Connect multiview components together (#7671)

Plumb through multiview configurations to the pipeline so that the
engine draws scenes using multiview extension. Users need to prepare
shaders compiled with the `multiview` param and set the
`stereoscopicType` flag to MULTIVIEW in the Engine::Config to enable
multiview feature.

In this change, postprocessings for multiview are not yet supported. So
we all disable them until they're supported.

The debug option `combineMultiviewImages` combines layers as one image,
which allows us to check the final result.

filament/src/RenderPass.cpp

@@ -184,8 +184,7 @@ void RenderPass::appendCommands(FEngine& engine,
  Command* curr = commands.data();
  size_t const commandCount = commands.size();
 
- auto stereoscopicEyeCount =
- renderFlags & IS_STEREOSCOPIC ? engine.getConfig().stereoscopicEyeCount : 1;
+ auto stereoscopicEyeCount = engine.getConfig().stereoscopicEyeCount;
 
  const float3 cameraPosition(mCameraPosition);
  const float3 cameraForwardVector(mCameraForwardVector);
@@ -521,7 +520,7 @@ RenderPass::Command* RenderPass::generateCommandsImpl(RenderPass::CommandTypeFla
 
  const bool hasShadowing = renderFlags & HAS_SHADOWING;
  const bool viewInverseFrontFaces = renderFlags & HAS_INVERSE_FRONT_FACES;
- const bool hasStereo = renderFlags & IS_STEREOSCOPIC;
+ const bool hasInstancedStereo = renderFlags & IS_INSTANCED_STEREOSCOPIC;
 
  Command cmdColor;
 
@@ -590,7 +589,7 @@ RenderPass::Command* RenderPass::generateCommandsImpl(RenderPass::CommandTypeFla
  // soaInstanceInfo[i].count is the number of instances the user has requested, either for
  // manual or hybrid instancing. Instanced stereo multiplies the number of instances by the
  // eye count.
- if (UTILS_UNLIKELY(hasStereo)) {
+ if (UTILS_UNLIKELY(hasInstancedStereo)) {
  cmdColor.primitive.instanceCount =
  (soaInstanceInfo[i].count * stereoEyeCount) |
  PrimitiveInfo::USER_INSTANCE_MASK;
@@ -624,7 +623,7 @@ RenderPass::Command* RenderPass::generateCommandsImpl(RenderPass::CommandTypeFla
  cmdDepth.primitive.skinningTexture = skinning.handleSampler;
  cmdDepth.primitive.morphWeightBuffer = morphing.handle;
 
- if (UTILS_UNLIKELY(hasStereo)) {
+ if (UTILS_UNLIKELY(hasInstancedStereo)) {
  cmdColor.primitive.instanceCount =
  (soaInstanceInfo[i].count * stereoEyeCount) |
  PrimitiveInfo::USER_INSTANCE_MASK;

filament/src/RenderPass.h

@@ -275,9 +275,9 @@ class RenderPass {
  "Command isn't trivially destructible");
 
  using RenderFlags = uint8_t;
- static constexpr RenderFlags HAS_SHADOWING = 0x01;
- static constexpr RenderFlags HAS_INVERSE_FRONT_FACES = 0x02;
- static constexpr RenderFlags IS_STEREOSCOPIC  = 0x04;
+ static constexpr RenderFlags HAS_SHADOWING  = 0x01;
+ static constexpr RenderFlags HAS_INVERSE_FRONT_FACES  = 0x02;
+ static constexpr RenderFlags IS_INSTANCED_STEREOSCOPIC = 0x04;
 
  // Arena used for commands
  using Arena = utils::Arena<

filament/src/RendererUtils.cpp

@@ -125,7 +125,9 @@ FrameGraphId<FrameGraphTexture> RendererUtils::colorPass(
  // depth auto-resolve, in which case we must allocate the depth
  // buffer with MS and manually resolve it (see "Resolved Depth Buffer"
  // pass).
+ .depth = colorBufferDesc.depth,
  .samples = canAutoResolveDepth ? colorBufferDesc.samples : uint8_t(config.msaa),
+ .type = colorBufferDesc.type,
  .format = format,
  });
  if (config.enabledStencilBuffer) {

filament/src/details/Engine.h

@@ -588,6 +588,9 @@ class FEngine : public Engine {
  struct {
  bool debug_froxel_visualization = false;
  } lighting;
+ struct {
+ bool combine_multiview_images = true;
+ } stereo;
  matdbg::DebugServer* server = nullptr;
  } debug;
 };

filament/src/details/Renderer.cpp

@@ -111,6 +111,8 @@ FRenderer::FRenderer(FEngine& engine) :
  &engine.debug.shadowmap.display_shadow_texture_level_count);
  debugRegistry.registerProperty("d.shadowmap.display_shadow_texture_power",
  &engine.debug.shadowmap.display_shadow_texture_power);
+ debugRegistry.registerProperty("d.stereo.combine_multiview_images",
+ &engine.debug.stereo.combine_multiview_images);
 
  DriverApi& driver = engine.getDriverApi();
 
@@ -520,7 +522,7 @@ void FRenderer::renderJob(RootArenaScope& rootArenaScope, FView& view) {
  // DEBUG: driver commands must all happen from the same thread. Enforce that on debug builds.
  driver.debugThreading();
 
- const bool hasPostProcess = view.hasPostProcessPass();
+ bool hasPostProcess = view.hasPostProcessPass();
  bool hasScreenSpaceRefraction = false;
  bool hasColorGrading = hasPostProcess;
  bool hasDithering = view.getDithering() == Dithering::TEMPORAL;
@@ -536,7 +538,16 @@ void FRenderer::renderJob(RootArenaScope& rootArenaScope, FView& view) {
  auto colorGrading = view.getColorGrading();
  auto ssReflectionsOptions = view.getScreenSpaceReflectionsOptions();
  auto guardBandOptions = view.getGuardBandOptions();
+ const bool isRenderingMultiview = view.hasStereo() &&
+ engine.getConfig().stereoscopicType == backend::StereoscopicType::MULTIVIEW;
+ // FIXME: This is to override some settings that are not supported for multiview at the moment.
+ // Remove this when all features are supported.
+ if (isRenderingMultiview) {
+ hasPostProcess = false;
+ msaaOptions.enabled = false;
+ }
  const uint8_t msaaSampleCount = msaaOptions.enabled ? msaaOptions.sampleCount : 1u;
+
  if (!hasPostProcess) {
  // disable all effects that are part of post-processing
  dofOptions.enabled = false;
@@ -700,7 +711,10 @@ void FRenderer::renderJob(RootArenaScope& rootArenaScope, FView& view) {
  RenderPass::RenderFlags renderFlags = 0;
  if (view.hasShadowing()) renderFlags |= RenderPass::HAS_SHADOWING;
  if (view.isFrontFaceWindingInverted()) renderFlags |= RenderPass::HAS_INVERSE_FRONT_FACES;
- if (view.hasStereo()) renderFlags |= RenderPass::IS_STEREOSCOPIC;
+ if (view.hasStereo() &&
+ engine.getConfig().stereoscopicType == backend::StereoscopicType::INSTANCED) {
+ renderFlags |= RenderPass::IS_INSTANCED_STEREOSCOPIC;
+ }
 
  RenderPassBuilder passBuilder(commandArena);
  passBuilder.renderFlags(renderFlags);
@@ -977,20 +991,26 @@ void FRenderer::renderJob(RootArenaScope& rootArenaScope, FView& view) {
 
  RenderPass const pass{ passBuilder.build(engine) };
 
- FrameGraphTexture::Descriptor const desc = {
+ FrameGraphTexture::Descriptor colorBufferDesc = {
  .width = config.physicalViewport.width,
  .height = config.physicalViewport.height,
  .format = config.hdrFormat
  };
 
+ // Set the depth to the number of layers if we're rendering multiview.
+ if (isRenderingMultiview) {
+ colorBufferDesc.depth = engine.getConfig().stereoscopicEyeCount;
+ colorBufferDesc.type = backend::SamplerType::SAMPLER_2D_ARRAY;
+ }
+
  // a non-drawing pass to prepare everything that need to be before the color passes execute
  fg.addTrivialSideEffectPass("Prepare Color Passes",
  [=, &js, &view, &ppm](DriverApi& driver) {
  // prepare color grading as subpass material
  if (colorGradingConfig.asSubpass) {
  ppm.colorGradingPrepareSubpass(driver,
  colorGrading, colorGradingConfig, vignetteOptions,
- desc.width, desc.height);
+ colorBufferDesc.width, colorBufferDesc.height);
  } else if (colorGradingConfig.customResolve) {
  ppm.customResolvePrepareSubpass(driver,
  PostProcessManager::CustomResolveOp::COMPRESS);
@@ -1008,7 +1028,7 @@ void FRenderer::renderJob(RootArenaScope& rootArenaScope, FView& view) {
 
  // the color pass itself + color-grading as subpass if needed
  auto colorPassOutput = RendererUtils::colorPass(fg, "Color Pass", mEngine, view,
- desc, config, colorGradingConfigForColor, pass.getExecutor());
+ colorBufferDesc, config, colorGradingConfigForColor, pass.getExecutor());
 
  if (view.isScreenSpaceRefractionEnabled() && !pass.empty()) {
  // this cancels the colorPass() call above if refraction is active.
@@ -1152,6 +1172,14 @@ void FRenderer::renderJob(RootArenaScope& rootArenaScope, FView& view) {
  }
  }
 
+ // Debug: combine the array texture for multiview into a single image.
+ if (UTILS_UNLIKELY(isRenderingMultiview && engine.debug.stereo.combine_multiview_images)) {
+ input = ppm.debugCombineArrayTexture(fg, blendModeTranslucent, input, xvp, {
+ .width = vp.width, .height = vp.height,
+ .format = colorGradingConfig.ldrFormat },
+ SamplerMagFilter::NEAREST, SamplerMinFilter::NEAREST);
+ }
+
  // We need to do special processing when rendering directly into the swap-chain, that is when
  // the viewRenderTarget is the default render target (mRenderTarget) and we're rendering into
  // it.

samples/CMakeLists.txt

@@ -50,8 +50,10 @@ file(MAKE_DIRECTORY ${MATERIAL_DIR})
 set (MATC_FLAGS ${MATC_BASE_FLAGS})
 if (FILAMENT_SAMPLES_STEREO_TYPE STREQUAL "instanced")
  set (MATC_FLAGS ${MATC_FLAGS} -PstereoscopicType=instanced)
+ add_definitions(-DFILAMENT_SAMPLES_STEREO_TYPE_INSTANCED)
 elseif (FILAMENT_SAMPLES_STEREO_TYPE STREQUAL "multiview")
  set (MATC_FLAGS ${MATC_FLAGS} -PstereoscopicType=multiview)
+ add_definitions(-DFILAMENT_SAMPLES_STEREO_TYPE_MULTIVIEW)
 endif ()
 
 foreach (mat_src ${MATERIAL_SRCS})

samples/gltf_viewer.cpp

@@ -893,6 +893,10 @@ int main(int argc, char** argv) {
  "d.shadowmap.display_shadow_texture_channel"), 0, 3);
  ImGui::Unindent();
  }
+#if defined(FILAMENT_SAMPLES_STEREO_TYPE_MULTIVIEW)
+ ImGui::Checkbox("Combine Multiview Images",
+ debug.getPropertyAddress<bool>("d.stereo.combine_multiview_images"));
+#endif
 
  bool debugFroxelVisualization;
  if (debug.getProperty("d.lighting.debug_froxel_visualization",