From aaa220fba4c0c46c2b340013b8e75beb5dd115fc Mon Sep 17 00:00:00 2001 From: Daosheng Mu Date: Sat, 24 Aug 2019 00:58:20 -0700 Subject: [PATCH] Using Surface from swap chains to render immersive frames instead of framebuffer textures. --- .../mozilla/vrbrowser/VRBrowserActivity.java | 16 ++++ .../browser/engine/SessionUtils.java | 7 ++ app/src/main/cpp/BrowserWorld.cpp | 19 ++-- app/src/main/cpp/DeviceDelegate.h | 1 + app/src/main/cpp/ExternalVR.cpp | 4 +- app/src/main/cpp/ExternalVR.h | 4 +- app/src/main/cpp/VRBrowser.cpp | 33 +++++++ app/src/main/cpp/VRBrowser.h | 3 + app/src/main/cpp/moz_external_vr.h | 43 ++++----- .../oculusvr/cpp/DeviceDelegateOculusVR.cpp | 89 ++++++++++++++++--- app/src/oculusvr/cpp/DeviceDelegateOculusVR.h | 4 + 11 files changed, 182 insertions(+), 41 deletions(-) diff --git a/app/src/common/shared/org/mozilla/vrbrowser/VRBrowserActivity.java b/app/src/common/shared/org/mozilla/vrbrowser/VRBrowserActivity.java index 8252feab6..914d11b5d 100644 --- a/app/src/common/shared/org/mozilla/vrbrowser/VRBrowserActivity.java +++ b/app/src/common/shared/org/mozilla/vrbrowser/VRBrowserActivity.java @@ -41,6 +41,7 @@ import org.mozilla.geckoview.GeckoRuntime; import org.mozilla.geckoview.GeckoSession; import org.mozilla.geckoview.GeckoVRManager; +import org.mozilla.gecko.gfx.ExternalVRSurface; import org.mozilla.vrbrowser.audio.AudioEngine; import org.mozilla.vrbrowser.browser.Accounts; import org.mozilla.vrbrowser.browser.PermissionDelegate; @@ -1070,6 +1071,21 @@ public void dismiss() { }); } + public void setExternalVRSurfaceIndex(int aIndex) { + GeckoVRManager.setExternalSurfaceIndex(aIndex); + } + + public void insertExternalSurface(int aWidth, int aHeight, int aIndex, Surface aSurface) { + if (aSurface == null) { + Log.e(LOGTAG, "aSurface is null in setExternalVRSurface..."); + } + GeckoVRManager.insertExternalSurface(new ExternalVRSurface(aIndex, aWidth, aHeight, aSurface)); + } + + public void releaseExternalVRSurfaces() { + GeckoVRManager.releaseExternalSurfaces(); + } + private SurfaceTexture createSurfaceTexture() { int[] ids = new int[1]; GLES20.glGenTextures(1, ids, 0); diff --git a/app/src/common/shared/org/mozilla/vrbrowser/browser/engine/SessionUtils.java b/app/src/common/shared/org/mozilla/vrbrowser/browser/engine/SessionUtils.java index 11ac4bf99..c7d272303 100644 --- a/app/src/common/shared/org/mozilla/vrbrowser/browser/engine/SessionUtils.java +++ b/app/src/common/shared/org/mozilla/vrbrowser/browser/engine/SessionUtils.java @@ -10,6 +10,7 @@ import org.mozilla.gecko.GeckoProfile; import org.mozilla.vrbrowser.browser.SettingsStore; import org.mozilla.vrbrowser.utils.SystemUtils; +import org.mozilla.vrbrowser.utils.DeviceType; import java.io.File; import java.io.FileNotFoundException; @@ -56,6 +57,12 @@ public static void vrPrefsWorkAround(Context aContext, Bundle aExtras) { out.write("pref(\"media.autoplay.enabled.ask-permission\", false);\n".getBytes()); out.write("pref(\"media.autoplay.default\", 0);\n".getBytes()); } + // In Oculus platform, we can render WebGL immersive frames info AndroidSurface. + if (DeviceType.isOculusBuild()) { + out.write("pref(\"webgl.enable-externalvr-surface\", true);\n".getBytes()); + } else { + out.write("pref(\"webgl.enable-externalvr-surface\", false);\n".getBytes()); + } } catch (FileNotFoundException e) { Log.e(LOGTAG, "Unable to create file: '" + prefFileName + "' got exception: " + e.toString()); } catch (IOException e) { diff --git a/app/src/main/cpp/BrowserWorld.cpp b/app/src/main/cpp/BrowserWorld.cpp index 809e1e3b4..fb8271010 100644 --- a/app/src/main/cpp/BrowserWorld.cpp +++ b/app/src/main/cpp/BrowserWorld.cpp @@ -1299,6 +1299,7 @@ void BrowserWorld::DrawWorld() { m.externalVR->SetCompositorEnabled(true); m.device->SetRenderMode(device::RenderMode::StandAlone); + m.device->EnableExternalSurfaceRender(false); if (m.fadeAnimation) { m.fadeAnimation->UpdateAnimation(); } @@ -1363,23 +1364,29 @@ BrowserWorld::DrawImmersive() { m.device->StartFrame(); VRB_GL_CHECK(glDepthMask(GL_FALSE)); m.externalVR->PushFramePoses(m.device->GetHeadTransform(), m.controllers->GetControllers(), m.context->GetTimestamp()); + mozilla::gfx::VRLayerTextureType surfaceType; int32_t surfaceHandle, textureWidth, textureHeight = 0; device::EyeRect leftEye, rightEye; bool aDiscardFrame = !m.externalVR->WaitFrameResult(); - m.externalVR->GetFrameResult(surfaceHandle, textureWidth, textureHeight, leftEye, rightEye); + m.externalVR->GetFrameResult(surfaceType, surfaceHandle, textureWidth, textureHeight, leftEye, rightEye); ExternalVR::VRState state = m.externalVR->GetVRState(); if (state == ExternalVR::VRState::Rendering) { if (!aDiscardFrame) { if (textureWidth > 0 && textureHeight > 0) { m.device->SetImmersiveSize((uint32_t) textureWidth/2, (uint32_t) textureHeight); } - m.blitter->StartFrame(surfaceHandle, leftEye, rightEye); - m.device->BindEye(device::Eye::Left); - m.blitter->Draw(device::Eye::Left); + m.device->EnableExternalSurfaceRender(surfaceType == + mozilla::gfx::VRLayerTextureType::LayerTextureType_ExternalVRSurface); + // In Oculus platform, we can render WebGL immersive frames info AndroidSurface. + if (surfaceType != mozilla::gfx::VRLayerTextureType::LayerTextureType_ExternalVRSurface) { + m.blitter->StartFrame(surfaceHandle, leftEye, rightEye); + m.device->BindEye(device::Eye::Left); + m.blitter->Draw(device::Eye::Left); #if !defined(VRBROWSER_NO_VR_API) - m.device->BindEye(device::Eye::Right); - m.blitter->Draw(device::Eye::Right); + m.device->BindEye(device::Eye::Right); + m.blitter->Draw(device::Eye::Right); #endif // !defined(VRBROWSER_NO_VR_API) + } } m.device->EndFrame(aDiscardFrame); m.blitter->EndFrame(); diff --git a/app/src/main/cpp/DeviceDelegate.h b/app/src/main/cpp/DeviceDelegate.h index d680c4015..ddd710e42 100644 --- a/app/src/main/cpp/DeviceDelegate.h +++ b/app/src/main/cpp/DeviceDelegate.h @@ -67,6 +67,7 @@ class DeviceDelegate { virtual void StartFrame() = 0; virtual void BindEye(const device::Eye aWhich) = 0; virtual void EndFrame(bool aDiscard = false) = 0; + virtual void EnableExternalSurfaceRender(bool aEnable) {} virtual VRLayerQuadPtr CreateLayerQuad(int32_t aWidth, int32_t aHeight, VRLayerSurface::SurfaceType aSurfaceType) { return nullptr; } virtual VRLayerQuadPtr CreateLayerQuad(const VRLayerSurfacePtr& aMoveLayer) { return nullptr; } diff --git a/app/src/main/cpp/ExternalVR.cpp b/app/src/main/cpp/ExternalVR.cpp index 540f837ca..94f19858f 100644 --- a/app/src/main/cpp/ExternalVR.cpp +++ b/app/src/main/cpp/ExternalVR.cpp @@ -476,10 +476,10 @@ ExternalVR::CompleteEnumeration() m.system.enumerationCompleted = true; } - void -ExternalVR::GetFrameResult(int32_t& aSurfaceHandle, int32_t& aTextureWidth, int32_t& aTextureHeight, +ExternalVR::GetFrameResult(mozilla::gfx::VRLayerTextureType& aSurfaceType, int32_t& aSurfaceHandle, int32_t& aTextureWidth, int32_t& aTextureHeight, device::EyeRect& aLeftEye, device::EyeRect& aRightEye) const { + aSurfaceType = m.browser.layerState[0].layer_stereo_immersive.textureType; aSurfaceHandle = (int32_t)m.browser.layerState[0].layer_stereo_immersive.textureHandle; mozilla::gfx::VRLayerEyeRect& left = m.browser.layerState[0].layer_stereo_immersive.leftEyeRect; mozilla::gfx::VRLayerEyeRect& right = m.browser.layerState[0].layer_stereo_immersive.rightEyeRect; diff --git a/app/src/main/cpp/ExternalVR.h b/app/src/main/cpp/ExternalVR.h index da66b8216..2b0a686a9 100644 --- a/app/src/main/cpp/ExternalVR.h +++ b/app/src/main/cpp/ExternalVR.h @@ -10,6 +10,7 @@ #include "Controller.h" #include "DeviceDelegate.h" #include "Device.h" +#include "moz_external_vr.h" #include #include #include @@ -54,7 +55,8 @@ class ExternalVR : public ImmersiveDisplay { VRState GetVRState() const; void PushFramePoses(const vrb::Matrix& aHeadTransform, const std::vector& aControllers, const double aTimestamp); bool WaitFrameResult(); - void GetFrameResult(int32_t& aSurfaceHandle, + void GetFrameResult(mozilla::gfx::VRLayerTextureType& aSurfaceType, + int32_t& aSurfaceHandle, int32_t& aTextureWidth, int32_t& aTextureHeight, device::EyeRect& aLeftEye, diff --git a/app/src/main/cpp/VRBrowser.cpp b/app/src/main/cpp/VRBrowser.cpp index 20ffd81c3..f901ef544 100644 --- a/app/src/main/cpp/VRBrowser.cpp +++ b/app/src/main/cpp/VRBrowser.cpp @@ -52,6 +52,12 @@ const char* kHaltActivity = "haltActivity"; const char* kHaltActivitySignature = "(I)V"; const char* kHandlePoorPerformance = "handlePoorPerformance"; const char* kHandlePoorPerformanceSignature = "()V"; +const char* kSetExternalVRSurfaceIndex = "setExternalVRSurfaceIndex"; +const char* kSetExternalVRSurfaceIndexSignature = "(I)V"; +const char* kInsertExternalVRSurface = "insertExternalSurface"; +const char* kInsertExternalVRSurfaceSignature = "(IIILandroid/view/Surface;)V"; +const char* kReleaseExternalSurfaces = "releaseExternalVRSurfaces"; +const char* kReleaseExternalSurfacesSignature = "()V"; JNIEnv* sEnv = nullptr; jclass sBrowserClass = nullptr; @@ -77,6 +83,9 @@ jmethodID sAreLayersEnabled = nullptr; jmethodID sSetDeviceType = nullptr; jmethodID sHaltActivity = nullptr; jmethodID sHandlePoorPerformance = nullptr; +jmethodID sSetExternalVRSurfaceIndex = nullptr; +jmethodID sInsertExternalVRSurface = nullptr; +jmethodID sReleaseExternalSurfaces = nullptr; } namespace crow { @@ -117,6 +126,9 @@ VRBrowser::InitializeJava(JNIEnv* aEnv, jobject aActivity) { sSetDeviceType = FindJNIMethodID(sEnv, sBrowserClass, kSetDeviceType, kSetDeviceTypeSignature); sHaltActivity = FindJNIMethodID(sEnv, sBrowserClass, kHaltActivity, kHaltActivitySignature); sHandlePoorPerformance = FindJNIMethodID(sEnv, sBrowserClass, kHandlePoorPerformance, kHandlePoorPerformanceSignature); + sInsertExternalVRSurface = FindJNIMethodID(sEnv, sBrowserClass, kInsertExternalVRSurface, kInsertExternalVRSurfaceSignature); + sSetExternalVRSurfaceIndex = FindJNIMethodID(sEnv, sBrowserClass, kSetExternalVRSurfaceIndex, kSetExternalVRSurfaceIndexSignature); + sReleaseExternalSurfaces = FindJNIMethodID(sEnv, sBrowserClass, kReleaseExternalSurfaces, kReleaseExternalSurfacesSignature); } void @@ -151,6 +163,7 @@ VRBrowser::ShutdownJava() { sAreLayersEnabled = nullptr; sSetDeviceType = nullptr; sHaltActivity = nullptr; + sInsertExternalVRSurface = nullptr; sEnv = nullptr; } @@ -339,4 +352,24 @@ VRBrowser::HandlePoorPerformance() { CheckJNIException(sEnv, __FUNCTION__); } +void +VRBrowser::SetExternalVRSurfaceId(jint aId) { + if (!ValidateMethodID(sEnv, sActivity, sSetExternalVRSurfaceIndex, __FUNCTION__)) { return; } + sEnv->CallVoidMethod(sActivity, sSetExternalVRSurfaceIndex, aId); + CheckJNIException(sEnv, __FUNCTION__); +} + +void +VRBrowser::InsertExternalVRSurface(jint aWidth, jint aHeight, jint aIndex, jobject aSurface) { + if (!ValidateMethodID(sEnv, sActivity, sInsertExternalVRSurface, __FUNCTION__)) { return; } + sEnv->CallVoidMethod(sActivity, sInsertExternalVRSurface, aWidth, aHeight, aIndex, aSurface); + CheckJNIException(sEnv, __FUNCTION__); +} + +void +VRBrowser::ReleaseExternalVRSurfaces() { + if (!ValidateMethodID(sEnv, sActivity, sReleaseExternalSurfaces, __FUNCTION__)) { return; } + sEnv->CallVoidMethod(sActivity, sReleaseExternalSurfaces); + CheckJNIException(sEnv, __FUNCTION__); +} } // namespace crow diff --git a/app/src/main/cpp/VRBrowser.h b/app/src/main/cpp/VRBrowser.h index 49e74d0e1..b289964ea 100644 --- a/app/src/main/cpp/VRBrowser.h +++ b/app/src/main/cpp/VRBrowser.h @@ -39,6 +39,9 @@ bool AreLayersEnabled(); void SetDeviceType(const jint aType); void HaltActivity(const jint aReason); void HandlePoorPerformance(); +void SetExternalVRSurfaceId(jint aId); +void InsertExternalVRSurface(jint aWidth, jint aHeight, jint aIndex, jobject aSurface); +void ReleaseExternalVRSurfaces(); } // namespace VRBrowser; } // namespace crow diff --git a/app/src/main/cpp/moz_external_vr.h b/app/src/main/cpp/moz_external_vr.h index 3f78c6cc8..06d73f22a 100644 --- a/app/src/main/cpp/moz_external_vr.h +++ b/app/src/main/cpp/moz_external_vr.h @@ -47,8 +47,8 @@ namespace gfx { // and mapped files if we have both release and nightlies // running at the same time? Or...what if we have multiple // release builds running on same machine? (Bug 1563232) -#define SHMEM_VERSION "0.0.4" -static const int32_t kVRExternalVersion = 11; +#define SHMEM_VERSION "0.0.5" +static const int32_t kVRExternalVersion = 12; // We assign VR presentations to groups with a bitmask. // Currently, we will only display either content or chrome. @@ -102,26 +102,26 @@ enum class ControllerCapabilityFlags : uint16_t { /** * Cap_Position is set if the Gamepad is capable of tracking its position. */ - Cap_Position = 1 << 1, + Cap_Position = 1 << 1, /** * Cap_Orientation is set if the Gamepad is capable of tracking its * orientation. */ - Cap_Orientation = 1 << 2, + Cap_Orientation = 1 << 2, /** * Cap_AngularAcceleration is set if the Gamepad is capable of tracking its * angular acceleration. */ - Cap_AngularAcceleration = 1 << 3, + Cap_AngularAcceleration = 1 << 3, /** * Cap_LinearAcceleration is set if the Gamepad is capable of tracking its * linear acceleration. */ - Cap_LinearAcceleration = 1 << 4, + Cap_LinearAcceleration = 1 << 4, /** * Cap_All used for validity checking during IPC serialization */ - Cap_All = (1 << 5) - 1 + Cap_All = (1 << 5) - 1 }; #endif // ifndef MOZILLA_INTERNAL_API @@ -133,12 +133,12 @@ enum class VRDisplayCapabilityFlags : uint16_t { /** * Cap_Position is set if the VRDisplay is capable of tracking its position. */ - Cap_Position = 1 << 1, + Cap_Position = 1 << 1, /** * Cap_Orientation is set if the VRDisplay is capable of tracking its * orientation. */ - Cap_Orientation = 1 << 2, + Cap_Orientation = 1 << 2, /** * Cap_Present is set if the VRDisplay is capable of presenting content to an * HMD or similar device. Can be used to indicate "magic window" devices that @@ -146,7 +146,7 @@ enum class VRDisplayCapabilityFlags : uint16_t { * meaningful. If false then calls to requestPresent should always fail, and * getEyeParameters should return null. */ - Cap_Present = 1 << 3, + Cap_Present = 1 << 3, /** * Cap_External is set if the VRDisplay is separate from the device's * primary display. If presenting VR content will obscure @@ -154,54 +154,54 @@ enum class VRDisplayCapabilityFlags : uint16_t { * un-set, the application should not attempt to mirror VR content * or update non-VR UI because that content will not be visible. */ - Cap_External = 1 << 4, + Cap_External = 1 << 4, /** * Cap_AngularAcceleration is set if the VRDisplay is capable of tracking its * angular acceleration. */ - Cap_AngularAcceleration = 1 << 5, + Cap_AngularAcceleration = 1 << 5, /** * Cap_LinearAcceleration is set if the VRDisplay is capable of tracking its * linear acceleration. */ - Cap_LinearAcceleration = 1 << 6, + Cap_LinearAcceleration = 1 << 6, /** * Cap_StageParameters is set if the VRDisplay is capable of room scale VR * and can report the StageParameters to describe the space. */ - Cap_StageParameters = 1 << 7, + Cap_StageParameters = 1 << 7, /** * Cap_MountDetection is set if the VRDisplay is capable of sensing when the * user is wearing the device. */ - Cap_MountDetection = 1 << 8, + Cap_MountDetection = 1 << 8, /** * Cap_PositionEmulated is set if the VRDisplay is capable of setting a * emulated position (e.g. neck model) even if still doesn't support 6DOF * tracking. */ - Cap_PositionEmulated = 1 << 9, + Cap_PositionEmulated = 1 << 9, /** * Cap_Inline is set if the device can be used for WebXR inline sessions * where the content is displayed within an element on the page. */ - Cap_Inline = 1 << 10, + Cap_Inline = 1 << 10, /** * Cap_ImmersiveVR is set if the device can give exclusive access to the * XR device display and that content is not intended to be integrated * with the user's environment */ - Cap_ImmersiveVR = 1 << 11, + Cap_ImmersiveVR = 1 << 11, /** * Cap_ImmersiveAR is set if the device can give exclusive access to the * XR device display and that content is intended to be integrated with * the user's environment. */ - Cap_ImmersiveAR = 1 << 12, + Cap_ImmersiveAR = 1 << 12, /** * Cap_All used for validity checking during IPC serialization */ - Cap_All = (1 << 13) - 1 + Cap_All = (1 << 13) - 1 }; #ifdef MOZILLA_INTERNAL_API @@ -372,7 +372,8 @@ enum class VRLayerTextureType : uint16_t { LayerTextureType_None = 0, LayerTextureType_D3D10SurfaceDescriptor = 1, LayerTextureType_MacIOSurface = 2, - LayerTextureType_GeckoSurfaceTexture = 3 + LayerTextureType_GeckoSurfaceTexture = 3, + LayerTextureType_ExternalVRSurface = 4 }; struct VRLayer_2D_Content { diff --git a/app/src/oculusvr/cpp/DeviceDelegateOculusVR.cpp b/app/src/oculusvr/cpp/DeviceDelegateOculusVR.cpp index cab14b47d..9c9dd3942 100644 --- a/app/src/oculusvr/cpp/DeviceDelegateOculusVR.cpp +++ b/app/src/oculusvr/cpp/DeviceDelegateOculusVR.cpp @@ -635,6 +635,8 @@ const vrb::Vector kAverageHeight(0.0f, 1.7f, 0.0f); const vrb::Vector kAverageOculusHeight(0.0f, 1.65f, 0.0f); struct DeviceDelegateOculusVR::State { + const static uint32_t EXTERNAL_SURFACE_BUFFER_SIZE = 1; + struct ControllerState { const int32_t index; const ElbowModel::HandEnum hand; @@ -663,6 +665,7 @@ struct DeviceDelegateOculusVR::State { ovrJava java = {}; ovrMobile* ovr = nullptr; OculusEyeSwapChainPtr eyeSwapChains[VRAPI_EYE_COUNT]; + ovrTextureSwapChain* eyeSurfaceSwapChain[EXTERNAL_SURFACE_BUFFER_SIZE]; OculusLayerCubePtr cubeLayer; OculusLayerEquirectPtr equirectLayer; std::vector uiLayers; @@ -726,6 +729,9 @@ struct DeviceDelegateOculusVR::State { exit(status); return; } + for (int i = 0; i < EXTERNAL_SURFACE_BUFFER_SIZE; ++i) { + eyeSurfaceSwapChain[i] = nullptr; + } initialized = true; SetRenderSize(device::RenderMode::StandAlone); @@ -1220,13 +1226,31 @@ DeviceDelegateOculusVR::SetImmersiveSize(const uint32_t aEyeWidth, const uint32_ DeviceUtils::GetTargetImmersiveSize(aEyeWidth, aEyeHeight, recommendedWidth, recommendedHeight, targetWidth, targetHeight); if (targetWidth != m.renderWidth || targetHeight != m.renderHeight) { + VRB_LOG("Resizing immersive mode swapChain: %dx%d", targetWidth, targetHeight); + m.renderWidth = targetWidth; m.renderHeight = targetHeight; vrb::RenderContextPtr render = m.context.lock(); for (int i = 0; i < VRAPI_EYE_COUNT; ++i) { m.eyeSwapChains[i]->Init(render, m.renderMode, m.renderWidth, m.renderHeight); } - VRB_LOG("Resize immersive mode swapChain: %dx%d", targetWidth, targetHeight); + + VRBrowser::ReleaseExternalVRSurfaces(); + for (int i = 0; i < DeviceDelegateOculusVR::State::EXTERNAL_SURFACE_BUFFER_SIZE; ++i) { + if (m.eyeSurfaceSwapChain[i]) { + vrapi_DestroyTextureSwapChain(m.eyeSurfaceSwapChain[i]); + m.eyeSurfaceSwapChain[i] = nullptr; + } + m.eyeSurfaceSwapChain[i] = vrapi_CreateAndroidSurfaceSwapChain(m.renderWidth, m.renderHeight); + auto surfaceOut = vrapi_GetTextureSwapChainAndroidSurface(m.eyeSurfaceSwapChain[i]); + surfaceOut = m.java.Env->NewGlobalRef(surfaceOut); + VRBrowser::InsertExternalVRSurface(m.renderWidth, m.renderHeight, i, surfaceOut); + } + + mNeedResize = true; + VRBrowser::SetExternalVRSurfaceId(externalSurfaceId); + } else { + mNeedResize = false; } } @@ -1386,8 +1410,6 @@ DeviceDelegateOculusVR::StartFrame() { ovrMatrix4f matrix = vrapi_GetTransformFromPose(&m.predictedTracking.HeadPose.Pose); vrb::Matrix head = vrb::Matrix::FromRowMajor(matrix.M[0]); - - if (m.renderMode == device::RenderMode::StandAlone) { head.TranslateInPlace(kAverageHeight); } @@ -1405,6 +1427,18 @@ DeviceDelegateOculusVR::StartFrame() { caps |= device::PositionEmulated; } m.immersiveDisplay->SetCapabilityFlags(caps); + + uint32_t width, height; + m.GetImmersiveRenderSize(width, height); + for (int i = 0; i < DeviceDelegateOculusVR::State::EXTERNAL_SURFACE_BUFFER_SIZE; ++i) { + if (!m.eyeSurfaceSwapChain[i]) { + m.eyeSurfaceSwapChain[i] = vrapi_CreateAndroidSurfaceSwapChain(width, height); + auto surfaceOut = vrapi_GetTextureSwapChainAndroidSurface(m.eyeSurfaceSwapChain[i]); + surfaceOut = m.java.Env->NewGlobalRef(surfaceOut); + VRBrowser::InsertExternalVRSurface(width, height, i, surfaceOut); + } + } + VRBrowser::SetExternalVRSurfaceId(externalSurfaceId); } int lastReorientCount = m.reorientCount; @@ -1504,13 +1538,33 @@ DeviceDelegateOculusVR::EndFrame(const bool aDiscard) { projection.HeadPose = m.predictedTracking.HeadPose; projection.Header.SrcBlend = VRAPI_FRAME_LAYER_BLEND_ONE; projection.Header.DstBlend = VRAPI_FRAME_LAYER_BLEND_ONE_MINUS_SRC_ALPHA; - for (int i = 0; i < VRAPI_FRAME_LAYER_EYE_MAX; ++i) { - const auto &eyeSwapChain = m.eyeSwapChains[i]; - const int swapChainIndex = m.frameIndex % eyeSwapChain->swapChainLength; - // Set up OVR layer textures - projection.Textures[i].ColorSwapChain = eyeSwapChain->ovrSwapChain; - projection.Textures[i].SwapChainIndex = swapChainIndex; - projection.Textures[i].TexCoordsFromTanAngles = ovrMatrix4f_TanAngleMatrixFromProjection(&projectionMatrix); + + if (!enableExternalSurfaceRender) { + for (int i = 0; i < VRAPI_FRAME_LAYER_EYE_MAX; ++i) { + const auto &eyeSwapChain = m.eyeSwapChains[i]; + const int swapChainIndex = m.frameIndex % eyeSwapChain->swapChainLength; + // Set up OVR layer textures + projection.Textures[i].ColorSwapChain = eyeSwapChain->ovrSwapChain; + projection.Textures[i].SwapChainIndex = swapChainIndex; + projection.Textures[i].TexCoordsFromTanAngles = ovrMatrix4f_TanAngleMatrixFromProjection(&projectionMatrix); + } + } else if (!mNeedResize){ // If mNeedResize, we need to wait for the next frame. + const int swapChainIndex = externalSurfaceId; + ovrMatrix4f proj(projectionMatrix); + // Flip texCoord in vertical when using WebGL frame textures. + proj.M[1][1] *= -1; + proj = ovrMatrix4f_TanAngleMatrixFromProjection(&proj); + + for (int i = 0; i < VRAPI_FRAME_LAYER_EYE_MAX; ++i) { + const auto eyeSwapChain = m.eyeSurfaceSwapChain[swapChainIndex]; + // Set up OVR layer textures + projection.Textures[i].ColorSwapChain = eyeSwapChain; + projection.Textures[i].SwapChainIndex = 0; + projection.Textures[i].TexCoordsFromTanAngles = proj; + } + + // Switch to the next surface. + externalSurfaceId = (++externalSurfaceId) % State::EXTERNAL_SURFACE_BUFFER_SIZE; } layers[layerCount++] = &projection.Header; @@ -1540,6 +1594,11 @@ DeviceDelegateOculusVR::EndFrame(const bool aDiscard) { vrapi_SubmitFrame2(m.ovr, &frameDesc); } +void +DeviceDelegateOculusVR::EnableExternalSurfaceRender(bool aEnable) { + enableExternalSurfaceRender = aEnable; +} + VRLayerQuadPtr DeviceDelegateOculusVR::CreateLayerQuad(int32_t aWidth, int32_t aHeight, VRLayerSurface::SurfaceType aSurfaceType) { @@ -1725,6 +1784,13 @@ DeviceDelegateOculusVR::LeaveVR() { for (int i = 0; i < VRAPI_EYE_COUNT; ++i) { m.eyeSwapChains[i]->Destroy(); } + for (int i = 0; i < DeviceDelegateOculusVR::State::EXTERNAL_SURFACE_BUFFER_SIZE; ++i) { + if (m.eyeSurfaceSwapChain[i]) { + vrapi_DestroyTextureSwapChain(m.eyeSurfaceSwapChain[i]); + m.eyeSurfaceSwapChain[i] = nullptr; + } + } + VRBrowser::ReleaseExternalVRSurfaces(); if (m.cubeLayer) { m.cubeLayer->Destroy(); } @@ -1756,7 +1822,8 @@ DeviceDelegateOculusVR::ExitApp() { return true; } -DeviceDelegateOculusVR::DeviceDelegateOculusVR(State &aState) : m(aState) {} +DeviceDelegateOculusVR::DeviceDelegateOculusVR(State &aState) : m(aState), + externalSurfaceId(0), enableExternalSurfaceRender(false), mNeedResize(false) {} DeviceDelegateOculusVR::~DeviceDelegateOculusVR() { m.Shutdown(); } diff --git a/app/src/oculusvr/cpp/DeviceDelegateOculusVR.h b/app/src/oculusvr/cpp/DeviceDelegateOculusVR.h index ae54323ec..db15398e2 100644 --- a/app/src/oculusvr/cpp/DeviceDelegateOculusVR.h +++ b/app/src/oculusvr/cpp/DeviceDelegateOculusVR.h @@ -45,6 +45,7 @@ class DeviceDelegateOculusVR : public DeviceDelegate { void StartFrame() override; void BindEye(const device::Eye aWhich) override; void EndFrame(const bool aDiscard) override; + void EnableExternalSurfaceRender(bool aEnable) override; VRLayerQuadPtr CreateLayerQuad(int32_t aWidth, int32_t aHeight, VRLayerSurface::SurfaceType aSurfaceType) override; VRLayerQuadPtr CreateLayerQuad(const VRLayerSurfacePtr& aMoveLayer) override; @@ -65,6 +66,9 @@ class DeviceDelegateOculusVR : public DeviceDelegate { virtual ~DeviceDelegateOculusVR(); private: State& m; + int32_t externalSurfaceId; + bool enableExternalSurfaceRender; + bool mNeedResize; VRB_NO_DEFAULTS(DeviceDelegateOculusVR) };