Render WebVR frames to OVR VR surfaces

app/src/common/shared/org/mozilla/vrbrowser/VRBrowserActivity.java

@@ -41,6 +41,7 @@
 import org.mozilla.geckoview.GeckoRuntime;
 import org.mozilla.geckoview.GeckoSession;
 import org.mozilla.geckoview.GeckoVRManager;
+import org.mozilla.gecko.gfx.ExternalVRSurface;
 import org.mozilla.vrbrowser.audio.AudioEngine;
 import org.mozilla.vrbrowser.browser.Accounts;
 import org.mozilla.vrbrowser.browser.PermissionDelegate;
@@ -1070,6 +1071,21 @@ public void dismiss() {
         });
     }
 
+    public void setExternalVRSurfaceIndex(int aIndex) {
+        GeckoVRManager.setExternalSurfaceIndex(aIndex);
+    }
+
+    public void insertExternalSurface(int aWidth, int aHeight, int aIndex, Surface aSurface) {
+        if (aSurface == null) {
+            Log.e(LOGTAG, "aSurface is null in setExternalVRSurface...");
+        }
+        GeckoVRManager.insertExternalSurface(new ExternalVRSurface(aIndex, aWidth, aHeight, aSurface));
+    }
+
+    public void releaseExternalVRSurfaces() {
+        GeckoVRManager.releaseExternalSurfaces();
+    }
+
     private SurfaceTexture createSurfaceTexture() {
         int[] ids = new int[1];
         GLES20.glGenTextures(1, ids, 0);

app/src/common/shared/org/mozilla/vrbrowser/browser/engine/SessionUtils.java

@@ -10,6 +10,7 @@
 import org.mozilla.gecko.GeckoProfile;
 import org.mozilla.vrbrowser.browser.SettingsStore;
 import org.mozilla.vrbrowser.utils.SystemUtils;
+import org.mozilla.vrbrowser.utils.DeviceType;
 
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -56,6 +57,12 @@ public static void vrPrefsWorkAround(Context aContext, Bundle aExtras) {
                 out.write("pref(\"media.autoplay.enabled.ask-permission\", false);\n".getBytes());
                 out.write("pref(\"media.autoplay.default\", 0);\n".getBytes());
             }
+            // In Oculus platform, we can render WebGL immersive frames info AndroidSurface.
+            if (DeviceType.isOculusBuild()) {
+                out.write("pref(\"webgl.enable-externalvr-surface\", true);\n".getBytes());
+            } else {
+                out.write("pref(\"webgl.enable-externalvr-surface\", false);\n".getBytes());
+            }
         } catch (FileNotFoundException e) {
             Log.e(LOGTAG, "Unable to create file: '" + prefFileName + "' got exception: " + e.toString());
         } catch (IOException e) {

app/src/main/cpp/BrowserWorld.cpp

@@ -1299,6 +1299,7 @@ void
 BrowserWorld::DrawWorld() {
   m.externalVR->SetCompositorEnabled(true);
   m.device->SetRenderMode(device::RenderMode::StandAlone);
+  m.device->EnableExternalSurfaceRender(false);
   if (m.fadeAnimation) {
     m.fadeAnimation->UpdateAnimation();
   }
@@ -1363,23 +1364,29 @@ BrowserWorld::DrawImmersive() {
   m.device->StartFrame();
   VRB_GL_CHECK(glDepthMask(GL_FALSE));
   m.externalVR->PushFramePoses(m.device->GetHeadTransform(), m.controllers->GetControllers(), m.context->GetTimestamp());
+  mozilla::gfx::VRLayerTextureType surfaceType;
   int32_t surfaceHandle, textureWidth, textureHeight = 0;
   device::EyeRect leftEye, rightEye;
   bool aDiscardFrame = !m.externalVR->WaitFrameResult();
-  m.externalVR->GetFrameResult(surfaceHandle, textureWidth, textureHeight, leftEye, rightEye);
+  m.externalVR->GetFrameResult(surfaceType, surfaceHandle, textureWidth, textureHeight, leftEye, rightEye);
   ExternalVR::VRState state = m.externalVR->GetVRState();
   if (state == ExternalVR::VRState::Rendering) {
     if (!aDiscardFrame) {
       if (textureWidth > 0 && textureHeight > 0) {
         m.device->SetImmersiveSize((uint32_t) textureWidth/2, (uint32_t) textureHeight);
       }
-      m.blitter->StartFrame(surfaceHandle, leftEye, rightEye);
-      m.device->BindEye(device::Eye::Left);
-      m.blitter->Draw(device::Eye::Left);
+      m.device->EnableExternalSurfaceRender(surfaceType ==
+        mozilla::gfx::VRLayerTextureType::LayerTextureType_ExternalVRSurface);
+      // In Oculus platform, we can render WebGL immersive frames info AndroidSurface.
+      if (surfaceType != mozilla::gfx::VRLayerTextureType::LayerTextureType_ExternalVRSurface) {
+        m.blitter->StartFrame(surfaceHandle, leftEye, rightEye);
+        m.device->BindEye(device::Eye::Left);
+        m.blitter->Draw(device::Eye::Left);
 #if !defined(VRBROWSER_NO_VR_API)
-      m.device->BindEye(device::Eye::Right);
-      m.blitter->Draw(device::Eye::Right);
+        m.device->BindEye(device::Eye::Right);
+        m.blitter->Draw(device::Eye::Right);
 #endif // !defined(VRBROWSER_NO_VR_API)
+      }
     }
     m.device->EndFrame(aDiscardFrame);
     m.blitter->EndFrame();

app/src/main/cpp/DeviceDelegate.h

@@ -67,6 +67,7 @@ class DeviceDelegate {
   virtual void StartFrame() = 0;
   virtual void BindEye(const device::Eye aWhich) = 0;
   virtual void EndFrame(bool aDiscard = false) = 0;
+  virtual void EnableExternalSurfaceRender(bool aEnable) {}
   virtual VRLayerQuadPtr CreateLayerQuad(int32_t aWidth, int32_t aHeight,
                                          VRLayerSurface::SurfaceType aSurfaceType) { return nullptr; }
   virtual VRLayerQuadPtr CreateLayerQuad(const VRLayerSurfacePtr& aMoveLayer) { return nullptr; }

app/src/main/cpp/ExternalVR.cpp

@@ -476,10 +476,10 @@ ExternalVR::CompleteEnumeration()
   m.system.enumerationCompleted = true;
 }
 
-
 void
-ExternalVR::GetFrameResult(int32_t& aSurfaceHandle, int32_t& aTextureWidth, int32_t& aTextureHeight,
+ExternalVR::GetFrameResult(mozilla::gfx::VRLayerTextureType& aSurfaceType, int32_t& aSurfaceHandle, int32_t& aTextureWidth, int32_t& aTextureHeight,
     device::EyeRect& aLeftEye, device::EyeRect& aRightEye) const {
+  aSurfaceType = m.browser.layerState[0].layer_stereo_immersive.textureType;
   aSurfaceHandle = (int32_t)m.browser.layerState[0].layer_stereo_immersive.textureHandle;
   mozilla::gfx::VRLayerEyeRect& left = m.browser.layerState[0].layer_stereo_immersive.leftEyeRect;
   mozilla::gfx::VRLayerEyeRect& right = m.browser.layerState[0].layer_stereo_immersive.rightEyeRect;

app/src/main/cpp/ExternalVR.h

@@ -10,6 +10,7 @@
 #include "Controller.h"
 #include "DeviceDelegate.h"
 #include "Device.h"
+#include "moz_external_vr.h"
 #include <memory>
 #include <string>
 #include <vector>
@@ -54,7 +55,8 @@ class ExternalVR : public ImmersiveDisplay {
   VRState GetVRState() const;
   void PushFramePoses(const vrb::Matrix& aHeadTransform, const std::vector<Controller>& aControllers, const double aTimestamp);
   bool WaitFrameResult();
-  void GetFrameResult(int32_t& aSurfaceHandle,
+  void GetFrameResult(mozilla::gfx::VRLayerTextureType& aSurfaceType,
+                      int32_t& aSurfaceHandle,
                       int32_t& aTextureWidth,
                       int32_t& aTextureHeight,
                       device::EyeRect& aLeftEye,

app/src/main/cpp/VRBrowser.cpp

@@ -52,6 +52,12 @@ const char* kHaltActivity = "haltActivity";
 const char* kHaltActivitySignature = "(I)V";
 const char* kHandlePoorPerformance = "handlePoorPerformance";
 const char* kHandlePoorPerformanceSignature = "()V";
+const char* kSetExternalVRSurfaceIndex = "setExternalVRSurfaceIndex";
+const char* kSetExternalVRSurfaceIndexSignature = "(I)V";
+const char* kInsertExternalVRSurface = "insertExternalSurface";
+const char* kInsertExternalVRSurfaceSignature = "(IIILandroid/view/Surface;)V";
+const char* kReleaseExternalSurfaces = "releaseExternalVRSurfaces";
+const char* kReleaseExternalSurfacesSignature = "()V";
 
 JNIEnv* sEnv = nullptr;
 jclass sBrowserClass = nullptr;
@@ -77,6 +83,9 @@ jmethodID sAreLayersEnabled = nullptr;
 jmethodID sSetDeviceType = nullptr;
 jmethodID sHaltActivity = nullptr;
 jmethodID sHandlePoorPerformance = nullptr;
+jmethodID sSetExternalVRSurfaceIndex = nullptr;
+jmethodID sInsertExternalVRSurface = nullptr;
+jmethodID sReleaseExternalSurfaces = nullptr;
 }
 
 namespace crow {
@@ -117,6 +126,9 @@ VRBrowser::InitializeJava(JNIEnv* aEnv, jobject aActivity) {
   sSetDeviceType = FindJNIMethodID(sEnv, sBrowserClass, kSetDeviceType, kSetDeviceTypeSignature);
   sHaltActivity = FindJNIMethodID(sEnv, sBrowserClass, kHaltActivity, kHaltActivitySignature);
   sHandlePoorPerformance = FindJNIMethodID(sEnv, sBrowserClass, kHandlePoorPerformance, kHandlePoorPerformanceSignature);
+  sInsertExternalVRSurface = FindJNIMethodID(sEnv, sBrowserClass, kInsertExternalVRSurface, kInsertExternalVRSurfaceSignature);
+  sSetExternalVRSurfaceIndex = FindJNIMethodID(sEnv, sBrowserClass, kSetExternalVRSurfaceIndex, kSetExternalVRSurfaceIndexSignature);
+  sReleaseExternalSurfaces = FindJNIMethodID(sEnv, sBrowserClass, kReleaseExternalSurfaces, kReleaseExternalSurfacesSignature);
 }
 
 void
@@ -151,6 +163,7 @@ VRBrowser::ShutdownJava() {
   sAreLayersEnabled = nullptr;
   sSetDeviceType = nullptr;
   sHaltActivity = nullptr;
+  sInsertExternalVRSurface = nullptr;
   sEnv = nullptr;
 }
 
@@ -339,4 +352,24 @@ VRBrowser::HandlePoorPerformance() {
   CheckJNIException(sEnv, __FUNCTION__);
 }
 
+void
+VRBrowser::SetExternalVRSurfaceId(jint aId) {
+  if (!ValidateMethodID(sEnv, sActivity, sSetExternalVRSurfaceIndex, __FUNCTION__)) { return; }
+  sEnv->CallVoidMethod(sActivity, sSetExternalVRSurfaceIndex, aId);
+  CheckJNIException(sEnv, __FUNCTION__);
+}
+
+void
+VRBrowser::InsertExternalVRSurface(jint aWidth, jint aHeight, jint aIndex, jobject aSurface) {
+  if (!ValidateMethodID(sEnv, sActivity, sInsertExternalVRSurface, __FUNCTION__)) { return; }
+  sEnv->CallVoidMethod(sActivity, sInsertExternalVRSurface, aWidth, aHeight, aIndex, aSurface);
+  CheckJNIException(sEnv, __FUNCTION__);
+}
+
+void
+VRBrowser::ReleaseExternalVRSurfaces() {
+  if (!ValidateMethodID(sEnv, sActivity, sReleaseExternalSurfaces, __FUNCTION__)) { return; }
+  sEnv->CallVoidMethod(sActivity, sReleaseExternalSurfaces);
+  CheckJNIException(sEnv, __FUNCTION__);
+}
 } // namespace crow

app/src/main/cpp/VRBrowser.h

@@ -39,6 +39,9 @@ bool AreLayersEnabled();
 void SetDeviceType(const jint aType);
 void HaltActivity(const jint aReason);
 void HandlePoorPerformance();
+void SetExternalVRSurfaceId(jint aId);
+void InsertExternalVRSurface(jint aWidth, jint aHeight, jint aIndex, jobject aSurface);
+void ReleaseExternalVRSurfaces();
 } // namespace VRBrowser;
 
 } // namespace crow

app/src/main/cpp/moz_external_vr.h

@@ -47,8 +47,8 @@ namespace gfx {
 // and mapped files if we have both release and nightlies
 // running at the same time? Or...what if we have multiple
 // release builds running on same machine? (Bug 1563232)
-#define SHMEM_VERSION "0.0.4"
-static const int32_t kVRExternalVersion = 11;
+#define SHMEM_VERSION "0.0.5"
+static const int32_t kVRExternalVersion = 12;
 
 // We assign VR presentations to groups with a bitmask.
 // Currently, we will only display either content or chrome.
@@ -102,26 +102,26 @@ enum class ControllerCapabilityFlags : uint16_t {
   /**
    * Cap_Position is set if the Gamepad is capable of tracking its position.
    */
-      Cap_Position = 1 << 1,
+  Cap_Position = 1 << 1,
   /**
    * Cap_Orientation is set if the Gamepad is capable of tracking its
    * orientation.
    */
-      Cap_Orientation = 1 << 2,
+  Cap_Orientation = 1 << 2,
   /**
    * Cap_AngularAcceleration is set if the Gamepad is capable of tracking its
    * angular acceleration.
    */
-      Cap_AngularAcceleration = 1 << 3,
+  Cap_AngularAcceleration = 1 << 3,
   /**
    * Cap_LinearAcceleration is set if the Gamepad is capable of tracking its
    * linear acceleration.
    */
-      Cap_LinearAcceleration = 1 << 4,
+  Cap_LinearAcceleration = 1 << 4,
   /**
    * Cap_All used for validity checking during IPC serialization
    */
-      Cap_All = (1 << 5) - 1
+  Cap_All = (1 << 5) - 1
 };
 
 #endif  // ifndef MOZILLA_INTERNAL_API
@@ -133,75 +133,75 @@ enum class VRDisplayCapabilityFlags : uint16_t {
   /**
    * Cap_Position is set if the VRDisplay is capable of tracking its position.
    */
-      Cap_Position = 1 << 1,
+  Cap_Position = 1 << 1,
   /**
    * Cap_Orientation is set if the VRDisplay is capable of tracking its
    * orientation.
    */
-      Cap_Orientation = 1 << 2,
+  Cap_Orientation = 1 << 2,
   /**
    * Cap_Present is set if the VRDisplay is capable of presenting content to an
    * HMD or similar device.  Can be used to indicate "magic window" devices that
    * are capable of 6DoF tracking but for which requestPresent is not
    * meaningful. If false then calls to requestPresent should always fail, and
    * getEyeParameters should return null.
    */
-      Cap_Present = 1 << 3,
+  Cap_Present = 1 << 3,
   /**
    * Cap_External is set if the VRDisplay is separate from the device's
    * primary display. If presenting VR content will obscure
    * other content on the device, this should be un-set. When
    * un-set, the application should not attempt to mirror VR content
    * or update non-VR UI because that content will not be visible.
    */
-      Cap_External = 1 << 4,
+  Cap_External = 1 << 4,
   /**
    * Cap_AngularAcceleration is set if the VRDisplay is capable of tracking its
    * angular acceleration.
    */
-      Cap_AngularAcceleration = 1 << 5,
+  Cap_AngularAcceleration = 1 << 5,
   /**
    * Cap_LinearAcceleration is set if the VRDisplay is capable of tracking its
    * linear acceleration.
    */
-      Cap_LinearAcceleration = 1 << 6,
+  Cap_LinearAcceleration = 1 << 6,
   /**
    * Cap_StageParameters is set if the VRDisplay is capable of room scale VR
    * and can report the StageParameters to describe the space.
    */
-      Cap_StageParameters = 1 << 7,
+  Cap_StageParameters = 1 << 7,
   /**
    * Cap_MountDetection is set if the VRDisplay is capable of sensing when the
    * user is wearing the device.
    */
-      Cap_MountDetection = 1 << 8,
+  Cap_MountDetection = 1 << 8,
   /**
    * Cap_PositionEmulated is set if the VRDisplay is capable of setting a
    * emulated position (e.g. neck model) even if still doesn't support 6DOF
    * tracking.
    */
-      Cap_PositionEmulated = 1 << 9,
+  Cap_PositionEmulated = 1 << 9,
   /**
    * Cap_Inline is set if the device can be used for WebXR inline sessions
    * where the content is displayed within an element on the page.
    */
-      Cap_Inline = 1 << 10,
+  Cap_Inline = 1 << 10,
   /**
    * Cap_ImmersiveVR is set if the device can give exclusive access to the
    * XR device display and that content is not intended to be integrated
    * with the user's environment
    */
-      Cap_ImmersiveVR = 1 << 11,
+  Cap_ImmersiveVR = 1 << 11,
   /**
    * Cap_ImmersiveAR is set if the device can give exclusive access to the
    * XR device display and that content is intended to be integrated with
    * the user's environment.
    */
-      Cap_ImmersiveAR = 1 << 12,
+  Cap_ImmersiveAR = 1 << 12,
   /**
    * Cap_All used for validity checking during IPC serialization
    */
-      Cap_All = (1 << 13) - 1
+  Cap_All = (1 << 13) - 1
 };
 
 #ifdef MOZILLA_INTERNAL_API
@@ -372,7 +372,8 @@ enum class VRLayerTextureType : uint16_t {
   LayerTextureType_None = 0,
   LayerTextureType_D3D10SurfaceDescriptor = 1,
   LayerTextureType_MacIOSurface = 2,
-  LayerTextureType_GeckoSurfaceTexture = 3
+  LayerTextureType_GeckoSurfaceTexture = 3,
+  LayerTextureType_ExternalVRSurface = 4
 };
 
 struct VRLayer_2D_Content {