Skip to content

Commit

Permalink
Merge pull request #187 from EpicGamesExt/xr-backport
Browse files Browse the repository at this point in the history 
Make XR changes compatible with UE 5.4.2
  • Loading branch information
@lukehb
lukehb authored Jul 11, 2024
2 parents 87595cb + 6113921 commit e34dc2b
Show file tree
Hide file tree
Showing 3 changed files with 28 additions and 106 deletions.
2 changes: 1 addition & 1 deletion Frontend/library/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@epicgames-ps/lib-pixelstreamingfrontend-ue5.4",
"version": "1.1.1",
"version": "1.1.2",
"description": "Frontend library for Unreal Engine 5.4 Pixel Streaming",
"main": "dist/lib-pixelstreamingfrontend.js",
"module": "dist/lib-pixelstreamingfrontend.esm.js",
Expand Down
2 changes: 1 addition & 1 deletion Frontend/library/src/Inputs/XRGamepadController.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ export class XRGamepadController {
if (currButton.pressed) {
// press
let isRepeat = prevButton.pressed ? 1 : 0;
this.toStreamerMessagesProvider.toStreamerHandlers.get('XRButtonPressed')([handedness, i, isRepeat, currButton.value]);
this.toStreamerMessagesProvider.toStreamerHandlers.get('XRButtonPressed')([handedness, i, isRepeat]);
} else if (prevButton.pressed) {
this.toStreamerMessagesProvider.toStreamerHandlers.get('XRButtonReleased')([handedness, i, 0]);
}
Expand Down
130 changes: 26 additions & 104 deletions Frontend/library/src/WebXR/WebXRController.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,9 @@ export class WebXRController {
private xrRefSpace: XRReferenceSpace;
private gl: WebGL2RenderingContext;
private xrViewerPose : XRViewerPose = null;
// Used for comparisons to ensure two numbers are close enough.
private EPSILON = 0.0000001;

private positionLocation: number;
private texcoordLocation: number;
private textureOffsetUniform: WebGLUniformLocation;

private positionBuffer: WebGLBuffer;
private texcoordBuffer: WebGLBuffer;
Expand All @@ -31,12 +28,6 @@ export class WebXRController {
private leftView: XRView = null;
private rightView: XRView = null;

// Store the HMD data we have last sent (not all of it is needed every frame unless it changes)
private lastSentLeftEyeProj: Float32Array = null;
private lastSentRightEyeProj: Float32Array = null;
private lastSentRelativeLeftEyePos: DOMPointReadOnly = null;
private lastSentRelativeRightEyePos: DOMPointReadOnly = null;

onSessionStarted: EventTarget;
onSessionEnded: EventTarget;
onFrame: EventTarget;
Expand Down Expand Up @@ -324,109 +315,40 @@ export class WebXRController {
this.onSessionStarted.dispatchEvent(new Event('xrSessionStarted'));
}

areArraysEqual(a: Float32Array, b: Float32Array) : boolean {
return a.length === b.length && a.every((element, index) => Math.abs(element - b[index]) <= this.EPSILON);
}

arePointsEqual(a: DOMPointReadOnly, b: DOMPointReadOnly) : boolean {
return Math.abs(a.x - b.x) >= this.EPSILON && Math.abs(a.y - b.y) >= this.EPSILON && Math.abs(a.z - b.z) >= this.EPSILON;
}

sendXRDataToUE() {
if(this.leftView == null || this.rightView == null) {
return;
}

// We selectively send either the `XREyeViews` or `XRHMDTransform`
// messages over the datachannel. The reason for this selective sending is that
// the `XREyeViews` is a much larger message and changes infrequently (e.g. only when user changes headset IPD).
// Therefore, we only need to send it once on startup and then any time it changes.
// The rest of the time we can send the `XRHMDTransform` message.
let shouldSendEyeViews = this.lastSentLeftEyeProj == null ||
this.lastSentRightEyeProj == null ||
this.lastSentRelativeLeftEyePos == null ||
this.lastSentRelativeRightEyePos == null;

const leftEyeTrans = this.leftView.transform.matrix;
const leftEyeProj = this.leftView.projectionMatrix;
const rightEyeTrans = this.rightView.transform.matrix;
const rightEyeProj = this.rightView.projectionMatrix;
const hmdTrans = this.xrViewerPose.transform.matrix;

// Check if projection matrices have changed
if(!shouldSendEyeViews && this.lastSentLeftEyeProj != null && this.lastSentRightEyeProj != null) {
let leftEyeProjUnchanged = this.areArraysEqual(leftEyeProj, this.lastSentLeftEyeProj);
let rightEyeProjUnchanged = this.areArraysEqual(rightEyeProj, this.lastSentRightEyeProj);
shouldSendEyeViews = leftEyeProjUnchanged == false || rightEyeProjUnchanged == false;
}

let leftEyeRelativePos = new DOMPointReadOnly(
this.leftView.transform.position.x - this.xrViewerPose.transform.position.x,
this.leftView.transform.position.y - this.xrViewerPose.transform.position.y,
this.leftView.transform.position.z - this.xrViewerPose.transform.position.z,
1.0
);

let rightEyeRelativePos = new DOMPointReadOnly(
this.leftView.transform.position.x - this.xrViewerPose.transform.position.x,
this.leftView.transform.position.y - this.xrViewerPose.transform.position.y,
this.leftView.transform.position.z - this.xrViewerPose.transform.position.z,
1.0
);

// Check if relative eye pos has changed (e.g IPD changed)
if(!shouldSendEyeViews && this.lastSentRelativeLeftEyePos != null && this.lastSentRelativeRightEyePos != null) {
let leftEyePosUnchanged = this.arePointsEqual(leftEyeRelativePos, this.lastSentRelativeLeftEyePos);
let rightEyePosUnchanged = this.arePointsEqual(rightEyeRelativePos, this.lastSentRelativeRightEyePos);
shouldSendEyeViews = leftEyePosUnchanged == false || rightEyePosUnchanged == false;
// Note: We are not checking if EyeView rotation changes (afaict no HMD's support changing this value at runtime).
}

if(shouldSendEyeViews) {
// send transform (4x4) and projection matrix (4x4) data for each eye (left first, then right)
// prettier-ignore
this.webRtcController.streamMessageController.toStreamerHandlers.get('XREyeViews')([
// Left eye 4x4 transform matrix
leftEyeTrans[0], leftEyeTrans[4], leftEyeTrans[8], leftEyeTrans[12],
leftEyeTrans[1], leftEyeTrans[5], leftEyeTrans[9], leftEyeTrans[13],
leftEyeTrans[2], leftEyeTrans[6], leftEyeTrans[10], leftEyeTrans[14],
leftEyeTrans[3], leftEyeTrans[7], leftEyeTrans[11], leftEyeTrans[15],
// Left eye 4x4 projection matrix
leftEyeProj[0], leftEyeProj[4], leftEyeProj[8], leftEyeProj[12],
leftEyeProj[1], leftEyeProj[5], leftEyeProj[9], leftEyeProj[13],
leftEyeProj[2], leftEyeProj[6], leftEyeProj[10], leftEyeProj[14],
leftEyeProj[3], leftEyeProj[7], leftEyeProj[11], leftEyeProj[15],
// Right eye 4x4 transform matrix
rightEyeTrans[0], rightEyeTrans[4], rightEyeTrans[8], rightEyeTrans[12],
rightEyeTrans[1], rightEyeTrans[5], rightEyeTrans[9], rightEyeTrans[13],
rightEyeTrans[2], rightEyeTrans[6], rightEyeTrans[10], rightEyeTrans[14],
rightEyeTrans[3], rightEyeTrans[7], rightEyeTrans[11], rightEyeTrans[15],
// right eye 4x4 projection matrix
rightEyeProj[0], rightEyeProj[4], rightEyeProj[8], rightEyeProj[12],
rightEyeProj[1], rightEyeProj[5], rightEyeProj[9], rightEyeProj[13],
rightEyeProj[2], rightEyeProj[6], rightEyeProj[10], rightEyeProj[14],
rightEyeProj[3], rightEyeProj[7], rightEyeProj[11], rightEyeProj[15],
// HMD 4x4 transform
hmdTrans[0], hmdTrans[4], hmdTrans[8], hmdTrans[12],
hmdTrans[1], hmdTrans[5], hmdTrans[9], hmdTrans[13],
hmdTrans[2], hmdTrans[6], hmdTrans[10], hmdTrans[14],
hmdTrans[3], hmdTrans[7], hmdTrans[11], hmdTrans[15],
]);
this.lastSentLeftEyeProj = leftEyeProj;
this.lastSentRightEyeProj = rightEyeProj;
this.lastSentRelativeLeftEyePos = leftEyeRelativePos;
this.lastSentRelativeRightEyePos = rightEyeRelativePos;
}
else {
// If we don't need to the entire eye views being sent just send the HMD transform
this.webRtcController.streamMessageController.toStreamerHandlers.get('XRHMDTransform')([
// HMD 4x4 transform
hmdTrans[0], hmdTrans[4], hmdTrans[8], hmdTrans[12],
hmdTrans[1], hmdTrans[5], hmdTrans[9], hmdTrans[13],
hmdTrans[2], hmdTrans[6], hmdTrans[10], hmdTrans[14],
hmdTrans[3], hmdTrans[7], hmdTrans[11], hmdTrans[15],
]);
}
const rightEyeProj = this.rightView.projectionMatrix;;

// send transform (4x4) and projection matrix (4x4) data for each eye (left first, then right)
// prettier-ignore
this.webRtcController.streamMessageController.toStreamerHandlers.get('XREyeViews')([
// Left eye 4x4 transform matrix
leftEyeTrans[0], leftEyeTrans[4], leftEyeTrans[8], leftEyeTrans[12],
leftEyeTrans[1], leftEyeTrans[5], leftEyeTrans[9], leftEyeTrans[13],
leftEyeTrans[2], leftEyeTrans[6], leftEyeTrans[10], leftEyeTrans[14],
leftEyeTrans[3], leftEyeTrans[7], leftEyeTrans[11], leftEyeTrans[15],
// Left eye 4x4 projection matrix
leftEyeProj[0], leftEyeProj[4], leftEyeProj[8], leftEyeProj[12],
leftEyeProj[1], leftEyeProj[5], leftEyeProj[9], leftEyeProj[13],
leftEyeProj[2], leftEyeProj[6], leftEyeProj[10], leftEyeProj[14],
leftEyeProj[3], leftEyeProj[7], leftEyeProj[11], leftEyeProj[15],
// Right eye 4x4 transform matrix
rightEyeTrans[0], rightEyeTrans[4], rightEyeTrans[8], rightEyeTrans[12],
rightEyeTrans[1], rightEyeTrans[5], rightEyeTrans[9], rightEyeTrans[13],
rightEyeTrans[2], rightEyeTrans[6], rightEyeTrans[10], rightEyeTrans[14],
rightEyeTrans[3], rightEyeTrans[7], rightEyeTrans[11], rightEyeTrans[15],
// right eye 4x4 projection matrix
rightEyeProj[0], rightEyeProj[4], rightEyeProj[8], rightEyeProj[12],
rightEyeProj[1], rightEyeProj[5], rightEyeProj[9], rightEyeProj[13],
rightEyeProj[2], rightEyeProj[6], rightEyeProj[10], rightEyeProj[14],
rightEyeProj[3], rightEyeProj[7], rightEyeProj[11], rightEyeProj[15]
]);
}

onXrFrame(time: DOMHighResTimeStamp, frame: XRFrame) {
Expand Down

0 comments on commit e34dc2b

Please sign in to comment.