feat: Hand Tracking Implementation

stalgiag · Oct 16, 2024 · 1a5fd65 · 1a5fd65
1 parent 927cc74
commit 1a5fd65
Show file tree

Hide file tree

Showing 7 changed files with 283 additions and 2 deletions.
diff --git a/examples/ar/handtracking-finger/example.js b/examples/ar/handtracking-finger/example.js
@@ -0,0 +1,13 @@
+function setup() {
+  createCanvas(windowWidth, windowHeight, AR);
+  describe('A sphere on your right index finger');
+  mainHandMode(RIGHT);
+}
+
+function draw() {
+  normalMaterial();
+  push();
+  translate(finger.x, finger.y, finger.z);
+  sphere(0.01);
+  pop();
+}
diff --git a/examples/ar/handtracking-finger/index.html b/examples/ar/handtracking-finger/index.html
@@ -0,0 +1,20 @@
+<!doctype html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <meta
+      name="viewport"
+      content="width=device-width, initial-scale=1, user-scalable=no"
+    />
+    <meta name="mobile-web-app-capable" content="yes" />
+    <meta name="apple-mobile-web-app-capable" content="yes" />
+
+    <title>AR EXAMPLE: HandTracking Finger</title>
+    <script src="../../../node_modules/p5/lib/p5.js"></script>
+    <script src="../../../dist/p5xr.min.js"></script>
+  </head>
+  <body>
+    <header></header>
+    <script src="example.js"></script>
+  </body>
+</html>
diff --git a/examples/ar/handtracking-hands/example.js b/examples/ar/handtracking-hands/example.js
@@ -0,0 +1,15 @@
+function setup() {
+  createCanvas(windowWidth, windowHeight, AR);
+  describe("Spheres on all the skeletal joints of the hand.");
+}
+
+function draw() {
+  normalMaterial();
+  for (let i = 0; i < hands.length; i++) {
+    const joint = hands[i];
+    push();
+    translate(joint.x, joint.y, joint.z);
+    sphere(0.01);
+    pop();
+  }
+}
diff --git a/examples/ar/handtracking-hands/index.html b/examples/ar/handtracking-hands/index.html
@@ -0,0 +1,20 @@
+<!doctype html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <meta
+      name="viewport"
+      content="width=device-width, initial-scale=1, user-scalable=no"
+    />
+    <meta name="mobile-web-app-capable" content="yes" />
+    <meta name="apple-mobile-web-app-capable" content="yes" />
+
+    <title>AR EXAMPLE: HandTracking Hands</title>
+    <script src="../../../node_modules/p5/lib/p5.js"></script>
+    <script src="../../../dist/p5xr.min.js"></script>
+  </head>
+  <body>
+    <header></header>
+    <script src="example.js"></script>
+  </body>
+</html>
diff --git a/src/app.js b/src/app.js
@@ -1,5 +1,6 @@
 import * as constants from './p5xr/core/constants';
 import './p5xr/core/p5overrides';
+import './p5xr/features/handtracking';
 import p5vr from './p5xr/p5vr/p5vr';
 import p5ar from './p5xr/p5ar/p5ar';
 

diff --git a/src/p5xr/core/p5xr.js b/src/p5xr/core/p5xr.js
@@ -1,6 +1,7 @@
 import p5xrViewer from './p5xrViewer';
 import p5xrButton from './p5xrButton';
 import p5xrInput from './p5xrInput';
+import '../features/handtracking';
 
 /**
  * p5vr class holds all state and methods that are specific to VR
@@ -27,7 +28,7 @@ import p5xrInput from './p5xrInput';
  */
 export default class p5xr {
   constructor(options = {}) {
-    const { requiredFeatures = [], optionalFeatures = [] } = options;
+    const { requiredFeatures = [], optionalFeatures = ['hand-tracking'] } = options;
 
     this.xrDevice = null;
     this.isVR = null;
@@ -360,6 +361,11 @@ export default class p5xr {
     const viewer = frame.getViewerPose(this.xrRefSpace);
     const glLayer = session.renderState.baseLayer;
     this.frame = frame;
+
+    for (const inputSource of session.inputSources) {
+      _handleHandInput(frame, this.xrRefSpace, inputSource);
+    }
+
     // Getting the pose may fail if, for example, tracking is lost. So we
     // have to check to make sure that we got a valid pose before attempting
     // to render with it. If not in this case we'll just leave the
@@ -402,7 +408,6 @@ export default class p5xr {
           viewport.height * scaleFactor,
         );
         this.__updateViewport(viewport);
-
         this.__drawEye(i);
         i++;
       }

diff --git a/src/p5xr/features/handtracking.js b/src/p5xr/features/handtracking.js
@@ -0,0 +1,207 @@
+/*
+ * @module HandTracking
+ *
+ * Hand Input Implementation (https://www.w3.org/TR/webxr-hand-input-1/)
+ */
+
+/**
+ * One of the five possible values for mainFingerMode (THUMB, INDEX, MIDDLE, RING, PINKY).
+ * Represents the thumb.
+ * @property {number} THUMB
+ * @final
+ */
+export const THUMB = 0;
+
+/**
+ * One of the five possible values for mainFingerMode (THUMB, INDEX, MIDDLE, RING, PINKY).
+ * Represents the index finger.
+ * @property {number} INDEX
+ * @final
+ */
+export const INDEX = 1;
+
+/**
+ * One of the five possible values for mainFingerMode (THUMB, INDEX, MIDDLE, RING, PINKY).
+ * Represents the middle finger.
+ * @property {number} MIDDLE
+ * @final
+ */
+export const MIDDLE = 2;
+
+/**
+ * One of the five possible values for mainFingerMode (THUMB, INDEX, MIDDLE, RING, PINKY).
+ * Represents the ring finger.
+ * @property {number} RING
+ * @final
+ */
+export const RING = 3;
+
+/**
+ * One of the five possible values for mainFingerMode (THUMB, INDEX, MIDDLE, RING, PINKY).
+ * Represents the pinky finger.
+ * @property {number} PINKY
+ * @final
+ */
+export const PINKY = 4;
+
+p5.prototype._mainHandMode = p5.prototype.RIGHT;
+
+const p = p5.prototype;
+p5.prototype.hands = Array.from({ length: 50 }, () => ({
+  x: 0,
+  y: 0,
+  z: 0,
+  rad: 0.1,
+  mat: new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]), // Identity matrix
+}));
+
+// TODO: implement other fingers, currently index only
+p5.prototype.fingerLeft = p.hands[9];
+p5.prototype.fingerRight = p.hands[9 + 25];
+
+/**
+ * An Object representing the main finger tip
+ * By default it is the right index finger tip
+ */
+p5.prototype.finger = p.fingerRight;
+p5.prototype.fingerMain = p.fingerRight;
+p5.prototype.fingerAlt = p.fingerLeft;
+
+p5.prototype.fingers = [
+  p.hands[4],
+  p.hands[9],
+  p.hands[14],
+  p.hands[19],
+  p.hands[24],
+  p.hands[4 + 25],
+  p.hands[9 + 25],
+  p.hands[14 + 25],
+  p.hands[19 + 25],
+  p.hands[24 + 25],
+];
+
+p5.prototype.fingersLeft = p.fingers.slice(0, 5);
+p5.prototype.fingersRight = p.fingers.slice(5, 10);
+
+p5.prototype.fingersMain = p.fingersRight;
+p5.prototype.fingersAlt = p.fingersLeft;
+
+p5.prototype.handLeft = p.hands.slice(0, 25);
+p5.prototype.handRight = p.hands.slice(25, 50);
+p5.prototype.handMain = p.handRight;
+p5.prototype.hand = p.handMain;
+p5.prototype.handAlt = p.handLeft;
+
+p5.prototype.flatMatrices = new Float32Array(16 * 25); // one 4x4 mat for 25 joints
+p5.prototype.flatMatricesFull = new Float32Array(16 * 50); // one 4x4 mat for 25 joints
+p5.prototype.radii = new Float32Array(25);
+p5.prototype.radiiFull = new Float32Array(50);
+
+p5.prototype._pinchTreshold = 25e-3; // about 25 mm
+p5.prototype.fingersArePinched = false;
+p5.prototype.pFingersArePinched = false;
+
+p5.prototype._handleOnPinch = function () {
+  const finger = this.fingerMain;
+  const thumb = this.fingersRight[0];
+  const dist = this.dist(
+    finger.x,
+    finger.y,
+    finger.z,
+    thumb.x,
+    thumb.y,
+    thumb.z,
+  );
+
+  this._setProperty('fingersArePinched', dist < this._pinchTreshold);
+
+  if (!this.pFingersArePinched && this.fingersArePinched) {
+    const context = this._isGlobal ? window : this;
+    if (typeof context.fingersPinched === 'function') {
+      context.fingersPinched();
+    }
+  }
+};
+
+p5.prototype._handleHandInput = function (frame, refSpace, inputSource) {
+  // TODO: Refactor to only do this once
+  this._setProperty('flatMatrices', this.flatMatrices);
+  this._setProperty('radii', this.radii);
+  this._setProperty('hands', this.hands);
+  this._setProperty('fingerLeft', this.fingerLeft);
+  this._setProperty('fingerRight', this.fingerRight);
+  this._setProperty('finger', this.finger);
+  this._setProperty('fingerMain', this.fingerMain);
+  this._setProperty('fingerAlt', this.fingerAlt);
+
+  this._setProperty('fingers', this.fingers);
+  this._setProperty('fingersLeft', this.fingersLeft);
+  this._setProperty('fingersRight', this.fingersRight);
+  this._setProperty('fingersMain', this.fingersMain);
+  this._setProperty('fingersAlt', this.fingersAlt);
+
+  this._setProperty('handLeft', this.handLeft);
+  this._setProperty('handRight', this.handRight);
+  this._setProperty('hand', this.hand);
+  this._setProperty('handMain', this.handMain);
+  this._setProperty('handAlt', this.handAlt);
+
+  if (!inputSource.hand) {
+    return;
+  }
+
+  if (
+    !frame.fillPoses(inputSource.hand.values(), refSpace, this.flatMatrices)
+  ) {
+    return;
+  }
+
+  // eslint-disable-next-line no-unused-vars
+  const areRadiiFilled = frame.fillJointRadii(
+    inputSource.hand.values(),
+    this.radii,
+  ); // todo - handle when we don't get them
+  if (!areRadiiFilled) {
+    console.log('radii not filled');
+  }
+
+  const off = inputSource.handedness === 'left' ? 0 : 25;
+  this.flatMatricesFull.set(this.flatMatrices, off * 16);
+  this.radiiFull.set(this.radii, off);
+
+  for (let i = 0; i < 25; i++) {
+    const mat = this.flatMatrices.slice(i * 16, (i + 1) * 16);
+    this.hands[i + off].x = mat[12];
+    this.hands[i + off].y = mat[13];
+    this.hands[i + off].z = mat[14];
+    this.hands[i + off].mat4 = mat;
+    this.hands[i + off].rad = radii[i];
+  }
+
+  this._handleOnPinch();
+  this._setProperty('pFingersArePinched', this.fingersArePinched);
+};
+
+/**
+ * Changes default handedness
+ *
+ * Adjusts variables for `finger`, `fingerMain`, `fingerAlt`, `hand`, `handMain`, `handAlt`
+ * @param  {Constant} mode either RIGHT, LEFT
+ */
+p5.prototype.mainHandMode = function (mode) {
+  if (mode === this.LEFT) {
+    this.finger = this.fingerLeft;
+    this.fingerMain = this.fingerLeft;
+    this.fingerAlt = this.fingerRight;
+    this.hand = this.handLeft;
+    this.handMain = this.handLeft;
+    this.handAlt = this.handRight;
+  } else if (mode === this.RIGHT) {
+    this.finger = this.fingerRight;
+    this.fingerMain = this.fingerRight;
+    this.fingerAlt = this.fingerLeft;
+    this.hand = this.handRight;
+    this.handMain = this.handRight;
+    this.handAlt = this.handLeft;
+  }
+};