wasm: Add support for Unicode categories

Author: pdubroy

packages/miniohm-js/index.js

@@ -3,6 +3,32 @@
 const WASM_PAGE_SIZE = 64 * 1024;
 const INPUT_BUFFER_OFFSET = WASM_PAGE_SIZE;
 
+// Bit flags for Unicode categories, based on the order that they appear in
+// https://www.unicode.org/Public/16.0.0/ucd/extracted/DerivedGeneralCategory.txt
+
+const UnicodeCategoryNames = [
+  'Cn', // Unassigned
+  'Lu', // Uppercase_Letter
+  'Ll', // Lowercase_Letter
+  'Lt', // Titlecase_Letter
+  'Lm', // Modifier_Letter
+  'Lo', // Other_Letter
+];
+
+const utf8 = new TextDecoder('utf-8');
+
+function regexFromCategoryBitmap(bitmap) {
+  const cats = [];
+  for (let i = 0; i < 32; i++) {
+    const mask = 1 << i;
+    if (bitmap & mask) cats.push(UnicodeCategoryNames[i]);
+  }
+  return new RegExp(
+      cats.map(cat => `\\p{${cat}}`).join('|'),
+      'uy', // u: unicode, y: sticky
+  );
+}
+
 function assert(cond, msg) {
   if (!cond) {
     throw new Error(msg ?? 'assertion failed');
@@ -32,12 +58,38 @@ export class WasmMatcher {
           return name[0] === name[0].toUpperCase();
         },
         fillInputBuffer: this._fillInputBuffer.bind(this),
+        matchUnicodeChar: (catBitmap, pos) => {
+          const re = regexFromCategoryBitmap(catBitmap);
+          return re.test(this._nextCodePoint());
+        },
       },
     };
     this._ruleIds = new Map();
     this._ruleNames = [];
   }
 
+  // Return a JavaScript string containing the next code point from the input
+  // buffer, and advance pos past it.
+  _nextCodePoint() {
+    const {pos, memory} = this._instance.exports;
+    const offset = pos.value;
+    const byteArr = new Uint8Array(memory.buffer, INPUT_BUFFER_OFFSET + offset);
+    const firstByte = byteArr[0];
+    let len;
+    if ((firstByte & 0b10000000) === 0) {
+      len = 1;
+    } else if ((firstByte & 0b11100000) === 0b11000000) {
+      len = 2;
+    } else if ((firstByte & 0b11110000) === 0b11100000) {
+      len = 3;
+    } else {
+      len = 4;
+    }
+    const str = utf8.decode(byteArr.subarray(0, len));
+    pos.value += len;
+    return str;
+  }
+
   _extractRuleIds(module) {
     const sections = WebAssembly.Module.customSections(module, 'ruleNames');
     if (sections.length === 0) {

packages/wasm/runtime/ohmRuntime.ts

@@ -3,6 +3,7 @@ type ApplyResult = bool;
 declare function fillInputBuffer(offset: i32, maxLen: i32): i32;
 declare function printI32(val: i32): void;
 declare function isRuleSyntactic(ruleId: i32): bool;
+declare function matchUnicodeChar(categoryBitmap: i32): bool;
 
 @inline const IMPLICIT_SPACE_SKIPPING = true;
 
@@ -50,6 +51,10 @@ let rightmostFailurePos: i32 = 0;
 let sp: usize = 0;
 let bindings: Array<i32> = new Array<i32>();
 
+export function dummy(i: i32): void {
+  printI32(i);
+}
+
 @inline function max<T>(a: T, b: T): T {
   return a > b ? a : b;
 }
@@ -289,3 +294,8 @@ export function getCstRoot(): usize {
   // TODO: Figure out how to handle this w.r.t. leading and trailing space.
   return bindings[0];
 }
+
+// TODO: Find a way to call this directly from generated code.
+export function doMatchUnicodeChar(categoryBitmap: i32): bool {
+  return matchUnicodeChar(categoryBitmap)
+}

packages/wasm/src/index.js

@@ -792,6 +792,7 @@ export class Compiler {
     asm.addBlocktype([w.valtype.i32], [w.valtype.i32]);
     asm.addBlocktype([], [w.valtype.i32]); // Rule eval
     // (global $runtime/ohmRuntime/pos (mut i32) (i32.const 0))
+    // (global $runtime/ohmRuntime/rightmostFailurePos (mut i32) (i32.const 0))
     // (global $runtime/ohmRuntime/sp (mut i32) (i32.const 0))
     // (global $~lib/shared/runtime/Runtime.Stub i32 (i32.const 0))
     // (global $~lib/shared/runtime/Runtime.Minimal i32 (i32.const 1))
@@ -1630,35 +1631,50 @@ export class Compiler {
         const isLowercase = 'a' <= c && c <= 'z';
         const isUppercase = 'A' <= c && c <= 'Z';
         if ((exp.category === 'Lu' && isUppercase) || (exp.category === 'Ll' && isLowercase)) {
-          return w.labelidx(asm.depthOf('innerSuccess'));
+          return w.labelidx(asm.depthOf('fastSuccess'));
         }
         return w.labelidx(asm.depthOf('failure'));
       });
+
     this.wrapTerminalLike(() => {
       asm.block(
           w.blocktype.empty,
           () => {
             asm.block(
                 w.blocktype.empty,
                 () => {
-                  asm.currCharCode();
-                  asm.brTable(makeLabels(), w.labelidx(asm.depthOf('default')));
+                  // Fast path: a jump table for ASCII characters.
+                  asm.block(
+                      w.blocktype.empty,
+                      () => {
+                        asm.currCharCode();
+                        asm.brTable(makeLabels(), w.labelidx(asm.depthOf('default')));
+                      },
+                      'default',
+                  );
+                  // Fall through: not an ASCII character.
+
+                  // Push the arg: a bitmap indicating the categories.
+                  // prettier-ignore
+                  switch (exp.category) {
+                    case 'Lu': asm.i32Const(1 << 1); break;
+                    case 'Ll': asm.i32Const(1 << 2); break;
+                    case 'Ltmo': asm.i32Const((1 << 3) | (1 << 4) | (1 << 5)); break;
+                    default: assert(false, 'not handled');
+                  }
+                  asm.callPrebuiltFunc('doMatchUnicodeChar');
+                  asm.ifElse(
+                      w.blocktype.empty,
+                      () => asm.break(asm.depthOf('slowSuccess')),
+                      () => asm.break(asm.depthOf('failure')),
+                  );
                 },
-                'default',
+                'fastSuccess',
             );
-            // Check for 0xff (end)
-            asm.currCharCode();
-            asm.i32Const(0xff);
-            asm.i32Eq();
-            asm.condBreak(asm.depthOf('failure'));
-
-            // Otherwise, trap.
-            // TODO: Replace this with a proper, out-of-line implementation.
-            asm.emit(instr.unreachable);
+            asm.incPos();
           },
-          'innerSuccess',
+          'slowSuccess',
       );
-      asm.incPos();
     });
   }
 }

packages/wasm/test/test-failurePos.js

@@ -76,19 +76,25 @@ function arbitraryEdit(input) {
 // - for some randomly-corrupted input, which fails to parse
 // - the rightmostFailurePosition reported by a JS matcher and a Wasm matcher
 //   is the same.
-const sameFailurePos = (t, wasmMatcher) =>
-  fc.property(arbitraryEdit(validInput), input => {
+function sameFailurePos(wasmMatcher) {
+  return fc.property(arbitraryEdit(validInput), input => {
     wasmMatcher.setInput(input);
     fc.pre(wasmMatcher.match() === 0);
     assert.equal(
         ns.LiquidHTML.match(input).getRightmostFailurePosition(),
         wasmMatcher.getRightmostFailurePosition(),
     );
   });
+}
 
 test('failure pos (fast-check)', async t => {
   const m = await wasmMatcherForGrammar(ns.LiquidHTML);
-  t.notThrows(() => fc.assert(sameFailurePos(t, m), {verbose, includeErrorInReport: true}));
+  const details = fc.check(sameFailurePos(m), {
+    includeErrorInReport: true,
+    interruptAfterTimeLimit: 1000,
+  });
+  t.log(`numRuns: ${details.numRuns}`);
+  t.is(details.failed, false, `${fc.defaultReportMessage(details)}`);
 });
 
 test('failure pos: basic 1', async t => {

packages/wasm/test/test-wasm.js

@@ -1,4 +1,6 @@
 import test from 'ava';
+import assert from 'node:assert/strict';
+import * as fc from 'fast-check';
 import * as ohm from 'ohm-js';
 import {performance} from 'perf_hooks';
 
@@ -34,14 +36,6 @@ function unparse(m, root) {
   return ans;
 }
 
-// const dumpMemoTable = pos => {
-//   const arr = [];
-//   for (let i = 0; i < 6; i++) {
-//     arr.push(view.getUint32(pos * Constants.MEMO_COL_SIZE_BYTES + i * 4, true));
-//   }
-//   console.log(arr.map(v => v.toString(16).padStart(8, '0')).join(' '));
-// };
-
 test('input in memory', async t => {
   const g = ohm.grammar('G { start = "x" }');
   const matcher = await wasmMatcherForGrammar(g);
@@ -960,15 +954,23 @@ test('space skipping & lex', async t => {
   }
 });
 
-test('unicode built-ins', async t => {
-  const g = ohm.grammar(`
-    G {
-      Start = lower upper
-    }`);
+// fast-check's stringMatching combiner doesn't support unicode regexes.
+const arbitraryStringMatching = regex =>
+  fc.string({maxLength: 2, unit: 'binary'}).filter(str => regex.test(str));
+
+test('unicode built-ins: non-ASII (fast-check)', async t => {
+  const g = ohm.grammar('G { Start = letter letter }');
   const m = await wasmMatcherForGrammar(g);
-  t.is(matchWithInput(m, 'aA'), 1);
-  t.is(matchWithInput(m, ' aZ'), 1);
-  t.is(matchWithInput(m, ' zA'), 1);
-  t.is(matchWithInput(m, 'a@'), 0);
-  t.is(matchWithInput(m, 'a['), 0);
+  const hasExpectedResult = wasmMatcher => {
+    return fc.property(arbitraryStringMatching(/^\p{L}\p{L}$/u), str => {
+      wasmMatcher.setInput(str);
+      assert.equal(wasmMatcher.match(), 1);
+    });
+  };
+  const details = fc.check(hasExpectedResult(m), {
+    includeErrorInReport: true,
+    interruptAfterTimeLimit: 200,
+  });
+  t.log(`numRuns: ${details.numRuns}`);
+  t.is(details.failed, false, `${fc.defaultReportMessage(details)}`);
 });