[js/web] Add package export tests for onnxruntime-web

js/web/test/e2e/exports/README.md

@@ -0,0 +1,57 @@
+This folder includes test data, scripts and source code that used to test the export functionality of onnxruntime-web package.
+
+## nextjs-default
+
+### Summary
+
+This is a Next.js application created by `npx create-next-app@latest` using the following config:
+
+```
+<ORT_ROOT>\js\web\test\e2e\exports\testcases>npx create-next-app@latest
+
+√ What is your project named? ... nextjs-default
+√ Would you like to use TypeScript? ... No / Yes
+√ Would you like to use ESLint? ... No / Yes
+√ Would you like to use Tailwind CSS? ... No / Yes
+√ Would you like your code inside a `src/` directory? ... No / Yes
+√ Would you like to use App Router? (recommended) ... No / Yes
+√ Would you like to use Turbopack for `next dev`? ... No / Yes
+√ Would you like to customize the import alias (`@/*` by default)? ... No / Yes
+Creating a new Next.js app in <ORT_ROOT>\js\web\test\e2e\exports\testcases\nextjs-default.
+
+Using npm.
+
+Initializing project with template: app
+
+
+Installing dependencies:
+- react
+- react-dom
+- next
+```
+
+Small changes are made based on the application template, including:
+
+- Add a client side rendering (CSR) component which contains:
+  - a checkbox for multi-thread
+  - a checkbox for proxy
+  - a "Load Model" button
+  - a "Run Model" button
+  - a state DIV
+  - a log DIV
+- Add a helper module for creating ORT session and run
+
+### Tests
+
+Uses puppeteer to simulate the following tests:
+
+- Tests on `npm run dev` (dev server)
+  - multi-thread OFF, proxy OFF
+  - multi-thread OFF, proxy ON
+  - multi-thread ON, proxy OFF
+  - multi-thread ON, proxy ON
+- Tests on `npm run build` + `npm run serve` (prod)
+  - multi-thread OFF, proxy OFF
+  - multi-thread OFF, proxy ON
+  - multi-thread ON, proxy OFF
+  - multi-thread ON, proxy ON

js/web/test/e2e/exports/main.js

@@ -0,0 +1,202 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+'use strict';
+
+const path = require('path');
+const { spawn } = require('child_process');
+const EventEmitter = require('node:events');
+const treeKill = require('tree-kill');
+
+/**
+ * Entry point for package exports tests.
+ *
+ * @param {string[]} packagesToInstall
+ */
+module.exports = async function main(PRESERVE, PACKAGES_TO_INSTALL) {
+  console.log('Running exports tests...');
+
+  // testcases/nextjs-default
+  {
+    const wd = path.join(__dirname, 'testcases/nextjs-default');
+    if (!PRESERVE) {
+      if (PACKAGES_TO_INSTALL.length === 0) {
+        await runShellCmd('npm ci', { wd });
+      } else {
+        await runShellCmd(`npm install ${PACKAGES_TO_INSTALL.map((i) => `"${i}"`).join(' ')}`, { wd });
+      }
+    }
+
+    const launchBrowserAndRunTests = async (logPrefix, port = 3000) => {
+      const testResults = [];
+      const puppeteer = require('puppeteer-core');
+      let browser;
+      try {
+        browser = await puppeteer.launch({
+          executablePath: 'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
+          browser: 'chrome',
+          headless: true,
+          args: ['--enable-features=SharedArrayBuffer', '--no-sandbox', '--disable-setuid-sandbox'],
+        });
+
+        for await (const flags of [
+          [false, false],
+          //[false, true],
+          //[true, false],
+          //[true, true],
+        ]) {
+          const [multiThread, proxy] = flags;
+          console.log(`[${logPrefix}] Running test with multi-thread: ${multiThread}, proxy: ${proxy}...`);
+
+          const page = await browser.newPage();
+          await page.goto(`http://localhost:${port}`);
+          // wait for the page to load
+          await page.waitForSelector('#ortstate', { visible: true });
+          // if multi-thread is enabled, check the checkbox
+          if (multiThread) {
+            await page.locator('#cb-mt').click();
+          }
+          // if proxy is enabled, check the checkbox
+          if (proxy) {
+            await page.locator('#cb-px').click();
+          }
+          // click the load model button
+          await page.locator('#btn-load').click();
+          // wait for the model to load or fail
+          await page.waitForFunction("['2','3'].includes(document.getElementById('ortstate').innerText)");
+          // verify the model is loaded
+          const modelLoadState = await page.$eval('#ortstate', (el) => el.innerText);
+          if (modelLoadState !== '2') {
+            const ortLog = await page.$eval('#ortlog', (el) => el.innerText);
+            testResults.push({ multiThread, proxy, success: false, message: `Failed to load model: ${ortLog}` });
+            continue;
+          }
+
+          // click the run test button
+          await page.locator('#btn-run').click();
+          // wait for the inference run to complete or fail
+          await page.waitForFunction("['5','6'].includes(document.getElementById('ortstate').innerText)");
+          // verify the inference run result
+          const runState = await page.$eval('#ortstate', (el) => el.innerText);
+          if (runState !== '5') {
+            const ortLog = await page.$eval('#ortlog', (el) => el.innerText);
+            testResults.push({ multiThread, proxy, success: false, message: `Failed to run model: ${ortLog}` });
+            continue;
+          }
+
+          testResults.push({ multiThread, proxy, success: true });
+        }
+
+        return testResults;
+      } finally {
+        console.log(`[${logPrefix}] Closing the browser...`);
+        // close the browser
+        if (browser) {
+          await browser.close();
+        }
+      }
+    };
+
+    // test dev mode
+    {
+      console.log('Testing Next.js default (dev mode)...');
+      const npmRunDevEvent = new EventEmitter();
+      const npmRunDev = runShellCmd('npm run dev', {
+        wd,
+        event: npmRunDevEvent,
+        ready: '✓ Ready in',
+        ignoreExitCode: true,
+      });
+
+      let testResults;
+      npmRunDevEvent.on('serverReady', async () => {
+        try {
+          testResults = await launchBrowserAndRunTests('default:dev');
+        } finally {
+          console.log('Killing the server...');
+          // kill the server as the last step
+          npmRunDevEvent.emit('kill');
+        }
+      });
+
+      await npmRunDev;
+
+      console.log('Next.js default test (dev mode) result:', testResults);
+      if (testResults.some((r) => !r.success)) {
+        throw new Error('Next.js default test (dev mode) failed.');
+      }
+    } // test dev mode
+
+    // test prod mode
+    {
+      console.log('Testing Next.js default (prod mode)...');
+      // run 'npm run build'
+      await runShellCmd('npm run build', { wd });
+      const npmRunStartEvent = new EventEmitter();
+      const npmRunStart = runShellCmd('npm run start', {
+        wd,
+        event: npmRunStartEvent,
+        ready: '✓ Ready in',
+        ignoreExitCode: true,
+      });
+
+      let testResults;
+      npmRunStartEvent.on('serverReady', async () => {
+        try {
+          testResults = await launchBrowserAndRunTests('default:prod');
+        } finally {
+          console.log('Killing the server...');
+          // kill the server as the last step
+          npmRunStartEvent.emit('kill');
+        }
+      });
+
+      await npmRunStart;
+
+      console.log('Next.js default test (prod mode) result:', testResults);
+      if (testResults.some((r) => !r.success)) {
+        throw new Error('Next.js default test (prod mode) failed.');
+      }
+    } // test prod mode
+  }
+};
+
+async function runShellCmd(cmd, { wd = __dirname, event = null, ready = null, ignoreExitCode = false }) {
+  console.log('===============================================================');
+  console.log(' Running command in shell:');
+  console.log(' > ' + cmd);
+  console.log('===============================================================');
+
+  return new Promise((resolve, reject) => {
+    const childProcess = spawn(cmd, { shell: true, stdio: ['ignore', 'pipe', 'inherit'], cwd: wd });
+    childProcess.on('close', function (code, signal) {
+      if (code === 0 || ignoreExitCode) {
+        resolve();
+      } else {
+        reject(`Process exits with code ${code}`);
+      }
+    });
+    childProcess.stdout.on('data', (data) => {
+      process.stdout.write(data);
+
+      if (ready && event && data.toString().includes(ready)) {
+        event.emit('serverReady');
+      }
+    });
+    if (event) {
+      event.on('kill', () => {
+        childProcess.stdout.destroy();
+        treeKill(childProcess.pid);
+        console.log('killing process...');
+      });
+    }
+  });
+}
+
+async function delay(ms) {
+  return new Promise(function (resolve) {
+    setTimeout(function () {
+      resolve();
+    }, ms);
+  });
+}

js/web/test/e2e/exports/testcases/nextjs-default/.gitignore

@@ -0,0 +1,41 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/versions
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.pnpm-debug.log*
+
+# env files (can opt-in for committing if needed)
+.env*
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

js/web/test/e2e/exports/testcases/nextjs-default/README.md

@@ -0,0 +1,36 @@
+This is a [Next.js](https://nextjs.org) project bootstrapped with [`create-next-app`](https://nextjs.org/docs/app/api-reference/cli/create-next-app).
+
+## Getting Started
+
+First, run the development server:
+
+```bash
+npm run dev
+# or
+yarn dev
+# or
+pnpm dev
+# or
+bun dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+You can start editing the page by modifying `app/page.js`. The page auto-updates as you edit the file.
+
+This project uses [`next/font`](https://nextjs.org/docs/app/building-your-application/optimizing/fonts) to automatically optimize and load [Geist](https://vercel.com/font), a new font family for Vercel.
+
+## Learn More
+
+To learn more about Next.js, take a look at the following resources:
+
+- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
+- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
+
+You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome!
+
+## Deploy on Vercel
+
+The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
+
+Check out our [Next.js deployment documentation](https://nextjs.org/docs/app/building-your-application/deploying) for more details.

js/web/test/e2e/exports/testcases/nextjs-default/app/globals.css

@@ -0,0 +1,42 @@
+:root {
+  --background: #ffffff;
+  --foreground: #171717;
+}
+
+@media (prefers-color-scheme: dark) {
+  :root {
+    --background: #0a0a0a;
+    --foreground: #ededed;
+  }
+}
+
+html,
+body {
+  max-width: 100vw;
+  overflow-x: hidden;
+}
+
+body {
+  color: var(--foreground);
+  background: var(--background);
+  font-family: Arial, Helvetica, sans-serif;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+* {
+  box-sizing: border-box;
+  padding: 0;
+  margin: 0;
+}
+
+a {
+  color: inherit;
+  text-decoration: none;
+}
+
+@media (prefers-color-scheme: dark) {
+  html {
+    color-scheme: dark;
+  }
+}