From ee2caa78dd7b2e5e50c5247561dd7b8e77347a8b Mon Sep 17 00:00:00 2001 From: Will Fancher Date: Wed, 1 Aug 2018 08:08:29 -0400 Subject: [PATCH 01/10] Begin reorganizing the kernel with kernel-space typescript fs --- example/index.html | 9 ++ example/package.json | 19 ++++ example/src/index.ts | 14 +++ example/tsconfig.json | 15 ++++ example/webpack.config.js | 16 ++++ kernel/package.json | 26 ++++++ kernel/src/index.ts | 98 +++++++++++++++++++++ kernel/src/stdio_handles.ts | 170 ++++++++++++++++++++++++++++++++++++ kernel/tsconfig.json | 15 ++++ 9 files changed, 382 insertions(+) create mode 100644 example/index.html create mode 100644 example/package.json create mode 100644 example/src/index.ts create mode 100644 example/tsconfig.json create mode 100644 example/webpack.config.js create mode 100644 kernel/package.json create mode 100644 kernel/src/index.ts create mode 100644 kernel/src/stdio_handles.ts create mode 100644 kernel/tsconfig.json diff --git a/example/index.html b/example/index.html new file mode 100644 index 0000000..a581623 --- /dev/null +++ b/example/index.html @@ -0,0 +1,9 @@ + + + + Getting Started + + + + + diff --git a/example/package.json b/example/package.json new file mode 100644 index 0000000..ad1fe29 --- /dev/null +++ b/example/package.json @@ -0,0 +1,19 @@ +{ + "name": "webabi-example", + "version": "0.0.1", + "description": "", + "scripts": { + "build": "tsc", + "install": "npm run build", + "test": "npm run build && webpack && node build/main.js" + }, + "author": "Will Fancher", + "dependencies": { + "typescript": "^2.9.2", + "webabi-kernel": "file:../kernel" + }, + "devDependencies": { + "webpack": "^4.16.3", + "webpack-cli": "^3.1.0" + } +} diff --git a/example/src/index.ts b/example/src/index.ts new file mode 100644 index 0000000..eb2f0d4 --- /dev/null +++ b/example/src/index.ts @@ -0,0 +1,14 @@ +import { Device, configureFileSystem, BFSCallback, Stats, File, FileFlag } from "webabi-kernel"; + +class JSaddleDevice implements Device { + open(flag: FileFlag, cb: BFSCallback): void { + } + stat(isLstat: boolean | null, cb: BFSCallback): void { + } +} + +configureFileSystem({ "/jsaddle": new JSaddleDevice() }, (err, fs) => { + console.log(err); + let buf = Buffer.from("hi\n"); + fs.write(1, buf, 0, buf.length, null, () => {}); +}); diff --git a/example/tsconfig.json b/example/tsconfig.json new file mode 100644 index 0000000..156c09a --- /dev/null +++ b/example/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "es6", + "outDir": "dist", + "lib": ["dom", "es2015", "es2016", "es2017"], + "module": "commonjs", + "declaration": true + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "node_modules" + ] +} diff --git a/example/webpack.config.js b/example/webpack.config.js new file mode 100644 index 0000000..fc441df --- /dev/null +++ b/example/webpack.config.js @@ -0,0 +1,16 @@ +const path = require('path'); + +console.log(require.resolve("webabi-kernel")); + +module.exports = { + entry: './dist/index.js', + mode: "production", + output: { + filename: 'main.js', + path: path.resolve(__dirname, 'build') + }, + resolve: { + // Using file:../kernel in package.json requires this + symlinks: false + } +}; diff --git a/kernel/package.json b/kernel/package.json new file mode 100644 index 0000000..e904b18 --- /dev/null +++ b/kernel/package.json @@ -0,0 +1,26 @@ +{ + "name": "webabi-kernel", + "version": "0.0.1", + "description": "", + "scripts": { + "build": "tsc", + "main": "npm run tsc && node dist/foo.js", + "install": "npm run build" + }, + "main": "dist/index.js", + "author": "Will Fancher", + "dependencies": { + "@types/archiver": "^2.0.0", + "@types/async": "^2.0.49", + "@types/body-parser": "^1.16.4", + "@types/dropboxjs": "0.0.29", + "@types/express": "^4.0.36", + "@types/filesystem": "0.0.28", + "@types/isomorphic-fetch": "^0.0.34", + "@types/mocha": "^5.2.5", + "@types/node": "^7.0", + "@types/rimraf": "^2.0.2", + "browserfs": "^1.4.3", + "typescript": "^2.9.2" + } +} diff --git a/kernel/src/index.ts b/kernel/src/index.ts new file mode 100644 index 0000000..0500894 --- /dev/null +++ b/kernel/src/index.ts @@ -0,0 +1,98 @@ +import MountableFileSystem from "browserfs/dist/node/backend/MountableFileSystem"; +import * as handles from "./stdio_handles"; +import { BaseFileSystem, FileSystemConstructor, BFSCallback, + BFSOneArgCallback, BFSThreeArgCallback, FileSystem, + FileSystemOptions } from "browserfs/dist/node/core/file_system"; +import { FileType } from 'browserfs/dist/node/core/node_fs_stats'; +import Stats from 'browserfs/dist/node/core/node_fs_stats'; +import { File } from "browserfs/dist/node/core/file"; +import { FileFlag } from "browserfs/dist/node/core/file_flag"; +import { ApiError, ErrorCode } from 'browserfs/dist/node/core/api_error'; +import FS from "browserfs/dist/node/core/FS"; + +export interface Device { + open(flag: FileFlag, cb: BFSCallback): void; + stat(isLstat: boolean | null, cb: BFSCallback): void; +} + +export interface DeviceFileSystemOptions { + devices: {[name: string]: Device}; +} + +export class DeviceFileSystem extends BaseFileSystem implements FileSystem { + public static readonly Name = "DeviceFileSystem"; + public static readonly Options: FileSystemOptions = {}; + + public static Create(opts: DeviceFileSystemOptions, cb: BFSCallback): void { + return cb(null, new DeviceFileSystem(opts)); + } + + public static isAvailable(): boolean { + return true; + } + + options: DeviceFileSystemOptions; + + constructor(options: DeviceFileSystemOptions) { + super(); + this.options = options; + } + + public getName() { + return "DeviceFileSystem"; + } + public isReadOnly() { + return false; + } + public supportsProps() { + return false; + } + public supportsSynch() { + return false; + } + + public openFile(p: string, flag: FileFlag, cb: BFSCallback): void { + if (this.options.devices.hasOwnProperty(p)) { + return this.options.devices[p].open(flag, cb); + } else { + return cb(ApiError.ENOENT(p)); + } + } + public stat(p: string, isLstat: boolean | null, cb: BFSCallback): void { + if (this.options.devices.hasOwnProperty(p)) { + return this.options.devices[p].stat(isLstat, cb); + } else { + return cb(ApiError.ENOENT(p)); + } + } +} + +export function configureFileSystem(devices: { [name: string]: Device }, cb: BFSCallback): void { + DeviceFileSystem.Create({ devices: devices }, (e, dfs) => { + if (e) { + cb(e); + return; + } + MountableFileSystem.Create({ + "/dev": dfs + }, (e, mfs) => { + if (e) { + cb(e); + return + } + + const fs = new FS(); + fs.initialize(mfs); + + const fdMap: {[id: number]: File} = (fs as any).fdMap; + fdMap[0] = handles.stdin; + fdMap[1] = handles.stdout; + fdMap[2] = handles.stderr; + + cb(undefined, fs); + }); + }); +} + +// Re-export for device implementors +export { BFSCallback, Stats, File, FileFlag }; diff --git a/kernel/src/stdio_handles.ts b/kernel/src/stdio_handles.ts new file mode 100644 index 0000000..1c4ea94 --- /dev/null +++ b/kernel/src/stdio_handles.ts @@ -0,0 +1,170 @@ +import { File, BaseFile } from "browserfs/dist/node/core/file"; +import { BaseFileSystem, FileSystemConstructor, BFSCallback, BFSOneArgCallback, BFSThreeArgCallback, FileSystem, FileSystemOptions } from "browserfs/dist/node/core/file_system"; +import { FileType } from 'browserfs/dist/node/core/node_fs_stats'; +import Stats from 'browserfs/dist/node/core/node_fs_stats'; +import { ApiError, ErrorCode } from 'browserfs/dist/node/core/api_error'; + +export let stdin: File; +export let stdout: File; +export let stderr: File; + +class UselessFile extends BaseFile implements File { + getPos(): number | undefined { + return undefined; + } + stat(cb: BFSCallback): void { + return cb(undefined, new Stats(FileType.FILE, 0)); + } + statSync(): Stats { + return new Stats(FileType.FILE, 0) + } + close(cb: BFSOneArgCallback): void { + cb(new ApiError(ErrorCode.ENOTSUP)); + } + closeSync(): void { + throw new ApiError(ErrorCode.ENOTSUP); + } + truncate(len: number, cb: BFSOneArgCallback): void { + cb(new ApiError(ErrorCode.ENOTSUP)); + } + truncateSync(len: number): void { + throw new ApiError(ErrorCode.ENOTSUP); + } + write(buffer: Buffer, offset: number, length: number, position: number | null, cb: BFSThreeArgCallback): void { + cb(new ApiError(ErrorCode.ENOTSUP)); + } + writeSync(buffer: Buffer, offset: number, length: number, position: number | null): number { + throw new ApiError(ErrorCode.ENOTSUP); + } + read(buffer: Buffer, offset: number, length: number, position: number | null, cb: BFSThreeArgCallback): void { + cb(new ApiError(ErrorCode.ENOTSUP)); + } + readSync(buffer: Buffer, offset: number, length: number, position: number): number { + throw new ApiError(ErrorCode.ENOTSUP); + } +} + +if (process && !(process as any).browser) { + interface Request { + buffer: Buffer; + offset: number; + length: number; + cb: BFSThreeArgCallback; + } + class ReadWriteStreamFile extends UselessFile implements File { + stream: NodeJS.ReadWriteStream; + requests: [Request] = <[Request]> []; + leftover?: Buffer = null; + + constructor(stream: NodeJS.ReadWriteStream) { + super(); + this.stream = stream; + this.stream.pause(); + this.stream.on("error", (err) => { + const reqs = this.requests; + this.requests = <[Request]> []; + for (const req of reqs) { + req.cb(err, undefined, undefined); + } + }); + this.stream.on("data", (buf) => { + this.stream.pause(); + if (this.leftover) { + buf = Buffer.concat([this.leftover, buf]); + this.leftover = null; + } + this.onData(buf); + }); + } + + onData(buf: Buffer): void { + const reqs = this.requests; + this.requests = <[Request]> []; + let nextBuf: Buffer | null = null; + for (const req of reqs) { + if (buf.length > req.length) { + nextBuf = buf.slice(req.length); + buf = buf.slice(0, req.length); + } else { + nextBuf = null; + } + + const copied = buf.copy(req.buffer, req.offset); + req.cb(undefined, copied, req.buffer); + + buf = nextBuf; + } + + if (nextBuf) { + // nextBuf may still have the old leftover underlying it. + // Use Buffer.from to avoid retaining the entire history. + this.leftover = Buffer.from(nextBuf); + } + }; + close(cb: BFSOneArgCallback): void { + this.stream.end(cb); + } + write(buffer: Buffer, offset: number, length: number, position: number | null, cb: BFSThreeArgCallback): void { + this.stream.write(buffer.slice(offset, offset + length), (err) => { + if (err) { + cb(err); + } else { + cb(undefined, length, buffer); + } + }); + } + read(buffer: Buffer, offset: number, length: number, position: number | null, cb: BFSThreeArgCallback): void { + this.stream.resume(); + this.requests.push({ + buffer: buffer, + offset: offset, + length: length, + cb: cb + }); + } + } + + stdin = new ReadWriteStreamFile(process.stdin); + stdout = new ReadWriteStreamFile(process.stdout); + stderr = new ReadWriteStreamFile(process.stderr); +} else { + class ConsoleFile extends UselessFile implements File { + log: (msg: string) => void; + buffer?: Buffer = null; + + constructor(log: (msg: string) => void) { + super(); + this.log = log; + } + + write(buffer: Buffer, offset: number, length: number, position: number | null, cb: BFSThreeArgCallback): void { + let slicedBuffer = buffer.slice(offset, offset + length); + let n = slicedBuffer.lastIndexOf("\n"); + if (n < 0) { + if (this.buffer) { + this.buffer = Buffer.concat([this.buffer, slicedBuffer]); + } else { + this.buffer = slicedBuffer; + } + } else { + let logBuffer = slicedBuffer.slice(0, n); + if (this.buffer) { + logBuffer = Buffer.concat([this.buffer, logBuffer]); + } + this.log(logBuffer.toString()); + + // + 1 to skip the \n + if (n + 1 < slicedBuffer.length) { + this.buffer = slicedBuffer.slice(n + 1); + } else { + this.buffer = null; + } + } + cb(undefined, length, buffer); + } + } + + stdin = new UselessFile(); + stdout = new ConsoleFile((msg) => console.log(msg)); + stderr = new ConsoleFile((msg) => console.error(msg)); +} diff --git a/kernel/tsconfig.json b/kernel/tsconfig.json new file mode 100644 index 0000000..156c09a --- /dev/null +++ b/kernel/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "es6", + "outDir": "dist", + "lib": ["dom", "es2015", "es2016", "es2017"], + "module": "commonjs", + "declaration": true + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "node_modules" + ] +} From 79830bee65c766f969c14a5536c62000cad3d5f2 Mon Sep 17 00:00:00 2001 From: Will Fancher Date: Wed, 1 Aug 2018 08:37:19 -0400 Subject: [PATCH 02/10] Fix stdin --- example/package.json | 2 +- example/src/index.ts | 8 ++++++-- example/webpack.config.js | 3 +++ kernel/src/stdio_handles.ts | 7 +++++-- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/example/package.json b/example/package.json index ad1fe29..9ef23df 100644 --- a/example/package.json +++ b/example/package.json @@ -5,7 +5,7 @@ "scripts": { "build": "tsc", "install": "npm run build", - "test": "npm run build && webpack && node build/main.js" + "test": "npm run build && webpack && (echo bar | node build/main.js)" }, "author": "Will Fancher", "dependencies": { diff --git a/example/src/index.ts b/example/src/index.ts index eb2f0d4..d67fc5c 100644 --- a/example/src/index.ts +++ b/example/src/index.ts @@ -9,6 +9,10 @@ class JSaddleDevice implements Device { configureFileSystem({ "/jsaddle": new JSaddleDevice() }, (err, fs) => { console.log(err); - let buf = Buffer.from("hi\n"); - fs.write(1, buf, 0, buf.length, null, () => {}); + let buf = Buffer.from("foo\n"); + fs.write(1, buf, 0, buf.length, null, () => { + fs.read(0, buf, 0, 4, null, (err, n, buf) => { + console.log({ err: err, n: n, buf: buf && buf.toString() }); + }); + }); }); diff --git a/example/webpack.config.js b/example/webpack.config.js index fc441df..61e6378 100644 --- a/example/webpack.config.js +++ b/example/webpack.config.js @@ -12,5 +12,8 @@ module.exports = { resolve: { // Using file:../kernel in package.json requires this symlinks: false + }, + node: { + process: false } }; diff --git a/kernel/src/stdio_handles.ts b/kernel/src/stdio_handles.ts index 1c4ea94..6c686b2 100644 --- a/kernel/src/stdio_handles.ts +++ b/kernel/src/stdio_handles.ts @@ -79,9 +79,9 @@ if (process && !(process as any).browser) { onData(buf: Buffer): void { const reqs = this.requests; - this.requests = <[Request]> []; let nextBuf: Buffer | null = null; - for (const req of reqs) { + let req: Request; + while (req = this.requests.shift()) { if (buf.length > req.length) { nextBuf = buf.slice(req.length); buf = buf.slice(0, req.length); @@ -92,6 +92,9 @@ if (process && !(process as any).browser) { const copied = buf.copy(req.buffer, req.offset); req.cb(undefined, copied, req.buffer); + if (!nextBuf || nextBuf.length == 0) { + break; + } buf = nextBuf; } From aa9cf49e2e7de4a19adf1e789524681035843b5f Mon Sep 17 00:00:00 2001 From: Will Fancher Date: Wed, 1 Aug 2018 08:54:49 -0400 Subject: [PATCH 03/10] Fix process detection in browser --- kernel/src/stdio_handles.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/src/stdio_handles.ts b/kernel/src/stdio_handles.ts index 6c686b2..bb89a8f 100644 --- a/kernel/src/stdio_handles.ts +++ b/kernel/src/stdio_handles.ts @@ -44,7 +44,7 @@ class UselessFile extends BaseFile implements File { } } -if (process && !(process as any).browser) { +if (!(typeof process === "undefined" || (process as any).browser)) { interface Request { buffer: Buffer; offset: number; From e31053473d20c6ac4b74fe918814354cc1756744 Mon Sep 17 00:00:00 2001 From: Will Fancher Date: Wed, 1 Aug 2018 08:56:48 -0400 Subject: [PATCH 04/10] Resume the stream if we still have requests to fill --- kernel/src/stdio_handles.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernel/src/stdio_handles.ts b/kernel/src/stdio_handles.ts index bb89a8f..386f042 100644 --- a/kernel/src/stdio_handles.ts +++ b/kernel/src/stdio_handles.ts @@ -103,6 +103,9 @@ if (!(typeof process === "undefined" || (process as any).browser)) { // Use Buffer.from to avoid retaining the entire history. this.leftover = Buffer.from(nextBuf); } + if (this.requests.length > 0) { + this.stream.resume(); + } }; close(cb: BFSOneArgCallback): void { this.stream.end(cb); From b3d55d9194e7a98ef0abcc2b4e26436654be8646 Mon Sep 17 00:00:00 2001 From: Will Fancher Date: Thu, 2 Aug 2018 01:14:23 -0400 Subject: [PATCH 05/10] Use async / await, like civilized people. --- example/src/index.ts | 18 ++++++++------- kernel/src/index.ts | 55 ++++++++++++++++++++++++++------------------ 2 files changed, 42 insertions(+), 31 deletions(-) diff --git a/example/src/index.ts b/example/src/index.ts index d67fc5c..71d38dc 100644 --- a/example/src/index.ts +++ b/example/src/index.ts @@ -1,4 +1,4 @@ -import { Device, configureFileSystem, BFSCallback, Stats, File, FileFlag } from "webabi-kernel"; +import { Device, configureFileSystem, BFSCallback, Stats, File, FileFlag, FS, asyncRead, asyncWrite } from "webabi-kernel"; class JSaddleDevice implements Device { open(flag: FileFlag, cb: BFSCallback): void { @@ -7,12 +7,14 @@ class JSaddleDevice implements Device { } } -configureFileSystem({ "/jsaddle": new JSaddleDevice() }, (err, fs) => { - console.log(err); +async function main() { + let fs = await configureFileSystem({ "/jsaddle": new JSaddleDevice() }); let buf = Buffer.from("foo\n"); - fs.write(1, buf, 0, buf.length, null, () => { - fs.read(0, buf, 0, 4, null, (err, n, buf) => { - console.log({ err: err, n: n, buf: buf && buf.toString() }); - }); - }); + await asyncWrite(fs, 1, buf, 0, buf.length, null); + const { byteLength } = await asyncRead(fs, 0, buf, 0, buf.length, null); + console.log({ byteLength: byteLength, buffer: buf.toString() }); +} + +main().catch(e => { + console.error("Error: ", e); }); diff --git a/kernel/src/index.ts b/kernel/src/index.ts index 0500894..ecc5092 100644 --- a/kernel/src/index.ts +++ b/kernel/src/index.ts @@ -67,32 +67,41 @@ export class DeviceFileSystem extends BaseFileSystem implements FileSystem { } } -export function configureFileSystem(devices: { [name: string]: Device }, cb: BFSCallback): void { - DeviceFileSystem.Create({ devices: devices }, (e, dfs) => { - if (e) { - cb(e); - return; - } +export async function configureFileSystem(devices: { [name: string]: Device }): Promise { + const dfs = await new Promise((resolve, reject) => { + DeviceFileSystem.Create({ devices: devices }, (e, dfs) => e ? reject(e) : resolve(dfs)) + }); + const mfs = await new Promise((resolve, reject) => { MountableFileSystem.Create({ "/dev": dfs - }, (e, mfs) => { - if (e) { - cb(e); - return - } - - const fs = new FS(); - fs.initialize(mfs); - - const fdMap: {[id: number]: File} = (fs as any).fdMap; - fdMap[0] = handles.stdin; - fdMap[1] = handles.stdout; - fdMap[2] = handles.stderr; - - cb(undefined, fs); - }); + }, (e, mfs) => e ? reject(e) : resolve(mfs)); + }); + + const fs = new FS(); + fs.initialize(mfs); + + const fdMap: {[id: number]: File} = (fs as any).fdMap; + fdMap[0] = handles.stdin; + fdMap[1] = handles.stdout; + fdMap[2] = handles.stderr; + + return fs; +} + +export async function asyncRead(fs: FS, fd: number, buffer: Buffer, offset: number, length: number, position: number | null) +: Promise<{ byteLength: number, buffer: Buffer }> { + return new Promise<{ byteLength: number, buffer: Buffer }>((resolve, reject) => { + fs.read(fd, buffer, offset, length, position, + (err, n, buf) => err ? reject(err) : resolve({ byteLength: n, buffer: buf })); + }); +} + +export async function asyncWrite(fs: FS, fd: number, buffer: Buffer, offset: number, length: number, position: number | null) +: Promise { + return new Promise((resolve, reject) => { + fs.write(fd, buffer, offset, length, position, e => e ? reject(e) : resolve()) }); } // Re-export for device implementors -export { BFSCallback, Stats, File, FileFlag }; +export { BFSCallback, Stats, File, FileFlag, FS }; From 133230cc5abef5bd776015788a16654ad19f8551 Mon Sep 17 00:00:00 2001 From: Will Fancher Date: Thu, 2 Aug 2018 02:48:12 -0400 Subject: [PATCH 06/10] Use async / await in Device interface --- example/src/index.ts | 10 +++++++--- kernel/src/index.ts | 14 +++++++------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/example/src/index.ts b/example/src/index.ts index 71d38dc..85881aa 100644 --- a/example/src/index.ts +++ b/example/src/index.ts @@ -1,9 +1,13 @@ -import { Device, configureFileSystem, BFSCallback, Stats, File, FileFlag, FS, asyncRead, asyncWrite } from "webabi-kernel"; +import { Device, configureFileSystem, BFSCallback, + Stats, File, FileFlag, FS, ApiError, + ErrorCode, asyncRead, asyncWrite } from "webabi-kernel"; class JSaddleDevice implements Device { - open(flag: FileFlag, cb: BFSCallback): void { + async open(flag: FileFlag): Promise { + throw new ApiError(ErrorCode.ENOTSUP); } - stat(isLstat: boolean | null, cb: BFSCallback): void { + async stat(isLstat: boolean | null): Promise { + throw new ApiError(ErrorCode.ENOTSUP); } } diff --git a/kernel/src/index.ts b/kernel/src/index.ts index ecc5092..5c4c2a9 100644 --- a/kernel/src/index.ts +++ b/kernel/src/index.ts @@ -11,8 +11,8 @@ import { ApiError, ErrorCode } from 'browserfs/dist/node/core/api_error'; import FS from "browserfs/dist/node/core/FS"; export interface Device { - open(flag: FileFlag, cb: BFSCallback): void; - stat(isLstat: boolean | null, cb: BFSCallback): void; + open(flag: FileFlag): Promise; + stat(isLstat: boolean | null): Promise; } export interface DeviceFileSystemOptions { @@ -53,16 +53,16 @@ export class DeviceFileSystem extends BaseFileSystem implements FileSystem { public openFile(p: string, flag: FileFlag, cb: BFSCallback): void { if (this.options.devices.hasOwnProperty(p)) { - return this.options.devices[p].open(flag, cb); + this.options.devices[p].open(flag).then(f => cb(undefined, f), e => cb(e)); } else { - return cb(ApiError.ENOENT(p)); + cb(ApiError.ENOENT(p)); } } public stat(p: string, isLstat: boolean | null, cb: BFSCallback): void { if (this.options.devices.hasOwnProperty(p)) { - return this.options.devices[p].stat(isLstat, cb); + this.options.devices[p].stat(isLstat).then(s => cb(undefined, s), e => cb(e)); } else { - return cb(ApiError.ENOENT(p)); + cb(ApiError.ENOENT(p)); } } } @@ -104,4 +104,4 @@ export async function asyncWrite(fs: FS, fd: number, buffer: Buffer, offset: num } // Re-export for device implementors -export { BFSCallback, Stats, File, FileFlag, FS }; +export { BFSCallback, Stats, File, FileFlag, FS, ApiError, ErrorCode }; From 0bb945458d0b85ae2d246bdb9b4009bc6a179adf Mon Sep 17 00:00:00 2001 From: Will Fancher Date: Thu, 2 Aug 2018 05:19:00 -0400 Subject: [PATCH 07/10] Basic worker module for kernel --- kernel/src/node_parent.ts | 7 +++++++ kernel/src/worker.ts | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 kernel/src/node_parent.ts create mode 100644 kernel/src/worker.ts diff --git a/kernel/src/node_parent.ts b/kernel/src/node_parent.ts new file mode 100644 index 0000000..29aa571 --- /dev/null +++ b/kernel/src/node_parent.ts @@ -0,0 +1,7 @@ +declare module "worker_threads" { + export class Worker { + constructor(path: string); + postMessage(msg: any): void; + on(event: "message", handler: (msg: any) => void); + } +} diff --git a/kernel/src/worker.ts b/kernel/src/worker.ts new file mode 100644 index 0000000..961bcb2 --- /dev/null +++ b/kernel/src/worker.ts @@ -0,0 +1,24 @@ +import * as nodeWorkers from "worker_threads"; + +export interface OnMessage { + onMessage(msg: any): void; +} + +export interface PostMessage { + postMessage(msg: any): void; +} + +export let makeWorker: (path: string, receiver: OnMessage) => PostMessage; +if (typeof Worker !== "undefined") { + makeWorker = (path, receiver) => { + const worker = new Worker(path); + worker.onmessage = msg => receiver.onMessage(msg.data); + return { postMessage: msg => worker.postMessage(msg) }; + }; +} else { + makeWorker = (path, receiver) => { + const worker = new nodeWorkers.Worker(path); + worker.on("message", msg => receiver.onMessage(msg)); + return { postMessage: msg => worker.postMessage(msg) }; + } +} From 2dabad6aae2280cedd8d9b2a4d8ffbc7b722dc6d Mon Sep 17 00:00:00 2001 From: Will Fancher Date: Thu, 2 Aug 2018 05:19:06 -0400 Subject: [PATCH 08/10] Remove dumb foo.js from package.json --- kernel/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/kernel/package.json b/kernel/package.json index e904b18..6251a89 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -4,7 +4,6 @@ "description": "", "scripts": { "build": "tsc", - "main": "npm run tsc && node dist/foo.js", "install": "npm run build" }, "main": "dist/index.js", From 02d323e2ff8d1d58f42815570657cbc05617d610 Mon Sep 17 00:00:00 2001 From: Will Fancher Date: Thu, 2 Aug 2018 07:07:46 -0400 Subject: [PATCH 09/10] Remove extraneous log in webpack.config.js --- example/webpack.config.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/example/webpack.config.js b/example/webpack.config.js index 61e6378..4ecf468 100644 --- a/example/webpack.config.js +++ b/example/webpack.config.js @@ -1,7 +1,5 @@ const path = require('path'); -console.log(require.resolve("webabi-kernel")); - module.exports = { entry: './dist/index.js', mode: "production", From 74083f77c962e862f2a3aca354946dc86eb106ea Mon Sep 17 00:00:00 2001 From: Will Fancher Date: Fri, 3 Aug 2018 08:11:55 -0400 Subject: [PATCH 10/10] Example workers, working with webpack in node and browser --- .gitignore | 4 ++ example/package.json | 4 +- example/src/index.ts | 16 +++++++- example/webpack.config.js | 8 ++++ exec/package.json | 18 +++++++++ exec/src/index.ts | 14 +++++++ exec/src/node_worker.ts | 8 ++++ exec/src/worker.ts | 28 +++++++++++++ exec/tsconfig.json | 15 +++++++ exec/webpack.config.js | 18 +++++++++ example/index.html => index.html | 2 +- kernel/src/DeviceFileSystem.ts | 63 +++++++++++++++++++++++++++++ kernel/src/index.ts | 69 +++----------------------------- kernel/src/kernel.ts | 9 +++++ kernel/src/node_parent.ts | 3 +- kernel/src/worker.ts | 22 ++++++---- 16 files changed, 224 insertions(+), 77 deletions(-) create mode 100644 .gitignore create mode 100644 exec/package.json create mode 100644 exec/src/index.ts create mode 100644 exec/src/node_worker.ts create mode 100644 exec/src/worker.ts create mode 100644 exec/tsconfig.json create mode 100644 exec/webpack.config.js rename example/index.html => index.html (66%) create mode 100644 kernel/src/DeviceFileSystem.ts create mode 100644 kernel/src/kernel.ts diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..71b7ebc --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +build/ +dist/ +package-lock.json diff --git a/example/package.json b/example/package.json index 9ef23df..45959e4 100644 --- a/example/package.json +++ b/example/package.json @@ -3,9 +3,9 @@ "version": "0.0.1", "description": "", "scripts": { - "build": "tsc", + "build": "tsc && webpack", "install": "npm run build", - "test": "npm run build && webpack && (echo bar | node build/main.js)" + "test": "npm run build && (cd ..; echo bar | node --experimental-worker example/build/main.js)" }, "author": "Will Fancher", "dependencies": { diff --git a/example/src/index.ts b/example/src/index.ts index 85881aa..36f9b85 100644 --- a/example/src/index.ts +++ b/example/src/index.ts @@ -1,6 +1,7 @@ -import { Device, configureFileSystem, BFSCallback, +import { Device, configureKernel, BFSCallback, Stats, File, FileFlag, FS, ApiError, ErrorCode, asyncRead, asyncWrite } from "webabi-kernel"; +import { makeWorker, PostMessage, OnMessage } from "webabi-kernel/dist/worker"; class JSaddleDevice implements Device { async open(flag: FileFlag): Promise { @@ -12,11 +13,22 @@ class JSaddleDevice implements Device { } async function main() { - let fs = await configureFileSystem({ "/jsaddle": new JSaddleDevice() }); + let kernel = await configureKernel({ "/jsaddle": new JSaddleDevice() }); + let fs = kernel.fs; let buf = Buffer.from("foo\n"); await asyncWrite(fs, 1, buf, 0, buf.length, null); const { byteLength } = await asyncRead(fs, 0, buf, 0, buf.length, null); console.log({ byteLength: byteLength, buffer: buf.toString() }); + + console.log("working"); + let worker: PostMessage; + worker = await makeWorker("./exec/build/main.js", { + onMessage: msg => { + console.log(msg); + worker.close(); + } + }); + worker.postMessage("foo"); } main().catch(e => { diff --git a/example/webpack.config.js b/example/webpack.config.js index 4ecf468..618670b 100644 --- a/example/webpack.config.js +++ b/example/webpack.config.js @@ -3,6 +3,14 @@ const path = require('path'); module.exports = { entry: './dist/index.js', mode: "production", + externals: [ + ((context, request, callback) => { + if (request == "worker_threads") { + return callback(null, "commonjs " + request); + } + callback(); + }) + ], output: { filename: 'main.js', path: path.resolve(__dirname, 'build') diff --git a/exec/package.json b/exec/package.json new file mode 100644 index 0000000..58d128d --- /dev/null +++ b/exec/package.json @@ -0,0 +1,18 @@ +{ + "name": "webabi-exec", + "version": "0.0.1", + "description": "", + "scripts": { + "build": "tsc && webpack", + "install": "npm run build" + }, + "main": "dist/index.js", + "author": "Will Fancher", + "dependencies": { + "typescript": "^2.9.2" + }, + "devDependencies": { + "webpack": "^4.16.3", + "webpack-cli": "^3.1.0" + } +} diff --git a/exec/src/index.ts b/exec/src/index.ts new file mode 100644 index 0000000..1c749f5 --- /dev/null +++ b/exec/src/index.ts @@ -0,0 +1,14 @@ +import { connectParent, PostMessage } from "./worker"; + +async function main() { + let parent: PostMessage; + parent = await connectParent({ + onMessage: msg => { + console.log(msg); + parent.close(); + } + }); + parent.postMessage("bar"); +} + +main(); diff --git a/exec/src/node_worker.ts b/exec/src/node_worker.ts new file mode 100644 index 0000000..fdfb35a --- /dev/null +++ b/exec/src/node_worker.ts @@ -0,0 +1,8 @@ +declare module "worker_threads" { + interface MessagePort { + postMessage(msg: any, transferList: [any]); + on(event: "message", handler: (msg: any) => void); + unref(): void; + } + export const parentPort: MessagePort; +} diff --git a/exec/src/worker.ts b/exec/src/worker.ts new file mode 100644 index 0000000..6f54961 --- /dev/null +++ b/exec/src/worker.ts @@ -0,0 +1,28 @@ +export interface OnMessage { + onMessage(msg: any): void; +} + +export interface PostMessage { + postMessage(msg: any, transferList?: [any]): void; + close(): void; +} + +export let connectParent: (receiver: OnMessage) => Promise; +if (typeof self !== "undefined") { + connectParent = async (receiver) => { + self.onmessage = msg => receiver.onMessage(msg.data); + return { + postMessage: (msg, transferList) => self.postMessage(msg, transferList as any /*tsc complains wrongly*/), + close: () => {} + }; + }; +} else { + connectParent = async (receiver) => { + const nodeWorkers = await import("worker_threads"); + nodeWorkers.parentPort.on("message", msg => receiver.onMessage(msg)); + return { + postMessage: (msg, transferList) => nodeWorkers.parentPort.postMessage(msg, transferList), + close: () => nodeWorkers.parentPort.unref() + }; + } +} diff --git a/exec/tsconfig.json b/exec/tsconfig.json new file mode 100644 index 0000000..156c09a --- /dev/null +++ b/exec/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "es6", + "outDir": "dist", + "lib": ["dom", "es2015", "es2016", "es2017"], + "module": "commonjs", + "declaration": true + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "node_modules" + ] +} diff --git a/exec/webpack.config.js b/exec/webpack.config.js new file mode 100644 index 0000000..bf47314 --- /dev/null +++ b/exec/webpack.config.js @@ -0,0 +1,18 @@ +const path = require('path'); + +module.exports = { + entry: './dist/index.js', + mode: "production", + externals: [ + ((context, request, callback) => { + if (request == "worker_threads") { + return callback(null, "commonjs " + request); + } + callback(); + }) + ], + output: { + filename: 'main.js', + path: path.resolve(__dirname, 'build') + } +}; diff --git a/example/index.html b/index.html similarity index 66% rename from example/index.html rename to index.html index a581623..25e7984 100644 --- a/example/index.html +++ b/index.html @@ -4,6 +4,6 @@ Getting Started - + diff --git a/kernel/src/DeviceFileSystem.ts b/kernel/src/DeviceFileSystem.ts new file mode 100644 index 0000000..384fdb8 --- /dev/null +++ b/kernel/src/DeviceFileSystem.ts @@ -0,0 +1,63 @@ +import { BaseFileSystem, BFSCallback, FileSystem, FileSystemOptions + } from "browserfs/dist/node/core/file_system"; +import Stats from 'browserfs/dist/node/core/node_fs_stats'; +import { File } from "browserfs/dist/node/core/file"; +import { FileFlag } from "browserfs/dist/node/core/file_flag"; +import { ApiError } from 'browserfs/dist/node/core/api_error'; + +export interface Device { + open(flag: FileFlag): Promise; + stat(isLstat: boolean | null): Promise; +} + +export interface DeviceFileSystemOptions { + devices: {[name: string]: Device}; +} + +export class DeviceFileSystem extends BaseFileSystem implements FileSystem { + public static readonly Name = "DeviceFileSystem"; + public static readonly Options: FileSystemOptions = {}; + + public static Create(opts: DeviceFileSystemOptions, cb: BFSCallback): void { + return cb(null, new DeviceFileSystem(opts)); + } + + public static isAvailable(): boolean { + return true; + } + + options: DeviceFileSystemOptions; + + constructor(options: DeviceFileSystemOptions) { + super(); + this.options = options; + } + + public getName() { + return "DeviceFileSystem"; + } + public isReadOnly() { + return false; + } + public supportsProps() { + return false; + } + public supportsSynch() { + return false; + } + + public openFile(p: string, flag: FileFlag, cb: BFSCallback): void { + if (this.options.devices.hasOwnProperty(p)) { + this.options.devices[p].open(flag).then(f => cb(undefined, f), e => cb(e)); + } else { + cb(ApiError.ENOENT(p)); + } + } + public stat(p: string, isLstat: boolean | null, cb: BFSCallback): void { + if (this.options.devices.hasOwnProperty(p)) { + this.options.devices[p].stat(isLstat).then(s => cb(undefined, s), e => cb(e)); + } else { + cb(ApiError.ENOENT(p)); + } + } +} diff --git a/kernel/src/index.ts b/kernel/src/index.ts index 5c4c2a9..f523d54 100644 --- a/kernel/src/index.ts +++ b/kernel/src/index.ts @@ -1,73 +1,15 @@ import MountableFileSystem from "browserfs/dist/node/backend/MountableFileSystem"; import * as handles from "./stdio_handles"; -import { BaseFileSystem, FileSystemConstructor, BFSCallback, - BFSOneArgCallback, BFSThreeArgCallback, FileSystem, - FileSystemOptions } from "browserfs/dist/node/core/file_system"; -import { FileType } from 'browserfs/dist/node/core/node_fs_stats'; +import { BFSCallback } from "browserfs/dist/node/core/file_system"; import Stats from 'browserfs/dist/node/core/node_fs_stats'; import { File } from "browserfs/dist/node/core/file"; import { FileFlag } from "browserfs/dist/node/core/file_flag"; import { ApiError, ErrorCode } from 'browserfs/dist/node/core/api_error'; import FS from "browserfs/dist/node/core/FS"; +import { DeviceFileSystem, Device } from "./DeviceFileSystem"; +import Kernel from "./kernel"; -export interface Device { - open(flag: FileFlag): Promise; - stat(isLstat: boolean | null): Promise; -} - -export interface DeviceFileSystemOptions { - devices: {[name: string]: Device}; -} - -export class DeviceFileSystem extends BaseFileSystem implements FileSystem { - public static readonly Name = "DeviceFileSystem"; - public static readonly Options: FileSystemOptions = {}; - - public static Create(opts: DeviceFileSystemOptions, cb: BFSCallback): void { - return cb(null, new DeviceFileSystem(opts)); - } - - public static isAvailable(): boolean { - return true; - } - - options: DeviceFileSystemOptions; - - constructor(options: DeviceFileSystemOptions) { - super(); - this.options = options; - } - - public getName() { - return "DeviceFileSystem"; - } - public isReadOnly() { - return false; - } - public supportsProps() { - return false; - } - public supportsSynch() { - return false; - } - - public openFile(p: string, flag: FileFlag, cb: BFSCallback): void { - if (this.options.devices.hasOwnProperty(p)) { - this.options.devices[p].open(flag).then(f => cb(undefined, f), e => cb(e)); - } else { - cb(ApiError.ENOENT(p)); - } - } - public stat(p: string, isLstat: boolean | null, cb: BFSCallback): void { - if (this.options.devices.hasOwnProperty(p)) { - this.options.devices[p].stat(isLstat).then(s => cb(undefined, s), e => cb(e)); - } else { - cb(ApiError.ENOENT(p)); - } - } -} - -export async function configureFileSystem(devices: { [name: string]: Device }): Promise { +export async function configureKernel(devices: { [name: string]: Device }): Promise { const dfs = await new Promise((resolve, reject) => { DeviceFileSystem.Create({ devices: devices }, (e, dfs) => e ? reject(e) : resolve(dfs)) }); @@ -85,7 +27,7 @@ export async function configureFileSystem(devices: { [name: string]: Device }): fdMap[1] = handles.stdout; fdMap[2] = handles.stderr; - return fs; + return new Kernel(fs); } export async function asyncRead(fs: FS, fd: number, buffer: Buffer, offset: number, length: number, position: number | null) @@ -105,3 +47,4 @@ export async function asyncWrite(fs: FS, fd: number, buffer: Buffer, offset: num // Re-export for device implementors export { BFSCallback, Stats, File, FileFlag, FS, ApiError, ErrorCode }; +export * from "./DeviceFileSystem"; diff --git a/kernel/src/kernel.ts b/kernel/src/kernel.ts new file mode 100644 index 0000000..e65ff80 --- /dev/null +++ b/kernel/src/kernel.ts @@ -0,0 +1,9 @@ +import FS from "browserfs/dist/node/core/FS"; +import { makeWorker, PostMessage, OnMessage } from "./worker"; + +export default class Kernel { + fs: FS; + constructor(fs: FS) { + this.fs = fs; + }; +} diff --git a/kernel/src/node_parent.ts b/kernel/src/node_parent.ts index 29aa571..6f74e3f 100644 --- a/kernel/src/node_parent.ts +++ b/kernel/src/node_parent.ts @@ -1,7 +1,8 @@ declare module "worker_threads" { export class Worker { constructor(path: string); - postMessage(msg: any): void; + postMessage(msg: any, transferList: [any]): void; on(event: "message", handler: (msg: any) => void); + unref(): void; } } diff --git a/kernel/src/worker.ts b/kernel/src/worker.ts index 961bcb2..3afac8d 100644 --- a/kernel/src/worker.ts +++ b/kernel/src/worker.ts @@ -1,24 +1,30 @@ -import * as nodeWorkers from "worker_threads"; - export interface OnMessage { onMessage(msg: any): void; } export interface PostMessage { - postMessage(msg: any): void; + postMessage(msg: any, transferList?: [any]): void; + close(): void; } -export let makeWorker: (path: string, receiver: OnMessage) => PostMessage; +export let makeWorker: (path: string, receiver: OnMessage) => Promise; if (typeof Worker !== "undefined") { - makeWorker = (path, receiver) => { + makeWorker = async (path, receiver) => { const worker = new Worker(path); worker.onmessage = msg => receiver.onMessage(msg.data); - return { postMessage: msg => worker.postMessage(msg) }; + return { + postMessage: (msg, transferList) => worker.postMessage(msg, transferList), + close: () => {} + }; }; } else { - makeWorker = (path, receiver) => { + makeWorker = async (path, receiver) => { + const nodeWorkers = await import("worker_threads"); const worker = new nodeWorkers.Worker(path); worker.on("message", msg => receiver.onMessage(msg)); - return { postMessage: msg => worker.postMessage(msg) }; + return { + postMessage: (msg, transferList) => worker.postMessage(msg, transferList), + close: () => worker.unref() + }; } }