diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..71b7ebc --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +build/ +dist/ +package-lock.json diff --git a/example/package.json b/example/package.json new file mode 100644 index 0000000..45959e4 --- /dev/null +++ b/example/package.json @@ -0,0 +1,19 @@ +{ + "name": "webabi-example", + "version": "0.0.1", + "description": "", + "scripts": { + "build": "tsc && webpack", + "install": "npm run build", + "test": "npm run build && (cd ..; echo bar | node --experimental-worker example/build/main.js)" + }, + "author": "Will Fancher", + "dependencies": { + "typescript": "^2.9.2", + "webabi-kernel": "file:../kernel" + }, + "devDependencies": { + "webpack": "^4.16.3", + "webpack-cli": "^3.1.0" + } +} diff --git a/example/src/index.ts b/example/src/index.ts new file mode 100644 index 0000000..36f9b85 --- /dev/null +++ b/example/src/index.ts @@ -0,0 +1,36 @@ +import { Device, configureKernel, BFSCallback, + Stats, File, FileFlag, FS, ApiError, + ErrorCode, asyncRead, asyncWrite } from "webabi-kernel"; +import { makeWorker, PostMessage, OnMessage } from "webabi-kernel/dist/worker"; + +class JSaddleDevice implements Device { + async open(flag: FileFlag): Promise { + throw new ApiError(ErrorCode.ENOTSUP); + } + async stat(isLstat: boolean | null): Promise { + throw new ApiError(ErrorCode.ENOTSUP); + } +} + +async function main() { + let kernel = await configureKernel({ "/jsaddle": new JSaddleDevice() }); + let fs = kernel.fs; + let buf = Buffer.from("foo\n"); + await asyncWrite(fs, 1, buf, 0, buf.length, null); + const { byteLength } = await asyncRead(fs, 0, buf, 0, buf.length, null); + console.log({ byteLength: byteLength, buffer: buf.toString() }); + + console.log("working"); + let worker: PostMessage; + worker = await makeWorker("./exec/build/main.js", { + onMessage: msg => { + console.log(msg); + worker.close(); + } + }); + worker.postMessage("foo"); +} + +main().catch(e => { + console.error("Error: ", e); +}); diff --git a/example/tsconfig.json b/example/tsconfig.json new file mode 100644 index 0000000..156c09a --- /dev/null +++ b/example/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "es6", + "outDir": "dist", + "lib": ["dom", "es2015", "es2016", "es2017"], + "module": "commonjs", + "declaration": true + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "node_modules" + ] +} diff --git a/example/webpack.config.js b/example/webpack.config.js new file mode 100644 index 0000000..618670b --- /dev/null +++ b/example/webpack.config.js @@ -0,0 +1,25 @@ +const path = require('path'); + +module.exports = { + entry: './dist/index.js', + mode: "production", + externals: [ + ((context, request, callback) => { + if (request == "worker_threads") { + return callback(null, "commonjs " + request); + } + callback(); + }) + ], + output: { + filename: 'main.js', + path: path.resolve(__dirname, 'build') + }, + resolve: { + // Using file:../kernel in package.json requires this + symlinks: false + }, + node: { + process: false + } +}; diff --git a/exec/package.json b/exec/package.json new file mode 100644 index 0000000..58d128d --- /dev/null +++ b/exec/package.json @@ -0,0 +1,18 @@ +{ + "name": "webabi-exec", + "version": "0.0.1", + "description": "", + "scripts": { + "build": "tsc && webpack", + "install": "npm run build" + }, + "main": "dist/index.js", + "author": "Will Fancher", + "dependencies": { + "typescript": "^2.9.2" + }, + "devDependencies": { + "webpack": "^4.16.3", + "webpack-cli": "^3.1.0" + } +} diff --git a/exec/src/index.ts b/exec/src/index.ts new file mode 100644 index 0000000..1c749f5 --- /dev/null +++ b/exec/src/index.ts @@ -0,0 +1,14 @@ +import { connectParent, PostMessage } from "./worker"; + +async function main() { + let parent: PostMessage; + parent = await connectParent({ + onMessage: msg => { + console.log(msg); + parent.close(); + } + }); + parent.postMessage("bar"); +} + +main(); diff --git a/exec/src/node_worker.ts b/exec/src/node_worker.ts new file mode 100644 index 0000000..fdfb35a --- /dev/null +++ b/exec/src/node_worker.ts @@ -0,0 +1,8 @@ +declare module "worker_threads" { + interface MessagePort { + postMessage(msg: any, transferList: [any]); + on(event: "message", handler: (msg: any) => void); + unref(): void; + } + export const parentPort: MessagePort; +} diff --git a/exec/src/worker.ts b/exec/src/worker.ts new file mode 100644 index 0000000..6f54961 --- /dev/null +++ b/exec/src/worker.ts @@ -0,0 +1,28 @@ +export interface OnMessage { + onMessage(msg: any): void; +} + +export interface PostMessage { + postMessage(msg: any, transferList?: [any]): void; + close(): void; +} + +export let connectParent: (receiver: OnMessage) => Promise; +if (typeof self !== "undefined") { + connectParent = async (receiver) => { + self.onmessage = msg => receiver.onMessage(msg.data); + return { + postMessage: (msg, transferList) => self.postMessage(msg, transferList as any /*tsc complains wrongly*/), + close: () => {} + }; + }; +} else { + connectParent = async (receiver) => { + const nodeWorkers = await import("worker_threads"); + nodeWorkers.parentPort.on("message", msg => receiver.onMessage(msg)); + return { + postMessage: (msg, transferList) => nodeWorkers.parentPort.postMessage(msg, transferList), + close: () => nodeWorkers.parentPort.unref() + }; + } +} diff --git a/exec/tsconfig.json b/exec/tsconfig.json new file mode 100644 index 0000000..156c09a --- /dev/null +++ b/exec/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "es6", + "outDir": "dist", + "lib": ["dom", "es2015", "es2016", "es2017"], + "module": "commonjs", + "declaration": true + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "node_modules" + ] +} diff --git a/exec/webpack.config.js b/exec/webpack.config.js new file mode 100644 index 0000000..bf47314 --- /dev/null +++ b/exec/webpack.config.js @@ -0,0 +1,18 @@ +const path = require('path'); + +module.exports = { + entry: './dist/index.js', + mode: "production", + externals: [ + ((context, request, callback) => { + if (request == "worker_threads") { + return callback(null, "commonjs " + request); + } + callback(); + }) + ], + output: { + filename: 'main.js', + path: path.resolve(__dirname, 'build') + } +}; diff --git a/index.html b/index.html new file mode 100644 index 0000000..25e7984 --- /dev/null +++ b/index.html @@ -0,0 +1,9 @@ + + + + Getting Started + + + + + diff --git a/kernel/package.json b/kernel/package.json new file mode 100644 index 0000000..6251a89 --- /dev/null +++ b/kernel/package.json @@ -0,0 +1,25 @@ +{ + "name": "webabi-kernel", + "version": "0.0.1", + "description": "", + "scripts": { + "build": "tsc", + "install": "npm run build" + }, + "main": "dist/index.js", + "author": "Will Fancher", + "dependencies": { + "@types/archiver": "^2.0.0", + "@types/async": "^2.0.49", + "@types/body-parser": "^1.16.4", + "@types/dropboxjs": "0.0.29", + "@types/express": "^4.0.36", + "@types/filesystem": "0.0.28", + "@types/isomorphic-fetch": "^0.0.34", + "@types/mocha": "^5.2.5", + "@types/node": "^7.0", + "@types/rimraf": "^2.0.2", + "browserfs": "^1.4.3", + "typescript": "^2.9.2" + } +} diff --git a/kernel/src/DeviceFileSystem.ts b/kernel/src/DeviceFileSystem.ts new file mode 100644 index 0000000..384fdb8 --- /dev/null +++ b/kernel/src/DeviceFileSystem.ts @@ -0,0 +1,63 @@ +import { BaseFileSystem, BFSCallback, FileSystem, FileSystemOptions + } from "browserfs/dist/node/core/file_system"; +import Stats from 'browserfs/dist/node/core/node_fs_stats'; +import { File } from "browserfs/dist/node/core/file"; +import { FileFlag } from "browserfs/dist/node/core/file_flag"; +import { ApiError } from 'browserfs/dist/node/core/api_error'; + +export interface Device { + open(flag: FileFlag): Promise; + stat(isLstat: boolean | null): Promise; +} + +export interface DeviceFileSystemOptions { + devices: {[name: string]: Device}; +} + +export class DeviceFileSystem extends BaseFileSystem implements FileSystem { + public static readonly Name = "DeviceFileSystem"; + public static readonly Options: FileSystemOptions = {}; + + public static Create(opts: DeviceFileSystemOptions, cb: BFSCallback): void { + return cb(null, new DeviceFileSystem(opts)); + } + + public static isAvailable(): boolean { + return true; + } + + options: DeviceFileSystemOptions; + + constructor(options: DeviceFileSystemOptions) { + super(); + this.options = options; + } + + public getName() { + return "DeviceFileSystem"; + } + public isReadOnly() { + return false; + } + public supportsProps() { + return false; + } + public supportsSynch() { + return false; + } + + public openFile(p: string, flag: FileFlag, cb: BFSCallback): void { + if (this.options.devices.hasOwnProperty(p)) { + this.options.devices[p].open(flag).then(f => cb(undefined, f), e => cb(e)); + } else { + cb(ApiError.ENOENT(p)); + } + } + public stat(p: string, isLstat: boolean | null, cb: BFSCallback): void { + if (this.options.devices.hasOwnProperty(p)) { + this.options.devices[p].stat(isLstat).then(s => cb(undefined, s), e => cb(e)); + } else { + cb(ApiError.ENOENT(p)); + } + } +} diff --git a/kernel/src/index.ts b/kernel/src/index.ts new file mode 100644 index 0000000..f523d54 --- /dev/null +++ b/kernel/src/index.ts @@ -0,0 +1,50 @@ +import MountableFileSystem from "browserfs/dist/node/backend/MountableFileSystem"; +import * as handles from "./stdio_handles"; +import { BFSCallback } from "browserfs/dist/node/core/file_system"; +import Stats from 'browserfs/dist/node/core/node_fs_stats'; +import { File } from "browserfs/dist/node/core/file"; +import { FileFlag } from "browserfs/dist/node/core/file_flag"; +import { ApiError, ErrorCode } from 'browserfs/dist/node/core/api_error'; +import FS from "browserfs/dist/node/core/FS"; +import { DeviceFileSystem, Device } from "./DeviceFileSystem"; +import Kernel from "./kernel"; + +export async function configureKernel(devices: { [name: string]: Device }): Promise { + const dfs = await new Promise((resolve, reject) => { + DeviceFileSystem.Create({ devices: devices }, (e, dfs) => e ? reject(e) : resolve(dfs)) + }); + const mfs = await new Promise((resolve, reject) => { + MountableFileSystem.Create({ + "/dev": dfs + }, (e, mfs) => e ? reject(e) : resolve(mfs)); + }); + + const fs = new FS(); + fs.initialize(mfs); + + const fdMap: {[id: number]: File} = (fs as any).fdMap; + fdMap[0] = handles.stdin; + fdMap[1] = handles.stdout; + fdMap[2] = handles.stderr; + + return new Kernel(fs); +} + +export async function asyncRead(fs: FS, fd: number, buffer: Buffer, offset: number, length: number, position: number | null) +: Promise<{ byteLength: number, buffer: Buffer }> { + return new Promise<{ byteLength: number, buffer: Buffer }>((resolve, reject) => { + fs.read(fd, buffer, offset, length, position, + (err, n, buf) => err ? reject(err) : resolve({ byteLength: n, buffer: buf })); + }); +} + +export async function asyncWrite(fs: FS, fd: number, buffer: Buffer, offset: number, length: number, position: number | null) +: Promise { + return new Promise((resolve, reject) => { + fs.write(fd, buffer, offset, length, position, e => e ? reject(e) : resolve()) + }); +} + +// Re-export for device implementors +export { BFSCallback, Stats, File, FileFlag, FS, ApiError, ErrorCode }; +export * from "./DeviceFileSystem"; diff --git a/kernel/src/kernel.ts b/kernel/src/kernel.ts new file mode 100644 index 0000000..e65ff80 --- /dev/null +++ b/kernel/src/kernel.ts @@ -0,0 +1,9 @@ +import FS from "browserfs/dist/node/core/FS"; +import { makeWorker, PostMessage, OnMessage } from "./worker"; + +export default class Kernel { + fs: FS; + constructor(fs: FS) { + this.fs = fs; + }; +} diff --git a/kernel/src/node_parent.ts b/kernel/src/node_parent.ts new file mode 100644 index 0000000..6f74e3f --- /dev/null +++ b/kernel/src/node_parent.ts @@ -0,0 +1,8 @@ +declare module "worker_threads" { + export class Worker { + constructor(path: string); + postMessage(msg: any, transferList: [any]): void; + on(event: "message", handler: (msg: any) => void); + unref(): void; + } +} diff --git a/kernel/src/stdio_handles.ts b/kernel/src/stdio_handles.ts new file mode 100644 index 0000000..386f042 --- /dev/null +++ b/kernel/src/stdio_handles.ts @@ -0,0 +1,176 @@ +import { File, BaseFile } from "browserfs/dist/node/core/file"; +import { BaseFileSystem, FileSystemConstructor, BFSCallback, BFSOneArgCallback, BFSThreeArgCallback, FileSystem, FileSystemOptions } from "browserfs/dist/node/core/file_system"; +import { FileType } from 'browserfs/dist/node/core/node_fs_stats'; +import Stats from 'browserfs/dist/node/core/node_fs_stats'; +import { ApiError, ErrorCode } from 'browserfs/dist/node/core/api_error'; + +export let stdin: File; +export let stdout: File; +export let stderr: File; + +class UselessFile extends BaseFile implements File { + getPos(): number | undefined { + return undefined; + } + stat(cb: BFSCallback): void { + return cb(undefined, new Stats(FileType.FILE, 0)); + } + statSync(): Stats { + return new Stats(FileType.FILE, 0) + } + close(cb: BFSOneArgCallback): void { + cb(new ApiError(ErrorCode.ENOTSUP)); + } + closeSync(): void { + throw new ApiError(ErrorCode.ENOTSUP); + } + truncate(len: number, cb: BFSOneArgCallback): void { + cb(new ApiError(ErrorCode.ENOTSUP)); + } + truncateSync(len: number): void { + throw new ApiError(ErrorCode.ENOTSUP); + } + write(buffer: Buffer, offset: number, length: number, position: number | null, cb: BFSThreeArgCallback): void { + cb(new ApiError(ErrorCode.ENOTSUP)); + } + writeSync(buffer: Buffer, offset: number, length: number, position: number | null): number { + throw new ApiError(ErrorCode.ENOTSUP); + } + read(buffer: Buffer, offset: number, length: number, position: number | null, cb: BFSThreeArgCallback): void { + cb(new ApiError(ErrorCode.ENOTSUP)); + } + readSync(buffer: Buffer, offset: number, length: number, position: number): number { + throw new ApiError(ErrorCode.ENOTSUP); + } +} + +if (!(typeof process === "undefined" || (process as any).browser)) { + interface Request { + buffer: Buffer; + offset: number; + length: number; + cb: BFSThreeArgCallback; + } + class ReadWriteStreamFile extends UselessFile implements File { + stream: NodeJS.ReadWriteStream; + requests: [Request] = <[Request]> []; + leftover?: Buffer = null; + + constructor(stream: NodeJS.ReadWriteStream) { + super(); + this.stream = stream; + this.stream.pause(); + this.stream.on("error", (err) => { + const reqs = this.requests; + this.requests = <[Request]> []; + for (const req of reqs) { + req.cb(err, undefined, undefined); + } + }); + this.stream.on("data", (buf) => { + this.stream.pause(); + if (this.leftover) { + buf = Buffer.concat([this.leftover, buf]); + this.leftover = null; + } + this.onData(buf); + }); + } + + onData(buf: Buffer): void { + const reqs = this.requests; + let nextBuf: Buffer | null = null; + let req: Request; + while (req = this.requests.shift()) { + if (buf.length > req.length) { + nextBuf = buf.slice(req.length); + buf = buf.slice(0, req.length); + } else { + nextBuf = null; + } + + const copied = buf.copy(req.buffer, req.offset); + req.cb(undefined, copied, req.buffer); + + if (!nextBuf || nextBuf.length == 0) { + break; + } + buf = nextBuf; + } + + if (nextBuf) { + // nextBuf may still have the old leftover underlying it. + // Use Buffer.from to avoid retaining the entire history. + this.leftover = Buffer.from(nextBuf); + } + if (this.requests.length > 0) { + this.stream.resume(); + } + }; + close(cb: BFSOneArgCallback): void { + this.stream.end(cb); + } + write(buffer: Buffer, offset: number, length: number, position: number | null, cb: BFSThreeArgCallback): void { + this.stream.write(buffer.slice(offset, offset + length), (err) => { + if (err) { + cb(err); + } else { + cb(undefined, length, buffer); + } + }); + } + read(buffer: Buffer, offset: number, length: number, position: number | null, cb: BFSThreeArgCallback): void { + this.stream.resume(); + this.requests.push({ + buffer: buffer, + offset: offset, + length: length, + cb: cb + }); + } + } + + stdin = new ReadWriteStreamFile(process.stdin); + stdout = new ReadWriteStreamFile(process.stdout); + stderr = new ReadWriteStreamFile(process.stderr); +} else { + class ConsoleFile extends UselessFile implements File { + log: (msg: string) => void; + buffer?: Buffer = null; + + constructor(log: (msg: string) => void) { + super(); + this.log = log; + } + + write(buffer: Buffer, offset: number, length: number, position: number | null, cb: BFSThreeArgCallback): void { + let slicedBuffer = buffer.slice(offset, offset + length); + let n = slicedBuffer.lastIndexOf("\n"); + if (n < 0) { + if (this.buffer) { + this.buffer = Buffer.concat([this.buffer, slicedBuffer]); + } else { + this.buffer = slicedBuffer; + } + } else { + let logBuffer = slicedBuffer.slice(0, n); + if (this.buffer) { + logBuffer = Buffer.concat([this.buffer, logBuffer]); + } + this.log(logBuffer.toString()); + + // + 1 to skip the \n + if (n + 1 < slicedBuffer.length) { + this.buffer = slicedBuffer.slice(n + 1); + } else { + this.buffer = null; + } + } + cb(undefined, length, buffer); + } + } + + stdin = new UselessFile(); + stdout = new ConsoleFile((msg) => console.log(msg)); + stderr = new ConsoleFile((msg) => console.error(msg)); +} diff --git a/kernel/src/worker.ts b/kernel/src/worker.ts new file mode 100644 index 0000000..3afac8d --- /dev/null +++ b/kernel/src/worker.ts @@ -0,0 +1,30 @@ +export interface OnMessage { + onMessage(msg: any): void; +} + +export interface PostMessage { + postMessage(msg: any, transferList?: [any]): void; + close(): void; +} + +export let makeWorker: (path: string, receiver: OnMessage) => Promise; +if (typeof Worker !== "undefined") { + makeWorker = async (path, receiver) => { + const worker = new Worker(path); + worker.onmessage = msg => receiver.onMessage(msg.data); + return { + postMessage: (msg, transferList) => worker.postMessage(msg, transferList), + close: () => {} + }; + }; +} else { + makeWorker = async (path, receiver) => { + const nodeWorkers = await import("worker_threads"); + const worker = new nodeWorkers.Worker(path); + worker.on("message", msg => receiver.onMessage(msg)); + return { + postMessage: (msg, transferList) => worker.postMessage(msg, transferList), + close: () => worker.unref() + }; + } +} diff --git a/kernel/tsconfig.json b/kernel/tsconfig.json new file mode 100644 index 0000000..156c09a --- /dev/null +++ b/kernel/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "es6", + "outDir": "dist", + "lib": ["dom", "es2015", "es2016", "es2017"], + "module": "commonjs", + "declaration": true + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "node_modules" + ] +}