Skip to content

Commit

Permalink
V3: Added default thread count and build options (#9798)
Browse files Browse the repository at this point in the history
* added default thread count and build options

* ParcelNapiOptions
  • Loading branch information
alshdavid authored Jun 18, 2024
1 parent f02ddc0 commit 07dfb42
Show file tree
Hide file tree
Showing 10 changed files with 88 additions and 51 deletions.
2 changes: 1 addition & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion crates/node-bindings/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ toml = "0.8.12"
tracing = "0.1.40"
tracing-subscriber = "0.3.18"
xxhash-rust = { version = "0.8.2", features = ["xxh3"] }
num_cpus = "1.16.0"

[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
parcel = { path = "../parcel", features = ["nodejs"] }
Expand Down
11 changes: 0 additions & 11 deletions crates/node-bindings/src/file_system/file_system_napi.rs
Original file line number Diff line number Diff line change
@@ -1,14 +1,12 @@
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;

use napi::bindgen_prelude::FromNapiValue;
use napi::threadsafe_function::ThreadsafeFunctionCallMode;
use napi::Env;
use napi::JsFunction;
use napi::JsObject;
use parcel::file_system::FileSystem;
use parcel::file_system::FileSystemRef;
use serde::de::DeserializeOwned;
use serde::Serialize;

Expand Down Expand Up @@ -40,15 +38,6 @@ impl FileSystemNapi {
)?),
})
}

pub fn from_options(env: &Env, options: &JsObject) -> napi::Result<Option<FileSystemRef>> {
let mut fs = None::<FileSystemRef>;
if options.has_named_property("fs")? {
let fs_raw: JsObject = options.get_named_property("fs")?;
fs.replace(Arc::new(FileSystemNapi::new(&env, &fs_raw)?));
}
Ok(fs)
}
}

// These methods must be run off the nodejs main/worker
Expand Down
79 changes: 56 additions & 23 deletions crates/node-bindings/src/parcel/parcel.rs
Original file line number Diff line number Diff line change
@@ -1,61 +1,94 @@
use std::path::PathBuf;
use std::sync::Arc;
use std::thread;

use napi::Env;
use napi::JsNumber;
use napi::JsFunction;
use napi::JsObject;
use napi_derive::napi;
use parcel::rpc::nodejs::RpcHostNodejs;
use parcel::BuildOptions;
use parcel::Parcel;
use parcel::ParcelOptions;

use crate::file_system::FileSystemNapi;

#[napi(object)]
pub struct ParcelNapiBuildOptions {}

#[napi(object)]
pub struct ParcelNapiBuildResult {}

#[napi(object)]
pub struct ParcelNapiOptions {
pub threads: Option<u32>,
pub node_workers: Option<u32>,
pub fs: Option<JsObject>,
pub rpc: JsFunction,
}

#[napi]
pub struct ParcelNapi {
pub node_worker_count: u32,
parcel: Arc<Parcel>,
}

#[napi]
impl ParcelNapi {
#[napi(constructor)]
pub fn new(env: Env, options: JsObject) -> napi::Result<Self> {
pub fn new(env: Env, options: ParcelNapiOptions) -> napi::Result<Self> {
// Debugging Instrumentation
let _ = tracing_subscriber::fmt::try_init();
let thread_id = std::thread::current().id();
tracing::trace!(?thread_id, "parcel-napi initialize");

// Parcel Core Options
let mut parcel_options = ParcelOptions::default();

// Wrap the JavaScript-supplied FileSystem
let fs = FileSystemNapi::from_options(&env, &options)?;
if let Some(fs) = options.fs {
parcel_options.fs = Some(Arc::new(FileSystemNapi::new(&env, &fs)?));
}

// Assign Rust thread count from JavaScript
if let Some(threads) = options.threads {
parcel_options.threads = threads as usize;
}

// Set up Nodejs plugin bindings
let node_workers: JsNumber = options.get_property(env.create_string("nodeWorkers")?)?;
let node_workers = node_workers.get_uint32()?;
let rpc_host_nodejs =
RpcHostNodejs::new(&env, options.get_named_property("rpc")?, node_workers)?;

// Initialize Parcel
let parcel = Parcel::new(ParcelOptions {
fs,
rpc: Some(Arc::new(rpc_host_nodejs)),
});
let node_worker_count: usize;
if let Some(node_workers) = options.node_workers {
node_worker_count = node_workers as usize;
} else {
node_worker_count = parcel_options.threads;
}

let rpc_host_nodejs = RpcHostNodejs::new(&env, options.rpc, node_worker_count.clone())?;
parcel_options.rpc = Some(Arc::new(rpc_host_nodejs));

// Return self
Ok(Self {
parcel: Arc::new(parcel),
node_worker_count: node_worker_count as u32,
parcel: Arc::new(Parcel::new(parcel_options)),
})
}

#[napi]
pub async fn build(&self) -> napi::Result<()> {
self.parcel.build().unwrap();
Ok(())
}
pub fn build(&self, env: Env, _options: ParcelNapiBuildOptions) -> napi::Result<JsObject> {
let (deferred, promise) = env.create_deferred()?;
// Parse build options from JS options
let build_options = BuildOptions {};

#[napi]
pub fn default_thread_count(env: Env) -> napi::Result<JsNumber> {
let cpus = num_cpus::get();
let cpus = env.create_int32(cpus as i32)?;
Ok(cpus)
// Call build in its own dedicated system thread
thread::spawn({
let parcel = self.parcel.clone();
move || match parcel.build(build_options) {
Ok(_result) => deferred.resolve(|_env| Ok(ParcelNapiBuildResult {})),
Err(error) => deferred.reject(napi::Error::from_reason(format!("{:?}", error))),
}
});

Ok(promise)
}

// Temporary, for testing
Expand Down
1 change: 1 addition & 0 deletions crates/parcel/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,4 @@ anyhow = "1.0.82"
dyn-hash = "0.x"
petgraph = "0.x"
xxhash-rust = { version = "0.8.2", features = ["xxh3"] }
num_cpus = "1.16.0"
20 changes: 18 additions & 2 deletions crates/parcel/src/parcel.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,28 @@ use parcel_plugin_rpc::RpcHostRef;
pub struct Parcel {
pub fs: FileSystemRef,
pub rpc: Option<RpcHostRef>,
pub threads: usize,
}

pub struct ParcelOptions {
pub fs: Option<FileSystemRef>,
pub rpc: Option<RpcHostRef>,
pub threads: usize,
}

impl Default for ParcelOptions {
fn default() -> Self {
Self {
fs: Some(Arc::new(OsFileSystem::default())),
rpc: Default::default(),
threads: num_cpus::get(),
}
}
}

pub struct BuildOptions;
pub struct BuildResult;

impl Parcel {
pub fn new(options: ParcelOptions) -> Self {
let fs = options
Expand All @@ -28,16 +43,17 @@ impl Parcel {
Self {
fs,
rpc: options.rpc,
threads: options.threads,
}
}

pub fn build(&self) -> anyhow::Result<()> {
pub fn build(&self, options: BuildOptions) -> anyhow::Result<BuildResult> {
let mut _rpc_connection = None::<RpcConnectionRef>;

if let Some(rpc_host) = &self.rpc {
_rpc_connection = Some(rpc_host.start()?);
}

Ok(())
Ok(BuildResult {})
}
}
4 changes: 2 additions & 2 deletions crates/parcel_plugin_rpc/src/nodejs/rpc_host_nodejs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,11 @@ use super::RpcConnectionNodejsMulti;
// the lazy initialization of Nodejs worker threads.
pub struct RpcHostNodejs {
threadsafe_function: ThreadsafeFunction<RpcHostMessage>,
node_workers: u32,
node_workers: usize,
}

impl RpcHostNodejs {
pub fn new(env: &Env, callback: JsFunction, node_workers: u32) -> napi::Result<Self> {
pub fn new(env: &Env, callback: JsFunction, node_workers: usize) -> napi::Result<Self> {
// Create a threadsafe function that casts the incoming message data to something
// accessible in JavaScript. The function accepts a return value from a JS callback
let mut threadsafe_function: ThreadsafeFunction<RpcHostMessage> = env
Expand Down
16 changes: 7 additions & 9 deletions packages/core/core/src/parcel-v3/Parcel.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,16 @@ export type ParcelV3Options = {|
fs?: FileSystem,
|};

export type ParcelV3BuildOptions = {||};

export class ParcelV3 {
_internal: napi.ParcelNapi;
#nodeWorkerCount: number;

constructor({
threads = napi.ParcelNapi.defaultThreadCount(),
nodeWorkers,
}: ParcelV3Options) {
this.#nodeWorkerCount = nodeWorkers || threads;
constructor({threads, nodeWorkers, fs}: ParcelV3Options) {
this._internal = new napi.ParcelNapi({
threads,
nodeWorkers,
fs,
rpc: async (err, id, data, done) => {
try {
if (err) {
Expand All @@ -49,12 +47,12 @@ export class ParcelV3 {
}
}

async build(): Promise<any> {
async build(options: ParcelV3BuildOptions): Promise<any> {
// initialize workers lazily
const workers = this.#startWorkers();

// Run the Parcel build
let result = await this._internal.build();
let result = await this._internal.build(options);

// Stop workers
this.#stopWorkers(await workers);
Expand All @@ -65,7 +63,7 @@ export class ParcelV3 {
const workersOnLoad = [];
const workers = [];

for (let i = 0; i < this.#nodeWorkerCount; i++) {
for (let i = 0; i < this._internal.nodeWorkerCount; i++) {
let worker = new Worker(path.join(__dirname, 'worker', 'index.js'));
workers.push(worker);
workersOnLoad.push(
Expand Down
2 changes: 1 addition & 1 deletion packages/core/integration-tests/test/parcel-v3.js
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,6 @@ describe('parcel-v3', function () {
assert(await parcel._internal.testingTempFsIsFile(__filename));
await parcel._internal.testingRpcPing();

await parcel.build();
await parcel.build({});
});
});
3 changes: 2 additions & 1 deletion packages/core/rust/index.js.flow
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,9 @@ export interface ParcelNapiOptions {
}

declare export class ParcelNapi {
nodeWorkerCount: number,
constructor(options: ParcelNapiOptions): ParcelNapi;
build(): Promise<void>;
build(options: {||}): Promise<void>;
static defaultThreadCount(): number;
testingTempFsReadToString(path: string): string;
testingTempFsIsDir(path: string): boolean;
Expand Down

0 comments on commit 07dfb42

Please sign in to comment.