Parcel V3 - Native asset request (#9808)

crates/parcel/src/plugins.rs

@@ -1,4 +1,8 @@
+use std::fmt::Debug;
+use std::hash::Hash;
+use std::hash::Hasher;
 use std::path::Path;
+use std::u64;
 
 use anyhow::anyhow;
 use parcel_config::map::NamedPattern;
@@ -152,14 +156,17 @@ impl<'a> Plugins<'a> {
     &self,
     path: &Path,
     pipeline: Option<&str>,
-  ) -> Result<Vec<Box<dyn TransformerPlugin>>, anyhow::Error> {
+  ) -> Result<TransformerPipeline, anyhow::Error> {
     let mut transformers: Vec<Box<dyn TransformerPlugin>> = Vec::new();
     let named_pattern = pipeline.map(|pipeline| NamedPattern {
       pipeline,
       use_fallback: false,
     });
 
+    let mut hasher = parcel_core::hash::IdentifierHasher::default();
+
     for transformer in self.config.transformers.get(path, named_pattern).iter() {
+      transformer.hash(&mut hasher);
       if transformer.package_name == "@parcel/transformer-swc" {
         transformers.push(Box::new(ParcelTransformerJs::new(self.ctx)));
         continue;
@@ -175,14 +182,36 @@ impl<'a> Plugins<'a> {
       };
     }
 
-    Ok(transformers)
+    Ok(TransformerPipeline {
+      transformers,
+      hash: hasher.finish(),
+    })
   }
 
   pub fn validators(&self, _path: &Path) -> Result<Vec<Box<dyn ValidatorPlugin>>, anyhow::Error> {
     todo!()
   }
 }
 
+pub struct TransformerPipeline {
+  pub transformers: Vec<Box<dyn TransformerPlugin>>,
+  hash: u64,
+}
+
+impl PartialEq for TransformerPipeline {
+  fn eq(&self, other: &Self) -> bool {
+    self.hash == other.hash
+  }
+}
+
+impl Debug for TransformerPipeline {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    f.debug_struct("TransformerPipeline")
+      .field("transformers", &self.transformers)
+      .finish()
+  }
+}
+
 #[cfg(test)]
 mod tests {
   use std::path::PathBuf;
@@ -283,6 +312,9 @@ mod tests {
       .transformers(Path::new("a.ts"), None)
       .expect("Not to panic");
 
-    assert_eq!(format!("{:?}", transformers), "[RpcTransformerPlugin]")
+    assert_eq!(
+      format!("{:?}", transformers),
+      r"TransformerPipeline { transformers: [RpcTransformerPlugin] }"
+    )
   }
 }

crates/parcel/src/requests.rs

@@ -1 +1,2 @@
+mod asset_request;
 mod path_request;

crates/parcel/src/requests/asset_request.rs

@@ -0,0 +1,168 @@
+use std::hash::Hash;
+use std::hash::Hasher;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use anyhow::anyhow;
+use parcel_core::cache::CacheRef;
+use parcel_core::plugin::AssetBuildEvent;
+use parcel_core::plugin::BuildProgressEvent;
+use parcel_core::plugin::InitialAsset;
+use parcel_core::plugin::ReporterEvent;
+use parcel_core::plugin::RunTransformContext;
+use parcel_core::plugin::TransformResult;
+use parcel_core::plugin::TransformationInput;
+use parcel_core::types::Asset;
+use parcel_core::types::AssetStats;
+use parcel_core::types::Dependency;
+use parcel_core::types::Environment;
+use parcel_core::types::FileType;
+use parcel_filesystem::FileSystemRef;
+
+use crate::plugins::Plugins;
+use crate::plugins::TransformerPipeline;
+use crate::request_tracker::{Request, RequestResult, RunRequestContext, RunRequestError};
+
+/// The AssetRequest runs transformer plugins on discovered Assets.
+/// - Decides which transformer pipeline to run from the input Asset type
+/// - Runs the pipeline in series, switching pipeline if the Asset type changes
+/// - Stores the final Asset source code in the cache, for access in packaging
+/// - Finally, returns the complete Asset and it's discovered Dependencies
+pub struct AssetRequest<'a> {
+  pub env: Arc<Environment>,
+  pub file_path: PathBuf,
+  pub code: Option<String>,
+  pub pipeline: Option<String>,
+  pub side_effects: bool,
+  // TODO: move the following to RunRequestContext
+  pub cache: CacheRef,
+  pub file_system: FileSystemRef,
+  pub plugins: Arc<Plugins<'a>>,
+}
+
+impl<'a> Hash for AssetRequest<'a> {
+  fn hash<H: Hasher>(&self, state: &mut H) {
+    // TODO: Just derive this once the contextual params are moved to RunRequestContext
+    self.file_path.hash(state);
+    self.code.hash(state);
+    self.pipeline.hash(state);
+    self.env.hash(state);
+    self.side_effects.hash(state);
+  }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct AssetResult {
+  pub asset: Asset,
+  pub dependencies: Vec<Dependency>,
+}
+
+impl<'a> Request<AssetResult> for AssetRequest<'a> {
+  fn run(
+    &self,
+    request_context: RunRequestContext<AssetResult>,
+  ) -> Result<RequestResult<AssetResult>, RunRequestError> {
+    request_context.report(ReporterEvent::BuildProgress(BuildProgressEvent::Building(
+      AssetBuildEvent {
+        // TODO: Should we try avoid a clone here?
+        file_path: self.file_path.clone(),
+      },
+    )));
+
+    let pipeline = self
+      .plugins
+      .transformers(&self.file_path, self.pipeline.as_deref())?;
+    let asset_type = FileType::from_extension(
+      self
+        .file_path
+        .extension()
+        .and_then(|s| s.to_str())
+        .unwrap_or(""),
+    );
+    let mut transform_ctx = RunTransformContext::new(self.file_system.clone());
+
+    let result = run_pipeline(
+      pipeline,
+      TransformationInput::InitialAsset(InitialAsset {
+        // TODO: Are these clones neccessary?
+        file_path: self.file_path.clone(),
+        code: self.code.clone(),
+        env: self.env.clone(),
+        side_effects: self.side_effects,
+      }),
+      asset_type,
+      &self.plugins,
+      &mut transform_ctx,
+    )?;
+
+    // Write the Asset source code to the cache, this is read later in packaging
+    // TODO: Clarify the correct content key
+    let content_key = result.asset.id().to_string();
+    self
+      .cache
+      .set_blob(&content_key, result.asset.code.bytes())?;
+
+    Ok(RequestResult {
+      result: AssetResult {
+        asset: Asset {
+          stats: AssetStats {
+            size: result.asset.code.size(),
+            time: 0,
+          },
+          ..result.asset
+        },
+        dependencies: result.dependencies,
+      },
+      // TODO: Support invalidations
+      invalidations: vec![],
+    })
+  }
+}
+
+fn run_pipeline(
+  pipeline: TransformerPipeline,
+  input: TransformationInput,
+  asset_type: FileType,
+  plugins: &Plugins,
+  transform_ctx: &mut RunTransformContext,
+) -> anyhow::Result<TransformResult> {
+  let mut dependencies = vec![];
+  let mut invalidations = vec![];
+
+  let mut transform_input = input;
+
+  for transformer in &pipeline.transformers {
+    let transform_result = transformer.transform(transform_ctx, transform_input)?;
+    let is_different_asset_type = transform_result.asset.asset_type != asset_type;
+
+    transform_input = TransformationInput::Asset(transform_result.asset);
+
+    // If the Asset has changed type then we may need to trigger a different pipeline
+    if is_different_asset_type {
+      let next_pipeline = plugins.transformers(transform_input.file_path(), None)?;
+
+      if next_pipeline != pipeline {
+        return run_pipeline(
+          next_pipeline,
+          transform_input,
+          asset_type,
+          plugins,
+          transform_ctx,
+        );
+      };
+    }
+
+    dependencies.extend(transform_result.dependencies);
+    invalidations.extend(transform_result.invalidate_on_file_change);
+  }
+
+  if let TransformationInput::Asset(asset) = transform_input {
+    Ok(TransformResult {
+      asset,
+      dependencies,
+      invalidate_on_file_change: invalidations,
+    })
+  } else {
+    Err(anyhow!("No transformations for Asset"))
+  }
+}

crates/parcel_config/src/map/named_pipelines_map.rs

@@ -1,3 +1,5 @@
+use std::hash::Hash;
+use std::hash::Hasher;
 use std::path::Path;
 
 use indexmap::IndexMap;
@@ -45,6 +47,14 @@ pub struct NamedPipelinesMap(
   IndexMap<String, Vec<PluginNode>>,
 );
 
+impl Hash for NamedPipelinesMap {
+  fn hash<H: Hasher>(&self, state: &mut H) {
+    for item in self.0.iter() {
+      item.hash(state);
+    }
+  }
+}
+
 impl NamedPipelinesMap {
   pub fn new(map: IndexMap<String, Vec<PluginNode>>) -> Self {
     Self(map)

crates/parcel_config/src/parcel_config.rs

@@ -11,7 +11,7 @@ use crate::map::NamedPipelinesMap;
 use crate::map::PipelineMap;
 use crate::map::PipelinesMap;
 
-#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
+#[derive(Clone, Debug, Deserialize, Hash, PartialEq, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct PluginNode {
   pub package_name: String,

crates/parcel_core/src/cache.rs

@@ -0,0 +1,11 @@
+use std::sync::Arc;
+
+use anyhow::anyhow;
+
+pub type CacheRef = Arc<dyn Cache + Sync + Send>;
+
+pub trait Cache {
+  fn set_blob(&self, _key: &str, _blob: &[u8]) -> anyhow::Result<()> {
+    Err(anyhow!("Not implmented"))
+  }
+}

crates/parcel_core/src/lib.rs

@@ -1,4 +1,5 @@
 pub mod bundle_graph;
+pub mod cache;
 pub mod hash;
 pub mod plugin;
 pub mod types;

crates/parcel_core/src/plugin/reporter_plugin.rs

@@ -1,14 +1,19 @@
-use std::fmt::Debug;
 use std::sync::Arc;
+use std::{fmt::Debug, path::PathBuf};
 
 use crate::types::Dependency;
 
 pub struct ResolvingEvent {
   pub dependency: Arc<Dependency>,
 }
 
+pub struct AssetBuildEvent {
+  pub file_path: PathBuf,
+}
+
 pub enum BuildProgressEvent {
   Resolving(ResolvingEvent),
+  Building(AssetBuildEvent),
 }
 
 // TODO Flesh these out