feat(frontend-canister): create_chunks (#3898)

CHANGELOG.md

@@ -36,6 +36,14 @@ This applies to the following:
  - tech stack value computation
  - packtool (vessel, mops etc)
 
+## Dependencies
+
+### Frontend canister
+
+Added `create_chunks`. It has the same behavior as `create_chunk`, except that it takes a `vec blob` and returns a `vec BatchId` instead of non-`vec` variants.
+
+Module hash: 3a533f511b3960b4186e76cf9abfbd8222a2c507456a66ec55671204ee70cae3
+
 # 0.23.0
 
 ### feat: Add canister snapshots

docs/design/asset-canister-interface.md

@@ -267,7 +267,7 @@ Required Permission: [Prepare](#permission-prepare)
  create_chunk: (
  record { 
  batch_id: BatchId;
- content: blob 
+ content: blob
  }
  ) -> (record { 
  chunk_id: ChunkId
@@ -286,6 +286,31 @@ Preconditions:
 
 Required Permission: [Prepare](#permission-prepare)
 
+### Method: `create_chunks`
+
+```candid
+ create_chunks: (
+ record {
+ batch_id: BatchId;
+ content: vec blob;
+ }
+ ) -> (
+ chunk_ids: vec ChunkId;
+ );
+```
+
+This method stores a number of chunks and extends the batch expiry time.
+
+When creating chunks for a given content encoding, the size of each chunk except the last must be the same.
+
+The asset canister must retain all data related to a batch for at least the [Minimum Batch Retention Duration](#constant-minimum-batch-retention-duration) after creating a chunk in a batch.
+
+Preconditions:
+- The batch exists.
+- Creation of the chunk would not exceed chunk creation limits.
+
+Required Permission: [Prepare](#permission-prepare)
+
 ### Method: `commit_batch`
 
 ```candid

src/canisters/frontend/ic-certified-assets/assets.did

@@ -199,6 +199,7 @@ service: (asset_canister_args: opt AssetCanisterArgs) -> {
  create_batch : (record {}) -> (record { batch_id: BatchId });
 
  create_chunk: (record { batch_id: BatchId; content: blob }) -> (record { chunk_id: ChunkId });
+ create_chunks: (record { batch_id: BatchId; content: vec blob }) -> (record { chunk_ids: vec ChunkId });
 
  // Perform all operations successfully, or reject
  commit_batch: (CommitBatchArguments) -> ();

src/canisters/frontend/ic-certified-assets/src/lib.rs

@@ -167,6 +167,15 @@ fn create_chunk(arg: CreateChunkArg) -> CreateChunkResponse {
  })
 }
 
+#[update(guard = "can_prepare")]
+#[candid_method(update)]
+fn create_chunks(arg: CreateChunksArg) -> CreateChunksResponse {
+ STATE.with(|s| match s.borrow_mut().create_chunks(arg, time()) {
+ Ok(chunk_ids) => CreateChunksResponse { chunk_ids },
+ Err(msg) => trap(&msg),
+ })
+}
+
 #[update(guard = "can_commit")]
 #[candid_method(update)]
 fn create_asset(arg: CreateAssetArguments) {

src/canisters/frontend/ic-certified-assets/src/state_machine.rs

@@ -582,43 +582,73 @@ impl State {
  }
 
  pub fn create_chunk(&mut self, arg: CreateChunkArg, now: u64) -> Result<ChunkId, String> {
+ let ids = self.create_chunks_helper(arg.batch_id, vec![arg.content], now)?;
+ ids.into_iter()
+ .next()
+ .ok_or_else(|| "Bug: created chunk did not return a chunk id.".to_string())
+ }
+
+ pub fn create_chunks(
+ &mut self,
+ CreateChunksArg {
+ batch_id,
+ content: chunks,
+ }: CreateChunksArg,
+ now: u64,
+ ) -> Result<Vec<ChunkId>, String> {
+ self.create_chunks_helper(batch_id, chunks, now)
+ }
+
+ /// Post-condition: `chunks.len() == output_chunk_ids.len()`
+ fn create_chunks_helper(
+ &mut self,
+ batch_id: Nat,
+ chunks: Vec<ByteBuf>,
+ now: u64,
+ ) -> Result<Vec<ChunkId>, String> {
  if let Some(max_chunks) = self.configuration.max_chunks {
- if self.chunks.len() + 1 > max_chunks as usize {
+ if self.chunks.len() + chunks.len() > max_chunks as usize {
  return Err("chunk limit exceeded".to_string());
  }
  }
  if let Some(max_bytes) = self.configuration.max_bytes {
  let current_total_bytes = &self.batches.iter().fold(0, |acc, (_batch_id, batch)| {
  acc + batch.chunk_content_total_size
  });
-
- if current_total_bytes + arg.content.as_ref().len() > max_bytes as usize {
+ let new_bytes: usize = chunks.iter().map(|chunk| chunk.len()).sum();
+ if current_total_bytes + new_bytes > max_bytes as usize {
  return Err("byte limit exceeded".to_string());
  }
  }
  let batch = self
  .batches
- .get_mut(&arg.batch_id)
+ .get_mut(&batch_id)
  .ok_or_else(|| "batch not found".to_string())?;
  if batch.commit_batch_arguments.is_some() {
  return Err("batch has been proposed".to_string());
  }
 
  batch.expires_at = Int::from(now + BATCH_EXPIRY_NANOS);
 
- let chunk_id = self.next_chunk_id.clone();
- self.next_chunk_id += 1_u8;
- batch.chunk_content_total_size += arg.content.as_ref().len();
-
- self.chunks.insert(
- chunk_id.clone(),
- Chunk {
- batch_id: arg.batch_id,
- content: RcBytes::from(arg.content),
- },
- );
+ let chunks_len = chunks.len();
+
+ let mut chunk_ids = Vec::with_capacity(chunks.len());
+ for chunk in chunks {
+ let chunk_id = self.next_chunk_id.clone();
+ self.next_chunk_id += 1_u8;
+ batch.chunk_content_total_size += chunk.len();
+ self.chunks.insert(
+ chunk_id.clone(),
+ Chunk {
+ batch_id: batch_id.clone(),
+ content: RcBytes::from(chunk),
+ },
+ );
+ chunk_ids.push(chunk_id);
+ }
 
- Ok(chunk_id)
+ debug_assert!(chunks_len == chunk_ids.len());
+ Ok(chunk_ids)
  }
 
  pub fn commit_batch(&mut self, arg: CommitBatchArguments, now: u64) -> Result<(), String> {

src/canisters/frontend/ic-certified-assets/src/tests.rs

@@ -9,6 +9,7 @@ use crate::types::{
  SetAssetPropertiesArguments,
 };
 use crate::url_decode::{url_decode, UrlDecodeError};
+use crate::CreateChunksArg;
 use candid::{Nat, Principal};
 use ic_certification_testing::CertificateBuilder;
 use ic_crypto_tree_hash::Digest;
@@ -724,14 +725,24 @@ fn cannot_create_chunk_in_proposed_batch_() {
  const BODY: &[u8] = b"<!DOCTYPE html><html></html>";
  match state.create_chunk(
  CreateChunkArg {
- batch_id: batch_1,
+ batch_id: batch_1.clone(),
  content: ByteBuf::from(BODY.to_vec()),
  },
  time_now,
  ) {
  Err(err) if err == *"batch has been proposed" => {}
  other => panic!("expected batch already proposed error, got: {:?}", other),
  }
+ match state.create_chunks(
+ CreateChunksArg {
+ batch_id: batch_1,
+ content: vec![ByteBuf::from(BODY.to_vec())],
+ },
+ time_now,
+ ) {
+ Err(err) if err == *"batch has been proposed" => {}
+ other => panic!("expected batch already proposed error, got: {:?}", other),
+ }
 }
 
 #[test]
@@ -3765,6 +3776,18 @@ mod enforce_limits {
  time_now,
  )
  .unwrap();
+ assert_eq!(
+ state
+ .create_chunks(
+ CreateChunksArg {
+ batch_id: batch_2.clone(),
+ content: vec![ByteBuf::new(), ByteBuf::new()]
+ },
+ time_now
+ )
+ .unwrap_err(),
+ "chunk limit exceeded"
+ );
  state
  .create_chunk(
  CreateChunkArg {
@@ -3818,20 +3841,27 @@ mod enforce_limits {
 
  let batch_1 = state.create_batch(time_now).unwrap();
  let batch_2 = state.create_batch(time_now).unwrap();
+ assert_eq!(
+ state
+ .create_chunks(
+ CreateChunksArg {
+ batch_id: batch_1.clone(),
+ content: vec![
+ ByteBuf::from(c0.clone()),
+ ByteBuf::from(c1.clone()),
+ ByteBuf::from(c2.clone())
+ ]
+ },
+ time_now
+ )
+ .unwrap_err(),
+ "byte limit exceeded"
+ );
  state
- .create_chunk(
- CreateChunkArg {
+ .create_chunks(
+ CreateChunksArg {
  batch_id: batch_1.clone(),
- content: ByteBuf::from(c0),
- },
- time_now,
- )
- .unwrap();
- state
- .create_chunk(
- CreateChunkArg {
- batch_id: batch_2.clone(),
- content: ByteBuf::from(c1),
+ content: vec![ByteBuf::from(c0), ByteBuf::from(c1)],
  },
  time_now,
  )

src/canisters/frontend/ic-certified-assets/src/types.rs

@@ -134,6 +134,17 @@ pub struct CreateChunkResponse {
  pub chunk_id: ChunkId,
 }
 
+#[derive(Clone, Debug, CandidType, Deserialize)]
+pub struct CreateChunksArg {
+ pub batch_id: BatchId,
+ pub content: Vec<ByteBuf>,
+}
+
+#[derive(Clone, Debug, CandidType, Deserialize)]
+pub struct CreateChunksResponse {
+ pub chunk_ids: Vec<ChunkId>,
+}
+
 #[derive(Clone, Debug, CandidType, Deserialize, PartialEq, Eq)]
 pub struct AssetProperties {
  pub max_age: Option<u64>,

src/distributed/assetstorage.did

@@ -199,6 +199,7 @@ service: (asset_canister_args: opt AssetCanisterArgs) -> {
  create_batch : (record {}) -> (record { batch_id: BatchId });
 
  create_chunk: (record { batch_id: BatchId; content: blob }) -> (record { chunk_id: ChunkId });
+ create_chunks: (record { batch_id: BatchId; content: vec blob }) -> (record { chunk_ids: vec ChunkId });
 
  // Perform all operations successfully, or reject
  commit_batch: (CommitBatchArguments) -> ();