Merge branch 'rooch-network:main' into main

.github/workflows/deploy_mainnet.yml

@@ -3,19 +3,59 @@ on:
   workflow_dispatch:
     inputs:
       tagName:
-        description: 'Tag Name to Checkout'
+        description: 'Tag or branch to deploy'
+        default: 'main'
         required: true
-        default: 'latest'
   # workflow_run:
   #   workflows: ["Build Docker And Deploy Seed"]
   #   types:
   #     - completed
 
 jobs:
   deploy-rooch-mainnet:
-    name: deploy rooch mainnet
+    name: Deploy Rooch Mainnet
     runs-on: self-hosted
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      (github.event.workflow_run.conclusion == 'success' &&
+      github.event.workflow_run.event == 'release' &&
+      (github.event.release.prerelease == true || github.event.release.prerelease == false))
     steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.ref }}
+
+      - name: Download Docker tag artifact
+        if: github.event_name == 'workflow_run'
+        uses: actions/download-artifact@v4
+        with:
+          name: docker_tag
+          path: .
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ secrets.GIT_PACKAGE_TOKEN }}
+
+      - name: Determine ref
+        id: determine-ref
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "REF=${{ github.event.inputs.ref }}" >> $GITHUB_ENV
+          else
+            if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
+              echo "REF=$(cat docker_tag.txt)" >> $GITHUB_ENV
+            else
+              if [[ "${{ github.ref }}" == refs/tags/* ]]; then
+                REF=${GITHUB_REF#refs/tags/}
+              elif [[ "${{ github.ref }}" == refs/heads/* ]]; then
+                REF=${GITHUB_REF#refs/heads/}
+              fi
+              echo "REF=$REF" >> $GITHUB_ENV
+            fi
+          fi
+
+      - name: Print ref
+        run: echo "The Tag Ref is ${{ env.REF }} and The Branch Ref is ${{ github.ref }}"
+
       - name: Deploy to GCP MAINNET VM
         env:
           PRIVATE_KEY: ${{ secrets.GCP_MAINNET_SSH_PRIVATE_KEY }}
@@ -26,5 +66,11 @@ jobs:
           chmod 600 private_key.pem
           sudo apt update
           sudo apt install -y --no-install-recommends openssh-server
-          ssh -o StrictHostKeyChecking=no -i private_key.pem $USER@$HOST bash -c "'sleep 30' && && docker image prune -a -f && docker ps | grep rooch | awk '{print \$1}' | xargs -r docker stop && docker ps -a | grep rooch | awk '{print \$1}' | xargs -r docker rm -f && docker pull 'ghcr.io/rooch-network/rooch:${{ github.event.inputs.tagName }}' && docker run -d -v /data:/root -p 6767:6767 'ghcr.io/rooch-network/rooch:${{ github.event.inputs.tagName }}' server start -n main --btc-rpc-url '${{secrets.BTC_MAIN_RPC_URL}}' --btc-rpc-username rooch-main --btc-rpc-password '${{secrets.BTC_MAIN_RPC_PWD}}' --btc-start-block-height 0 --btc-end-block-height 767420 --data-import-mode 1"
-
+          BRANCH=$(basename ${{ github.ref }})
+          ssh -o StrictHostKeyChecking=no -i private_key.pem $USER@$HOST \
+            "cd /root/rooch && git fetch origin && git checkout -B $BRANCH origin/$BRANCH || git checkout $BRANCH && bash scripts/deploy_rooch_mainnet.sh \
+            '${{ env.REF }}' \
+            '${{ secrets.BTC_MAIN_RPC_URL }}' \
+            '${{ secrets.BTC_MAIN_RPC_PWD }}' \
+            '${{ secrets.OPENDA_GCP_MAINNET_BUCKET }}' \
+            '${{ secrets.OPENDA_GCP_MAINNET_CREDENTIAL }}'"

.github/workflows/deploy_testnet.yml

@@ -68,7 +68,7 @@ jobs:
           sudo apt install -y --no-install-recommends openssh-server
           BRANCH=$(basename ${{ github.ref }})
           ssh -o StrictHostKeyChecking=no -i private_key.pem $USER@$HOST \
-            "cd /root/rooch && git fetch origin && git checkout -B $BRANCH origin/$BRANCH || git checkout $BRANCH && bash scripts/deploy_rooch.sh \
+            "cd /root/rooch && git fetch origin && git checkout -B $BRANCH origin/$BRANCH || git checkout $BRANCH && bash scripts/deploy_rooch_testnet.sh \
             '${{ env.REF }}' \
             '${{ secrets.BTC_TEST_RPC_URL }}' \
             '${{ secrets.BTC_TEST_RPC_PWD }}' \

Cargo.toml

@@ -332,6 +332,7 @@ uuid = { version = "1.10.0", features = ["v4", "fast-rng"] }
 protobuf = { version = "2.28", features = ["with-bytes"] }
 redb = { version = "2.1.1" }
 rocksdb = { git = "https://github.com/rooch-network/rust-rocksdb.git", rev = "41d102327ba3cf9a2335d1192e8312c92bc3d6f9", features = ["lz4"] }
+lz4 = { version = "1.27.0" }
 
 ripemd = { version = "0.1.3" }
 fastcrypto-zkp = { version = "0.1.3" }

crates/rooch-da/Cargo.toml

@@ -30,3 +30,4 @@ opendal = { workspace = true }
 rooch-config = { workspace = true }
 serde_yaml = { workspace = true }
 xxhash-rust = { workspace = true, features = ["xxh3"] }
+lz4 = { workspace = true }

crates/rooch-da/docs/stream.md

@@ -0,0 +1,32 @@
+DA Stream
+====
+
+DA Stream is a continuous flow of data from sequencer to verifier. It is a sequence of DA Batch.
+
+## Batch
+
+A batch is a collection of transactions. It is the unit of data flow in DA Stream.
+
+Each batch maps to a L2 block.
+
+## Chunk
+
+A chunk is a collection of DA Batch for better compression ratio.
+
+Components:
+
+- Chunk Header: Version, Chunk Number, Chunk Checksum
+- Chunk Body: One or more DA batch after compression
+
+Version of chunk determines:
+
+1. chunk format: serialization/deserialization, compression algorithm of chunk body
+2. segment format
+
+### Segment
+
+Segment consists of chunk bytes split by a certain size.
+
+Segment is the unit submitted to DA backend, designed to comply with the block size restrictions of the DA backend.
+
+Version of segment inherits from chunk version.

crates/rooch-da/docs/todo.md

@@ -0,0 +1,21 @@
+TODO
+===
+
+## Chunk Builder
+
+Chunk Builder is a component to build chunks from batches, avoiding burst I/O to DA backend.
+
+1. Persist batch into buffer(local persistence layer or other faster media) first, then return ok(if high-performance is
+   preferred).
+2. Split chunk into segments and submit segments to DA backend asynchronously.
+3. Clean up batch buffer after segments being submitted to DA backend schedule.
+
+## DA Proxy
+
+### Get API
+
+1. get by block number
+2. get by batch hash
+3. pull by stream
+
+add FEC for SDC protection (wrong response attacks)

crates/rooch-da/src/actor/da.rs

@@ -36,7 +36,6 @@ impl Actor for DAActor {}
 impl DAActor {
     pub async fn new(da_config: DAConfig, actor_system: &ActorSystem) -> Result<Self> {
         // internal servers
-
         let mut servers: Vec<Arc<dyn DAServerProxy + Send + Sync>> = Vec::new();
         let mut submit_threshold = 1;
         let mut success_count = 0;