ci: build all packages

.github/workflows/build.yml

@@ -32,6 +32,8 @@ jobs:
           done
       - name: Check formatting
         run: nix fmt -- --fail-on-change
+      - name: Check all packages
+        run: nix flake -L check
       - name: Build default
         run: nix --log-format raw -L build '.#deku'
       - name: Build static

deku-c/client/src/deku-c/contract.ts

@@ -37,6 +37,10 @@ const parseContractState = (json: JSONType): JSONType => {
       const first = json[1] as Array<JSONType>;
       return first.map((json) => parseContractState(json));
     }
+    case "Set": {
+      const first = json[1] as Array<JSONType>;
+      return first.map((json) => parseContractState(json));
+    }
     case "Union": {
       const first = json[1] as Array<JSONType>;
       const type = first[0] as string;

deku-c/deku-cli/commands/generate-identity.ts

@@ -16,7 +16,7 @@ export default function make(command: Commander.Command) {
           }
         } catch (e: any) {
           console.error("An error occurred:");
-          console.error(e.message);
+          console.error(e);
           process.exit(1);
         }
       });

deku-c/deku-cli/commands/invoke.ts

@@ -76,7 +76,8 @@ async function invokeLigoMain(
     console.log("Operation hash:", hash);
   } catch (e: any) {
     console.error("An error occurred:");
-    console.error(e.message);
+    // TODO: this error is totally opaque if the expression is malformed.
+    console.error(e);
     process.exit(1);
   }
 }

deku-c/deku-cli/commands/show-entrypoints.ts

@@ -19,7 +19,7 @@ async function main(
     }
   } catch (e: any) {
     console.error("An error occurred:");
-    console.error(e.message);
+    console.error(e);
     process.exit(1);
   }
 }

deku-c/deku-cli/commands/show-storage.ts

@@ -24,7 +24,7 @@ async function main(
     console.log(state);
   } catch (e: any) {
     console.error("An error occurred:");
-    console.error(e.message);
+    console.error(e);
     process.exit(1);
   }
 }

deku-p/src/core/bin/api/deku_api.ml

@@ -36,7 +36,7 @@ let make_dump_loop ~sw ~env ~data_folder =
 
 let save_block ~sw ~indexer ~block =
   let on_error err =
-    Format.eprintf "save_block.error: %s\n%!" (Printexc.to_string err)
+    Logs.err (fun m -> m "save_block.error: %s\n%!" (Printexc.to_string err))
   in
   (* TODO: better logging *)
   Eio.Fiber.fork_sub ~sw ~on_error @@ fun _sw ->

deku-p/src/core/bin/api/handlers.ml

@@ -64,9 +64,9 @@ module Get_block_by_level_or_hash : NO_BODY_HANDLERS = struct
   open Api_path
 
   type path = Level_or_hash.t
-  type response = Data_encoding.Json.t
+  type response = Block.t
 
-  let response_encoding = Data_encoding.Json.encoding
+  let response_encoding = Block.encoding
   let meth = `GET
 
   let path =

deku-p/src/core/bin/node/node.ml

@@ -69,8 +69,7 @@ and handle_chain_action ~sw ~env ~action node =
       | Some indexer ->
           Eio.Fiber.fork_sub ~sw ~on_error @@ fun _sw ->
           let (Block { level; _ }) = block in
-          Block_storage.save_block_and_votes ~level ~network indexer;
-          Block_storage.save_block ~block indexer
+          Block_storage.save_block_and_votes ~level ~network indexer
       | None -> ())
   | Chain_send_blocks { connection; above } ->
       send_blocks ~sw ~connection ~above node

deku-p/src/core/block_storage/block_storage.mli

@@ -15,11 +15,8 @@ val save_block_and_votes :
 val save_message : message:Message.Network.t -> storage -> unit
 
 (* TODO: optimize store and load Message.Network.t *)
-val find_block_by_level :
-  level:Level.t -> storage -> Data_encoding.Json.t option
-
-val find_block_by_hash :
-  block_hash:Block_hash.t -> storage -> Data_encoding.Json.t option
+val find_block_by_level : level:Level.t -> storage -> Block.t option
+val find_block_by_hash : block_hash:Block_hash.t -> storage -> Block.t option
 
 val find_block_and_votes_by_level :
   level:Level.t -> storage -> Message.Network.t option

deku-p/src/core/block_storage/query.ml

@@ -1,4 +1,3 @@
-open Deku_stdlib
 open Deku_consensus
 open Deku_gossip
 
@@ -71,10 +70,6 @@ let insert_message =
 
 let insert_block ~block ~timestamp pool =
   let (Block.Block { hash = block_hash; level; _ }) = block in
-  let block =
-    Parallel.parallel @@ fun () ->
-    Data_encoding.Json.construct Block.encoding block
-  in
   Caqti_eio.Pool.use (insert_block ~block_hash ~level ~block ~timestamp) pool
 
 let insert_block_and_votes ~level ~network ~timestamp pool =

deku-p/src/core/block_storage/query.mli

@@ -26,11 +26,10 @@ val insert_message :
   pool ->
   unit result_promise
 
-val find_block_by_level :
-  level:Level.t -> pool -> Data_encoding.Json.t option result_promise
+val find_block_by_level : level:Level.t -> pool -> Block.t option result_promise
 
 val find_block_by_hash :
-  hash:Block_hash.t -> pool -> Data_encoding.Json.t option result_promise
+  hash:Block_hash.t -> pool -> Block.t option result_promise
 
 val find_block_and_votes_by_level :
   level:Level.t -> pool -> Message.Network.t option result_promise

deku-p/src/core/block_storage/types.ml

@@ -41,25 +41,31 @@ module Timestamp : Rapper.CUSTOM with type t = Timestamp.t = struct
     Caqti_type.(custom ~encode ~decode float)
 end
 
-module Block : Rapper.CUSTOM with type t = Data_encoding.Json.t = struct
-  type t = Data_encoding.Json.t
+module Block : Rapper.CUSTOM with type t = Block.t = struct
+  type t = Block.t
 
   let t =
     let encode block =
-      block |> Data_encoding.Json.to_string |> Ezgzip.compress |> Result.ok
+      (* TODO: in this commit we moved block storage onto the main thread,
+         slowing us down considerably. We should explore moving it back off
+         main thread. *)
+      match Data_encoding.Binary.to_string Block.encoding block with
+      | Ok binary -> Ok (Ezgzip.compress binary)
+      | Error error ->
+          Error
+            (Format.asprintf "write error: %a"
+               Data_encoding.Binary.pp_write_error error)
     in
-    let decode json =
-      try
-        let maybe_string =
-          json |> Ezgzip.decompress |> Result.map Data_encoding.Json.from_string
-        in
-        match maybe_string with
-        | Ok value -> value
-        | _ -> failwith "cannot decompress block"
-      with exn ->
-        Error
-          (Format.sprintf "cannot decode block from the database: %s"
-             (Printexc.to_string exn))
+    let decode compressed =
+      match Ezgzip.decompress compressed with
+      | Ok binary -> (
+          match Data_encoding.Binary.of_string Block.encoding binary with
+          | Ok network -> Ok network
+          | Error error ->
+              Error
+                (Format.asprintf "%a" Data_encoding.Binary.pp_read_error error))
+      | Error (`Gzip error) ->
+          Error (Format.asprintf "%a" Ezgzip.pp_error error)
     in
     Caqti_type.(custom ~encode ~decode string)
 end

deku-p/src/core/chain/chain.ml

@@ -111,6 +111,12 @@ let commit ~current_level ~block ~votes ~validators =
       withdrawal_handles_hash;
     }
 
+(* TODO: move this to a transparently configurable thing in Cmdliner in Deku_node *)
+let minimum_block_latency =
+  match Sys.getenv_opt "DEKU_MINIMUM_BLOCK_LATENCY" with
+  | Some x -> Option.value ~default:0.0 (Float.of_string_opt x)
+  | None -> 0.0
+
 (* after gossip *)
 let apply_consensus_action chain consensus_action =
   let open Consensus in
@@ -128,6 +134,9 @@ let apply_consensus_action chain consensus_action =
       let fragment =
         Fragment_produce { producer; above; withdrawal_handles_hash }
       in
+      (match minimum_block_latency with
+      | 0. -> ()
+      | minimum_block_latency -> Unix.sleepf minimum_block_latency);
       (chain, [ Chain_fragment { fragment } ])
   | Consensus_vote { level; vote } ->
       let content = Message.Content.vote ~level ~vote in
@@ -191,7 +200,22 @@ let incoming_vote ~current ~level ~vote chain =
   apply_consensus_actions chain actions
 
 let incoming_operation ~operation chain =
-  Logs.info (fun m -> m "Incoming operation: %a" Operation.Signed.pp operation);
+  let () =
+    let open Operation in
+    let (Signed.Signed_operation
+          { initial = Initial.Initial_operation { operation; hash; _ }; _ }) =
+      operation
+    in
+    let hash = Operation_hash.to_b58 hash in
+    match operation with
+    | Operation_ticket_transfer _ ->
+        Logs.info (fun m -> m "Incoming ticket transfer: %s" hash)
+    | Operation_vm_transaction _ ->
+        Logs.info (fun m -> m "Incoming vm transaction: %s" hash)
+    | Operation_withdraw _ ->
+        Logs.info (fun m -> m "Incoming vm withdraw: %s" hash)
+    | Operation_noop _ -> Logs.info (fun m -> m "Incoming noop: %s" hash)
+  in
   let (Chain ({ producer; _ } as chain)) = chain in
   let producer = Producer.incoming_operation ~operation producer in
   let chain = Chain { chain with producer } in

deku-p/src/core/gossip/message.ml

@@ -15,16 +15,10 @@ module Header = struct
       (tup2 Deku_crypto.BLAKE2b.encoding Level.encoding)
 
   let make ~hash ~level = Message_header { hash; level }
-
-  let encode header =
-    let json = Data_encoding.Json.construct encoding header in
-    Data_encoding.Json.to_string json
+  let encode header = Data_encoding.Binary.to_string_exn encoding header
 
   let decode ~raw_header =
-    let json = Data_encoding.Json.from_string raw_header in
-    match json with
-    | Ok json -> Data_encoding.Json.destruct encoding json
-    | _ -> failwith "impossible to decode"
+    Data_encoding.Binary.of_string_exn encoding raw_header
 end
 
 module Content = struct

docs/Deku-Canonical/deku_c_cli.md

@@ -104,7 +104,7 @@ the contract and the raw code of the contract.
 The `invoke` command allows to call an entrypoint with a specific parameter. Usage:
 
 ```bash
-deku-cli wallet contract_address parameter
+deku-cli invoke wallet contract_address parameter
 ```
 
 where
@@ -118,7 +118,7 @@ The `parameter` can be provided by the Ligo compiler using the `ligo compile par
 To invoke a contract with a Ligo expression directly, use the `invoke-ligo` command:
 
 ```bash
-deku-cli wallet contract_address contract_path expression
+deku-cli invoke wallet contract_address contract_path expression
 ```
 
 where

examples/deku-p-react/package.json

@@ -10,7 +10,7 @@
     "@taquito/taquito": "^13.0.1",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
-    "react-scripts": "^2.1.3",
+    "react-scripts": "5.0.1",
     "web-vitals": "^2.1.4"
   },
   "devDependencies": {
@@ -26,11 +26,13 @@
     "ts-node": "^10.9.1",
     "tsconfig-paths": "^4.0.0",
     "tsconfig-paths-jest": "^0.0.1",
-    "typescript": "^4.7.4"
+    "typescript": "^4.7.4",
+    "os-browserify": "^0.3.0",
+    "stream-browserify": "^3.0.0"
   },
   "scripts": {
-    "start": "set port=9090 && react-scripts --openssl-legacy-provider start",
-    "build": "react-scripts --openssl-legacy-provider build"
+    "start": "react-app-rewired start",
+    "build": "react-app-rewired build"
   },
   "browserslist": {
     "production": [

examples/number-go-up/src/App.tsx

@@ -57,8 +57,7 @@ export const App = () => {
           "https://deku-canonical-vm0.deku-v1.marigold.dev/api/v1/state/unix"
         );
         const data = await result.json();
-        const storage = JSON.parse(data[contractAddr]);
-        const counterState = storage.state[1];
+        const counterState = data[contractAddr].state[1];
         console.log("setting state:", counterState);
         setCounterState(counterState);
       })();

examples/number-go-up/tsconfig.json

@@ -14,7 +14,7 @@
     "resolveJsonModule": true,
     "isolatedModules": true,
     "noEmit": true,
-    "jsx": "react-jsx"
+    "jsx": "preserve"
   },
   "include": ["src"]
 }

flake.nix

@@ -84,6 +84,7 @@
             tuna = self'.packages.tuna;
             deploy-rs = deploy-rs.packages.${system}.default;
           };
+          checks = self.outputs.packages.${system};
           formatter = treefmt.legacyPackages.${system}.withConfig {
             settings = with pkgs; {
               excludes = [

nix/deku-p/default.nix

@@ -39,7 +39,11 @@
       static = true;
     };
 
-    ligo = inputs.ligo.packages.${system}.ligoLight;
+    ligo =
+      if system == "x86_64-linux"
+      then inputs.ligo.packages.${system}.ligoLight
+      # TODO: fix ligo for other systems
+      else pkgs.hello;
 
     docker = pkgs.callPackage ./docker.nix {inherit deku ligo;};
   in {

scripts/reset.sh

@@ -11,6 +11,7 @@ for N in 0 1 2 3; do
     rm -rf "./flextesa_chain/data/$N/api_vm_pipe_read"
     rm -rf "./flextesa_chain/data/$N/api_vm_pipe_write"
     rm -rf "./flextesa_chain/data/$N/deku_api.json"
+    rm -rf "./flextesa_chain/data/$N/deku_api.tmp.json"
 done
 
 rm -rf /tmp/api_database.db