From 340a69ed72920aba264510a5a06ea628bf273166 Mon Sep 17 00:00:00 2001 From: sveitser Date: Tue, 29 Oct 2024 12:55:37 +0100 Subject: [PATCH 1/2] WIP: basic mutation testing with cargo mutants - Update nixpkgs to get latest cargo-mutants. - Remove nixWithFlakes from flake.nix because it would have to be changed to work with the latest version of nixpkgs and I think we no longer need it. - Create a non-flaky test profile. - Run with `cargo mutants`. - Inspect output and `./mutants.out` directory For example `./mutants.out/missed.txt` looks like this contract-bindings/src/plonk_verifier_2.rs:463:13: replace plonk_verifier_2::::encode -> Vec with vec![] hotshot-state-prover/src/mock_ledger.rs:496:57: replace + with * in gen_circuit_for_test hotshot-state-prover/src/mock_ledger.rs:496:53: replace * with / in gen_circuit_for_test hotshot-state-prover/src/mock_ledger.rs:149:9: replace MockLedger::elapse_without_block with () contract-bindings/src/light_client_mock.rs:1928:21: replace == with != in light_client_mock::::valid_selector contract-bindings/src/erc1967_proxy.rs:319:13: replace erc1967_proxy::::encode -> ::std::vec::Vec with vec![0] hotshot-state-prover/src/circuit.rs:236:29: replace % with / in build To make it really useful further configuration is likely necessary, such as marking some functions to not mutate them. --- .config/nextest.toml | 27 +++++++++++++++++++++++++-- Cargo.toml | 10 ++++++++++ flake.lock | 8 ++++---- flake.nix | 11 +++-------- 4 files changed, 42 insertions(+), 14 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index af8b6a75a..2701e8815 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -1,7 +1,15 @@ +# cargo-mutants does not seem to support specifying the nextest profile? :( [profile.default] slow-timeout = "2m" -default-filter = 'not (test(slow_) | package(tests))' -retries = 2 +default-filter = """not (test(slow_) | package(tests) \ + | test(test_fee_upgrade_time_based) \ + | test(test_fee_upgrade_view_based) \ + | test(test_marketplace_upgrade_time_based) \ + | test(test_marketplace_upgrade_view_based) \ + | test(test_process_client_handling_stream_subscribe_voters) \ + | test(test_process_client_handling_stream_subscribe_node_identity) \ + )""" +retries = 0 # The restart tests run an entire sequencing network, and so are quite resource intensive. [[profile.default.overrides]] @@ -22,3 +30,18 @@ retries = 2 slow-timeout = "2m" default-filter = 'package(tests)' retries = 2 + +# # Profile to be used with mutant testing. +# # The upgrade tests are slow. +# # The process_client_handling_stream_subscribe_... tests may hang. +# [profile.fast-non-flaky] +# slow-timeout = "2m" +# default-filter = """not (test(slow_) | package(tests) \ +# | test(test_fee_upgrade_time_based) \ +# | test(test_fee_upgrade_view_based) \ +# | test(test_marketplace_upgrade_time_based) \ +# | test(test_marketplace_upgrade_view_based) \ +# | test(test_process_client_handling_stream_subscribe_voters) \ +# | test(test_process_client_handling_stream_subscribe_node_identity) \ +# )""" +# retries = 0 diff --git a/Cargo.toml b/Cargo.toml index fa5ed74e1..d5d477ad6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -156,6 +156,16 @@ strip = "debuginfo" [profile.dev.package."*"] opt-level = 3 +[profile.test] +inherits = "dev" +# Minimal opt-level to run api::test::test_{catchup,restart} +opt-level = 1 + +# [profile.mutants] +# inherits = "dev" +# debug = "none" + + [patch."https://github.com/EspressoSystems/HotShot.git"] hotshot = { git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.78-patch5" } hotshot-builder-api = { git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.78-patch5" } diff --git a/flake.lock b/flake.lock index ad381da77..b1858ae67 100644 --- a/flake.lock +++ b/flake.lock @@ -264,16 +264,16 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1726463316, - "narHash": "sha256-gI9kkaH0ZjakJOKrdjaI/VbaMEo9qBbSUl93DnU7f4c=", + "lastModified": 1729980323, + "narHash": "sha256-eWPRZAlhf446bKSmzw6x7RWEE4IuZgAp8NW3eXZwRAY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "99dc8785f6a0adac95f5e2ab05cc2e1bf666d172", + "rev": "86e78d3d2084ff87688da662cf78c2af085d8e73", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-unstable", + "ref": "nixpkgs-unstable", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 6b4a7e467..b4065282e 100644 --- a/flake.nix +++ b/flake.nix @@ -12,7 +12,7 @@ ]; }; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; inputs.rust-overlay.url = "github:oxalica/rust-overlay"; inputs.nixpkgs-cross-overlay.url = @@ -175,11 +175,6 @@ nightlyToolchain = pkgs.rust-bin.selectLatestNightlyWith (toolchain: toolchain.minimal.override { extensions = [ "rust-analyzer" ]; }); - # nixWithFlakes allows pre v2.4 nix installations to use - # flake commands (like `nix flake update`) - nixWithFlakes = pkgs.writeShellScriptBin "nix" '' - exec ${pkgs.nixFlakes}/bin/nix --experimental-features "nix-command flakes" "$@" - ''; solc = pkgs.solc-bin.latest; in mkShell (rustEnvVars // { @@ -195,14 +190,14 @@ # Rust tools cargo-audit cargo-edit - cargo-sort + cargo-mutants cargo-nextest + cargo-sort typos just nightlyToolchain.passthru.availableComponents.rust-analyzer # Tools - nixWithFlakes nixpkgs-fmt entr process-compose From 3e9ccb772beacfd1e2077a8eecdae03597d09c2b Mon Sep 17 00:00:00 2001 From: sveitser Date: Tue, 29 Oct 2024 13:05:47 +0100 Subject: [PATCH 2/2] Add missing file --- .cargo/mutants.toml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .cargo/mutants.toml diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml new file mode 100644 index 000000000..d04de6bcc --- /dev/null +++ b/.cargo/mutants.toml @@ -0,0 +1,4 @@ +# cargo-mutants configuration + +# profile = "mutants" # Build without debug symbols +test_tool = "nextest"