From 07005a0c73a89e8d47d4657630cb2e5744cabb97 Mon Sep 17 00:00:00 2001 From: David Terry Date: Mon, 4 Oct 2021 20:42:42 +0200 Subject: [PATCH] dapp-mutate This adds a simple mutation testing framework to dapptools, allowing users to gain insight into which parts of their code are not well specified by their test suite. --- overlay.nix | 20 +++++++ src/dapp/default.nix | 5 +- src/dapp/libexec/dapp/dapp | 17 +++++- src/dapp/libexec/dapp/dapp-mutate | 95 +++++++++++++++++++++++++++++++ 4 files changed, 133 insertions(+), 4 deletions(-) create mode 100755 src/dapp/libexec/dapp/dapp-mutate diff --git a/overlay.nix b/overlay.nix index 96af1a376..b12fad0ec 100644 --- a/overlay.nix +++ b/overlay.nix @@ -159,4 +159,24 @@ in rec { secp256k1 = super.secp256k1.overrideDerivation (_: { dontDisableStatic = true; }); + + universalmutator = with super.python3Packages; buildPythonPackage rec { + pname = "universalmutator"; + version = "1.0.12"; + + format = "wheel"; + + propagatedBuildInputs = [ python-Levenshtein setuptools ]; + + src = fetchPypi { + inherit pname version format; + python = "py3"; + sha256 = "sha256-HHJuoqlKPV3aus+fZE+pAzeoiTNPrXO+Put5TyCmAOc="; + }; + + meta = with lib; { + maintainers = with maintainers; [ xwvvvvwx ]; + description = "regex based mutation testing tool"; + }; + }; } diff --git a/src/dapp/default.nix b/src/dapp/default.nix index f01ddabf6..0815add1d 100644 --- a/src/dapp/default.nix +++ b/src/dapp/default.nix @@ -1,6 +1,7 @@ { lib, stdenv, fetchFromGitHub, makeWrapper, glibcLocales , coreutils, git, gnused, gnumake, hevm, jshon, jq, nix -, nodejs, perl, python3, seth, shellcheck, solc, tre, dapptoolsSrc }: +, nodejs, perl, python3, seth, shellcheck, solc, tre +, dapptoolsSrc, parallel, universalmutator }: stdenv.mkDerivation rec { name = "dapp-${version}"; @@ -16,7 +17,7 @@ stdenv.mkDerivation rec { postInstall = let path = lib.makeBinPath [ - coreutils git gnused gnumake hevm jshon jq nix nodejs perl seth solc tre python3 + parallel coreutils git gnused gnumake hevm jshon jq nix nodejs perl seth solc tre python3 universalmutator ]; in '' diff --git a/src/dapp/libexec/dapp/dapp b/src/dapp/libexec/dapp/dapp index e07efbd82..bec3b8c82 100755 --- a/src/dapp/libexec/dapp/dapp +++ b/src/dapp/libexec/dapp/dapp @@ -22,7 +22,7 @@ ### cache= use the cache at directory ### ffi allow the use of the ffi cheatcode (WARNING: allows test authors to execute arbitrary code on your machine) ### coverage print coverage data - +### ### RPC options: ### rpc fetch remote state via ETH_RPC_URL ### rpc-url= fetch remote state via @@ -39,7 +39,7 @@ ### ### Contract verifying options: ### async don't wait for confirmation - +### ### Dapp testnet options: ### rpc-port=port change RPC port (default: 8545) ### rpc-addr=address change RPC address (default: 127.0.0.1) @@ -49,6 +49,11 @@ ### save=name after finishing, save snapshot ### load=name start from a previously saved snapshot ### dir=directory testnet directory +### +### Mutate Options: +### mutants-dir= the directory in which mutants should be stored +### test-command= the command to use for test execution when filtering mutants +### timeout= timeout to use when filtering mutants OPTS="dapp [] [] @@ -97,6 +102,11 @@ accounts=number create multiple accounts (default: 1) save=name after finishing, save snapshot load=name start from a previously saved snapshot dir=directory testnet directory + + Mutate Options: +mutants-dir= the directory in which mutants should be stored +test-command= the command to use for test execution when filtering mutants +timeout= timeout to use when filtering mutants " set -e @@ -177,6 +187,9 @@ while [[ $1 ]]; do --load) shift; export DAPP_TESTNET_LOAD=$1;; --dir) shift; export DAPP_TESTNET_gethdir=$1;; + --mutants-dir) shift; export DAPP_MUTANTS_DIR=$1;; + --test-command) shift; export DAPP_MUTANTS_TEST_COMMAND=$1;; + --timoeut) shift; export DAPP_MUTANTS_TIMEOUT=$1;; *) printf "${0##*/}: internal error: %q\\n" "$1"; exit 1 esac; shift diff --git a/src/dapp/libexec/dapp/dapp-mutate b/src/dapp/libexec/dapp/dapp-mutate new file mode 100755 index 000000000..65b6323ba --- /dev/null +++ b/src/dapp/libexec/dapp/dapp-mutate @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +### dapp mutate -- mutation testing framework +### Usage: dapp mutate [] +### +### dapp mutate provides utilities for mutation testing. Four subcommands are available: +### +### gen: apply mutations and store them in --mutants-dir +### filter: run the project test suite against each mutant, keep the ones where the test suite passes +### show-diffs: display the mutatations that were not detected by the test suite +### status: display the current status of the mutation campaign +### +### To begin a mutation testing campaign, run dapp mutate gen, this will +### generate mutated versions of all non test source files in your project and +### place them in ./mutants. +### Once you have some mutants you can run dapp mutate filter to check if the +### mutations are detected by the test suite. You can run dapp mutate show-diffs +### to display the undetected mutations and dapp mutate status to display the current +### progress of the mutation campgaign. +### +### Options: +### -m, --match only operate on files matching the given regex +### --timeout maximum time given for the test command when filtering +### --test-command the command to use for test execution when filtering mutants +### --mutants-dir the directory in which mutants should be stored + +set -euo pipefail + +export DAPP_TEST_MATCH=${DAPP_TEST_MATCH:-".+[^\.][^t]\.sol$"} +export DAPP_MUTANTS_DIR=${DAPP_MUTANTS_DIR:-mutants} +export DAPP_MUTANTS_TEST_COMMAND=${DAPP_MUTANTS_TEST_COMMAND:-"dapp test"} +export DAPP_MUTANTS_TIMEOUT=${DAPP_MUTANTS_TIMEOUT:-60} +export DAPP_SOLC=${DAPP_SOLC:-"solc"} + +mkdir -p "./$DAPP_MUTANTS_DIR" + +[[ -z $1 ]] && dapp help mutate && exit 0 + +export root +root=$(pwd) + +case "$1" in + "gen") + # shellcheck disable=SC2153 + files=$(find "$root/$DAPP_SRC" -type f -regex "$DAPP_TEST_MATCH") + + # we run in a tmpdir since universalmutator spams tmp files in the cwd... + tmp=$(mktemp -d) + cd "$tmp" || exit + # shellcheck disable=SC2064 + trap "rm -rf $tmp; exit 1" EXIT + + for f in $files; do + mutate \ + --mutantDir "$root/$DAPP_MUTANTS_DIR" \ + --cmd "eval $DAPP_SOLC $(dapp remappings) MUTANT --asm --optimize --allow-paths ." \ + "$f" + done + ;; + + "filter") + files=$(find "$root/$DAPP_SRC" -type f -regex "$DAPP_TEST_MATCH") + + for f in $files; do + analyze_mutants "$f" "$DAPP_MUTANTS_TEST_COMMAND" \ + --mutantDir "./$DAPP_MUTANTS_DIR" \ + --timeout "$DAPP_MUTANTS_TIMEOUT" \ + --prefix mutants \ + --show \ + --resume + done + ;; + + "status") + total=$(find "$DAPP_MUTANTS_DIR" -type f | wc -l) + killed=$(wc -l < mutants.killed.txt) + notkilled=$(wc -l < mutants.notkilled.txt) + tested=$(bc <<< "$killed + $notkilled") + + echo "$total mutants generated, $tested were tested" + echo "$killed mutations were detected by the test suite" + echo "$notkilled mutations were undetected by the test suite" + ;; + + "show-diffs") + mapfile -t notkilled < <(cat mutants.notkilled.txt) + for f in "${notkilled[@]}"; do + echo + src="$DAPP_SRC/$(echo "$f" | sed -e 's|\.mutant\..*\.sol$|\.sol|g')" + echo "$f -> $src" + diff --color "$DAPP_MUTANTS_DIR/$f" "$src" || : + done + ;; + + *) printf "${0##*/}: unrecognised subcommand: %q\\n" "$1"; exit 1 +esac