From 0eb8e918e96a2d3943b75f8ac0e87ed15f69c9c7 Mon Sep 17 00:00:00 2001
From: minh-bq <97180373+minh-bq@users.noreply.github.com>
Date: Thu, 26 Sep 2024 13:29:14 +0700
Subject: [PATCH] script/benchstat: introduce interleave mode when running
 benchmark (#583)

Currently, all iterations of the benchmark for old commit run before the one for
new commit. This can make the noise not evenly distributed across 2 benchmark
runs (e.g. the machine heats up too much after running all iterations the old
benchmark). This commit introduces interleave mode when running benchmark where
each iteration of benchmark for old commit is followed by an iteration of
benchmark for new one then back to an iteration of benchmark for old one.
---
 .github/workflows/evm-benchmark.yml |  8 ++---
 script/benchstat.sh                 | 53 ++++++++++++++++++++++-------
 2 files changed, 44 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/evm-benchmark.yml b/.github/workflows/evm-benchmark.yml
index 0d8b200e59..b365b98d08 100644
--- a/.github/workflows/evm-benchmark.yml
+++ b/.github/workflows/evm-benchmark.yml
@@ -44,9 +44,9 @@ jobs:
           go install golang.org/x/perf/cmd/benchstat@v0.0.0-20240604174448-3b48cf0e0164
           cd core/vm
           if [[ ${{ github.event_name }} = "pull_request" ]]; then
-            ../../script/benchstat.sh -c "go test -test.v -run=^$ -bench=BenchmarkEvm -count=10" \
-              -o $(git rev-parse --short origin/master) -n ${{ github.sha }}
+            ../../script/benchstat.sh -c "-test.v -test.run=^$ -test.bench=BenchmarkEvm" \
+              -o $(git rev-parse --short origin/master) -n ${{ github.sha }} -c 10 -i
           else
-            ../../script/benchstat.sh -c "go test -test.v -run=^$ -bench=BenchmarkEvm -count=10" \
-              -o "$OLD_COMMIT" -n "$NEW_COMMIT"
+            ../../script/benchstat.sh -c "-test.v -test.run=^$ -test.bench=BenchmarkEvm" \
+              -o "$OLD_COMMIT" -n "$NEW_COMMIT" -c 10 -i
           fi
diff --git a/script/benchstat.sh b/script/benchstat.sh
index 0af77e2ffc..f59adeefa4 100755
--- a/script/benchstat.sh
+++ b/script/benchstat.sh
@@ -3,24 +3,30 @@
 unset COMMAND
 unset OLD_COMMIT
 unset NEW_COMMIT
+unset COUNT
+unset INTERLEAVE
 PROGNAME=$(basename "$0")
-USAGE="$PROGNAME -c command -o old_commit -n new_commit
+USAGE="$PROGNAME -f flags -c count -o old_commit -n new_commit [-i]
 
 where:
-    command:     benchmark command
-    old_commit:  the old commit
-    new_commit:  the new commit
+    -f flags:       benchmark flags (e.g. -test.v -test.run=^$ -test.bench=BenchmarkGetAccount)
+    -o old_commit:  the old commit
+    -n new_commit:  the new commit
+    -c count:       the number of benchmark runs
+    -i:             interleave mode
 "
 
-while getopts :hc:o:n: option; do
+while getopts :hf:o:n:c:i option; do
   case $option in
     h)
       echo "$USAGE"
       exit 0
       ;;
-    c) COMMAND=$OPTARG ;;
+    f) COMMAND=$OPTARG ;;
     o) OLD_COMMIT=$OPTARG ;;
     n) NEW_COMMIT=$OPTARG ;;
+    c) COUNT=$OPTARG ;;
+    i) INTERLEAVE=true ;;
     :)
       echo "$PROGNAME: option requires an argument -- '$OPTARG'" >&2
       exit 1
@@ -34,7 +40,7 @@ while getopts :hc:o:n: option; do
 done
 shift $((OPTIND - 1))
 
-if [[ -z $COMMAND || -z $OLD_COMMIT || -z $NEW_COMMIT ]]; then
+if [[ -z $COMMAND || -z $OLD_COMMIT || -z $NEW_COMMIT || -z $COUNT ]]; then
   echo "$PROGNAME: missing mandatory option" >&2
   echo "Try '$(basename "$0") -h' for more information" >&2
   exit 1
@@ -42,22 +48,43 @@ fi
 
 CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
 if [[ $OLD_COMMIT =~ "HEAD" ]]; then
-	OLD_COMMIT=$(git rev-parse $OLD_COMMIT)
+  OLD_COMMIT=$(git rev-parse $OLD_COMMIT)
 fi
 
 if [[ $NEW_COMMIT =~ "HEAD" ]]; then
-	NEW_COMMIT=$(git rev-parse $NEW_COMMIT)
+  NEW_COMMIT=$(git rev-parse $NEW_COMMIT)
 fi
 
 set -ex
 
-git checkout $OLD_COMMIT
-$COMMAND | tee old.txt
-git checkout $NEW_COMMIT
-$COMMAND | tee new.txt
+if [[ "$INTERLEAVE" = "true" ]]; then
+  git checkout $OLD_COMMIT
+  go test -c -o old.test
+  OLD_COMMAND="./old.test $COMMAND"
+
+  git checkout $NEW_COMMIT
+  go test -c -o new.test
+  NEW_COMMAND="./new.test $COMMAND"
+
+  for i in $(seq 1 $COUNT); do
+    $OLD_COMMAND | tee -a old.txt
+    $NEW_COMMAND | tee -a new.txt
+  done
+
+else
+  COMMAND="go test $COMMAND -test.count=$COUNT"
+  git checkout $OLD_COMMIT
+  $COMMAND | tee old.txt
+  git checkout $NEW_COMMIT
+  $COMMAND | tee new.txt
+fi
 
 benchstat old.txt new.txt
 
 git checkout $CURRENT_BRANCH
 rm old.txt
 rm new.txt
+
+if [[ "$INTERLEAVE" = "true" ]]; then
+  rm old.test new.test
+fi