Extended comparison tool and included HELLO, GEOPOS, GEORADIUS and SI…

…SMEMBER new benchmarks (#266)

* Enabled running forks source built benchmarks

* Fixed server_name Null check

* Enabled passing baseline/comparison hash and github_repo to ensure proper data filtering on compare. Removed refs/heads/ usage from builder

* skipping cli builder test on ci

* Added --baseline-target-branch and --comparison-target-branch to the compare tool

* Added GEOPOS and GEOSEARCH WITHCOORD new benchmarks

* Included the connection setup benchmark using HELLO

* Bumping version from 0.1.218 to 0.1.219

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.218"
+version = "0.1.219"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "Readme.md"

redis_benchmarks_specification/__compare__/args.py

@@ -96,6 +96,9 @@ def create_compare_arguments(parser):
     parser.add_argument(
         "--baseline-target-version", type=str, default=None, required=False
     )
+    parser.add_argument(
+        "--baseline-target-branch", type=str, default=None, required=False
+    )
     parser.add_argument("--comparison-branch", type=str, default=None, required=False)
     parser.add_argument(
         "--baseline-github-repo", type=str, default="redis", required=False
@@ -108,6 +111,9 @@ def create_compare_arguments(parser):
     parser.add_argument(
         "--comparison-target-version", type=str, default=None, required=False
     )
+    parser.add_argument(
+        "--comparison-target-branch", type=str, default=None, required=False
+    )
     parser.add_argument("--print-regressions-only", type=bool, default=False)
     parser.add_argument("--print-improvements-only", type=bool, default=False)
     parser.add_argument("--skip-unstable", type=bool, default=False)

redis_benchmarks_specification/__compare__/compare.py

@@ -203,6 +203,8 @@ def compare_command_logic(args, project_name, project_version):
             )
         )
         baseline_branch = default_baseline_branch
+    if baseline_branch == "":
+        baseline_branch = None
     comparison_branch = args.comparison_branch
     simplify_table = args.simple_table
     print_regressions_only = args.print_regressions_only
@@ -250,6 +252,8 @@ def compare_command_logic(args, project_name, project_version):
     running_platform = args.running_platform
     baseline_target_version = args.baseline_target_version
     comparison_target_version = args.comparison_target_version
+    baseline_target_branch = args.baseline_target_branch
+    comparison_target_branch = args.comparison_target_branch
     baseline_github_repo = args.baseline_github_repo
     comparison_github_repo = args.comparison_github_repo
     baseline_hash = args.baseline_hash
@@ -318,6 +322,8 @@ def compare_command_logic(args, project_name, project_version):
         comparison_hash,
         baseline_github_repo,
         comparison_github_repo,
+        baseline_target_branch,
+        comparison_target_branch,
     )
     prepare_regression_comment(
         auto_approve,
@@ -547,6 +553,8 @@ def compute_regression_table(
     baseline_hash=None,
     baseline_github_repo="redis",
     comparison_github_repo="redis",
+    baseline_target_branch=None,
+    comparison_target_branch=None,
 ):
     START_TIME_NOW_UTC, _, _ = get_start_time_vars()
     START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31)
@@ -574,6 +582,8 @@ def compute_regression_table(
         comparison_target_version,
         comparison_hash,
         baseline_hash,
+        baseline_target_branch,
+        comparison_target_branch,
     )
     logging.info(f"Using baseline filter {by_str_baseline}={baseline_str}")
     logging.info(f"Using comparison filter {by_str_comparison}={comparison_str}")
@@ -739,6 +749,11 @@ def compute_regression_table(
     )
 
 
+def get_by_error(name, by_str_arr):
+    by_string = ",".join(by_str_arr)
+    return f"--{name}-branch, --{name}-tag, --{name}-target-branch, --{name}-hash, and --{name}-target-version are mutually exclusive. You selected a total of {len(by_str_arr)}: {by_string}. Pick one..."
+
+
 def get_by_strings(
     baseline_branch,
     comparison_branch,
@@ -748,57 +763,73 @@ def get_by_strings(
     comparison_target_version=None,
     baseline_hash=None,
     comparison_hash=None,
+    baseline_target_branch=None,
+    comparison_target_branch=None,
 ):
     baseline_covered = False
     comparison_covered = False
     by_str_baseline = ""
     by_str_comparison = ""
     baseline_str = ""
     comparison_str = ""
+    baseline_by_arr = []
+    comparison_by_arr = []
+
+    ################# BASELINE BY ....
+
     if baseline_branch is not None:
-        baseline_covered = True
         by_str_baseline = "branch"
+        baseline_covered = True
         baseline_str = baseline_branch
-    if comparison_branch is not None:
-        comparison_covered = True
-        by_str_comparison = "branch"
-        comparison_str = comparison_branch
+        baseline_by_arr.append(by_str_baseline)
 
     if baseline_tag is not None:
-        if comparison_covered:
-            logging.error(
-                "--baseline-branch and --baseline-tag are mutually exclusive. Pick one..."
-            )
+        by_str_baseline = "version"
+        if baseline_covered:
+            baseline_by_arr.append(by_str_baseline)
+            logging.error(get_by_error("baseline", baseline_by_arr))
             exit(1)
         baseline_covered = True
-        by_str_baseline = "version"
         baseline_str = baseline_tag
 
     if baseline_target_version is not None:
-        if comparison_covered:
-            logging.error(
-                "--baseline-branch, --baseline-tag and --baseline-target-version are mutually exclusive. Pick one..."
-            )
+        by_str_baseline = "target+version"
+        if baseline_covered:
+            baseline_by_arr.append(by_str_baseline)
+            logging.error(get_by_error("baseline", baseline_by_arr))
             exit(1)
         baseline_covered = True
-        by_str_baseline = "target+version"
         baseline_str = baseline_target_version
 
     if baseline_hash is not None:
-        if comparison_covered:
-            logging.error(
-                "--baseline-branch, --baseline-tag, --baseline-hash, and --baseline-target-version are mutually exclusive. Pick one..."
-            )
+        by_str_baseline = "hash"
+        if baseline_covered:
+            baseline_by_arr.append(by_str_baseline)
+            logging.error(get_by_error("baseline", baseline_by_arr))
             exit(1)
         baseline_covered = True
-        by_str_baseline = "hash"
         baseline_str = baseline_hash
+    if baseline_target_branch is not None:
+        by_str_baseline = "target+branch"
+        if baseline_covered:
+            baseline_by_arr.append(by_str_baseline)
+            logging.error(get_by_error("baseline", baseline_by_arr))
+            exit(1)
+        baseline_covered = True
+        baseline_str = baseline_target_branch
+
+    ################# COMPARISON BY ....
+
+    if comparison_branch is not None:
+        by_str_comparison = "branch"
+        comparison_covered = True
+        comparison_str = comparison_branch
 
     if comparison_tag is not None:
         # check if we had already covered comparison
         if comparison_covered:
             logging.error(
-                "--comparison-branch and --comparison-tag are mutually exclusive. Pick one..."
+                "--comparison-branch and --comparison-tag, --comparison-hash, --comparison-target-branch, and --comparison-target-table are mutually exclusive. Pick one..."
             )
             exit(1)
         comparison_covered = True
@@ -808,18 +839,29 @@ def get_by_strings(
         # check if we had already covered comparison
         if comparison_covered:
             logging.error(
-                "--comparison-branch, --comparison-tag, and --comparison-target-table are mutually exclusive. Pick one..."
+                "--comparison-branch, --comparison-tag, --comparison-hash, --comparison-target-branch, and --comparison-target-table are mutually exclusive. Pick one..."
             )
             exit(1)
         comparison_covered = True
         by_str_comparison = "target+version"
         comparison_str = comparison_target_version
 
+    if comparison_target_branch is not None:
+        # check if we had already covered comparison
+        if comparison_covered:
+            logging.error(
+                "--comparison-branch, --comparison-tag, --comparison-hash, --comparison-target-branch, and --comparison-target-table are mutually exclusive. Pick one..."
+            )
+            exit(1)
+        comparison_covered = True
+        by_str_comparison = "target+branch"
+        comparison_str = comparison_target_branch
+
     if comparison_hash is not None:
         # check if we had already covered comparison
         if comparison_covered:
             logging.error(
-                "--comparison-branch, --comparison-tag, --comparison-hash, and --comparison-target-table are mutually exclusive. Pick one..."
+                "--comparison-branch, --comparison-tag, --comparison-hash, --comparison-target-branch, and --comparison-target-table are mutually exclusive. Pick one..."
             )
             exit(1)
         comparison_covered = True
@@ -829,13 +871,13 @@ def get_by_strings(
     if baseline_covered is False:
         logging.error(
             "You need to provider either "
-            + "( --baseline-branch, --baseline-tag, --baseline-hash, or --baseline-target-version ) "
+            + "( --baseline-branch, --baseline-tag, --baseline-hash, --baseline-target-branch or --baseline-target-version ) "
         )
         exit(1)
     if comparison_covered is False:
         logging.error(
             "You need to provider either "
-            + "( --comparison-branch, --comparison-tag, --comparison-hash, or --comparison-target-version ) "
+            + "( --comparison-branch, --comparison-tag, --comparison-hash, --comparison-target-branch or --comparison-target-version ) "
         )
         exit(1)
     return baseline_str, by_str_baseline, comparison_str, by_str_comparison
@@ -1252,9 +1294,9 @@ def get_v_pct_change_and_largest_var(
             if last_n < 0 or (last_n > 0 and len(comparison_values) < last_n):
                 comparison_values.append(tuple[1])
         comparison_df = pd.DataFrame(comparison_values)
-        comparison_median = float(comparison_df.median())
+        comparison_median = float(comparison_df.median().iloc[0])
         comparison_v = comparison_median
-        comparison_std = float(comparison_df.std())
+        comparison_std = float(comparison_df.std().iloc[0])
         if verbose:
             logging.info(
                 "comparison_datapoints: {} value: {}; std-dev: {}; median: {}".format(

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-2-elements-geopos.yml

@@ -0,0 +1,32 @@
+version: 0.4
+name: memtier_benchmark-1key-geo-2-elements-geopos
+description: 'Runs memtier_benchmark, for a keyspace length of 1 GEO key. The GEO key contains 2 elements and comes from the example of https://redis.io/docs/latest/commands/geopos, and we query it using GEOPOS command.'
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  resources:
+    requests:
+      memory: 1g
+  init_commands:
+   - '"GEOADD" "Sicily" "13.361389" "38.115556" "Palermo" "15.087269" "37.502669" "Catania"'
+tested-groups:
+- geo
+tested-commands:
+- geopos
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: -c 50 -t 4 --command="GEOPOS Sicily Palermo Catania"  --hide-histogram --test-time 120
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 138

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-geo-2-elements-geosearch-fromlonlat-withcoord.yml

@@ -0,0 +1,33 @@
+version: 0.4
+name: memtier_benchmark-1key-geo-2-elements-geosearch-fromlonlat-withcoord
+description: 'Runs memtier_benchmark, for a keyspace length of 1 GEO key. The GEO key contains 2 elements and comes from the example of https://redis.io/docs/latest/commands/geosearch, and we query it using GEOSEARCH command.'
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  resources:
+    requests:
+      memory: 1g
+  init_commands:
+   - '"GEOADD" "Sicily" "13.361389" "38.115556" "Palermo" "15.087269" "37.502669" "Catania"'
+   - '"GEOADD" "Sicily" "12.758489" "38.788135" "edge1" "17.241510" "38.788135" "edge2"'
+tested-groups:
+- geo
+tested-commands:
+- geosearch
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: -c 50 -t 4 --command="GEOSEARCH Sicily FROMLONLAT 15 37 BYBOX 400 400 km ASC WITHCOORD WITHDIST"  --hide-histogram --test-time 120
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 138

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sismember-is-a-member.yml

@@ -0,0 +1,32 @@
+version: 0.4
+name: memtier_benchmark-1key-set-100-elements-sismember-is-a-member
+description: 'Runs memtier_benchmark, for a keyspace length of 1 SET key. The SET contains 100 elements in it and we query it using SISMEMBER in which the value is a member. '
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  resources:
+    requests:
+      memory: 1g
+  init_commands:
+   - '"SADD" "set:100" "vyoomgwuzv" "xamjodnbpf" "ewomnmugfa" "ljcgdooafo" "pcxdhdjwnf" "djetcyfxuc" "licotqplim" "alqlzsvuuz" "ijsmoyesvd" "whmotknaff" "rkaznetutk" "ksqpdywgdd" "gorgpnnqwr" "gekntrykfh" "rjkknoigmu" "luemuetmia" "gxephxbdru" "ncjfckgkcl" "hhjclfbbka" "cgoeihlnei" "zwnitejtpg" "upodnpqenn" "mibvtmqxcy" "htvbwmfyic" "rqvryfvlie" "nxcdcaqgit" "gfdqdrondm" "lysbgqqfqw" "nxzsnkmxvi" "nsxaigrnje" "cwaveajmcz" "xsepfhdizi" "owtkxlzaci" "agsdggdghc" "tcjvjofxtd" "kgqrovsxce" "ouuybhtvyb" "ueyrvldzwl" "vpbkvwgxsf" "pytrnqdhvs" "qbiwbqiubb" "ssjqrsluod" "urvgxwbiiz" "ujrxcmpvsq" "mtccjerdon" "xczfmrxrja" "imyizmhzjk" "oguwnmniig" "mxwgdcutnb" "pqyurbvifk" "ccagtnjilc" "mbxohpancs" "lgrkndhekf" "eqlgkwosie" "jxoxtnzujs" "lbtpbknelm" "ichqzmiyot" "mbgehjiauu" "aovfsvbwjg" "nmgxcctxpn" "vyqqkuszzh" "rojeolnopp" "ibhohmfxzt" "qbyhorvill" "nhfnbxqgol" "wkbasfyzqz" "mjjuylgssm" "imdqxmkzdj" "oapbvnisyq" "bqntlsaqjb" "ocrcszcznp" "hhniikmtsx" "hlpdstpvzw" "wqiwdbncmt" "vymjzlzqcn" "hhjchwjlmc" "ypfeltycpy" "qjyeqcfhjj" "uapsgmizgh" "owbbdezgxn" "qrosceblyo" "sahqeskveq" "dapacykoah" "wvcnqbvlnf" "perfwnpvkl" "ulbrotlhze" "fhuvzpxjbc" "holjcdpijr" "onzjrteqmu" "pquewclxuy" "vpmpffdoqz" "eouliovvra" "vxcbagyymm" "jekkafodvk" "ypekeuutef" "dlbqcynhrn" "erxulvebrj" "qwxrsgafzy" "dlsjwmqzhx" "exvhmqxvvp"'
+tested-groups:
+- set
+tested-commands:
+- sismember
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --command="SISMEMBER set:100 lysbgqqfqw"  --hide-histogram --test-time 180
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 1

redis_benchmarks_specification/test-suites/memtier_benchmark-1key-set-100-elements-sismember-not-a-member.yml

@@ -0,0 +1,32 @@
+version: 0.4
+name: memtier_benchmark-1key-set-100-elements-sismember-not-a-member
+description: 'Runs memtier_benchmark, for a keyspace length of 1 SET key. The SET contains 100 elements in it and we query it using SISMEMBER in which the value is not a member. '
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1
+  resources:
+    requests:
+      memory: 1g
+  init_commands:
+   - '"SADD" "set:100" "vyoomgwuzv" "xamjodnbpf" "ewomnmugfa" "ljcgdooafo" "pcxdhdjwnf" "djetcyfxuc" "licotqplim" "alqlzsvuuz" "ijsmoyesvd" "whmotknaff" "rkaznetutk" "ksqpdywgdd" "gorgpnnqwr" "gekntrykfh" "rjkknoigmu" "luemuetmia" "gxephxbdru" "ncjfckgkcl" "hhjclfbbka" "cgoeihlnei" "zwnitejtpg" "upodnpqenn" "mibvtmqxcy" "htvbwmfyic" "rqvryfvlie" "nxcdcaqgit" "gfdqdrondm" "lysbgqqfqw" "nxzsnkmxvi" "nsxaigrnje" "cwaveajmcz" "xsepfhdizi" "owtkxlzaci" "agsdggdghc" "tcjvjofxtd" "kgqrovsxce" "ouuybhtvyb" "ueyrvldzwl" "vpbkvwgxsf" "pytrnqdhvs" "qbiwbqiubb" "ssjqrsluod" "urvgxwbiiz" "ujrxcmpvsq" "mtccjerdon" "xczfmrxrja" "imyizmhzjk" "oguwnmniig" "mxwgdcutnb" "pqyurbvifk" "ccagtnjilc" "mbxohpancs" "lgrkndhekf" "eqlgkwosie" "jxoxtnzujs" "lbtpbknelm" "ichqzmiyot" "mbgehjiauu" "aovfsvbwjg" "nmgxcctxpn" "vyqqkuszzh" "rojeolnopp" "ibhohmfxzt" "qbyhorvill" "nhfnbxqgol" "wkbasfyzqz" "mjjuylgssm" "imdqxmkzdj" "oapbvnisyq" "bqntlsaqjb" "ocrcszcznp" "hhniikmtsx" "hlpdstpvzw" "wqiwdbncmt" "vymjzlzqcn" "hhjchwjlmc" "ypfeltycpy" "qjyeqcfhjj" "uapsgmizgh" "owbbdezgxn" "qrosceblyo" "sahqeskveq" "dapacykoah" "wvcnqbvlnf" "perfwnpvkl" "ulbrotlhze" "fhuvzpxjbc" "holjcdpijr" "onzjrteqmu" "pquewclxuy" "vpmpffdoqz" "eouliovvra" "vxcbagyymm" "jekkafodvk" "ypekeuutef" "dlbqcynhrn" "erxulvebrj" "qwxrsgafzy" "dlsjwmqzhx" "exvhmqxvvp"'
+tested-groups:
+- set
+tested-commands:
+- sismember
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --command="SISMEMBER set:100 not-a-member"  --hide-histogram --test-time 180
+  resources:
+    requests:
+      cpus: '4'
+      memory: 2g
+
+priority: 1