benchmarks: import jetscii's benchmarks

There was some discussion about how to compare jetscii with Teddy and
some interesting benchmark results[1]. I decided to import the
benchmarks and see what things look like here.

[1]: shepmaster/jetscii#57

.vim/coc-settings.json

@@ -3,6 +3,7 @@
     "aho-corasick-debug/Cargo.toml",
     "benchmarks/engines/rust-aho-corasick/Cargo.toml",
     "benchmarks/engines/rust-daachorse/Cargo.toml",
+    "benchmarks/engines/rust-jetscii/Cargo.toml",
     "benchmarks/engines/naive/Cargo.toml",
     "benchmarks/shared/Cargo.toml",
     "fuzz/Cargo.toml",

benchmarks/definitions/jetscii.toml

@@ -0,0 +1,136 @@
+analysis = '''
+These benchmarks were ported out of the jetscii crate, specifically from
+[Dr-Emann's PR]. There were some irregularities in the benchmark results, so
+I thought it might be interesting to include it here.
+
+We add "real" variants of each benchmark as well using a small XML data set on
+mental health. The original benchmarks search a haystack consisting entirely of
+`a` repeated, with the last byte corresponding to one of the needle bytes. This
+is useful for measuring pure throughput, but less good for approximating real
+world performance. In this case, for at least `xml-delim3` and `xml-delim5`, it
+seems like an XML haystack would be better suited.
+
+[Dr-Emann's PR]: https://github.com/shepmaster/jetscii/pull/57
+'''
+
+[[bench]]
+model = "count"
+name = "space-repeateda"
+regex = [' ']
+haystack = { contents = "a", repeat = 5_242_880, append = " " }
+count = 1
+engines = [
+  "daachorse/bytewise/leftmost-first",
+  "rust/aho-corasick/dfa/leftmost-first",
+  "rust/aho-corasick/packed/leftmost-first",
+  "rust/old-aho-corasick/packed/leftmost-first",
+  "rust/jetscii/ascii-chars/prebuilt",
+]
+
+[[bench]]
+model = "count"
+name = "xmldelim3-repeateda"
+regex = ['<', '>', '&']
+haystack = { contents = "a", repeat = 5_242_880, append = "&" }
+count = 1
+engines = [
+  "daachorse/bytewise/leftmost-first",
+  "rust/aho-corasick/dfa/leftmost-first",
+  "rust/aho-corasick/packed/leftmost-first",
+  "rust/old-aho-corasick/packed/leftmost-first",
+  "rust/jetscii/ascii-chars/prebuilt",
+]
+
+[[bench]]
+model = "count"
+name = "xmldelim5-repeateda"
+regex = ['<', '>', '&', "'", '"']
+haystack = { contents = "a", repeat = 5_242_880, append = '"' }
+count = 1
+engines = [
+  "daachorse/bytewise/leftmost-first",
+  "rust/aho-corasick/dfa/leftmost-first",
+  "rust/aho-corasick/packed/leftmost-first",
+  "rust/old-aho-corasick/packed/leftmost-first",
+  "rust/jetscii/ascii-chars/prebuilt",
+]
+
+[[bench]]
+model = "count"
+name = "big16-repeateda"
+regex = [
+  'A', 'B', 'C', 'D',
+  'E', 'F', 'G', 'H',
+  'I', 'J', 'K', 'L',
+  'M', 'N', 'O', 'P',
+]
+haystack = { contents = "a", repeat = 5_242_880, append = "P" }
+count = 1
+engines = [
+  "daachorse/bytewise/leftmost-first",
+  "rust/aho-corasick/dfa/leftmost-first",
+  "rust/aho-corasick/packed/leftmost-first",
+  "rust/old-aho-corasick/packed/leftmost-first",
+  "rust/jetscii/ascii-chars/prebuilt",
+]
+
+[[bench]]
+model = "count"
+name = "space-mentalhealth"
+regex = [' ']
+haystack = { path = "catalog.data.gov/mental-health-4weeks.xml" }
+count = 1_181_201
+engines = [
+  "daachorse/bytewise/leftmost-first",
+  "rust/aho-corasick/dfa/leftmost-first",
+  "rust/aho-corasick/packed/leftmost-first",
+  "rust/old-aho-corasick/packed/leftmost-first",
+  "rust/jetscii/ascii-chars/prebuilt",
+]
+
+[[bench]]
+model = "count"
+name = "xmldelim3-mentalhealth"
+regex = ['<', '>', '&']
+haystack = { path = "catalog.data.gov/mental-health-4weeks.xml" }
+count = 604_714
+engines = [
+  "daachorse/bytewise/leftmost-first",
+  "rust/aho-corasick/dfa/leftmost-first",
+  "rust/aho-corasick/packed/leftmost-first",
+  "rust/old-aho-corasick/packed/leftmost-first",
+  "rust/jetscii/ascii-chars/prebuilt",
+]
+
+[[bench]]
+model = "count"
+name = "xmldelim5-mentalhealth"
+regex = ['<', '>', '&', "'", '"']
+haystack = { path = "catalog.data.gov/mental-health-4weeks.xml" }
+count = 688_252
+engines = [
+  "daachorse/bytewise/leftmost-first",
+  "rust/aho-corasick/dfa/leftmost-first",
+  "rust/aho-corasick/packed/leftmost-first",
+  "rust/old-aho-corasick/packed/leftmost-first",
+  "rust/jetscii/ascii-chars/prebuilt",
+]
+
+[[bench]]
+model = "count"
+name = "big16-mentalhealth"
+regex = [
+  'A', 'B', 'C', 'D',
+  'E', 'F', 'G', 'H',
+  'I', 'J', 'K', 'L',
+  'M', 'N', 'O', 'P',
+]
+haystack = { path = "catalog.data.gov/mental-health-4weeks.xml" }
+count = 176_447
+engines = [
+  "daachorse/bytewise/leftmost-first",
+  "rust/aho-corasick/dfa/leftmost-first",
+  "rust/aho-corasick/packed/leftmost-first",
+  "rust/old-aho-corasick/packed/leftmost-first",
+  "rust/jetscii/ascii-chars/prebuilt",
+]

benchmarks/engines.toml

@@ -317,6 +317,42 @@
     bin = "cargo"
     args = ["clean"]
 
+# Engines based on the `jetscii` Rust crate. This is somewhat more appropriately
+# compared with routines in `memchr`, but there is some overlap in use cases
+# with Teddy's packed searcher for multiple single-byte needles.
+
+[[engine]]
+  name = "rust/jetscii/ascii-chars/prebuilt"
+  cwd = "./engines/rust-jetscii"
+  [engine.version]
+    bin = "./target/release/main"
+    args = ["--version"]
+  [engine.run]
+    bin = "./target/release/main"
+    args = ["ascii-chars-prebuilt"]
+  [[engine.build]]
+    bin = "cargo"
+    args = ["build", "--release"]
+  [[engine.clean]]
+    bin = "cargo"
+    args = ["clean"]
+
+[[engine]]
+  name = "rust/jetscii/ascii-chars/oneshot"
+  cwd = "./engines/rust-jetscii"
+  [engine.version]
+    bin = "./target/release/main"
+    args = ["--version"]
+  [engine.run]
+    bin = "./target/release/main"
+    args = ["ascii-chars-oneshot"]
+  [[engine.build]]
+    bin = "cargo"
+    args = ["build", "--release"]
+  [[engine.clean]]
+    bin = "cargo"
+    args = ["clean"]
+
 # Naive engines. Useful for comparisons and to determine the crossover point
 # where a multi-substring algorithm is beneficial over multi single-substring
 # algorithms. We include both the `memchr` crate and `std`.

benchmarks/engines/naive/Cargo.toml

@@ -18,4 +18,4 @@ path = "../../shared"
 [profile.release]
 debug = true
 codegen-units = 1
-lto = "thin"
+lto = "fat"

benchmarks/engines/rust-aho-corasick/Cargo.toml

@@ -21,4 +21,4 @@ path = "../../shared"
 [profile.release]
 debug = true
 codegen-units = 1
-lto = "thin"
+lto = "fat"

benchmarks/engines/rust-daachorse/Cargo.toml

@@ -18,4 +18,4 @@ path = "../../shared"
 [profile.release]
 debug = true
 codegen-units = 1
-lto = "thin"
+lto = "fat"

benchmarks/engines/rust-jetscii/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+publish = false
+name = "main"
+version = "0.5.3"
+edition = "2021"
+
+[workspace]
+
+[dependencies]
+anyhow = "1.0.72"
+jetscii = "=0.5.3"
+
+[dependencies.shared]
+path = "../../shared"
+
+[[bin]]
+name = "main"
+path = "main.rs"
+
+[profile.release]
+debug = true
+codegen-units = 1
+lto = "fat"

benchmarks/engines/rust-jetscii/main.rs

@@ -0,0 +1,100 @@
+use std::io::Write;
+
+use shared::{Benchmark, Sample};
+
+fn main() -> anyhow::Result<()> {
+    let Some(arg) = std::env::args_os().nth(1) else {
+        anyhow::bail!("Usage: runner (<engine-name> | --version)")
+    };
+    let Ok(arg) = arg.into_string() else {
+        anyhow::bail!("argument given is not valid UTF-8")
+    };
+    if arg == "--version" {
+        writeln!(std::io::stdout(), env!("CARGO_PKG_VERSION"))?;
+        return Ok(());
+    }
+    let engine = arg;
+    let b = Benchmark::from_stdin()?;
+    let samples = match (&*engine, &*b.model) {
+        ("ascii-chars-prebuilt", "count") => memmem_prebuilt_count(&b)?,
+        ("ascii-chars-oneshot", "count") => memmem_oneshot_count(&b)?,
+        (engine, model) => {
+            anyhow::bail!("unrecognized engine '{engine}' and model '{model}'")
+        }
+    };
+    let mut stdout = std::io::stdout().lock();
+    for s in samples.iter() {
+        writeln!(stdout, "{},{}", s.duration.as_nanos(), s.count)?;
+    }
+    Ok(())
+}
+
+fn memmem_prebuilt_count(b: &Benchmark) -> anyhow::Result<Vec<Sample>> {
+    let Ok(haystack) = std::str::from_utf8(&b.haystack) else {
+        anyhow::bail!("jetscii ASCII search requires valid UTF-8 haystack")
+    };
+    let (needles, len) = needle_array(b)?;
+    let fallback = jetscii_fallback(b)?;
+    let finder = jetscii::AsciiChars::new(needles, len, fallback);
+    shared::run(b, || {
+        let mut haystack = haystack;
+        let mut count = 0;
+        while let Some(i) = finder.find(haystack) {
+            count += 1;
+            haystack = &haystack[i + 1..];
+        }
+        Ok(count)
+    })
+}
+
+fn memmem_oneshot_count(b: &Benchmark) -> anyhow::Result<Vec<Sample>> {
+    let Ok(haystack) = std::str::from_utf8(&b.haystack) else {
+        anyhow::bail!("jetscii ASCII search requires valid UTF-8 haystack")
+    };
+    let (needles, len) = needle_array(b)?;
+    let fallback = jetscii_fallback(b)?;
+    shared::run(b, || {
+        let finder = jetscii::AsciiChars::new(needles, len, &fallback);
+        let mut haystack = haystack;
+        let mut count = 0;
+        while let Some(i) = finder.find(haystack) {
+            count += 1;
+            haystack = &haystack[i + 1..];
+        }
+        Ok(count)
+    })
+}
+
+/// Converts the needles from the given benchmark into a fixed size 16-element
+/// array along with the number of actual needles in the array (which may be
+/// less than 16).
+///
+/// If any needle is more than one byte or there are too many needles to fit
+/// into a 16-element array, then this returns an error. This also returns an
+/// error if any of the bytes are not ASCII.
+fn needle_array(b: &Benchmark) -> anyhow::Result<([u8; 16], i32)> {
+    let mut array = [0u8; 16];
+    let needles = b.needle_bytes()?;
+    let Ok(len) = i32::try_from(needles.len()) else {
+        anyhow::bail!("needle length {} could not fit into i32", needles.len())
+    };
+    anyhow::ensure!(
+        needles.len() <= 16,
+        "jetscii only supports at most 16 single byte needles, \
+         but found {} needles",
+        needles.len(),
+    );
+    for (i, byte) in needles.into_iter().enumerate() {
+        array[i] = byte;
+    }
+    Ok((array, len))
+}
+
+/// Create a fallback predicate for jetscii's up-to-16-bytes search.
+fn jetscii_fallback(b: &Benchmark) -> anyhow::Result<impl Fn(u8) -> bool> {
+    let mut set = vec![false; 256];
+    for byte in b.needle_bytes()? {
+        set[usize::from(byte)] = true;
+    }
+    Ok(move |byte| set[usize::from(byte)])
+}

benchmarks/engines/rust-old-aho-corasick/Cargo.toml

@@ -18,4 +18,4 @@ path = "../../shared"
 [profile.release]
 debug = true
 codegen-units = 1
-lto = "thin"
+lto = "fat"

benchmarks/haystacks/catalog.data.gov/README.md

@@ -0,0 +1,4 @@
+Source: https://catalog.data.gov/dataset/
+
+Mental health:
+https://catalog.data.gov/dataset/mental-health-care-in-the-last-4-weeks