feat: simple performance bench for foyer-memory

.cargo/config.toml

@@ -0,0 +1,2 @@
+[target.'cfg(all())']
+rustflags = ["--cfg", "tokio_unstable"]

.config/hakari.toml

@@ -0,0 +1,30 @@
+# This file contains settings for `cargo hakari`.
+# See https://docs.rs/cargo-hakari/latest/cargo_hakari/config for a full list of options.
+
+hakari-package = "foyer-workspace-hack"
+
+# Format version for hakari's output. Version 4 requires cargo-hakari 0.9.22 or above.
+dep-format-version = "4"
+
+# Setting workspace.resolver = "2" in the root Cargo.toml is HIGHLY recommended.
+# Hakari works much better with the new feature resolver.
+# For more about the new feature resolver, see:
+# https://blog.rust-lang.org/2021/03/25/Rust-1.51.0.html#cargos-new-feature-resolver
+resolver = "2"
+
+# Add triples corresponding to platforms commonly used by developers here.
+# https://doc.rust-lang.org/rustc/platform-support.html
+platforms = [
+    # "x86_64-unknown-linux-gnu",
+    # "x86_64-apple-darwin",
+    # "x86_64-pc-windows-msvc",
+]
+
+# Write out exact versions rather than a semver range. (Defaults to false.)
+# exact-versions = true
+
+[traversal-excludes]
+workspace-members = ["examples"]
+third-party = [
+    { name = "miniz_oxide" },
+]

.github/dependabot.yml

@@ -0,0 +1,12 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "cargo" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "daily"
+    rebase-strategy: "disabled"

.github/pull_request_template.md

@@ -0,0 +1,11 @@
+## What's changed and what's your intention?
+
+> Please explain **IN DETAIL** what the changes are in this PR and why they are needed. :D
+
+## Checklist
+
+- [ ] I have written the necessary rustdoc comments
+- [ ] I have added the necessary unit tests and integration tests
+- [ ] I have passed `make all` (or `make fast` instead if the old tests are not modified) in my local environment.
+
+## Related issues or PRs (optional)

.github/semantic.yml

@@ -0,0 +1,16 @@
+# Ref: https://github.com/zeke/semantic-pull-requests#configuration .
+titleAndCommits: true
+anyCommit: true
+types:
+  - feat
+  - fix
+  - docs
+  - style
+  - refactor
+  - perf
+  - test
+  - build
+  - ci
+  - chore
+  - revert
+allowMergeCommits: true

.github/template/generate.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# You will need to install yq >= 4.16 to use this tool.
+# brew install yq
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR"
+
+
+HEADER="""
+# ================= THIS FILE IS AUTOMATICALLY GENERATED =================
+#
+# Please run generate.sh and commit after editing the workflow templates.
+#
+# ========================================================================
+"""
+
+# Generate workflow for main branch
+echo "$HEADER" > ../workflows/main.yml
+# shellcheck disable=SC2016
+yq ea '. as $item ireduce ({}; . * $item )' template.yml main-override.yml | yq eval '... comments=""' - >> ../workflows/main.yml
+echo "$HEADER" >> ../workflows/main.yml
+
+# Generate workflow for pull requests
+echo "$HEADER" > ../workflows/pull-request.yml
+# shellcheck disable=SC2016
+yq ea '. as $item ireduce ({}; . * $item )' template.yml pr-override.yml | yq eval '... comments=""' - >> ../workflows/pull-request.yml
+echo "$HEADER" >> ../workflows/pull-request.yml
+
+if [ "$1" == "--check" ] ; then
+ if ! git diff --exit-code; then
+    echo "Please run generate.sh and commit after editing the workflow templates."
+    exit 1
+ fi
+fi

.github/template/main-override.yml

@@ -0,0 +1,6 @@
+name: CI (main)
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:

.github/template/pr-override.yml

@@ -0,0 +1,9 @@
+name: CI
+
+on:
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: environment-${{ github.ref }}
+  cancel-in-progress: true

.github/template/template.yml

@@ -0,0 +1,241 @@
+name:
+
+on:
+
+env:
+  RUST_TOOLCHAIN: stable
+  RUST_TOOLCHAIN_NIGHTLY: nightly-2024-03-17
+  CARGO_TERM_COLOR: always
+  CACHE_KEY_SUFFIX: 20240410-2
+
+jobs:
+  misc-check:
+    name: misc check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Run typos check
+        uses: crate-ci/typos@master
+      - name: Install yq
+        run: |
+          wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${BINARY}.tar.gz -O - | tar xz && sudo mv ${BINARY} /usr/bin/yq
+        env:
+          YQ_VERSION: v4.16.1
+          BINARY: yq_linux_amd64
+          BUF_VERSION: 1.0.0-rc6
+      - name: Install jq
+        uses: dcarbone/[email protected]
+      - name: Check if CI workflows are up-to-date
+        run: |
+          ./.github/template/generate.sh --check
+      - name: Check if Grafana dashboards are minimized
+        run: |
+          ./scripts/minimize-dashboards.sh --check
+      - name: Run ShellCheck
+        uses: ludeeus/action-shellcheck@master
+  rust-test:
+    name: rust test with codecov
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          components: rustfmt, clippy, llvm-tools-preview
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-rust-test
+      - name: Install cargo-binstall
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
+      - name: Install cargo tools
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          cargo binstall -y cargo-sort cargo-hakari
+      - name: Run rust cargo-sort check
+        run: |
+          cargo sort -w -c
+      - name: Run hakari check
+        run: |
+          cargo hakari generate --diff
+          cargo hakari manage-deps --dry-run
+      - name: Run rust format check
+        run: |
+          cargo fmt --all -- --check
+      - name: Run rust clippy check
+        run: |
+          cargo clippy --all-targets --features tokio-console -- -D warnings
+          cargo clippy --all-targets --features deadlock -- -D warnings
+          cargo clippy --all-targets -- -D warnings
+      - if: steps.cache.outputs.cache-hit != 'true'
+        uses: taiki-e/install-action@cargo-llvm-cov
+      - if: steps.cache.outputs.cache-hit != 'true'
+        uses: taiki-e/install-action@nextest
+      - name: Run rust test with coverage (igored tests)
+        run: |
+          cargo llvm-cov --no-report nextest --run-ignored ignored-only --no-capture --workspace
+      - name: Run rust test with coverage
+        run: |
+          cargo llvm-cov --no-report nextest
+      - name: Generate codecov report
+        run: |
+          cargo llvm-cov report --lcov --output-path lcov.info
+      - uses: codecov/codecov-action@v3
+        if: matrix.os == 'ubuntu-latest'
+        with:
+          verbose: true
+  deadlock:
+    name: run with single worker thread and deadlock detection
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deadlock
+      - name: Run foyer-storage-bench with single worker thread and deadlock detection
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "--cfg tokio_unstable"
+          RUST_LOG: info
+          TOKIO_WORKER_THREADS: 1
+        run: |-
+          cargo build --all --features deadlock
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock
+          timeout 2m ./target/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/deadlock --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
+  asan:
+    name: run with address saniziter
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN_NIGHTLY }}
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-asan
+      - name: Run Unit Tests With Address Sanitizer
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "-Zsanitizer=address --cfg tokio_unstable"
+          RUST_LOG: info
+        run: |-
+          cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
+      - name: Run foyer-storage-bench With Address Sanitizer
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "-Zsanitizer=address --cfg tokio_unstable"
+          RUST_LOG: info
+        run: |-
+          cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/asan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
+  lsan:
+    name: run with leak saniziter
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN_NIGHTLY }}
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-lsan
+      - name: Run Unit Tests With Leak Sanitizer
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "-Zsanitizer=leak --cfg tokio_unstable"
+          RUST_LOG: info
+        run: |-
+          cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} test --lib --bins --tests --target x86_64-unknown-linux-gnu -- --nocapture
+      - name: Run foyer-storage-bench With Leak Sanitizer
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "-Zsanitizer=leak --cfg tokio_unstable"
+          RUST_LOG: info
+        run: |-
+          cargo +${{ env.RUST_TOOLCHAIN_NIGHTLY }} build --all --target x86_64-unknown-linux-gnu
+          mkdir -p $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan
+          timeout 2m ./target/x86_64-unknown-linux-gnu/debug/foyer-storage-bench --dir $GITHUB_WORKSPACE/foyer-data/foyer-storage-bench/lsan --capacity 256 --region-size 16 --lookup-range 1000 --w-rate 1 --r-rate 1 --ticket-insert-rate-limit 10 --time 60
+  deterministic-test:
+    name: run deterministic test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install rust toolchain@v1
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+          components: rustfmt, clippy
+      - name: Cache Cargo home
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}-${{ env.CACHE_KEY_SUFFIX }}-deterministic-test
+      - if: steps.cache.outputs.cache-hit != 'true'
+        uses: taiki-e/install-action@nextest
+      - name: Run rust clippy check (madsim)
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "--cfg tokio_unstable --cfg madsim"
+          RUST_LOG: info
+          TOKIO_WORKER_THREADS: 1
+        run: |-
+          cargo clippy --all-targets
+      - name: Run nexmark test (madsim)
+        env:
+          RUST_BACKTRACE: 1
+          RUSTFLAGS: "--cfg tokio_unstable --cfg madsim"
+          RUST_LOG: info
+          TOKIO_WORKER_THREADS: 1
+        run: |-
+          cargo nextest run --all

.github/workflows/license_check.yml

@@ -0,0 +1,19 @@
+name: License Checker
+
+on:
+  push:
+    branches:
+    - main
+    - "forks/*"
+  pull_request:
+    branches: 
+    - main
+    - "v*.*.*-rc"
+jobs:
+  license-header-check:
+    runs-on: ubuntu-latest
+    name: license-header-check
+    steps:
+    - uses: actions/checkout@v3
+    - name: Check License Header
+      uses: apache/skywalking-eyes/header@df70871af1a8109c9a5b1dc824faaf65246c5236