From cca20d44c03e26a2a24a0b854bf5e19388d9dd80 Mon Sep 17 00:00:00 2001
From: Yuchen Liang <70461588+yliang412@users.noreply.github.com>
Date: Tue, 10 Dec 2024 00:01:25 -0500
Subject: [PATCH]  plannertest: criterion-rs based benchmark for both planning
 and execution (#262)

Part of #263.

## Problem

There is a lack of performance evaluation in optd. Without proper
benchmarking, we will not know if some changes cause significant
regression in performance. We should keep track of the two aspects of
optimizer performance:

- plan generation time
- actual execution time

## Summary of changes

- Add a [Criterion-rs](https://bheisler.github.io/criterion.rs/book/)
based benchmark to measure optd performance.
- Use the `sqlplannertest` yaml-based setup, bench if there is a `bench`
task.
- Bump `sqlplannertest` to version 0.4.
- better individual test matching added in
https://github.com/risinglightdb/sqlplannertest-rs/pull/9

Planning:
  - Input: SQL statement (aka `datafusion_sql::parser::Statement`).
  - Measures the time it takes to generate a datafusion physical plan.

Execution:
- Input: datafusion physical plan outputted from optd (aka `Arc<dyn
ExecutionPlan>`).
- Measures the time it takes to execute the entire query.

## Next Steps
https://github.com/benchmark-action/github-action-benchmark can directly
take `cargo bench` results as input. So it will be easy to set up CI.

---------

Signed-off-by: Yuchen Liang <yuchenl3@andrew.cmu.edu>
---
 Cargo.lock                                    | 629 +++++++++++-------
 optd-sqlplannertest/Cargo.toml                |  10 +-
 optd-sqlplannertest/README.md                 |  57 +-
 optd-sqlplannertest/benches/planner_bench.rs  | 150 +++++
 optd-sqlplannertest/src/bench_helper.rs       |  64 ++
 .../src/bench_helper/execution.rs             |  71 ++
 .../src/bench_helper/planning.rs              |  52 ++
 .../src/bin/planner_test_apply.rs             |  55 +-
 optd-sqlplannertest/src/lib.rs                | 162 +++--
 .../tests/tpch/bench_populate.sql             |  18 +
 optd-sqlplannertest/tests/tpch/q1.yml         |   1 +
 optd-sqlplannertest/tests/tpch/q10.yml        |   1 +
 optd-sqlplannertest/tests/tpch/q11.yml        |   1 +
 optd-sqlplannertest/tests/tpch/q12.yml        |   1 +
 .../tests/tpch/q13.planner.sql                |  74 +++
 optd-sqlplannertest/tests/tpch/q13.yml        |  26 +
 optd-sqlplannertest/tests/tpch/q14.yml        |   1 +
 .../tests/tpch/q16.yml.disabled               |  35 +
 .../tests/tpch/q18.yml.disabled               |  38 ++
 optd-sqlplannertest/tests/tpch/q2.yml         |   1 +
 .../tests/tpch/q20.yml.disabled               |  42 ++
 .../tests/tpch/q21.yml.disabled               |  45 ++
 .../tests/tpch/q22.yml.disabled               |  42 ++
 optd-sqlplannertest/tests/tpch/q3.yml         |   1 +
 .../tests/tpch/q4.yml.disabled                |  26 +
 optd-sqlplannertest/tests/tpch/q5.yml         |   1 +
 optd-sqlplannertest/tests/tpch/q6.yml         |   1 +
 optd-sqlplannertest/tests/tpch/q7.yml         |   1 +
 optd-sqlplannertest/tests/tpch/q8.yml         |   1 +
 optd-sqlplannertest/tests/tpch/q9.yml         |   1 +
 30 files changed, 1269 insertions(+), 339 deletions(-)
 create mode 100644 optd-sqlplannertest/benches/planner_bench.rs
 create mode 100644 optd-sqlplannertest/src/bench_helper.rs
 create mode 100644 optd-sqlplannertest/src/bench_helper/execution.rs
 create mode 100644 optd-sqlplannertest/src/bench_helper/planning.rs
 create mode 100644 optd-sqlplannertest/tests/tpch/bench_populate.sql
 create mode 100644 optd-sqlplannertest/tests/tpch/q13.planner.sql
 create mode 100644 optd-sqlplannertest/tests/tpch/q13.yml
 create mode 100644 optd-sqlplannertest/tests/tpch/q16.yml.disabled
 create mode 100644 optd-sqlplannertest/tests/tpch/q18.yml.disabled
 create mode 100644 optd-sqlplannertest/tests/tpch/q20.yml.disabled
 create mode 100644 optd-sqlplannertest/tests/tpch/q21.yml.disabled
 create mode 100644 optd-sqlplannertest/tests/tpch/q22.yml.disabled
 create mode 100644 optd-sqlplannertest/tests/tpch/q4.yml.disabled

diff --git a/Cargo.lock b/Cargo.lock
index f948fde5..f828d1af 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -63,9 +63,9 @@ dependencies = [
 
 [[package]]
 name = "allocator-api2"
-version = "0.2.18"
+version = "0.2.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f"
+checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
 
 [[package]]
 name = "android-tzdata"
@@ -82,6 +82,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "anes"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
+
 [[package]]
 name = "anstream"
 version = "0.6.18"
@@ -133,9 +139,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.93"
+version = "1.0.94"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775"
+checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7"
 dependencies = [
  "backtrace",
 ]
@@ -161,7 +167,7 @@ dependencies = [
  "snap",
  "strum 0.25.0",
  "strum_macros 0.25.3",
- "thiserror 1.0.68",
+ "thiserror 1.0.69",
  "typed-builder",
  "uuid",
  "xz2",
@@ -229,7 +235,7 @@ dependencies = [
  "chrono",
  "chrono-tz",
  "half",
- "hashbrown 0.15.1",
+ "hashbrown 0.15.2",
  "num",
 ]
 
@@ -324,7 +330,7 @@ dependencies = [
  "arrow-schema",
  "chrono",
  "half",
- "indexmap 2.6.0",
+ "indexmap 2.7.0",
  "lexical-core",
  "num",
  "serde",
@@ -429,9 +435,9 @@ dependencies = [
 
 [[package]]
 name = "async-compression"
-version = "0.4.17"
+version = "0.4.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0cb8f1d480b0ea3783ab015936d2a55c87e219676f0c0b7dec61494043f21857"
+checksum = "df895a515f70646414f4b45c0b79082783b80552b373a68283012928df56f522"
 dependencies = [
  "bzip2",
  "flate2",
@@ -512,7 +518,7 @@ dependencies = [
  "aws-sdk-sts",
  "aws-smithy-async",
  "aws-smithy-http",
- "aws-smithy-json",
+ "aws-smithy-json 0.60.7",
  "aws-smithy-runtime",
  "aws-smithy-runtime-api",
  "aws-smithy-types",
@@ -543,9 +549,9 @@ dependencies = [
 
 [[package]]
 name = "aws-runtime"
-version = "1.4.3"
+version = "1.4.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a10d5c055aa540164d9561a0e2e74ad30f0dcf7393c3a92f6733ddf9c5762468"
+checksum = "b5ac934720fbb46206292d2c75b57e67acfc56fe7dfd34fb9a02334af08409ea"
 dependencies = [
  "aws-credential-types",
  "aws-sigv4",
@@ -568,15 +574,15 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-sso"
-version = "1.49.0"
+version = "1.50.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09677244a9da92172c8dc60109b4a9658597d4d298b188dd0018b6a66b410ca4"
+checksum = "05ca43a4ef210894f93096039ef1d6fa4ad3edfabb3be92b80908b9f2e4b4eab"
 dependencies = [
  "aws-credential-types",
  "aws-runtime",
  "aws-smithy-async",
  "aws-smithy-http",
- "aws-smithy-json",
+ "aws-smithy-json 0.61.1",
  "aws-smithy-runtime",
  "aws-smithy-runtime-api",
  "aws-smithy-types",
@@ -590,15 +596,15 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-ssooidc"
-version = "1.50.0"
+version = "1.51.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81fea2f3a8bb3bd10932ae7ad59cc59f65f270fc9183a7e91f501dc5efbef7ee"
+checksum = "abaf490c2e48eed0bb8e2da2fb08405647bd7f253996e0f93b981958ea0f73b0"
 dependencies = [
  "aws-credential-types",
  "aws-runtime",
  "aws-smithy-async",
  "aws-smithy-http",
- "aws-smithy-json",
+ "aws-smithy-json 0.61.1",
  "aws-smithy-runtime",
  "aws-smithy-runtime-api",
  "aws-smithy-types",
@@ -612,15 +618,15 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-sts"
-version = "1.50.0"
+version = "1.51.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ada54e5f26ac246dc79727def52f7f8ed38915cb47781e2a72213957dc3a7d5"
+checksum = "b68fde0d69c8bfdc1060ea7da21df3e39f6014da316783336deff0a9ec28f4bf"
 dependencies = [
  "aws-credential-types",
  "aws-runtime",
  "aws-smithy-async",
  "aws-smithy-http",
- "aws-smithy-json",
+ "aws-smithy-json 0.61.1",
  "aws-smithy-query",
  "aws-smithy-runtime",
  "aws-smithy-runtime-api",
@@ -635,9 +641,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sigv4"
-version = "1.2.5"
+version = "1.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5619742a0d8f253be760bfbb8e8e8368c69e3587e4637af5754e488a611499b1"
+checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2"
 dependencies = [
  "aws-credential-types",
  "aws-smithy-http",
@@ -648,7 +654,7 @@ dependencies = [
  "hex",
  "hmac",
  "http 0.2.12",
- "http 1.1.0",
+ "http 1.2.0",
  "once_cell",
  "percent-encoding",
  "sha2",
@@ -696,6 +702,15 @@ dependencies = [
  "aws-smithy-types",
 ]
 
+[[package]]
+name = "aws-smithy-json"
+version = "0.61.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095"
+dependencies = [
+ "aws-smithy-types",
+]
+
 [[package]]
 name = "aws-smithy-query"
 version = "0.60.7"
@@ -708,9 +723,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-runtime"
-version = "1.7.3"
+version = "1.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be28bd063fa91fd871d131fc8b68d7cd4c5fa0869bea68daca50dcb1cbd76be2"
+checksum = "9f20685047ca9d6f17b994a07f629c813f08b5bce65523e47124879e60103d45"
 dependencies = [
  "aws-smithy-async",
  "aws-smithy-http",
@@ -743,7 +758,7 @@ dependencies = [
  "aws-smithy-types",
  "bytes",
  "http 0.2.12",
- "http 1.1.0",
+ "http 1.2.0",
  "pin-project-lite",
  "tokio",
  "tracing",
@@ -761,7 +776,7 @@ dependencies = [
  "bytes-utils",
  "futures-core",
  "http 0.2.12",
- "http 1.1.0",
+ "http 1.2.0",
  "http-body 0.4.6",
  "http-body 1.0.1",
  "http-body-util",
@@ -868,9 +883,9 @@ dependencies = [
 
 [[package]]
 name = "blake3"
-version = "1.5.4"
+version = "1.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7"
+checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e"
 dependencies = [
  "arrayref",
  "arrayvec",
@@ -911,9 +926,9 @@ dependencies = [
 
 [[package]]
 name = "bstr"
-version = "1.10.0"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c"
+checksum = "1a68f1f47cdf0ec8ee4b941b2eee2a80cb796db73118c0dd09ac63fbe405be22"
 dependencies = [
  "memchr",
  "regex-automata",
@@ -934,9 +949,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
-version = "1.8.0"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da"
+checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b"
 
 [[package]]
 name = "bytes-utils"
@@ -975,11 +990,17 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "02d88a780e6aa14b75d7be99f374d8b5c315aaf9c12ada1e2b1cb281468584c9"
 
+[[package]]
+name = "cast"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
+
 [[package]]
 name = "cc"
-version = "1.1.36"
+version = "1.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baee610e9452a8f6f0a1b6194ec09ff9e2d85dea54432acdae41aa0761c95d70"
+checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d"
 dependencies = [
  "jobserver",
  "libc",
@@ -1006,9 +1027,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
 
 [[package]]
 name = "chrono"
-version = "0.4.38"
+version = "0.4.39"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401"
+checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825"
 dependencies = [
  "android-tzdata",
  "iana-time-zone",
@@ -1040,11 +1061,38 @@ dependencies = [
  "phf_codegen",
 ]
 
+[[package]]
+name = "ciborium"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
+dependencies = [
+ "ciborium-io",
+ "ciborium-ll",
+ "serde",
+]
+
+[[package]]
+name = "ciborium-io"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
+
+[[package]]
+name = "ciborium-ll"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
+dependencies = [
+ "ciborium-io",
+ "half",
+]
+
 [[package]]
 name = "clap"
-version = "4.5.20"
+version = "4.5.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8"
+checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -1052,9 +1100,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.20"
+version = "4.5.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54"
+checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838"
 dependencies = [
  "anstream",
  "anstyle",
@@ -1076,9 +1124,9 @@ dependencies = [
 
 [[package]]
 name = "clap_lex"
-version = "0.7.2"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"
+checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
 
 [[package]]
 name = "clipboard-win"
@@ -1097,13 +1145,13 @@ checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
 
 [[package]]
 name = "comfy-table"
-version = "7.1.1"
+version = "7.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b34115915337defe99b2aff5c2ce6771e5fbc4079f4b506301f5cf394c8452f7"
+checksum = "24f165e7b643266ea80cb858aed492ad9280e3e05ce24d4a99d7d7b889b6a4d9"
 dependencies = [
  "strum 0.26.3",
  "strum_macros 0.26.4",
- "unicode-width",
+ "unicode-width 0.2.0",
 ]
 
 [[package]]
@@ -1115,7 +1163,7 @@ dependencies = [
  "encode_unicode 0.3.6",
  "lazy_static",
  "libc",
- "unicode-width",
+ "unicode-width 0.1.14",
  "windows-sys 0.52.0",
 ]
 
@@ -1182,9 +1230,9 @@ dependencies = [
 
 [[package]]
 name = "cpufeatures"
-version = "0.2.14"
+version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0"
+checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3"
 dependencies = [
  "libc",
 ]
@@ -1198,6 +1246,44 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "criterion"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f"
+dependencies = [
+ "anes",
+ "cast",
+ "ciborium",
+ "clap",
+ "criterion-plot",
+ "futures",
+ "is-terminal",
+ "itertools 0.10.5",
+ "num-traits",
+ "once_cell",
+ "oorandom",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "tinytemplate",
+ "tokio",
+ "walkdir",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
+dependencies = [
+ "cast",
+ "itertools 0.10.5",
+]
+
 [[package]]
 name = "crossbeam"
 version = "0.8.4"
@@ -1272,9 +1358,9 @@ dependencies = [
 
 [[package]]
 name = "csv"
-version = "1.3.0"
+version = "1.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe"
+checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf"
 dependencies = [
  "csv-core",
  "itoa",
@@ -1307,9 +1393,9 @@ dependencies = [
 
 [[package]]
 name = "ctor"
-version = "0.2.8"
+version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f"
+checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501"
 dependencies = [
  "quote",
  "syn",
@@ -1408,8 +1494,8 @@ dependencies = [
  "glob",
  "half",
  "hashbrown 0.14.5",
- "indexmap 2.6.0",
- "itertools",
+ "indexmap 2.7.0",
+ "itertools 0.13.0",
  "log",
  "num-traits",
  "num_cpus",
@@ -1459,7 +1545,7 @@ dependencies = [
  "chrono",
  "half",
  "hashbrown 0.14.5",
- "indexmap 2.6.0",
+ "indexmap 2.7.0",
  "instant",
  "libc",
  "num_cpus",
@@ -1517,7 +1603,7 @@ dependencies = [
  "datafusion-functions-aggregate-common",
  "datafusion-functions-window-common",
  "datafusion-physical-expr-common",
- "indexmap 2.6.0",
+ "indexmap 2.7.0",
  "paste",
  "serde_json",
  "sqlparser",
@@ -1533,7 +1619,7 @@ checksum = "4da0f3cb4669f9523b403d6b5a0ec85023e0ab3bf0183afd1517475b3e64fdd2"
 dependencies = [
  "arrow",
  "datafusion-common",
- "itertools",
+ "itertools 0.13.0",
  "paste",
 ]
 
@@ -1554,7 +1640,7 @@ dependencies = [
  "datafusion-expr",
  "hashbrown 0.14.5",
  "hex",
- "itertools",
+ "itertools 0.13.0",
  "log",
  "md-5",
  "rand",
@@ -1580,7 +1666,7 @@ dependencies = [
  "datafusion-physical-expr",
  "datafusion-physical-expr-common",
  "half",
- "indexmap 2.6.0",
+ "indexmap 2.7.0",
  "log",
  "paste",
 ]
@@ -1616,7 +1702,7 @@ dependencies = [
  "datafusion-functions",
  "datafusion-functions-aggregate",
  "datafusion-physical-expr-common",
- "itertools",
+ "itertools 0.13.0",
  "log",
  "paste",
  "rand",
@@ -1691,8 +1777,8 @@ dependencies = [
  "datafusion-expr",
  "datafusion-physical-expr",
  "hashbrown 0.14.5",
- "indexmap 2.6.0",
- "itertools",
+ "indexmap 2.7.0",
+ "itertools 0.13.0",
  "log",
  "paste",
  "regex-syntax",
@@ -1719,8 +1805,8 @@ dependencies = [
  "datafusion-physical-expr-common",
  "half",
  "hashbrown 0.14.5",
- "indexmap 2.6.0",
- "itertools",
+ "indexmap 2.7.0",
+ "itertools 0.13.0",
  "log",
  "paste",
  "petgraph",
@@ -1753,7 +1839,7 @@ dependencies = [
  "datafusion-expr-common",
  "datafusion-physical-expr",
  "datafusion-physical-plan",
- "itertools",
+ "itertools 0.13.0",
 ]
 
 [[package]]
@@ -1781,8 +1867,8 @@ dependencies = [
  "futures",
  "half",
  "hashbrown 0.14.5",
- "indexmap 2.6.0",
- "itertools",
+ "indexmap 2.7.0",
+ "itertools 0.13.0",
  "log",
  "once_cell",
  "parking_lot",
@@ -1802,7 +1888,7 @@ dependencies = [
  "arrow-schema",
  "datafusion-common",
  "datafusion-expr",
- "indexmap 2.6.0",
+ "indexmap 2.7.0",
  "log",
  "regex",
  "sqlparser",
@@ -2011,12 +2097,12 @@ dependencies = [
 
 [[package]]
 name = "errno"
-version = "0.3.9"
+version = "0.3.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba"
+checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
 dependencies = [
  "libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -2039,9 +2125,9 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
 
 [[package]]
 name = "fastrand"
-version = "2.1.1"
+version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6"
+checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
 
 [[package]]
 name = "fd-lock"
@@ -2072,9 +2158,9 @@ dependencies = [
 
 [[package]]
 name = "flate2"
-version = "1.0.34"
+version = "1.0.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0"
+checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c"
 dependencies = [
  "crc32fast",
  "miniz_oxide",
@@ -2163,9 +2249,9 @@ checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
 
 [[package]]
 name = "futures-lite"
-version = "2.4.0"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f1fa2f9765705486b33fd2acf1577f8ec449c2ba1f318ae5447697b7c08d210"
+checksum = "cef40d21ae2c515b51041df9ed313ed21e572df340ea58a922a0aefe7e8891a1"
 dependencies = [
  "fastrand",
  "futures-core",
@@ -2238,8 +2324,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
 dependencies = [
  "cfg-if",
+ "js-sys",
  "libc",
  "wasi",
+ "wasm-bindgen",
 ]
 
 [[package]]
@@ -2266,7 +2354,7 @@ dependencies = [
  "futures-sink",
  "futures-util",
  "http 0.2.12",
- "indexmap 2.6.0",
+ "indexmap 2.7.0",
  "slab",
  "tokio",
  "tokio-util",
@@ -2284,8 +2372,8 @@ dependencies = [
  "fnv",
  "futures-core",
  "futures-sink",
- "http 1.1.0",
- "indexmap 2.6.0",
+ "http 1.2.0",
+ "indexmap 2.7.0",
  "slab",
  "tokio",
  "tokio-util",
@@ -2322,9 +2410,9 @@ dependencies = [
 
 [[package]]
 name = "hashbrown"
-version = "0.15.1"
+version = "0.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3"
+checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
 
 [[package]]
 name = "heck"
@@ -2396,9 +2484,9 @@ dependencies = [
 
 [[package]]
 name = "http"
-version = "1.1.0"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258"
+checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea"
 dependencies = [
  "bytes",
  "fnv",
@@ -2423,7 +2511,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
 dependencies = [
  "bytes",
- "http 1.1.0",
+ "http 1.2.0",
 ]
 
 [[package]]
@@ -2434,7 +2522,7 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f"
 dependencies = [
  "bytes",
  "futures-util",
- "http 1.1.0",
+ "http 1.2.0",
  "http-body 1.0.1",
  "pin-project-lite",
 ]
@@ -2491,7 +2579,7 @@ dependencies = [
  "futures-channel",
  "futures-util",
  "h2 0.4.7",
- "http 1.1.0",
+ "http 1.2.0",
  "http-body 1.0.1",
  "httparse",
  "itoa",
@@ -2524,14 +2612,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333"
 dependencies = [
  "futures-util",
- "http 1.1.0",
+ "http 1.2.0",
  "hyper 1.5.1",
  "hyper-util",
  "rustls 0.23.19",
  "rustls-native-certs 0.8.1",
  "rustls-pki-types",
  "tokio",
- "tokio-rustls 0.26.0",
+ "tokio-rustls 0.26.1",
  "tower-service",
 ]
 
@@ -2544,7 +2632,7 @@ dependencies = [
  "bytes",
  "futures-channel",
  "futures-util",
- "http 1.1.0",
+ "http 1.2.0",
  "http-body 1.0.1",
  "hyper 1.5.1",
  "pin-project-lite",
@@ -2735,12 +2823,12 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.6.0"
+version = "2.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da"
+checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f"
 dependencies = [
  "equivalent",
- "hashbrown 0.15.1",
+ "hashbrown 0.15.2",
  "serde",
 ]
 
@@ -2785,6 +2873,15 @@ version = "1.70.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
 
+[[package]]
+name = "itertools"
+version = "0.10.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itertools"
 version = "0.13.0"
@@ -2796,9 +2893,9 @@ dependencies = [
 
 [[package]]
 name = "itoa"
-version = "1.0.11"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
+checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
 
 [[package]]
 name = "jobserver"
@@ -2811,10 +2908,11 @@ dependencies = [
 
 [[package]]
 name = "js-sys"
-version = "0.3.72"
+version = "0.3.76"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9"
+checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7"
 dependencies = [
+ "once_cell",
  "wasm-bindgen",
 ]
 
@@ -2826,9 +2924,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 
 [[package]]
 name = "lexical-core"
-version = "1.0.2"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0431c65b318a590c1de6b8fd6e72798c92291d27762d94c9e6c37ed7a73d8458"
+checksum = "b765c31809609075565a70b4b71402281283aeda7ecaf4818ac14a7b2ade8958"
 dependencies = [
  "lexical-parse-float",
  "lexical-parse-integer",
@@ -2839,9 +2937,9 @@ dependencies = [
 
 [[package]]
 name = "lexical-parse-float"
-version = "1.0.2"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb17a4bdb9b418051aa59d41d65b1c9be5affab314a872e5ad7f06231fb3b4e0"
+checksum = "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2"
 dependencies = [
  "lexical-parse-integer",
  "lexical-util",
@@ -2850,9 +2948,9 @@ dependencies = [
 
 [[package]]
 name = "lexical-parse-integer"
-version = "1.0.2"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5df98f4a4ab53bf8b175b363a34c7af608fe31f93cc1fb1bf07130622ca4ef61"
+checksum = "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e"
 dependencies = [
  "lexical-util",
  "static_assertions",
@@ -2860,18 +2958,18 @@ dependencies = [
 
 [[package]]
 name = "lexical-util"
-version = "1.0.3"
+version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85314db53332e5c192b6bca611fb10c114a80d1b831ddac0af1e9be1b9232ca0"
+checksum = "5a82e24bf537fd24c177ffbbdc6ebcc8d54732c35b50a3f28cc3f4e4c949a0b3"
 dependencies = [
  "static_assertions",
 ]
 
 [[package]]
 name = "lexical-write-float"
-version = "1.0.2"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e7c3ad4e37db81c1cbe7cf34610340adc09c322871972f74877a712abc6c809"
+checksum = "c5afc668a27f460fb45a81a757b6bf2f43c2d7e30cb5a2dcd3abf294c78d62bd"
 dependencies = [
  "lexical-util",
  "lexical-write-integer",
@@ -2880,9 +2978,9 @@ dependencies = [
 
 [[package]]
 name = "lexical-write-integer"
-version = "1.0.2"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb89e9f6958b83258afa3deed90b5de9ef68eef090ad5086c791cd2345610162"
+checksum = "629ddff1a914a836fb245616a7888b62903aae58fa771e1d83943035efa0f978"
 dependencies = [
  "lexical-util",
  "static_assertions",
@@ -2890,9 +2988,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.162"
+version = "0.2.168"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398"
+checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d"
 
 [[package]]
 name = "libflate"
@@ -2964,9 +3062,9 @@ checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
 
 [[package]]
 name = "litemap"
-version = "0.7.3"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704"
+checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104"
 
 [[package]]
 name = "lock_api"
@@ -3046,11 +3144,10 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "1.0.2"
+version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec"
+checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd"
 dependencies = [
- "hermit-abi 0.3.9",
  "libc",
  "wasi",
  "windows-sys 0.52.0",
@@ -3211,7 +3308,7 @@ dependencies = [
  "futures",
  "humantime",
  "hyper 1.5.1",
- "itertools",
+ "itertools 0.13.0",
  "md-5",
  "parking_lot",
  "percent-encoding",
@@ -3235,6 +3332,12 @@ version = "1.20.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
 
+[[package]]
+name = "oorandom"
+version = "11.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9"
+
 [[package]]
 name = "openssl-probe"
 version = "0.1.5"
@@ -3263,7 +3366,7 @@ dependencies = [
  "arrow-schema",
  "chrono",
  "erased-serde",
- "itertools",
+ "itertools 0.13.0",
  "ordered-float 4.5.0",
  "pretty_assertions",
  "serde",
@@ -3281,7 +3384,7 @@ dependencies = [
  "datafusion-expr",
  "futures-lite",
  "futures-util",
- "itertools",
+ "itertools 0.13.0",
  "optd-core",
  "optd-datafusion-repr",
  "optd-datafusion-repr-adv-cost",
@@ -3298,7 +3401,7 @@ dependencies = [
  "camelpaste",
  "datafusion-expr",
  "heck 0.5.0",
- "itertools",
+ "itertools 0.13.0",
  "optd-core",
  "pretty-xmlish",
  "serde",
@@ -3314,7 +3417,7 @@ dependencies = [
  "arrow-schema",
  "assert_approx_eq",
  "datafusion",
- "itertools",
+ "itertools 0.13.0",
  "optd-core",
  "optd-datafusion-repr",
  "optd-gungnir",
@@ -3333,7 +3436,7 @@ version = "0.1.1"
 dependencies = [
  "crossbeam",
  "hashbrown 0.14.5",
- "itertools",
+ "itertools 0.13.0",
  "lazy_static",
  "murmur2",
  "optd-core",
@@ -3383,7 +3486,7 @@ dependencies = [
  "datafusion",
  "datafusion-optd-cli",
  "env_logger 0.9.3",
- "itertools",
+ "itertools 0.13.0",
  "lazy_static",
  "mimalloc",
  "optd-datafusion-bridge",
@@ -3391,7 +3494,7 @@ dependencies = [
  "optd-datafusion-repr-adv-cost",
  "regex",
  "sqllogictest",
- "thiserror 2.0.2",
+ "thiserror 2.0.6",
  "tokio",
 ]
 
@@ -3402,16 +3505,18 @@ dependencies = [
  "anyhow",
  "async-trait",
  "clap",
+ "criterion",
  "datafusion",
  "datafusion-optd-cli",
  "env_logger 0.9.3",
- "itertools",
+ "itertools 0.13.0",
  "lazy_static",
  "mimalloc",
  "optd-datafusion-bridge",
  "optd-datafusion-repr",
  "optd-datafusion-repr-adv-cost",
  "regex",
+ "serde_yaml",
  "sqlplannertest",
  "tokio",
 ]
@@ -3508,7 +3613,7 @@ dependencies = [
  "flate2",
  "futures",
  "half",
- "hashbrown 0.15.1",
+ "hashbrown 0.15.2",
  "lz4_flex",
  "num",
  "num-bigint",
@@ -3551,7 +3656,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db"
 dependencies = [
  "fixedbitset",
- "indexmap 2.6.0",
+ "indexmap 2.7.0",
 ]
 
 [[package]]
@@ -3610,6 +3715,34 @@ version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
 
+[[package]]
+name = "plotters"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"
+dependencies = [
+ "num-traits",
+ "plotters-backend",
+ "plotters-svg",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "plotters-backend"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"
+
+[[package]]
+name = "plotters-svg"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"
+dependencies = [
+ "plotters-backend",
+]
+
 [[package]]
 name = "postgres-protocol"
 version = "0.6.7"
@@ -3711,7 +3844,7 @@ dependencies = [
  "is-terminal",
  "lazy_static",
  "term",
- "unicode-width",
+ "unicode-width 0.1.14",
 ]
 
 [[package]]
@@ -3725,9 +3858,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.89"
+version = "1.0.92"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e"
+checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0"
 dependencies = [
  "unicode-ident",
 ]
@@ -3750,9 +3883,9 @@ dependencies = [
 
 [[package]]
 name = "quinn"
-version = "0.11.5"
+version = "0.11.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684"
+checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef"
 dependencies = [
  "bytes",
  "pin-project-lite",
@@ -3761,40 +3894,43 @@ dependencies = [
  "rustc-hash",
  "rustls 0.23.19",
  "socket2",
- "thiserror 1.0.68",
+ "thiserror 2.0.6",
  "tokio",
  "tracing",
 ]
 
 [[package]]
 name = "quinn-proto"
-version = "0.11.8"
+version = "0.11.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6"
+checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d"
 dependencies = [
  "bytes",
+ "getrandom",
  "rand",
  "ring",
  "rustc-hash",
  "rustls 0.23.19",
+ "rustls-pki-types",
  "slab",
- "thiserror 1.0.68",
+ "thiserror 2.0.6",
  "tinyvec",
  "tracing",
+ "web-time",
 ]
 
 [[package]]
 name = "quinn-udp"
-version = "0.5.7"
+version = "0.5.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d5a626c6807713b15cac82a6acaccd6043c9a5408c24baae07611fec3f243da"
+checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527"
 dependencies = [
  "cfg_aliases 0.2.1",
  "libc",
  "once_cell",
  "socket2",
  "tracing",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -3883,7 +4019,7 @@ checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43"
 dependencies = [
  "getrandom",
  "libredox",
- "thiserror 1.0.68",
+ "thiserror 1.0.69",
 ]
 
 [[package]]
@@ -3900,9 +4036,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.4.8"
+version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3"
+checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -3938,7 +4074,7 @@ dependencies = [
  "futures-core",
  "futures-util",
  "h2 0.4.7",
- "http 1.1.0",
+ "http 1.2.0",
  "http-body 1.0.1",
  "http-body-util",
  "hyper 1.5.1",
@@ -3961,7 +4097,7 @@ dependencies = [
  "serde_urlencoded",
  "sync_wrapper",
  "tokio",
- "tokio-rustls 0.26.0",
+ "tokio-rustls 0.26.1",
  "tokio-util",
  "tower-service",
  "url",
@@ -4046,15 +4182,15 @@ dependencies = [
 
 [[package]]
 name = "rustix"
-version = "0.38.39"
+version = "0.38.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee"
+checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85"
 dependencies = [
  "bitflags 2.6.0",
  "errno",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -4130,6 +4266,9 @@ name = "rustls-pki-types"
 version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b"
+dependencies = [
+ "web-time",
+]
 
 [[package]]
 name = "rustls-webpki"
@@ -4175,7 +4314,7 @@ dependencies = [
  "nix",
  "radix_trie",
  "unicode-segmentation",
- "unicode-width",
+ "unicode-width 0.1.14",
  "utf8parse",
  "windows-sys 0.52.0",
 ]
@@ -4197,9 +4336,9 @@ dependencies = [
 
 [[package]]
 name = "schannel"
-version = "0.1.26"
+version = "0.1.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1"
+checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d"
 dependencies = [
  "windows-sys 0.59.0",
 ]
@@ -4248,9 +4387,9 @@ dependencies = [
 
 [[package]]
 name = "security-framework-sys"
-version = "2.12.0"
+version = "2.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6"
+checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2"
 dependencies = [
  "core-foundation-sys",
  "libc",
@@ -4270,18 +4409,18 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
 
 [[package]]
 name = "serde"
-version = "1.0.214"
+version = "1.0.215"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5"
+checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.214"
+version = "1.0.215"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766"
+checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4290,9 +4429,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.132"
+version = "1.0.133"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03"
+checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377"
 dependencies = [
  "itoa",
  "memchr",
@@ -4322,7 +4461,7 @@ dependencies = [
  "chrono",
  "hex",
  "indexmap 1.9.3",
- "indexmap 2.6.0",
+ "indexmap 2.7.0",
  "serde",
  "serde_derive",
  "serde_json",
@@ -4348,7 +4487,7 @@ version = "0.9.34+deprecated"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
 dependencies = [
- "indexmap 2.6.0",
+ "indexmap 2.7.0",
  "itoa",
  "ryu",
  "serde",
@@ -4432,7 +4571,7 @@ version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917"
 dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
  "proc-macro2",
  "quote",
  "syn",
@@ -4446,9 +4585,9 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
 
 [[package]]
 name = "socket2"
-version = "0.5.7"
+version = "0.5.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c"
+checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8"
 dependencies = [
  "libc",
  "windows-sys 0.52.0",
@@ -4472,7 +4611,7 @@ dependencies = [
  "futures",
  "glob",
  "humantime",
- "itertools",
+ "itertools 0.13.0",
  "libtest-mimic",
  "md-5",
  "owo-colors",
@@ -4480,7 +4619,7 @@ dependencies = [
  "similar",
  "subst",
  "tempfile",
- "thiserror 1.0.68",
+ "thiserror 1.0.69",
  "tracing",
 ]
 
@@ -4507,15 +4646,16 @@ dependencies = [
 
 [[package]]
 name = "sqlplannertest"
-version = "0.3.0"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f785c6f7bf880d40b33c9edffda8244096b2ffc2f6716e645583600118f9139a"
+checksum = "3510ad2ca50c652ccef035ae6fe3596c306f365a1319d8ca116f01381127a335"
 dependencies = [
  "anyhow",
  "async-trait",
  "console",
  "futures-util",
  "glob",
+ "itertools 0.13.0",
  "libtest-mimic",
  "serde",
  "serde_yaml",
@@ -4600,7 +4740,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7a3c1ba4fd019bc866333a61fe205fc9b686e3cf5971dd8dfc116657d933031c"
 dependencies = [
  "memchr",
- "unicode-width",
+ "unicode-width 0.1.14",
 ]
 
 [[package]]
@@ -4611,9 +4751,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
 [[package]]
 name = "syn"
-version = "2.0.87"
+version = "2.0.90"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d"
+checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4642,9 +4782,9 @@ dependencies = [
 
 [[package]]
 name = "tempfile"
-version = "3.13.0"
+version = "3.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b"
+checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c"
 dependencies = [
  "cfg-if",
  "fastrand",
@@ -4714,27 +4854,27 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.68"
+version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892"
+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
 dependencies = [
- "thiserror-impl 1.0.68",
+ "thiserror-impl 1.0.69",
 ]
 
 [[package]]
 name = "thiserror"
-version = "2.0.2"
+version = "2.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "037e29b009aa709f293b974da5cd33b15783c049e07f8435778ce8c4871525d8"
+checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47"
 dependencies = [
- "thiserror-impl 2.0.2",
+ "thiserror-impl 2.0.6",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.68"
+version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e"
+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4743,9 +4883,9 @@ dependencies = [
 
 [[package]]
 name = "thiserror-impl"
-version = "2.0.2"
+version = "2.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea4778c7e8ff768bdb32a58a2349903859fe719a320300d7d4ce8636f19a1e69"
+checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4784,9 +4924,9 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.36"
+version = "0.3.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885"
+checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21"
 dependencies = [
  "deranged",
  "itoa",
@@ -4805,9 +4945,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
 
 [[package]]
 name = "time-macros"
-version = "0.2.18"
+version = "0.2.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf"
+checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de"
 dependencies = [
  "num-conv",
  "time-core",
@@ -4832,6 +4972,16 @@ dependencies = [
  "zerovec",
 ]
 
+[[package]]
+name = "tinytemplate"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "tinyvec"
 version = "1.8.0"
@@ -4849,9 +4999,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.41.1"
+version = "1.42.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33"
+checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551"
 dependencies = [
  "backtrace",
  "bytes",
@@ -4914,20 +5064,19 @@ dependencies = [
 
 [[package]]
 name = "tokio-rustls"
-version = "0.26.0"
+version = "0.26.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4"
+checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37"
 dependencies = [
  "rustls 0.23.19",
- "rustls-pki-types",
  "tokio",
 ]
 
 [[package]]
 name = "tokio-util"
-version = "0.7.12"
+version = "0.7.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a"
+checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078"
 dependencies = [
  "bytes",
  "futures-core",
@@ -4948,7 +5097,7 @@ version = "0.22.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5"
 dependencies = [
- "indexmap 2.6.0",
+ "indexmap 2.7.0",
  "toml_datetime",
  "winnow",
 ]
@@ -4961,9 +5110,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
 
 [[package]]
 name = "tracing"
-version = "0.1.40"
+version = "0.1.41"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
+checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
 dependencies = [
  "pin-project-lite",
  "tracing-attributes",
@@ -4972,9 +5121,9 @@ dependencies = [
 
 [[package]]
 name = "tracing-attributes"
-version = "0.1.27"
+version = "0.1.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
+checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4983,9 +5132,9 @@ dependencies = [
 
 [[package]]
 name = "tracing-core"
-version = "0.1.32"
+version = "0.1.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
+checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c"
 dependencies = [
  "once_cell",
  "valuable",
@@ -5004,9 +5153,9 @@ dependencies = [
 
 [[package]]
 name = "tracing-subscriber"
-version = "0.3.18"
+version = "0.3.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b"
+checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008"
 dependencies = [
  "nu-ansi-term",
  "sharded-slab",
@@ -5072,9 +5221,9 @@ checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.13"
+version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe"
+checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
 
 [[package]]
 name = "unicode-normalization"
@@ -5103,6 +5252,12 @@ version = "0.1.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
 
+[[package]]
+name = "unicode-width"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"
+
 [[package]]
 name = "unsafe-libyaml"
 version = "0.2.11"
@@ -5117,9 +5272,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
 
 [[package]]
 name = "url"
-version = "2.5.3"
+version = "2.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada"
+checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60"
 dependencies = [
  "form_urlencoded",
  "idna",
@@ -5220,9 +5375,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.95"
+version = "0.2.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e"
+checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396"
 dependencies = [
  "cfg-if",
  "once_cell",
@@ -5231,13 +5386,12 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.95"
+version = "0.2.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358"
+checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79"
 dependencies = [
  "bumpalo",
  "log",
- "once_cell",
  "proc-macro2",
  "quote",
  "syn",
@@ -5246,21 +5400,22 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.45"
+version = "0.4.49"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b"
+checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2"
 dependencies = [
  "cfg-if",
  "js-sys",
+ "once_cell",
  "wasm-bindgen",
  "web-sys",
 ]
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.95"
+version = "0.2.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56"
+checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -5268,9 +5423,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.95"
+version = "0.2.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68"
+checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5281,9 +5436,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.95"
+version = "0.2.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d"
+checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6"
 
 [[package]]
 name = "wasm-streams"
@@ -5300,9 +5455,19 @@ dependencies = [
 
 [[package]]
 name = "web-sys"
-version = "0.3.72"
+version = "0.3.76"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "web-time"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112"
+checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -5341,7 +5506,7 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
 dependencies = [
- "windows-sys 0.48.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -5581,9 +5746,9 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049"
 
 [[package]]
 name = "yoke"
-version = "0.7.4"
+version = "0.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5"
+checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40"
 dependencies = [
  "serde",
  "stable_deref_trait",
@@ -5593,9 +5758,9 @@ dependencies = [
 
 [[package]]
 name = "yoke-derive"
-version = "0.7.4"
+version = "0.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95"
+checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5626,18 +5791,18 @@ dependencies = [
 
 [[package]]
 name = "zerofrom"
-version = "0.1.4"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55"
+checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e"
 dependencies = [
  "zerofrom-derive",
 ]
 
 [[package]]
 name = "zerofrom-derive"
-version = "0.1.4"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5"
+checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808"
 dependencies = [
  "proc-macro2",
  "quote",
diff --git a/optd-sqlplannertest/Cargo.toml b/optd-sqlplannertest/Cargo.toml
index 6adbd772..7ea9e0e4 100644
--- a/optd-sqlplannertest/Cargo.toml
+++ b/optd-sqlplannertest/Cargo.toml
@@ -13,7 +13,7 @@ repository = { workspace = true }
 [dependencies]
 clap = { version = "4.5.4", features = ["derive"] }
 anyhow = { version = "1", features = ["backtrace"] }
-sqlplannertest = "0.3"
+sqlplannertest = "0.4"
 async-trait = "0.1"
 datafusion-optd-cli = { path = "../datafusion-optd-cli", version = "43.0.0" }
 optd-datafusion-repr-adv-cost = { path = "../optd-datafusion-repr-adv-cost", version = "0.1" }
@@ -40,6 +40,14 @@ optd-datafusion-repr = { path = "../optd-datafusion-repr", version = "0.1" }
 itertools = "0.13"
 lazy_static = "1.4.0"
 
+[dev-dependencies]
+criterion = { version = "0.5.1", features = ["async_tokio"] }
+serde_yaml = "0.9"
+
 [[test]]
 name = "planner_test"
 harness = false
+
+[[bench]]
+name = "planner_bench"
+harness = false
diff --git a/optd-sqlplannertest/README.md b/optd-sqlplannertest/README.md
index 085dc455..8295c5d5 100644
--- a/optd-sqlplannertest/README.md
+++ b/optd-sqlplannertest/README.md
@@ -3,14 +3,55 @@
 These test cases use the [sqlplannertest](https://crates.io/crates/sqlplannertest) crate to execute SQL queries and inspect their output.
 They do not check whether a plan is correct, and instead rely on a text-based diff of the query's output to determine whether something is different than the expected output.
 
+We are also using this crate to generate benchmarks for evaluating optd's performance. With the help with the [criterion](https://crates.io/crates/criterion) crate, we can benchmark planning time and the execution time of physical plan produced by the optimizer.
 
 ## Execute Test Cases
 
+**Running all test cases**
+
 ```shell
 cargo test -p optd-sqlplannertest
 # or use nextest
 cargo nextest run -p optd-sqlplannertest
 ```
+
+**Running tests in specfic modules or files**
+
+```shell
+# Running all test cases in the tpch module
+cargo nextest run -p optd-sqlplannertest tpch
+# Running all test cases in the tests/subqueries/subquery_unnesting.yml
+cargo nextest run -p optd-sqlplannertest subquery::subquery_unnesting
+```
+
+## Executing Benchmarks
+
+There are two metrics we care about when evaluating 
+
+### Usage
+
+```shell
+# Benchmark all TPC-H queries with "bench" task enabled
+cargo bench --bench planner_bench tpch/
+
+# Benchmark TPC-H Q1
+cargo bench --bench planner_bench tpch/q1/
+
+# Benchmark TPC-H Q1 planning
+cargo bench --bench planner_bench tpch/q1/planning
+
+# Benchmark TPC-H Q1 execution
+cargo bench --bench planner_bench tpch/q1/execution
+
+# View the HTML report
+python3 -m http.server -d ./target/criterion/
+```
+
+### Limitations
+
+`planner_bench` can only handle `sqlplannertest` yaml-based test file with single test case.
+
+
 ## Add New Test Case
 
 To add a SQL query tests, create a YAML file in a subdir in "tests".
@@ -30,11 +71,11 @@ Each file can contain multiple tests that are executed in sequential order from
     - explain:logical_optd,physical_optd
   desc: Equality predicate
 ```
-| Name       | Description                                                        |
-| ---------- | ------------------------------------------------------------------ |
-| `sql`      | List of SQL statements to execute separate by newlines             |
-| `tasks`    | How to execute the SQL statements. See [Tasks](#tasks) below       |
-| `desc`     | (Optional) Text description of what the test cases represents      |
+| Name    | Description                                                   |
+| ------- | ------------------------------------------------------------- |
+| `sql`   | List of SQL statements to execute separate by newlines        |
+| `tasks` | How to execute the SQL statements. See [Tasks](#tasks) below  |
+| `desc`  | (Optional) Text description of what the test cases represents |
 
 After adding the YAML file, you then need to use the update command to automatically create the matching SQL file that contains the expected output of the test cases.
 
@@ -46,14 +87,16 @@ The following commands will automatically update all of them for you. You should
 ```shell
 # Update all test cases
 cargo run -p optd-sqlplannertest --bin planner_test_apply
-# or, supply a list of directories to update
-cargo run -p optd-sqlplannertest --bin planner_test_apply -- subqueries
+# or, supply a list of modules or files to update
+cargo run -p optd-sqlplannertest --bin planner_test_apply -- subqueries tpch::q1
 ```
 
 ## Tasks
 
 The `explain` and `execute` task will be run with datafusion's logical optimizer disabled. Each task has some toggleable flags to control its behavior.
 
+The `bench` task is only used in benchmarks. A test case can only be executed as a benchmark if a bench task exists.
+
 ### `execute` Task
 
 #### Flags
diff --git a/optd-sqlplannertest/benches/planner_bench.rs b/optd-sqlplannertest/benches/planner_bench.rs
new file mode 100644
index 00000000..5900c50b
--- /dev/null
+++ b/optd-sqlplannertest/benches/planner_bench.rs
@@ -0,0 +1,150 @@
+use std::{
+    future::Future,
+    path::{Path, PathBuf},
+};
+
+use anyhow::{bail, Context, Result};
+use criterion::{black_box, criterion_group, criterion_main, BatchSize, Criterion};
+use optd_sqlplannertest::bench_helper::{
+    bench_run, bench_setup, ExecutionBenchRunner, PlannerBenchRunner, PlanningBenchRunner,
+};
+use sqlplannertest::{discover_tests_with_selections, parse_test_cases, TestCase};
+use tokio::runtime::Runtime;
+
+fn criterion_benchmark(c: &mut Criterion) {
+    let selection = "tpch";
+    let selections = vec![selection.to_string()];
+
+    let tests_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests");
+    planner_bench_runner(
+        &tests_dir,
+        || async { PlanningBenchRunner::new().await },
+        &selections,
+        c,
+    )
+    .unwrap();
+
+    let path = tests_dir.join(format!("{selection}/bench_populate.sql"));
+    let populate_sql = std::fs::read_to_string(&path)
+        .with_context(|| format!("failed to read {}", path.display()))
+        .unwrap();
+
+    planner_bench_runner(
+        &tests_dir,
+        move || {
+            let populate_sql = populate_sql.clone();
+            async { ExecutionBenchRunner::new(populate_sql).await }
+        },
+        &selections,
+        c,
+    )
+    .unwrap();
+}
+
+/// Discovers and bench each test case.
+///
+/// The user needs to provide a runner function that creates a runner that
+/// implements the [`PlannerBenchRunner`] trait.
+///
+/// A test case will be selected if:
+///
+/// 1. it's included in the `tests_dir` as part of `selections`.
+/// 2. has `bench` listed in the task list.
+///
+/// ## Limitation
+///
+/// Currently only accept sqlplannertest files with single test case.
+fn planner_bench_runner<F, Ft, R>(
+    tests_dir: impl AsRef<Path>,
+    runner_fn: F,
+    selections: &[String],
+    c: &mut Criterion,
+) -> Result<()>
+where
+    F: Fn() -> Ft + Send + Sync + 'static + Clone,
+    Ft: Future<Output = Result<R>> + Send,
+    R: PlannerBenchRunner + 'static,
+{
+    let tests = discover_tests_with_selections(&tests_dir, selections)?
+        .map(|path| {
+            let path = path?;
+            let relative_path = path
+                .strip_prefix(&tests_dir)
+                .context("unable to relative path")?
+                .as_os_str();
+            let testname = relative_path
+                .to_str()
+                .context("unable to convert to string")?
+                .to_string();
+            Ok::<_, anyhow::Error>((path, testname))
+        })
+        .collect::<Result<Vec<_>, _>>()?;
+
+    for (path, testname) in tests {
+        bench_runner(path, testname, runner_fn.clone(), c)?;
+    }
+
+    Ok(())
+}
+
+/// Bench runner for a test case.
+fn bench_runner<F, Ft, R>(
+    path: PathBuf,
+    testname: String,
+    runner_fn: F,
+    c: &mut Criterion,
+) -> Result<()>
+where
+    F: Fn() -> Ft + Send + Sync + 'static + Clone,
+    Ft: Future<Output = Result<R>> + Send,
+    R: PlannerBenchRunner,
+{
+    fn build_runtime() -> Runtime {
+        tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .unwrap()
+    }
+
+    let testcases = std::fs::read(&path)?;
+    let testcases: Vec<TestCase> = serde_yaml::from_slice(&testcases)?;
+
+    let testcases = parse_test_cases(
+        {
+            let mut path = path.clone();
+            path.pop();
+            path
+        },
+        testcases,
+    )?;
+
+    if testcases.len() != 1 {
+        bail!(
+            "planner_bench can only handle sqlplannertest yml file with one test cases, {} has {}",
+            path.display(),
+            testcases.len()
+        );
+    }
+
+    let testcase = &testcases[0];
+
+    let should_bench = testcase.tasks.iter().any(|x| x.starts_with("bench"));
+    if should_bench {
+        let mut group = c.benchmark_group(testname.strip_suffix(".yml").unwrap());
+        let runtime = build_runtime();
+        group.bench_function(R::BENCH_NAME, |b| {
+            b.iter_batched(
+                || bench_setup(&runtime, runner_fn.clone(), testcase),
+                |(runner, input, flags)| {
+                    bench_run(&runtime, runner, black_box(input), testcase, &flags)
+                },
+                BatchSize::PerIteration,
+            );
+        });
+        group.finish();
+    }
+    Ok(())
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);
diff --git a/optd-sqlplannertest/src/bench_helper.rs b/optd-sqlplannertest/src/bench_helper.rs
new file mode 100644
index 00000000..3831015c
--- /dev/null
+++ b/optd-sqlplannertest/src/bench_helper.rs
@@ -0,0 +1,64 @@
+pub mod execution;
+pub mod planning;
+
+use std::future::Future;
+
+use crate::TestFlags;
+use anyhow::Result;
+use tokio::runtime::Runtime;
+
+pub use execution::ExecutionBenchRunner;
+pub use planning::PlanningBenchRunner;
+
+pub trait PlannerBenchRunner {
+    /// Describes what the benchmark is evaluating.
+    const BENCH_NAME: &str;
+    /// Benchmark's input.
+    type BenchInput;
+
+    /// Setups the necessary environment for the benchmark based on the test case.
+    /// Returns the input needed for the benchmark.
+    fn setup(
+        &mut self,
+        test_case: &sqlplannertest::ParsedTestCase,
+    ) -> impl std::future::Future<Output = Result<(Self::BenchInput, TestFlags)>> + Send;
+
+    /// Runs the actual benchmark based on the test case and input.
+    fn bench(
+        self,
+        input: Self::BenchInput,
+        test_case: &sqlplannertest::ParsedTestCase,
+        flags: &TestFlags,
+    ) -> impl std::future::Future<Output = Result<()>> + Send;
+}
+
+/// Sync wrapper for [`PlannerBenchRunner::setup`]
+pub fn bench_setup<F, Ft, R>(
+    runtime: &Runtime,
+    runner_fn: F,
+    testcase: &sqlplannertest::ParsedTestCase,
+) -> (R, R::BenchInput, TestFlags)
+where
+    F: Fn() -> Ft + Send + Sync + 'static + Clone,
+    Ft: Future<Output = Result<R>> + Send,
+    R: PlannerBenchRunner,
+{
+    runtime.block_on(async {
+        let mut runner = runner_fn().await.unwrap();
+        let (input, flags) = runner.setup(testcase).await.unwrap();
+        (runner, input, flags)
+    })
+}
+
+/// Sync wrapper for [`PlannerBenchRunner::bench`]
+pub fn bench_run<R>(
+    runtime: &Runtime,
+    runner: R,
+    input: R::BenchInput,
+    testcase: &sqlplannertest::ParsedTestCase,
+    flags: &TestFlags,
+) where
+    R: PlannerBenchRunner,
+{
+    runtime.block_on(async { runner.bench(input, testcase, flags).await.unwrap() });
+}
diff --git a/optd-sqlplannertest/src/bench_helper/execution.rs b/optd-sqlplannertest/src/bench_helper/execution.rs
new file mode 100644
index 00000000..7baa5b5f
--- /dev/null
+++ b/optd-sqlplannertest/src/bench_helper/execution.rs
@@ -0,0 +1,71 @@
+use std::sync::Arc;
+
+use crate::{extract_flags, DatafusionDBMS, TestFlags};
+use anyhow::Result;
+use datafusion::{execution::TaskContext, physical_plan::ExecutionPlan};
+
+use super::PlannerBenchRunner;
+
+/// A benchmark runner for evaluating execution time of optimized plan.
+pub struct ExecutionBenchRunner {
+    pub dbms: DatafusionDBMS,
+    /// DDLs and DMLs to populate the tables.
+    pub populate_sql: String,
+}
+
+impl ExecutionBenchRunner {
+    pub async fn new(populate_sql: String) -> Result<Self> {
+        Ok(ExecutionBenchRunner {
+            dbms: DatafusionDBMS::new().await?,
+            populate_sql,
+        })
+    }
+}
+
+/// With physical execution plan as input,
+/// measures the time it takes to execute the plan generated by the optimizer.
+impl PlannerBenchRunner for ExecutionBenchRunner {
+    const BENCH_NAME: &str = "execution";
+    type BenchInput = Vec<(Arc<dyn ExecutionPlan>, Arc<TaskContext>)>;
+    async fn setup(
+        &mut self,
+        test_case: &sqlplannertest::ParsedTestCase,
+    ) -> Result<(Self::BenchInput, TestFlags)> {
+        for sql in &test_case.before_sql {
+            self.dbms.execute(sql, &TestFlags::default()).await?;
+        }
+
+        // Populate the existing tables.
+        for sql in self.populate_sql.split(";\n") {
+            self.dbms.execute(sql, &TestFlags::default()).await?;
+        }
+
+        let bench_task = test_case
+            .tasks
+            .iter()
+            .find(|x| x.starts_with("bench"))
+            .unwrap();
+        let flags = extract_flags(bench_task)?;
+
+        self.dbms.setup(&flags).await?;
+        let statements = self.dbms.parse_sql(&test_case.sql).await?;
+
+        let mut physical_plans = Vec::new();
+        for statement in statements {
+            physical_plans.push(self.dbms.create_physical_plan(statement, &flags).await?);
+        }
+
+        Ok((physical_plans, flags))
+    }
+    async fn bench(
+        self,
+        input: Self::BenchInput,
+        _test_case: &sqlplannertest::ParsedTestCase,
+        _flags: &TestFlags,
+    ) -> Result<()> {
+        for (physical_plan, task_ctx) in input {
+            self.dbms.execute_physical(physical_plan, task_ctx).await?;
+        }
+        Ok(())
+    }
+}
diff --git a/optd-sqlplannertest/src/bench_helper/planning.rs b/optd-sqlplannertest/src/bench_helper/planning.rs
new file mode 100644
index 00000000..4546b467
--- /dev/null
+++ b/optd-sqlplannertest/src/bench_helper/planning.rs
@@ -0,0 +1,52 @@
+use std::collections::VecDeque;
+
+use crate::{extract_flags, DatafusionDBMS, TestFlags};
+use anyhow::Result;
+use datafusion::sql::parser::Statement;
+
+use super::PlannerBenchRunner;
+
+/// A benchmark runner for evaluating optimizer planning time.
+pub struct PlanningBenchRunner(DatafusionDBMS);
+
+impl PlanningBenchRunner {
+    pub async fn new() -> Result<Self> {
+        Ok(PlanningBenchRunner(DatafusionDBMS::new().await?))
+    }
+}
+
+/// With parsed statements as input,
+/// measures the time it takes to generate datafusion physical plans.
+impl PlannerBenchRunner for PlanningBenchRunner {
+    const BENCH_NAME: &str = "planning";
+    type BenchInput = VecDeque<Statement>;
+    async fn setup(
+        &mut self,
+        test_case: &sqlplannertest::ParsedTestCase,
+    ) -> Result<(Self::BenchInput, TestFlags)> {
+        for sql in &test_case.before_sql {
+            self.0.execute(sql, &TestFlags::default()).await?;
+        }
+        let bench_task = test_case
+            .tasks
+            .iter()
+            .find(|x| x.starts_with("bench"))
+            .unwrap();
+        let flags = extract_flags(bench_task)?;
+        self.0.setup(&flags).await?;
+        let statements = self.0.parse_sql(&test_case.sql).await?;
+
+        Ok((statements, flags))
+    }
+    async fn bench(
+        self,
+        input: Self::BenchInput,
+        _test_case: &sqlplannertest::ParsedTestCase,
+        flags: &TestFlags,
+    ) -> Result<()> {
+        for stmt in input {
+            self.0.create_physical_plan(stmt, flags).await?;
+        }
+        Ok(())
+    }
+}
diff --git a/optd-sqlplannertest/src/bin/planner_test_apply.rs b/optd-sqlplannertest/src/bin/planner_test_apply.rs
index f555bfba..6afca93c 100644
--- a/optd-sqlplannertest/src/bin/planner_test_apply.rs
+++ b/optd-sqlplannertest/src/bin/planner_test_apply.rs
@@ -12,8 +12,8 @@ use sqlplannertest::PlannerTestApplyOptions;
 #[derive(Parser)]
 #[command(version, about, long_about = None)]
 struct Cli {
-    /// Optional list of directories to apply the test; if empty, apply all tests
-    directories: Vec<String>,
+    /// Optional list of test modules or test files to apply the test; if empty, apply all tests
+    selections: Vec<String>,
     /// Use the advanced cost model
     #[clap(long)]
     enable_advanced_cost_model: bool,
@@ -29,40 +29,23 @@ async fn main() -> Result<()> {
     let cli = Cli::parse();
 
     let enable_advanced_cost_model = cli.enable_advanced_cost_model;
-    let opts = PlannerTestApplyOptions { serial: cli.serial };
+    let opts = PlannerTestApplyOptions {
+        serial: cli.serial,
+        selections: cli.selections,
+    };
+
+    sqlplannertest::planner_test_apply_with_options(
+        Path::new(env!("CARGO_MANIFEST_DIR")).join("tests"),
+        move || async move {
+            if enable_advanced_cost_model {
+                optd_sqlplannertest::DatafusionDBMS::new_advanced_cost().await
+            } else {
+                optd_sqlplannertest::DatafusionDBMS::new().await
+            }
+        },
+        opts,
+    )
+    .await?;
 
-    if cli.directories.is_empty() {
-        println!("Running all tests");
-        sqlplannertest::planner_test_apply_with_options(
-            Path::new(env!("CARGO_MANIFEST_DIR")).join("tests"),
-            move || async move {
-                if enable_advanced_cost_model {
-                    optd_sqlplannertest::DatafusionDBMS::new_advanced_cost().await
-                } else {
-                    optd_sqlplannertest::DatafusionDBMS::new().await
-                }
-            },
-            opts,
-        )
-        .await?;
-    } else {
-        for directory in cli.directories {
-            println!("Running tests in {}", directory);
-            sqlplannertest::planner_test_apply_with_options(
-                Path::new(env!("CARGO_MANIFEST_DIR"))
-                    .join("tests")
-                    .join(directory),
-                move || async move {
-                    if enable_advanced_cost_model {
-                        optd_sqlplannertest::DatafusionDBMS::new_advanced_cost().await
-                    } else {
-                        optd_sqlplannertest::DatafusionDBMS::new().await
-                    }
-                },
-                opts.clone(),
-            )
-            .await?;
-        }
-    }
     Ok(())
 }
diff --git a/optd-sqlplannertest/src/lib.rs b/optd-sqlplannertest/src/lib.rs
index 5d572aa0..3ea88975 100644
--- a/optd-sqlplannertest/src/lib.rs
+++ b/optd-sqlplannertest/src/lib.rs
@@ -3,13 +3,18 @@
 // Use of this source code is governed by an MIT-style license that can be found in the LICENSE file or at
 // https://opensource.org/licenses/MIT.
 
-use std::collections::HashSet;
+pub mod bench_helper;
+
+use std::collections::{HashSet, VecDeque};
 use std::sync::Arc;
 
+use datafusion::arrow::array::RecordBatch;
 use datafusion::arrow::util::display::{ArrayFormatter, FormatOptions};
 use datafusion::catalog::CatalogProviderList;
+use datafusion::execution::TaskContext;
+use datafusion::physical_plan::ExecutionPlan;
 use datafusion::prelude::SessionContext;
-use datafusion::sql::parser::DFParser;
+use datafusion::sql::parser::{DFParser, Statement};
 use datafusion::sql::sqlparser::dialect::GenericDialect;
 use datafusion_optd_cli::helper::unescape_input;
 use itertools::Itertools;
@@ -76,76 +81,107 @@ impl DatafusionDBMS {
         Ok((ctx, optimizer))
     }
 
-    pub(crate) async fn execute(&self, sql: &str, flags: &TestFlags) -> Result<Vec<Vec<String>>> {
-        {
-            let mut guard = self
-                .optd_optimizer
-                .as_ref()
-                .unwrap()
-                .optimizer
-                .lock()
-                .unwrap();
-            let optimizer = guard.as_mut().unwrap().optd_optimizer_mut();
-            if flags.panic_on_budget {
-                optimizer.panic_on_explore_limit(true);
-            } else {
-                optimizer.panic_on_explore_limit(false);
-            }
-            if flags.disable_pruning {
-                optimizer.disable_pruning(true);
-            } else {
-                optimizer.disable_pruning(false);
+    /// Sets up test specific behaviors based on `flags`.
+    pub(crate) async fn setup(&self, flags: &TestFlags) -> Result<()> {
+        let mut guard = self
+            .optd_optimizer
+            .as_ref()
+            .unwrap()
+            .optimizer
+            .lock()
+            .unwrap();
+        let optimizer = guard.as_mut().unwrap().optd_optimizer_mut();
+        if flags.panic_on_budget {
+            optimizer.panic_on_explore_limit(true);
+        } else {
+            optimizer.panic_on_explore_limit(false);
+        }
+        if flags.disable_pruning {
+            optimizer.disable_pruning(true);
+        } else {
+            optimizer.disable_pruning(false);
+        }
+        let rules = optimizer.rules();
+        if flags.enable_logical_rules.is_empty() {
+            for r in 0..rules.len() {
+                optimizer.enable_rule(r);
             }
-            let rules = optimizer.rules();
-            if flags.enable_logical_rules.is_empty() {
-                for r in 0..rules.len() {
-                    optimizer.enable_rule(r);
-                }
-                guard.as_mut().unwrap().enable_heuristic(true);
-            } else {
-                for (rule_id, rule) in rules.as_ref().iter().enumerate() {
-                    if rule.is_impl_rule() {
-                        optimizer.enable_rule(rule_id);
-                    } else {
-                        optimizer.disable_rule(rule_id);
-                    }
-                }
-                let mut rules_to_enable = flags
-                    .enable_logical_rules
-                    .iter()
-                    .map(|x| x.as_str())
-                    .collect::<HashSet<_>>();
-                for (rule_id, rule) in rules.as_ref().iter().enumerate() {
-                    if rules_to_enable.remove(rule.name()) {
-                        optimizer.enable_rule(rule_id);
-                    }
+            guard.as_mut().unwrap().enable_heuristic(true);
+        } else {
+            for (rule_id, rule) in rules.as_ref().iter().enumerate() {
+                if rule.is_impl_rule() {
+                    optimizer.enable_rule(rule_id);
+                } else {
+                    optimizer.disable_rule(rule_id);
                 }
-                if !rules_to_enable.is_empty() {
-                    bail!("Unknown logical rule: {:?}", rules_to_enable);
+            }
+            let mut rules_to_enable = flags
+                .enable_logical_rules
+                .iter()
+                .map(|x| x.as_str())
+                .collect::<HashSet<_>>();
+            for (rule_id, rule) in rules.as_ref().iter().enumerate() {
+                if rules_to_enable.remove(rule.name()) {
+                    optimizer.enable_rule(rule_id);
                 }
-                guard.as_mut().unwrap().enable_heuristic(false);
             }
+            if !rules_to_enable.is_empty() {
+                bail!("Unknown logical rule: {:?}", rules_to_enable);
+            }
+            guard.as_mut().unwrap().enable_heuristic(false);
         }
+
+        Ok(())
+    }
+
+    /// Parses input SQL string into statements.
+    pub async fn parse_sql(&self, sql: &str) -> Result<VecDeque<Statement>> {
         let sql = unescape_input(sql)?;
         let dialect = Box::new(GenericDialect);
         let statements = DFParser::parse_sql_with_dialect(&sql, dialect.as_ref())?;
-        let mut result = Vec::new();
-        for statement in statements {
-            let df = if flags.enable_df_logical {
-                let plan = self
-                    .use_df_logical_ctx
-                    .state()
-                    .statement_to_plan(statement)
-                    .await?;
-                self.use_df_logical_ctx.execute_logical_plan(plan).await?
-            } else {
-                let plan = self.ctx.state().statement_to_plan(statement).await?;
+        Ok(statements)
+    }
+
+    /// Creates a physical execution plan with associated task context from a SQL statement.
+    pub(crate) async fn create_physical_plan(
+        &self,
+        stmt: Statement,
+        flags: &TestFlags,
+    ) -> Result<(Arc<dyn ExecutionPlan>, Arc<TaskContext>)> {
+        let df = if flags.enable_df_logical {
+            let plan = self
+                .use_df_logical_ctx
+                .state()
+                .statement_to_plan(stmt)
+                .await?;
+            self.use_df_logical_ctx.execute_logical_plan(plan).await?
+        } else {
+            let plan = self.ctx.state().statement_to_plan(stmt).await?;
 
-                self.ctx.execute_logical_plan(plan).await?
-            };
+            self.ctx.execute_logical_plan(plan).await?
+        };
+        let task_ctx = Arc::new(df.task_ctx());
+        let plan = df.create_physical_plan().await?;
+        Ok((plan, task_ctx))
+    }
 
-            let batches = df.collect().await?;
+    /// Executes the physical [`ExecutionPlan`] and collect the results in memory.
+    pub(crate) async fn execute_physical(
+        &self,
+        plan: Arc<dyn ExecutionPlan>,
+        task_ctx: Arc<TaskContext>,
+    ) -> Result<Vec<RecordBatch>> {
+        let batches = datafusion::physical_plan::collect(plan, task_ctx).await?;
+        Ok(batches)
+    }
 
+    pub async fn execute(&self, sql: &str, flags: &TestFlags) -> Result<Vec<Vec<String>>> {
+        self.setup(flags).await?;
+        let statements = self.parse_sql(sql).await?;
+        let mut result = Vec::new();
+        for statement in statements {
+            let (plan, task_ctx) = self.create_physical_plan(statement, flags).await?;
+            let batches = self.execute_physical(plan, task_ctx).await?;
             let options = FormatOptions::default().with_null("NULL");
 
             for batch in batches {
@@ -307,7 +343,7 @@ lazy_static! {
 }
 
 #[derive(Default, Debug)]
-struct TestFlags {
+pub struct TestFlags {
     verbose: bool,
     enable_df_logical: bool,
     enable_logical_rules: Vec<String>,
@@ -319,7 +355,7 @@ struct TestFlags {
 /// Extract the flags from a task. The flags are specified in square brackets.
 /// For example, the flags for the task `explain[use_df_logical, verbose]` are `["use_df_logical",
 /// "verbose"]`.
-fn extract_flags(task: &str) -> Result<TestFlags> {
+pub fn extract_flags(task: &str) -> Result<TestFlags> {
     if let Some(captures) = FLAGS_REGEX.captures(task) {
         let flags = captures
             .get(1)
diff --git a/optd-sqlplannertest/tests/tpch/bench_populate.sql b/optd-sqlplannertest/tests/tpch/bench_populate.sql
new file mode 100644
index 00000000..a37f1de1
--- /dev/null
+++ b/optd-sqlplannertest/tests/tpch/bench_populate.sql
@@ -0,0 +1,18 @@
+-- A special version of DDL/DML for popluating the TPC-H tables, sf=0.01
+
+CREATE EXTERNAL TABLE customer_tbl STORED AS CSV OPTIONS (HAS_HEADER false, DELIMITER '|') LOCATION '../datafusion-optd-cli/tpch-sf0_01/customer.csv';
+insert into customer select column_1, column_2, column_3, column_4, column_5, column_6, column_7, column_8 from customer_tbl;
+CREATE EXTERNAL TABLE lineitem_tbl STORED AS CSV OPTIONS (HAS_HEADER false, DELIMITER '|') LOCATION '../datafusion-optd-cli/tpch-sf0_01/lineitem.csv';
+insert into lineitem select column_1, column_2, column_3, column_4, column_5, column_6, column_7, column_8, column_9, column_10, column_11, column_12, column_13, column_14, column_15, column_16 from lineitem_tbl;
+CREATE EXTERNAL TABLE nation_tbl STORED AS CSV OPTIONS (HAS_HEADER false, DELIMITER '|') LOCATION '../datafusion-optd-cli/tpch-sf0_01/nation.csv';
+insert into nation select column_1, column_2, column_3, column_4 from nation_tbl;
+CREATE EXTERNAL TABLE orders_tbl STORED AS CSV OPTIONS (HAS_HEADER false, DELIMITER '|') LOCATION '../datafusion-optd-cli/tpch-sf0_01/orders.csv';
+insert into orders select column_1, column_2, column_3, column_4, column_5, column_6, column_7, column_8, column_9 from orders_tbl;
+CREATE EXTERNAL TABLE part_tbl STORED AS CSV OPTIONS (HAS_HEADER false, DELIMITER '|') LOCATION '../datafusion-optd-cli/tpch-sf0_01/part.csv';
+insert into part select column_1, column_2, column_3, column_4, column_5, column_6, column_7, column_8, column_9 from part_tbl;
+CREATE EXTERNAL TABLE partsupp_tbl STORED AS CSV OPTIONS (HAS_HEADER false, DELIMITER '|') LOCATION '../datafusion-optd-cli/tpch-sf0_01/partsupp.csv';
+insert into partsupp select column_1, column_2, column_3, column_4, column_5 from partsupp_tbl;
+CREATE EXTERNAL TABLE region_tbl STORED AS CSV OPTIONS (HAS_HEADER false, DELIMITER '|') LOCATION '../datafusion-optd-cli/tpch-sf0_01/region.csv';
+insert into region select column_1, column_2, column_3 from region_tbl;
+CREATE EXTERNAL TABLE supplier_tbl STORED AS CSV OPTIONS (HAS_HEADER false, DELIMITER '|') LOCATION '../datafusion-optd-cli/tpch-sf0_01/supplier.csv';
+insert into supplier select column_1, column_2, column_3, column_4, column_5, column_6, column_7 from supplier_tbl;
diff --git a/optd-sqlplannertest/tests/tpch/q1.yml b/optd-sqlplannertest/tests/tpch/q1.yml
index 8746ed3d..b9e1dc7c 100644
--- a/optd-sqlplannertest/tests/tpch/q1.yml
+++ b/optd-sqlplannertest/tests/tpch/q1.yml
@@ -22,3 +22,4 @@
   before: ["include_sql:schema.sql"]
   tasks:
     - explain:logical_optd,physical_optd
+    - bench
diff --git a/optd-sqlplannertest/tests/tpch/q10.yml b/optd-sqlplannertest/tests/tpch/q10.yml
index 726faa2b..69d96632 100644
--- a/optd-sqlplannertest/tests/tpch/q10.yml
+++ b/optd-sqlplannertest/tests/tpch/q10.yml
@@ -35,3 +35,4 @@
   before: ["include_sql:schema.sql"]
   tasks:
     - explain:logical_optd,physical_optd
+    - bench
diff --git a/optd-sqlplannertest/tests/tpch/q11.yml b/optd-sqlplannertest/tests/tpch/q11.yml
index b7b1038e..22a2860c 100644
--- a/optd-sqlplannertest/tests/tpch/q11.yml
+++ b/optd-sqlplannertest/tests/tpch/q11.yml
@@ -30,3 +30,4 @@
   before: ["include_sql:schema.sql"]
   tasks:
     - explain:logical_optd,physical_optd
+    - bench
diff --git a/optd-sqlplannertest/tests/tpch/q12.yml b/optd-sqlplannertest/tests/tpch/q12.yml
index ccbfa2ca..c115725c 100644
--- a/optd-sqlplannertest/tests/tpch/q12.yml
+++ b/optd-sqlplannertest/tests/tpch/q12.yml
@@ -25,3 +25,4 @@
   before: ["include_sql:schema.sql"]
   tasks:
     - explain:logical_optd,physical_optd
+    - bench
diff --git a/optd-sqlplannertest/tests/tpch/q13.planner.sql b/optd-sqlplannertest/tests/tpch/q13.planner.sql
new file mode 100644
index 00000000..5451f497
--- /dev/null
+++ b/optd-sqlplannertest/tests/tpch/q13.planner.sql
@@ -0,0 +1,74 @@
+-- TPC-H Q13
+select
+    c_count,
+    count(*) as custdist
+from
+    (
+        select
+            c_custkey,
+            count(o_orderkey)
+        from
+            customer left outer join orders on
+                c_custkey = o_custkey
+                and o_comment not like '%special%requests%'
+        group by
+            c_custkey
+    ) as c_orders (c_custkey, c_count)
+group by
+    c_count
+order by
+    custdist desc,
+    c_count desc;
+
+/*
+LogicalSort
+├── exprs:
+│   ┌── SortOrder { order: Desc }
+│   │   └── #1
+│   └── SortOrder { order: Desc }
+│       └── #0
+└── LogicalProjection { exprs: [ #0, #1 ] }
+    └── LogicalAgg
+        ├── exprs:Agg(Count)
+        │   └── [ 1(i64) ]
+        ├── groups: [ #1 ]
+        └── LogicalProjection { exprs: [ #0, #1 ] }
+            └── LogicalProjection { exprs: [ #0, #1 ] }
+                └── LogicalAgg
+                    ├── exprs:Agg(Count)
+                    │   └── [ #8 ]
+                    ├── groups: [ #0 ]
+                    └── LogicalJoin
+                        ├── join_type: LeftOuter
+                        ├── cond:And
+                        │   ├── Eq
+                        │   │   ├── #0
+                        │   │   └── #9
+                        │   └── Like { expr: #16, pattern: "%special%requests%", negated: true, case_insensitive: false }
+                        ├── LogicalScan { table: customer }
+                        └── LogicalScan { table: orders }
+PhysicalSort
+├── exprs:
+│   ┌── SortOrder { order: Desc }
+│   │   └── #1
+│   └── SortOrder { order: Desc }
+│       └── #0
+└── PhysicalAgg
+    ├── aggrs:Agg(Count)
+    │   └── [ 1(i64) ]
+    ├── groups: [ #1 ]
+    └── PhysicalAgg
+        ├── aggrs:Agg(Count)
+        │   └── [ #8 ]
+        ├── groups: [ #0 ]
+        └── PhysicalNestedLoopJoin
+            ├── join_type: LeftOuter
+            ├── cond:And
+            │   ├── Eq
+            │   │   ├── #0
+            │   │   └── #9
+            │   └── Like { expr: #16, pattern: "%special%requests%", negated: true, case_insensitive: false }
+            ├── PhysicalScan { table: customer }
+            └── PhysicalScan { table: orders }
+*/
+
diff --git a/optd-sqlplannertest/tests/tpch/q13.yml b/optd-sqlplannertest/tests/tpch/q13.yml
new file mode 100644
index 00000000..a960b155
--- /dev/null
+++ b/optd-sqlplannertest/tests/tpch/q13.yml
@@ -0,0 +1,26 @@
+- sql: |
+    select
+        c_count,
+        count(*) as custdist
+    from
+        (
+            select
+                c_custkey,
+                count(o_orderkey)
+            from
+                customer left outer join orders on
+                    c_custkey = o_custkey
+                    and o_comment not like '%special%requests%'
+            group by
+                c_custkey
+        ) as c_orders (c_custkey, c_count)
+    group by
+        c_count
+    order by
+        custdist desc,
+        c_count desc;
+  desc: TPC-H Q13
+  before: ["include_sql:schema.sql"]
+  tasks:
+    - explain:logical_optd,physical_optd
+    - bench
diff --git a/optd-sqlplannertest/tests/tpch/q14.yml b/optd-sqlplannertest/tests/tpch/q14.yml
index 35d0b620..c8541fa1 100644
--- a/optd-sqlplannertest/tests/tpch/q14.yml
+++ b/optd-sqlplannertest/tests/tpch/q14.yml
@@ -14,3 +14,4 @@
   before: ["include_sql:schema.sql"]
   tasks:
     - explain:logical_optd,physical_optd
+    - bench
diff --git a/optd-sqlplannertest/tests/tpch/q16.yml.disabled b/optd-sqlplannertest/tests/tpch/q16.yml.disabled
new file mode 100644
index 00000000..dfcb58ae
--- /dev/null
+++ b/optd-sqlplannertest/tests/tpch/q16.yml.disabled
@@ -0,0 +1,35 @@
+- sql: |
+    select
+        p_brand,
+        p_type,
+        p_size,
+        count(distinct ps_suppkey) as supplier_cnt
+    from
+        partsupp,
+        part
+    where
+        p_partkey = ps_partkey
+        and p_brand <> 'Brand#45'
+        and p_type not like 'MEDIUM POLISHED%'
+        and p_size in (49, 14, 23, 45, 19, 3, 36, 9)
+        and ps_suppkey not in (
+            select
+                s_suppkey
+            from
+                supplier
+            where
+                s_comment like '%Customer%Complaints%'
+        )
+    group by
+        p_brand,
+        p_type,
+        p_size
+    order by
+        supplier_cnt desc,
+        p_brand,
+        p_type,
+        p_size;
+  desc: TPC-H Q16
+  before: ["include_sql:schema.sql"]
+  tasks:
+    - explain:logical_optd,physical_optd
diff --git a/optd-sqlplannertest/tests/tpch/q18.yml.disabled b/optd-sqlplannertest/tests/tpch/q18.yml.disabled
new file mode 100644
index 00000000..50218463
--- /dev/null
+++ b/optd-sqlplannertest/tests/tpch/q18.yml.disabled
@@ -0,0 +1,38 @@
+- sql: |
+    select
+        c_name,
+        c_custkey,
+        o_orderkey,
+        o_orderdate,
+        o_totalprice,
+        sum(l_quantity)
+    from
+        customer,
+        orders,
+        lineitem
+    where
+        o_orderkey in (
+            select
+                l_orderkey
+            from
+                lineitem
+            group by
+                l_orderkey having
+                    sum(l_quantity) > 250 -- original: 300
+        )
+        and c_custkey = o_custkey
+        and o_orderkey = l_orderkey
+    group by
+        c_name,
+        c_custkey,
+        o_orderkey,
+        o_orderdate,
+        o_totalprice
+    order by
+        o_totalprice desc,
+        o_orderdate
+    limit 100;
+  desc: TPC-H Q18
+  before: ["include_sql:schema.sql"]
+  tasks:
+    - explain:logical_optd,physical_optd
diff --git a/optd-sqlplannertest/tests/tpch/q2.yml b/optd-sqlplannertest/tests/tpch/q2.yml
index d7bf0b17..28510116 100644
--- a/optd-sqlplannertest/tests/tpch/q2.yml
+++ b/optd-sqlplannertest/tests/tpch/q2.yml
@@ -47,3 +47,4 @@
   before: ["include_sql:schema.sql"]
   tasks:
     - explain:logical_optd,optimized_logical_optd,physical_optd
+
diff --git a/optd-sqlplannertest/tests/tpch/q20.yml.disabled b/optd-sqlplannertest/tests/tpch/q20.yml.disabled
new file mode 100644
index 00000000..e94ebb5e
--- /dev/null
+++ b/optd-sqlplannertest/tests/tpch/q20.yml.disabled
@@ -0,0 +1,42 @@
+- sql: |
+    select
+        s_name,
+        s_address
+    from
+        supplier,
+        nation
+    where
+        s_suppkey in (
+            select
+                ps_suppkey
+            from
+                partsupp
+            where
+                ps_partkey in (
+                    select
+                        p_partkey
+                    from
+                        part
+                    where
+                        p_name like 'indian%'
+                )
+                and ps_availqty > (
+                    select
+                        0.5 * sum(l_quantity)
+                    from
+                        lineitem
+                    where
+                        l_partkey = ps_partkey
+                        and l_suppkey = ps_suppkey
+                        and l_shipdate >= date '1996-01-01'
+                        and l_shipdate < date '1996-01-01' + interval '1' year
+                )
+        )
+        and s_nationkey = n_nationkey
+        and n_name = 'IRAQ'
+    order by
+        s_name;
+  desc: TPC-H Q20
+  before: ["include_sql:schema.sql"]
+  tasks:
+    - explain:logical_optd,physical_optd
diff --git a/optd-sqlplannertest/tests/tpch/q21.yml.disabled b/optd-sqlplannertest/tests/tpch/q21.yml.disabled
new file mode 100644
index 00000000..c239796c
--- /dev/null
+++ b/optd-sqlplannertest/tests/tpch/q21.yml.disabled
@@ -0,0 +1,45 @@
+- sql: |
+	select
+		s_name,
+		count(*) as numwait
+	from
+		supplier,
+		lineitem l1,
+		orders,
+		nation
+	where
+		s_suppkey = l1.l_suppkey
+		and o_orderkey = l1.l_orderkey
+		and o_orderstatus = 'F'
+		and l1.l_receiptdate > l1.l_commitdate
+		and exists (
+			select
+				*
+			from
+				lineitem l2
+			where
+				l2.l_orderkey = l1.l_orderkey
+				and l2.l_suppkey <> l1.l_suppkey
+		)
+		and not exists (
+			select
+				*
+			from
+				lineitem l3
+			where
+				l3.l_orderkey = l1.l_orderkey
+				and l3.l_suppkey <> l1.l_suppkey
+				and l3.l_receiptdate > l3.l_commitdate
+		)
+		and s_nationkey = n_nationkey
+		and n_name = 'SAUDI ARABIA'
+	group by
+		s_name
+	order by
+		numwait desc,
+		s_name
+	limit 100;
+  desc: TPC-H Q21
+  before: ["include_sql:schema.sql"]
+  tasks:
+    - explain:logical_optd,physical_optd
diff --git a/optd-sqlplannertest/tests/tpch/q22.yml.disabled b/optd-sqlplannertest/tests/tpch/q22.yml.disabled
new file mode 100644
index 00000000..55678824
--- /dev/null
+++ b/optd-sqlplannertest/tests/tpch/q22.yml.disabled
@@ -0,0 +1,42 @@
+- sql: |
+    select
+        cntrycode,
+        count(*) as numcust,
+        sum(c_acctbal) as totacctbal
+    from
+        (
+            select
+                substring(c_phone from 1 for 2) as cntrycode,
+                c_acctbal
+            from
+                customer
+            where
+                substring(c_phone from 1 for 2) in
+                    ('13', '31', '23', '29', '30', '18', '17')
+                and c_acctbal > (
+                    select
+                        avg(c_acctbal)
+                    from
+                        customer
+                    where
+                        c_acctbal > 0.00
+                        and substring(c_phone from 1 for 2) in
+                            ('13', '31', '23', '29', '30', '18', '17')
+                )
+                and not exists (
+                    select
+                        *
+                    from
+                        orders
+                    where
+                        o_custkey = c_custkey
+                )
+        ) as custsale
+    group by
+        cntrycode
+    order by
+        cntrycode;
+  desc: TPC-H Q22
+  before: ["include_sql:schema.sql"]
+  tasks:
+    - explain:logical_optd,physical_optd
diff --git a/optd-sqlplannertest/tests/tpch/q3.yml b/optd-sqlplannertest/tests/tpch/q3.yml
index 7f17f12f..3f0bbdaf 100644
--- a/optd-sqlplannertest/tests/tpch/q3.yml
+++ b/optd-sqlplannertest/tests/tpch/q3.yml
@@ -25,3 +25,4 @@
   before: ["include_sql:schema.sql"]
   tasks:
     - explain:logical_optd,physical_optd
+    - bench
diff --git a/optd-sqlplannertest/tests/tpch/q4.yml.disabled b/optd-sqlplannertest/tests/tpch/q4.yml.disabled
new file mode 100644
index 00000000..a30999b2
--- /dev/null
+++ b/optd-sqlplannertest/tests/tpch/q4.yml.disabled
@@ -0,0 +1,26 @@
+- sql: |
+    select
+        o_orderpriority,
+        count(*) as order_count
+    from
+        orders
+    where
+        o_orderdate >= date '1993-07-01'
+        and o_orderdate < date '1993-07-01' + interval '3' month
+        and exists (
+            select
+                *
+            from
+                lineitem
+            where
+                l_orderkey = o_orderkey
+                and l_commitdate < l_receiptdate
+        )
+    group by
+        o_orderpriority
+    order by
+        o_orderpriority;
+  desc: TPC-H Q4
+  before: ["include_sql:schema.sql"]
+  tasks:
+    - explain:logical_optd,physical_optd
diff --git a/optd-sqlplannertest/tests/tpch/q5.yml b/optd-sqlplannertest/tests/tpch/q5.yml
index e20ce915..daa36036 100644
--- a/optd-sqlplannertest/tests/tpch/q5.yml
+++ b/optd-sqlplannertest/tests/tpch/q5.yml
@@ -27,3 +27,4 @@
   before: ["include_sql:schema.sql"]
   tasks:
     - explain:logical_optd,physical_optd
+    - bench
diff --git a/optd-sqlplannertest/tests/tpch/q6.yml b/optd-sqlplannertest/tests/tpch/q6.yml
index c95f0b9c..de06ee45 100644
--- a/optd-sqlplannertest/tests/tpch/q6.yml
+++ b/optd-sqlplannertest/tests/tpch/q6.yml
@@ -12,3 +12,4 @@
   before: ["include_sql:schema.sql"]
   tasks:
     - explain:logical_optd,physical_optd
+    - bench
diff --git a/optd-sqlplannertest/tests/tpch/q7.yml b/optd-sqlplannertest/tests/tpch/q7.yml
index ae2ccf42..bd7dde0b 100644
--- a/optd-sqlplannertest/tests/tpch/q7.yml
+++ b/optd-sqlplannertest/tests/tpch/q7.yml
@@ -42,3 +42,4 @@
   before: ["include_sql:schema.sql"]
   tasks:
     - explain:logical_optd,physical_optd
+    - bench
diff --git a/optd-sqlplannertest/tests/tpch/q8.yml b/optd-sqlplannertest/tests/tpch/q8.yml
index 5483a70e..17ee68d2 100644
--- a/optd-sqlplannertest/tests/tpch/q8.yml
+++ b/optd-sqlplannertest/tests/tpch/q8.yml
@@ -40,3 +40,4 @@
   before: ["include_sql:schema.sql"]
   tasks:
     - explain:logical_optd,physical_optd
+    - bench
diff --git a/optd-sqlplannertest/tests/tpch/q9.yml b/optd-sqlplannertest/tests/tpch/q9.yml
index 92a298c7..d1601b17 100644
--- a/optd-sqlplannertest/tests/tpch/q9.yml
+++ b/optd-sqlplannertest/tests/tpch/q9.yml
@@ -35,3 +35,4 @@
   before: ["include_sql:schema.sql"]
   tasks:
     - explain:logical_optd,physical_optd
+    - bench