From c937b55bf2fabf5b64c0030022c854b46e4a5f4e Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 4 Jul 2024 15:14:41 +0200 Subject: [PATCH 01/10] fix scripts so fuzzing can continue --- scripts/reduce_sql.py | 6 +++--- scripts/run_fuzzer.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/reduce_sql.py b/scripts/reduce_sql.py index 245a89b..774095b 100644 --- a/scripts/reduce_sql.py +++ b/scripts/reduce_sql.py @@ -214,7 +214,7 @@ def reduce_query_log_query(start, shell, queries, query_index, max_time_seconds) return sql_query -def reduce_multi_statement(sql_queries, local_shell, local_data_load): +def reduce_multi_statement(sql_queries, local_shell, local_data_load, max_time=300): reducer = MultiStatementManager(sql_queries) last_statement = reducer.get_last_statement() print(f"testing if just last statement of multi statement creates the error") @@ -222,7 +222,7 @@ def reduce_multi_statement(sql_queries, local_shell, local_data_load): expected_error = sanitize_error(stderr).strip() if len(expected_error) > 0: # reduce just the last statement - return reduce(last_statement, local_data_load, local_shell, expected_error, int(args.max_time)) + return reduce(last_statement, local_data_load, local_shell, expected_error, max_time) queries = reduce_query_log(reducer.statements, local_shell, [local_data_load]) return "\n".join(queries) @@ -295,7 +295,7 @@ def reduce_query_log(queries, shell, data_load=[], max_time_seconds=300): print("===================================================") if MultiStatementManager.is_multi_statement(sql_query): - final_query = reduce_multi_statement(sql_query, shell, data_load) + final_query = reduce_multi_statement(sql_query, shell, data_load, int(args.max_time)) else: final_query = reduce(sql_query, data_load, shell, expected_error, int(args.max_time)) diff --git a/scripts/run_fuzzer.py b/scripts/run_fuzzer.py index f2bb8f3..4a09d0c 100644 --- a/scripts/run_fuzzer.py +++ b/scripts/run_fuzzer.py @@ -196,4 +196,5 @@ def run_shell_command(cmd): required_queries = reduce_sql.reduce_multi_statement(all_queries, shell, load_script) cmd = load_script + '\n' + last_query + "\n" -fuzzer_helper.file_issue(cmd, error_msg, fuzzer_name, seed, git_hash) +if not dry: + fuzzer_helper.file_issue(cmd, error_msg, fuzzer_name, seed, git_hash) From 811cc0ca7ad504ae822c54f1bb8a74cf84616985 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 4 Jul 2024 15:51:29 +0200 Subject: [PATCH 02/10] add a workflow to run fuzzer if fuzzing script changes --- .../workflows/test-fuzzer-ci-still-works.yml | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 .github/workflows/test-fuzzer-ci-still-works.yml diff --git a/.github/workflows/test-fuzzer-ci-still-works.yml b/.github/workflows/test-fuzzer-ci-still-works.yml new file mode 100644 index 0000000..1e57d33 --- /dev/null +++ b/.github/workflows/test-fuzzer-ci-still-works.yml @@ -0,0 +1,84 @@ +name: test-fuzzer-ci-still-works +on: + workflow_dispatch: + repository_dispatch: + push: + branches: + - '**' + - '!main' + paths-ignore: + - '**' + - '!scripts/fuzzer_helper.py' + - '!scripts/run_fuzzer.py' + - '!scripts/run_sqlancer.py' + - '!scripts/runsqlsmith.py' + +jobs: + build-duckdb: + name: Build DuckDB + runs-on: ubuntu-latest + timeout-minutes: 120 + outputs: + duckdb-hash: ${{ steps.find-hash.outputs.hash }} + env: + BUILD_ICU: 1 + BUILD_JSON: 1 + BUILD_TPCH: 1 + BUILD_TPCDS: 1 + BUILD_PARQUET: 1 + BUILD_JEMALLOC: 1 + CRASH_ON_ASSERT: 1 + GEN: ninja + + steps: + - name: Dependencies + shell: bash + run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build ccache + + - name: checkout + - uses: actions/checkout@v3 + + - name: Setup Ccache + uses: hendrikmuhs/ccache-action@main + + - uses: actions/upload-artifact@v3 + with: + name: duckdb + path: build/debug/duckdb + + fuzzer: + name: Fuzzer + needs: + - build-duckdb + runs-on: ubuntu-latest + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + fuzzer: [duckfuzz, sqlsmith, duckfuzz_functions] + data: [alltypes, tpch, emptyalltypes] + + steps: + - uses: actions/checkout@v3 + with: + repository: duckdb/duckdb_sqlsmith + fetch-depth: 0 + + - name: Download a single artifact + uses: actions/download-artifact@v3 + with: + name: duckdb + + - name: Fuzz + shell: bash + run: | + chmod +x duckdb + runtime="10 minute" + endtime=$(date -ud "$runtime" +%s) + + while [[ $(date -u +%s) -le $endtime ]] + do + echo "Time Now: `date +%H:%M:%S`" + python3 scripts/run_fuzzer.py --no_checks --${{ matrix.fuzzer }} --${{ matrix.data }} --shell=./duckdb --dry + done + From 529232ab8a4083c60e51ad25db5dddd202d9b844 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 4 Jul 2024 15:52:37 +0200 Subject: [PATCH 03/10] only run fuzzer for a minute --- .github/workflows/test-fuzzer-ci-still-works.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-fuzzer-ci-still-works.yml b/.github/workflows/test-fuzzer-ci-still-works.yml index 1e57d33..db45514 100644 --- a/.github/workflows/test-fuzzer-ci-still-works.yml +++ b/.github/workflows/test-fuzzer-ci-still-works.yml @@ -73,7 +73,7 @@ jobs: shell: bash run: | chmod +x duckdb - runtime="10 minute" + runtime="1 minute" endtime=$(date -ud "$runtime" +%s) while [[ $(date -u +%s) -le $endtime ]] From 9a32a0aea25d748290494e8b236c9ac489d6be67 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 4 Jul 2024 16:07:09 +0200 Subject: [PATCH 04/10] update ignore paths --- .github/workflows/test-fuzzer-ci-still-works.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/test-fuzzer-ci-still-works.yml b/.github/workflows/test-fuzzer-ci-still-works.yml index db45514..6f0b60f 100644 --- a/.github/workflows/test-fuzzer-ci-still-works.yml +++ b/.github/workflows/test-fuzzer-ci-still-works.yml @@ -6,12 +6,6 @@ on: branches: - '**' - '!main' - paths-ignore: - - '**' - - '!scripts/fuzzer_helper.py' - - '!scripts/run_fuzzer.py' - - '!scripts/run_sqlancer.py' - - '!scripts/runsqlsmith.py' jobs: build-duckdb: From 2d2519f0765373a512e19ac0e6d6551691da55ba Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 4 Jul 2024 16:08:40 +0200 Subject: [PATCH 05/10] remove '-' --- .github/workflows/test-fuzzer-ci-still-works.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-fuzzer-ci-still-works.yml b/.github/workflows/test-fuzzer-ci-still-works.yml index 6f0b60f..4ca1e8a 100644 --- a/.github/workflows/test-fuzzer-ci-still-works.yml +++ b/.github/workflows/test-fuzzer-ci-still-works.yml @@ -30,7 +30,7 @@ jobs: run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build ccache - name: checkout - - uses: actions/checkout@v3 + uses: actions/checkout@v3 - name: Setup Ccache uses: hendrikmuhs/ccache-action@main From 5600a5fb58415853db182cedc8da505183e6d033 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Thu, 4 Jul 2024 16:20:39 +0200 Subject: [PATCH 06/10] CI again From 6440a4d6618b3c4c899d407dfa38b92176fe379d Mon Sep 17 00:00:00 2001 From: Tmonster Date: Fri, 5 Jul 2024 13:00:50 +0200 Subject: [PATCH 07/10] update scripts --- scripts/run_fuzzer.py | 3 +-- scripts/runsqlsmith.py | 52 ------------------------------------------ 2 files changed, 1 insertion(+), 54 deletions(-) delete mode 100644 scripts/runsqlsmith.py diff --git a/scripts/run_fuzzer.py b/scripts/run_fuzzer.py index 4a09d0c..ed5a494 100644 --- a/scripts/run_fuzzer.py +++ b/scripts/run_fuzzer.py @@ -89,7 +89,6 @@ def get_fuzzer_name(fuzzer): def run_shell_command(cmd): command = [shell, '--batch', '-init', '/dev/null'] - res = subprocess.run(command, input=bytearray(cmd, 'utf8'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout = res.stdout.decode('utf8', 'ignore').strip() stderr = res.stderr.decode('utf8', 'ignore').strip() @@ -158,7 +157,7 @@ def run_shell_command(cmd): with open(complete_log_file, 'r') as f: all_queries = f.read() -(stdout, stderr, returncode) = run_shell_command(load_script + all_queries) +(stdout, stderr, returncode) = run_shell_command(load_script + '\n' + all_queries, True) if returncode == 0: print("Failed to reproduce the issue...") exit(0) diff --git a/scripts/runsqlsmith.py b/scripts/runsqlsmith.py deleted file mode 100644 index 9569750..0000000 --- a/scripts/runsqlsmith.py +++ /dev/null @@ -1,52 +0,0 @@ -# run SQL smith and collect breaking queries -import os -import re -import subprocess -import sys -import sqlite3 -from python_helpers import open_utf8 - -sqlsmith_db = 'sqlsmith.db' -sqlsmith_test_dir = 'test/sqlsmith/queries' - -export_queries = False - -con = sqlite3.connect(sqlsmith_db) -c = con.cursor() - -if len(sys.argv) == 2: - if sys.argv[1] == '--export': - export_queries = True - elif sys.argv[1] == '--reset': - c.execute('DROP TABLE IF EXISTS sqlsmith_errors') - else: - print('Unknown query option ' + sys.argv[1]) - exit(1) - -if export_queries: - c.execute('SELECT query FROM sqlsmith_errors') - results = c.fetchall() - for fname in os.listdir(sqlsmith_test_dir): - os.remove(os.path.join(sqlsmith_test_dir, fname)) - - for i in range(len(results)): - with open(os.path.join(sqlsmith_test_dir, 'sqlsmith-%d.sql' % (i + 1)), 'w+') as f: - f.write(results[i][0] + "\n") - exit(0) - - -def run_sqlsmith(): - subprocess.call(['build/debug/third_party/sqlsmith/sqlsmith', '--duckdb=:memory:']) - - -c.execute('CREATE TABLE IF NOT EXISTS sqlsmith_errors(query VARCHAR)') - -while True: - # run SQL smith - run_sqlsmith() - # get the breaking query - with open_utf8('sqlsmith.log', 'r') as f: - text = re.sub('[ \t\n]+', ' ', f.read()) - - c.execute('INSERT INTO sqlsmith_errors VALUES (?)', (text,)) - con.commit() From 565a3cc56e0955bd526ff5029a2b91fbafc11a98 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Mon, 8 Jul 2024 16:47:38 +0200 Subject: [PATCH 08/10] reduce max_queries so timeout isn't reached --- scripts/run_fuzzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run_fuzzer.py b/scripts/run_fuzzer.py index ed5a494..059db09 100644 --- a/scripts/run_fuzzer.py +++ b/scripts/run_fuzzer.py @@ -102,7 +102,7 @@ def run_shell_command(cmd): else: current_errors = fuzzer_helper.extract_github_issues(shell, perform_checks) -max_queries = 2000 +max_queries = 1000 last_query_log_file = 'sqlsmith.log' complete_log_file = 'sqlsmith.complete.log' From 720fca122da889f7623e2bfdf9a8732549bd6c7d Mon Sep 17 00:00:00 2001 From: Tmonster Date: Tue, 9 Jul 2024 10:03:53 +0200 Subject: [PATCH 09/10] lower max_queries even more to see if fuzzing CI will pass --- scripts/run_fuzzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run_fuzzer.py b/scripts/run_fuzzer.py index 059db09..8424f24 100644 --- a/scripts/run_fuzzer.py +++ b/scripts/run_fuzzer.py @@ -102,7 +102,7 @@ def run_shell_command(cmd): else: current_errors = fuzzer_helper.extract_github_issues(shell, perform_checks) -max_queries = 1000 +max_queries = 20 last_query_log_file = 'sqlsmith.log' complete_log_file = 'sqlsmith.complete.log' From 0c107319e4a0ee6fffaea18bf29e8c964725b227 Mon Sep 17 00:00:00 2001 From: Tmonster Date: Fri, 12 Jul 2024 11:14:58 +0200 Subject: [PATCH 10/10] bump max_queries up to 1000 --- scripts/run_fuzzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run_fuzzer.py b/scripts/run_fuzzer.py index 8424f24..059db09 100644 --- a/scripts/run_fuzzer.py +++ b/scripts/run_fuzzer.py @@ -102,7 +102,7 @@ def run_shell_command(cmd): else: current_errors = fuzzer_helper.extract_github_issues(shell, perform_checks) -max_queries = 20 +max_queries = 1000 last_query_log_file = 'sqlsmith.log' complete_log_file = 'sqlsmith.complete.log'