diff --git a/.github/workflows/test-fuzzer-ci-still-works.yml b/.github/workflows/test-fuzzer-ci-still-works.yml new file mode 100644 index 0000000..4ca1e8a --- /dev/null +++ b/.github/workflows/test-fuzzer-ci-still-works.yml @@ -0,0 +1,78 @@ +name: test-fuzzer-ci-still-works +on: + workflow_dispatch: + repository_dispatch: + push: + branches: + - '**' + - '!main' + +jobs: + build-duckdb: + name: Build DuckDB + runs-on: ubuntu-latest + timeout-minutes: 120 + outputs: + duckdb-hash: ${{ steps.find-hash.outputs.hash }} + env: + BUILD_ICU: 1 + BUILD_JSON: 1 + BUILD_TPCH: 1 + BUILD_TPCDS: 1 + BUILD_PARQUET: 1 + BUILD_JEMALLOC: 1 + CRASH_ON_ASSERT: 1 + GEN: ninja + + steps: + - name: Dependencies + shell: bash + run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build ccache + + - name: checkout + uses: actions/checkout@v3 + + - name: Setup Ccache + uses: hendrikmuhs/ccache-action@main + + - uses: actions/upload-artifact@v3 + with: + name: duckdb + path: build/debug/duckdb + + fuzzer: + name: Fuzzer + needs: + - build-duckdb + runs-on: ubuntu-latest + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + fuzzer: [duckfuzz, sqlsmith, duckfuzz_functions] + data: [alltypes, tpch, emptyalltypes] + + steps: + - uses: actions/checkout@v3 + with: + repository: duckdb/duckdb_sqlsmith + fetch-depth: 0 + + - name: Download a single artifact + uses: actions/download-artifact@v3 + with: + name: duckdb + + - name: Fuzz + shell: bash + run: | + chmod +x duckdb + runtime="1 minute" + endtime=$(date -ud "$runtime" +%s) + + while [[ $(date -u +%s) -le $endtime ]] + do + echo "Time Now: `date +%H:%M:%S`" + python3 scripts/run_fuzzer.py --no_checks --${{ matrix.fuzzer }} --${{ matrix.data }} --shell=./duckdb --dry + done + diff --git a/scripts/reduce_sql.py b/scripts/reduce_sql.py index 245a89b..774095b 100644 --- a/scripts/reduce_sql.py +++ b/scripts/reduce_sql.py @@ -214,7 +214,7 @@ def reduce_query_log_query(start, shell, queries, query_index, max_time_seconds) return sql_query -def reduce_multi_statement(sql_queries, local_shell, local_data_load): +def reduce_multi_statement(sql_queries, local_shell, local_data_load, max_time=300): reducer = MultiStatementManager(sql_queries) last_statement = reducer.get_last_statement() print(f"testing if just last statement of multi statement creates the error") @@ -222,7 +222,7 @@ def reduce_multi_statement(sql_queries, local_shell, local_data_load): expected_error = sanitize_error(stderr).strip() if len(expected_error) > 0: # reduce just the last statement - return reduce(last_statement, local_data_load, local_shell, expected_error, int(args.max_time)) + return reduce(last_statement, local_data_load, local_shell, expected_error, max_time) queries = reduce_query_log(reducer.statements, local_shell, [local_data_load]) return "\n".join(queries) @@ -295,7 +295,7 @@ def reduce_query_log(queries, shell, data_load=[], max_time_seconds=300): print("===================================================") if MultiStatementManager.is_multi_statement(sql_query): - final_query = reduce_multi_statement(sql_query, shell, data_load) + final_query = reduce_multi_statement(sql_query, shell, data_load, int(args.max_time)) else: final_query = reduce(sql_query, data_load, shell, expected_error, int(args.max_time)) diff --git a/scripts/run_fuzzer.py b/scripts/run_fuzzer.py index f2bb8f3..059db09 100644 --- a/scripts/run_fuzzer.py +++ b/scripts/run_fuzzer.py @@ -89,7 +89,6 @@ def get_fuzzer_name(fuzzer): def run_shell_command(cmd): command = [shell, '--batch', '-init', '/dev/null'] - res = subprocess.run(command, input=bytearray(cmd, 'utf8'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout = res.stdout.decode('utf8', 'ignore').strip() stderr = res.stderr.decode('utf8', 'ignore').strip() @@ -103,7 +102,7 @@ def run_shell_command(cmd): else: current_errors = fuzzer_helper.extract_github_issues(shell, perform_checks) -max_queries = 2000 +max_queries = 1000 last_query_log_file = 'sqlsmith.log' complete_log_file = 'sqlsmith.complete.log' @@ -158,7 +157,7 @@ def run_shell_command(cmd): with open(complete_log_file, 'r') as f: all_queries = f.read() -(stdout, stderr, returncode) = run_shell_command(load_script + all_queries) +(stdout, stderr, returncode) = run_shell_command(load_script + '\n' + all_queries, True) if returncode == 0: print("Failed to reproduce the issue...") exit(0) @@ -196,4 +195,5 @@ def run_shell_command(cmd): required_queries = reduce_sql.reduce_multi_statement(all_queries, shell, load_script) cmd = load_script + '\n' + last_query + "\n" -fuzzer_helper.file_issue(cmd, error_msg, fuzzer_name, seed, git_hash) +if not dry: + fuzzer_helper.file_issue(cmd, error_msg, fuzzer_name, seed, git_hash) diff --git a/scripts/runsqlsmith.py b/scripts/runsqlsmith.py deleted file mode 100644 index 9569750..0000000 --- a/scripts/runsqlsmith.py +++ /dev/null @@ -1,52 +0,0 @@ -# run SQL smith and collect breaking queries -import os -import re -import subprocess -import sys -import sqlite3 -from python_helpers import open_utf8 - -sqlsmith_db = 'sqlsmith.db' -sqlsmith_test_dir = 'test/sqlsmith/queries' - -export_queries = False - -con = sqlite3.connect(sqlsmith_db) -c = con.cursor() - -if len(sys.argv) == 2: - if sys.argv[1] == '--export': - export_queries = True - elif sys.argv[1] == '--reset': - c.execute('DROP TABLE IF EXISTS sqlsmith_errors') - else: - print('Unknown query option ' + sys.argv[1]) - exit(1) - -if export_queries: - c.execute('SELECT query FROM sqlsmith_errors') - results = c.fetchall() - for fname in os.listdir(sqlsmith_test_dir): - os.remove(os.path.join(sqlsmith_test_dir, fname)) - - for i in range(len(results)): - with open(os.path.join(sqlsmith_test_dir, 'sqlsmith-%d.sql' % (i + 1)), 'w+') as f: - f.write(results[i][0] + "\n") - exit(0) - - -def run_sqlsmith(): - subprocess.call(['build/debug/third_party/sqlsmith/sqlsmith', '--duckdb=:memory:']) - - -c.execute('CREATE TABLE IF NOT EXISTS sqlsmith_errors(query VARCHAR)') - -while True: - # run SQL smith - run_sqlsmith() - # get the breaking query - with open_utf8('sqlsmith.log', 'r') as f: - text = re.sub('[ \t\n]+', ' ', f.read()) - - c.execute('INSERT INTO sqlsmith_errors VALUES (?)', (text,)) - con.commit()