Update fuzzing scripts

.github/workflows/test-fuzzer-ci-still-works.yml

@@ -0,0 +1,78 @@
+name: test-fuzzer-ci-still-works
+on:
+  workflow_dispatch:
+  repository_dispatch:
+  push:
+    branches:
+      - '**'
+      - '!main'
+
+jobs:
+  build-duckdb:
+    name: Build DuckDB
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    outputs:
+      duckdb-hash: ${{ steps.find-hash.outputs.hash }}
+    env:
+      BUILD_ICU: 1
+      BUILD_JSON: 1
+      BUILD_TPCH: 1
+      BUILD_TPCDS: 1
+      BUILD_PARQUET: 1
+      BUILD_JEMALLOC: 1
+      CRASH_ON_ASSERT: 1
+      GEN: ninja
+
+    steps:
+      - name: Dependencies
+        shell: bash
+        run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build ccache
+
+      - name: checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Ccache
+        uses: hendrikmuhs/ccache-action@main
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: duckdb
+          path: build/debug/duckdb
+
+  fuzzer:
+    name: Fuzzer
+    needs:
+    - build-duckdb
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+      matrix:
+        fuzzer: [duckfuzz, sqlsmith, duckfuzz_functions]
+        data: [alltypes, tpch, emptyalltypes]
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: duckdb/duckdb_sqlsmith
+          fetch-depth: 0
+
+      - name: Download a single artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: duckdb
+
+      - name: Fuzz
+        shell: bash
+        run: |
+            chmod +x duckdb
+            runtime="1 minute"
+            endtime=$(date -ud "$runtime" +%s)
+
+            while [[ $(date -u +%s) -le $endtime ]]
+            do
+                echo "Time Now: `date +%H:%M:%S`"
+                python3 scripts/run_fuzzer.py --no_checks --${{ matrix.fuzzer }} --${{ matrix.data }} --shell=./duckdb --dry
+            done
+

scripts/reduce_sql.py

@@ -214,15 +214,15 @@ def reduce_query_log_query(start, shell, queries, query_index, max_time_seconds)
     return sql_query
 
 
-def reduce_multi_statement(sql_queries, local_shell, local_data_load):
+def reduce_multi_statement(sql_queries, local_shell, local_data_load, max_time=300):
     reducer = MultiStatementManager(sql_queries)
     last_statement = reducer.get_last_statement()
     print(f"testing if just last statement of multi statement creates the error")
     (stdout, stderr, returncode) = run_shell_command(local_shell, local_data_load + last_statement)
     expected_error = sanitize_error(stderr).strip()
     if len(expected_error) > 0:
         # reduce just the last statement
-        return reduce(last_statement, local_data_load, local_shell, expected_error, int(args.max_time))
+         return reduce(last_statement, local_data_load, local_shell, expected_error, max_time)
     queries = reduce_query_log(reducer.statements, local_shell, [local_data_load])
     return "\n".join(queries)
 
@@ -295,7 +295,7 @@ def reduce_query_log(queries, shell, data_load=[], max_time_seconds=300):
     print("===================================================")
 
     if MultiStatementManager.is_multi_statement(sql_query):
-        final_query = reduce_multi_statement(sql_query, shell, data_load)
+        final_query = reduce_multi_statement(sql_query, shell, data_load, int(args.max_time))
     else:
         final_query = reduce(sql_query, data_load, shell, expected_error, int(args.max_time))
 

scripts/run_fuzzer.py

@@ -89,7 +89,6 @@ def get_fuzzer_name(fuzzer):
 
 def run_shell_command(cmd):
     command = [shell, '--batch', '-init', '/dev/null']
-
     res = subprocess.run(command, input=bytearray(cmd, 'utf8'), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     stdout = res.stdout.decode('utf8', 'ignore').strip()
     stderr = res.stderr.decode('utf8', 'ignore').strip()
@@ -103,7 +102,7 @@ def run_shell_command(cmd):
 else:
     current_errors = fuzzer_helper.extract_github_issues(shell, perform_checks)
 
-max_queries = 2000
+max_queries = 1000
 last_query_log_file = 'sqlsmith.log'
 complete_log_file = 'sqlsmith.complete.log'
 
@@ -158,7 +157,7 @@ def run_shell_command(cmd):
 with open(complete_log_file, 'r') as f:
     all_queries = f.read()
 
-(stdout, stderr, returncode) = run_shell_command(load_script + all_queries)
+(stdout, stderr, returncode) = run_shell_command(load_script + '\n' + all_queries, True)
 if returncode == 0:
     print("Failed to reproduce the issue...")
     exit(0)
@@ -196,4 +195,5 @@ def run_shell_command(cmd):
 required_queries = reduce_sql.reduce_multi_statement(all_queries, shell, load_script)
 cmd = load_script + '\n' + last_query + "\n"
 
-fuzzer_helper.file_issue(cmd, error_msg, fuzzer_name, seed, git_hash)
+if not dry:
+    fuzzer_helper.file_issue(cmd, error_msg, fuzzer_name, seed, git_hash)