chore: add sqlfluff to lint spark sql queries (#113)

.pre-commit-config.yaml

@@ -5,3 +5,9 @@ repos:
       - id: ruff
         args: [ --fix, --exit-non-zero-on-fix ]
       - id: ruff-format
+  - repo: https://github.com/sqlfluff/sqlfluff
+    rev: 3.2.5
+    hooks:
+      - id: sqlfluff-lint
+      - id: sqlfluff-fix
+        args: [--config, "./pyproject.toml"]

examples/airflow/iceberg_query.sql

@@ -3,12 +3,19 @@ CREATE TABLE IF NOT EXISTS prod.db.sample (
     data string,
     category string
 )
-USING iceberg
+USING ICEBERG
 PARTITIONED BY (category);
 
 MERGE INTO prod.db.sample t
-USING (SELECT * FROM prod.db.another_sample WHERE category = 'foo') s
-ON t.id = s.id
+USING (
+    SELECT
+        id,
+        data,
+        category
+    FROM prod.db.another_sample
+    WHERE category = 'foo'
+) s
+    ON t.id = s.id
 WHEN MATCHED THEN UPDATE SET *
 WHEN NOT MATCHED THEN INSERT *
-WHEN NOT MATCHED BY SOURCE THEN DELETE;
+WHEN NOT MATCHED BY SOURCE THEN DELETE; -- noqa: PRS

examples/airflow/query.sql

@@ -7,8 +7,8 @@ USING PARQUET
 PARTITIONED BY (student_id INT);
 
 INSERT INTO students VALUES
-    ('Amy Smith', '123 Park Ave, San Jose', {{ ts }}, 111111);
+('Amy Smith', '123 Park Ave, San Jose', '{{ ts }}', 111111);
 
 INSERT INTO students VALUES
-    ('Bob Brown', '456 Taylor St, Cupertino', {{ ts }}, 222222);,
-    ('Cathy Johnson', '789 Race Ave, Palo Alto', {{ ts}}, 333333);
+('Bob Brown', '456 Taylor St, Cupertino', '{{ ts }}', 222222),
+('Cathy Johnson', '789 Race Ave, Palo Alto', '{{ ts }}', 333333);

pyproject.toml

@@ -46,6 +46,7 @@ mkdocstrings = {version = "^0.25.1", extras = ["python"]}
 mkdocs-gen-files = "^0.5.0"
 mkdocs-literate-nav = "^0.6.1"
 helm-mkdocs = "^0.0.5"
+sqlfluff = "^3.2.5"
 
 [tool.poetry.extras]
 api = ["fastapi", "kubernetes-asyncio", "uvicorn", "httpx", "jinja2", "aiohttp", "websockets"]
@@ -90,3 +91,17 @@ docstring-code-format = true
 
 [tool.ruff.lint.pydocstyle]
 convention = "google"
+
+[tool.sqlfluff.core]
+dialect = "sparksql"
+sql_file_exts = ".sql"
+ignore = "templating"
+exclude_rules="AL01"
+
+[tool.sqlfluff.indentation]
+indented_joins = false
+indented_using_on = true
+template_blocks_indent = false
+
+[tool.sqlfluff.rules.capitalisation.keywords]
+capitalisation_policy = "upper"