diff --git a/sanitychecks/flows/dlt_duckdb.yaml b/sanitychecks/flows/dlt_duckdb.yaml index 079d217..c7f3b8c 100644 --- a/sanitychecks/flows/dlt_duckdb.yaml +++ b/sanitychecks/flows/dlt_duckdb.yaml @@ -1,5 +1,5 @@ -id: dlt_duckdb -namespace: qa +id: dlt_duckdb_python +namespace: company.team tasks: @@ -20,6 +20,7 @@ tasks: response.raise_for_status() data.append(response.json()) # Extract, normalize, and load the data + print(data) pipeline.run(data, table_name='product') extension: .py @@ -49,12 +50,12 @@ tasks: data.duckdb: "{{ outputs.product_api_to_duckdb.outputFiles['product_pipeline.duckdb']}}" sql: | ATTACH '{{workingDir}}/data.duckdb' AS data; - SELECT category, SUM(price) AS sum_price FROM data.product_data.product GROUP BY 1; + SELECT category, SUM(price) AS sum_price FROM data.product_data.product GROUP BY 1 ORDER BY 2; fetchType: STORE - id: assert type: io.kestra.plugin.core.execution.Assert conditions: - - "{{ fromIon(read(outputs.duckdb['uri'])).category == 'home-decoration' }}" - - "{{ fromIon(read(outputs.duckdb['uri'])).sum_price == 194.95000000000002 }}" + - "{{ fromIon(read(outputs.duckdb['uri'])).category == 'kitchen-accessories' }}" + - "{{ fromIon(read(outputs.duckdb['uri'])).sum_price == 13.98 }}"