Skip to content

Commit 235dfc9

Browse files
LewisDaviesLewisDavies
authored
Fix required vars check & prefer recent config (#35)
* Update tests * Fix database vars check * Update boolean checks * Update test script * Bump version & clean up
1 parent 0379c10 commit 235dfc9

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+574
-99
lines changed

‎README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,7 @@ Now that your dev models are using prod data, you DAG would look like this:
272272
```
273273

274274
## Compatibility
275-
`upstream-prod` was initially designed on Snowflake and is now primarily tested on Databricks. Based on my experience and user reports, it is known to work on:
275+
`upstream-prod` is built and tested on Snowflake. Based on my experience and user reports, it is known to work on:
276276
- Snowflake
277277
- Databricks
278278
- BigQuery

‎dbt_project.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: 'upstream_prod'
2-
version: '0.8.0'
2+
version: '0.8.1'
33
config-version: 2
44

55
require-dbt-version: [">=1.5.0", "<2.0.0"]

‎integration_tests/_template/macros/ref.sql

+2-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66
fallback=var("upstream_prod_fallback", False),
77
env_schemas=var("upstream_prod_env_schemas", False),
88
version=None,
9-
prefer_recent=var("upstream_prod_prefer_recent", False)
9+
prefer_recent=var("upstream_prod_prefer_recent", False),
10+
prod_database_replace=var("upstream_prod_database_replace", None)
1011
) %}
1112

1213
{% do return(upstream_prod.ref(parent_model, prod_database, prod_schema, enabled, fallback, env_schemas, version)) %}

‎integration_tests/_template/run_tests.sh

+9-3
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,24 @@
11
# Setup
22
set -e
3-
echo "\nSETTING UP ENVIRONMENT\n"
3+
echo ""
4+
echo "## SETTING UP ENVIRONMENT"
5+
echo ""
46
dbt clean
57
dbt deps
68
dbt run-operation create_test_db --args '{db: upstream__prod_db}'
79
dbt run-operation create_test_db --args '{db: upstream__dev_db}'
810

911
# Create staging model in appropriate envs
10-
echo "\nBUILDING STAGING MODELS\n"
12+
echo ""
13+
echo "## BUILDING STAGING MODELS"
14+
echo ""
1115
dbt build -s stg__defer_prod stg__defer_vers stg__dev_newer --target prod
1216
dbt build -s stg__dev_fallback stg__dev_newer
1317

1418
# Build & test downstream models
15-
echo "\nBUILDING DOWNSTREAM MODELS\n"
19+
echo ""
20+
echo "## BUILDING DOWNSTREAM MODELS"
21+
echo ""
1622
dbt build -s models/marts
1723

1824
# Check dbt-codegen compatibility

‎integration_tests/create_symlinks.sh

-11
This file was deleted.

‎integration_tests/dev_db/macros/ref.sql

+2-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66
fallback=var("upstream_prod_fallback", False),
77
env_schemas=var("upstream_prod_env_schemas", False),
88
version=None,
9-
prefer_recent=var("upstream_prod_prefer_recent", False)
9+
prefer_recent=var("upstream_prod_prefer_recent", False),
10+
prod_database_replace=var("upstream_prod_database_replace", None)
1011
) %}
1112

1213
{% do return(upstream_prod.ref(parent_model, prod_database, prod_schema, enabled, fallback, env_schemas, version)) %}

‎integration_tests/dev_db/run_tests.sh

+9-3
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,24 @@
11
# Setup
22
set -e
3-
echo "\nSETTING UP ENVIRONMENT\n"
3+
echo ""
4+
echo "## SETTING UP ENVIRONMENT"
5+
echo ""
46
dbt clean
57
dbt deps
68
dbt run-operation create_test_db --args '{db: upstream__prod_db}'
79
dbt run-operation create_test_db --args '{db: upstream__dev_db}'
810

911
# Create staging model in appropriate envs
10-
echo "\nBUILDING STAGING MODELS\n"
12+
echo ""
13+
echo "## BUILDING STAGING MODELS"
14+
echo ""
1115
dbt build -s stg__defer_prod stg__defer_vers stg__dev_newer --target prod
1216
dbt build -s stg__dev_fallback stg__dev_newer
1317

1418
# Build & test downstream models
15-
echo "\nBUILDING DOWNSTREAM MODELS\n"
19+
echo ""
20+
echo "## BUILDING DOWNSTREAM MODELS"
21+
echo ""
1622
dbt build -s models/marts
1723

1824
# Check dbt-codegen compatibility

‎integration_tests/dev_db_dev_sch/macros/ref.sql

+2-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66
fallback=var("upstream_prod_fallback", False),
77
env_schemas=var("upstream_prod_env_schemas", False),
88
version=None,
9-
prefer_recent=var("upstream_prod_prefer_recent", False)
9+
prefer_recent=var("upstream_prod_prefer_recent", False),
10+
prod_database_replace=var("upstream_prod_database_replace", None)
1011
) %}
1112

1213
{% do return(upstream_prod.ref(parent_model, prod_database, prod_schema, enabled, fallback, env_schemas, version)) %}

‎integration_tests/dev_db_dev_sch/run_tests.sh

+9-3
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,24 @@
11
# Setup
22
set -e
3-
echo "\nSETTING UP ENVIRONMENT\n"
3+
echo ""
4+
echo "## SETTING UP ENVIRONMENT"
5+
echo ""
46
dbt clean
57
dbt deps
68
dbt run-operation create_test_db --args '{db: upstream__prod_db}'
79
dbt run-operation create_test_db --args '{db: upstream__dev_db}'
810

911
# Create staging model in appropriate envs
10-
echo "\nBUILDING STAGING MODELS\n"
12+
echo ""
13+
echo "## BUILDING STAGING MODELS"
14+
echo ""
1115
dbt build -s stg__defer_prod stg__defer_vers stg__dev_newer --target prod
1216
dbt build -s stg__dev_fallback stg__dev_newer
1317

1418
# Build & test downstream models
15-
echo "\nBUILDING DOWNSTREAM MODELS\n"
19+
echo ""
20+
echo "## BUILDING DOWNSTREAM MODELS"
21+
echo ""
1622
dbt build -s models/marts
1723

1824
# Check dbt-codegen compatibility

‎integration_tests/dev_db_env_sch/macros/ref.sql

+2-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66
fallback=var("upstream_prod_fallback", False),
77
env_schemas=var("upstream_prod_env_schemas", False),
88
version=None,
9-
prefer_recent=var("upstream_prod_prefer_recent", False)
9+
prefer_recent=var("upstream_prod_prefer_recent", False),
10+
prod_database_replace=var("upstream_prod_database_replace", None)
1011
) %}
1112

1213
{% do return(upstream_prod.ref(parent_model, prod_database, prod_schema, enabled, fallback, env_schemas, version)) %}

‎integration_tests/dev_db_env_sch/run_tests.sh

+9-3
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,24 @@
11
# Setup
22
set -e
3-
echo "\nSETTING UP ENVIRONMENT\n"
3+
echo ""
4+
echo "## SETTING UP ENVIRONMENT"
5+
echo ""
46
dbt clean
57
dbt deps
68
dbt run-operation create_test_db --args '{db: upstream__prod_db}'
79
dbt run-operation create_test_db --args '{db: upstream__dev_db}'
810

911
# Create staging model in appropriate envs
10-
echo "\nBUILDING STAGING MODELS\n"
12+
echo ""
13+
echo "## BUILDING STAGING MODELS"
14+
echo ""
1115
dbt build -s stg__defer_prod stg__defer_vers stg__dev_newer --target prod
1216
dbt build -s stg__dev_fallback stg__dev_newer
1317

1418
# Build & test downstream models
15-
echo "\nBUILDING DOWNSTREAM MODELS\n"
19+
echo ""
20+
echo "## BUILDING DOWNSTREAM MODELS"
21+
echo ""
1622
dbt build -s models/marts
1723

1824
# Check dbt-codegen compatibility

‎integration_tests/dev_db_replace/dbt_project.yml

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
name: 'upstream_prod_integration_tests'
2+
version: '1.0.0'
3+
config-version: 2
4+
5+
profile: 'upstream_prod_db'
6+
7+
model-paths: ["models"]
8+
test-paths: ["tests"]
9+
macro-paths: ["macros"]
10+
11+
target-path: "target" # directory which will store compiled SQL files
12+
clean-targets: # directories to be removed by `dbt clean`
13+
- "target"
14+
- "dbt_packages"
15+
16+
17+
models:
18+
upstream_prod_integration_tests:
19+
marts:
20+
+schema: marts
21+
staging:
22+
+schema: stg
23+
24+
vars:
25+
upstream_prod_database_replace: [dev, prod]
26+
upstream_prod_fallback: true
27+
upstream_prod_prefer_recent: true

‎integration_tests/dev_db_replace/macros/create_test_db.sql

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{% macro create_test_db(db) -%}
2+
3+
{% if target.type == "snowflake" %}
4+
{% do run_query("create or replace database " ~ db) %}
5+
{% elif target.type == "databricks" %}
6+
{% do run_query("drop catalog if exists " ~ db ~ " cascade") %}
7+
{% do run_query("create catalog " ~ db) %}
8+
{% endif %}
9+
10+
{%- endmacro %}

‎integration_tests/dev_db_replace/macros/ref.sql

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{% macro ref(
2+
parent_model,
3+
prod_database=var("upstream_prod_database", None),
4+
prod_schema=var("upstream_prod_schema", None),
5+
enabled=var("upstream_prod_enabled", True),
6+
fallback=var("upstream_prod_fallback", False),
7+
env_schemas=var("upstream_prod_env_schemas", False),
8+
version=None,
9+
prefer_recent=var("upstream_prod_prefer_recent", False),
10+
prod_database_replace=var("upstream_prod_database_replace", None)
11+
) %}
12+
13+
{% do return(upstream_prod.ref(parent_model, prod_database, prod_schema, enabled, fallback, env_schemas, version)) %}
14+
15+
{% endmacro %}

0 commit comments

Comments
 (0)