feature: create base tpdm/edfi_3 models iteration 2, add staging models

Author: Vasili Zakharko

models/staging/edfi_3/base/_edfi_3__base.yml

@@ -79,9 +79,6 @@ models:
   - name: base_ef3__parents
     config:
       tags: ['core']
-  - name: base_ef3__persons
-    config:
-      tags: ['core']
   - name: base_ef3__programs
     config:
       tags: ['core']

models/staging/edfi_3/stage/_edfi_3__stage.yml

@@ -401,6 +401,10 @@ models:
     config:
       tags: ['core']
 
+  - name: stg_ef3__school_year_types
+    config:
+      tags: ['core']
+
   - name: stg_ef3__schools
     config:
       tags: ['core']
@@ -460,6 +464,10 @@ models:
     config:
       tags: ['core']
 
+  - name: stg_ef3__staff_absence_events
+    config:
+      tags: ['core']
+
   - name: stg_ef3__staff_education_organization_assignment_associations
     config:
       tags: ['core']

models/staging/edfi_3/stage/stg_ef3__school_year_types.sql

@@ -0,0 +1,28 @@
+with school_year_types as (
+    select * from {{ ref('base_ef3__school_year_types') }}
+    where not is_deleted
+),
+keyed as (
+    select
+        {{ dbt_utils.surrogate_key(
+            [
+                'tenant_code',
+                'api_year',
+                'lower(school_year)',
+                'lower(current_school_year)'
+            ]
+        )}} as k_school_year_types,
+        api_year as school_year,
+        school_year_types.*
+        {{ extract_extension(model_name=this.name, flatten=True) }}
+    from school_year_types
+),
+deduped as (
+    {{
+        dbt_utils.deduplicate(
+            relation='keyed',
+            partition_by='k_school_year_types',
+            order_by='pull_timestamp desc'
+        )
+    }}
+)

models/staging/edfi_3/stage/stg_ef3__staff_absence_events.sql

@@ -0,0 +1,28 @@
+with staff_absence_events as (
+    select * from {{ ref('base_ef3__staff_absence_events') }}
+),
+keyed as (
+    select
+        {{ dbt_utils.surrogate_key(
+            [
+                'tenant_code',
+                'lower(absence_event_category)',
+                'lower(survey_identifier)',
+                'lower(namespace)',
+                'lower(course_code)'
+            ]
+        ) }} as k_staff_absence_events,
+        staff_absence_events.*
+        {{ extract_extension(model_name=this.name, flatten=True) }}
+    from staff_absence_events
+),
+deduped as (
+    {{
+        dbt_utils.deduplicate(
+            relation='keyed',
+            partition_by='k_staff_absence_events',
+            order_by='api_year desc, pull_timestamp desc'
+        )
+    }}
+)
+select * from deduped

models/staging/edfi_3/stage/stg_ef3__survey_course_associations.sql

@@ -0,0 +1,29 @@
+with base_survey_course_associations as (
+    select * from {{ ref('base_ef3__survey_course_associations') }}
+    where not is_deleted
+),
+keyed as (
+    select
+        {{ gen_skey('k_survey') }},
+        {{ gen_skey('k_course') }},
+        {{ dbt_utils.surrogate_key(
+            [
+                'tenant_code',
+                'lower(education_organization_id)',
+                'lower(event_date)'
+            ]
+        ) }} as k_survey_course_associations,
+        base_survey_course_associations.*
+        {{ extract_extension(model_name=this.name, flatten=True) }}
+    from base_survey_course_associations
+),
+deduped as (
+    {{
+        dbt_utils.deduplicate(
+            relation='keyed',
+            partition_by='k_survey, k_course, entry_date, k_survey_course_associations',
+            order_by='pull_timestamp desc'
+        )
+    }}
+)
+select * from deduped