From 2f483db09115f3b842c732a9a181df41be3e23d0 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Mon, 30 Jun 2025 13:04:31 -0500 Subject: [PATCH 01/35] Create base and stage models for tpdm survey_section_response_person_target_associations and update their respective yml files. --- models/staging/tpdm/base/_tpdm__base.yml | 4 +++ ...on_response_person_target_associations.sql | 29 +++++++++++++++++ models/staging/tpdm/stage/_tpdm__stage.yml | 18 +++++++++++ ...on_response_person_target_associations.sql | 31 +++++++++++++++++++ 4 files changed, 82 insertions(+) create mode 100644 models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql create mode 100644 models/staging/tpdm/stage/stg_tpdm__survey_section_response_person_target_associations.sql diff --git a/models/staging/tpdm/base/_tpdm__base.yml b/models/staging/tpdm/base/_tpdm__base.yml index 81f41dca..651e8389 100644 --- a/models/staging/tpdm/base/_tpdm__base.yml +++ b/models/staging/tpdm/base/_tpdm__base.yml @@ -86,3 +86,7 @@ models: tags: ['tpdm'] enabled: "{{ var('edu:tpdmcommunity:enabled', False) }}" + - name: base_tpdm__survey_section_response_person_target_associations + config: + tags: ['tpdm'] + enabled: "{{ var('edu:tpdmcommunity:enabled', False) }}" diff --git a/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql b/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql new file mode 100644 index 00000000..563b7db3 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql @@ -0,0 +1,29 @@ +with survey_section_response_person_target_associations as ( + {{ source_edfi3('survey_section_response_person_target_associations') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:surveySectionResponseReference:namespace::string as namespace, + v:personReference:personId::string as person_id, + {{ extract_descriptor('v:personReference:sourceSystemDescriptor::string') }} as source_system, + v:surveySectionResponseReference:surveyIdentifier::string as survey_id, + v:surveySectionResponseReference:surveyResponseIdentifier::string as survey_response_id, + v:surveySectionResponseReference:surveySectionTitle::string as survey_section_title, + -- references + v:surveySectionResponseReference as survey_section_response_reference, + v:personReference as person_reference, + -- edfi extensions + v:_ext as v_ext + from survey_section_response_person_target_associations +) +select * from renamed diff --git a/models/staging/tpdm/stage/_tpdm__stage.yml b/models/staging/tpdm/stage/_tpdm__stage.yml index 2e19a4a5..dff4c65b 100644 --- a/models/staging/tpdm/stage/_tpdm__stage.yml +++ b/models/staging/tpdm/stage/_tpdm__stage.yml @@ -90,6 +90,12 @@ referential_integrity_tests: field: k_evaluation_objective_rating tags: ['ref_integrity'] + - k_survey_section_response: &ref_k_survey_section_response + - relationships: + to: ref('stg_ef3__survey_section_responses') + field: k_survey_section_response + tags: ['ref_integrity'] + models: - name: stg_tpdm__candidate_educator_preparation_program_associations config: @@ -274,3 +280,15 @@ models: - name: k_educator_prep_program tests: *ref_k_educator_prep_program + + - name: stg_tpdm__survey_section_response_person_target_associations + config: + tags: ['tpdm'] + enabled: "{{ var('edu:tpdmcommunity:enabled', False) }}" + columns: + - name: k_person + tests: + *ref_k_person + - name: k_survey_section_response + tests: + *ref_k_survey_section_response diff --git a/models/staging/tpdm/stage/stg_tpdm__survey_section_response_person_target_associations.sql b/models/staging/tpdm/stage/stg_tpdm__survey_section_response_person_target_associations.sql new file mode 100644 index 00000000..d77290c8 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__survey_section_response_person_target_associations.sql @@ -0,0 +1,31 @@ +with survey_section_response_person_target_associations as ( + select * from {{ ref('base_tpdm__survey_section_response_person_target_associations') }} +), +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + ['tenant_code', + 'api_year', + 'lower(namespace)', + 'lower(person_id)', + 'lower(source_system)', + 'lower(survey_id)', + 'lower(survey_response_id)', + 'lower(survey_section_title)'] + ) }} as k_survey_section_response_person_target_association, + {{ gen_skey('k_survey_section_response') }}, + {{ gen_skey('k_person') }}, + survey_section_response_person_target_associations.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from survey_section_response_person_target_associations +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_survey_section_response_person_target_association', + order_by='last_modified_timestamp desc, pull_timestamp desc') + }} +) +select * from deduped +where not is_deleted From 0d53c5210818b833cdcc6d936e4d598462ebc67d Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Thu, 3 Jul 2025 12:08:33 -0500 Subject: [PATCH 02/35] Modify attribute order in base model and add relevant key to the gen_skey macro --- macros/gen_skey.sql | 5 +++++ ...survey_section_response_person_target_associations.sql | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index 939f7975..e2b781ba 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -342,6 +342,11 @@ 'col_list': [], 'annualize': False }, + 'k_survey_section_response': { + 'reference_name': 'survey_section_response_reference', + 'col_list': ['surveyIdentifier', 'namespace', 'surveySectionTitle'], + 'annualize': True + } } %} diff --git a/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql b/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql index 563b7db3..c2b2edbf 100644 --- a/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql +++ b/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql @@ -13,12 +13,12 @@ renamed as ( v:id::string as record_guid, -- identity components - v:surveySectionResponseReference:namespace::string as namespace, - v:personReference:personId::string as person_id, - {{ extract_descriptor('v:personReference:sourceSystemDescriptor::string') }} as source_system, v:surveySectionResponseReference:surveyIdentifier::string as survey_id, + v:surveySectionResponseReference:namespace::string as namespace, + v:surveySectionResponseReference:surveySectionTitle::string as survey_section_title, v:surveySectionResponseReference:surveyResponseIdentifier::string as survey_response_id, - v:surveySectionResponseReference:surveySectionTitle::string as survey_section_title, + v:personReference:personId::string as person_id, + {{ extract_descriptor('v:personReference:sourceSystemDescriptor::string') }} as source_system, -- references v:surveySectionResponseReference as survey_section_response_reference, v:personReference as person_reference, From 31cad930a3be22b14eeb63aca30ecfba8e497653 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Fri, 11 Jul 2025 16:12:57 -0500 Subject: [PATCH 03/35] Create base/stg models for application_events --- macros/gen_skey.sql | 5 +++ .../base/base_tpdm__application_events.sql | 36 +++++++++++++++++++ .../stage/stg_tpdm__application_events.sql | 22 ++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 models/staging/tpdm/base/base_tpdm__application_events.sql create mode 100644 models/staging/tpdm/stage/stg_tpdm__application_events.sql diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index e2b781ba..a190614d 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -346,6 +346,11 @@ 'reference_name': 'survey_section_response_reference', 'col_list': ['surveyIdentifier', 'namespace', 'surveySectionTitle'], 'annualize': True + }, + 'k_application': { + 'reference_name': 'application_reference', + 'col_list': ['applicantProfileIdentifier', 'applicationIdentifier', 'ed_org_id'], + 'annualize': True, } } diff --git a/models/staging/tpdm/base/base_tpdm__application_events.sql b/models/staging/tpdm/base/base_tpdm__application_events.sql new file mode 100644 index 00000000..bdcc10cb --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__application_events.sql @@ -0,0 +1,36 @@ +with app_events as ( + {{ source_edfi3('application_events') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:sequenceNumber::int as sequence_number, + v:eventDate::string as event_date, -- this could also be event_start_date + {{ extract_descriptor('v:applicationEventTypeDescriptor::string') }} as application_event_type, + v:applicationReference:applicationIdentifier::string as application_identifier, + v:applicationReference:applicantProfileIdentifier::string as applicant_profile_identifier, + v:applicationReference:educationOrganizationId::int as education_organization_id, + -- non-identity components + v:eventEndDate::string as event_end_date + v:applicationEvaluationScore::float as application_evaluation_score + v:schoolYearTypeReference:schoolYear:int as school_year + -- references + v:applicationReference as application_reference + -- descriptors + {{ extract_descriptor('v:applicationEventResultDescriptor::string') }} as application_event_result_descriptor + {{ extract_descriptor('v:termDescriptor::string') }} as term + -- edfi extensions + v:_ext as v_ext + + from app_events +) +select * from renamed diff --git a/models/staging/tpdm/stage/stg_tpdm__application_events.sql b/models/staging/tpdm/stage/stg_tpdm__application_events.sql new file mode 100644 index 00000000..246dcea1 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__application_events.sql @@ -0,0 +1,22 @@ +with base_app_events as ( + select * from {{ ref('base_tpdm__application_events')}} +), +keyed as ( + select + {{ gen_skey('k_application') }}, + {{ gen_skey('k_ed_org') }}, + * + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_app_events +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_school, k_student, restraint_event_identifier', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file From 72b398f440353239e440b47e91ecf0b85c15a897 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Tue, 15 Jul 2025 17:25:47 -0500 Subject: [PATCH 04/35] Update naming in staging models --- ...dm__survey_section_response_person_target_associations.sql | 4 ++-- ...dm__survey_section_response_person_target_associations.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql b/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql index c2b2edbf..cc92b7bc 100644 --- a/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql +++ b/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql @@ -16,12 +16,12 @@ renamed as ( v:surveySectionResponseReference:surveyIdentifier::string as survey_id, v:surveySectionResponseReference:namespace::string as namespace, v:surveySectionResponseReference:surveySectionTitle::string as survey_section_title, - v:surveySectionResponseReference:surveyResponseIdentifier::string as survey_response_id, + v:surveySectionResponseReference:surveyResponseIdentifier::string as survey_response_identifier, v:personReference:personId::string as person_id, {{ extract_descriptor('v:personReference:sourceSystemDescriptor::string') }} as source_system, -- references v:surveySectionResponseReference as survey_section_response_reference, - v:personReference as person_reference, + v:personReference as person_reference, -- edfi extensions v:_ext as v_ext from survey_section_response_person_target_associations diff --git a/models/staging/tpdm/stage/stg_tpdm__survey_section_response_person_target_associations.sql b/models/staging/tpdm/stage/stg_tpdm__survey_section_response_person_target_associations.sql index d77290c8..702210c7 100644 --- a/models/staging/tpdm/stage/stg_tpdm__survey_section_response_person_target_associations.sql +++ b/models/staging/tpdm/stage/stg_tpdm__survey_section_response_person_target_associations.sql @@ -10,7 +10,7 @@ keyed as ( 'lower(person_id)', 'lower(source_system)', 'lower(survey_id)', - 'lower(survey_response_id)', + 'lower(survey_response_identifier)', 'lower(survey_section_title)'] ) }} as k_survey_section_response_person_target_association, {{ gen_skey('k_survey_section_response') }}, From 78a2569b1970f728d25ba8c826b8785c9a1d40c8 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Tue, 15 Jul 2025 17:26:03 -0500 Subject: [PATCH 05/35] Fix gen_skey column naming --- macros/gen_skey.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index a190614d..390c7f27 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -229,7 +229,7 @@ 'k_survey_section': { 'reference_name': 'survey_section_reference', - 'col_list': ['namespace', 'surveyIdentifier', 'survey_section_title'], + 'col_list': ['namespace', 'surveyIdentifier', 'surveySectionTitle'], 'annualize': True }, @@ -349,7 +349,7 @@ }, 'k_application': { 'reference_name': 'application_reference', - 'col_list': ['applicantProfileIdentifier', 'applicationIdentifier', 'ed_org_id'], + 'col_list': ['applicantProfileIdentifier', 'applicationIdentifier', 'educationOrganizationId'], 'annualize': True, } From 90064606f56bbf9c62d0d32f9ea34dba8b9e7a4b Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Tue, 15 Jul 2025 17:34:16 -0500 Subject: [PATCH 06/35] Fix dtype for date in base model --- models/staging/tpdm/base/base_tpdm__application_events.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/staging/tpdm/base/base_tpdm__application_events.sql b/models/staging/tpdm/base/base_tpdm__application_events.sql index bdcc10cb..639d7630 100644 --- a/models/staging/tpdm/base/base_tpdm__application_events.sql +++ b/models/staging/tpdm/base/base_tpdm__application_events.sql @@ -14,13 +14,13 @@ renamed as ( v:id::string as record_guid, -- identity components v:sequenceNumber::int as sequence_number, - v:eventDate::string as event_date, -- this could also be event_start_date + v:eventDate::date as event_date, -- this could also be event_start_date {{ extract_descriptor('v:applicationEventTypeDescriptor::string') }} as application_event_type, v:applicationReference:applicationIdentifier::string as application_identifier, v:applicationReference:applicantProfileIdentifier::string as applicant_profile_identifier, v:applicationReference:educationOrganizationId::int as education_organization_id, -- non-identity components - v:eventEndDate::string as event_end_date + v:eventEndDate::date as event_end_date v:applicationEvaluationScore::float as application_evaluation_score v:schoolYearTypeReference:schoolYear:int as school_year -- references From 78fc5da2263a804aeb5a086d56548dd93e60cefa Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Wed, 16 Jul 2025 10:58:14 -0500 Subject: [PATCH 07/35] Fix stage model for application_events --- models/staging/tpdm/stage/stg_tpdm__application_events.sql | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/models/staging/tpdm/stage/stg_tpdm__application_events.sql b/models/staging/tpdm/stage/stg_tpdm__application_events.sql index 246dcea1..15f4effa 100644 --- a/models/staging/tpdm/stage/stg_tpdm__application_events.sql +++ b/models/staging/tpdm/stage/stg_tpdm__application_events.sql @@ -4,8 +4,7 @@ with base_app_events as ( keyed as ( select {{ gen_skey('k_application') }}, - {{ gen_skey('k_ed_org') }}, - * + base_app_events.* {{ extract_extension(model_name=this.name, flatten=True) }} from base_app_events ), @@ -13,7 +12,7 @@ deduped as ( {{ dbt_utils.deduplicate( relation='keyed', - partition_by='k_school, k_student, restraint_event_identifier', + partition_by='k_application, sequence_number, event_date, application_event_type', order_by='last_modified_timestamp desc, pull_timestamp desc' ) }} From eade733eee53c39ff807da5dbecaf4bd9c29ddf4 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Wed, 16 Jul 2025 11:03:07 -0500 Subject: [PATCH 08/35] Base/stg models for StaffEducatorPreparationProgramAssociation --- ...cator_preparation_program_associations.sql | 32 +++++++++++++++++++ ...cator_preparation_program_associations.sql | 22 +++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 models/staging/tpdm/base/base_tpdm_staff_educator_preparation_program_associations.sql create mode 100644 models/staging/tpdm/stage/stg_tpdm_staff_educator_preparation_program_associations.sql diff --git a/models/staging/tpdm/base/base_tpdm_staff_educator_preparation_program_associations.sql b/models/staging/tpdm/base/base_tpdm_staff_educator_preparation_program_associations.sql new file mode 100644 index 00000000..af389cc8 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm_staff_educator_preparation_program_associations.sql @@ -0,0 +1,32 @@ +with staff_educator_preparation_program_associations as ( + {{ source_edfi3('staff_educator_preparation_program_associations') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:staffReference:staffUniqueId::string as staff_id, + v:educatorPreparationProgramReference:educationOrganizationId::int as ed_org_id, + v:educatorPreparationProgramReference:programName::string as program_name, + {{ extract_descriptor('v:educatorPreparationProgramReference:programTypeDescriptor') }} as program_type, + -- non-identity components + v:beginDate::date as begin_date, + v:educatorPreparationProgramReference:link:rel::string as ed_org_type, -- this was declareed in base_tpdm__candidate_educator_preparation_program_associations, is it correct? + v:endDate::date as end_date, + v:completer::boolean as is_completer, + -- references + v:staffReference as staff_reference, + v:educatorPreparationProgramReference as educator_preparation_program_reference, + -- edfi extensions + v:_ext as v_ext + from staff_educator_preparation_program_associations +) +select * from renamed diff --git a/models/staging/tpdm/stage/stg_tpdm_staff_educator_preparation_program_associations.sql b/models/staging/tpdm/stage/stg_tpdm_staff_educator_preparation_program_associations.sql new file mode 100644 index 00000000..8d4f4482 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm_staff_educator_preparation_program_associations.sql @@ -0,0 +1,22 @@ +with staff_educator_preparation_program_associations as ( + select * from {{ ref('base_tpdm__staff_educator_preparation_program_associations') }} +), +keyed as ( + select + {{ gen_skey('k_staff') }}, + {{ gen_skey('k_educator_prep_program') }}, + staff_educator_preparation_program_associations.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from staff_educator_preparation_program_associations +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_staff, k_educator_prep_program', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted From 6f0f87ddaa387dd577fc5fa306ffd2b35de12867 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Wed, 16 Jul 2025 11:32:02 -0500 Subject: [PATCH 09/35] Base/stg models for CredentialEvent --- macros/gen_skey.sql | 5 ++++ .../base/base_tpdm__credential_events.sql | 29 +++++++++++++++++++ .../stage/stg_tpdm__credential_events.sql | 21 ++++++++++++++ 3 files changed, 55 insertions(+) create mode 100644 models/staging/tpdm/base/base_tpdm__credential_events.sql create mode 100644 models/staging/tpdm/stage/stg_tpdm__credential_events.sql diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index 390c7f27..a7aa1127 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -351,6 +351,11 @@ 'reference_name': 'application_reference', 'col_list': ['applicantProfileIdentifier', 'applicationIdentifier', 'educationOrganizationId'], 'annualize': True, + }, + 'k_credential': { + 'reference_name': 'credential_reference', + 'col_list': ['credentialIdentifier', 'stateOfIssueStateAbbreviationDescriptor'], + 'annualize': True, } } diff --git a/models/staging/tpdm/base/base_tpdm__credential_events.sql b/models/staging/tpdm/base/base_tpdm__credential_events.sql new file mode 100644 index 00000000..178785ce --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__credential_events.sql @@ -0,0 +1,29 @@ +with app_events as ( + {{ source_edfi3('application_events') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:credentialEventDate::date as credential_event_date, + {{ extract_descriptor('v:credentialEventTypeDescriptor::string') }} as credential_event_type, + v:credentialReference:credentialIdentifier::string as credential_identifier, + {{ extract_descriptor('v:credentialReference:stateOfIssueStateAbbreviationDescriptor::string') }} as state_of_issue, + -- non-identity components + v:credentialEventReason::string as credential_event_reason + -- references + v:credentialReference as credential_reference + -- edfi extensions + v:_ext as v_ext + + from app_events +) +select * from renamed diff --git a/models/staging/tpdm/stage/stg_tpdm__credential_events.sql b/models/staging/tpdm/stage/stg_tpdm__credential_events.sql new file mode 100644 index 00000000..30b65572 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__credential_events.sql @@ -0,0 +1,21 @@ +with base_credential_events as ( + select * from {{ ref('base_tpdm__credential_events')}} +), +keyed as ( + select + {{ gen_skey('k_credential') }}, + base_credential_events.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_credential_events +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_credential, credential_event_date, credential_event_type, credential_identifier', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file From 8f03b6cd658a4760ccd676ee3da6c6b1891db70a Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Wed, 16 Jul 2025 16:13:48 -0500 Subject: [PATCH 10/35] Base/stg models for FieldworkExperienceSectionAssociation --- macros/gen_skey.sql | 12 ++++++- ...ldwork_experience_section_associations.sql | 31 +++++++++++++++++++ ...ldwork_experience_section_associations.sql | 22 +++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 models/staging/tpdm/base/base_tpdm__fieldwork_experience_section_associations.sql create mode 100644 models/staging/tpdm/stage/stg_tpdm__fieldwork_experience_section_associations.sql diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index a7aa1127..e586ffd2 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -356,7 +356,17 @@ 'reference_name': 'credential_reference', 'col_list': ['credentialIdentifier', 'stateOfIssueStateAbbreviationDescriptor'], 'annualize': True, - } + }, + 'k_fieldwork_experience': { + 'reference_name': 'fieldwork_experience_reference', + 'col_list': ['beginDate', 'fieldworkIdentifier', 'studentUniqueId'], + 'annualize': True, + }, + 'k_section': { + 'reference_name': 'section_reference', + 'col_list': ['localCourseCode', 'schoolId', 'schoolYear', 'sectionIdentifier', 'sessionName'], + 'annualize': False, + } } %} diff --git a/models/staging/tpdm/base/base_tpdm__fieldwork_experience_section_associations.sql b/models/staging/tpdm/base/base_tpdm__fieldwork_experience_section_associations.sql new file mode 100644 index 00000000..d12b5691 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__fieldwork_experience_section_associations.sql @@ -0,0 +1,31 @@ +with student_section_associations as ( + {{ source_edfi3('student_section_associations') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:fieldworkExperienceReference:beginDate::date as begin_date, + v:fieldworkExperienceReference:fieldworkIdentifier::string as fieldwork_identifier, + v:fieldworkExperienceReference:studentUniqueId::string as student_unique_id, + v:sectionReference:sectionIdentifier::string as section_identifier, + v:sectionReference:localCourseCode::string as local_course_code, + v:sectionReference:sessionName::string as session_name, + v:sectionReference:schoolId::int as school_id, + v:sectionReference:schoolYear::int as school_year, + -- references + v:fieldworkExperienceReference as fieldwork_experience_reference, + v:sectionReference as section_reference, + -- edfi extensions + v:_ext as v_ext + from student_section_associations +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__fieldwork_experience_section_associations.sql b/models/staging/tpdm/stage/stg_tpdm__fieldwork_experience_section_associations.sql new file mode 100644 index 00000000..b2ee0a9d --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__fieldwork_experience_section_associations.sql @@ -0,0 +1,22 @@ +with base_student_section_associations as ( + select * from {{ ref('stg_tpdm__fieldwork_experience_section_associations') }} +), +keyed as ( + select + {{ gen_skey('k_fieldwork_experience') }}, + {{ gen_skey('k_section') }}, -- should this be k_course_section instead? + base_student_section_associations.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_student_section_associations +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_fieldwork_experience, k_section', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file From e0f7b0b71cb06ebedcac8280ed52e323b6cb4325 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Wed, 16 Jul 2025 17:02:30 -0500 Subject: [PATCH 11/35] Base/stg models for Goal --- macros/gen_skey.sql | 15 +++++--- models/staging/tpdm/base/base_tpdm__goals.sql | 37 +++++++++++++++++++ models/staging/tpdm/stage/stg_tpdm__goals.sql | 34 +++++++++++++++++ 3 files changed, 81 insertions(+), 5 deletions(-) create mode 100644 models/staging/tpdm/base/base_tpdm__goals.sql create mode 100644 models/staging/tpdm/stage/stg_tpdm__goals.sql diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index e586ffd2..f57dedf7 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -350,23 +350,28 @@ 'k_application': { 'reference_name': 'application_reference', 'col_list': ['applicantProfileIdentifier', 'applicationIdentifier', 'educationOrganizationId'], - 'annualize': True, + 'annualize': True }, 'k_credential': { 'reference_name': 'credential_reference', 'col_list': ['credentialIdentifier', 'stateOfIssueStateAbbreviationDescriptor'], - 'annualize': True, + 'annualize': True }, 'k_fieldwork_experience': { 'reference_name': 'fieldwork_experience_reference', 'col_list': ['beginDate', 'fieldworkIdentifier', 'studentUniqueId'], - 'annualize': True, + 'annualize': True }, 'k_section': { 'reference_name': 'section_reference', 'col_list': ['localCourseCode', 'schoolId', 'schoolYear', 'sectionIdentifier', 'sessionName'], - 'annualize': False, - } + 'annualize': False + }, + 'k_parent_goal': { + 'reference_name': 'parent_goal_reference', + 'col_list': ['assignmentDate', 'goalTitle', 'personId', 'sourceSystemDescriptor'], + 'annualize': False + } } %} diff --git a/models/staging/tpdm/base/base_tpdm__goals.sql b/models/staging/tpdm/base/base_tpdm__goals.sql new file mode 100644 index 00000000..9cdab378 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__goals.sql @@ -0,0 +1,37 @@ +with base_goals as ( + select * from {{ source_edfi3('goals') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:personReference:personId::string as person_id, + v:personReference:sourceSystemDescriptor::string as person_source_system, + v:goalTitle::string as goal_title, + v:assignmentDate::date as assignment_date, + -- non-identity components + v:goalDescription::string as goal_description, + v:comments::string as comments, + v:completed_indicator::boolean as is_completed, + v:completedDate::date as completed_date, + v:dueDate::date as due_date, + -- descriptors + {{ extract_descriptor('v:goalTypeDescriptor::string') }} as goal_type, + -- references + v:evaluationElementReference as evaluation_element_reference, + v:evaluationObjectiveReference as evaluation_objective_reference, + v:parentGoalReference as parent_goal_reference, + v:personReference as person_reference, + -- edfi extensions + v:_ext as v_ext + from base_goals +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__goals.sql b/models/staging/tpdm/stage/stg_tpdm__goals.sql new file mode 100644 index 00000000..774851a0 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__goals.sql @@ -0,0 +1,34 @@ +with base_goals as ( + select * from {{ ref('base_tpdm__goals') }} +), +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'assignment_date', + 'lower(goal_title)', + 'lower(person_id)', + 'lower(person_source_system)' + ] + ) }} as k_goal, + {{ gen_skey('k_person') }}, + {{ gen_skey('k_evaluation_element') }}, + {{ gen_skey('k_evaluation_objective') }}, + {{ gen_skey('k_parent_goal') }}, + base_goals.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_goals +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_goal', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file From 75cf15d7386c787f755d47a94f358d4f9994c284 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Thu, 17 Jul 2025 13:02:01 -0500 Subject: [PATCH 12/35] Fix k_goal in stg model --- macros/gen_skey.sql | 7 +------ models/staging/tpdm/stage/stg_tpdm__goals.sql | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index f57dedf7..cd118d38 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -366,12 +366,7 @@ 'reference_name': 'section_reference', 'col_list': ['localCourseCode', 'schoolId', 'schoolYear', 'sectionIdentifier', 'sessionName'], 'annualize': False - }, - 'k_parent_goal': { - 'reference_name': 'parent_goal_reference', - 'col_list': ['assignmentDate', 'goalTitle', 'personId', 'sourceSystemDescriptor'], - 'annualize': False - } + } } %} diff --git a/models/staging/tpdm/stage/stg_tpdm__goals.sql b/models/staging/tpdm/stage/stg_tpdm__goals.sql index 774851a0..74c1b1d6 100644 --- a/models/staging/tpdm/stage/stg_tpdm__goals.sql +++ b/models/staging/tpdm/stage/stg_tpdm__goals.sql @@ -16,7 +16,7 @@ keyed as ( {{ gen_skey('k_person') }}, {{ gen_skey('k_evaluation_element') }}, {{ gen_skey('k_evaluation_objective') }}, - {{ gen_skey('k_parent_goal') }}, + {{ gen_skey('k_goal', alt_ref='parent_goal_reference', alt_k_name='k_goal__parent') }}, base_goals.* {{ extract_extension(model_name=this.name, flatten=True) }} from base_goals From c1d5324860aeca3a3de0af64a9189f6fe3df9c77 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Thu, 17 Jul 2025 15:19:10 -0500 Subject: [PATCH 13/35] Create base/stg models for OpenStaffPositionEvent --- macros/gen_skey.sql | 7 ++++- .../base_tpdm__open_staff_position_events.sql | 29 +++++++++++++++++++ .../stg_tpdm__open_staff_position_events.sql | 21 ++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 models/staging/tpdm/base/base_tpdm__open_staff_position_events.sql create mode 100644 models/staging/tpdm/stage/stg_tpdm__open_staff_position_events.sql diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index cd118d38..18074430 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -366,7 +366,12 @@ 'reference_name': 'section_reference', 'col_list': ['localCourseCode', 'schoolId', 'schoolYear', 'sectionIdentifier', 'sessionName'], 'annualize': False - } + }, + 'k_open_staff_position': { + 'reference_name': 'open_staff_position_reference', + 'col_list': ['educationOrganizationId', 'eventDate', 'openStaffPositionEventTypeDescriptor', 'requisitionNumber'], + 'annualize': False + } } %} diff --git a/models/staging/tpdm/base/base_tpdm__open_staff_position_events.sql b/models/staging/tpdm/base/base_tpdm__open_staff_position_events.sql new file mode 100644 index 00000000..aff3b7d8 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__open_staff_position_events.sql @@ -0,0 +1,29 @@ +with open_staff_position_events as ( + {{ source_edfi3('open_staff_position_events') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:eventDate::date as event_date, + {{ extract_descriptor('v:openStaffPositionEventTypeDescriptor::string') }} as open_staff_position_event_type, -- name event_type instead? + v:openStaffPositionReference:educationOrganizationId::int as ed_org_id, + v:openStaffPositionReference:requisitionNumber::string as requisition_number, + -- descriptors + {{ extract_descriptor('v:openStaffPositionEventStatusDescriptor::string') }} as open_staff_position_event_status, -- name event_status instead? + -- references + v:openStaffPositionReference as open_staff_position_reference, + -- edfi extensions + v:_ext as v_ext + + from open_staff_position_events +) +select * from renamed diff --git a/models/staging/tpdm/stage/stg_tpdm__open_staff_position_events.sql b/models/staging/tpdm/stage/stg_tpdm__open_staff_position_events.sql new file mode 100644 index 00000000..1f7659a8 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__open_staff_position_events.sql @@ -0,0 +1,21 @@ +with open_staff_position_events as ( + select * from {{ ref('base_tpdm__open_staff_position_events')}} +), +keyed as ( + select + {{ gen_skey('k_open_staff_position') }}, + open_staff_position_events.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from open_staff_position_events +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_open_staff_position, event_date, open_staff_position_event_type', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file From e26e8a497d2dc5f5c7bf7219a185ac83ce6c8590 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Thu, 17 Jul 2025 15:41:33 -0500 Subject: [PATCH 14/35] Create base/stg models for ProfessionalDevelopmentEvent --- ..._tpdm__professional_development_events.sql | 32 +++++++++++++++++++ ..._tpdm__professional_development_events.sql | 28 ++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 models/staging/tpdm/base/base_tpdm__professional_development_events.sql create mode 100644 models/staging/tpdm/stage/stg_tpdm__professional_development_events.sql diff --git a/models/staging/tpdm/base/base_tpdm__professional_development_events.sql b/models/staging/tpdm/base/base_tpdm__professional_development_events.sql new file mode 100644 index 00000000..26b9a287 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__professional_development_events.sql @@ -0,0 +1,32 @@ +with professional_development_events as ( + {{ source_edfi3('professional_development_events') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:professionalDevelopmentTitle::string as professional_development_title, + v:namespace::string as namespace, + -- non-identity components + v:multipleSession::boolean as is_multiple_session, + v:professionalDevelopmentReason::string as professional_development_reason, + v:required::boolean as is_required, + v:totalHoura::integer as total_hours, -- should this be float or number type instead? + -- descriptors + {{ extract_descriptor('v:professionalDevelopmentOfferedByDescriptor::string') }} as offered_by, + -- references + v:openStaffPositionReference as open_staff_position_reference, + -- edfi extensions + v:_ext as v_ext + + from professional_development_events +) +select * from renamed diff --git a/models/staging/tpdm/stage/stg_tpdm__professional_development_events.sql b/models/staging/tpdm/stage/stg_tpdm__professional_development_events.sql new file mode 100644 index 00000000..39f779bb --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__professional_development_events.sql @@ -0,0 +1,28 @@ +with professional_dev_events as ( + select * from {{ ref('base_tpdm__professional_development_events') }} +), +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'lower(namespace)', + 'lower(professional_development_title)' + ] + ) }} as k_professional_development_event, + professional_dev_events.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from professional_dev_events +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_professional_development_event', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file From db61ff6adc269db1a5d33b4011abcd6a78b38306 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Thu, 17 Jul 2025 15:46:24 -0500 Subject: [PATCH 15/35] Fix base model --- .../tpdm/base/base_tpdm__professional_development_events.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/models/staging/tpdm/base/base_tpdm__professional_development_events.sql b/models/staging/tpdm/base/base_tpdm__professional_development_events.sql index 26b9a287..5d87f3ab 100644 --- a/models/staging/tpdm/base/base_tpdm__professional_development_events.sql +++ b/models/staging/tpdm/base/base_tpdm__professional_development_events.sql @@ -19,11 +19,9 @@ renamed as ( v:multipleSession::boolean as is_multiple_session, v:professionalDevelopmentReason::string as professional_development_reason, v:required::boolean as is_required, - v:totalHoura::integer as total_hours, -- should this be float or number type instead? + v:totalHours::integer as total_hours, -- should this be float or number type instead? -- descriptors {{ extract_descriptor('v:professionalDevelopmentOfferedByDescriptor::string') }} as offered_by, - -- references - v:openStaffPositionReference as open_staff_position_reference, -- edfi extensions v:_ext as v_ext From 576f6c5ec0843f6bb51d9bab778f80251d6bbf96 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Thu, 17 Jul 2025 16:13:26 -0500 Subject: [PATCH 16/35] Create base/stg models for ProfessionalDevelopmentEventAttendance --- macros/gen_skey.sql | 7 +++- ...essional_development_event_attendances.sql | 32 +++++++++++++++++++ ...essional_development_event_attendances.sql | 22 +++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 models/staging/tpdm/base/base_tpdm__professional_development_event_attendances.sql create mode 100644 models/staging/tpdm/stage/stg_tpdm__professional_development_event_attendances.sql diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index 18074430..0434f872 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -370,7 +370,12 @@ 'k_open_staff_position': { 'reference_name': 'open_staff_position_reference', 'col_list': ['educationOrganizationId', 'eventDate', 'openStaffPositionEventTypeDescriptor', 'requisitionNumber'], - 'annualize': False + 'annualize': True + }, + 'k_professional_development_event': { + 'reference_name': 'professional_development_event_reference', + 'col_list': ['namespace', 'professionalDevelopmentTitle'], + 'annualize': True } } diff --git a/models/staging/tpdm/base/base_tpdm__professional_development_event_attendances.sql b/models/staging/tpdm/base/base_tpdm__professional_development_event_attendances.sql new file mode 100644 index 00000000..c32b15e6 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__professional_development_event_attendances.sql @@ -0,0 +1,32 @@ +with professional_development_event_attendances as ( + {{ source_edfi3('professional_development_event_attendances') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:attendanceDate::int as attendance_date, + v:personReference:personId::string as person_id, + {{ extract_descriptor('v:personReference:sourceSystemDescriptor::string') }} as person_source_system, + v:professionalDevelopmentEventReference:namespace::string as professional_development_event_namespace, + v:professionalDevelopmentEventReference:professionalDevelopmentTitle::string as professional_development_title, + -- non-identity components + v:attendanceEventReason::string as attendance_event_reason, + -- descriptors + {{ extract_descriptor('v:attendanceEventCategoryDescriptor::string') }} as attendance_event_category, + -- references + v:personReference as person_reference, + v:professionalDevelopmentEventReference as professional_development_event_reference, + -- edfi extensions + v:_ext as v_ext + from professional_development_event_attendances +) +select * from renamed diff --git a/models/staging/tpdm/stage/stg_tpdm__professional_development_event_attendances.sql b/models/staging/tpdm/stage/stg_tpdm__professional_development_event_attendances.sql new file mode 100644 index 00000000..e62c7a68 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__professional_development_event_attendances.sql @@ -0,0 +1,22 @@ +with professional_dev_event_attendances as ( + select * from {{ ref('base_tpdm__professional_development_event_attendances')}} +), +keyed as ( + select + {{ gen_skey('k_person') }}, + {{ gen_skey('k_professional_development_event') }}, + professional_dev_event_attendances.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from professional_dev_event_attendances +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_person, k_professional_development_event, attendance_date', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file From b1721ebfd87c9f27761b3196045f954fb96e1c91 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Thu, 17 Jul 2025 16:15:05 -0500 Subject: [PATCH 17/35] Fix date dtype --- .../base_tpdm__professional_development_event_attendances.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/staging/tpdm/base/base_tpdm__professional_development_event_attendances.sql b/models/staging/tpdm/base/base_tpdm__professional_development_event_attendances.sql index c32b15e6..d62669dd 100644 --- a/models/staging/tpdm/base/base_tpdm__professional_development_event_attendances.sql +++ b/models/staging/tpdm/base/base_tpdm__professional_development_event_attendances.sql @@ -13,7 +13,7 @@ renamed as ( v:id::string as record_guid, -- identity components - v:attendanceDate::int as attendance_date, + v:attendanceDate::date as attendance_date, v:personReference:personId::string as person_id, {{ extract_descriptor('v:personReference:sourceSystemDescriptor::string') }} as person_source_system, v:professionalDevelopmentEventReference:namespace::string as professional_development_event_namespace, From 44f73d6d1140de78abffdd6c451c461e9f8e76fa Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Thu, 17 Jul 2025 17:10:07 -0500 Subject: [PATCH 18/35] Create base/stg models for SurveySectionAggregateResponse --- macros/gen_skey.sql | 18 ++++++++- ...dm__survey_section_aggregate_responses.sql | 40 +++++++++++++++++++ ...dm__survey_section_aggregate_responses.sql | 21 ++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 models/staging/tpdm/base/base_tpdm__survey_section_aggregate_responses.sql create mode 100644 models/staging/tpdm/stage/stg_tpdm__survey_section_aggregate_responses.sql diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index 0434f872..3fc7c5e5 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -376,7 +376,23 @@ 'reference_name': 'professional_development_event_reference', 'col_list': ['namespace', 'professionalDevelopmentTitle'], 'annualize': True - } + }, + 'k_evaluation_element_rating': { + 'reference_name': 'evaluation_element_rating_reference', + 'col_list': ['educationOrganizationId', + 'evaluationDate', + 'evaluationElementTitle', + 'evaluationObjectiveTitle', + 'evaluationPeriodDescriptor', + 'evaluationTitle', + 'performanceEvaluationTitle', + 'performanceEvaluationTypeDescriptor', + 'personId', + 'schoolYear', + 'sourceSystemDescriptor', + 'termDescriptor'], + 'annualize': True + } } %} diff --git a/models/staging/tpdm/base/base_tpdm__survey_section_aggregate_responses.sql b/models/staging/tpdm/base/base_tpdm__survey_section_aggregate_responses.sql new file mode 100644 index 00000000..9afbd19a --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__survey_section_aggregate_responses.sql @@ -0,0 +1,40 @@ +with survey_section_aggregate_responses as ( + {{ source_edfi3('survey_section_aggregate_responses') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:evaluationElementRatingReference:educationOrganizationId::string as ed_org_id, + v:evaluationElementRatingReference:evaluationDate::string as evaluation_date, + v:evaluationElementRatingReference:evaluationElementTitle::string as evaluation_element_title, + {{ extract_descriptor('v:evaluationElementRatingReference:sourceSystemDescriptor::string') }} as evaluation_element_rating_source_system, + v:evaluationElementRatingReference:evaluationObjectiveTitle::string as evaluation_objective_title, + {{ extract_descriptor('v:evaluationElementRatingReference:evaluationPeriodDescriptor::string') }} as evaluation_period, + v:evaluationElementRatingReference:evaluationTitle::string as evaluation_title, + v:evaluationElementRatingReference:performanceEvaluationTitle::string as performance_evaluation_title, + {{ extract_descriptor('v:evaluationElementRatingReference:performanceEvaluationTypeDescriptor::string') }} as performance_evaluation_type, + v:evaluationElementRatingReference:personId::string as person_id, + v:evaluationElementRatingReference:schoolYear::string as school_year, + {{ extract_descriptor('v:evaluationElementRatingReference:termDescriptor::string') }} as academic_term, + v:surveySectionReference:surveyIdentifier::string as survey_identifier, + v:surveySectionReference:namespace::string as survey_section_namespace, + v:surveySectionReference:surveySectionTitle::string as survey_section_title, + -- non-identity components + v:scoreValue::float as score_value, + -- references + v:evaluationElementRatingReference as evaluation_element_rating_reference, + v:surveySectionReference as survey_section_reference, + -- edfi extensions + v:_ext as v_ext + from survey_section_aggregate_responses +) +select * from renamed diff --git a/models/staging/tpdm/stage/stg_tpdm__survey_section_aggregate_responses.sql b/models/staging/tpdm/stage/stg_tpdm__survey_section_aggregate_responses.sql new file mode 100644 index 00000000..afd5802f --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__survey_section_aggregate_responses.sql @@ -0,0 +1,21 @@ +with base_survey_section_aggregate_responses as ( + select * from {{ ref('base_tpdm__survey_section_aggregate_responses') }} +), +keyed as ( + select + {{ gen_skey('k_evaluation_element_rating') }}, + {{ gen_skey('k_survey_section') }}, + base_survey_section_aggregate_responses.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_survey_section_aggregate_responses +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_survey_section_response, k_survey_section', + order_by='last_modified_timestamp desc, pull_timestamp desc') + }} +) +select * from deduped +where not is_deleted From 09c3960ef35801798b1278dd3d4b30b32d48e856 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Thu, 17 Jul 2025 17:10:47 -0500 Subject: [PATCH 19/35] Fix base/stg models --- .../tpdm/base/base_tpdm__application_events.sql | 4 ++-- ...y_section_response_person_target_associations.sql | 12 +----------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/models/staging/tpdm/base/base_tpdm__application_events.sql b/models/staging/tpdm/base/base_tpdm__application_events.sql index 639d7630..830f8d16 100644 --- a/models/staging/tpdm/base/base_tpdm__application_events.sql +++ b/models/staging/tpdm/base/base_tpdm__application_events.sql @@ -18,7 +18,7 @@ renamed as ( {{ extract_descriptor('v:applicationEventTypeDescriptor::string') }} as application_event_type, v:applicationReference:applicationIdentifier::string as application_identifier, v:applicationReference:applicantProfileIdentifier::string as applicant_profile_identifier, - v:applicationReference:educationOrganizationId::int as education_organization_id, + v:applicationReference:educationOrganizationId::int as ed_org_id, -- non-identity components v:eventEndDate::date as event_end_date v:applicationEvaluationScore::float as application_evaluation_score @@ -27,7 +27,7 @@ renamed as ( v:applicationReference as application_reference -- descriptors {{ extract_descriptor('v:applicationEventResultDescriptor::string') }} as application_event_result_descriptor - {{ extract_descriptor('v:termDescriptor::string') }} as term + {{ extract_descriptor('v:termDescriptor::string') }} as academic_term -- edfi extensions v:_ext as v_ext diff --git a/models/staging/tpdm/stage/stg_tpdm__survey_section_response_person_target_associations.sql b/models/staging/tpdm/stage/stg_tpdm__survey_section_response_person_target_associations.sql index 702210c7..7015ce26 100644 --- a/models/staging/tpdm/stage/stg_tpdm__survey_section_response_person_target_associations.sql +++ b/models/staging/tpdm/stage/stg_tpdm__survey_section_response_person_target_associations.sql @@ -3,16 +3,6 @@ with survey_section_response_person_target_associations as ( ), keyed as ( select - {{ dbt_utils.generate_surrogate_key( - ['tenant_code', - 'api_year', - 'lower(namespace)', - 'lower(person_id)', - 'lower(source_system)', - 'lower(survey_id)', - 'lower(survey_response_identifier)', - 'lower(survey_section_title)'] - ) }} as k_survey_section_response_person_target_association, {{ gen_skey('k_survey_section_response') }}, {{ gen_skey('k_person') }}, survey_section_response_person_target_associations.* @@ -23,7 +13,7 @@ deduped as ( {{ dbt_utils.deduplicate( relation='keyed', - partition_by='k_survey_section_response_person_target_association', + partition_by='k_survey_section_response, k_person', order_by='last_modified_timestamp desc, pull_timestamp desc') }} ) From e7f069fee2ad97c590ab184607fe88d1b8255797 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Fri, 18 Jul 2025 09:49:11 -0500 Subject: [PATCH 20/35] Create base/stg models for QuantitativeMeasure --- .../base/base_tpdm__quantitative_measures.sql | 36 ++++++++++++++++++ .../stage/stg_tpdm__quantitative_measures.sql | 37 +++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 models/staging/tpdm/base/base_tpdm__quantitative_measures.sql create mode 100644 models/staging/tpdm/stage/stg_tpdm__quantitative_measures.sql diff --git a/models/staging/tpdm/base/base_tpdm__quantitative_measures.sql b/models/staging/tpdm/base/base_tpdm__quantitative_measures.sql new file mode 100644 index 00000000..14215e6f --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__quantitative_measures.sql @@ -0,0 +1,36 @@ +with quantitative_measures as ( + {{ source_edfi3('quantitative_measures') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:evaluationElementReference:evaluationTitle::string as evaluation_title, + v:evaluationElementReference:evaluationElementTitle::string as evaluation_element_title, + v:evaluationElementReference:evaluationObjectiveTitle::string as evaluation_objective_title, + {{ extract_descriptor('v:evaluationElementReference:evaluationPeriodDescriptor::string') }} as evaluation_period, + v:evaluationElementReference:performanceEvaluationTitle::string as performance_evaluation_title, + v:evaluationElementReference:performanceEvaluationTypeDescriptor::string as performance_evaluation_type, + v:evaluationElementReference:educationOrganizationId::int as ed_org_id, + v:evaluationElementReference:schoolYear::int as school_year, + {{ extract_descriptor('v:evaluationElementReference:termDescriptor::string') }} as academic_term, + v:quantitativeMeasureIdentifier::string as quantitative_measure_identifier, + -- descriptors + {{ extract_descriptor('v:quantitativeMeasureTypeDescriptor::string') }} as quantitative_measure_type, + {{ extract_descriptor('v:quantitativeMeasureDatatypeDescriptor::string') }} as quantitative_measure_datatype, + -- references + v:evaluationElementReference as evaluation_element_reference, + -- edfi extensions + v:_ext as v_ext + + from quantitative_measures +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__quantitative_measures.sql b/models/staging/tpdm/stage/stg_tpdm__quantitative_measures.sql new file mode 100644 index 00000000..fa464c81 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__quantitative_measures.sql @@ -0,0 +1,37 @@ +with base_quantitative_measures as ( + select * from {{ ref('base_tpdm__quantitative_measures') }} +), +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'ed_org_id', + 'lower(evaluation_element_title)', + 'lower(evaluation_objective_title)', + 'lower(evaluation_period)', + 'lower(evaluation_title)', + 'lower(performance_evaluation_title)', + 'lower(performance_evaluation_type)', + 'lower(quantitative_measure_identifier)', + 'school_year', + 'lower(academic_term)' + ] + ) }} as k_quantitative_measure, + {{ gen_skey('k_evaluation_element') }}, + base_quantitative_measures.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_quantitative_measures +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_quantitative_measure', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file From a1700fbbb5dab88f311df3a2d9bfe5ab227a265b Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Fri, 18 Jul 2025 12:29:05 -0500 Subject: [PATCH 21/35] Create base/stg models for QuantitativeMeasureScore --- macros/gen_skey.sql | 18 ++++++- ...base_tpdm__quantitative_measure_scores.sql | 50 +++++++++++++++++++ .../stg_tpdm__quantitative_measure_scores.sql | 22 ++++++++ 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 models/staging/tpdm/base/base_tpdm__quantitative_measure_scores.sql create mode 100644 models/staging/tpdm/stage/stg_tpdm__quantitative_measure_scores.sql diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index 3fc7c5e5..4954f53e 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -392,7 +392,23 @@ 'sourceSystemDescriptor', 'termDescriptor'], 'annualize': True - } + }, + 'k_quantitative_measure': { + 'reference_name': 'quantitative_measure_reference', + 'col_list': [ + 'educationOrganizationId', + 'evaluationElementTitle', + 'evaluationObjectiveTitle', + 'evaluationPeriodDescriptor', + 'evaluationTitle', + 'performanceEvaluationTitle', + 'performanceEvaluationTypeDescriptor', + 'quantitative_measure_identifier', + 'schoolYear', + 'termDescriptor' + ], + 'annualize': True + }, } %} diff --git a/models/staging/tpdm/base/base_tpdm__quantitative_measure_scores.sql b/models/staging/tpdm/base/base_tpdm__quantitative_measure_scores.sql new file mode 100644 index 00000000..70ee31d7 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__quantitative_measure_scores.sql @@ -0,0 +1,50 @@ +with quantitative_measure_scores as ( + {{ source_edfi3('quantitative_measure_scores') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:quantitativeMeasureReference:quantitativeMeasureIdentifier::string as quantitative_measure_identifier, + v:quantitativeMeasureReference:educationOrganizationId::int as quant_msr_ed_org_id, + v:quantitativeMeasureReference:evaluationElementTitle::string as quant_msr_evaluation_element_title, + v:quantitativeMeasureReference:evaluationObjectiveTitle::string as quant_msr_evaluation_objective_title, + {{ extract_descriptor('v:quantitativeMeasureReference:evaluationPeriodDescriptor::string') }} as quant_msr_evaluation_period, + v:quantitativeMeasureReference:evaluationTitle::string as quant_msr_evaluation_title, + v:quantitativeMeasureReference:performanceEvaluationTitle::string as quant_msr_performance_evaluation_title, + {{ extract_descriptor('v:quantitativeMeasureReference:performanceEvaluationTypeDescriptor::string') }} as quant_msr_performance_evaluation_type, + v:quantitativeMeasureReference:schoolYear::int as quant_msr_school_year, + {{ extract_descriptor('v:quantitativeMeasureReference:termDescriptor::string') }} as quant_msr_academic_term, + + v:evaluationElementRatingReference:evaluationDate::date as eval_elem_rating_evaluation_date, + v:evaluationElementRatingReference:personId::string as eval_elem_rating_person_id, + v:evaluationElementRatingReference:educationOrganizationId::int as eval_elem_rating_ed_org_id, + v:evaluationElementRatingReference:evaluationElementTitle::string as eval_elem_rating_evaluation_element_title, + v:evaluationElementRatingReference:evaluationObjectiveTitle::string as eval_elem_rating_evaluation_objective_title, + {{ extract_descriptor('v:evaluationElementRatingReference:evaluationPeriodDescriptor::string') }} as eval_elem_rating_evaluation_period, + v:evaluationElementRatingReference:evaluationTitle::string as eval_elem_rating_evaluation_title, + v:evaluationElementRatingReference:performanceEvaluationTitle::string as eval_elem_rating_performance_evaluation_title, + {{ extract_descriptor('v:evaluationElementRatingReference:performanceEvaluationTypeDescriptor::string') }} as eval_elem_rating_evaluation_type, + v:evaluationElementRatingReference:schoolYear::int as eval_elem_rating_school_year, + {{ extract_descriptor('v:evaluationElementRatingReference:termDescriptor::string') }} as eval_elem_rating_academic_term, + {{ extract_descriptor('v:evaluationElementRatingReference:sourceSystemDescriptor::string') }} as eval_elem_rating_source_system, + -- non-identity components + v:scoreValue::string as score_value, + v:standardError::string as standard_error, + -- references + v:quantitativeMeasureReference as quantitative_measure_reference, + v:evaluationElementRatingReference as evaluation_element_rating_reference, + -- edfi extensions + v:_ext as v_ext + + from quantitative_measure_scores +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__quantitative_measure_scores.sql b/models/staging/tpdm/stage/stg_tpdm__quantitative_measure_scores.sql new file mode 100644 index 00000000..e0059193 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__quantitative_measure_scores.sql @@ -0,0 +1,22 @@ +with base_quantitative_measure_scores as ( + select * from {{ ref('base_tpdm__quantitative_measure_scores') }} +), +keyed as ( + select + {{ gen_skey('k_quantitative_measure') }}, + {{ gen_skey('k_evaluation_element_rating') }}, + base_quantitative_measure_scores.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_quantitative_measure_scores +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_evaluation_element_rating, k_quantitative_measure', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file From 03f9475d54d4ca8d2570cfa1b0aa59053a86bb0c Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Fri, 18 Jul 2025 15:50:44 -0500 Subject: [PATCH 22/35] Create base/stg models for RecruitmentEvent --- .../base/base_tpdm__recruitment_events.sql | 31 +++++++++++++++++++ .../stage/stg_tpdm__recruitment_events.sql | 30 ++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 models/staging/tpdm/base/base_tpdm__recruitment_events.sql create mode 100644 models/staging/tpdm/stage/stg_tpdm__recruitment_events.sql diff --git a/models/staging/tpdm/base/base_tpdm__recruitment_events.sql b/models/staging/tpdm/base/base_tpdm__recruitment_events.sql new file mode 100644 index 00000000..49458d60 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__recruitment_events.sql @@ -0,0 +1,31 @@ +with recruitment_events as ( + {{ source_edfi3('recruitment_events') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:eventDate::date as event_date, + v:eventTitle::string as event_title, + v:educationOrganizationReference:educationOrganizationId::int as ed_org_id, + -- non-identity components + v:eventDescription::string as event_description, + v:eventLocation::string as event_location, + -- descriptors + {{ extract_descriptor('v:recruitmentEventTypeDescriptor::string') }} as recruitment_event_type, + -- references + v:educationOrganizationReference as education_organization_reference, + -- edfi extensions + v:_ext as v_ext + + from recruitment_events +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__recruitment_events.sql b/models/staging/tpdm/stage/stg_tpdm__recruitment_events.sql new file mode 100644 index 00000000..6550dac8 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__recruitment_events.sql @@ -0,0 +1,30 @@ +with base_recruitment_events as ( + select * from {{ ref('base_tpdm__recruitment_events') }} +), +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'ed_org_id', + 'event_date', + 'lower(event_title)', + ] + ) }} as k_recruitment_event, + {{ gen_skey('k_education_organization') }}, + base_recruitment_events.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_recruitment_events +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_recruitment_event', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file From 8a9e0b8b6a5402aba7463f057d337648786b8778 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Fri, 18 Jul 2025 17:57:52 -0500 Subject: [PATCH 23/35] Create base/stg models for RecruitmentEventAttendance --- macros/gen_skey.sql | 6 +- ...se_tpdm__recruitment_event_attendances.sql | 63 +++++++++++++++++++ ...tg_tpdm__recruitment_event_attendances.sql | 31 +++++++++ 3 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 models/staging/tpdm/base/base_tpdm__recruitment_event_attendances.sql create mode 100644 models/staging/tpdm/stage/stg_tpdm__recruitment_event_attendances.sql diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index 4954f53e..7182e6e5 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -409,7 +409,11 @@ ], 'annualize': True }, - + 'k_recruitment_event': { + 'reference_name': 'recruitment_event_reference', + 'col_list': ['educationOrganizationId', 'eventDate', 'eventTitle'], + 'annualize': True + } } %} {#- retrieve key def for then decompose parts -#} diff --git a/models/staging/tpdm/base/base_tpdm__recruitment_event_attendances.sql b/models/staging/tpdm/base/base_tpdm__recruitment_event_attendances.sql new file mode 100644 index 00000000..93420f27 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__recruitment_event_attendances.sql @@ -0,0 +1,63 @@ +with recruitment_event_attendances as ( + {{ source_edfi3('recruitment_event_attendances') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:recruitmentEventAttendeeIdentifier::string as attendee_identifier, + v:recruitmentEventReference:eventTitle::string as event_title, + v:recruitmentEventReference:eventDate::date as event_date, + v:recruitmentEventReference:educationOrganizationId::int as ed_org_id, + -- non-identity components + v:currentPosition:positionTitle::string as current_position_title, + {{ extract_descriptor('v:currentPosition:academicSubjectDescriptor::string') }} as current_position_academic_subject, + v:currentPosition:nameOfInstitution::string as current_position_institution_name, + v:currentPosition:location::string as current_position_location, + v:firstName::string as first_name, + v:lastSurname::string as last_surname, + v:maidenName::string as maiden_name, + v:middleName::string as middle_name, + v:generationCodeSuffix::string as generation_code_suffix, + v:personalTitlePrefix::string as personal_title_prefix, + v:electronicMailAddress::string as email_address, + v:socialMediaNetworkName::string as social_media_network_name, + v:socialMediaUserName::string as social_media_username, + v:met::boolean as was_met, -- indicates if person was met by a representative of the ed org, is there a better naming? + v:preScreeningRating::integer as pre_screening_rating, + v:applied::boolean as has_applied, -- indicates whether the prospect applied for a position. There might be a better name for this. + v:recruitmentEventAttendeeQualifications:eligible::boolean as is_eligible, + v:recruitmentEventAttendeeQualifications:capacityToServe::boolean as capacity_to_serve, + v:recruitmentEventAttendeeQualifications:yearsOfServiceCurrentPlacement::float as years_of_service_current_placement, + v:recruitmentEventAttendeeQualifications:yearsOfServiceTotal::float as years_of_service_total, + v:referral::boolean as is_referral, + v:referredBy::string as referred_by, + v:notes::string as additional_notes, + v:hispanicLatinoEthnicity::boolean as has_hispanic_latino_ethnicity, + -- descriptors + {{ extract_descriptor('v:genderDescriptor::string') }} as gender, + {{ extract_descriptor('v:sexDescriptor::string') }} as sex, + {{ extract_descriptor('v:recruitmentEventAttendeeTypeDescriptor::string') }} as recruitment_event_attendee_type, + -- unflattened lists + v:currentPosition:gradeLevels as current_position_grade_levels, -- is this the correct way of naming a list that is nested within a composit part? + v:disabilities as v_disabilities, + v:personalIdentificationDocuments as v_personal_identification_documents, + v:races as v_races, + v:telephones as v_telephones, + v:touchpoints as v_touchpoints, + -- references + v:recruitmentEventReference as recruitment_event_reference, + -- edfi extensions + v:_ext as v_ext + + from recruitment_event_attendances +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__recruitment_event_attendances.sql b/models/staging/tpdm/stage/stg_tpdm__recruitment_event_attendances.sql new file mode 100644 index 00000000..aefb359b --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__recruitment_event_attendances.sql @@ -0,0 +1,31 @@ +with base_recruitment_event_attendances as ( + select * from {{ ref('base_tpdm__recruitment_event_attendances') }} +), +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'lower(attendee_identifier)', + 'ed_org_id', + 'event_date', + 'lower(event_title)' + ] + ) }} as k_recruitment_event_attendance, + {{ gen_skey('k_recruitment_event') }}, + base_recruitment_event_attendances.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_recruitment_event_attendances +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_recruitment_event_attendance', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file From c622cbcc75c3e52d01845dbf22a5793bafbc6944 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Fri, 18 Jul 2025 18:01:38 -0500 Subject: [PATCH 24/35] Fix model filenames --- ...ase_tpdm__staff_educator_preparation_program_associations.sql} | 0 ...stg_tpdm__staff_educator_preparation_program_associations.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename models/staging/tpdm/base/{base_tpdm_staff_educator_preparation_program_associations.sql => base_tpdm__staff_educator_preparation_program_associations.sql} (100%) rename models/staging/tpdm/stage/{stg_tpdm_staff_educator_preparation_program_associations.sql => stg_tpdm__staff_educator_preparation_program_associations.sql} (100%) diff --git a/models/staging/tpdm/base/base_tpdm_staff_educator_preparation_program_associations.sql b/models/staging/tpdm/base/base_tpdm__staff_educator_preparation_program_associations.sql similarity index 100% rename from models/staging/tpdm/base/base_tpdm_staff_educator_preparation_program_associations.sql rename to models/staging/tpdm/base/base_tpdm__staff_educator_preparation_program_associations.sql diff --git a/models/staging/tpdm/stage/stg_tpdm_staff_educator_preparation_program_associations.sql b/models/staging/tpdm/stage/stg_tpdm__staff_educator_preparation_program_associations.sql similarity index 100% rename from models/staging/tpdm/stage/stg_tpdm_staff_educator_preparation_program_associations.sql rename to models/staging/tpdm/stage/stg_tpdm__staff_educator_preparation_program_associations.sql From 028234787f1cb9a697be933aedcb49848bc48806 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Fri, 18 Jul 2025 18:16:00 -0500 Subject: [PATCH 25/35] Fix surrogate key for ed_org in stage model --- models/staging/tpdm/stage/stg_tpdm__recruitment_events.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/staging/tpdm/stage/stg_tpdm__recruitment_events.sql b/models/staging/tpdm/stage/stg_tpdm__recruitment_events.sql index 6550dac8..705d47bf 100644 --- a/models/staging/tpdm/stage/stg_tpdm__recruitment_events.sql +++ b/models/staging/tpdm/stage/stg_tpdm__recruitment_events.sql @@ -12,7 +12,7 @@ keyed as ( 'lower(event_title)', ] ) }} as k_recruitment_event, - {{ gen_skey('k_education_organization') }}, + {{ gen_skey('k_ed_org') }}, base_recruitment_events.* {{ extract_extension(model_name=this.name, flatten=True) }} from base_recruitment_events From e7afdabef544bccad4d94774efba33dc91aa75f0 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Fri, 18 Jul 2025 18:16:41 -0500 Subject: [PATCH 26/35] Create base/stg models for Path --- models/staging/tpdm/base/base_tpdm__paths.sql | 30 +++++++++++++++++++ models/staging/tpdm/stage/stg_tpdm__paths.sql | 30 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 models/staging/tpdm/base/base_tpdm__paths.sql create mode 100644 models/staging/tpdm/stage/stg_tpdm__paths.sql diff --git a/models/staging/tpdm/base/base_tpdm__paths.sql b/models/staging/tpdm/base/base_tpdm__paths.sql new file mode 100644 index 00000000..caccda28 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__paths.sql @@ -0,0 +1,30 @@ +with paths as ( + {{ source_edfi3('paths') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:pathName::string as path_name, + v:educationOrganizationReference:educationOrganizationId::int as ed_org_id, + -- non-identity components + v:graduationPlanReference:educationOrganizationId::int as graduation_plan_ed_org_id, + {{ extract_descriptor('v:graduationPlanReference:graduationPlanTypeDescriptor::string') }} as graduation_plan_type, + v:graduationPlanReference:graduationSchoolYear::int as graduation_school_year, + -- references + v:educationOrganizationReference as education_organization_reference, + v:graduationPlanReference as graduation_plan_reference, + -- edfi extensions + v:_ext as v_ext + + from paths +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__paths.sql b/models/staging/tpdm/stage/stg_tpdm__paths.sql new file mode 100644 index 00000000..368671e3 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__paths.sql @@ -0,0 +1,30 @@ +with base_paths as ( + select * from {{ ref('base_tpdm__paths') }} +), +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'ed_org_id', + 'lower(path_name)' + ] + ) }} as k_path, + {{ gen_skey('k_ed_org') }}, + {{ gen_skey('k_graduation_plan') }}, + base_paths.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_paths +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_path', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file From 97e42f468c35ead87911333b4d9d4628340a6b83 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Sat, 19 Jul 2025 13:49:33 -0500 Subject: [PATCH 27/35] Create base/stg models for PathMilestone --- .../tpdm/base/base_tpdm__path_milestones.sql | 27 ++++++++++++++++++ .../tpdm/stage/stg_tpdm__path_milestones.sql | 28 +++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 models/staging/tpdm/base/base_tpdm__path_milestones.sql create mode 100644 models/staging/tpdm/stage/stg_tpdm__path_milestones.sql diff --git a/models/staging/tpdm/base/base_tpdm__path_milestones.sql b/models/staging/tpdm/base/base_tpdm__path_milestones.sql new file mode 100644 index 00000000..aa85ab0d --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__path_milestones.sql @@ -0,0 +1,27 @@ +with path_milestones as ( + {{ source_edfi3('path_milestones') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:pathMilestoneName::string as path_milestone_name, + {{ extract_descriptor('v:pathMilestoneTypeDescriptor::string') }} as path_milestone_type, + + -- non-identity components + v:pathMilestoneCode::string as path_milestone_code, + v:pathMilestoneDescription::string as path_milestone_description, + -- edfi extensions + v:_ext as v_ext + + from path_milestones +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__path_milestones.sql b/models/staging/tpdm/stage/stg_tpdm__path_milestones.sql new file mode 100644 index 00000000..e9bb4d2f --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__path_milestones.sql @@ -0,0 +1,28 @@ +with base_path_milestones as ( + select * from {{ ref('base_tpdm__path_milestones') }} +), +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'lower(path_milestone_name)', + 'lower(path_milestone_type)' + ] + ) }} as k_path_milestone, + base_path_milestones.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_path_milestones +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_path_milestone', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file From 77c8f66feca9dc498d8f496106da3ee7d8ed63e6 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Sat, 19 Jul 2025 14:00:01 -0500 Subject: [PATCH 28/35] Create base/stg models for PathPhase --- macros/gen_skey.sql | 7 ++++- .../tpdm/base/base_tpdm__path_phases.sql | 31 +++++++++++++++++++ .../tpdm/stage/stg_tpdm__path_phases.sql | 30 ++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 models/staging/tpdm/base/base_tpdm__path_phases.sql create mode 100644 models/staging/tpdm/stage/stg_tpdm__path_phases.sql diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index 7182e6e5..50321ffd 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -413,7 +413,12 @@ 'reference_name': 'recruitment_event_reference', 'col_list': ['educationOrganizationId', 'eventDate', 'eventTitle'], 'annualize': True - } + }, + 'k_path': { + 'reference_name': 'path_reference', + 'col_list': ['educationOrganizationId', 'pathName'], + 'annualize': True + } %} {#- retrieve key def for then decompose parts -#} diff --git a/models/staging/tpdm/base/base_tpdm__path_phases.sql b/models/staging/tpdm/base/base_tpdm__path_phases.sql new file mode 100644 index 00000000..dab6a3fb --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__path_phases.sql @@ -0,0 +1,31 @@ +with path_phases as ( + {{ source_edfi3('path_phases') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:pathPhaseName::string as path_phase_name, + v:pathReference:educationOrganizationId::int as ed_org_id, + v:pathReference:pathName::string as path_name, + -- non-identity components + v:pathPhaseSequence::int as path_phase_sequence, + v:phasePathDescription::string as phase_path_description, + -- unflattened lists + v:pathMilestones as v_path_milestones, + -- references + v:pathReference as path_reference, + -- edfi extensions + v:_ext as v_ext + + from path_phases +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__path_phases.sql b/models/staging/tpdm/stage/stg_tpdm__path_phases.sql new file mode 100644 index 00000000..7e3c80a1 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__path_phases.sql @@ -0,0 +1,30 @@ +with base_path_phases as ( + select * from {{ ref('base_tpdm__path_phases') }} +), +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'ed_org_id', + 'lower(path_name)', + 'lower(path_phase_name)' + ] + ) }} as k_path_phase, + {{ gen_skey('k_path') }}, + base_path_phases.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_path_phases +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_path_phase', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file From be737f5c4bba2bb23529c583c7491e1145af55c9 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Sat, 19 Jul 2025 14:11:53 -0500 Subject: [PATCH 29/35] Create base/stg models for StudentPath --- .../tpdm/base/base_tpdm__student_paths.sql | 29 +++++++++++++++++++ .../tpdm/stage/stg_tpdm__student_paths.sql | 22 ++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 models/staging/tpdm/base/base_tpdm__student_paths.sql create mode 100644 models/staging/tpdm/stage/stg_tpdm__student_paths.sql diff --git a/models/staging/tpdm/base/base_tpdm__student_paths.sql b/models/staging/tpdm/base/base_tpdm__student_paths.sql new file mode 100644 index 00000000..b4586311 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__student_paths.sql @@ -0,0 +1,29 @@ +with student_paths as ( + {{ source_edfi3('student_paths') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:pathReference:pathName::string as path_name, + v:studentReference:studentUniqueId::string as student_unique_id, + v:pathReference:educationOrganizationId::int as ed_org_id, + -- unflattened lists + v:periods as v_periods, + -- references + v:pathReference as path_reference, + v:studentReference as student_reference, + -- edfi extensions + v:_ext as v_ext + + from student_paths +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__student_paths.sql b/models/staging/tpdm/stage/stg_tpdm__student_paths.sql new file mode 100644 index 00000000..eb703aae --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__student_paths.sql @@ -0,0 +1,22 @@ +with base_student_paths as ( + select * from {{ ref('base_tpdm__student_paths') }} +), +keyed as ( + select + {{ gen_skey('k_path') }}, + {{ gen_skey('k_student') }}, + base_student_paths.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_student_paths +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_path, k_student', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file From da73bb33b1cc1e6d5258932b9770ba3544109be0 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Sat, 19 Jul 2025 14:34:59 -0500 Subject: [PATCH 30/35] Fix models for StudentPath --- models/staging/tpdm/base/base_tpdm__student_paths.sql | 6 +++--- models/staging/tpdm/stage/stg_tpdm__student_paths.sql | 11 ++++++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/models/staging/tpdm/base/base_tpdm__student_paths.sql b/models/staging/tpdm/base/base_tpdm__student_paths.sql index b4586311..afc8c853 100644 --- a/models/staging/tpdm/base/base_tpdm__student_paths.sql +++ b/models/staging/tpdm/base/base_tpdm__student_paths.sql @@ -13,13 +13,13 @@ renamed as ( v:id::string as record_guid, -- identity components - v:pathReference:pathName::string as path_name, - v:studentReference:studentUniqueId::string as student_unique_id, + v:pathReference:pathName::string as path_name, + v:studentReference:studentUniqueId::string as student_unique_id, v:pathReference:educationOrganizationId::int as ed_org_id, -- unflattened lists v:periods as v_periods, -- references - v:pathReference as path_reference, + v:pathReference as path_reference, v:studentReference as student_reference, -- edfi extensions v:_ext as v_ext diff --git a/models/staging/tpdm/stage/stg_tpdm__student_paths.sql b/models/staging/tpdm/stage/stg_tpdm__student_paths.sql index eb703aae..2c9b4728 100644 --- a/models/staging/tpdm/stage/stg_tpdm__student_paths.sql +++ b/models/staging/tpdm/stage/stg_tpdm__student_paths.sql @@ -3,6 +3,15 @@ with base_student_paths as ( ), keyed as ( select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'ed_org_id', + 'lower(path_name)', + 'lower(student_unique_id)' + ] + ) }} as k_student_path, {{ gen_skey('k_path') }}, {{ gen_skey('k_student') }}, base_student_paths.* @@ -13,7 +22,7 @@ deduped as ( {{ dbt_utils.deduplicate( relation='keyed', - partition_by='k_path, k_student', + partition_by='k_student_path', order_by='last_modified_timestamp desc, pull_timestamp desc' ) }} From 7b9dd8ef5aa40c1aded35ad947d1ec611e403890 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Sat, 19 Jul 2025 14:40:41 -0500 Subject: [PATCH 31/35] Base/stg models for SudentPathMilestone --- macros/gen_skey.sql | 12 +++++- ..._tpdm__student_path_milestone_statuses.sql | 37 +++++++++++++++++++ ..._tpdm__student_path_milestone_statuses.sql | 22 +++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 models/staging/tpdm/base/base_tpdm__student_path_milestone_statuses.sql create mode 100644 models/staging/tpdm/stage/stg_tpdm__student_path_milestone_statuses.sql diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index 50321ffd..e2ab34a9 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -418,7 +418,17 @@ 'reference_name': 'path_reference', 'col_list': ['educationOrganizationId', 'pathName'], 'annualize': True - + }, + 'k_student_path': { + 'reference_name': 'student_path_reference', + 'col_list': ['educationOrganizationId', 'pathName', 'studentUniqueId'], + 'annualize': True + }, + 'k_path_milestone': { + 'reference_name': 'path_milestone_reference', + 'col_list': ['educationOrganizationId', 'pathMilestoneName', 'pathMilestoneTypeDescriptor', 'pathName'], + 'annualize': True + } } %} {#- retrieve key def for then decompose parts -#} diff --git a/models/staging/tpdm/base/base_tpdm__student_path_milestone_statuses.sql b/models/staging/tpdm/base/base_tpdm__student_path_milestone_statuses.sql new file mode 100644 index 00000000..deb24f12 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__student_path_milestone_statuses.sql @@ -0,0 +1,37 @@ +with student_path_milestone_statuses as ( + {{ source_edfi3('student_path_milestone_statuses') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:pathMilestoneReference:pathMilestoneName::string as path_milestone_name, + {{ extract_descriptor('v:pathMilestoneReference:pathMilestoneTypeDescriptor::string') }} as path_milestone_type, + v:studentPathReference:educationOrganizationId::int as ed_org_id, + v:studentPathReference:pathName::string as path_name, + v:studentPathReference:studentUniqueId::string as student_unique_id, + -- non-identity components + v:completionIndicator::boolean as has_completed, + v:event:description::string as status_description, + v:event:pathMilestoneStatusDate::string as status_date, + -- descriptors + {{ extract_descriptor('v:event:pathMilestoneStatusDescriptor::string') }} as status_descriptor, + -- references + v:pathMilestoneReference as path_milestone_reference, + v:studentPathReference as student_path_reference, + v:pathPhaseReference as path_phase_reference, + + -- edfi extensions + v:_ext as v_ext + + from student_path_milestone_statuses +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__student_path_milestone_statuses.sql b/models/staging/tpdm/stage/stg_tpdm__student_path_milestone_statuses.sql new file mode 100644 index 00000000..844bfc5c --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__student_path_milestone_statuses.sql @@ -0,0 +1,22 @@ +with base_student_path_milestone_statuses as ( + select * from {{ ref('base_tpdm__student_path_milestone_statuses') }} +), +keyed as ( + select + {{ gen_skey('k_student_path') }}, + {{ gen_skey('k_path_milestone') }}, + base_student_path_milestone_statuses.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_student_path_milestone_statuses +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_student_path, k_path_milestone', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file From 4cd694e21661f1771c9cf6e3dba3586e3a464929 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Sat, 19 Jul 2025 14:41:25 -0500 Subject: [PATCH 32/35] Fix structure in base model --- models/staging/tpdm/base/base_tpdm__path_milestones.sql | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/models/staging/tpdm/base/base_tpdm__path_milestones.sql b/models/staging/tpdm/base/base_tpdm__path_milestones.sql index aa85ab0d..fa0c521a 100644 --- a/models/staging/tpdm/base/base_tpdm__path_milestones.sql +++ b/models/staging/tpdm/base/base_tpdm__path_milestones.sql @@ -13,11 +13,10 @@ renamed as ( v:id::string as record_guid, -- identity components - v:pathMilestoneName::string as path_milestone_name, + v:pathMilestoneName::string as path_milestone_name, {{ extract_descriptor('v:pathMilestoneTypeDescriptor::string') }} as path_milestone_type, - -- non-identity components - v:pathMilestoneCode::string as path_milestone_code, + v:pathMilestoneCode::string as path_milestone_code, v:pathMilestoneDescription::string as path_milestone_description, -- edfi extensions v:_ext as v_ext From fba970095976cc8a7185cc42432ab82486e0cdb6 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Sat, 19 Jul 2025 15:06:12 -0500 Subject: [PATCH 33/35] Create base/stg models for StudentPathPhaseStatus --- macros/gen_skey.sql | 5 +++ ...base_tpdm__student_path_phase_statuses.sql | 35 +++++++++++++++++++ .../stg_tpdm__student_path_phase_statuses.sql | 22 ++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 models/staging/tpdm/base/base_tpdm__student_path_phase_statuses.sql create mode 100644 models/staging/tpdm/stage/stg_tpdm__student_path_phase_statuses.sql diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index e2ab34a9..2ea6236f 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -428,6 +428,11 @@ 'reference_name': 'path_milestone_reference', 'col_list': ['educationOrganizationId', 'pathMilestoneName', 'pathMilestoneTypeDescriptor', 'pathName'], 'annualize': True + }, + 'k_path_phase': { + 'reference_name': 'path_phase_reference', + 'col_list': ['coleducationOrganizationId_a', 'pathName', 'pathPhaseName'], + 'annualize': True } } %} diff --git a/models/staging/tpdm/base/base_tpdm__student_path_phase_statuses.sql b/models/staging/tpdm/base/base_tpdm__student_path_phase_statuses.sql new file mode 100644 index 00000000..1dba9b69 --- /dev/null +++ b/models/staging/tpdm/base/base_tpdm__student_path_phase_statuses.sql @@ -0,0 +1,35 @@ +with student_path_phase_statuses as ( + {{ source_edfi3('student_path_phase_statuses') }} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + filename, + file_row_number, + is_deleted, + + v:id::string as record_guid, + -- identity components + v:pathPhaseReference:pathName::string as path_phase_path_name, + v:pathPhaseReference:pathPhaseName::string as path_phase_name, + v:pathPhaseReference:educationOrganizationId::int as path_phase_ed_org_id, + v:studentPathReference:pathName::string as student_path_path_name, + v:studentPathReference:studentUniqueId::string as student+unique_id, + v:studentPathReference:educationOrganizationId::int as student_path_ed_org_id, + -- non-identity components + v:completionIndicator::boolean as has_completed, + -- unflattened lists + v:events as v_events, + v:periods as v_periods, + -- references + v:pathPhaseReference as path_phase_reference, + v:studentPathReference as student_path_reference, + -- edfi extensions + v:_ext as v_ext + + from student_path_phase_statuses +) +select * from renamed \ No newline at end of file diff --git a/models/staging/tpdm/stage/stg_tpdm__student_path_phase_statuses.sql b/models/staging/tpdm/stage/stg_tpdm__student_path_phase_statuses.sql new file mode 100644 index 00000000..7306a9d8 --- /dev/null +++ b/models/staging/tpdm/stage/stg_tpdm__student_path_phase_statuses.sql @@ -0,0 +1,22 @@ +with base_student_path_phase_statuses as ( + select * from {{ ref('base_tpdm__student_path_phase_statuses') }} +), +keyed as ( + select + {{ gen_skey('k_path_phase') }}, + {{ gen_skey('k_student_path') }}, + base_student_path_phase_statuses.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_student_path_phase_statuses +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_path_phase, k_student_path', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +) +select * from deduped +where not is_deleted \ No newline at end of file From bc77faa121f48787597bc3083f0d28df7418b043 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Tue, 5 Aug 2025 15:11:43 -0500 Subject: [PATCH 34/35] Reorder and rename attributes in base model --- ...dm__survey_section_response_person_target_associations.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql b/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql index cc92b7bc..f5563e8f 100644 --- a/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql +++ b/models/staging/tpdm/base/base_tpdm__survey_section_response_person_target_associations.sql @@ -13,12 +13,12 @@ renamed as ( v:id::string as record_guid, -- identity components - v:surveySectionResponseReference:surveyIdentifier::string as survey_id, v:surveySectionResponseReference:namespace::string as namespace, + v:surveySectionResponseReference:surveyIdentifier::string as survey_id, v:surveySectionResponseReference:surveySectionTitle::string as survey_section_title, v:surveySectionResponseReference:surveyResponseIdentifier::string as survey_response_identifier, v:personReference:personId::string as person_id, - {{ extract_descriptor('v:personReference:sourceSystemDescriptor::string') }} as source_system, + {{ extract_descriptor('v:personReference:sourceSystemDescriptor::string') }} as person_source_system, -- references v:surveySectionResponseReference as survey_section_response_reference, v:personReference as person_reference, From e86d784bfc18e8f797bc78f921323851ff0245e3 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Tue, 5 Aug 2025 15:28:41 -0500 Subject: [PATCH 35/35] Fix column name in gen_skey --- macros/gen_skey.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index 2ea6236f..6e659162 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -431,7 +431,7 @@ }, 'k_path_phase': { 'reference_name': 'path_phase_reference', - 'col_list': ['coleducationOrganizationId_a', 'pathName', 'pathPhaseName'], + 'col_list': ['educationOrganizationId', 'pathName', 'pathPhaseName'], 'annualize': True } }