Feature/epdm models evaluations

CHANGELOG.md

@@ -3,6 +3,14 @@
 ## Under the hood
 ## Fixes
 
+# edu_wh v0.6.1
+## New features
+- Add `school_year` column to `fct_student_grades`, `fct_student_discipline_incident_behaviors`, `fct_student_discipline_incident_summary`, `fct_student_discipline_actions_summary`, `fct_student_school_attendance_event`, `fct_student_section_attendance_event`, `fct_student_daily_attendance`
+## Under the hood
+- Expand yml documentation for `fct_student_section_attendance_event`
+## Fixes
+- Fix `period_duration` calculation in `dim_class_period` to account for different time formats.
+
 # edu_wh v0.6.0
 ## New features
 - Add `attendance_excusal_status` and `consecutive_days_by_excusal_status` to `fct_student_daily_attendance`, where:

dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 
 name: 'edu_wh'
-version: '0.6.0'
+version: '0.6.1'
 require-dbt-version: [">=1.10.5", "<2.0.0"]
 
 # This setting configures which "profile" dbt uses for this project.

models/core_warehouse/dim_class_period.sql

@@ -43,7 +43,10 @@ formatted as (
                 else meeting_time:endTime::string
             end
         end as end_time,
-        timediff(MINUTE, concat('2020-01-01 ', start_time)::timestamp, concat('2020-01-01 ', end_time)::timestamp) as period_duration
+        case 
+          when start_time <= end_time then timediff(MINUTE, concat('2020-01-01 ', start_time)::timestamp, concat('2020-01-01 ', end_time)::timestamp) 
+          else timediff(MINUTE, concat('2020-01-01 ', start_time)::timestamp, dateadd(hour, 12 , concat('2020-01-01 ', end_time)::timestamp))   
+        end as period_duration
 
         -- custom indicators
         {% if custom_data_sources is not none and custom_data_sources | length -%}

models/core_warehouse/fct_student_daily_attendance.sql

@@ -93,6 +93,7 @@ stu_enr_att_cal as (
         enr.tenant_code,
         enr.entry_date,
         attendance_calendar.k_calendar_date,
+        attendance_calendar.school_year,
         attendance_calendar.calendar_date,
         enr.exit_withdraw_date
     from fct_student_school_assoc as enr
@@ -108,6 +109,7 @@ fill_positive_attendance as (
         stu_enr_att_cal.k_student_xyear,
         stu_enr_att_cal.k_school,
         stu_enr_att_cal.k_calendar_date,
+        stu_enr_att_cal.school_year,
         coalesce(
             fct_student_school_att.k_session, 
             bld_attendance_sessions.k_session
@@ -206,6 +208,7 @@ cumulatives as (
         excusal_status_streaks.k_student_xyear,
         excusal_status_streaks.k_school,
         excusal_status_streaks.k_calendar_date,
+        excusal_status_streaks.school_year,
         excusal_status_streaks.calendar_date,
         excusal_status_streaks.k_session,
         excusal_status_streaks.tenant_code,

models/core_warehouse/fct_student_daily_attendance.yml

@@ -34,6 +34,7 @@ models:
       - name: k_student
       - name: k_school
       - name: k_calendar_date
+      - name: school_year
       - name: calendar_date
       - name: k_session
       - name: tenant_code

models/core_warehouse/fct_student_discipline_actions_summary.sql

@@ -57,6 +57,7 @@ formatted as (
         fct_student_discipline_actions.k_student_xyear,
         fct_student_discipline_actions.k_discipline_actions_event,
         fct_student_discipline_actions.tenant_code,
+        fct_student_discipline_actions.school_year,
         behaviors_array.behavior_types_array,
         behaviors_array.incident_id_array,
         actions_array.discipline_actions_array,

models/core_warehouse/fct_student_discipline_actions_summary.yml

@@ -31,6 +31,7 @@ models:
         description: >
           A key containing discipline_action_id and discipline_date to represent a single discipline action event.
       - name: tenant_code
+      - name: school_year
       - name: behavior_types_array
         description: >
           An array of behavior types, which are defined as a category of behavior describing the 

models/core_warehouse/fct_student_discipline_incident_behaviors.sql

@@ -50,6 +50,7 @@ formatted as (
              'lower(stg_stu_discipline_incident_behaviors.behavior_type)']
         ) }} as k_student_discipline_incident_behavior,
         stg_stu_discipline_incident_behaviors.tenant_code,
+        stg_stu_discipline_incident_behaviors.school_year,
         stg_stu_discipline_incident_behaviors.school_id,
         stg_stu_discipline_incident_behaviors.incident_id,
         stg_stu_discipline_incident_behaviors.behavior_type,

models/core_warehouse/fct_student_discipline_incident_behaviors.yml

@@ -27,6 +27,7 @@ models:
         description: >
           A key containing incident_id and school_id to represent a single discipline incident.
       - name: tenant_code
+      - name: school_year
       - name: school_id
       - name: incident_id
         description: > 

models/core_warehouse/fct_student_discipline_incident_summary.sql

@@ -53,6 +53,7 @@ formatted as (
         fct_student_discipline_incident_behaviors.k_student_xyear,
         fct_student_discipline_incident_behaviors.k_discipline_incident,
         fct_student_discipline_incident_behaviors.tenant_code,
+        fct_student_discipline_incident_behaviors.school_year,
         behaviors_array.behavior_types_array,
         actions_array.discipline_actions_array,
         -- we want to include the most severe behavior type and discipline action

models/core_warehouse/fct_student_discipline_incident_summary.yml

@@ -31,6 +31,7 @@ models:
         description: >
           A key containing discipline_action_id and discipline_date to represent a single discipline event.
       - name: tenant_code
+      - name: school_year
       - name: behavior_types_array
         description: >
           An array of behavior types, which are defined as a category of behavior describing the 

models/core_warehouse/fct_student_grades.sql

@@ -41,6 +41,7 @@ formatted as (
         dim_grading_period.k_grading_period,
         stg_grades.grade_type,
         stg_grades.tenant_code,
+        stg_grades.school_year,
         stg_grades.letter_grade_earned,
         stg_grades.numeric_grade_earned,
         stg_grades.diagnostic_statement,

models/core_warehouse/fct_student_grades.yml

@@ -45,6 +45,7 @@ models:
           The type of grade in a report card or transcript (e.g., Final, 
           Exam, Grading Period).
       - name: tenant_code
+      - name: school_year
       - name: letter_grade_earned
       - name: numeric_grade_earned
       - name: diagnostic_statement

models/core_warehouse/fct_student_school_attendance_event.sql

@@ -56,6 +56,7 @@ joined as (
         dim_calendar_date.calendar_date,
         dim_session.k_session,
         stg_stu_sch_attend.tenant_code,
+        stg_stu_sch_attend.school_year,
         stg_stu_sch_attend.attendance_event_category,
         stg_stu_sch_attend.attendance_event_reason,
         xwalk_att_events.is_absent,
@@ -103,6 +104,7 @@ formatted as (
         k_calendar_date,
         k_session,
         tenant_code,
+        school_year,
         calendar_date,
         attendance_event_category,
         attendance_event_reason,

models/core_warehouse/fct_student_school_attendance_event.yml

@@ -33,6 +33,7 @@ models:
       - name: calendar_date
       - name: k_session
       - name: tenant_code
+      - name: school_year
       - name: attendance_event_category
       - name: attendance_event_reason
       - name: is_absent

models/core_warehouse/fct_student_section_attendance_event.sql

@@ -31,6 +31,7 @@ formatted as (
         dim_course_section.k_school,
         dim_course_section.k_course_section,
         stg_stu_section_attendance.tenant_code,
+        stg_stu_section_attendance.school_year,
         stg_stu_section_attendance.attendance_event_date,
         stg_stu_section_attendance.attendance_event_category,
         stg_stu_section_attendance.attendance_event_reason,

models/core_warehouse/fct_student_section_attendance_event.yml

@@ -2,5 +2,53 @@ version: 2
 
 models:
   - name: fct_student_section_attendance_event
+    description: >
+      ##### Overview:
+        Defines student-section-level attendance events (period/class attendance).
+
+        Since attendance is generally recorded only in the negative, this is 
+        typically a table of absences only. This is more granular than school-level
+        attendance, tracking attendance at the individual class/section level.
+
+      ##### Primary Key:
+        `k_student, k_course_section, attendance_event_category, attendance_event_date`
+
     config:
       tags: ['core']
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          arguments:
+            combination_of_columns:
+              - k_student
+              - k_course_section
+              - attendance_event_category
+              - attendance_event_date
+    columns:
+      - name: k_student
+        description: Unique identifier for the student. Foreign key reference to dim_student.
+      - name: k_student_xyear
+        description: Unique identifier for the student across all years.
+      - name: k_school
+        description: Unique identifier for the school. Derived from the course section.
+      - name: k_course_section
+        description: Unique identifier for the course section. Foreign key reference to dim_course_section.
+      - name: tenant_code
+      - name: school_year
+      - name: attendance_event_date
+        description: The date of the attendance event.
+      - name: attendance_event_category
+        description: The descriptor value from AttendanceEventCategoryDescriptor.
+      - name: attendance_event_reason
+        description: The reason for the attendance event (e.g., reason for absence).
+      - name: is_absent
+        description: Indicator for absence. Defined via descriptor mapping in `xwalk_attendance_events`.
+      - name: event_duration
+        description: The amount of time for the event as recognized by the school.
+      - name: section_attendance_duration
+        description: The duration of the section attendance event.
+      - name: arrival_time
+        description: The time of arrival for the student.
+      - name: departure_time
+        description: The time of departure for the student.
+      - name: educational_environment
+        description: The setting in which a child receives education and related services.

models/tpdm_warehouse/dim_evaluation.sql

@@ -0,0 +1,33 @@
+{{
+  config(
+    post_hook=[
+        "alter table {{ this }} alter column k_evaluation set not null",
+        "alter table {{ this }} add primary key (k_evaluation)",
+    ]
+  )
+}}
+
+with stg_eval as (
+    select * from {{ ref('stg_tpdm__evaluations') }}
+),
+
+formatted as (
+    select
+        stg_eval.k_evaluation,
+        stg_eval.tenant_code,
+        stg_eval.school_year,
+        stg_eval.ed_org_id,
+        stg_eval.evaluation_period,
+        stg_eval.performance_evaluation_title,
+        stg_eval.performance_evaluation_type,
+        stg_eval.academic_term,
+        stg_eval.evaluation_description,
+        stg_eval.min_rating,
+        stg_eval.max_rating,
+        stg_eval.inter_rater_reliability_score,
+        stg_eval.evaluation_type,
+        stg_eval.v_rating_levels,
+        stg_eval.performance_evaluation_reference
+    from stg_eval
+)
+select * from formatted

models/tpdm_warehouse/dim_performance_evaluation.sql

@@ -0,0 +1,30 @@
+{{
+  config(
+    post_hook=[
+        "alter table {{ this }} alter column k_performance_evaluation set not null",
+        "alter table {{ this }} add primary key (k_performance_evaluation)",
+    ]
+  )
+}}
+
+with stg_perf_eval as (
+    select * from {{ ref('stg_tpdm__performance_evaluations') }}
+),
+
+formatted as (
+    select
+        stg_perf_eval.k_performance_evaluation,
+        stg_perf_eval.tenant_code,
+        stg_perf_eval.school_year,
+        stg_perf_eval.ed_org_id,
+        stg_perf_eval.evaluation_period,
+        stg_perf_eval.performance_evaluation_title,
+        stg_perf_eval.performance_evaluation_type,
+        stg_perf_eval.academic_term,
+        stg_perf_eval.academic_subject,
+        stg_perf_eval.v_grade_levels,
+        stg_perf_eval.v_rating_levels
+        {{ edu_edfi_source.extract_extension(model_name='stg_tpdm__performance_evaluations', flatten=False) }}
+    from stg_perf_eval
+)
+select * from formatted

models/tpdm_warehouse/dim_performance_evaluation.yml

@@ -0,0 +1,13 @@
+version: 2
+
+models:
+  - name: dim_performance_evaluation
+    description: >
+        ##### Overview:
+          A performance evaluation of an educator, typically regularly scheduled and uniformly applied, composed of one or more Evaluations.
+
+        ##### Primary Key:
+          `k_performance_evaluation` - There is one record per performance evaluation.
+    config:
+      tags: ['tpdm']
+      enabled: "{{ var('edu:tpdm:enabled', False) }}"

models/tpdm_warehouse/fct_performance_evaluation_rating.sql

@@ -0,0 +1,30 @@
+with stg_perf_eval_rating as (
+    select * from {{ ref('stg_tpdm__performance_evaluation_ratings') }}
+),
+
+formatted as (
+    select
+        stg_perf_eval_rating.k_person,
+        stg_perf_eval_rating.k_performance_evaluation,
+        stg_perf_eval_rating.tenant_code,
+        stg_perf_eval_rating.school_year,
+        stg_perf_eval_rating.ed_org_id,
+        stg_perf_eval_rating.evaluation_period,
+        stg_perf_eval_rating.performance_evaluation_title,
+        stg_perf_eval_rating.performance_evaluation_type,
+        stg_perf_eval_rating.person_id,
+        stg_perf_eval_rating.source_system,
+        stg_perf_eval_rating.academic_term,
+        stg_perf_eval_rating.actual_time,
+        stg_perf_eval_rating.schedule_date,
+        stg_perf_eval_rating.actual_date,
+        stg_perf_eval_rating.comments,
+        stg_perf_eval_rating.actual_duration,
+        stg_perf_eval_rating.is_announced,
+        stg_perf_eval_rating.performance_evaluation_rating_level,
+        stg_perf_eval_rating.coteaching_style_observed,
+        stg_perf_eval_rating.reviewers,
+        stg_perf_eval_rating.results
+    from stg_perf_eval_rating
+)
+select * from formatted

models/tpdm_warehouse/fct_performance_evaluation_rating.yml

@@ -0,0 +1,12 @@
+version: 2
+
+models:
+  - name: fct_performance_evaluation_rating
+    description: >
+        ##### Overview:
+          The summary rating for a Performance Evaluation across all Evaluation instruments for an individual educator.
+
+
+    config:
+      tags: ['tpdm']
+      enabled: "{{ var('edu:tpdm:enabled', False) }}"