diff --git a/elt-pipelines/facility_ops/transform/models/marts/accelerator/mcr_equipment_downtime_records.sql b/elt-pipelines/facility_ops/transform/models/marts/accelerator/mcr_equipment_downtime_records.sql new file mode 100644 index 00000000..c87a54f5 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/marts/accelerator/mcr_equipment_downtime_records.sql @@ -0,0 +1,130 @@ +{{ + config( + properties={ + "partitioning": "ARRAY['cycle_name']", + }, + on_table_exists = 'drop' +) +}} + +with + +records_sharepoint as ( select * from {{ ref('stg_accelerator_sharepoint__equipment_downtime_data_11_08_24') }} ), + +records_opralogweb as ( select * from {{ ref('stg_opralogweb__mcr_equipment_downtime') }} ), + +equipment_name_mappings as ( select * from {{ ref('stg_accelerator_sharepoint__edr_equipment_mapping') }} ), + +records_sharepoint_with_cycle_phase_col as ( + + select + + equipment, + fault_date, + cycle_name, + cast(NULL as varchar) as cycle_phase, + downtime_mins, + fault_occurred_at, + {{ adapter.quote('group') }}, + fault_description, + managers_comments + + from + + records_sharepoint + +), + +records_opralogweb_after_sharepoint_joined_with_cycles as ( + + select + + r.equipment, + r.fault_date, + c.name as cycle_name, + c.phase as cycle_phase, + r.downtime_mins, + r.fault_occurred_at, + r.{{ adapter.quote('group') }}, + r.fault_description, + r.managers_comments + + from records_opralogweb r + left join {{ ref("cycles") }} c on r.fault_occurred_at between c.started_at and c.ended_at + where fault_occurred_at > (select max(fault_occurred_at) from records_sharepoint_with_cycle_phase_col) +), + +all_records as ( + + select * from records_sharepoint_with_cycle_phase_col + union + select * from records_opralogweb_after_sharepoint_joined_with_cycles + +), + +equipment_up_at_col as ( + + select + + {{ normalize_whitespace('equipment') }} as equipment, + fault_date, + cycle_name, + cycle_phase, + downtime_mins, + fault_occurred_at, + fault_occurred_at + (interval '1' minute * downtime_mins) as equipment_up_at, + {{ adapter.quote('group') }}, + fault_description, + managers_comments + + from + + all_records d +), + +uptime_col as ( + + select + + equipment, + fault_date, + cycle_name, + cycle_phase, + downtime_mins, + fault_occurred_at, + equipment_up_at, + date_diff('minute', + lag(equipment_up_at, 1, null) over + (partition by cycle_name, equipment order by fault_occurred_at), fault_occurred_at + ) as uptime_before_fault_mins, + {{ adapter.quote('group') }}, + fault_description, + managers_comments + + from equipment_up_at_col +), + +equipment_category_col as ( + + select + + {{ normalize_whitespace('u.equipment') }} as equipment, + m.equipment_category as equipment_category, + fault_date, + cycle_name, + cycle_phase, + downtime_mins, + fault_occurred_at, + equipment_up_at, + uptime_before_fault_mins, + {{ adapter.quote('group') }}, + fault_description, + managers_comments + + from uptime_col u + left join equipment_name_mappings m on {{ create_equipment_category_key('u.equipment') }} = m.equipment + +) + +-- add order by clause for iceberg table sorting criterion +select * from equipment_category_col order by fault_occurred_at asc diff --git a/elt-pipelines/facility_ops/transform/models/marts/accelerator/mcr_equipment_downtime_records.yml b/elt-pipelines/facility_ops/transform/models/marts/accelerator/mcr_equipment_downtime_records.yml new file mode 100644 index 00000000..cbea8dbf --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/marts/accelerator/mcr_equipment_downtime_records.yml @@ -0,0 +1,4 @@ +models: + - name: mcr_equipment_downtime_records + description: > + A full history of MCR equipment failures, including downtime and fault descriptions diff --git a/elt-pipelines/facility_ops/transform/models/marts/accelerator/mcr_equipment_downtime_uncategorized_equipment.sql b/elt-pipelines/facility_ops/transform/models/marts/accelerator/mcr_equipment_downtime_uncategorized_equipment.sql new file mode 100644 index 00000000..35276ef8 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/marts/accelerator/mcr_equipment_downtime_uncategorized_equipment.sql @@ -0,0 +1,16 @@ +{{ + config( + on_table_exists = 'drop', + materialized = 'view' +) +}} + +select + + distinct(equipment) as uncategorized_equipment + +from + + {{ ref('mcr_equipment_downtime_records') }} + +where equipment_category is null diff --git a/elt-pipelines/facility_ops/transform/models/marts/accelerator/mcr_equipment_downtime_uncategorized_equipment.yml b/elt-pipelines/facility_ops/transform/models/marts/accelerator/mcr_equipment_downtime_uncategorized_equipment.yml new file mode 100644 index 00000000..50e19575 --- /dev/null +++ b/elt-pipelines/facility_ops/transform/models/marts/accelerator/mcr_equipment_downtime_uncategorized_equipment.yml @@ -0,0 +1,5 @@ +model: + - name: mcr_equipment_downtime_uncategorized_equipment + description: > + Lists the equipment names for which no category has been found in the + EDR equipment mapping table. diff --git a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_accelerator_sharepoint__equipment_downtime_data_11_08_24.sql b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_accelerator_sharepoint__equipment_downtime_data_11_08_24.sql index 89986a12..85d0e5aa 100644 --- a/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_accelerator_sharepoint__equipment_downtime_data_11_08_24.sql +++ b/elt-pipelines/facility_ops/transform/models/staging/accelerator/stg_accelerator_sharepoint__equipment_downtime_data_11_08_24.sql @@ -38,7 +38,7 @@ renamed as ( date(fault_date_str) as fault_date, -- Desktop Opralog used local time rather than UTC. Convert to UTC here. - {{ parse_utc_timestamp('fault_date_str', 'yyyy-MM-dd', 'fault_time_str', src_timezone='Europe/London') }} as fault_occurred_at, + cast({{ parse_utc_timestamp('fault_date_str', 'yyyy-MM-dd', 'fault_time_str', src_timezone='Europe/London') }} as timestamp(6)) as fault_occurred_at, {{ adapter.quote('group') }}, faultdescription as fault_description, @@ -48,12 +48,4 @@ renamed as ( ) -select -equipment, -cycle_name, -downtime_mins, -fault_date, -cast(fault_occurred_at as timestamp(6)) as fault_occurred_at, -fault_description, -managers_comments -from renamed +select * from renamed