build CBBR FY2027

.github/workflows/cbbr_build.yml

@@ -56,7 +56,10 @@ jobs:
           AWS_SECRET_ACCESS_KEY: "op://Data Engineering/DO_keys/AWS_SECRET_ACCESS_KEY"
           AWS_ACCESS_KEY_ID: "op://Data Engineering/DO_keys/AWS_ACCESS_KEY_ID"
           BUILD_ENGINE_SERVER: "op://Data Engineering/EDM_DATA/server_url"
-          EDM_DATA: "op://Data Engineering/EDM_DATA/defaultdb_url"
+          BUILD_ENGINE_HOST: "op://Data Engineering/EDM_DATA/server"
+          BUILD_ENGINE_USER: "op://Data Engineering/EDM_DATA/username"
+          BUILD_ENGINE_PASSWORD: "op://Data Engineering/EDM_DATA/password"
+          BUILD_ENGINE_PORT: "op://Data Engineering/EDM_DATA/port"
 
       - name: Finish container setup ...
         working-directory: ./
@@ -71,11 +74,16 @@ jobs:
 
       - name: Set recipe env vars
         working-directory: ./
-        run: source ./bash/export_recipe_env.sh products/cbbr/${{ inputs.recipe_file }}.lock.yml 
+        run: source ./bash/export_recipe_env.sh products/cbbr/${{ inputs.recipe_file }}.lock.yml
 
       - name: Dataloading
         run: python3 -m dcpy lifecycle builds load load
 
+      - name: Build
+        run: |
+          dbt deps
+          dbt debug
+
       - name: Create Build Tables and Normalize CBBR Values
         run: ./02_cbbr.sh
 

dcpy/lifecycle/scripts/_cli.py

@@ -1,7 +1,7 @@
 import typer
 
 from .compare_build_tables import app as compare_build_tables_app
-from .ingest_with_library_fallback import run as ingest_or_library_archive
+from .ingest_or_library_archive import run as ingest_or_library_archive
 from .package_and_distribute import app as package_dist_app
 from .product_metadata import app as product_metadata_app
 from .validate_ingest import app as ingest_validation_app

ingest_templates/dcp_cbbr_requests.yml

@@ -17,10 +17,12 @@ attributes:
 
 ingestion:
   source:
-    type: local_file
-    path: ./dcp_cbbr_requests.csv
+    type: s3
+    bucket: edm-recipes
+    key: inbox/dcp/dcp_cbbr_requests/{{ version }} BRs for Data Engineering (All Boards).csv
   file_format:
     type: csv
+    dtype: str
   processing_steps:
   - name: clean_column_names
-    args: { "replace": { " -": "", "-": "_", " ": "_" }, "lower": True }
+    args: { "replace": { "-": "_", " ": "_" }, "lower": True }

ingest_templates/omb_cbbr_agency_responses.yml

@@ -15,10 +15,13 @@ attributes:
 
 ingestion:
   source:
-    type: local_file
-    path: ./omb_cbbr_agency_responses.csv
+    type: s3
+    bucket: edm-recipes
+    key: inbox/dcp/omb_cbbr_agency_responses/{{ version }} OMB Raw Export.xlsx
   file_format:
-    type: csv
+    type: xlsx
+    sheet_name: AGCY reponses
+    dtype: str
   processing_steps:
   - name: clean_column_names
     args: { "replace": { " ": "_" }, "lower": True }

products/cbbr/02_cbbr.sh

@@ -6,6 +6,7 @@ echo "CBBR Version ${VERSION} : 02 CBBR"
 
 # TODO delete tables that are created by this stage
 echo "Create build tables to modify ..."
+dbt build --select staging
 run_sql_file sql/preprocessing.sql
 run_sql_file sql/cbbr_submissions.sql
 
@@ -15,7 +16,9 @@ run_sql_file sql/normalize_agency.sql
 echo "Normalize commdist values ..."
 run_sql_file sql/normalize_commdist.sql
 
-echo "Normalize denominator values ..."
-run_sql_file sql/normalize_denominator.sql
+echo "Normalize request values ..."
+run_sql_file sql/normalize_requests.sql
+
+dbt test --select intermediate
 
 echo "Done!"

products/cbbr/dbt_project.yml

@@ -0,0 +1,10 @@
+name: "cbbr"
+
+profile: "dcp-de-postgres"
+
+model-paths: [ "models" ]
+test-paths: [ "tests" ]
+
+tests:
+  +store_failures: true
+  +schema: "_tests"

products/cbbr/models/_sources.yml

@@ -0,0 +1,6 @@
+sources:
+  - name: recipe_sources
+    schema: "{{ env_var('BUILD_ENGINE_SCHEMA') }}"
+    tables:
+      - name: source_data_versions
+      - name: doitt_buildingfootprints

products/cbbr/models/intermediate/_cbbr_submissions.sql

@@ -0,0 +1 @@
+-- PLACEHOLDER

products/cbbr/models/intermediate/_intermediate.yml

@@ -0,0 +1,5 @@
+models:
+  - name: _cbbr_submissions
+    columns:
+      - name: need
+        tests: [ not_null ]

products/cbbr/models/staging/_staging.yml

@@ -0,0 +1,9 @@
+models:
+  - name: stg__doitt_buildingfootprints
+    columns:
+      - name: base_bbl
+        tests: [ not_null ]
+      - name: bin
+        tests: [ not_null ]
+      - name: geom
+        tests: [ not_null ]

products/cbbr/models/staging/stg__doitt_buildingfootprints.sql

@@ -0,0 +1,4 @@
+select
+    *,
+    wkb_geometry as geom
+from {{ source('recipe_sources', 'doitt_buildingfootprints') }}

products/cbbr/packages.yml

@@ -0,0 +1,7 @@
+packages:
+- package: dbt-labs/dbt_utils
+  version: 1.3.3
+- package: dbt-labs/audit_helper
+  version: 0.13.0
+- package: metaplane/dbt_expectations
+  version: 0.10.10

products/cbbr/profiles.yml

@@ -0,0 +1,11 @@
+dcp-de-postgres:
+  target: dev
+  outputs:
+    dev:
+      type: postgres
+      host: "{{ env_var('BUILD_ENGINE_HOST') }}"
+      user: "{{ env_var('BUILD_ENGINE_USER') }}"
+      password: "{{ env_var('BUILD_ENGINE_PASSWORD') }}"
+      port: "{{ env_var('BUILD_ENGINE_PORT') | as_number }}"
+      dbname: "{{ env_var('BUILD_ENGINE_DB') }}"
+      schema: "{{ env_var('BUILD_ENGINE_SCHEMA') }}"

products/cbbr/recipe.yml

@@ -1,6 +1,6 @@
 name: Community Board Budget Requests
 product: db-cbbr
-version: FY2026
+version: FY2027
 inputs:
   missing_versions_strategy: find_latest
   datasets:
@@ -12,3 +12,5 @@ inputs:
   - name: dpr_parksproperties
   - name: dcp_facilities
   - name: doitt_buildingfootprints
+  # other data
+  - name: dcp_managing_agencies_lookup

products/cbbr/sql/assign_geoms.sql

@@ -25,14 +25,14 @@ SET
     geom = (
         CASE
             WHEN a.geo_bbl || a.geo_bin IS NOT NULL
-                THEN st_centroid(b.geom)
+                THEN st_centroid(footprints.geom)
             ELSE a.geom
         END
     )
 FROM
-    _doitt_buildingfootprints AS b
+    stg__doitt_buildingfootprints AS footprints
 WHERE
-    a.geo_bbl || a.geo_bin = b.base_bbl || b.bin;
+    a.geo_bbl || a.geo_bin = footprints.base_bbl || footprints.bin;
 
 -- Convert from_x_coord, from_y_coord to from_geom
 -- Convert to_x_coord, to_y_coord to to_geom

products/cbbr/sql/cbbr_submissions.sql

@@ -4,7 +4,7 @@ DROP TABLE IF EXISTS _cbbr_submissions;
 CREATE TABLE _cbbr_submissions AS
 SELECT
     omb.dcpuniqid AS unique_id,
-    omb.trkno AS tracking_code,
+    TRIM(omb.trkno) AS tracking_code,
     omb.borough,
     RIGHT(omb.cb_label, 2) AS cd,
     omb.cb_label,
@@ -14,18 +14,18 @@ SELECT
     dcp.policy_area,
     dcp.need_group,
     dcp.need,
-    dcp.budget_request_title AS title,
+    dcp.budget_request_title_please_give_your_budget_request_a_short_an AS title,
     dcp.request,
     -- TODO - should probably just use dcp.explanation but clean special charactesr
     REPLACE(omb.reason, E'\n', ' ') AS explanation,
-    dcp.location_specific,
-    dcp.address,
-    dcp.site_or_facility_name,
-    dcp.on_street,
-    dcp.cross_street_1,
-    dcp.cross_street_2,
-    dcp.intersection_street_1,
-    dcp.intersection_street_2,
+    dcp."is_this_request_specific_to_a_particular_site,_location,_or_fac" AS location_specific,
+    dcp."address:request_location_(complete_all_that_apply)_for_addresse" AS address,
+    dcp."site_or_facility_name:request_location_(complete_all_that_apply" AS site_or_facility_name,
+    dcp."on_street_(enter_street_name):complete_the_following_if_your_re" AS on_street,
+    dcp."cross_street_1_(for_street_segment):request_location_(complete_" AS cross_street_1,
+    dcp."cross_street_2_(for_street_segment):request_location_(complete_" AS cross_street_2,
+    dcp."intersection___street_1:complete_the_following_if_your_request_" AS intersection_street_1,
+    dcp."intersection___street_2:complete_the_following_if_your_request_" AS intersection_street_2,
     omb.lowsgrp1 AS supporters_1,
     omb.lowsgrp2 AS supporters_2,
     omb.capis1 AS project_id_1, -- all null

products/cbbr/sql/export.sql

@@ -101,7 +101,7 @@ SELECT
     commdist,
     cb_label,
     type_br,
-    -- type,
+    type,
     priority,
     policy_area,
     need_group,

products/cbbr/sql/normalize_commdist.sql

@@ -50,5 +50,5 @@ SET
     borough_code = '5',
     commdist = '5' || lpad(cd, 2, '0')
 WHERE
-    a.borough = 'SI'
+    a.borough = 'Staten Island'
     AND commdist IS NULL;

products/cbbr/sql/normalize_requests.sql

@@ -0,0 +1,9 @@
+UPDATE _cbbr_submissions a
+SET
+    policy_area = 'Other Needs',
+    need_group = 'Other',
+    need = 'Other Capital Needs'
+WHERE
+    need IS NULL
+    AND agency_acronym = 'DSNY'
+    AND type = 'Capital';