From a656291825a9eed4d203588577f78b591b3542aa Mon Sep 17 00:00:00 2001 From: Damon McCullough Date: Tue, 17 Mar 2026 15:55:58 -0400 Subject: [PATCH 1/6] update kpdb envrc file --- products/knownprojects/.envrc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/products/knownprojects/.envrc b/products/knownprojects/.envrc index a3b4179b92..16f9c699b2 100644 --- a/products/knownprojects/.envrc +++ b/products/knownprojects/.envrc @@ -1,5 +1,6 @@ source_up +export BUILD_ENGINE_DB=kpdb export DATASET=knownprojects # Load local .env if it exists @@ -7,3 +8,5 @@ if [ -f .env ]; then dotenv .env watch_file .env fi + +export BUILD_ENGINE=$(echo_build_engine) From 5453c851230905c78f7b23551c4a69dd6553084b Mon Sep 17 00:00:00 2001 From: Damon McCullough Date: Wed, 3 Dec 2025 15:54:37 -0500 Subject: [PATCH 2/6] pin KPDB sources to 20250429 version --- products/knownprojects/recipe.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/products/knownprojects/recipe.yml b/products/knownprojects/recipe.yml index 71e6342ae8..1be846197b 100644 --- a/products/knownprojects/recipe.yml +++ b/products/knownprojects/recipe.yml @@ -15,13 +15,24 @@ inputs: - name: dcp_dcpprojectteams version_env_var: ZAP_VERSION - name: dcp_mappluto_wi + version: 25v1.1 - name: dcp_boroboundaries + version: 25a - name: dcp_housing + version: 24Q4 - name: dcp_zoningmapamendments + version: 20250331 - name: doe_eszones + version: 20240313 - name: doe_school_subdistricts + version: 2017 - name: dcp_school_districts + version: 25a - name: dcp_ct2020_wi + version: 25a - name: dcp_nta2020 + version: 25a - name: dcp_cdta2020 + version: 25a - name: dcp_cdboundaries_wi + version: 25a From c95fa490943654db6de9d0f68d76aed7db51b151 Mon Sep 17 00:00:00 2001 From: Damon McCullough Date: Tue, 17 Mar 2026 22:25:52 -0400 Subject: [PATCH 3/6] don't pin doe_school_subdistricts otherwise seems to cause error in boundaries_school_subdistricts.sql: ST_Intersects: Operation on mixed SRID geometries (Polygon, 4326) != (MultiPolygon, 0) --- products/knownprojects/recipe.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/products/knownprojects/recipe.yml b/products/knownprojects/recipe.yml index 1be846197b..78751eba13 100644 --- a/products/knownprojects/recipe.yml +++ b/products/knownprojects/recipe.yml @@ -25,7 +25,7 @@ inputs: - name: doe_eszones version: 20240313 - name: doe_school_subdistricts - version: 2017 + # version: 2017 - name: dcp_school_districts version: 25a - name: dcp_ct2020_wi From 7a7f3d0eccc95b1565fafc088c3383ce268306c1 Mon Sep 17 00:00:00 2001 From: Damon McCullough Date: Tue, 17 Mar 2026 16:09:47 -0400 Subject: [PATCH 4/6] disable use of future neighborhood studies There are no future neighborhood studies at the momemnt --- products/knownprojects/bash/01_dataloading.sh | 3 ++- products/knownprojects/python/__init__.py | 3 ++- products/knownprojects/sql/combine.sql | 5 ++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/products/knownprojects/bash/01_dataloading.sh b/products/knownprojects/bash/01_dataloading.sh index 54bae3538f..aeb3992ae3 100755 --- a/products/knownprojects/bash/01_dataloading.sh +++ b/products/knownprojects/bash/01_dataloading.sh @@ -15,7 +15,8 @@ python3 -m python.extractors esd_projects python3 -m python.extractors edc_projects python3 -m python.extractors edc_dcp_inputs python3 -m python.extractors dcp_n_study -python3 -m python.extractors dcp_n_study_future +# There are no future neighborhood studies at the momemnt +# python3 -m python.extractors dcp_n_study_future python3 -m python.extractors dcp_n_study_projected python3 -m python.extractors hpd_rfp python3 -m python.extractors dcp_planneradded diff --git a/products/knownprojects/python/__init__.py b/products/knownprojects/python/__init__.py index bdac7abda2..73d5a60919 100644 --- a/products/knownprojects/python/__init__.py +++ b/products/knownprojects/python/__init__.py @@ -28,7 +28,8 @@ "edc_projects": "2022.11.18 EDC inputs for DCP housing projections.xlsx", "edc_dcp_inputs": "edc_shapefile_20250225.zip", "dcp_n_study": "nstudy_rezoning_commitments_shapefile_20250430.zip", - "dcp_n_study_future": "future_neighborhoodstudies_20250304.zip", + # There are no future neighborhood studies at the momemnt + # "dcp_n_study_future": "future_neighborhoodstudies_20250304.zip", "dcp_n_study_projected": "past_neighborhoodstudies_20250304.zip", "hpd_rfp": "HPD_RFPs_20250110.xlsx", "dcp_planneradded": "dcp_planneradded_2025_04_30.csv", diff --git a/products/knownprojects/sql/combine.sql b/products/knownprojects/sql/combine.sql index fb6206c67e..8fe56ec2ab 100644 --- a/products/knownprojects/sql/combine.sql +++ b/products/knownprojects/sql/combine.sql @@ -167,7 +167,7 @@ _dcp_planneradded AS ( flag_senior_housing(a::text) AS senior_housing FROM dcp_planneradded AS a ), - +/* There are no future neighborhood studies at the momemnt _dcp_n_study_future AS ( SELECT 'Future Neighborhood Studies' AS source, @@ -189,6 +189,7 @@ _dcp_n_study_future AS ( record_nam || ' ' || 'Future Rezoning Development' AS record_name FROM dcp_n_study_future ), +*/ _dcp_n_study_projected AS ( SELECT @@ -415,6 +416,7 @@ FROM ( record_id_input, st_makevalid(geom) AS geom FROM _dcp_n_study + /* There are no future neighborhood studies at the momemnt UNION SELECT source, @@ -435,6 +437,7 @@ FROM ( record_id_input, st_makevalid(geom) AS geom FROM _dcp_n_study_future + */ UNION SELECT source, From 0a30849888e97200bb3057791ef445e99b4ac7bc Mon Sep 17 00:00:00 2001 From: Damon McCullough Date: Tue, 17 Mar 2026 16:11:11 -0400 Subject: [PATCH 5/6] update past neighborhood studies --- products/knownprojects/python/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/products/knownprojects/python/__init__.py b/products/knownprojects/python/__init__.py index 73d5a60919..72765afa6b 100644 --- a/products/knownprojects/python/__init__.py +++ b/products/knownprojects/python/__init__.py @@ -30,7 +30,7 @@ "dcp_n_study": "nstudy_rezoning_commitments_shapefile_20250430.zip", # There are no future neighborhood studies at the momemnt # "dcp_n_study_future": "future_neighborhoodstudies_20250304.zip", - "dcp_n_study_projected": "past_neighborhoodstudies_20250304.zip", + "dcp_n_study_projected": "past_neighborhoodstudies_20260305.zip", "hpd_rfp": "HPD_RFPs_20250110.xlsx", "dcp_planneradded": "dcp_planneradded_2025_04_30.csv", } From a112505659656c8ff92d6bc72fdd23cf5934c975 Mon Sep 17 00:00:00 2001 From: Damon McCullough Date: Tue, 17 Mar 2026 15:27:03 -0400 Subject: [PATCH 6/6] use Hudson Square project geometry source data to correct a ZAP project until we use the lates ZAP data --- products/knownprojects/bash/01_dataloading.sh | 1 + products/knownprojects/python/__init__.py | 1 + products/knownprojects/python/extractors.py | 11 +++++++++++ products/knownprojects/sql/dcp_application.sql | 16 +++++++++++++++- 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/products/knownprojects/bash/01_dataloading.sh b/products/knownprojects/bash/01_dataloading.sh index aeb3992ae3..a6df600893 100755 --- a/products/knownprojects/bash/01_dataloading.sh +++ b/products/knownprojects/bash/01_dataloading.sh @@ -21,6 +21,7 @@ python3 -m python.extractors dcp_n_study_projected python3 -m python.extractors hpd_rfp python3 -m python.extractors dcp_planneradded python3 -m python.extractors dcp_knownprojects +python3 -m python.extractors dcp_hudsonsquare # Load corrections tables run_sql_file sql/create_corrections.sql diff --git a/products/knownprojects/python/__init__.py b/products/knownprojects/python/__init__.py index 72765afa6b..e723e342ca 100644 --- a/products/knownprojects/python/__init__.py +++ b/products/knownprojects/python/__init__.py @@ -33,6 +33,7 @@ "dcp_n_study_projected": "past_neighborhoodstudies_20260305.zip", "hpd_rfp": "HPD_RFPs_20250110.xlsx", "dcp_planneradded": "dcp_planneradded_2025_04_30.csv", + "dcp_hudsonsquare": "HudsonSquare_P2012M0255_4326.zip", } DCP_HOUSING_CORRECTIONS_FILENAMES = { "corrections_dob": "corrections_dob.csv", diff --git a/products/knownprojects/python/extractors.py b/products/knownprojects/python/extractors.py index b7091ded40..2e02995100 100644 --- a/products/knownprojects/python/extractors.py +++ b/products/knownprojects/python/extractors.py @@ -72,6 +72,17 @@ def dcp_n_study_projected(filename: str) -> gpd.GeoDataFrame: return df +@ETL +def dcp_hudsonsquare(filename: str) -> gpd.GeoDataFrame: + # df = gpd.read_file(f"zip://{RAW_DATA_PATH}/{filename}") + + filename_prefix = f"{filename}".split(".")[0] + with zipfile.ZipFile(f"{RAW_DATA_PATH}/{filename}", "r") as zip_ref: + zip_ref.extractall(f"{RAW_DATA_PATH}") + df = gpd.read_file(f"{RAW_DATA_PATH}/{filename_prefix}/{filename_prefix}.shp") + return df + + @ETL def hpd_rfp(filename: str) -> pd.DataFrame: df = pd.read_excel(f"{RAW_DATA_PATH}/{filename}", dtype=str) diff --git a/products/knownprojects/sql/dcp_application.sql b/products/knownprojects/sql/dcp_application.sql index 0c94e50dc9..96faa3b650 100644 --- a/products/knownprojects/sql/dcp_application.sql +++ b/products/knownprojects/sql/dcp_application.sql @@ -347,6 +347,20 @@ geom_ulurp AS ( ) AS a LEFT JOIN dcp_zoningmapamendments AS zma ON a.dcp_ulurpnumber = zma.ulurpno GROUP BY a.record_id, a.geom +), + +-- Correcting geometry for Hudson Square project (P2012M0255) while using old ZAP data +geom_corrections AS ( + SELECT + a.record_id, + CASE + WHEN a.record_id = 'P2012M0255' + THEN b.geometry + ELSE a.geom + END AS geom + FROM geom_ulurp AS a + LEFT JOIN dcp_hudsonsquare AS b + ON a.record_id = b.projectid ) -- Main table with the geometry lookup @@ -355,5 +369,5 @@ SELECT b.geom INTO dcp_application FROM _dcp_application AS a -LEFT JOIN geom_ulurp AS b +LEFT JOIN geom_corrections AS b ON a.record_id = b.record_id;