From 2ea5cbda8ccf716a9d601a78b7119ecd57a5a818 Mon Sep 17 00:00:00 2001 From: gabe-levin Date: Mon, 24 Nov 2025 15:02:51 -0500 Subject: [PATCH 1/4] fix: database .pgdata del issue and autoregister new db in postgres --- dev-db/docker-compose.yaml | 19 +++++++++++-------- dev-db/pgadmin-servers.json | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 8 deletions(-) create mode 100644 dev-db/pgadmin-servers.json diff --git a/dev-db/docker-compose.yaml b/dev-db/docker-compose.yaml index 69c60c79..226a5f9f 100644 --- a/dev-db/docker-compose.yaml +++ b/dev-db/docker-compose.yaml @@ -1,13 +1,14 @@ services: - database: + db: + container_name: dev-db platform: linux/amd64 build: context: . dockerfile: Dockerfile environment: - - POSTGRES_USER=username - - POSTGRES_PASSWORD=password - - POSTGRES_DB=postgres + POSTGRES_USER: username + POSTGRES_PASSWORD: password + POSTGRES_DB: postgres ports: - 5439:5432 command: > @@ -29,7 +30,7 @@ services: -c min_wal_size=1GB -c max_wal_size=4GB volumes: - - ./.pgdata:/var/lib/postgresql/data + - dev_db_data:/var/lib/postgresql/data - ./init-scripts:/docker-entrypoint-initdb.d - ./space2stats_sample_cs.parquet:/docker-entrypoint-initdb.d/data/space2stats_sample_cs.parquet @@ -43,15 +44,17 @@ services: pgadmin: image: dpage/pgadmin4:latest environment: - - PGADMIN_DEFAULT_EMAIL=admin@space2stats.com - - PGADMIN_DEFAULT_PASSWORD=admin + PGADMIN_DEFAULT_EMAIL: admin@space2stats.com + PGADMIN_DEFAULT_PASSWORD: admin ports: - 8080:80 depends_on: - database: + db: condition: service_healthy volumes: - pgadmin_data:/var/lib/pgadmin + - ./pgadmin-servers.json:/pgadmin4/servers.json:ro volumes: pgadmin_data: + dev_db_data: diff --git a/dev-db/pgadmin-servers.json b/dev-db/pgadmin-servers.json new file mode 100644 index 00000000..15162df2 --- /dev/null +++ b/dev-db/pgadmin-servers.json @@ -0,0 +1,14 @@ +{ + "Servers": { + "1": { + "Name": "Space2Stats Dev", + "Group": "Development", + "Host": "db", + "Port": 5432, + "MaintenanceDB": "postgres", + "Username": "username", + "SSLMode": "prefer", + "Comment": "Auto-registered by docker-compose" + } + } +} From fcc66dc92f1baea7d2d3f158c855d60ea0d70b22 Mon Sep 17 00:00:00 2001 From: gabe-levin Date: Mon, 24 Nov 2025 15:03:06 -0500 Subject: [PATCH 2/4] fix: use modern docker compose syntax --- dev-db/Makefile | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/dev-db/Makefile b/dev-db/Makefile index b9cfc41a..fb3a8a30 100644 --- a/dev-db/Makefile +++ b/dev-db/Makefile @@ -12,7 +12,7 @@ help: up: @echo "Starting development database..." - docker-compose up -d + docker compose up -d @echo "Waiting for database to be ready..." @$(MAKE) wait @echo "Database ready at localhost:5439" @@ -20,10 +20,10 @@ up: @$(MAKE) seed down: - docker-compose down + docker compose down logs: - docker-compose logs -f database + docker compose logs -f database reset: @echo "Resetting database (this removes all data)..." @@ -33,15 +33,13 @@ reset: apply-schema: @echo "Applying database schema updates..." - docker-compose exec -T database psql -U username -d postgres -f /docker-entrypoint-initdb.d/01-init.sql + docker compose exec -T database psql -U username -d postgres -f /docker-entrypoint-initdb.d/01-init.sql seed: @python seed_sample_data.py clean: - docker-compose down -v --remove-orphans - @echo "Removing local database files..." - @rm -rf .pgdata + docker compose down -v --remove-orphans # Cross-platform wait command wait: From 0ee27c415ba270d5c2c9dd49400359c5c4dcf26f Mon Sep 17 00:00:00 2001 From: gabe-levin Date: Mon, 24 Nov 2025 15:51:35 -0500 Subject: [PATCH 3/4] fix: dir for sample data parquet files. works on ubuntu now --- dev-db/docker-compose.yaml | 1 - dev-db/seed_sample_data.py | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-db/docker-compose.yaml b/dev-db/docker-compose.yaml index 226a5f9f..126d612c 100644 --- a/dev-db/docker-compose.yaml +++ b/dev-db/docker-compose.yaml @@ -32,7 +32,6 @@ services: volumes: - dev_db_data:/var/lib/postgresql/data - ./init-scripts:/docker-entrypoint-initdb.d - - ./space2stats_sample_cs.parquet:/docker-entrypoint-initdb.d/data/space2stats_sample_cs.parquet healthcheck: test: ["CMD-SHELL", "pg_isready -U username -d postgres"] diff --git a/dev-db/seed_sample_data.py b/dev-db/seed_sample_data.py index aa44a44b..a470b998 100644 --- a/dev-db/seed_sample_data.py +++ b/dev-db/seed_sample_data.py @@ -134,12 +134,13 @@ def insert_data_to_db( def main() -> None: """Load the Space2Stats sample datasets into the development database.""" script_dir = Path(__file__).parent + data_dir = script_dir / "init-scripts" / "data" main_table = get_table_name("PGTABLENAME", DEFAULT_MAIN_TABLE) ts_table = get_table_name("TIMESERIES_TABLE_NAME", DEFAULT_TS_TABLE) - cs_file = script_dir / "space2stats_sample_cs.parquet" - ts_file = script_dir / "space2stats_sample_ts.parquet" + cs_file = data_dir / "space2stats_sample_cs.parquet" + ts_file = data_dir / "space2stats_sample_ts.parquet" print(f"Preparing to seed cross-sectional data into '{main_table}'.") cs_df = load_parquet_data(cs_file) From d7d067b3570e867c730148223485a26f36bca4ce Mon Sep 17 00:00:00 2001 From: gabe-levin Date: Mon, 24 Nov 2025 15:57:50 -0500 Subject: [PATCH 4/4] update: readme with new dir for the sample data --- dev-db/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev-db/README.md b/dev-db/README.md index 5f7fd321..f3aef157 100644 --- a/dev-db/README.md +++ b/dev-db/README.md @@ -36,6 +36,8 @@ The seeding script loads two parquet datasets into the development database: - `space2stats_sample_cs.parquet` → inserted into the `space2stats` table. It contains administrative boundaries, demographic breakdowns, nighttime lights, built area statistics, GHS indicators, and flood exposure metrics. - `space2stats_sample_ts.parquet` → inserted into the `climate` table. It provides Standardized Precipitation Index (SPI) climate time series aggregated by H3 hexagon. +Both parquet files must live in `init-scripts/data/` with the exact filenames above (the seed script reads from that directory). + The latest copies of these parquet files are stored in `s3://wbg-geography01/Space2Stats/sample_data/local_db/`. ## Troubleshooting