feat: add geocode cache

bin/cmd.ts

@@ -3,6 +3,7 @@ import { SignJWT } from "jose";
 import ensureOrganisationMap from "@/server/commands/ensureOrganisationMap";
 import importAreaSet from "@/server/commands/importAreaSet";
 import importPostcodes from "@/server/commands/importPostcodes";
+import populateGeocodeCache from "@/server/commands/populateGeocodeCache";
 import regeocode from "@/server/commands/regeocode";
 import removeDevWebhooks from "@/server/commands/removeDevWebhooks";
 import Invite from "@/server/emails/Invite";
@@ -79,6 +80,15 @@ program
     await importDataSource({ dataSourceId });
   });
 
+program
+  .command("populateGeocodeCache")
+  .description(
+    "Populate the geocode cache from existing address-geocoded data sources",
+  )
+  .action(async () => {
+    await populateGeocodeCache();
+  });
+
 program
   .command("upsertUser")
   .option("--email <email>")

migrations/1774658921006_geocode_cache.ts

@@ -0,0 +1,17 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { type Kysely, sql } from "kysely";
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .createTable("geocodeCache")
+    .addColumn("address", "text", (col) => col.primaryKey())
+    .addColumn("point", sql`geography`)
+    .addColumn("createdAt", "timestamp", (col) =>
+      col.notNull().defaultTo(sql`CURRENT_TIMESTAMP`),
+    )
+    .execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.schema.dropTable("geocodeCache").execute();
+}

src/server/commands/populateGeocodeCache.ts

@@ -0,0 +1,71 @@
+import { sql } from "kysely";
+import { GeocodingType } from "@/models/DataSource";
+import { db } from "@/server/services/database";
+import logger from "@/server/services/logger";
+import type { AddressGeocodingConfig } from "@/models/DataSource";
+import type { Point } from "@/models/shared";
+
+export default async function populateGeocodeCache() {
+  const dataSources = await db
+    .selectFrom("dataSource")
+    .select(["id", "name", "geocodingConfig"])
+    .where(sql`geocoding_config->>'type'`, "=", GeocodingType.Address)
+    .execute();
+
+  logger.info(`Found ${dataSources.length} address-geocoded data sources`);
+
+  let totalInserted = 0;
+
+  for (const dataSource of dataSources) {
+    const config = dataSource.geocodingConfig as AddressGeocodingConfig;
+    const addressColumns = config.columns;
+
+    const records = await db
+      .selectFrom("dataRecord")
+      .select(["json", "geocodePoint"])
+      .where("dataSourceId", "=", dataSource.id)
+      .execute();
+
+    const entries: { address: string; point: Point | null }[] = [];
+    for (const record of records) {
+      const json = record.json as Record<string, unknown>;
+      const address = addressColumns
+        .map((c) => json[c] || "")
+        .filter(Boolean)
+        .join(", ")
+        .trim();
+      if (!address) continue;
+
+      entries.push({ address, point: record.geocodePoint as Point | null });
+    }
+
+    if (entries.length === 0) {
+      logger.info(`  ${dataSource.name}: no records, skipping`);
+      continue;
+    }
+
+    // Deduplicate by address within this data source
+    const uniqueByAddress = new Map(entries.map((e) => [e.address, e]));
+    const deduplicated = Array.from(uniqueByAddress.values());
+
+    // Insert in batches of 500
+    const batchSize = 500;
+    let inserted = 0;
+    for (let i = 0; i < deduplicated.length; i += batchSize) {
+      const batch = deduplicated.slice(i, i + batchSize);
+      await db
+        .insertInto("geocodeCache")
+        .values(batch)
+        .onConflict((oc) => oc.column("address").doNothing())
+        .execute();
+      inserted += batch.length;
+    }
+
+    logger.info(
+      `  ${dataSource.name}: inserted ${inserted} cache entries from ${records.length} records`,
+    );
+    totalInserted += inserted;
+  }
+
+  logger.info(`Done. Inserted ${totalInserted} total cache entries.`);
+}

src/server/mapping/geocode.ts

@@ -1,3 +1,4 @@
+import { sql } from "kysely";
 import { getBooleanEnvVar } from "@/env";
 import { AreaSetCode } from "@/models/AreaSet";
 import {
@@ -12,6 +13,7 @@ import {
   findAreaByName,
   findAreasByPoint,
 } from "@/server/repositories/Area";
+import { db } from "@/server/services/database";
 import logger from "@/server/services/logger";
 import { geojsonPointToPoint } from "../utils/geo";
 import type { GeocodeResult } from "@/models/DataRecord";
@@ -199,32 +201,10 @@ const geocodeRecordByAddress = async (
   }
 
   if (!point) {
-    const geocodeUrl = new URL(
-      "https://api.mapbox.com/search/geocode/v6/forward",
-    );
-    geocodeUrl.searchParams.set("q", address);
-    geocodeUrl.searchParams.set("country", "GB");
-    geocodeUrl.searchParams.set(
-      "access_token",
-      process.env.MAPBOX_SECRET_TOKEN || "",
-    );
-
-    const response = await fetch(geocodeUrl);
-    if (!response.ok) {
-      throw new Error(`Geocode request failed: ${response.status}`);
-    }
-    const results = (await response.json()) as {
-      features?: { id: string; geometry: GeoJSONPoint }[];
-    };
-    if (!results.features?.length) {
+    point = await mapboxGeocode(address);
+    if (!point) {
       throw new Error(`Geocode request returned no features`);
     }
-
-    const feature = results.features[0];
-    point = {
-      lng: feature.geometry.coordinates[0],
-      lat: feature.geometry.coordinates[1],
-    };
   }
 
   const geocodeResult: GeocodeResult = {
@@ -352,3 +332,53 @@ const postcodesIOLookup = async (
     },
   };
 };
+
+const mapboxGeocode = async (address: string): Promise<Point | null> => {
+  const cached = await db
+    .selectFrom("geocodeCache")
+    .select("point")
+    .where("address", "=", address)
+    .where("createdAt", ">", sql<Date>`now() - interval '4 weeks'`)
+    .executeTakeFirst();
+
+  if (cached) {
+    logger.debug(`Geocode cache hit for "${address}"`);
+    return cached.point;
+  }
+
+  logger.info(`Geocode cache miss for "${address}", calling Mapbox API`);
+  const geocodeUrl = new URL(
+    "https://api.mapbox.com/search/geocode/v6/forward",
+  );
+  geocodeUrl.searchParams.set("q", address);
+  geocodeUrl.searchParams.set("country", "GB");
+  geocodeUrl.searchParams.set(
+    "access_token",
+    process.env.MAPBOX_SECRET_TOKEN || "",
+  );
+
+  const response = await fetch(geocodeUrl);
+  if (!response.ok) {
+    throw new Error(`Geocode request failed: ${response.status}`);
+  }
+  const results = (await response.json()) as {
+    features?: { id: string; geometry: GeoJSONPoint }[];
+  };
+
+  const point: Point | null = results.features?.length
+    ? {
+        lng: results.features[0].geometry.coordinates[0],
+        lat: results.features[0].geometry.coordinates[1],
+      }
+    : null;
+
+  await db
+    .insertInto("geocodeCache")
+    .values({ address, point })
+    .onConflict((oc) =>
+      oc.column("address").doUpdateSet({ point, createdAt: sql`now()` }),
+    )
+    .execute();
+
+  return point;
+};

src/server/models/GeocodeCache.ts

@@ -0,0 +1,10 @@
+import type { Point } from "@/models/shared";
+import type { ColumnType, Insertable } from "kysely";
+
+export interface GeocodeCacheTable {
+  address: string;
+  point: Point | null;
+  createdAt: ColumnType<Date, Date | undefined, Date>;
+}
+
+export type NewGeocodeCache = Insertable<GeocodeCacheTable>;

src/server/services/database/index.ts

@@ -11,6 +11,7 @@ import type { DataRecordTable } from "@/server/models/DataRecord";
 import type { DataSourceTable } from "@/server/models/DataSource";
 import type { DataSourceOrganisationOverrideTable } from "@/server/models/DataSourceOrganisationOverride";
 import type { FolderTable } from "@/server/models/Folder";
+import type { GeocodeCacheTable } from "@/server/models/GeocodeCache";
 import type { InspectorDataSourceConfigTable } from "@/server/models/InspectorDataSourceConfig";
 import type { InvitationTable } from "@/server/models/Invitation";
 import type { JobTable } from "@/server/models/Job";
@@ -62,6 +63,7 @@ export interface Database {
   dataSource: DataSourceTable;
   dataRecord: DataRecordTable;
   folder: FolderTable;
+  geocodeCache: GeocodeCacheTable;
   invitation: InvitationTable;
   map: MapTable;
   mapView: MapViewTable;

src/server/services/database/schema.ts

@@ -374,6 +374,16 @@ export interface DataSourceOrganisationOverride {
   inspectorColumns: unknown[]; // jsonb, NOT NULL, DEFAULT []
 }
 
+/**
+ * geocodeCache Table
+ * Caches Mapbox geocoding API responses to avoid redundant requests
+ */
+export interface GeocodeCache {
+  address: string; // text, PRIMARY KEY
+  point: unknown; // geography, NULLABLE
+  createdAt: Date; // timestamp, NOT NULL, DEFAULT CURRENT_TIMESTAMP
+}
+
 // ============================================================================
 // TYPE EXPORTS
 // ============================================================================
@@ -397,6 +407,9 @@ export interface Database {
   // Webhooks & Integrations
   airtableWebhook: AirtableWebhook;
 
+  // Caches
+  geocodeCache: GeocodeCache;
+
   // Maps & Views
   map: Map;
   mapView: MapView;