Species refactor for both trefle and plantnet sources

README.md

@@ -115,6 +115,33 @@ These commands boot a test API at `http://localhost:3050` using an in-memory Mon
 - The shared package ensures type consistency between all applications
 - The API server handles all database connections and business logic
 
+## Database backups and restores
+
+### Creating a backup
+
+- **Run backup script**:
+
+```bash
+npm run dbBackup
+```
+
+This will:
+
+- Connect to the MongoDB instance configured via `apps/api/.env`
+- Save and zip the result of `mongodump` to `mongodb_backup_YYYY-MM-DD_HH-MM-SS.zip`
+
+### Restoring from a backup
+
+- **Run restore script**:
+
+```bash
+# From repo root:
+npm run dbRestore backups/mongodb_backup_YYYY-MM-DD_HH-MM-SS.zip
+
+# Or, if you have an already-extracted folder:
+npm run dbRestore backups/mongodb_backup_YYYY-MM-DD_HH-MM-SS
+```
+
 ## Verbose debug logging
 
 Verbose debug logging is facilitated by the [debug](https://www.npmjs.com/package/debug) package, and is turned off by default.

apps/api/.env.example

@@ -8,7 +8,7 @@ API_BASE_URL=http://localhost:3000
 MOBILE_APP_BASE_URL=plannting://
 
 # Enforce a minimum required mobile app version
-MOBILE_APP_MINIMUM_REQUIRED_VERSION=0.13.0
+MOBILE_APP_MINIMUM_REQUIRED_VERSION=0.14.0
 
 # Debug logging
 # See readme for "Verbose debug logging"

apps/api/jest.config.js

@@ -1,6 +1,7 @@
 module.exports = {
   preset: 'ts-jest',
   testEnvironment: 'node',
+  setupFiles: ['<rootDir>/jest.setup.js'],
   roots: ['<rootDir>/src'],
   testMatch: [
     '**/__tests__/**/*.test.ts',

apps/api/jest.setup.js

@@ -0,0 +1,6 @@
+// Ensure a valid MongoDB URI is set when config/db load, so MongoClient doesn't throw
+// "Invalid scheme, expected connection string to start with mongodb:// or mongodb+srv://".
+// Tests that need the DB mock rawClient; no real connection is made.
+if (!process.env.MONGO_URI) {
+  process.env.MONGO_URI = 'mongodb://localhost:27017/jest'
+}

apps/api/package.json

@@ -10,7 +10,9 @@
     "start": "node dist/index.js",
     "lint": "tsc --noEmit",
     "test": "jest",
-    "test:watch": "jest --watch"
+    "test:watch": "jest --watch",
+    "dbBackup": "tsx src/scripts/dbBackup.ts",
+    "dbRestore": "tsx src/scripts/dbRestore.ts"
   },
   "dependencies": {
     "@trpc/server": "^11.5.1",

apps/api/src/config/index.ts

@@ -16,7 +16,7 @@ export const config: ApiConfig = {
     baseUrl: process.env.MOBILE_APP_BASE_URL,
     // Used by the mobile app to determine whether it must hard-block the UI until the user updates.
     // Default is 0.0.0 so we never block unless explicitly configured.
-    minimumRequiredVersion: (process.env.MOBILE_APP_MINIMUM_REQUIRED_VERSION ?? '0.13.0') as SemVerString,
+    minimumRequiredVersion: (process.env.MOBILE_APP_MINIMUM_REQUIRED_VERSION ?? '0.14.0') as SemVerString,
     storeUrls: {
       android: process.env.MOBILE_APP_STORE_URL_ANDROID ?? 'https://play.google.com/store/apps/details?id=com.completecodesolutions.***',
       ios: process.env.MOBILE_APP_STORE_URL_IOS ?? 'https://apps.apple.com/us/app/***/***',
@@ -25,10 +25,12 @@ export const config: ApiConfig = {
   mongo: {
     host: process.env.MONGO_HOST,
     appName: process.env.MONGO_APP_NAME,
+    dbName: process.env.MONGO_DB_NAME,
     password: process.env.MONGO_PASSWORD,
     user: process.env.MONGO_USER,
     uri: process.env.MONGO_URI // For automated tests, we use a in-memory MongoDB
-      || `mongodb+srv://${process.env.MONGO_USER}:${process.env.MONGO_PASSWORD}@${process.env.MONGO_HOST}/${process.env.MONGO_DB_NAME}?retryWrites=true&w=majority&appName=${process.env.MONGO_APP_NAME}`,
+      || (process.env.MONGO_HOST && `mongodb+srv://${process.env.MONGO_USER}:${process.env.MONGO_PASSWORD}@${process.env.MONGO_HOST}/${process.env.MONGO_DB_NAME}?retryWrites=true&w=majority&appName=${process.env.MONGO_APP_NAME}`)
+      || '',
   },
   openai: {
     apiKey: process.env.OPENAI_API_KEY,

apps/api/src/config/types.ts

@@ -25,6 +25,7 @@ export type ApiConfig = {
   mongo: {
     host: string | undefined,
     appName: string | undefined,
+    dbName: string | undefined,
     password: string | undefined,
     user: string | undefined,
     uri: string,

apps/api/src/endpoints/trpc/plants/chat.ts

@@ -62,7 +62,8 @@ export const chat = authProcedure
 
     const species = plant.species
     if (species) {
-      systemParts.push('', '**Species**', `- Common name: ${species.commonName}`)
+      const primaryCommonName = species.commonNames?.[0]?.name
+      systemParts.push('', '**Species**', `- Common name: ${primaryCommonName ?? '(none)'}`)
       if (species.scientificName) systemParts.push(`- Scientific name: ${species.scientificName}`)
       if (species.familyCommonName) systemParts.push(`- Family: ${species.familyCommonName}`)
       if (species.genus) systemParts.push(`- Genus: ${species.genus}`)

apps/api/src/endpoints/trpc/species/listSpecies.ts

@@ -1,7 +1,5 @@
 import { z } from 'zod'
 
-import { Species } from '../../../models'
-
 import { authProcedure } from '../../../procedures/authProcedure'
 
 import * as speciesLookupService from '../../../services/speciesLookup'
@@ -14,35 +12,7 @@ export const listSpecies = authProcedure
     .default({})
   )
   .query(async ({ input }) => {
-    const speciesFromService = await speciesLookupService.findSpecies({ q: input.q })
-
-    // Find or create species in database
-    const speciesPromises = speciesFromService.map(async (speciesData) => {
-      // Try to find existing species by source and sourceId
-      let species = await Species.findOne({
-        source: speciesData.source,
-        sourceId: speciesData.sourceId,
-      })
-
-      // If not found, create it
-      if (!species) {
-        species = await Species.create({
-          source: speciesData.source,
-          sourceId: speciesData.sourceId,
-          imageUrl: speciesData.imageUrl,
-          commonName: speciesData.commonName,
-          scientificName: speciesData.scientificName,
-          familyCommonName: speciesData.familyCommonName,
-          synonyms: speciesData.synonyms,
-          genus: speciesData.genus,
-          family: speciesData.family,
-        })
-      }
-
-      return species.toJSON()
-    })
-
-    const species = await Promise.all(speciesPromises)
+    const species = await speciesLookupService.findAndCacheSpecies({ q: input.q })
 
     return { species }
   })

apps/api/src/models/Plant.ts

@@ -12,7 +12,7 @@ export interface IPlant {
   chores: mongoose.Types.ObjectId[],
   plantedAt: Date | null,
   species: mongoose.Types.ObjectId | null,
-  /** User-uploaded photo URL (e.g. Vercel Blob). For species image use populated plant.species.imageUrl. */
+  /** User-uploaded photo URL (e.g. Vercel Blob). For species image use populated plant.species.images.*.url. */
   photoUrl: string | null,
   createdAt: Date,
   updatedAt: Date,

apps/api/src/models/Species.ts

@@ -2,58 +2,76 @@ import mongoose from 'mongoose'
 
 export interface ISpecies {
   _id: string,
-  source: 'trefle',
-  sourceId: number,
-  imageUrl: string | null,
-  commonName: string,
-  scientificName: string | null,
-  familyCommonName: string | null,
-  synonyms: string[] | null,
+  sourceId_plantNet: string | null,
+  sourceId_trefle: number | null,
+  commonNames: SpeciesCommonName[],
+  images: SpeciesImage[],
   genus: string | null,
   family: string | null,
+  familyCommonName: string | null,
+  scientificName: string,
+  altScientificNames: string[], // Alternative scientific names (aka synonyms). These are often older outdated names.
   fertilizerSuggestions: string | null,
   createdAt: Date,
   updatedAt: Date,
 }
 
+export type SpeciesImage = {
+  source: SpeciesDatumSource,
+  url: string,
+}
+
+export type SpeciesCommonName = {
+  name: string,
+  sources: SpeciesDatumSource[],
+}
+
+export type SpeciesDatumSource = 'trefle' | 'plantNet'
+
 export const speciesSchema = new mongoose.Schema<ISpecies>({
-  source: {
+  sourceId_plantNet: {
     type: String,
-    enum: ['trefle'],
-    required: true,
+    default: null,
+    trim: true,
   },
-  sourceId: {
+  sourceId_trefle: {
     type: Number,
-    required: true,
-  },
-  imageUrl: {
-    type: String,
     default: null,
   },
-  commonName: {
-    type: String,
-    required: true,
-    trim: true,
+  commonNames: {
+    type: [{
+      name: { type: String, required: true, trim: true },
+      sources: { type: [String], enum: ['trefle', 'plantNet'], default: [] },
+    }],
+    default: [],
   },
-  scientificName: {
-    type: String,
-    default: null,
+  images: {
+    type: [{
+      url: { type: String, required: true, trim: true },
+      source: { type: String, enum: ['trefle', 'plantNet'], required: true },
+    }],
+    default: [],
   },
-  familyCommonName: {
+  genus: {
     type: String,
     default: null,
   },
-  synonyms: {
-    type: [String],
+  family: {
+    type: String,
     default: null,
   },
-  genus: {
+  familyCommonName: {
     type: String,
     default: null,
   },
-  family: {
+  scientificName: {
     type: String,
-    default: null,
+    required: true,
+    trim: true,
+  },
+  altScientificNames: {
+    type: [String],
+    default: [],
   },
   fertilizerSuggestions: {
     type: String,
@@ -64,7 +82,33 @@ export const speciesSchema = new mongoose.Schema<ISpecies>({
   timestamps: true,
 })
 
-// Create unique index on source + sourceId to prevent duplicates
-speciesSchema.index({ source: 1, sourceId: 1 }, { unique: true })
+// Unique scientific name (case-insensitive due to schema collation)
+speciesSchema.index({ scientificName: 1 }, { unique: true })
+
+speciesSchema.pre('validate', function normalizeSpecies(next) {
+  if (typeof this.scientificName === 'string') {
+    this.scientificName = normalizeScientificName(this.scientificName)
+  }
+
+  this.altScientificNames = Array.from(new Set((this.altScientificNames ?? []).map(normalizeScientificName).filter(Boolean)))
+
+  this.images = (this.images ?? [])
+    .map(img => ({ url: img.url?.trim?.() ?? img.url, source: img.source }))
+    .filter(img => !!img.url)
+
+  this.commonNames = (this.commonNames ?? [])
+    .map(cn => ({ name: cn.name?.trim?.() ?? cn.name, sources: Array.from(new Set(cn.sources ?? [])) }))
+    .filter(cn => !!cn.name)
+
+  next()
+})
+
+// For scientific names always have the first letter capitalized and all other letters lowercase
+const normalizeScientificName = (name: string) => {
+  const trimmed = name.trim().replace(/\s+/g, ' ') // Trim and collapse any whitespace runs (multiple spaces/tabs/newlines) into a single space
+  if (!trimmed) return trimmed
+
+  return trimmed[0]!.toUpperCase() + trimmed.slice(1).toLowerCase()
+}
 
 export const Species = mongoose.model<ISpecies>('Species', speciesSchema)

apps/api/src/scripts/dbBackup.ts

@@ -0,0 +1,83 @@
+import 'dotenv/config'
+import { spawn } from 'node:child_process'
+import fs from 'node:fs/promises'
+import path from 'node:path'
+
+import { config } from '../config'
+
+function runCommand(
+  command: string,
+  args: string[],
+  options?: { cwd?: string },
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, args, {
+      cwd: options?.cwd,
+      stdio: 'inherit',
+      env: process.env,
+    })
+
+    child.on('error', reject)
+    child.on('close', (code) => {
+      if (code === 0) return resolve()
+      reject(new Error(`${command} exited with code ${code ?? 'unknown'}`))
+    })
+  })
+}
+
+function timestampStampString(now = new Date()) {
+  const pad2 = (n: number) => String(n).padStart(2, '0')
+  const yyyy = now.getFullYear()
+  const mm = pad2(now.getMonth() + 1)
+  const dd = pad2(now.getDate())
+  const hh = pad2(now.getHours())
+  const min = pad2(now.getMinutes())
+  const ss = pad2(now.getSeconds())
+
+  return `${yyyy}-${mm}-${dd}_${hh}-${min}-${ss}`
+}
+
+async function main() {
+  if (!config.mongo.uri) {
+    throw new Error('Missing MONGO_URI / config.mongo.uri')
+  }
+
+  if (!config.mongo.dbName) {
+    throw new Error('Missing MONGO_DB_NAME / config.mongo.dbName')
+  }
+
+  const repoRoot = process.cwd()
+  const backupsDir = path.join(repoRoot, 'backups')
+  await fs.mkdir(backupsDir, { recursive: true })
+
+  const stamp = timestampStampString()
+  const dumpName = `mongodb_backup_${stamp}`
+  const dumpPath = path.join(backupsDir, dumpName)
+
+  // Stream output so the user sees mongodump/zip progress in the CLI.
+  await runCommand('mongodump', [`--uri=${config.mongo.uri}`, `--out=${dumpPath}`])
+
+  const dbSubDir = path.join(dumpPath, config.mongo.dbName)
+  // Remove the subdirectory named after the db. This is important so that future restores can restore a db with a different name.
+  await runCommand('sh', [
+    '-c',
+    'mv "$1"/* "$2"/ && rmdir "$1"',
+    '_',
+    dbSubDir,
+    dumpPath,
+  ])
+
+  console.log(`Backup files created in ${dumpPath}, zipping...`)
+  await runCommand('zip', ['-r', `${dumpName}.zip`, dumpName], { cwd: backupsDir })
+
+  // Remove the original dump directory now that the zip exists
+  await fs.rm(dumpPath, { recursive: true, force: true })
+
+  console.log(`Backup zip saved to ${dumpName}.zip`)
+}
+
+main().catch((err) => {
+  // eslint-disable-next-line no-console
+  console.error(err)
+  process.exit(1)
+})