Switch database to Postgres

.github/workflows/main.yml

@@ -26,11 +26,18 @@ jobs:
 
      - name: Run Ruff (linting)
        uses: astral-sh/ruff-action@v3
+       with:
+        version: "0.15.11"
 
      - name: Run pytest (unit tests)
        env:
-        DB_PATH: test.db
-       run: uv run alembic upgrade head && uv run pytest
+        DATABASE_ENGINE: sqlite
+        DATABASE_HOST: localhost
+        DATABASE_PORT: "5432"
+        DATABASE_USER: test
+        DATABASE_PASSWORD: test
+        DATABASE_DB: test
+       run: uv run pytest
 
      - name: Run mypy (type checking)
        run: uv run mypy app/

.pre-commit-config.yaml

@@ -12,7 +12,7 @@ repos:
 
       - id: pytest
         name: pytest
-        entry: sh -c 'DB_PATH="${DB_PATH:-test.db}" uv run pytest'
+        entry: uv run pytest
         language: system
         pass_filenames: false
 

Dockerfile

@@ -1,7 +1,6 @@
 FROM python:3.11-slim
 COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
-ENV DB_PATH="database.db"
 
 WORKDIR /app
 

Taskfile.yml

@@ -1,8 +1,5 @@
 version: "3"
 
-env:
-  DB_PATH: "database.db"
-
 tasks:
   setup:
     desc: Initial project setup
@@ -30,6 +27,6 @@ tasks:
       - "docker compose up --build"
 
   test:
-    desc: Run tests with separate DB
+    desc: Run tests locally with uv (SQLite in-memory)
     cmds:
-      - "docker compose exec backend sh -c 'DB_PATH=test.db uv run pytest -vvs'"
+      - "uv run pytest"

app/config.py

@@ -1,24 +1,32 @@
 from pathlib import Path
+from typing import Literal
 
-from pydantic import field_validator
-from pydantic_settings import BaseSettings
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 UPLOAD_DIR = Path("data")
 
-# Used to catch missing DB_PATH.
-# If this value is still set, it means DB_PATH wasn't configured.
-DEFAULT_DB_PLACEHOLDER = "__MISSING_DB_PATH__"
 
+class DatabaseSettings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_file=".env", extra="ignore"
+    )
 
-class Settings(BaseSettings):
-    db_path: str = DEFAULT_DB_PLACEHOLDER
+    database_engine: Literal["sqlite", "postgres"] = Field(default="sqlite")
+    database_host: str
+    database_port: int
+    database_user: str
+    database_password: str
+    database_db: str
 
-    @field_validator("db_path", mode="before")
-    @classmethod
-    def validate_db_path(cls, value):
-        if value == DEFAULT_DB_PLACEHOLDER:
-            raise ValueError("You must set DB_PATH in your environment or .env file.")
-        return value
+    @property
+    def database_url(self) -> str:
+        if self.database_engine == "sqlite":
+            return "sqlite:///:memory:"
+        return (
+            f"postgresql+psycopg://{self.database_user}:{self.database_password}"
+            f"@{self.database_host}:{self.database_port}/{self.database_db}"
+        )
 
 
-settings = Settings()
+settings = DatabaseSettings()

app/database.py

@@ -1,8 +1,17 @@
+from sqlalchemy.pool import StaticPool
 from sqlmodel import create_engine
 
 from app.config import settings
 
-connect_args = {"check_same_thread": False}
-engine = create_engine(
-    f"sqlite:///{settings.db_path}", echo=True, connect_args=connect_args
-)
+if settings.database_engine == "sqlite":
+    engine = create_engine(
+        settings.database_url,
+        echo=True,
+        connect_args={"check_same_thread": False},
+        poolclass=StaticPool,
+    )
+else:
+    engine = create_engine(
+        settings.database_url,
+        echo=True,
+    )

compose.yaml

@@ -1,20 +1,37 @@
 services:
-  database:
-    image: alpine:3.20
-    command: sh -c "echo database started"
+  db:
+    image: postgres:17-alpine
+    env_file: .env
+    restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${DATABASE_USER} -d ${DATABASE_DB}"]
+      start_period: 30s
+      start_interval: 5s
+      interval: 30s
+      timeout: 5s
+      retries: 10
     volumes:
-      - sqlite_data:/db
+      - pgdata:/var/lib/postgresql/data
+  adminer:
+    image: adminer
+    restart: always
+    ports:
+      - 8080:8080
 
   backend:
     build: .
+    env_file: .env
     environment:
-      DB_PATH: /db/database.db
+      DATABASE_ENGINE: postgres
+      DATABASE_HOST: db
+      DATABASE_PORT: 5432
     ports:
       - "8000:8000"
     depends_on:
-      - database
+      db:
+        condition: service_healthy
     volumes:
-      - sqlite_data:/db
+      - ./data:/app/data
     command: >
       sh -c "uv run alembic upgrade head &&
              uv run uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload"
@@ -40,4 +57,4 @@ services:
           action: rebuild
 
 volumes:
-  sqlite_data:
+  pgdata:

pyproject.toml

@@ -8,8 +8,10 @@ dependencies = [
     "alembic>=1.16.2",
     "fastapi[standard]>=0.115.12",
     "pandas>=2.2.3",
+    "psycopg[binary]>=3.3.4",
     "pyarrow>=22.0.0",
     "pydantic-settings>=2.8.1",
+    "sqlalchemy>=2.0.49",
     "sqlmodel>=0.0.24",
     "uvicorn>=0.34.0",
 ]
@@ -29,12 +31,20 @@ dev = [
 [tool.pytest.ini_options]
 addopts = ["-vvs"]
 env = [
-    "DB_PATH=test.db",
+    "DATABASE_ENGINE=sqlite",
+    "DATABASE_HOST=localhost",
+    "DATABASE_PORT=5432",
+    "DATABASE_USER=test",
+    "DATABASE_PASSWORD=test",
+    "DATABASE_DB=test",
 ]
 
 [[tool.mypy.overrides]]
 module = ["pyarrow", "pyarrow.csv", "pyarrow.parquet"]
 ignore_missing_imports = true
 
 [tool.ruff]
-exclude = [".venv", "uv.lock"]
+exclude = [".venv", "uv.lock", "alembic"]
+
+[tool.mypy]
+plugins = ["pydantic.mypy"]

tests/conftest.py

@@ -1,17 +1,21 @@
 import logging
-import os
 from pathlib import Path
 
 import pytest
 from fastapi.testclient import TestClient
-from sqlmodel import Session, SQLModel, create_engine
+from sqlmodel import Session, SQLModel
 
+from app.config import settings
+from app.database import engine
 from app.dependencies import get_session, get_upload_dir
 from app.main import app
 
 
 @pytest.fixture(scope="session", autouse=True)
-def disable_logging():
+def test_setup():
+    assert settings.database_engine == "sqlite", (
+        f"Tests must use SQLite, got engine: {settings.engine}"
+    )
     logging.disable(logging.CRITICAL)
     yield
     logging.disable(logging.NOTSET)
@@ -24,17 +28,8 @@ def disable_logging():
 
 @pytest.fixture(scope="session")
 def test_engine(tmp_path_factory):
-    db_path = tmp_path_factory.mktemp("db") / "test.db"
-    os.environ["DB_PATH"] = str(db_path)
-    engine = create_engine(
-        f"sqlite:///{db_path}",
-        connect_args={"check_same_thread": False},
-        echo=False,
-    )
     SQLModel.metadata.create_all(engine)
     yield engine
-    db_path.unlink(missing_ok=True)
-
 
 @pytest.fixture
 def session(test_engine):