diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ace2b85 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.venv/ +__pycache__/ +.pytest_cache/ +venv/ \ No newline at end of file diff --git a/company-api/README.md b/company-api/README.md index e69de29..2f9f66e 100644 --- a/company-api/README.md +++ b/company-api/README.md @@ -0,0 +1,45 @@ +# Company Data Service + +This is the core backend service for the Data-Forge System. + +## Prerequisites + +- Python 3.10+ +- `pip` and `venv` + +## Local Development Setup + +1. **Navigate to the service directory**: + ```bash + cd company-data/company-api + +## Create and activate virtual environment + +```bash +python -m venv venv +# On macOS/Linux: +source venv/bin/activate +# On Windows: +venv\Scripts\activate +``` + +## Install Dependencies + +```bash +pip install -r requirements.txt +``` +## Run the development server + +```bash +uvicorn main:app --reload +``` + +## Verify it's working: +- Open your browser to ```http://localhost:8000``` → Should return ```{"status": "ok"}``` +- Interactive API docs are available at http://localhost:8000/docs + +## Running the tests +```bash +# From the company-api directory +python -m pytest +``` diff --git a/company-api/error_handlers.py b/company-api/error_handlers.py new file mode 100644 index 0000000..c32c29e --- /dev/null +++ b/company-api/error_handlers.py @@ -0,0 +1,37 @@ +""" +Exception handler registration for company-api. + +Kept separate from main.py so tests can mount these handlers on a +throwaway FastAPI app without needing the full production app (real DB +connection, all routes, etc) - see tests/test_error_handling.py. +""" + +import logging + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse + +from exceptions import AppError + +logger = logging.getLogger(__name__) + + +def register_exception_handlers(app: FastAPI) -> None: + @app.exception_handler(AppError) + async def app_error_handler(request: Request, exc: AppError) -> JSONResponse: + logger.warning( + "app_error", + extra={"error_code": exc.error_code, "path": str(request.url), "details": exc.details}, + ) + return JSONResponse( + status_code=exc.status_code, + content={"error": exc.error_code, "message": exc.message, "details": exc.details}, + ) + + @app.exception_handler(Exception) + async def unhandled_exception_handler(request: Request, exc: Exception) -> JSONResponse: + logger.exception("unhandled_exception", extra={"path": str(request.url)}) + return JSONResponse( + status_code=500, + content={"error": "internal_error", "message": "An unexpected error occurred", "details": {}}, + ) \ No newline at end of file diff --git a/company-api/main.py b/company-api/main.py new file mode 100644 index 0000000..5560f60 --- /dev/null +++ b/company-api/main.py @@ -0,0 +1,24 @@ +""" +company-api entrypoint. + +Deliberately minimal right now - no DB dependency, no business routes yet. +This exists so error-handling conventions have somewhere to attach and +get tested before routes/DB integration land on top. +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "company-common")) + +from fastapi import FastAPI + +from error_handlers import register_exception_handlers + +app = FastAPI(title="Company API", version="0.1.0") +register_exception_handlers(app) + + +@app.get("/") +async def health() -> dict: + return {"status": "ok"} \ No newline at end of file diff --git a/company-api/pytest.ini b/company-api/pytest.ini new file mode 100644 index 0000000..ce652b9 --- /dev/null +++ b/company-api/pytest.ini @@ -0,0 +1,9 @@ +[pytest] +testpaths = tests +python_files = test_*.py +python_classes = Test* +python_functions = test_* +addopts = -v --strict-markers +markers = + unit: Unit tests + integration: Integration tests \ No newline at end of file diff --git a/company-api/requirements.txt b/company-api/requirements.txt new file mode 100644 index 0000000..44810e8 --- /dev/null +++ b/company-api/requirements.txt @@ -0,0 +1,5 @@ +fastapi>=0.110.0 +uvicorn[standard]>=0.30.0 +httpx>=0.27.0 +pytest>=9.0.0 +pydantic>=2.0.0 \ No newline at end of file diff --git a/company-api/tests/conftest.py b/company-api/tests/conftest.py new file mode 100644 index 0000000..73bc623 --- /dev/null +++ b/company-api/tests/conftest.py @@ -0,0 +1,14 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent / "company-common")) + +import pytest +from fastapi.testclient import TestClient + +from main import app + + +@pytest.fixture +def client() -> TestClient: + return TestClient(app) \ No newline at end of file diff --git a/company-api/tests/test_error_handling.py b/company-api/tests/test_error_handling.py new file mode 100644 index 0000000..f93d13c --- /dev/null +++ b/company-api/tests/test_error_handling.py @@ -0,0 +1,61 @@ +""" +Tests the exception -> JSON response contract itself, independent of any +real route. A throwaway app with routes that deliberately raise is mounted +here so this contract stays tested even before company-api has real +business routes to exercise it through. +""" + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from error_handlers import register_exception_handlers +from exceptions import AppError, NotFoundError, DuplicateRecordError + + +@pytest.fixture +def error_client() -> TestClient: + app = FastAPI() + register_exception_handlers(app) + + @app.get("/boom/not-found") + async def boom_not_found(): + raise NotFoundError("company techcorp.com not found", details={"uid": "techcorp.com"}) + + @app.get("/boom/duplicate") + async def boom_duplicate(): + raise DuplicateRecordError("company already exists") + + @app.get("/boom/unhandled") + async def boom_unhandled(): + raise RuntimeError("something broke that we didn't anticipate") + + return TestClient(app, raise_server_exceptions=False) + + +class TestAppErrorHandling: + def test_not_found_error_shape(self, error_client): + response = error_client.get("/boom/not-found") + assert response.status_code == 404 + body = response.json() + assert body["error"] == "not_found" + assert body["message"] == "company techcorp.com not found" + assert body["details"] == {"uid": "techcorp.com"} + + def test_duplicate_error_shape(self, error_client): + response = error_client.get("/boom/duplicate") + assert response.status_code == 409 + assert response.json()["error"] == "duplicate_record" + + def test_app_error_status_codes_are_respected(self): + assert NotFoundError("x").status_code == 404 + assert DuplicateRecordError("x").status_code == 409 + assert AppError("x").status_code == 500 # base class default + + def test_unhandled_exception_returns_generic_500(self, error_client): + response = error_client.get("/boom/unhandled") + assert response.status_code == 500 + body = response.json() + assert body["error"] == "internal_error" + # message should NOT leak internal exception details to the client + assert "something broke" not in body["message"] \ No newline at end of file diff --git a/company-api/tests/test_health.py b/company-api/tests/test_health.py new file mode 100644 index 0000000..70cbfb9 --- /dev/null +++ b/company-api/tests/test_health.py @@ -0,0 +1,5 @@ +class TestHealth: + def test_health_returns_ok(self, client): + response = client.get("/") + assert response.status_code == 200 + assert response.json() == {"status": "ok"} \ No newline at end of file diff --git a/company-common/exceptions.py b/company-common/exceptions.py new file mode 100644 index 0000000..e7203cb --- /dev/null +++ b/company-common/exceptions.py @@ -0,0 +1,70 @@ +""" +Shared exception hierarchy (company-common). + +Repository, service, and crawler code raises these instead of generic +Exceptions or DB-specific errors. company-api catches AppError once, at +the top, and turns it into a consistent JSON error response - no route +handler needs its own try/except for expected failure cases. + +Keep this file free of framework imports (no FastAPI, no duckdb) - it has +to be importable by every package, including ones that never touch HTTP. +""" + +from typing import Any, Optional + + +class AppError(Exception): + """Base class for all expected application errors. + + status_code: the HTTP status the API layer should respond with. + error_code: a stable, machine-readable string for API consumers to + branch on - don't rely on the human-readable message for that. + """ + + status_code: int = 500 + error_code: str = "internal_error" + + def __init__(self, message: str, *, details: Optional[dict[str, Any]] = None): + super().__init__(message) + self.message = message + self.details = details or {} + + +class NotFoundError(AppError): + """Requested resource doesn't exist, e.g. get(uid) found nothing.""" + + status_code = 404 + error_code = "not_found" + + +class DuplicateRecordError(AppError): + """Attempted to create a record that already exists under a unique key.""" + + status_code = 409 + error_code = "duplicate_record" + + +class ValidationFailedError(AppError): + """Input failed domain-level validation beyond what Pydantic checks + (e.g. business rules), as opposed to malformed request payloads - + FastAPI/Pydantic already handle those with their own 422s.""" + + status_code = 422 + error_code = "validation_failed" + + +class DatabaseError(AppError): + """Underlying storage layer failed (connection issue, constraint + violation that isn't a simple duplicate, etc).""" + + status_code = 503 + error_code = "database_error" + + +class UpstreamSourceError(AppError): + """A data source (crawler target, third-party API) failed or returned + something unusable. Distinct from DatabaseError since the fix and the + retry strategy are different.""" + + status_code = 502 + error_code = "upstream_source_error" \ No newline at end of file diff --git a/company-common/models/company_data.py b/company-common/models/company_data.py index efea89b..c8ab9e9 100644 --- a/company-common/models/company_data.py +++ b/company-common/models/company_data.py @@ -1,26 +1,25 @@ -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator from typing import List, Optional from enum import Enum -from datetime import datetime +from datetime import datetime, timezone + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def make_uid(company_domain: str) -> str: + """Stable, idempotent id derived from domain - re-upserting the same + company (e.g. after a re-scrape) naturally overwrites rather than dupes.""" + return company_domain.strip().lower() class CompanyFundingInfo(BaseModel): funding_round: str = Field(..., description="The funding round of the company") funding_amount: float = Field(..., description="The funding amount of the company") funding_currency: str = Field(..., description="The currency of the funding amount") - funding_date: Optional[str] = Field( - default=None, description="The date of the funding round" - ) - investors: List[str] = Field( - default_factory=list, description="List of investors in the funding round" - ) - - -class CompanyFinancials(BaseModel): - revenue: float = Field(..., description="The revenue of the company") - currency: str = Field(..., description="The currency of the revenue") - financial_year: int = Field(..., description="The financial year of the revenue") - ebitda: float = Field(..., description="The EBITDA of the company") + funding_date: Optional[str] = Field(default=None, description="The date of the funding round") + investors: List[str] = Field(default_factory=list, description="List of investors in the funding round") class CompanyOperatingMetrics(BaseModel): @@ -41,19 +40,13 @@ class CompanyStatus(Enum): class CurrentCompanyStatus(BaseModel): status: CompanyStatus = Field(..., description="The status of the company") - last_updated: Optional[datetime] = Field( - default=None, description="The last updated date of the company status" - ) + last_updated: Optional[datetime] = Field(default=None, description="The last updated date of the company status") class CompanyEmployeeCount(BaseModel): total_employees: int = Field(..., description="The total number of employees") - month: Optional[int] = Field( - default=None, description="The month of the employee count" - ) - year: Optional[int] = Field( - default=None, description="The year of the employee count" - ) + month: Optional[int] = Field(default=None, description="The month of the employee count") + year: Optional[int] = Field(default=None, description="The year of the employee count") class CompanyLocation(BaseModel): @@ -64,60 +57,39 @@ class CompanyLocation(BaseModel): class CompanyData(BaseModel): - company_name: str = Field(..., description="The name of the company") - company_website: Optional[str] = Field( - default=None, description="The website URL of the company" + uid: Optional[str] = Field( + default=None, description="Stable id, auto-derived from company_domain if not set" ) + source: str = Field(..., description="Where this record came from, e.g. 'yahoo_finance', 'edgar'") + created_at: datetime = Field(default_factory=_now) + updated_at: datetime = Field(default_factory=_now) + + company_name: str = Field(..., description="The name of the company") + company_website: Optional[str] = Field(default=None, description="The website URL of the company") company_domain: str = Field(..., description="The domain of the company") - company_industries: List[str] = Field( - default_factory=list, description="The industries of the company" - ) + company_industries: List[str] = Field(default_factory=list, description="The industries of the company") company_size: str = Field(..., description="The size of the company") - company_founded_year: int = Field( - ..., description="The year the company was founded" - ) - company_website_url: Optional[str] = Field( - default=None, description="The website URL of the company" - ) - company_funding_info: List[CompanyFundingInfo] = Field( - default_factory=list, description="The funding information of the company" - ) - company_logo_url: Optional[str] = Field( - default=None, description="The logo URL of the company" - ) + company_founded_year: int = Field(..., description="The year the company was founded") + company_website_url: Optional[str] = Field(default=None, description="The website URL of the company") + company_funding_info: List[CompanyFundingInfo] = Field(default_factory=list, description="The funding information of the company") + company_logo_url: Optional[str] = Field(default=None, description="The logo URL of the company") company_status: CurrentCompanyStatus = Field( - default_factory=lambda: CurrentCompanyStatus( - status=CompanyStatus.ACTIVE, last_updated=None - ), + default_factory=lambda: CurrentCompanyStatus(status=CompanyStatus.ACTIVE, last_updated=None), description="The status of the company", ) - company_description: Optional[str] = Field( - default="", description="The description of the company" - ) - company_headquarters: Optional[str] = Field( - default=None, description="The headquarters location of the company" - ) - company_ceo: Optional[str] = Field( - default=None, description="The CEO of the company" - ) - company_type: Optional[str] = Field( - default=None, description="The type of the company (e.g., private, public)" - ) - company_linkedin_url: Optional[str] = Field( - default=None, description="The LinkedIn URL of the company" - ) - company_twitter_url: Optional[str] = Field( - default=None, description="The Twitter URL of the company" - ) - company_symbol: Optional[str] = Field( - default=None, description="The stock symbol of the company" - ) - company_operating_metrics: Optional[CompanyOperatingMetrics] = Field( - default=None, description="The operating metrics of the company" - ) - company_employee_counts: List[CompanyEmployeeCount] = Field( - default_factory=list, description="The employee counts of the company" - ) - company_locations: List[CompanyLocation] = Field( - default_factory=list, description="The location information of the company" - ) + company_description: Optional[str] = Field(default="", description="The description of the company") + company_headquarters: Optional[str] = Field(default=None, description="The headquarters location of the company") + company_ceo: Optional[str] = Field(default=None, description="The CEO of the company") + company_type: Optional[str] = Field(default=None, description="The type of the company (e.g., private, public)") + company_linkedin_url: Optional[str] = Field(default=None, description="The LinkedIn URL of the company") + company_twitter_url: Optional[str] = Field(default=None, description="The Twitter URL of the company") + company_symbol: Optional[str] = Field(default=None, description="The stock symbol of the company") + company_operating_metrics: Optional[CompanyOperatingMetrics] = Field(default=None, description="The operating metrics of the company") + company_employee_counts: List[CompanyEmployeeCount] = Field(default_factory=list, description="The employee counts of the company") + company_locations: List[CompanyLocation] = Field(default_factory=list, description="The location information of the company") + + @model_validator(mode="after") + def _fill_uid(self): + if not self.uid: + self.uid = make_uid(self.company_domain) + return self \ No newline at end of file