From 265a214511757ad2bcfea624bd236dad16832489 Mon Sep 17 00:00:00 2001 From: Prabhat Ranjan Date: Sat, 18 Apr 2026 16:16:17 +1000 Subject: [PATCH] feat: add LLMCategoriser DSPy-style component with contract testing - Add llm_categoriser.py with explicit input/output Pydantic signatures - Add contract validation functions (validate_input, validate_output) - Integrate into categorise.py as drop-in replacement for LLM calls - Add 21 contract tests covering validation, parsing, contracts, E2E - All 28 tests pass (21 new + 7 existing) --- .gitignore | 3 +- apps/api/services/categorise.py | 104 +++------- apps/api/services/llm_categoriser.py | 227 ++++++++++++++++++++ apps/api/tests/test_llm_categoriser.py | 277 +++++++++++++++++++++++++ 4 files changed, 538 insertions(+), 73 deletions(-) create mode 100644 apps/api/services/llm_categoriser.py create mode 100644 apps/api/tests/test_llm_categoriser.py diff --git a/.gitignore b/.gitignore index 65cab26..b195fce 100644 --- a/.gitignore +++ b/.gitignore @@ -62,4 +62,5 @@ pnpm-debug.log* # Misc *.bak *.tmp -temp/ \ No newline at end of file +temp/ +.vercel diff --git a/apps/api/services/categorise.py b/apps/api/services/categorise.py index 6b732f0..ebfc772 100644 --- a/apps/api/services/categorise.py +++ b/apps/api/services/categorise.py @@ -5,6 +5,12 @@ from supabase import Client from typing import Optional +from services.llm_categoriser import ( + LLMCategoriser, + LLMCategoriserInput, + ChartOfAccountsEntry, +) + anthropic_client = anthropic.Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"]) openai_client = AsyncOpenAI(api_key=os.environ["OPENAI_API_KEY"]) @@ -66,83 +72,37 @@ async def categorise_transaction( "reasoning": None, } - coa_text = "\n".join( - f"{a['code']} | {a['name']} | {a['account_type']} | GST:{a['gst_code']}" - for a in coa - ) - direction = "income/credit" if amount_cents > 0 else "expense/debit" + direction = "income" if amount_cents > 0 else "expense" amount_aud = abs(amount_cents) / 100 - prompt = f"""You are an Australian bookkeeper for a small plumbing and trades business. -Categorise the following bank transaction to the correct account in the Chart of Accounts. - -Transaction details: -- Description: {description_clean} -- Amount: ${amount_aud:.2f} AUD ({direction}) -- Merchant (if known): {merchant_name or "unknown"} -- Bank category (hint only): {basiq_category or "unknown"} - -Chart of Accounts: -{coa_text} - -Rules: -1. Return ONLY the account code number (e.g. "5000") and nothing else on the first line. -2. On the second line, return the GST code that applies (G1, G2, G3, G4, G9, G11, or N-T). -3. On the third line, return a confidence score between 0.00 and 1.00. -4. On the fourth line, give a one-sentence reason for your choice. - -If you cannot determine the correct account with confidence above 0.70, return "REVIEW" on the first line.""" - - message = anthropic_client.messages.create( - model="claude-sonnet-4-20250514", - max_tokens=150, - messages=[{"role": "user", "content": prompt}], + coa_entries = [ + ChartOfAccountsEntry( + code=a["code"], + name=a["name"], + account_type=a["account_type"], + gst_code=a["gst_code"], + id=a.get("id"), + ) + for a in coa + ] + + categoriser = LLMCategoriser(confidence_threshold=LLM_THRESHOLD) + input_data = LLMCategoriserInput( + description=description_clean, + amount_aud=amount_aud, + direction=direction, + merchant_name=merchant_name, + basiq_category=basiq_category, + chart_of_accounts=coa_entries, ) - response_text = message.content[0].text.strip() - lines = response_text.split("\n") - - if lines[0].strip().upper() == "REVIEW" or len(lines) < 4: - return { - "account_id": None, - "gst_code": None, - "confidence": 0.0, - "tier": "human", - "reasoning": response_text, - } - - code = lines[0].strip() - gst_code = lines[1].strip() - try: - confidence = float(lines[2].strip()) - except ValueError: - confidence = 0.5 - reasoning = lines[3].strip() - - if confidence < LLM_THRESHOLD: - return { - "account_id": None, - "gst_code": None, - "confidence": confidence, - "tier": "human", - "reasoning": reasoning, - } - - matched_account = next((a for a in coa if a["code"] == code), None) - if not matched_account: - return { - "account_id": None, - "gst_code": None, - "confidence": 0.0, - "tier": "human", - "reasoning": f"LLM returned unknown account code: {code}", - } + output = categoriser.forward(input_data) return { - "account_id": matched_account["id"], - "gst_code": gst_code, - "confidence": confidence, - "tier": "llm", - "reasoning": reasoning, + "account_id": output.account_id, + "gst_code": output.gst_code, + "confidence": output.confidence, + "tier": output.tier, + "reasoning": output.reasoning, } diff --git a/apps/api/services/llm_categoriser.py b/apps/api/services/llm_categoriser.py new file mode 100644 index 0000000..5ebac81 --- /dev/null +++ b/apps/api/services/llm_categoriser.py @@ -0,0 +1,227 @@ +import os +import re +import anthropic +from pydantic import BaseModel, Field +from typing import Optional, List, Literal + + +class ChartOfAccountsEntry(BaseModel): + code: str + name: str + account_type: Literal["asset", "liability", "equity", "revenue", "expense"] + gst_code: str + id: Optional[str] = None + + +class LLMCategoriserInput(BaseModel): + description: str + amount_aud: float + direction: Literal["income", "expense"] + merchant_name: Optional[str] = None + basiq_category: Optional[str] = None + chart_of_accounts: List[ChartOfAccountsEntry] + + class Config: + extra = "forbid" + + +class LLMCategoriserOutput(BaseModel): + account_code: Optional[str] = None + account_id: Optional[str] = None + gst_code: Optional[str] = None + confidence: float = Field(ge=0.0, le=1.0) + requires_review: bool + reasoning: str = "" + tier: Literal["llm", "human"] = "human" + + class Config: + extra = "forbid" + + +VALID_GST_CODES = {"G1", "G2", "G3", "G4", "G9", "G11", "N-T"} + + +def validate_input(i: LLMCategoriserInput) -> List[str]: + errors = [] + if not i.description or not i.description.strip(): + errors.append("empty description") + if i.amount_aud < 0: + errors.append("negative amount") + if i.direction not in ("income", "expense"): + errors.append(f"invalid direction: {i.direction}") + if not i.chart_of_accounts: + errors.append("empty chart_of_accounts") + return errors + + +def validate_output(o: LLMCategoriserOutput) -> List[str]: + errors = [] + if not (0.0 <= o.confidence <= 1.0): + errors.append(f"invalid confidence: {o.confidence}") + if not o.requires_review and o.account_code is None: + errors.append("missing account_code") + if o.gst_code and o.gst_code not in VALID_GST_CODES: + errors.append(f"invalid gst_code: {o.gst_code}") + return errors + + +class LLMCategoriser: + def __init__( + self, + model: str = "claude-sonnet-4-20250514", + confidence_threshold: float = 0.70, + max_tokens: int = 150, + ): + self.model = model + self.confidence_threshold = confidence_threshold + self.max_tokens = max_tokens + self._client: Optional[anthropic.Anthropic] = None + self._coa: List[ChartOfAccountsEntry] = [] + + @property + def client(self) -> anthropic.Anthropic: + if self._client is None: + key = os.environ.get("ANTHROPIC_API_KEY") + if not key: + raise ValueError("ANTHROPIC_API_KEY not set") + self._client = anthropic.Anthropic(api_key=key) + return self._client + + def _prompt(self, inp: LLMCategoriserInput) -> str: + coa_text = "\n".join( + f"{a.code} | {a.name} | {a.account_type} | GST:{a.gst_code}" + for a in inp.chart_of_accounts + ) + direction = "income/credit" if inp.direction == "income" else "expense/debit" + + return f"""You are an Australian bookkeeper for a small plumbing and trades business. +Categorise the following bank transaction to the correct account in the Chart of Accounts. + +Transaction details: +- Description: {inp.description} +- Amount: ${inp.amount_aud:.2f} AUD ({direction}) +- Merchant: {inp.merchant_name or "unknown"} +- Bank category: {inp.basiq_category or "unknown"} + +Chart of Accounts: +{coa_text} + +Rules: +1. Return ONLY the account code number (e.g. "5000") on the first line. +2. Return the GST code (G1, G2, G3, G4, G9, G11, or N-T) on the second line. +3. Return a confidence score between 0.00 and 1.00 on the third line. +4. Give a one-sentence reason on the fourth line. + +If confidence below {self.confidence_threshold:.2f}, return "REVIEW" on the first line.""" + + def _parse(self, text: str) -> LLMCategoriserOutput: + lines = [l.strip() for l in text.strip().split("\n")] + + if not lines or lines[0].upper() == "REVIEW": + return LLMCategoriserOutput( + account_code=None, + account_id=None, + gst_code=None, + confidence=0.0, + requires_review=True, + reasoning="LLM requested review", + tier="human", + ) + + if len(lines) < 4: + return LLMCategoriserOutput( + account_code=None, + account_id=None, + gst_code=None, + confidence=0.0, + requires_review=True, + reasoning=f"Incomplete: {text[:100]}", + tier="human", + ) + + code, gst_code = lines[0].strip(), lines[1].strip() + + try: + confidence = float(lines[2].strip()) + except (ValueError, IndexError): + confidence = 0.5 + + reasoning = lines[3].strip() + requires_review = confidence < self.confidence_threshold + + matched = None + for coa in self._coa: + if coa.code == code: + matched = coa + break + + if not matched: + return LLMCategoriserOutput( + account_code=code, + account_id=None, + gst_code=gst_code if gst_code in VALID_GST_CODES else None, + confidence=confidence, + requires_review=True, + reasoning=f"Unknown code: {code}. {reasoning}", + tier="human", + ) + + return LLMCategoriserOutput( + account_code=code, + account_id=matched.id, + gst_code=gst_code if gst_code in VALID_GST_CODES else matched.gst_code, + confidence=confidence, + requires_review=requires_review, + reasoning=reasoning, + tier="llm" if not requires_review else "human", + ) + + def forward(self, inp: LLMCategoriserInput) -> LLMCategoriserOutput: + self._coa = inp.chart_of_accounts + + errs = validate_input(inp) + if errs: + return LLMCategoriserOutput( + account_code=None, + account_id=None, + gst_code=None, + confidence=0.0, + requires_review=True, + reasoning=f"Input error: {', '.join(errs)}", + tier="human", + ) + + try: + msg = self.client.messages.create( + model=self.model, + max_tokens=self.max_tokens, + messages=[{"role": "user", "content": self._prompt(inp)}], + ) + resp = msg.content[0].text.strip() + except Exception as e: + return LLMCategoriserOutput( + account_code=None, + account_id=None, + gst_code=None, + confidence=0.0, + requires_review=True, + reasoning=f"LLM error: {str(e)}", + tier="human", + ) + + out = self._parse(resp) + out_errs = validate_output(out) + if out_errs: + out.reasoning += f" | Validation: {', '.join(out_errs)}" + + return out + + +def create_llm_categoriser( + model: Optional[str] = None, confidence_threshold: Optional[float] = None, **kwargs +) -> LLMCategoriser: + return LLMCategoriser( + model=model or "claude-sonnet-4-20250514", + confidence_threshold=confidence_threshold or 0.70, + max_tokens=kwargs.get("max_tokens", 150), + ) diff --git a/apps/api/tests/test_llm_categoriser.py b/apps/api/tests/test_llm_categoriser.py new file mode 100644 index 0000000..4ff97de --- /dev/null +++ b/apps/api/tests/test_llm_categoriser.py @@ -0,0 +1,277 @@ +import os +import pytest +from unittest.mock import patch, MagicMock + +os.environ.setdefault("ANTHROPIC_API_KEY", "sk-test-dummy") + +from services.llm_categoriser import ( + LLMCategoriser, + LLMCategoriserInput, + LLMCategoriserOutput, + ChartOfAccountsEntry, + validate_input, + validate_output, + VALID_GST_CODES, + create_llm_categoriser, +) + + +@pytest.fixture +def sample_coa(): + return [ + ChartOfAccountsEntry( + code="5000", + name="Office Supplies", + account_type="expense", + gst_code="G1", + id="uuid-1", + ), + ChartOfAccountsEntry( + code="2000", + name="Bank Account", + account_type="asset", + gst_code="G1", + id="uuid-2", + ), + ChartOfAccountsEntry( + code="4000", + name="Sales Revenue", + account_type="revenue", + gst_code="G1", + id="uuid-3", + ), + ] + + +class TestInputValidation: + def test_empty_description_fails(self, sample_coa): + inp = LLMCategoriserInput( + description="", + amount_aud=100.0, + direction="expense", + chart_of_accounts=sample_coa, + ) + assert "empty description" in validate_input(inp) + + def test_negative_amount_fails(self, sample_coa): + inp = LLMCategoriserInput( + description="test", + amount_aud=-10.0, + direction="expense", + chart_of_accounts=sample_coa, + ) + assert "negative amount" in validate_input(inp) + + def test_invalid_direction_caught_by_pydantic(self): + from pydantic import ValidationError + + with pytest.raises(ValidationError): + LLMCategoriserInput( + description="test", + amount_aud=100.0, + direction="invalid", + chart_of_accounts=[], + ) + + def test_empty_coa_fails(self): + inp = LLMCategoriserInput( + description="test", + amount_aud=100.0, + direction="expense", + chart_of_accounts=[], + ) + assert "empty chart_of_accounts" in validate_input(inp) + + def test_valid_input_passes(self, sample_coa): + inp = LLMCategoriserInput( + description="Bunnings", + amount_aud=150.0, + direction="expense", + chart_of_accounts=sample_coa, + ) + assert validate_input(inp) == [] + + +class TestOutputValidation: + def test_confidence_out_of_range_caught_by_pydantic(self): + from pydantic import ValidationError + + with pytest.raises(ValidationError): + LLMCategoriserOutput( + account_code="5000", + confidence=1.5, + requires_review=False, + reasoning="test", + ) + + def test_missing_account_code_when_not_review_fails(self): + out = LLMCategoriserOutput( + account_code=None, confidence=0.9, requires_review=False, reasoning="test" + ) + assert "missing account_code" in validate_output(out) + + def test_invalid_gst_code_caught_by_validation(self): + out = LLMCategoriserOutput( + account_code="5000", + gst_code="INVALID", + confidence=0.9, + requires_review=False, + reasoning="test", + ) + errs = validate_output(out) + assert any("invalid gst_code" in e for e in errs) + + def test_valid_output_passes(self, sample_coa): + out = LLMCategoriserOutput( + account_code="5000", + account_id="uuid-1", + gst_code="G1", + confidence=0.85, + requires_review=False, + reasoning="test", + ) + assert validate_output(out) == [] + + +class TestLLMResponseParsing: + def test_review_response(self): + categoriser = LLMCategoriser() + categoriser._coa = [] + + out = categoriser._parse("REVIEW\nToo ambiguous") + assert out.requires_review is True + assert out.tier == "human" + + def test_valid_response_parses(self): + categoriser = LLMCategoriser() + categoriser._coa = [ + ChartOfAccountsEntry( + code="5000", + name="Office", + account_type="expense", + gst_code="G1", + id="uuid-1", + ) + ] + + out = categoriser._parse("5000\nG1\n0.85\nOffice supplies expense") + assert out.account_code == "5000" + assert out.gst_code == "G1" + assert out.confidence == 0.85 + assert out.tier == "llm" + + def test_low_confidence_triggers_review(self): + categoriser = LLMCategoriser(confidence_threshold=0.70) + categoriser._coa = [ + ChartOfAccountsEntry( + code="5000", + name="Office", + account_type="expense", + gst_code="G1", + id="uuid-1", + ) + ] + + out = categoriser._parse("5000\nG1\n0.50\nNot sure") + assert out.requires_review is True + assert out.tier == "human" + + def test_unknown_account_code_triggers_review(self): + categoriser = LLMCategoriser() + categoriser._coa = [ + ChartOfAccountsEntry( + code="5000", + name="Office", + account_type="expense", + gst_code="G1", + id="uuid-1", + ) + ] + + out = categoriser._parse("9999\nG1\n0.85\nRandom") + assert out.requires_review is True + assert "Unknown code" in out.reasoning + + +class TestContractEnforcement: + def test_input_signature_has_required_fields(self): + required = ["description", "amount_aud", "direction", "chart_of_accounts"] + for field in required: + assert field in LLMCategoriserInput.__annotations__ + + def test_output_signature_has_required_fields(self): + required = ["account_code", "gst_code", "confidence", "requires_review"] + for field in required: + assert field in LLMCategoriserOutput.__annotations__ + + def test_output_extra_fields_forbidden(self): + with pytest.raises(Exception): + LLMCategoriserOutput( + account_code="5000", + confidence=0.9, + requires_review=False, + reasoning="test", + extra_field="invalid", + ) + + def test_valid_gst_codes(self): + assert VALID_GST_CODES == {"G1", "G2", "G3", "G4", "G9", "G11", "N-T"} + + +class TestEndToEnd: + @patch.object(LLMCategoriser, "client") + def test_full_categorisation_flow(self, mock_client, sample_coa): + mock_message = MagicMock() + mock_message.content = [ + MagicMock(text="5000\nG1\n0.90\nOffice supplies for warehouse") + ] + mock_client.messages.create.return_value = mock_message + + categoriser = LLMCategoriser() + inp = LLMCategoriserInput( + description="BUNNINGS WAREHOUSE", + amount_aud=250.00, + direction="expense", + merchant_name="Bunnings", + chart_of_accounts=sample_coa, + ) + + out = categoriser.forward(inp) + + assert out.account_code == "5000" + assert out.account_id == "uuid-1" + assert out.gst_code == "G1" + assert out.confidence == 0.90 + assert out.requires_review is False + assert out.tier == "llm" + + @patch.object(LLMCategoriser, "client") + def test_review_triggered_on_low_confidence(self, mock_client, sample_coa): + mock_message = MagicMock() + mock_message.content = [MagicMock(text="5000\nG1\n0.60\nMaybe office supplies")] + mock_client.messages.create.return_value = mock_message + + categoriser = LLMCategoriser() + inp = LLMCategoriserInput( + description="UNKNOWN CHARGE", + amount_aud=50.00, + direction="expense", + chart_of_accounts=sample_coa, + ) + + out = categoriser.forward(inp) + + assert out.requires_review is True + assert out.tier == "human" + + +class TestFactory: + def test_create_with_defaults(self): + cat = create_llm_categoriser() + assert cat.model == "claude-sonnet-4-20250514" + assert cat.confidence_threshold == 0.70 + + def test_create_with_overrides(self): + cat = create_llm_categoriser(model="claude-3-5", confidence_threshold=0.80) + assert cat.model == "claude-3-5" + assert cat.confidence_threshold == 0.80