From 8a396bad7bf4a9c24cf53430b93c38f9935b32fa Mon Sep 17 00:00:00 2001 From: CrewCircle Date: Wed, 29 Jul 2026 16:21:57 +1000 Subject: [PATCH] fix: accuracy scorer now checks the section field, not just citation label Found via q09 of the production-path benchmark: retrieval hit exactly the right section (heading breadcrumb "...Section 25-35 (Bad debts)" in the citation dict's `section` field, an exact match for the expected "ITAA 1997 s.25-35"), but the citation LABEL stayed a bare "ITAA 1997" - score_answer only ever read `citation`, never the richer `section` field hierarchical chunking already populates. Co-Authored-By: Claude Sonnet 5 --- .../tests/accuracy/test_research_accuracy.py | 20 +++++++++++-- apps/backend/tests/test_accuracy_scoring.py | 28 +++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/apps/backend/tests/accuracy/test_research_accuracy.py b/apps/backend/tests/accuracy/test_research_accuracy.py index f7876a0..8d2c032 100644 --- a/apps/backend/tests/accuracy/test_research_accuracy.py +++ b/apps/backend/tests/accuracy/test_research_accuracy.py @@ -40,6 +40,16 @@ def score_answer(question: dict, result: dict) -> dict: """ answer = result.get("answer", "").lower() citations = [c.get("citation", "").lower() for c in result.get("citations", [])] + # Hierarchical chunking (RAG-quality audit Fix 1) carries the retrieved + # section's own heading breadcrumb in `section` (e.g. "... > Section 25-35 + # (Bad debts)") - the `citation` field alone often stays a bare Act name + # ("ITAA 1997"). A citation dict can be exactly right at the section level + # while `citation` says nothing more specific than the Act - found via q09 + # of the production-path benchmark: real citations = ["ITAA 1997"] only, + # but its `section` field was "...Section 25-35 (Bad debts)", an EXACT + # match for the expected "ITAA 1997 s.25-35" that the citation-only check + # could never see. + sections = [(c.get("section") or "").lower() for c in result.get("citations", [])] expected_topics = [t.lower() for t in question.get("expected_topics", [])] topics_covered = sum(1 for t in expected_topics if t in answer) @@ -54,12 +64,18 @@ def _cited(ec: str) -> bool: # 820 of ITAA 1997" won't match the literal "gst act s.9-80" / # "itaa 1997 div 820" strings above - accept a bare match on just the # section/division token too (e.g. "9-80", "820") so natural phrasing - # isn't scored as a miss when the actual citation is right. + # isn't scored as a miss when the actual citation is right. Checked + # against the answer text, the citation label, AND the section + # breadcrumb (see `sections` above). for pattern in (_SECTION_TOKEN_RE, _DIVISION_TOKEN_RE): m = pattern.search(ec) if m: token = m.group(1) - if len(token) >= 2 and (token in answer or any(token in c for c in citations)): + if len(token) >= 2 and ( + token in answer + or any(token in c for c in citations) + or any(token in s for s in sections) + ): return True return False diff --git a/apps/backend/tests/test_accuracy_scoring.py b/apps/backend/tests/test_accuracy_scoring.py index 376a1da..8d0e918 100644 --- a/apps/backend/tests/test_accuracy_scoring.py +++ b/apps/backend/tests/test_accuracy_scoring.py @@ -82,6 +82,34 @@ def test_wrong_division_does_not_score_credit(): assert s["cit_ratio"] == 0.0 +def test_section_field_credited_even_when_citation_label_is_bare(): + """Hierarchical chunking carries the retrieved section's own heading + breadcrumb in the citation dict's `section` field (e.g. "... > Section + 25-35 (Bad debts)"), while `citation` often stays a bare Act name. Found + via q09 of the production-path benchmark: real citations = ["ITAA 1997"] + only, but its `section` field was an exact match for the expected + "ITAA 1997 s.25-35" - the citation-only check could never see it.""" + result = { + "answer": "A company can deduct a bad debt written off in the income year.", + "citations": [ + {"citation": "ITAA 1997", "section": "Division 25 > Section 25-35 (Bad debts)"} + ], + } + s = score_answer(_question(["bad debt"], ["ITAA 1997 s.25-35"]), result) + assert s["cit_ratio"] == 1.0 + + +def test_section_field_wrong_section_still_not_credited(): + result = { + "answer": "See the relevant division for details.", + "citations": [ + {"citation": "ITAA 1997", "section": "Division 165 > Section 165-120 (To deduct a bad debt)"} + ], + } + s = score_answer(_question([], ["ITAA 1997 s.25-35"]), result) + assert s["cit_ratio"] == 0.0 + + def test_citation_without_section_marker_unaffected(): """Expected citations with no 's.N' section marker (e.g. a ruling number) are untouched by the loosening - only the original literal