diff --git a/Makefile b/Makefile index 0fa6129..376409f 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ MODEL_EVAL_FLAGS ?= --ollama-model gpt-oss-safeguard:20b ADLINT_OLLAMA_TIMEOUT ?= 180 ADLINT_OLLAMA_NUM_PREDICT ?= 1024 -.PHONY: api creative-assets-eval demo dev landing-page-consistency eval benchmark benchmark-data policy-coverage policy-coverage-validate rewrite-quality model-benchmark model-smoke model-usefulness pr-preflight real-cases real-cases-ci real-cases-hybrid real-cases-model-quality real-cases-validate real-world-blind-candidates real-world-blind-ci real-world-blind-validate real-world-blind real-world-blind-model-quality research-summary scan test install +.PHONY: api creative-assets-eval demo dev false-positive-near-miss landing-page-consistency eval benchmark benchmark-data policy-coverage policy-coverage-validate rewrite-quality model-benchmark model-smoke model-usefulness pr-preflight real-cases real-cases-ci real-cases-hybrid real-cases-model-quality real-cases-validate real-world-blind-candidates real-world-blind-ci real-world-blind-validate real-world-blind real-world-blind-model-quality research-summary scan test install install: $(STAMP) @@ -36,6 +36,9 @@ landing-page-consistency: $(STAMP) creative-assets-eval: $(STAMP) $(BIN)/python evals/run_eval.py evals/datasets/creative_asset_metadata_v1.jsonl --min-decision-accuracy 1.0 --output evals/results/creative_asset_metadata_v1.json --markdown-output evals/results/creative_asset_metadata_v1.md +false-positive-near-miss: $(STAMP) + $(BIN)/python evals/run_eval.py evals/datasets/false_positive_near_miss_v1.jsonl --min-decision-accuracy 1.0 --max-policy-false-positives 0 --summary-only --output evals/results/false_positive_near_miss_v1.json --markdown-output evals/results/false_positive_near_miss_v1.md + research-summary: $(STAMP) $(BIN)/python evals/run_eval.py evals/datasets/seed_ads.jsonl --summary-only --summary-format json --min-decision-accuracy 0 $(BIN)/python evals/run_eval.py evals/datasets/rule_benchmark_v1.jsonl --summary-only --summary-format json --min-decision-accuracy 0 diff --git a/PRD.md b/PRD.md index bb2e71e..e63a354 100644 --- a/PRD.md +++ b/PRD.md @@ -50,10 +50,15 @@ The review target includes: - Reports: JSON and Markdown reports include policy hits, evidence, recommended actions, rewrites, landing-page details, and the decision-support disclaimer. +- Review packet export: local batch archives or single report directories can + be packaged into a manifest, reviewer summary, and copied Markdown reports + for local handoff without adding raw creative or raw landing-page data to the + manifest. - Rewrites: deterministic safer rewrite suggestions are generated for high-risk or review-required hits where feasible. -- Seed evals: `evals/run_eval.py` scores the 50-example - `evals/datasets/seed_ads.jsonl` dataset. +- Seed and focused evals: `evals/run_eval.py` scores the seed dataset plus + focused landing-page, creative-asset, false-positive near-miss, real-case, + and blind-holdout datasets. - Review hardening: tests cover CLI scanning, API endpoints, policy loading and filtering, report generation, documented examples, the seed eval runner, and opt-in logging behavior. @@ -63,6 +68,9 @@ The review target includes: when explicitly enabled. - Opt-in SQLite metadata storage records scan and eval summaries only when explicitly enabled, without raw ad or landing-page fields. +- Creative asset metadata validation checks declared MIME type, dimensions, + duration, file size, and asset-type compatibility without reading or + analyzing raw media. ### 2.2 Partial or future work diff --git a/README.md b/README.md index d8f88a3..1123963 100644 --- a/README.md +++ b/README.md @@ -113,8 +113,9 @@ make api # then open http://127.0.0.1:8000/ui/ a small stdlib HTML parser that can read inline HTML, local files, or fetchable HTML URLs. - Native image/video analysis. Creative asset metadata can be attached to - reports and supplied text metadata can be checked by text rules, but AdLint - does not yet run OCR, frame analysis, speech-to-text, or visual policy checks. + reports, declared metadata can be validated, and supplied text metadata can + be checked by text rules, but AdLint does not yet run OCR, frame analysis, + speech-to-text, or visual policy checks. - Fine-tuning. Local model support is available for decision support, but deterministic rules are the production baseline until live evals prove incremental value. @@ -149,6 +150,7 @@ make benchmark # run the 213-row synthetic policy regression benchmark make policy-coverage # refresh docs/policy_coverage_matrix.md make policy-coverage-validate # check the committed coverage matrix make rewrite-quality # run the deterministic rewrite-quality rubric eval +make false-positive-near-miss # guard benign near-misses against overbroad policy hits make pr-preflight # verify generated eval assets before opening an eval PR make real-cases # run balanced public-source real-case diagnostics make real-cases-ci # run the strict real-case CI gate @@ -266,7 +268,10 @@ those fields so the existing text rules can inspect them: ``` Reports include sanitized asset metadata such as `filename`, dimensions, media -type, and booleans showing which text metadata fields were supplied. Batch +type, metadata-validation status, and booleans showing which text metadata +fields were supplied. Metadata validation checks only declared fields such as +MIME type, dimensions, duration, file size, and asset-type compatibility. It +does not read, upload, OCR, transcribe, or visually classify raw media. Batch summaries omit the `creative_assets` column by default; per-row reports remain local when `--output-dir` is used. @@ -300,6 +305,27 @@ With `--output-dir`, AdLint writes: The batch summary intentionally omits raw ad copy and landing-page HTML. Per-row reports stay local and contain the evidence snippets needed for review. +### Review packet export + +Use `adlint review-packet` to turn a local batch archive or single scan report +directory into a reviewer handoff packet: + +```bash +adlint batch examples/batch_campaigns.csv --output-dir reports/batch +adlint review-packet reports/batch --output-dir reports/review-packet +``` + +The packet includes: + +- `manifest.json` with decisions, risk scores, policy IDs, policy source links, + relative Markdown report paths, and generation metadata. +- `review-packet.md` with a compact reviewer summary. +- copied per-row Markdown reports under `reports/`. + +The packet manifest omits raw ad copy, landing-page URLs, raw landing-page HTML, +raw creative paths, and raw media. It is for local reviewer handoff only and +does not change AdLint's decision-support boundary. + ## Scoring configuration By default, AdLint uses the built-in MVP scoring weights and thresholds. To @@ -494,6 +520,18 @@ for clarity, risk reduction, policy fit, and intent preservation. It reports a Deterministic rewrites remain the baseline before any model-generated rewrites are introduced. +Run the false-positive near-miss guardrail before broadening policy signals: + +```bash +make false-positive-near-miss +``` + +`false_positive_near_miss_v1` covers benign health-adjacent copy, soft +professional claims, offer/discount language, creator disclosures, and +platform-specific review labels. The target requires 1.000 rule-only decision +accuracy and zero policy false positives, and the report includes a separate +false-positive pressure section with reviewer notes. + Run the public-source real-case diagnostics: ```bash diff --git a/adlint/cli.py b/adlint/cli.py index ca4564e..71024d7 100644 --- a/adlint/cli.py +++ b/adlint/cli.py @@ -8,6 +8,7 @@ from adlint.config import load_config from adlint.engine import analyze from adlint.reports import to_markdown +from adlint.review_packet import build_review_packet DEMO_CONFIG = { @@ -122,6 +123,17 @@ def main(argv: Sequence[str] | None = None) -> int: help="Format printed to stdout.", ) + packet_parser = subparsers.add_parser("review-packet", help="Build a local reviewer handoff packet.") + packet_parser.add_argument( + "source_dir", + help="Directory containing an AdLint batch archive or single scan reports.", + ) + packet_parser.add_argument( + "--output-dir", + required=True, + help="Write the review packet manifest, summary, and copied Markdown reports to this directory.", + ) + args = parser.parse_args(argv) if args.command == "scan": @@ -174,5 +186,10 @@ def main(argv: Sequence[str] | None = None) -> int: print(to_markdown(result)) return 0 + if args.command == "review-packet": + packet = build_review_packet(args.source_dir, args.output_dir) + print(json.dumps(packet, indent=2, sort_keys=True)) + return 0 + parser.error(f"Unknown command: {args.command}") return 2 diff --git a/adlint/models.py b/adlint/models.py index d0893bc..6acd801 100644 --- a/adlint/models.py +++ b/adlint/models.py @@ -119,6 +119,20 @@ def to_dict(self) -> dict[str, Any]: } +@dataclass(frozen=True) +class CreativeAssetValidationFinding: + field: str + severity: str + message: str + + def to_dict(self) -> dict[str, str]: + return { + "field": self.field, + "severity": self.severity, + "message": self.message, + } + + @dataclass(frozen=True) class CreativeAsset: asset_id: str @@ -133,24 +147,40 @@ class CreativeAsset: transcript_excerpt: str = "" alt_text: str = "" labels: tuple[str, ...] = () + validation_findings: tuple[CreativeAssetValidationFinding, ...] = () @classmethod def from_dict(cls, raw: dict[str, Any], *, index: int = 1) -> "CreativeAsset": filename = _basename(raw.get("filename") or raw.get("file_name") or raw.get("path") or raw.get("url")) asset_id = str(raw.get("asset_id") or raw.get("id") or filename or f"asset-{index}") + asset_type = _asset_type(raw.get("asset_type") or raw.get("kind") or raw.get("type") or raw.get("media_type")) + mime_type = _optional_str(raw.get("mime_type") or raw.get("content_type")) + width = _optional_int(raw.get("width")) + height = _optional_int(raw.get("height")) + duration_seconds = _optional_float(raw.get("duration_seconds") or raw.get("duration")) + file_size_bytes = _optional_int(raw.get("file_size_bytes") or raw.get("size_bytes")) return cls( asset_id=_safe_asset_id(asset_id, fallback=f"asset-{index}"), - asset_type=_asset_type(raw.get("asset_type") or raw.get("kind") or raw.get("type") or raw.get("media_type")), + asset_type=asset_type, filename=filename, - mime_type=_optional_str(raw.get("mime_type") or raw.get("content_type")), - width=_optional_int(raw.get("width")), - height=_optional_int(raw.get("height")), - duration_seconds=_optional_float(raw.get("duration_seconds") or raw.get("duration")), - file_size_bytes=_optional_int(raw.get("file_size_bytes") or raw.get("size_bytes")), + mime_type=mime_type, + width=width, + height=height, + duration_seconds=duration_seconds, + file_size_bytes=file_size_bytes, text_overlay=_optional_str(raw.get("text_overlay") or raw.get("ocr_text")) or "", transcript_excerpt=_optional_str(raw.get("transcript_excerpt") or raw.get("audio_transcript")) or "", alt_text=_optional_str(raw.get("alt_text") or raw.get("description")) or "", labels=tuple(str(item).strip() for item in _raw_list(raw.get("labels")) if str(item).strip()), + validation_findings=_validate_creative_asset_metadata( + raw, + asset_type=asset_type, + mime_type=mime_type, + width=width, + height=height, + duration_seconds=duration_seconds, + file_size_bytes=file_size_bytes, + ), ) def text_fields(self, *, index: int) -> dict[str, str]: @@ -187,6 +217,13 @@ def to_dict(self) -> dict[str, Any]: ): if value not in (None, ""): payload[key] = value + if self.validation_findings: + payload["metadata_validation"] = { + "status": "needs_review", + "findings": [finding.to_dict() for finding in self.validation_findings], + } + else: + payload["metadata_validation"] = {"status": "passed", "findings": []} return payload @@ -318,6 +355,79 @@ def _asset_type(value: Any) -> str: return "unknown" +def _validate_creative_asset_metadata( + raw: dict[str, Any], + *, + asset_type: str, + mime_type: str | None, + width: int | None, + height: int | None, + duration_seconds: float | None, + file_size_bytes: int | None, +) -> tuple[CreativeAssetValidationFinding, ...]: + findings: list[CreativeAssetValidationFinding] = [] + + for field, keys, parsed in ( + ("width", ("width",), width), + ("height", ("height",), height), + ("duration_seconds", ("duration_seconds", "duration"), duration_seconds), + ("file_size_bytes", ("file_size_bytes", "size_bytes"), file_size_bytes), + ): + if _has_any(raw, keys) and parsed is None: + findings.append(_asset_finding(field, "Declared value must be a non-negative number.")) + + for field, value in ( + ("width", width), + ("height", height), + ("duration_seconds", duration_seconds), + ("file_size_bytes", file_size_bytes), + ): + if value is not None and value <= 0: + findings.append(_asset_finding(field, "Declared value must be greater than zero.")) + + if (width is None) != (height is None): + findings.append(_asset_finding("dimensions", "Width and height should be supplied together.")) + + if mime_type: + findings.extend(_validate_mime_type(asset_type, mime_type)) + + if asset_type == "image" and duration_seconds is not None: + findings.append(_asset_finding("duration_seconds", "Image assets should not declare duration.")) + if asset_type == "audio" and (width is not None or height is not None): + findings.append(_asset_finding("dimensions", "Audio assets should not declare visual dimensions.")) + + return tuple(findings) + + +def _validate_mime_type(asset_type: str, mime_type: str) -> list[CreativeAssetValidationFinding]: + normalized = mime_type.strip().lower() + findings: list[CreativeAssetValidationFinding] = [] + if "/" not in normalized: + findings.append(_asset_finding("mime_type", "Declared MIME type should use type/subtype format.")) + return findings + + expected: dict[str, tuple[str, ...]] = { + "image": ("image/",), + "video": ("video/",), + "audio": ("audio/",), + "display": ("image/", "text/html", "application/zip", "application/x-zip-compressed"), + "html5": ("text/html", "application/zip", "application/x-zip-compressed"), + } + prefixes = expected.get(asset_type) + if prefixes and not any(normalized.startswith(prefix) for prefix in prefixes): + allowed = ", ".join(prefix.rstrip("/") for prefix in prefixes) + findings.append(_asset_finding("mime_type", f"{asset_type} assets should use one of: {allowed}.")) + return findings + + +def _asset_finding(field: str, message: str) -> CreativeAssetValidationFinding: + return CreativeAssetValidationFinding(field=field, severity="warning", message=message) + + +def _has_any(raw: dict[str, Any], keys: tuple[str, ...]) -> bool: + return any(key in raw and raw[key] not in (None, "") for key in keys) + + def _basename(value: Any) -> str | None: text = _optional_str(value) if not text: diff --git a/adlint/reports.py b/adlint/reports.py index b81930f..8f24720 100644 --- a/adlint/reports.py +++ b/adlint/reports.py @@ -106,6 +106,12 @@ def to_markdown(result: AnalysisResult) -> str: if asset.duration_seconds is not None: details.append(f"{asset.duration_seconds:g}s") lines.append(f"- Asset {index}: {' | '.join(details)}") + if asset.validation_findings: + lines.append(" - Metadata validation: needs review") + for finding in asset.validation_findings: + lines.append(f" - `{finding.field}`: {finding.message}") + else: + lines.append(" - Metadata validation: passed") text_metadata = asset.to_dict()["text_metadata"] if any(text_metadata.values()): lines.append(" - Text metadata supplied for local rule checks; text contents are not reprinted in this section.") diff --git a/adlint/review_packet.py b/adlint/review_packet.py new file mode 100644 index 0000000..efde90e --- /dev/null +++ b/adlint/review_packet.py @@ -0,0 +1,279 @@ +from __future__ import annotations + +import json +import shutil +from collections import Counter +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + + +PACKET_VERSION = 1 +PRIVATE_MANIFEST_FIELDS = ( + "body", + "cta", + "creative_assets", + "headline", + "landing_page_html", + "landing_page_url", + "raw_media", +) + + +def build_review_packet(source_dir: str | Path, output_dir: str | Path) -> dict[str, Any]: + source = Path(source_dir) + output = Path(output_dir) + if not source.exists() or not source.is_dir(): + raise ValueError(f"Review packet source must be an existing directory: {source}") + + output.mkdir(parents=True, exist_ok=True) + reports_dir = output / "reports" + reports_dir.mkdir(parents=True, exist_ok=True) + + if (source / "adlint-batch-summary.json").exists(): + manifest = _manifest_from_batch(source, output, reports_dir) + elif (source / "adlint-report.json").exists(): + manifest = _manifest_from_single_report(source, output, reports_dir) + else: + raise ValueError( + "Review packet source must contain adlint-batch-summary.json or adlint-report.json." + ) + + manifest_path = output / "manifest.json" + markdown_path = output / "review-packet.md" + manifest["packet"] |= { + "manifest": str(manifest_path.relative_to(output)), + "markdown": str(markdown_path.relative_to(output)), + } + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + markdown_path.write_text(packet_to_markdown(manifest), encoding="utf-8") + return { + "manifest": str(manifest_path), + "markdown": str(markdown_path), + "row_reports": [ + str(output / row["reports"]["markdown"]) + for row in manifest["rows"] + if row.get("reports", {}).get("markdown") + ], + } + + +def packet_to_markdown(manifest: dict[str, Any]) -> str: + summary = manifest["summary"] + lines = [ + "# AdLint Review Packet", + "", + f"- Packet version: `{manifest['packet']['version']}`", + f"- Source type: `{manifest['source']['type']}`", + f"- Total rows: `{summary['total_rows']}`", + f"- Approved: `{summary['decision_counts'].get('approved', 0)}`", + f"- Needs review: `{summary['decision_counts'].get('needs_review', 0)}`", + f"- High risk: `{summary['decision_counts'].get('high_risk', 0)}`", + "", + "## Rows", + "", + "| ID | Decision | Risk score | Review | Policy hits | Markdown report |", + "| --- | --- | ---: | --- | --- | --- |", + ] + for row in manifest["rows"]: + policy_ids = ", ".join(row["policy_ids"]) if row["policy_ids"] else "none" + report = row.get("reports", {}).get("markdown", "") + report_link = f"[{report}]({report})" if report else "" + lines.append( + "| {id} | `{decision}` | {risk_score:.2f} | `{requires_review}` | {policy_ids} | {report} |".format( + id=row["id"], + decision=row["decision"], + risk_score=row["risk_score"], + requires_review=str(row["requires_review"]).lower(), + policy_ids=policy_ids, + report=report_link, + ) + ) + + lines.extend( + [ + "", + "## Policy Sources", + "", + ] + ) + sources = manifest.get("policy_sources", []) + if not sources: + lines.append("- No policy sources in this packet.") + for source in sources: + url = source.get("url", "") + note = source.get("note") or url or "Policy source" + policy_ids = ", ".join(source.get("policy_ids", [])) + if url: + lines.append(f"- [{note}]({url}) - {policy_ids}") + else: + lines.append(f"- {note} - {policy_ids}") + + lines.extend( + [ + "", + "## Privacy Boundary", + "", + "- The packet manifest omits raw ad copy, raw landing-page HTML, landing-page URLs, raw creative paths, and raw media.", + "- Per-row Markdown reports are copied for local reviewer handoff and retain only the evidence snippets produced by AdLint.", + "- AdLint is decision-support software, not legal advice or a platform approval guarantee.", + "", + ] + ) + return "\n".join(lines) + + +def _manifest_from_batch(source: Path, output: Path, reports_dir: Path) -> dict[str, Any]: + summary_path = source / "adlint-batch-summary.json" + summary = json.loads(summary_path.read_text(encoding="utf-8")) + rows: list[dict[str, Any]] = [] + policy_sources_by_key: dict[tuple[str, str], dict[str, Any]] = {} + + for row in summary.get("rows", []): + row_id = str(row["id"]) + report_json = _resolve_report_path(source, row.get("reports", {}).get("json")) + report_markdown = _resolve_report_path(source, row.get("reports", {}).get("markdown")) + report_payload = _load_report_payload(report_json) + copied_markdown = _copy_markdown_report(report_markdown, reports_dir, row_id) + policy_ids = [str(policy_id) for policy_id in row.get("policy_ids", [])] + _collect_policy_sources(policy_sources_by_key, report_payload.get("policy_hits", [])) + rows.append( + { + "id": row_id, + "decision": row["decision"], + "risk_score": float(row["risk_score"]), + "requires_review": bool(row["requires_review"]), + "policy_ids": policy_ids, + "asset_count": int(row.get("asset_count", 0)), + "reports": {"markdown": str(copied_markdown.relative_to(output))}, + } + ) + + copied_summary = _copy_if_exists(source / "adlint-batch-summary.md", output / "adlint-batch-summary.md") + manifest = _base_manifest(source_type="batch_archive", row_count=len(rows)) + manifest["source"] |= { + "summary": summary_path.name, + "batch_summary_markdown": copied_summary.name if copied_summary else None, + } + manifest["summary"] = { + "total_rows": int(summary.get("total_rows", len(rows))), + "decision_counts": dict(sorted(summary.get("decision_counts", {}).items())), + "policy_counts": dict(sorted(summary.get("policy_counts", {}).items())), + } + manifest["rows"] = rows + manifest["policy_sources"] = _policy_source_list(policy_sources_by_key) + return manifest + + +def _manifest_from_single_report(source: Path, output: Path, reports_dir: Path) -> dict[str, Any]: + report_json = source / "adlint-report.json" + report_markdown = source / "adlint-report.md" + payload = json.loads(report_json.read_text(encoding="utf-8")) + row_id = "adlint-report" + copied_markdown = _copy_markdown_report(report_markdown, reports_dir, row_id) + policy_hits = payload.get("policy_hits", []) + policy_ids = [str(hit["policy_id"]) for hit in policy_hits] + policy_sources_by_key: dict[tuple[str, str], dict[str, Any]] = {} + _collect_policy_sources(policy_sources_by_key, policy_hits) + + manifest = _base_manifest(source_type="single_report", row_count=1) + manifest["source"] |= {"report": report_json.name} + manifest["summary"] = { + "total_rows": 1, + "decision_counts": dict(Counter([payload["decision"]])), + "policy_counts": dict(sorted(Counter(policy_ids).items())), + } + manifest["rows"] = [ + { + "id": row_id, + "decision": payload["decision"], + "risk_score": float(payload["risk_score"]), + "requires_review": bool(payload["requires_review"]), + "policy_ids": policy_ids, + "asset_count": len(payload.get("creative_assets", [])), + "reports": {"markdown": str(copied_markdown.relative_to(output))}, + } + ] + manifest["policy_sources"] = _policy_source_list(policy_sources_by_key) + return manifest + + +def _base_manifest(*, source_type: str, row_count: int) -> dict[str, Any]: + return { + "packet": { + "version": PACKET_VERSION, + "created_at": datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z"), + }, + "source": {"type": source_type}, + "summary": {"total_rows": row_count, "decision_counts": {}, "policy_counts": {}}, + "privacy": { + "raw_creative_included": False, + "manifest_omits_fields": list(PRIVATE_MANIFEST_FIELDS), + "raw_media_copied": False, + }, + "rows": [], + "policy_sources": [], + } + + +def _resolve_report_path(source: Path, raw_path: Any) -> Path | None: + if not raw_path: + return None + path = Path(str(raw_path)) + if path.is_absolute(): + return path + return source / path + + +def _load_report_payload(path: Path | None) -> dict[str, Any]: + if path is None or not path.exists(): + return {} + return json.loads(path.read_text(encoding="utf-8")) + + +def _copy_markdown_report(path: Path | None, reports_dir: Path, row_id: str) -> Path: + if path is None or not path.exists(): + raise ValueError(f"Missing Markdown report for review packet row: {row_id}") + destination = reports_dir / f"{_safe_filename(row_id)}.md" + shutil.copyfile(path, destination) + return destination + + +def _copy_if_exists(source: Path, destination: Path) -> Path | None: + if not source.exists(): + return None + shutil.copyfile(source, destination) + return destination + + +def _collect_policy_sources( + policy_sources_by_key: dict[tuple[str, str], dict[str, Any]], + policy_hits: list[dict[str, Any]], +) -> None: + for hit in policy_hits: + source = hit.get("policy_source") or {} + url = str(source.get("url", "")) + note = str(source.get("note", "")) + if not url and not note: + continue + key = (url, note) + entry = policy_sources_by_key.setdefault(key, {"url": url, "note": note, "policy_ids": set()}) + entry["policy_ids"].add(str(hit.get("policy_id", ""))) + + +def _policy_source_list(policy_sources_by_key: dict[tuple[str, str], dict[str, Any]]) -> list[dict[str, Any]]: + sources: list[dict[str, Any]] = [] + for _key, source in sorted(policy_sources_by_key.items(), key=lambda item: (item[0][1], item[0][0])): + sources.append( + { + "url": source["url"], + "note": source["note"], + "policy_ids": sorted(policy_id for policy_id in source["policy_ids"] if policy_id), + } + ) + return sources + + +def _safe_filename(value: str) -> str: + cleaned = "".join(char if char.isalnum() or char in "-._" else "-" for char in value.strip()) + return cleaned.strip("-._") or "row" diff --git a/docs/eval_report.md b/docs/eval_report.md index 4406139..08da09d 100644 --- a/docs/eval_report.md +++ b/docs/eval_report.md @@ -139,6 +139,21 @@ Live local-model quality runs remain manual or scheduled diagnostics. They are not required CI because local inference is slow and the current evidence still supports deterministic rules as the production baseline. +Run the false-positive near-miss guardrail before broadening policy signals: + +```bash +make false-positive-near-miss +``` + +`evals/datasets/false_positive_near_miss_v1.jsonl` is a small guardrail set +for benign near-misses and review-only labels: health-adjacent education, +soft professional claims, offer/discount consistency, creator sponsorship +disclosures, and platform-specific review boundaries. The target requires +1.000 rule-only decision accuracy and `--max-policy-false-positives 0`, so it +fails when a policy change adds overbroad hits even if the decision label still +passes. Its Markdown report includes a separate false-positive pressure +section with row-level reviewer notes. + Direct command used by the benchmark target: ```bash @@ -421,8 +436,11 @@ also show why model-only should not replace deterministic rules yet: the local model still undercalls rows and maps concerns to `model_policy_review` rather than the detailed YAML policy ids. -The current benchmark test now fails if the adjudicated rule benchmark or -real-case set produces policy false-positive or false-negative review notes. +The current benchmark tests fail if the adjudicated rule benchmark or real-case +set produces policy false-positive or false-negative review notes. The +near-miss target adds a narrower false-positive pressure gate for benign copy +and review-only labels that should not become high-risk or unrelated policy +hits. For a paper-style summary of the benchmark design and result interpretation, see `docs/research_paper.md`. For the compiled LaTeX paper with charts, diff --git a/evals/README.md b/evals/README.md index 023620f..990b1ff 100644 --- a/evals/README.md +++ b/evals/README.md @@ -14,6 +14,9 @@ platform approval. - `datasets/creative_asset_metadata_v1.jsonl`: focused rows for metadata-only creative asset inputs. These rows test supplied OCR/text-overlay metadata and empty asset metadata; they do not test native image/video analysis. +- `datasets/false_positive_near_miss_v1.jsonl`: focused benign near-miss and + review-only rows for false-positive discipline before broadening policy + signals. - `datasets/real_cases_v1.jsonl`: 75 public-source, paraphrased real-case diagnostics balanced across 25 `approved`, 25 `needs_review`, and 25 `high_risk` expected decisions. These rows come from public marketing pages, @@ -44,6 +47,7 @@ Run focused workflow evals: ```bash make landing-page-consistency make creative-assets-eval +make false-positive-near-miss ``` Validate or refresh policy coverage inventory: @@ -78,6 +82,20 @@ should become qualified wellness/nutrition copy, a health-form tracking row that should shift to consent and privacy review language, and a cure claim that should become general wellness support with professional-consult wording. +Run false-positive near-miss evaluation separately from broad accuracy gates: + +```bash +make false-positive-near-miss +``` + +`false_positive_near_miss_v1` rows are marked with +`false_positive_guardrail: true` and include `reviewer_notes` plus +`near_miss_policy_ids`. The eval runner reports a separate +`false_positive_pressure` section with guardrail row counts, decision +overcalls, rows with policy false positives, and row-level reviewer notes. The +Make target requires zero policy false positives with +`--max-policy-false-positives 0`. + Run compact research-loop summaries: ```bash @@ -87,8 +105,8 @@ make research-summary The target prints deterministic JSON summaries for the seed, benchmark, real-case, and blind-holdout datasets. These summaries include row counts, decision accuracy, decision mismatches, confusion deltas, policy false -positive/negative counts, top review-note row IDs, model status counts, and -elapsed seconds. +positive/negative counts, false-positive pressure counts when guardrail rows +are present, top review-note row IDs, model status counts, and elapsed seconds. The seed and benchmark datasets are complete-coverage gates for current policy ids. `real_cases_v1` is included in the inventory, but it remains diff --git a/evals/datasets/false_positive_near_miss_v1.jsonl b/evals/datasets/false_positive_near_miss_v1.jsonl new file mode 100644 index 0000000..7625e06 --- /dev/null +++ b/evals/datasets/false_positive_near_miss_v1.jsonl @@ -0,0 +1,8 @@ +{"id":"near_miss_health_habit_tracker_approved","false_positive_guardrail":true,"near_miss_policy_ids":["unsupported_health_claim","weight_loss_claim","google_health_restricted_category"],"reviewer_notes":"Benign wellness productivity copy should remain approved; it avoids treatment, weight-loss, supplement, and clinical-evidence claims.","input":{"platform":"google","industry":"wellness","headline":"Build a calmer daily wellness routine","body":"Use a habit tracker to plan hydration, sleep reminders, and movement breaks.","cta":"Start planning"},"expected_decision":"approved","expected_policy_ids":[]} +{"id":"near_miss_science_backed_sleep_review_only","false_positive_guardrail":true,"near_miss_policy_ids":["unsupported_health_claim","medical_cure_claim"],"reviewer_notes":"Evidence-framed wellness education should route to substantiation review without becoming high risk because it avoids disease treatment and guaranteed outcomes.","input":{"platform":"google","industry":"wellness","headline":"Science-backed sleep hygiene tips","body":"Join our newsletter for evidence-based routines that may support better rest. Results vary.","cta":"Sign up"},"expected_decision":"needs_review","expected_policy_ids":["wellness_claim_review"]} +{"id":"near_miss_linkedin_soft_output_review_only","false_positive_guardrail":true,"near_miss_policy_ids":["linkedin_professional_claim_review"],"reviewer_notes":"Soft professional-outcome language should remain review-only and not be treated like explicit salary, hiring, or promotion guarantees.","input":{"platform":"linkedin","industry":"saas","headline":"Improve team output with weekly planning","body":"Use shared launch checklists to make handoffs clearer across marketing and sales.","cta":"See workflow"},"expected_decision":"needs_review","expected_policy_ids":["linkedin_professional_claim_review"]} +{"id":"near_miss_meta_finance_education_approved","false_positive_guardrail":true,"near_miss_policy_ids":["meta_financial_services_authorization_review","meta_special_ad_category_review"],"reviewer_notes":"Financial education copy should not trigger Meta authorization or Special Ad Category review when it offers no application, quote, credit, loan, insurance, or housing/employment service.","input":{"platform":"meta","industry":"finance","headline":"Loan planning education webinar","body":"Learn budgeting basics and how loan terms work. No application or quote is offered.","cta":"Register"},"expected_decision":"approved","expected_policy_ids":[]} +{"id":"near_miss_visible_discount_approved","false_positive_guardrail":true,"near_miss_policy_ids":["landing_page_offer_mismatch","google_misrepresentation_risk"],"reviewer_notes":"Discount and promo-code language should stay approved when the extracted landing page repeats the material terms clearly.","input":{"platform":"tiktok","industry":"general","headline":"Limited time 50% off creator toolkit","body":"Use promo code LAUNCH50 before the sale ends.","cta":"Claim deal","landing_page_html":"
Limited time 50% off with promo code LAUNCH50.
"},"expected_decision":"approved","expected_policy_ids":[]} +{"id":"near_miss_creator_disclosure_review_only","false_positive_guardrail":true,"near_miss_policy_ids":["tiktok_disclosure_risk","tiktok_misleading_content"],"reviewer_notes":"Creator sponsorship wording should remain a disclosure review label, not a misleading-content finding, when it is transparent about the paid relationship.","input":{"platform":"tiktok","industry":"creator","headline":"Sponsored planner walkthrough","body":"A paid partnership showing how I organize launch tasks, with the discount terms shown on the page.","cta":"View planner"},"expected_decision":"needs_review","expected_policy_ids":["missing_affiliate_or_sponsor_disclosure","tiktok_disclosure_risk"]} +{"id":"near_miss_meta_paid_partner_review_only","false_positive_guardrail":true,"near_miss_policy_ids":["meta_branded_content_disclosure","meta_financial_services_authorization_review"],"reviewer_notes":"Meta paid-partnership language for a general budgeting app should require disclosure review without becoming a financial-services authorization finding.","input":{"platform":"meta","industry":"creator","headline":"Paid partnership with a budgeting workspace","body":"Use my promo code to try the template library for planning monthly expenses.","cta":"View templates"},"expected_decision":"needs_review","expected_policy_ids":["meta_branded_content_disclosure","missing_affiliate_or_sponsor_disclosure"]} +{"id":"near_miss_health_quiz_no_private_request_approved","false_positive_guardrail":true,"near_miss_policy_ids":["meta_private_information_request","privacy_health_data_collection","tracking_pixel_risk"],"reviewer_notes":"A generic wellness quiz teaser should remain approved when the ad does not request private health information, collect symptoms, or mention tracking.","input":{"platform":"meta","industry":"wellness","headline":"Find your wellness planning style","body":"Answer a short lifestyle quiz and get general habit ideas for your weekly routine.","cta":"Take quiz"},"expected_decision":"approved","expected_policy_ids":[]} diff --git a/evals/run_eval.py b/evals/run_eval.py index 11d6291..7b375af 100644 --- a/evals/run_eval.py +++ b/evals/run_eval.py @@ -54,6 +54,12 @@ def main() -> int: default=0.7, help="Minimum decision accuracy required for a zero exit code.", ) + parser.add_argument( + "--max-policy-false-positives", + type=int, + default=None, + help="Maximum total policy false positives allowed for a zero exit code.", + ) parser.add_argument( "--max-review-notes", type=int, @@ -123,7 +129,16 @@ def main() -> int: ) else: print(json.dumps(metrics, indent=2, sort_keys=True)) - return 0 if _passes(metrics, min_accuracy=args.min_decision_accuracy, require_model=args.require_model) else 1 + return ( + 0 + if _passes( + metrics, + min_accuracy=args.min_decision_accuracy, + require_model=args.require_model, + max_policy_false_positives=args.max_policy_false_positives, + ) + else 1 + ) def _positive_int(value: str) -> int: @@ -160,16 +175,32 @@ def _attach_dataset(metrics: dict[str, Any], dataset: str) -> None: _attach_dataset(mode_metrics, dataset) -def _passes(metrics: dict[str, Any], *, min_accuracy: float, require_model: bool) -> bool: +def _passes( + metrics: dict[str, Any], + *, + min_accuracy: float, + require_model: bool, + max_policy_false_positives: int | None = None, +) -> bool: if metrics.get("mode") == "all": return all( - _passes(mode_metrics, min_accuracy=min_accuracy, require_model=require_model) + _passes( + mode_metrics, + min_accuracy=min_accuracy, + require_model=require_model, + max_policy_false_positives=max_policy_false_positives, + ) for mode_metrics in metrics["modes"].values() ) if require_model and metrics.get("model_required_failures"): return False if metrics["total_examples"] == 0: return not require_model + if ( + max_policy_false_positives is not None + and _policy_metric_count(metrics, "false_positive_count") > max_policy_false_positives + ): + return False return metrics["decision_accuracy"] >= min_accuracy @@ -221,6 +252,7 @@ def _compact_summary( } review_notes = metrics.get("review_notes", {}) + false_positive_pressure = metrics.get("false_positive_pressure", {}) return { "summary_version": 1, "dataset": metrics.get("dataset", "