From a096738805b888fbc8c37d7236c7bcdf982661e6 Mon Sep 17 00:00:00 2001 From: ftchvs Date: Tue, 26 May 2026 12:38:20 -0700 Subject: [PATCH 1/2] Improve AdLint demo path --- Makefile | 5 ++++- README.md | 21 ++++++++++++++++---- adlint/cli.py | 36 ++++++++++++++++++++++++++++++++++ adlint/rewrites/suggestions.py | 6 ++++++ docs/adlint_paper_visual_qa.md | 6 +++--- docs/eval_report.md | 26 ++++++++++++------------ pyproject.toml | 26 ++++++++++++++++++++++++ tests/test_cli.py | 30 ++++++++++++++++++++++++++++ 8 files changed, 135 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 314fb15..0fa6129 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ MODEL_EVAL_FLAGS ?= --ollama-model gpt-oss-safeguard:20b ADLINT_OLLAMA_TIMEOUT ?= 180 ADLINT_OLLAMA_NUM_PREDICT ?= 1024 -.PHONY: api creative-assets-eval dev landing-page-consistency eval benchmark benchmark-data policy-coverage policy-coverage-validate rewrite-quality model-benchmark model-smoke model-usefulness pr-preflight real-cases real-cases-ci real-cases-hybrid real-cases-model-quality real-cases-validate real-world-blind-candidates real-world-blind-ci real-world-blind-validate real-world-blind real-world-blind-model-quality research-summary scan test install +.PHONY: api creative-assets-eval demo dev landing-page-consistency eval benchmark benchmark-data policy-coverage policy-coverage-validate rewrite-quality model-benchmark model-smoke model-usefulness pr-preflight real-cases real-cases-ci real-cases-hybrid real-cases-model-quality real-cases-validate real-world-blind-candidates real-world-blind-ci real-world-blind-validate real-world-blind real-world-blind-model-quality research-summary scan test install install: $(STAMP) @@ -18,6 +18,9 @@ $(STAMP): pyproject.toml dev: $(STAMP) $(BIN)/python -m adlint scan examples/high_risk_tiktok_health.json --output-dir reports +demo: $(STAMP) + $(BIN)/python -m adlint demo --output-dir reports/demo + api: $(STAMP) ADLINT_OLLAMA_TIMEOUT=$(ADLINT_OLLAMA_TIMEOUT) ADLINT_OLLAMA_NUM_PREDICT=$(ADLINT_OLLAMA_NUM_PREDICT) $(BIN)/uvicorn adlint.api:app --reload --reload-dir adlint diff --git a/README.md b/README.md index f4261b3..d8f88a3 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ Example generated reports: Reproduce the CLI demo: ```bash -adlint scan examples/meta_high_risk_health.json --format markdown --output-dir docs/assets/demo +adlint demo --output-dir reports/demo make api # then open http://127.0.0.1:8000/ui/ ``` @@ -127,7 +127,7 @@ Requirements: Python 3.11 or newer. Docker is optional. python3 -m venv .venv . .venv/bin/activate python -m pip install -e ".[dev]" -adlint scan examples/high_risk_tiktok_health.json --output-dir reports +adlint demo --output-dir reports/demo ``` Without activating the virtual environment: @@ -141,6 +141,7 @@ Makefile shortcuts: ```bash make dev # install and run the high-risk example, writing reports/ +make demo # install and run the bundled public-safe demo, writing reports/demo/ make scan # install and run the wellness example make api # start uvicorn with adlint.api:app make eval # run the seed evals and write evals/results/latest.json @@ -173,7 +174,7 @@ Fastest local demo: python3 -m venv .venv . .venv/bin/activate python -m pip install -e ".[dev]" -adlint scan examples/meta_high_risk_health.json --format markdown +adlint demo --output-dir reports/demo make api ``` @@ -181,6 +182,12 @@ Then open `http://127.0.0.1:8000/ui/` and try the bundled high-risk health example. Use synthetic examples only; do not paste private campaign or customer data into issues, docs, screenshots, or eval fixtures. +The demo command uses an embedded synthetic ad, so it works even before you +choose or edit one of the example config files. It writes: + +- `reports/demo/adlint-report.json` +- `reports/demo/adlint-report.md` + ## CLI ```bash @@ -204,6 +211,12 @@ adlint scan - `--storage-path ` sets the SQLite metadata database path and opts into storage. +Run the built-in synthetic demo without preparing a config file: + +```bash +adlint demo --output-dir reports/demo +``` + Example config: ```json @@ -517,7 +530,7 @@ make real-world-blind-model-quality balanced across 30 approved, 30 needs-review, and 30 high-risk expected decisions. It is marked as a rule-tuning holdout and should be used to measure generalization before changing deterministic rules. The CI gate uses a 0.90 -decision-accuracy threshold against the current 0.967 post-triage rule-only +decision-accuracy threshold against the current 0.989 rule-only baseline. Before opening eval/reliability PRs, run: diff --git a/adlint/cli.py b/adlint/cli.py index bf7bbcd..ca4564e 100644 --- a/adlint/cli.py +++ b/adlint/cli.py @@ -10,6 +10,21 @@ from adlint.reports import to_markdown +DEMO_CONFIG = { + "platform": "meta", + "country": "US", + "industry": "health", + "headline": "Lose 20 pounds in 30 days guaranteed", + "body": "Our clinically proven supplement melts fat fast with before and after results.", + "cta": "Buy now", + "landing_page_html": ( + "

Fast weight-loss results

" + "

No prescription needed. Limited-time discount.

" + ), + "policy_modules": ["health_claims", "platform", "privacy", "landing_page"], +} + + def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="adlint") subparsers = parser.add_subparsers(dest="command", required=True) @@ -94,6 +109,19 @@ def main(argv: Sequence[str] | None = None) -> int: help="Path to optional scoring.yml threshold and weight overrides.", ) + demo_parser = subparsers.add_parser("demo", help="Run the bundled public-safe demo scan.") + demo_parser.add_argument( + "--output-dir", + default="reports/demo", + help="Write demo JSON and Markdown reports to this directory.", + ) + demo_parser.add_argument( + "--format", + choices=("json", "markdown"), + default="markdown", + help="Format printed to stdout.", + ) + args = parser.parse_args(argv) if args.command == "scan": @@ -138,5 +166,13 @@ def main(argv: Sequence[str] | None = None) -> int: print(json.dumps(result, indent=2, sort_keys=True)) return 0 + if args.command == "demo": + result = analyze(dict(DEMO_CONFIG), output_dir=args.output_dir) + if args.format == "json": + print(json.dumps(result.to_dict(), indent=2, sort_keys=True)) + else: + print(to_markdown(result)) + return 0 + parser.error(f"Unknown command: {args.command}") return 2 diff --git a/adlint/rewrites/suggestions.py b/adlint/rewrites/suggestions.py index 92f247e..7a767b7 100644 --- a/adlint/rewrites/suggestions.py +++ b/adlint/rewrites/suggestions.py @@ -34,6 +34,12 @@ def _rewrite_for_hit(submission: Submission, hit: PolicyHit) -> dict[str, str]: "body": "Learn about options that may support healthy habits. Consult a qualified professional for medical questions.", "cta": "Learn more", } + if hit.category == "health_claims" or hit.policy_id.startswith("meta_health_"): + return { + "headline": "Support your wellness routine with daily nutrition", + "body": "Designed to complement healthy habits. Individual results vary.", + "cta": _soft_cta(submission.cta), + } if hit.category == "privacy": return { "headline": submission.headline or "Learn more before you sign up", diff --git a/docs/adlint_paper_visual_qa.md b/docs/adlint_paper_visual_qa.md index 1afc6d9..faa6efe 100644 --- a/docs/adlint_paper_visual_qa.md +++ b/docs/adlint_paper_visual_qa.md @@ -4,11 +4,11 @@ - Source reviewed: `docs/adlint_hybrid_eval_paper.tex` - Output reviewed: `docs/build/adlint_hybrid_eval_paper.pdf` -- Bundled Tectonic: `/Users/ftchvs/Developer/.agents/codex/plugins/cache/openai-bundled/latex-tectonic/0.1.0/bin/tectonic` -- Compile command, run from `/Users/ftchvs/Developer/AdLint`: +- Tectonic binary: use a local `tectonic` install or bundled Tectonic binary. +- Compile command, run from the repository root: ```bash -/Users/ftchvs/Developer/.agents/codex/plugins/cache/openai-bundled/latex-tectonic/0.1.0/bin/tectonic --outdir docs/build docs/adlint_hybrid_eval_paper.tex +tectonic --outdir docs/build docs/adlint_hybrid_eval_paper.tex ``` - Compile result: passed; PDF written to `docs/build/adlint_hybrid_eval_paper.pdf`. diff --git a/docs/eval_report.md b/docs/eval_report.md index ea4af00..4406139 100644 --- a/docs/eval_report.md +++ b/docs/eval_report.md @@ -5,7 +5,7 @@ Status: deterministic rule benchmark v1. AdLint includes four labeled JSONL datasets: - `evals/datasets/seed_ads.jsonl`: the 58-example smoke set. -- `evals/datasets/rule_benchmark_v1.jsonl`: a 209-example benchmark generated +- `evals/datasets/rule_benchmark_v1.jsonl`: a 213-example benchmark generated from the seed set plus policy-author authored synthetic variants. - `evals/datasets/real_cases_v1.jsonl`: a 75-example public-source diagnostic set balanced across 25 approved, 25 needs-review, and 25 high-risk expected @@ -29,7 +29,7 @@ Rebuild the committed benchmark dataset: make benchmark-data ``` -Run the 209-example benchmark and write JSON plus Markdown reports: +Run the 213-example benchmark and write JSON plus Markdown reports: ```bash make benchmark @@ -130,8 +130,8 @@ make real-world-blind-model-quality CI uses `make pr-preflight`, `make real-cases-ci`, and `make real-world-blind-ci`. The real-case gate requires 1.000 rule-only decision accuracy because this curated set should not regress. The blind -holdout gate uses 0.90 rule-only decision accuracy against the current 0.967 -post-triage baseline, preserving room for known misses while still catching broad +holdout gate uses 0.90 rule-only decision accuracy against the current 0.989 +rule-only baseline, preserving room for known misses while still catching broad reliability regressions. Both CI eval targets print compact summaries and upload full JSON/Markdown reports as workflow artifacts. @@ -171,11 +171,11 @@ Current `rule_benchmark_v1` results from the local deterministic rule runner: | Metric | Value | | --- | ---: | -| Total examples | 200 | +| Total examples | 213 | | Decision accuracy | 1.000 | -| Expected approved | 51 | -| Expected needs_review | 50 | -| Expected high_risk | 99 | +| Expected approved | 44 | +| Expected needs_review | 62 | +| Expected high_risk | 107 | | Policy false-positive review notes | 0 | | Policy false-negative review notes | 0 | @@ -183,9 +183,9 @@ Current confusion matrix: | Expected \ Actual | approved | needs_review | high_risk | | --- | ---: | ---: | ---: | -| approved | 51 | 0 | 0 | -| needs_review | 0 | 50 | 0 | -| high_risk | 0 | 0 | 99 | +| approved | 44 | 0 | 0 | +| needs_review | 0 | 62 | 0 | +| high_risk | 0 | 0 | 107 | Current category-level precision and recall: @@ -201,8 +201,8 @@ Current category-level precision and recall: Interpretation: the 1.000 score is strong evidence that the deterministic rules and current benchmark labels are internally consistent. It is not a -claim that future ads will pass review with 100% accuracy. If the 209 examples -were a representative random sample, 209/209 correct decisions would imply an +claim that future ads will pass review with 100% accuracy. If the 213 examples +were a representative random sample, 213/213 correct decisions would imply an approximate 95% Wilson lower bound of 0.981, but this benchmark is authored regression coverage rather than a random production sample. diff --git a/pyproject.toml b/pyproject.toml index 3e64c48..570a61c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,12 +12,38 @@ license = "MIT" authors = [ { name = "AdLint Contributors" } ] +keywords = [ + "advertising", + "compliance", + "brand-safety", + "growth-marketing", + "policy-as-code", + "local-first", + "fastapi" +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Framework :: FastAPI", + "Intended Audience :: Developers", + "Intended Audience :: Legal Industry", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Office/Business", + "Topic :: Software Development :: Quality Assurance" +] dependencies = [ "fastapi>=0.115", "PyYAML>=6.0.2", "uvicorn>=0.34" ] +[project.urls] +Homepage = "https://github.com/ftchvs/AdLint" +Repository = "https://github.com/ftchvs/AdLint" +Issues = "https://github.com/ftchvs/AdLint/issues" + [project.optional-dependencies] dev = [ "httpx>=0.27", diff --git a/tests/test_cli.py b/tests/test_cli.py index 73b729c..dcff804 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -81,6 +81,36 @@ def test_cli_scan_markdown_format_includes_disclaimer(tmp_path, capsys) -> None: assert "brand_safety_politics" in output +def test_cli_demo_runs_public_safe_sample_and_writes_reports(tmp_path, capsys) -> None: + output_dir = tmp_path / "demo" + + assert main(["demo", "--output-dir", str(output_dir)]) == 0 + + output = capsys.readouterr().out + assert output.startswith("# AdLint Report") + assert "Decision: `high_risk`" in output + assert "Decision-Support Disclaimer" in output + assert "developed with evidence-informed guidance" not in output + assert "Support your wellness routine with daily nutrition" in output + assert (output_dir / "adlint-report.json").exists() + assert (output_dir / "adlint-report.md").exists() + + +def test_cli_demo_json_stdout_remains_parseable(tmp_path, capsys) -> None: + output_dir = tmp_path / "demo" + + assert main(["demo", "--output-dir", str(output_dir), "--format", "json"]) == 0 + + output = json.loads(capsys.readouterr().out) + assert output["decision"] == "high_risk" + assert output["reports"] == { + "json": str(output_dir / "adlint-report.json"), + "markdown": str(output_dir / "adlint-report.md"), + } + assert output["logging_enabled"] is False + assert output["model"] == {"enabled": False, "provider": None, "status": "disabled"} + + def test_cli_batch_prints_private_json_summary(tmp_path, capsys) -> None: csv_path = tmp_path / "ads.csv" csv_path.write_text( From 9c05d143b9b2fb48ec39e2e61e67078f4cd70a5a Mon Sep 17 00:00:00 2001 From: ftchvs Date: Tue, 26 May 2026 13:15:03 -0700 Subject: [PATCH 2/2] Update CI actions runtime --- .github/workflows/test.yml | 6 +++--- tests/test_ci_eval_config.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f2c918f..6e360d2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,10 +18,10 @@ jobs: steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} cache: pip @@ -45,7 +45,7 @@ jobs: - name: Upload eval reports if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: eval-reports-py${{ matrix.python-version }} path: | diff --git a/tests/test_ci_eval_config.py b/tests/test_ci_eval_config.py index 9fdb57b..3fa4028 100644 --- a/tests/test_ci_eval_config.py +++ b/tests/test_ci_eval_config.py @@ -31,6 +31,6 @@ def test_github_actions_runs_packaging_gates_and_uploads_eval_artifacts() -> Non assert "make pr-preflight" in workflow assert "make real-cases-ci" in workflow assert "make real-world-blind-ci" in workflow - assert "actions/upload-artifact@v4" in workflow + assert "actions/upload-artifact@v6" in workflow assert "evals/results/*.json" in workflow assert "evals/results/*.md" in workflow