Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ jobs:

steps:
- name: Check out repository
uses: actions/checkout@v4
uses: actions/checkout@v6

- name: Set up Python
uses: actions/setup-python@v5
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
cache: pip
Expand All @@ -45,7 +45,7 @@ jobs:

- name: Upload eval reports
if: always()
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v6
with:
name: eval-reports-py${{ matrix.python-version }}
path: |
Expand Down
5 changes: 4 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ MODEL_EVAL_FLAGS ?= --ollama-model gpt-oss-safeguard:20b
ADLINT_OLLAMA_TIMEOUT ?= 180
ADLINT_OLLAMA_NUM_PREDICT ?= 1024

.PHONY: api creative-assets-eval dev landing-page-consistency eval benchmark benchmark-data policy-coverage policy-coverage-validate rewrite-quality model-benchmark model-smoke model-usefulness pr-preflight real-cases real-cases-ci real-cases-hybrid real-cases-model-quality real-cases-validate real-world-blind-candidates real-world-blind-ci real-world-blind-validate real-world-blind real-world-blind-model-quality research-summary scan test install
.PHONY: api creative-assets-eval demo dev landing-page-consistency eval benchmark benchmark-data policy-coverage policy-coverage-validate rewrite-quality model-benchmark model-smoke model-usefulness pr-preflight real-cases real-cases-ci real-cases-hybrid real-cases-model-quality real-cases-validate real-world-blind-candidates real-world-blind-ci real-world-blind-validate real-world-blind real-world-blind-model-quality research-summary scan test install

install: $(STAMP)

Expand All @@ -18,6 +18,9 @@ $(STAMP): pyproject.toml
dev: $(STAMP)
$(BIN)/python -m adlint scan examples/high_risk_tiktok_health.json --output-dir reports

demo: $(STAMP)
$(BIN)/python -m adlint demo --output-dir reports/demo

api: $(STAMP)
ADLINT_OLLAMA_TIMEOUT=$(ADLINT_OLLAMA_TIMEOUT) ADLINT_OLLAMA_NUM_PREDICT=$(ADLINT_OLLAMA_NUM_PREDICT) $(BIN)/uvicorn adlint.api:app --reload --reload-dir adlint

Expand Down
21 changes: 17 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ Example generated reports:
Reproduce the CLI demo:

```bash
adlint scan examples/meta_high_risk_health.json --format markdown --output-dir docs/assets/demo
adlint demo --output-dir reports/demo
make api # then open http://127.0.0.1:8000/ui/
```

Expand Down Expand Up @@ -127,7 +127,7 @@ Requirements: Python 3.11 or newer. Docker is optional.
python3 -m venv .venv
. .venv/bin/activate
python -m pip install -e ".[dev]"
adlint scan examples/high_risk_tiktok_health.json --output-dir reports
adlint demo --output-dir reports/demo
```

Without activating the virtual environment:
Expand All @@ -141,6 +141,7 @@ Makefile shortcuts:

```bash
make dev # install and run the high-risk example, writing reports/
make demo # install and run the bundled public-safe demo, writing reports/demo/
make scan # install and run the wellness example
make api # start uvicorn with adlint.api:app
make eval # run the seed evals and write evals/results/latest.json
Expand Down Expand Up @@ -173,14 +174,20 @@ Fastest local demo:
python3 -m venv .venv
. .venv/bin/activate
python -m pip install -e ".[dev]"
adlint scan examples/meta_high_risk_health.json --format markdown
adlint demo --output-dir reports/demo
make api
```

Then open `http://127.0.0.1:8000/ui/` and try the bundled high-risk health
example. Use synthetic examples only; do not paste private campaign or customer
data into issues, docs, screenshots, or eval fixtures.

The demo command uses an embedded synthetic ad, so it works even before you
choose or edit one of the example config files. It writes:

- `reports/demo/adlint-report.json`
- `reports/demo/adlint-report.md`

## CLI

```bash
Expand All @@ -204,6 +211,12 @@ adlint scan <config>
- `--storage-path <path>` sets the SQLite metadata database path and opts into
storage.

Run the built-in synthetic demo without preparing a config file:

```bash
adlint demo --output-dir reports/demo
```

Example config:

```json
Expand Down Expand Up @@ -517,7 +530,7 @@ make real-world-blind-model-quality
balanced across 30 approved, 30 needs-review, and 30 high-risk expected
decisions. It is marked as a rule-tuning holdout and should be used to measure
generalization before changing deterministic rules. The CI gate uses a 0.90
decision-accuracy threshold against the current 0.967 post-triage rule-only
decision-accuracy threshold against the current 0.989 rule-only
baseline.

Before opening eval/reliability PRs, run:
Expand Down
36 changes: 36 additions & 0 deletions adlint/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,21 @@
from adlint.reports import to_markdown


DEMO_CONFIG = {
"platform": "meta",
"country": "US",
"industry": "health",
"headline": "Lose 20 pounds in 30 days guaranteed",
"body": "Our clinically proven supplement melts fat fast with before and after results.",
"cta": "Buy now",
"landing_page_html": (
"<html><body><h1>Fast weight-loss results</h1>"
"<p>No prescription needed. Limited-time discount.</p></body></html>"
),
"policy_modules": ["health_claims", "platform", "privacy", "landing_page"],
}


def main(argv: Sequence[str] | None = None) -> int:
parser = argparse.ArgumentParser(prog="adlint")
subparsers = parser.add_subparsers(dest="command", required=True)
Expand Down Expand Up @@ -94,6 +109,19 @@ def main(argv: Sequence[str] | None = None) -> int:
help="Path to optional scoring.yml threshold and weight overrides.",
)

demo_parser = subparsers.add_parser("demo", help="Run the bundled public-safe demo scan.")
demo_parser.add_argument(
"--output-dir",
default="reports/demo",
help="Write demo JSON and Markdown reports to this directory.",
)
demo_parser.add_argument(
"--format",
choices=("json", "markdown"),
default="markdown",
help="Format printed to stdout.",
)

args = parser.parse_args(argv)

if args.command == "scan":
Expand Down Expand Up @@ -138,5 +166,13 @@ def main(argv: Sequence[str] | None = None) -> int:
print(json.dumps(result, indent=2, sort_keys=True))
return 0

if args.command == "demo":
result = analyze(dict(DEMO_CONFIG), output_dir=args.output_dir)
if args.format == "json":
print(json.dumps(result.to_dict(), indent=2, sort_keys=True))
else:
print(to_markdown(result))
return 0

parser.error(f"Unknown command: {args.command}")
return 2
6 changes: 6 additions & 0 deletions adlint/rewrites/suggestions.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,12 @@ def _rewrite_for_hit(submission: Submission, hit: PolicyHit) -> dict[str, str]:
"body": "Learn about options that may support healthy habits. Consult a qualified professional for medical questions.",
"cta": "Learn more",
}
if hit.category == "health_claims" or hit.policy_id.startswith("meta_health_"):
return {
"headline": "Support your wellness routine with daily nutrition",
"body": "Designed to complement healthy habits. Individual results vary.",
"cta": _soft_cta(submission.cta),
}
if hit.category == "privacy":
return {
"headline": submission.headline or "Learn more before you sign up",
Expand Down
6 changes: 3 additions & 3 deletions docs/adlint_paper_visual_qa.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@

- Source reviewed: `docs/adlint_hybrid_eval_paper.tex`
- Output reviewed: `docs/build/adlint_hybrid_eval_paper.pdf`
- Bundled Tectonic: `/Users/ftchvs/Developer/.agents/codex/plugins/cache/openai-bundled/latex-tectonic/0.1.0/bin/tectonic`
- Compile command, run from `/Users/ftchvs/Developer/AdLint`:
- Tectonic binary: use a local `tectonic` install or bundled Tectonic binary.
- Compile command, run from the repository root:

```bash
/Users/ftchvs/Developer/.agents/codex/plugins/cache/openai-bundled/latex-tectonic/0.1.0/bin/tectonic --outdir docs/build docs/adlint_hybrid_eval_paper.tex
tectonic --outdir docs/build docs/adlint_hybrid_eval_paper.tex
```

- Compile result: passed; PDF written to `docs/build/adlint_hybrid_eval_paper.pdf`.
Expand Down
26 changes: 13 additions & 13 deletions docs/eval_report.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ Status: deterministic rule benchmark v1.
AdLint includes four labeled JSONL datasets:

- `evals/datasets/seed_ads.jsonl`: the 58-example smoke set.
- `evals/datasets/rule_benchmark_v1.jsonl`: a 209-example benchmark generated
- `evals/datasets/rule_benchmark_v1.jsonl`: a 213-example benchmark generated
from the seed set plus policy-author authored synthetic variants.
- `evals/datasets/real_cases_v1.jsonl`: a 75-example public-source diagnostic
set balanced across 25 approved, 25 needs-review, and 25 high-risk expected
Expand All @@ -29,7 +29,7 @@ Rebuild the committed benchmark dataset:
make benchmark-data
```

Run the 209-example benchmark and write JSON plus Markdown reports:
Run the 213-example benchmark and write JSON plus Markdown reports:

```bash
make benchmark
Expand Down Expand Up @@ -130,8 +130,8 @@ make real-world-blind-model-quality
CI uses `make pr-preflight`, `make real-cases-ci`, and
`make real-world-blind-ci`. The real-case gate requires 1.000 rule-only
decision accuracy because this curated set should not regress. The blind
holdout gate uses 0.90 rule-only decision accuracy against the current 0.967
post-triage baseline, preserving room for known misses while still catching broad
holdout gate uses 0.90 rule-only decision accuracy against the current 0.989
rule-only baseline, preserving room for known misses while still catching broad
reliability regressions. Both CI eval targets print compact summaries and
upload full JSON/Markdown reports as workflow artifacts.

Expand Down Expand Up @@ -171,21 +171,21 @@ Current `rule_benchmark_v1` results from the local deterministic rule runner:

| Metric | Value |
| --- | ---: |
| Total examples | 200 |
| Total examples | 213 |
| Decision accuracy | 1.000 |
| Expected approved | 51 |
| Expected needs_review | 50 |
| Expected high_risk | 99 |
| Expected approved | 44 |
| Expected needs_review | 62 |
| Expected high_risk | 107 |
| Policy false-positive review notes | 0 |
| Policy false-negative review notes | 0 |

Current confusion matrix:

| Expected \ Actual | approved | needs_review | high_risk |
| --- | ---: | ---: | ---: |
| approved | 51 | 0 | 0 |
| needs_review | 0 | 50 | 0 |
| high_risk | 0 | 0 | 99 |
| approved | 44 | 0 | 0 |
| needs_review | 0 | 62 | 0 |
| high_risk | 0 | 0 | 107 |

Current category-level precision and recall:

Expand All @@ -201,8 +201,8 @@ Current category-level precision and recall:

Interpretation: the 1.000 score is strong evidence that the deterministic
rules and current benchmark labels are internally consistent. It is not a
claim that future ads will pass review with 100% accuracy. If the 209 examples
were a representative random sample, 209/209 correct decisions would imply an
claim that future ads will pass review with 100% accuracy. If the 213 examples
were a representative random sample, 213/213 correct decisions would imply an
approximate 95% Wilson lower bound of 0.981, but this benchmark is authored
regression coverage rather than a random production sample.

Expand Down
26 changes: 26 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,38 @@ license = "MIT"
authors = [
{ name = "AdLint Contributors" }
]
keywords = [
"advertising",
"compliance",
"brand-safety",
"growth-marketing",
"policy-as-code",
"local-first",
"fastapi"
]
classifiers = [
"Development Status :: 3 - Alpha",
"Environment :: Console",
"Framework :: FastAPI",
"Intended Audience :: Developers",
"Intended Audience :: Legal Industry",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Topic :: Office/Business",
"Topic :: Software Development :: Quality Assurance"
]
dependencies = [
"fastapi>=0.115",
"PyYAML>=6.0.2",
"uvicorn>=0.34"
]

[project.urls]
Homepage = "https://github.com/ftchvs/AdLint"
Repository = "https://github.com/ftchvs/AdLint"
Issues = "https://github.com/ftchvs/AdLint/issues"

[project.optional-dependencies]
dev = [
"httpx>=0.27",
Expand Down
2 changes: 1 addition & 1 deletion tests/test_ci_eval_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,6 @@ def test_github_actions_runs_packaging_gates_and_uploads_eval_artifacts() -> Non
assert "make pr-preflight" in workflow
assert "make real-cases-ci" in workflow
assert "make real-world-blind-ci" in workflow
assert "actions/upload-artifact@v4" in workflow
assert "actions/upload-artifact@v6" in workflow
assert "evals/results/*.json" in workflow
assert "evals/results/*.md" in workflow
30 changes: 30 additions & 0 deletions tests/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,36 @@ def test_cli_scan_markdown_format_includes_disclaimer(tmp_path, capsys) -> None:
assert "brand_safety_politics" in output


def test_cli_demo_runs_public_safe_sample_and_writes_reports(tmp_path, capsys) -> None:
output_dir = tmp_path / "demo"

assert main(["demo", "--output-dir", str(output_dir)]) == 0

output = capsys.readouterr().out
assert output.startswith("# AdLint Report")
assert "Decision: `high_risk`" in output
assert "Decision-Support Disclaimer" in output
assert "developed with evidence-informed guidance" not in output
assert "Support your wellness routine with daily nutrition" in output
assert (output_dir / "adlint-report.json").exists()
assert (output_dir / "adlint-report.md").exists()


def test_cli_demo_json_stdout_remains_parseable(tmp_path, capsys) -> None:
output_dir = tmp_path / "demo"

assert main(["demo", "--output-dir", str(output_dir), "--format", "json"]) == 0

output = json.loads(capsys.readouterr().out)
assert output["decision"] == "high_risk"
assert output["reports"] == {
"json": str(output_dir / "adlint-report.json"),
"markdown": str(output_dir / "adlint-report.md"),
}
assert output["logging_enabled"] is False
assert output["model"] == {"enabled": False, "provider": None, "status": "disabled"}


def test_cli_batch_prints_private_json_summary(tmp_path, capsys) -> None:
csv_path = tmp_path / "ads.csv"
csv_path.write_text(
Expand Down