diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..0f7a467 --- /dev/null +++ b/.env.example @@ -0,0 +1,16 @@ +# ─── IGenBench API Keys ─────────────────────────────────────────────────────── +# Copy this file to .env and fill in your keys. +# Never commit .env to version control. + +# Google AI Studio — generation + evaluation +# https://aistudio.google.com/app/apikey +GOOGLE_API_KEY= + +# OpenRouter — generation + evaluation (OpenAI-compatible) +# https://openrouter.ai/keys +OPENROUTER_API_KEY= + +# Replicate — generation only +# Requires: uv sync --extra replicate +# https://replicate.com/account/api-tokens +REPLICATE_API_TOKEN= diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..9d18712 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,65 @@ +name: Bug Report +description: Report something that is not working correctly. +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report. + Please search existing issues before submitting. + + - type: textarea + id: description + attributes: + label: What happened? + description: A clear description of the bug. + validations: + required: true + + - type: textarea + id: reproduce + attributes: + label: Steps to reproduce + description: The exact commands or code that trigger the bug. + placeholder: | + 1. Run `igenbench eval --info-path ...` + 2. See error ... + validations: + required: true + + - type: textarea + id: expected + attributes: + label: Expected behaviour + validations: + required: true + + - type: textarea + id: logs + attributes: + label: Relevant logs / error output + render: shell + + - type: input + id: version + attributes: + label: IGenBench version + placeholder: "e.g. 0.1.0 or git SHA" + validations: + required: true + + - type: input + id: python + attributes: + label: Python version + placeholder: "e.g. 3.11.9" + validations: + required: true + + - type: input + id: os + attributes: + label: Operating system + placeholder: "e.g. macOS 15, Ubuntu 24.04, Windows 11" + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..7f53c1a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,37 @@ +name: Feature Request +description: Suggest a new feature or improvement. +labels: ["enhancement"] +body: + - type: markdown + attributes: + value: | + Thanks for the suggestion! Please search existing issues first. + + - type: textarea + id: problem + attributes: + label: Problem or motivation + description: What problem does this feature solve, or what use case does it enable? + validations: + required: true + + - type: textarea + id: solution + attributes: + label: Proposed solution + description: Describe the feature you'd like and how it should work. + validations: + required: true + + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + description: Any other approaches you considered? + + - type: checkboxes + id: willing + attributes: + label: Are you willing to implement this? + options: + - label: Yes, I'd like to submit a PR. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..13b376c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,49 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + lint: + name: Lint & Format + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" + + - name: Install dependencies + run: uv sync --dev + + - name: Ruff lint + run: uv run ruff check . + + - name: Ruff format check + run: uv run ruff format --check . + + test: + name: Tests (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + + - name: Set up uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: uv sync --dev + + - name: Run unit tests + run: uv run pytest tests/ -v --tb=short diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..e6452c8 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,150 @@ +# AGENTS.md — IGenBench Codebase Guide for AI Coding Agents + +This file helps AI agents (Claude Code, GitHub Copilot, Cursor, etc.) navigate and modify this codebase correctly. + +--- + +## What This Project Does + +IGenBench is a CLI benchmark tool for evaluating the **reliability** of text-to-infographic generation models. Given a dataset of `VISItem` JSON files (each containing a prompt, reference image, and evaluation questions), it: + +1. **Generates** infographics using a text-to-image model (`igenbench gen` / `igenbench batch-gen`) +2. **Evaluates** generated images by asking an LLM to answer factual Q&A questions about the image (`igenbench eval` / `igenbench batch-eval`) +3. **Scores** the results to compute Q-ACC and I-ACC metrics (`igenbench score`) + +--- + +## Architecture + +``` +CLI (typer) + └── Workflow (orchestration + resume logic) + └── Engine (single-item gen/eval step) + └── LLMCaller (provider abstraction) + ├── GoogleCaller + ├── OpenrouterCaller + └── ReplicateCaller +``` + +### Key Files + +| File | Purpose | +|------|---------| +| `igenbench/vis_item.py` | Core data model. **Read this first.** | +| `igenbench/cli/main.py` | Typer app; registers all sub-commands | +| `igenbench/cli/gen_cli.py` | `igenbench gen` command | +| `igenbench/cli/eval_cli.py` | `igenbench eval` command | +| `igenbench/cli/batch_cli.py` | `igenbench batch-gen` and `batch-eval` | +| `igenbench/cli/score_cli.py` | `igenbench score` command | +| `igenbench/workflow/gen_workflow.py` | Generation orchestration | +| `igenbench/workflow/eval_workflow.py` | Evaluation orchestration with resume | +| `igenbench/engine/gen_engine.py` | Calls LLMCaller.generate_image for one item | +| `igenbench/engine/eval_engine.py` | Calls LLMCaller.understand_image + parses answer | +| `igenbench/utils/llm/llm_caller.py` | All provider implementations + `LLMCaller` base | +| `igenbench/utils/llm/caller_registry.py` | `@register_caller` decorator + lookup | +| `igenbench/utils/state_manager.py` | Reads/writes VISItem JSON to disk | +| `igenbench/utils/path_resolver.py` | Resolves output paths from item id + model name | +| `prompts/gen_text2image.py` | Prompt template for image generation | +| `prompts/eval_judge_factual_qa.py` | Prompt template for evaluation | + +--- + +## Data Model + +```python +VISItem + id: str # e.g. "001" + t2i_prompt: str # generation prompt + chart_type: str # e.g. "bar", "pie" + reference_image_url: str # URL of ground-truth image + generation: dict[model, path] # generated image paths keyed by model name + evaluation: list[EvalEntry] + +EvalEntry + source: str # "prompt" | "seed" + question: str # e.g. "How many bars are there?" + ground: str # ground-truth answer + question_type: str # reliability dimension (e.g. "count", "ordering") + judgments: list[Judgment] + +Judgment + gen_model: str # model that generated the image + eval_model: str # model that evaluated it + analysis: str # chain-of-thought reasoning + answer: str # "1" (correct) or "0" (incorrect) +``` + +All items are serialised to `{output_dir}/{item_id}/{item_id}.json`. + +--- + +## How to Add a New Provider + +```python +# igenbench/utils/llm/llm_caller.py +from .caller_registry import register_caller +from .llm_caller import LLMCaller + +@register_caller("my_provider") +class MyProviderCaller(LLMCaller): + def __init__(self) -> None: + # initialise your SDK / client here + pass + + def generate_image(self, model: str, prompt: str, **kwargs): + # return a PIL.Image.Image + ... + + def understand_image(self, model: str, prompt: str, image_path: str, **kwargs) -> str: + # return the model's text response + ... +``` + +Then call with `--provider my_provider`. + +--- + +## Dev Commands + +```bash +# Install (including dev deps) +uv sync --dev + +# Lint +uv run ruff check . + +# Format +uv run ruff format . + +# Tests (no API keys needed) +uv run pytest tests/ -v + +# Run a single gen (requires GOOGLE_API_KEY) +uv run igenbench gen \ + --info-path hf_datasets/data/1.json \ + --output-dir outputs/ \ + --provider google \ + --model gemini-2.5-flash-image +``` + +--- + +## Output Directory Layout + +``` +outputs/ +└── {item_id}/ + ├── {item_id}.json # VISItem state (generation + evaluation results) + └── {gen_model}/ + └── {item_id}.png # generated image +``` + +--- + +## Common Mistakes to Avoid + +- **Never** add API keys to source files. Use environment variables (see `.env.example`). +- `VISItem.from_dict(path)` takes a **file path**, not a dict. +- `EvalEntry.judgments` is a list — multiple (gen_model, eval_model) pairs can coexist on the same question (multi-model comparison). +- The `Question` dataclass in `vis_item.py` is **deprecated**; use `EvalEntry` instead. +- When editing CLI commands, register them via `igenbench/cli/main.py` imports (side-effect import pattern). diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e5a5345 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,21 @@ +# Changelog + +All notable changes to IGenBench are documented here. +Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/); +versioning follows [PEP 440](https://peps.python.org/pep-0440/). + +--- + +## [0.1.0] — 2026-06 + +### Added +- `igenbench gen` — generate a single infographic from a VISItem JSON +- `igenbench eval` — evaluate a single generated image against benchmark questions +- `igenbench batch-gen` — generate infographics for the full dataset with `--resume` +- `igenbench batch-eval` — evaluate the full dataset with `--resume` +- `igenbench score` — aggregate Q-ACC / I-ACC scores with optional `--by-source` / `--by-type` breakdown +- Provider support: Google (Gemini), OpenRouter, Replicate +- `@register_caller` plugin API for adding custom LLM providers +- `VISItem` data model with full JSON serialisation +- Incremental progress: every question result is saved immediately so interrupted runs resume cleanly +- ACL 2026 camera-ready release diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..7554cd0 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,132 @@ +# Contributing to IGenBench + +Thank you for your interest in contributing! This document covers everything you need to get started. + +--- + +## Prerequisites + +| Tool | Version | Notes | +|------|---------|-------| +| Python | ≥ 3.10 | `pyenv` or system Python | +| [uv](https://docs.astral.sh/uv/) | latest | fast package manager | +| Git | any | | + +--- + +## Dev Setup + +```bash +git clone https://github.com/MisterBrookT/IGenBench.git +cd IGenBench + +# Install all dependencies including dev extras +uv sync --dev + +# Install pre-commit hooks (ruff lint + commitizen) +uv run pre-commit install +``` + +Copy the environment template and fill in your API keys: + +```bash +cp .env.example .env +# Edit .env with your actual keys +``` + +--- + +## Code Style + +We use [Ruff](https://docs.astral.sh/ruff/) for linting and formatting. Pre-commit hooks run it automatically on every commit. + +To run manually: + +```bash +# Lint +uv run ruff check . + +# Format +uv run ruff format . + +# Lint + auto-fix +uv run ruff check --fix . +``` + +--- + +## Running Tests + +Unit tests (no API keys required): + +```bash +uv run pytest tests/ -v +``` + +Integration tests require live API keys and access to `hf_datasets/`. They are not run in CI to avoid incurring costs. + +--- + +## Adding a New LLM Provider + +See the [Adding Custom Models](README.md#adding-custom-models) section in the README. In short: + +1. Implement a `LLMCaller` subclass in `igenbench/utils/llm/llm_caller.py`. +2. Decorate it with `@register_caller("your_provider_name")`. +3. Implement `generate_image` and/or `understand_image`. +4. Add a test in `tests/test_llm_caller.py` that mocks the external API. + +--- + +## Commit Messages + +We follow [Conventional Commits](https://www.conventionalcommits.org/): + +``` +feat: add support for Azure OpenAI provider +fix: handle empty evaluation list in score_cli +docs: clarify batch-eval --resume behaviour +test: add VISItem serialization round-trip test +``` + +The pre-commit hook (commitizen) validates your commit messages automatically. + +--- + +## Pull Request Checklist + +- [ ] `uv run ruff check .` passes with no errors +- [ ] `uv run ruff format --check .` passes +- [ ] `uv run pytest tests/ -v` passes +- [ ] New public functions/classes have docstrings +- [ ] README updated if you added a new CLI flag or provider + +--- + +## Project Structure + +``` +igenbench/ +├── cli/ # Typer CLI commands (gen, eval, batch-gen, batch-eval, score) +├── engine/ # Thin wrappers that execute a single gen/eval step +├── utils/ +│ ├── llm/ # Provider implementations (GoogleCaller, OpenrouterCaller, …) +│ ├── io.py # File I/O helpers +│ ├── path_resolver.py +│ ├── state_manager.py +│ └── error_handler.py +├── workflow/ # Orchestration: batch loops, resume logic +└── vis_item.py # Core data model: VISItem, EvalEntry, Judgment +prompts/ # Prompt templates for generation and evaluation +tests/ # Unit tests (no API keys needed) +``` + +--- + +## Reporting Bugs + +Please use the [Bug Report template](.github/ISSUE_TEMPLATE/bug_report.yml). + +## Requesting Features + +Please use the [Feature Request template](.github/ISSUE_TEMPLATE/feature_request.yml). diff --git a/igenbench/cli/batch_cli.py b/igenbench/cli/batch_cli.py index e22ac6b..93cded4 100644 --- a/igenbench/cli/batch_cli.py +++ b/igenbench/cli/batch_cli.py @@ -6,7 +6,6 @@ from igenbench.cli.main import app from igenbench.utils.logger import logger -from igenbench.vis_item import VISItem from igenbench.workflow.eval_workflow import EvalWorkflow from igenbench.workflow.gen_workflow import GenWorkflow diff --git a/igenbench/cli/eval_cli.py b/igenbench/cli/eval_cli.py index 5121681..6a32ee9 100644 --- a/igenbench/cli/eval_cli.py +++ b/igenbench/cli/eval_cli.py @@ -40,7 +40,7 @@ def cmd_run_evaluation( """ try: workflow = EvalWorkflow(provider, model, output_dir, resume=resume) - + # Load item - use state_manager for resume functionality if resume: item = workflow.state_manager.load_item(info_path, resume=True) @@ -53,7 +53,9 @@ def cmd_run_evaluation( # Save result item.save_item(output_path=output_dir) save_path = item.build_save_path(output_dir) - logger.info(f"✅ Evaluation completed successfully for {item.id}.json, saved to {save_path}") + logger.info( + f"✅ Evaluation completed successfully for {item.id}.json, saved to {save_path}" + ) except FileNotFoundError as e: logger.error(f"❌ File not found: {e}") diff --git a/igenbench/cli/gen_cli.py b/igenbench/cli/gen_cli.py index e409ec5..4c3fd7c 100644 --- a/igenbench/cli/gen_cli.py +++ b/igenbench/cli/gen_cli.py @@ -32,7 +32,7 @@ def cmd_gen( """ try: workflow = GenWorkflow(provider, model, output_dir, resume=resume) - + # Load item - use state_manager for resume functionality if resume: item = workflow.state_manager.load_item(info_path, resume=True) diff --git a/igenbench/cli/score_cli.py b/igenbench/cli/score_cli.py index da380f3..fb931a8 100644 --- a/igenbench/cli/score_cli.py +++ b/igenbench/cli/score_cli.py @@ -101,9 +101,13 @@ def cmd_score( col_w = 60 typer.echo(f"\n{'=' * col_w}") - typer.echo(f"IGenBench Scores ({item_count} items{', ' + str(skipped) + ' skipped' if skipped else ''})") + typer.echo( + f"IGenBench Scores ({item_count} items{', ' + str(skipped) + ' skipped' if skipped else ''})" + ) typer.echo(f"{'=' * col_w}") - typer.echo(f"{'Gen Model':<28} {'Eval Model':<20} {'Accuracy':>9} {'(correct/total)':>15}") + typer.echo( + f"{'Gen Model':<28} {'Eval Model':<20} {'Accuracy':>9} {'(correct/total)':>15}" + ) typer.echo(f"{'-' * col_w}") for (gm, em), (correct, total) in sorted(overall.items()): @@ -120,9 +124,7 @@ def cmd_score( typer.echo(f" {'[' + src + ']':<26} {'':<20} {a:>8.1%} ({c}/{t})") if by_type: - qtypes = sorted( - {k[2] for k in by_type_scores if k[0] == gm and k[1] == em} - ) + qtypes = sorted({k[2] for k in by_type_scores if k[0] == gm and k[1] == em}) for qt in qtypes: c, t = by_type_scores[(gm, em, qt)] a = c / t if t else 0.0 diff --git a/igenbench/engine/eval_engine.py b/igenbench/engine/eval_engine.py index 6d6d26a..e8aeb13 100644 --- a/igenbench/engine/eval_engine.py +++ b/igenbench/engine/eval_engine.py @@ -30,13 +30,13 @@ def judge_entry( image_path: str, ) -> Judgment: """Judge an evaluation entry by calling LLM with the image. - + Args: eval_entry: Evaluation entry containing the question gen_model: Name of the model that generated the image eval_model: Name of the model performing evaluation image_path: Path to the generated image - + Returns: Judgment object with analysis and answer """ diff --git a/igenbench/engine/gen_engine.py b/igenbench/engine/gen_engine.py index 3719b3d..b5799a2 100644 --- a/igenbench/engine/gen_engine.py +++ b/igenbench/engine/gen_engine.py @@ -11,16 +11,16 @@ def __init__(self, llm_client: LLMClient, model: str): def text2image(self, item: VISItem): """Generate image from text prompt. - + Args: item: VISItem containing t2i_prompt - + Returns: PIL Image object """ if not item.t2i_prompt: raise ValueError("Generation requires t2i_prompt to be set") - + prompt = item.t2i_prompt response = self.llm_client.call_image_generation( model=self.model, prompt=prompt diff --git a/igenbench/utils/io.py b/igenbench/utils/io.py index 412746d..b3e0293 100644 --- a/igenbench/utils/io.py +++ b/igenbench/utils/io.py @@ -105,5 +105,3 @@ def split_senmantic_and_data_in_t2i_prompt(t2i_prompt: str) -> tuple[str, str]: Split the semantic and data parts of the T2I prompt. """ return t2i_prompt.split("The given data is:") - - diff --git a/igenbench/utils/llm/client.py b/igenbench/utils/llm/client.py index af39f07..1d83724 100644 --- a/igenbench/utils/llm/client.py +++ b/igenbench/utils/llm/client.py @@ -18,7 +18,9 @@ def _get_caller(self, provider: str) -> LLMCaller: ) return CALLER_REGISTRY[provider]() - def call_text_generation(self, model: str, prompt: str, **kwargs: Any) -> Union[dict, str]: + def call_text_generation( + self, model: str, prompt: str, **kwargs: Any + ) -> Union[dict, str]: text_response = self._caller.generate_text(model, prompt, **kwargs) return extract_from_markdown(text_response) diff --git a/igenbench/utils/llm/llm_caller.py b/igenbench/utils/llm/llm_caller.py index 0757a47..3741bbe 100644 --- a/igenbench/utils/llm/llm_caller.py +++ b/igenbench/utils/llm/llm_caller.py @@ -25,6 +25,7 @@ def _get_mime_type(image_path: str) -> str: def encode_image_to_base64(image_path: str) -> str: import base64 + with open(image_path, "rb") as image_file: return base64.b64encode(image_file.read()).decode("utf-8") @@ -32,6 +33,7 @@ def encode_image_to_base64(image_path: str) -> str: def base64_to_PILImage(base64_image_url: str) -> PILImage: import base64 from io import BytesIO + base64_data = base64_image_url.split(",")[1] image_bytes = base64.b64decode(base64_data) pil_image = Image.open(BytesIO(image_bytes)) @@ -72,7 +74,9 @@ def understand_image( response = self._client.models.generate_content( model=model, contents=[ - types.Part.from_bytes(data=image_bytes, mime_type=_get_mime_type(image_path)), + types.Part.from_bytes( + data=image_bytes, mime_type=_get_mime_type(image_path) + ), prompt, ], ) @@ -157,6 +161,7 @@ def __init__(self) -> None: def generate_text(self, model: str, prompt: str, **kwargs: Any) -> str: import replicate + input = {"prompt": prompt} output = replicate.run(model, input=input) return "".join(output) diff --git a/igenbench/vis_item.py b/igenbench/vis_item.py index 694ba0b..e4a7830 100644 --- a/igenbench/vis_item.py +++ b/igenbench/vis_item.py @@ -154,7 +154,7 @@ class VISItem: reference_image_url: Optional[str] = None t2i_prompt: Optional[str] = None chart_type: Optional[str] = None - + # generation result: {model_name: image_path} generation: Optional[dict] = field(default_factory=dict) @@ -172,11 +172,13 @@ def check_generation_exists(self, model: str) -> bool: def get_evaluation_by_source(self, source: str) -> List[EvalEntry]: """Get evaluation entries filtered by source (prompt or seed).""" - return [entry for entry in self.evaluation if hasattr(entry, 'source') and entry.source == source] + return [ + entry + for entry in self.evaluation + if hasattr(entry, "source") and entry.source == source + ] - def check_evaluation_complete( - self, gen_model: str, eval_model: str - ) -> bool: + def check_evaluation_complete(self, gen_model: str, eval_model: str) -> bool: """Check if all questions have been fully evaluated for the given models.""" if not self.evaluation: return False @@ -207,12 +209,14 @@ def from_dict(cls, info_path: str) -> "VISItem": kwargs["chart_type"] = info_data["chart_type"] if "generation" in info_data: kwargs["generation"] = info_data["generation"] - + # Handle evaluation - now a list with source field if "evaluation" in info_data: evaluation = [] for entry in info_data["evaluation"]: - eval_entry = EvalEntry.from_dict(entry) if isinstance(entry, dict) else entry + eval_entry = ( + EvalEntry.from_dict(entry) if isinstance(entry, dict) else entry + ) # Add source field to eval_entry if it exists in the data if isinstance(entry, dict) and "source" in entry: eval_entry.source = entry["source"] diff --git a/igenbench/workflow/eval_workflow.py b/igenbench/workflow/eval_workflow.py index fbb14bc..d04ea13 100644 --- a/igenbench/workflow/eval_workflow.py +++ b/igenbench/workflow/eval_workflow.py @@ -1,6 +1,5 @@ """Dedicated workflow for running evaluations on generated questions.""" -from typing import List from pathlib import Path from igenbench.workflow.workflow_config import EvalWorkflowConfig @@ -75,7 +74,7 @@ def run( logger.info( f"🔍 Evaluating item {item.id} with {self.eval_model} on {gen_model_name}" ) - + item = self._evaluate_all( item=item, image_path=image_path, diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/sample_item.json b/tests/fixtures/sample_item.json new file mode 100644 index 0000000..ae0db2e --- /dev/null +++ b/tests/fixtures/sample_item.json @@ -0,0 +1,30 @@ +{ + "id": "test-001", + "t2i_prompt": "A bar chart showing quarterly sales: Q1=10, Q2=20, Q3=15, Q4=25.", + "chart_type": "bar", + "reference_image_url": null, + "generation": {}, + "evaluation": [ + { + "source": "prompt", + "ground": "4", + "question": "How many bars are there in the chart?", + "question_type": "count", + "judgments": [] + }, + { + "source": "seed", + "ground": "Q4", + "question": "Which quarter has the highest sales?", + "question_type": "ordering", + "judgments": [ + { + "gen_model": "gemini-2.5-flash-image", + "eval_model": "gemini-2.5-flash", + "analysis": "The bar for Q4 is the tallest, representing 25.", + "answer": "1" + } + ] + } + ] +} diff --git a/tests/test_vis_item.py b/tests/test_vis_item.py new file mode 100644 index 0000000..3ce8d78 --- /dev/null +++ b/tests/test_vis_item.py @@ -0,0 +1,154 @@ +"""Unit tests for VISItem and related data classes. No API keys required.""" + +from pathlib import Path + +from igenbench.vis_item import EvalEntry, Judgment, VISItem + +FIXTURES_DIR = Path(__file__).parent / "fixtures" +SAMPLE_ITEM_PATH = FIXTURES_DIR / "sample_item.json" + + +# ─── Loading ────────────────────────────────────────────────────────────────── + + +def test_from_dict_loads_id(): + item = VISItem.from_dict(str(SAMPLE_ITEM_PATH)) + assert item.id == "test-001" + + +def test_from_dict_loads_prompt(): + item = VISItem.from_dict(str(SAMPLE_ITEM_PATH)) + assert "bar chart" in item.t2i_prompt + + +def test_from_dict_loads_evaluation_entries(): + item = VISItem.from_dict(str(SAMPLE_ITEM_PATH)) + assert len(item.evaluation) == 2 + + +def test_from_dict_eval_entry_fields(): + item = VISItem.from_dict(str(SAMPLE_ITEM_PATH)) + entry = item.evaluation[0] + assert entry.source == "prompt" + assert entry.question_type == "count" + assert entry.ground == "4" + + +def test_from_dict_loads_judgment(): + item = VISItem.from_dict(str(SAMPLE_ITEM_PATH)) + entry = item.evaluation[1] # has one judgment + assert len(entry.judgments) == 1 + j = entry.judgments[0] + assert isinstance(j, Judgment) + assert j.answer == "1" + assert j.gen_model == "gemini-2.5-flash-image" + + +# ─── has_judgment / add_judgment ────────────────────────────────────────────── + + +def test_has_judgment_true(): + item = VISItem.from_dict(str(SAMPLE_ITEM_PATH)) + entry = item.evaluation[1] + assert entry.has_judgment("gemini-2.5-flash-image", "gemini-2.5-flash") + + +def test_has_judgment_false_wrong_gen_model(): + item = VISItem.from_dict(str(SAMPLE_ITEM_PATH)) + entry = item.evaluation[1] + assert not entry.has_judgment("unknown-model", "gemini-2.5-flash") + + +def test_add_judgment_appends(): + entry = EvalEntry( + source="prompt", + ground="2", + question="How many axes?", + question_type="count", + ) + entry.add_judgment("gen-model", "eval-model", "reasoning", "1") + assert len(entry.judgments) == 1 + assert entry.judgments[0].answer == "1" + + +# ─── check_evaluation_complete ──────────────────────────────────────────────── + + +def test_check_evaluation_complete_false_when_missing_judgment(): + item = VISItem.from_dict(str(SAMPLE_ITEM_PATH)) + # evaluation[0] has no judgments + assert not item.check_evaluation_complete( + "gemini-2.5-flash-image", "gemini-2.5-flash" + ) + + +def test_check_evaluation_complete_true_when_all_judged(): + item = VISItem( + id="x", + evaluation=[ + EvalEntry( + judgments=[ + Judgment(gen_model="gm", eval_model="em", analysis="ok", answer="1") + ] + ) + ], + ) + assert item.check_evaluation_complete("gm", "em") + + +def test_check_evaluation_complete_false_empty(): + item = VISItem(id="x", evaluation=[]) + assert not item.check_evaluation_complete("gm", "em") + + +# ─── check_generation_exists ───────────────────────────────────────────────── + + +def test_check_generation_exists_false_when_empty(): + item = VISItem.from_dict(str(SAMPLE_ITEM_PATH)) + assert not item.check_generation_exists("any-model") + + +def test_check_generation_exists_true_after_update(): + item = VISItem(id="x", generation={}) + item.update_generation("my-model", "/path/to/img.png") + assert item.check_generation_exists("my-model") + + +# ─── save / round-trip ──────────────────────────────────────────────────────── + + +def test_save_item_creates_file(tmp_path): + item = VISItem.from_dict(str(SAMPLE_ITEM_PATH)) + item.save_item(str(tmp_path)) + expected = tmp_path / "test-001" / "test-001.json" + assert expected.exists() + + +def test_save_item_direct_path(tmp_path): + item = VISItem.from_dict(str(SAMPLE_ITEM_PATH)) + out = tmp_path / "out.json" + item.save_item(str(out)) + assert out.exists() + + +def test_round_trip_preserves_data(tmp_path): + item = VISItem.from_dict(str(SAMPLE_ITEM_PATH)) + item.save_item(str(tmp_path)) + saved_path = tmp_path / "test-001" / "test-001.json" + reloaded = VISItem.from_dict(str(saved_path)) + assert reloaded.id == item.id + assert len(reloaded.evaluation) == len(item.evaluation) + assert reloaded.evaluation[1].judgments[0].answer == "1" + + +# ─── get_evaluation_by_source ──────────────────────────────────────────────── + + +def test_get_evaluation_by_source(): + item = VISItem.from_dict(str(SAMPLE_ITEM_PATH)) + prompt_entries = item.get_evaluation_by_source("prompt") + seed_entries = item.get_evaluation_by_source("seed") + assert len(prompt_entries) == 1 + assert len(seed_entries) == 1 + assert prompt_entries[0].source == "prompt" diff --git a/uv.lock b/uv.lock index 7fae636..de4a401 100644 --- a/uv.lock +++ b/uv.lock @@ -349,6 +349,50 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, ] +[[package]] +name = "igenbench" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "google-genai" }, + { name = "openai" }, + { name = "pillow" }, + { name = "requests" }, + { name = "typer" }, +] + +[package.optional-dependencies] +replicate = [ + { name = "replicate" }, +] + +[package.dev-dependencies] +dev = [ + { name = "commitizen" }, + { name = "pre-commit" }, + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "google-genai", specifier = ">=1.49.0" }, + { name = "openai", specifier = ">=2.7.1" }, + { name = "pillow", specifier = ">=12.0.0" }, + { name = "replicate", marker = "extra == 'replicate'", specifier = ">=0.34.0" }, + { name = "requests", specifier = ">=2.31.0" }, + { name = "typer", specifier = ">=0.20.0" }, +] +provides-extras = ["replicate"] + +[package.metadata.requires-dev] +dev = [ + { name = "commitizen", specifier = ">=4.10.0" }, + { name = "pre-commit", specifier = ">=4.5.0" }, + { name = "pytest", specifier = ">=9.0.1" }, + { name = "ruff", specifier = ">=0.14.6" }, +] + [[package]] name = "iniconfig" version = "2.3.0" @@ -573,41 +617,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] -[[package]] -name = "nanochart" -version = "0.1.0" -source = { editable = "." } -dependencies = [ - { name = "google-genai" }, - { name = "openai" }, - { name = "pillow" }, - { name = "typer" }, -] - -[package.dev-dependencies] -dev = [ - { name = "commitizen" }, - { name = "pre-commit" }, - { name = "pytest" }, - { name = "ruff" }, -] - -[package.metadata] -requires-dist = [ - { name = "google-genai", specifier = ">=1.49.0" }, - { name = "openai", specifier = ">=2.7.1" }, - { name = "pillow", specifier = ">=12.0.0" }, - { name = "typer", specifier = ">=0.20.0" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "commitizen", specifier = ">=4.10.0" }, - { name = "pre-commit", specifier = ">=4.5.0" }, - { name = "pytest", specifier = ">=9.0.1" }, - { name = "ruff", specifier = ">=0.14.6" }, -] - [[package]] name = "nodeenv" version = "1.9.1" @@ -1046,6 +1055,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3c/26/1062c7ec1b053db9e499b4d2d5bc231743201b74051c973dadeac80a8f43/questionary-2.1.1-py3-none-any.whl", hash = "sha256:a51af13f345f1cdea62347589fbb6df3b290306ab8930713bfae4d475a7d4a59", size = 36753, upload-time = "2025-08-28T19:00:19.56Z" }, ] +[[package]] +name = "replicate" +version = "1.0.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx" }, + { name = "packaging" }, + { name = "pydantic" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4b/fd/caf6c59a6b8007366bd52ab5a320bf8d828f3860a60039309cfc0e375ec9/replicate-1.0.7.tar.gz", hash = "sha256:d88cb2c37ba39fb370c87fc3291601c67aae64bb918a20a85b5ce399c23ee84c", size = 62226, upload-time = "2025-05-27T11:29:08.111Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/5a/b3aa02a11a33de08e7771579154af3193decfb9d923b30b14c17b4e8bbce/replicate-1.0.7-py3-none-any.whl", hash = "sha256:667c50a9eb83be17de6278ff89483102b3b50f49a2c7fbcaa2e2b14df13816f9", size = 48626, upload-time = "2025-05-27T11:29:06.801Z" }, +] + [[package]] name = "requests" version = "2.32.5"