diff --git a/Makefile b/Makefile index c5c751c..184ecd2 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ # statt abgewiesen. # make setup Nur die lokale Umgebung anlegen/aktualisieren. # make generate KI-Aufgaben generieren (braucht einen API-Schluessel, siehe README). +# make export Ein Set fuer KI-Review exportieren (ARGS=" [--split-size N] ..."). # make audit Ueberblick ueber deine Inhalte ausgeben. # make clean Die lokale Umgebung entfernen. # @@ -38,7 +39,7 @@ PIP := $(VENV)/bin/pip ENGINE_PIN := $(shell cat schema/engine-version.txt) ENGINE_STAMP := node_modules/.engine-$(ENGINE_PIN) -.PHONY: validate lint lint-warnings setup generate audit clean help +.PHONY: validate lint lint-warnings setup generate export audit clean help help: @echo "make validate - Inhalte pruefen (richtet sich beim ersten Mal selbst ein)" @@ -46,6 +47,7 @@ help: @echo "make lint-warnings - derselbe Lauf, zusätzlich mit Warnungen (W-*)" @echo "make setup - lokale Umgebung anlegen" @echo "make generate - KI-Aufgaben generieren (API-Schluessel noetig; ARGS=\"--topic ...\")" + @echo "make export - Set fuer KI-Review exportieren (ARGS=\" [--split-size N] ...\")" @echo "make audit - Inhalts-Ueberblick" @echo "make clean - lokale Umgebung entfernen" @@ -86,6 +88,12 @@ lint-warnings: $(ENGINE_STAMP) generate: $(VENV)/.ready @$(PY) scripts/generate_exercises.py $(ARGS) +# Ein Set fuer KI-Review exportieren, z. B.: +# make export ARGS="" +# make export ARGS=" --split-size 5" +export: $(VENV)/.ready + @$(PY) scripts/export_set.py $(ARGS) + audit: $(VENV)/.ready @$(PY) scripts/audit_content.py diff --git a/README.md b/README.md index 0e87397..7b4a745 100644 --- a/README.md +++ b/README.md @@ -134,11 +134,19 @@ Full walkthrough: [docs/GETTING-STARTED.md](docs/GETTING-STARTED.md). `scripts/export_set.py` writes all lessons of ONE set into a single YAML (or JSON) file so an AI assistant or a human can review the whole -set in one pass (syntax, correctness, consistency across lessons): +set in one pass (syntax, correctness, consistency across lessons). + +**Recommended (via make; reuses the local environment `make validate` set up):** ```bash -python3 scripts/export_set.py react-grundlagen +make export ARGS="react-grundlagen" # -> exports/react-grundlagen-de-.yaml +``` + +**Direct (fallback; run it inside the venv from the Quick start):** + +```bash +python3 scripts/export_set.py react-grundlagen python3 scripts/export_set.py react-grundlagen --format json --out /tmp/review.json ``` @@ -149,6 +157,12 @@ source-language directories, `--lang` (default `de`) picks the `sets//` directory. Non-ASCII characters stay real UTF-8. An unknown slug aborts with a list of the available sets. +For a large set, `--split-size N` writes multiple self-contained files +of at most N lessons each instead of one huge file, e.g. +`make export ARGS="python-basics --split-size 5"` (each part keeps its +own `review_instructions` copy, so any one file can be handed to an AI +on its own). Cannot be combined with `--out`. + The export is self-contained: its first field `review_instructions` holds the complete review prompt from [`docs/ai-review-prompt-template.md`](docs/ai-review-prompt-template.md) diff --git a/docs/export-set-usage.de.md b/docs/export-set-usage.de.md index 0b79c5d..3a2892a 100644 --- a/docs/export-set-usage.de.md +++ b/docs/export-set-usage.de.md @@ -20,7 +20,9 @@ Lektions-JSONs unter `sets/` ein. ## Nutzung ```bash -python3 scripts/export_set.py [--lang ] [--format yaml|json] [--out PFAD] +make export ARGS=" [--lang ] [--format yaml|json] [--out PFAD] [--split-size N]" +# oder direkt (Fallback; innerhalb der venv aus dem Quick Start): +python3 scripts/export_set.py [--lang ] [--format yaml|json] [--out PFAD] [--split-size N] ``` | Argument | Bedeutung | Default | @@ -28,23 +30,37 @@ python3 scripts/export_set.py [--lang ] [--format yaml|json] [- | `` | Set-Id aus dem Wurzel-`manifest.yaml` (z. B. `react-grundlagen-from-de`) oder der Ordnername des Set-Pfads (z. B. `react-grundlagen` für `sets/de/react-grundlagen`) | Pflicht | | `--lang` | Quellsprachen-Verzeichnis (`sets//`), das einen Ordnernamen-Slug eindeutig macht, der unter mehreren Quellsprachen existiert | `de` | | `--format` | Ausgabeformat: `yaml` oder `json` | `yaml` | -| `--out` | Pfad der Ausgabedatei | `exports/--.` | +| `--out` | Pfad der Ausgabedatei (nicht kombinierbar mit `--split-size`) | `exports/--.` | +| `--split-size` | Export in mehrere Dateien von je hoechstens N Lektionen aufteilen, statt einer Datei | aus (eine Datei) | Beispiele: ```bash # Standardfall: YAML-Export nach exports/ (die Sets liegen unter sets/de/) -python3 scripts/export_set.py react-grundlagen +make export ARGS="react-grundlagen" # -> exports/react-grundlagen-de-.yaml # Sonderfall: JSON an einen eigenen Pfad (nur wenn ein Tooling explizit JSON braucht) -python3 scripts/export_set.py react-grundlagen --format json --out /tmp/review.json +make export ARGS="react-grundlagen --format json --out /tmp/review.json" + +# Grosses Set: in Teile von je hoechstens 5 Lektionen aufteilen, fuer eine +# KI mit begrenztem Kontextfenster +make export ARGS="python-basics --split-size 5" +# -> exports/python-basics-de--part01-of-3.yaml, part02-of-3, part03-of-3 ``` Ohne `--out` landet die Datei in `exports/` nach dem Muster -`--.`. Das Verzeichnis `exports/` -wird bei Bedarf angelegt und ist **gitignored**: Exportdateien sind -Wegwerf-Artefakte fürs Review und werden nie committet. +`--.` (bei `--split-size` je eine +Datei pro Teil nach `---partNN-of-MM.`). +Das Verzeichnis `exports/` wird bei Bedarf angelegt und ist +**gitignored**: Exportdateien sind Wegwerf-Artefakte fürs Review und +werden nie committet. + +Jeder von `--split-size` geschriebene Teil ist eigenstaendig: er traegt +seine eigene `review_instructions`-Kopie sowie die Felder +`part`/`of`/`lesson_count`/`total_lesson_count`, sodass jeder einzelne +Teil fuer sich, in beliebiger Reihenfolge, an eine KI zum Review +gegeben werden kann. Ein unbekannter oder mehrdeutiger Slug bricht mit Exit-Code 2 und einer Liste der verfügbaren Sets ab. Umlaute und alle anderen @@ -55,7 +71,7 @@ Nicht-ASCII-Zeichen bleiben echtes UTF-8. 1. **Export erzeugen:** ```bash - python3 scripts/export_set.py react-grundlagen + make export ARGS="react-grundlagen" ``` 2. **Exportdatei öffnen** und im `review_instructions`-Block am Anfang @@ -100,7 +116,8 @@ Nicht-ASCII-Zeichen bleiben echtes UTF-8. - **Quellkapitel bei jedem Review neu einfügen**, wenn es sich geändert hat; nicht aus einem alten Export kopieren. - **Große Sets in Portionen prüfen** (z. B. 8-10 Lektionen pro - Durchgang), wenn der Kontext der verwendeten KI begrenzt ist. + Durchgang), wenn der Kontext der verwendeten KI begrenzt ist - dafür + `--split-size` nutzen, statt den Export von Hand zu zerschneiden. - **YAML als Standard belassen**; JSON nur, wenn ein Tooling das explizit braucht. - **Kein Copy-Paste von KI-Vorschlägen ohne Gegenlesen.** Die KI diff --git a/docs/export-set-usage.md b/docs/export-set-usage.md index 6f51e46..f81efdc 100644 --- a/docs/export-set-usage.md +++ b/docs/export-set-usage.md @@ -20,7 +20,9 @@ JSON files under `sets/`. ## Usage ```bash -python3 scripts/export_set.py [--lang ] [--format yaml|json] [--out PATH] +make export ARGS=" [--lang ] [--format yaml|json] [--out PATH] [--split-size N]" +# or, direct (fallback; run it inside the venv from the Quick start): +python3 scripts/export_set.py [--lang ] [--format yaml|json] [--out PATH] [--split-size N] ``` | Argument | Meaning | Default | @@ -28,24 +30,37 @@ python3 scripts/export_set.py [--lang ] [--format yaml|json] [- | `` | Set id from the root `manifest.yaml` (e.g. `react-grundlagen-from-de`) or the folder name of the set path (e.g. `react-grundlagen` for `sets/de/react-grundlagen`) | required | | `--lang` | Source-language directory (`sets//`) that disambiguates a folder-name slug existing under several source languages | `de` | | `--format` | Output format: `yaml` or `json` | `yaml` | -| `--out` | Output file path | `exports/--.` | +| `--out` | Output file path (cannot be combined with `--split-size`) | `exports/--.` | +| `--split-size` | Split the export into multiple files of at most N lessons each, instead of one file | off (one file) | Examples: ```bash # Standard case: YAML export into exports/ (the sets live under sets/de/) -python3 scripts/export_set.py react-grundlagen +make export ARGS="react-grundlagen" # -> exports/react-grundlagen-de-.yaml # Special case: JSON to a custom path (only when a tool explicitly needs JSON) -python3 scripts/export_set.py react-grundlagen --format json --out /tmp/review.json +make export ARGS="react-grundlagen --format json --out /tmp/review.json" + +# Large set: split into parts of at most 5 lessons each for an AI with a +# limited context window +make export ARGS="python-basics --split-size 5" +# -> exports/python-basics-de--part01-of-3.yaml, part02-of-3, part03-of-3 ``` Without `--out`, the file is written to `exports/` following the -pattern `--.`. The `exports/` +pattern `--.` (or, with +`--split-size`, one file per part following +`---partNN-of-MM.`). The `exports/` directory is created on demand and is **gitignored**: export files are throwaway review artifacts and are never committed. +Each part written by `--split-size` is self-contained: it carries its +own `review_instructions` copy plus `part`/`of`/`lesson_count`/ +`total_lesson_count` fields, so any single part can be handed to an AI +reviewer on its own, in any order, without the others. + An unknown or ambiguous slug aborts with exit code 2 and a list of the available sets. Umlauts and all other non-ASCII characters stay real UTF-8. @@ -55,7 +70,7 @@ UTF-8. 1. **Create the export:** ```bash - python3 scripts/export_set.py react-grundlagen + make export ARGS="react-grundlagen" ``` 2. **Open the export file** and find the section "Quellkapitel" @@ -99,7 +114,8 @@ UTF-8. - **Re-insert the source chapter for every review** when it has changed; do not copy it out of an old export. - **Review large sets in slices** (e.g. 8-10 lessons per pass) when - the AI you use has a limited context window. + the AI you use has a limited context window - use `--split-size` + instead of manually cutting the export down. - **Keep YAML as the default**; use JSON only when a tool explicitly requires it. - **No copy-paste of AI suggestions without cross-reading.** The AI diff --git a/scripts/export_set.py b/scripts/export_set.py index bc4afb8..cf5bb5c 100644 --- a/scripts/export_set.py +++ b/scripts/export_set.py @@ -13,6 +13,17 @@ Usage: python3 scripts/export_set.py [--lang de] [--format yaml|json] [--out PATH] + python3 scripts/export_set.py --split-size N [--lang de] [--format yaml|json] + +``--split-size N`` splits a large set into multiple files of at most N +lessons each (e.g. for handing a course to an AI reviewer one digestible +chunk at a time instead of one huge file - a 115-lesson set has no business +being reviewed in one context window). Each part is self-contained: it +carries its own ``review_instructions`` copy plus ``part``/``of`` and +``total_lesson_count`` fields so a reviewer knows where the chunk sits in +the whole set. Incompatible with ``--out`` (a split export always writes +multiple files under ``exports/``, named +``---part-of-.``). ```` matches either a set id from the root manifest (e.g. ``fuehrerschein-uebung-from-de``) or the basename of a set path (e.g. @@ -212,6 +223,52 @@ def build_export(set_slug: str, lang: str) -> dict: } +def chunk_lessons(lesson_documents: list[dict], split_size: int) -> list[list[dict]]: + """Split ``lesson_documents`` into consecutive chunks of at most + ``split_size`` lessons each, preserving order. The last chunk may be + smaller. A ``split_size`` at or above the lesson count yields a single + chunk. Pure and independent of I/O so it is trivially unit-testable.""" + if split_size < 1: + raise ValueError(f"split_size must be >= 1, got {split_size}") + return [ + lesson_documents[start : start + split_size] + for start in range(0, len(lesson_documents), split_size) + ] or [[]] + + +def build_export_parts(set_slug: str, lang: str, split_size: int) -> list[dict]: + """Assemble one export payload PER CHUNK of at most ``split_size`` + lessons, for AI review sessions that cannot fit an entire large set in + one context window (e.g. a 115-lesson set). Every part is + self-contained: it carries its own copy of ``review_instructions`` so + each file can be handed to a reviewer independently, plus ``part``/``of`` + and ``total_lesson_count`` so the reviewer knows where a chunk sits in + the whole set. All parts share one ``generated_at`` timestamp (one + export run).""" + root_manifest = yaml.safe_load(ROOT_MANIFEST_PATH.read_text(encoding="utf-8")) + set_entry = resolve_set(root_manifest, set_slug, lang) + lesson_documents = load_lessons(REPO_ROOT / set_entry["path"]) + generated_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + engine_version = ENGINE_VERSION_PATH.read_text(encoding="utf-8").strip() + review_instructions = load_review_instructions() + chunks = chunk_lessons(lesson_documents, split_size) + return [ + { + "review_instructions": review_instructions, + "set": set_slug, + "language": lang, + "engine_version": engine_version, + "generated_at": generated_at, + "part": part_index + 1, + "of": len(chunks), + "lesson_count": len(chunk), + "total_lesson_count": len(lesson_documents), + "lessons": chunk, + } + for part_index, chunk in enumerate(chunks) + ] + + def render_export(export_payload: dict, export_format: str) -> str: """Serialize the payload; real UTF-8 in both formats, keys in insert order, multi-line strings (notably ``review_instructions``) as YAML block scalars.""" @@ -233,6 +290,20 @@ def default_output_path(set_slug: str, lang: str, export_format: str) -> Path: return EXPORTS_DIR / f"{set_slug}-{lang}-{file_timestamp}.{export_format}" +def default_split_output_path( + set_slug: str, lang: str, export_format: str, part: int, of: int, file_timestamp: str +) -> Path: + """Return ``exports/---part-of-.``, + one file per chunk of a ``--split-size`` export. ``file_timestamp`` is + computed ONCE by the caller and shared across every part of one export + run, so the parts sort together and are recognisable as one batch.""" + width = max(2, len(str(of))) + return ( + EXPORTS_DIR + / f"{set_slug}-{lang}-{file_timestamp}-part{part:0{width}d}-of-{of}.{export_format}" + ) + + def parse_arguments(argv: list[str] | None) -> argparse.Namespace: argument_parser = argparse.ArgumentParser( description=( @@ -259,13 +330,23 @@ def parse_arguments(argv: list[str] | None) -> argparse.Namespace: ) argument_parser.add_argument( "--out", - help="output file path (default: exports/--.)", + help="output file path (default: exports/--.). " + "Incompatible with --split-size (which writes multiple files).", + ) + argument_parser.add_argument( + "--split-size", + type=int, + default=None, + help="split the export into multiple files of at most N lessons each, " + "e.g. for AI review sessions that cannot fit a large set in one " + "context window. Each file is self-contained (its own " + "review_instructions) and carries part/of/total_lesson_count. " + "Default: one file for the whole set. Incompatible with --out.", ) return argument_parser.parse_args(argv) -def main(argv: list[str] | None = None) -> int: - cli_arguments = parse_arguments(argv) +def _write_single_export(cli_arguments: argparse.Namespace) -> int: try: export_payload = build_export(cli_arguments.set_slug, cli_arguments.lang) except SetResolutionError as resolution_error: @@ -290,5 +371,51 @@ def main(argv: list[str] | None = None) -> int: return 0 +def _write_split_export(cli_arguments: argparse.Namespace) -> int: + try: + export_parts = build_export_parts( + cli_arguments.set_slug, cli_arguments.lang, cli_arguments.split_size + ) + except (SetResolutionError, ValueError) as build_error: + print(f"ERROR: {build_error}", file=sys.stderr) + return 2 + + EXPORTS_DIR.mkdir(parents=True, exist_ok=True) + file_timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + of = len(export_parts) + for export_payload in export_parts: + output_path = default_split_output_path( + cli_arguments.set_slug, + cli_arguments.lang, + cli_arguments.export_format, + export_payload["part"], + of, + file_timestamp, + ) + output_path.write_text( + render_export(export_payload, cli_arguments.export_format), encoding="utf-8" + ) + print( + f"Exported part {export_payload['part']}/{of} " + f"({export_payload['lesson_count']} of {export_payload['total_lesson_count']} " + f"lessons) of set '{export_payload['set']}' to {output_path}" + ) + return 0 + + +def main(argv: list[str] | None = None) -> int: + cli_arguments = parse_arguments(argv) + if cli_arguments.split_size is not None and cli_arguments.out: + print( + "ERROR: --out cannot be combined with --split-size " + "(a split export writes multiple files)", + file=sys.stderr, + ) + return 2 + if cli_arguments.split_size is not None: + return _write_split_export(cli_arguments) + return _write_single_export(cli_arguments) + + if __name__ == "__main__": sys.exit(main()) diff --git a/tests/test_export_set.py b/tests/test_export_set.py index e2a4652..7c7ab08 100644 --- a/tests/test_export_set.py +++ b/tests/test_export_set.py @@ -15,7 +15,10 @@ * ``--format json`` produces valid JSON with the same lesson content, * the default output path lands under ``exports/`` (created on demand), * the manifest set id (``react-grundlagen-from-de``) resolves the same - set as the path basename slug (``react-grundlagen``). + set as the path basename slug (``react-grundlagen``), +* ``--split-size`` splits a large set's export into multiple + self-contained files, each carrying its own ``review_instructions`` + copy plus ``part``/``of``/``lesson_count``/``total_lesson_count``. """ from __future__ import annotations @@ -23,6 +26,7 @@ import sys from pathlib import Path +import pytest import yaml REPO_ROOT = Path(__file__).resolve().parents[1] @@ -215,3 +219,107 @@ def test_missing_review_template_fails_with_clear_error( captured_stderr = capsys.readouterr().err assert "ai-review-prompt-template" in captured_stderr or "not-there.md" in captured_stderr assert not out_path.exists() + + +def test_chunk_lessons_splits_into_consecutive_groups_preserving_order() -> None: + lessons = [{"id": f"l{i}"} for i in range(8)] + chunks = export_set.chunk_lessons(lessons, 5) + assert chunks == [lessons[0:5], lessons[5:8]] + + +def test_chunk_lessons_split_size_at_or_above_count_yields_one_chunk() -> None: + lessons = [{"id": f"l{i}"} for i in range(3)] + assert export_set.chunk_lessons(lessons, 5) == [lessons] + assert export_set.chunk_lessons(lessons, 3) == [lessons] + + +def test_chunk_lessons_rejects_non_positive_split_size() -> None: + with pytest.raises(ValueError): + export_set.chunk_lessons([{"id": "l0"}], 0) + + +EXPORTS_DIR = REPO_ROOT / "exports" + + +def run_split_export(tmp_path: Path, split_size: int, *extra_argv: str) -> list[Path]: + """Run a split export for the known set and return the written files, + sorted by name (which sorts by part number thanks to zero-padding).""" + before = set(EXPORTS_DIR.glob("*")) if EXPORTS_DIR.is_dir() else set() + exit_code = export_set.main( + [KNOWN_SLUG, "--split-size", str(split_size), *extra_argv] + ) + assert exit_code == 0 + created = sorted((set(EXPORTS_DIR.glob("*")) - before)) + for created_path in created: + tmp_path_copy = tmp_path / created_path.name + tmp_path_copy.write_bytes(created_path.read_bytes()) + created_path.unlink() + return sorted(tmp_path.glob("*")) + + +def test_split_size_writes_one_file_per_chunk_with_correct_lesson_counts( + tmp_path: Path, +) -> None: + source_lessons = load_source_lessons() + assert len(source_lessons) == 5 # KNOWN_SLUG fixture assumption + + created_files = run_split_export(tmp_path, 3) + assert len(created_files) == 2 + + part_1 = yaml.safe_load(created_files[0].read_text(encoding="utf-8")) + part_2 = yaml.safe_load(created_files[1].read_text(encoding="utf-8")) + assert part_1["lesson_count"] == 3 + assert part_2["lesson_count"] == 2 + assert part_1["total_lesson_count"] == 5 + assert part_2["total_lesson_count"] == 5 + assert part_1["part"] == 1 + assert part_1["of"] == 2 + assert part_2["part"] == 2 + assert part_2["of"] == 2 + + +def test_split_size_parts_concatenate_to_the_full_lesson_order(tmp_path: Path) -> None: + source_lessons = load_source_lessons() + created_files = run_split_export(tmp_path, 2) + + reassembled: list[dict] = [] + for created_path in created_files: + payload = yaml.safe_load(created_path.read_text(encoding="utf-8")) + reassembled.extend(payload["lessons"]) + assert reassembled == source_lessons + + +def test_split_size_each_part_is_self_contained_with_review_instructions( + tmp_path: Path, +) -> None: + created_files = run_split_export(tmp_path, 3) + template_text = REVIEW_TEMPLATE_PATH.read_text(encoding="utf-8") + for created_path in created_files: + payload = yaml.safe_load(created_path.read_text(encoding="utf-8")) + assert payload["review_instructions"] == template_text + + +def test_split_size_at_or_above_lesson_count_yields_single_file(tmp_path: Path) -> None: + created_files = run_split_export(tmp_path, 100) + assert len(created_files) == 1 + payload = yaml.safe_load(created_files[0].read_text(encoding="utf-8")) + assert payload["part"] == 1 + assert payload["of"] == 1 + assert payload["lesson_count"] == payload["total_lesson_count"] + + +def test_split_size_combined_with_out_is_a_usage_error(tmp_path: Path, capsys) -> None: + exit_code = export_set.main( + [ + KNOWN_SLUG, + "--split-size", + "5", + "--out", + str(tmp_path / "never-written.yaml"), + ] + ) + assert exit_code != 0 + captured_stderr = capsys.readouterr().err + assert "--split-size" in captured_stderr + assert "--out" in captured_stderr + assert not (tmp_path / "never-written.yaml").exists()