diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a1badba..32ed39f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing -Thanks for helping improve the DocLang standard and reference validator. +Thanks for helping improve the DocLang standard and reference toolkit. ## Prerequisites @@ -20,7 +20,7 @@ CI installs only the `ci` group (`uv sync --frozen --no-default-groups --group c ## Repository layout - **`spec.md`** — normative specification -- **`doclang/`** — reference validator (XSD, Schematron, CLI); see [doclang/README.md](./doclang/README.md) for package usage +- **`doclang/`** — reference toolkit (Python package, CLI); see [doclang/README.md](./doclang/README.md) for usage - **`reference/`** — source data for Appendix A (Excel, examples) - **`exports/`** — generated Word exports from `spec.md` - **`utils/`** — maintenance scripts (version sync, reference generation, DOCX export, release preparation) diff --git a/README.md b/README.md index ac8c772..92d3cda 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ **[DocLang](https://www.doclang.ai/) is the AI-native markup format for unstructured content** — including documents, images, and more. It maps cleanly to LLM tokens while preserving structure, semantics, layout, and geometry in a single, unambiguous representation. -This repository is the home of the normative specification and the reference validator for DocLang. If you build with LLMs and VLMs on real-world content, this is where the standard lives. +This repository is the home of the normative specification and the reference toolkit for DocLang. If you build with LLMs and VLMs on real-world content, this is where the standard lives. ## Specification @@ -24,20 +24,26 @@ The source of the specification is available in [spec.md](https://github.com/doc and exports to different formats can be found in the [exports/](https://github.com/doclang-project/doclang/tree/main/exports) directory. -## Reference Validator +## Reference Toolkit -You can install the validator from PyPI: +You can install the toolkit from PyPI: ```bash pip install doclang ``` -You can then validate a DocLang document as follows: +### Validation ```bash doclang validate -n my_document.dclg ``` +### Packaging + +```bash +doclang pack my_document.dclg +``` + For more details, see the [doclang/README.md](https://github.com/doclang-project/doclang/blob/main/doclang/README.md). ## Citation diff --git a/doclang/README.md b/doclang/README.md index baa318c..7ba6530 100644 --- a/doclang/README.md +++ b/doclang/README.md @@ -1,6 +1,6 @@ -# DocLang Validation +# DocLang Toolkit -Validate DocLang XML documents against XSD schema and Schematron rules. +Official Python toolkit for working with DocLang — CLI commands and library APIs. ## Installation @@ -8,15 +8,15 @@ Validate DocLang XML documents against XSD schema and Schematron rules. pip install doclang ``` -## Usage +## CLI -### Basic CLI Usage +### Validation ```bash doclang validate my_document.dclg ``` -### More CLI Usage Scenarios +#### More validation scenarios ```bash ## Inject DocLang namespace if document doesn't declare it: @@ -38,7 +38,26 @@ doclang validate my_document.dclg --quiet doclang --help ``` -### Python API +### Packaging + +```bash +doclang pack markup.dclg +``` + +#### More packaging scenarios + +```bash +doclang pack markup.dclg -o report.dclx +doclang pack markup.dclg --pages screenshots/ +doclang pack markup.dclg --page a.png --page b.png +doclang pack markup.dclg --asset chart.svg=exports/diagram.svg +doclang pack markup.dclg --assets payload/ +doclang pack markup.dclg --validate +``` + +## Python API + +### Validation ```python from doclang import validate, ValidationError @@ -52,6 +71,19 @@ except ValidationError as exc: print(f"{exc.schematron_errors=}") ``` +### Packaging + +```python +from doclang import pack, PackagingError + +path = pack( + "markup.dclg", + pages="screenshots/", + assets={"chart.svg": "exports/diagram.svg"}, +) +print(f"Created {path}") +``` + ## Validation Rules ### XSD Validation (doclang.xsd) diff --git a/doclang/__init__.py b/doclang/__init__.py index cbb74da..eff3a59 100644 --- a/doclang/__init__.py +++ b/doclang/__init__.py @@ -1,5 +1,6 @@ -"""DocLang reference validator.""" +"""DocLang reference toolkit.""" +from doclang.packaging import PackagingError, pack from doclang.validation import ValidationError, validate -__all__ = ["ValidationError", "validate"] +__all__ = ["PackagingError", "ValidationError", "pack", "validate"] diff --git a/doclang/_packaging.py b/doclang/_packaging.py new file mode 100644 index 0000000..2ce23a9 --- /dev/null +++ b/doclang/_packaging.py @@ -0,0 +1,184 @@ +"""Internal implementation for DocLang archive packaging.""" + +from __future__ import annotations + +import shutil +import tempfile +import zipfile +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Union + +_CONTENT_TYPES_XML = """\ + + + + + + + + + +""" + +_RELS_XML = """\ + + + + +""" + +PagesInput = Union[ + str, + Path, + Sequence[Union[str, Path]], + Mapping[int, Union[str, Path]], +] + +AssetsInput = Union[ + str, + Path, + Mapping[str, Union[str, Path]], +] + + +class PackagingError(Exception): + """Raised when DocLang archive packaging fails.""" + + +def _require_file(path: Path, *, label: str) -> None: + if not path.is_file(): + raise PackagingError(f"{label} not found or not a file: {path}") + + +def _require_directory(path: Path, *, label: str) -> None: + if not path.is_dir(): + raise PackagingError(f"{label} not found or not a directory: {path}") + + +def _validate_archive_relative_path(path: str, *, label: str) -> None: + if not path or path.startswith("/") or "\\" in path: + raise PackagingError(f"Invalid {label} path: {path!r}") + parts = Path(path).parts + if ".." in parts or path in {".", ".."}: + raise PackagingError(f"Invalid {label} path: {path!r}") + + +def _copy_tree_into(source: Path, destination: Path) -> None: + destination.mkdir(parents=True, exist_ok=True) + for item in source.iterdir(): + target = destination / item.name + if item.is_dir(): + shutil.copytree(item, target, dirs_exist_ok=True) + else: + shutil.copy2(item, target) + + +def _place_document(stage: Path, document: Path) -> None: + _require_file(document, label="Document") + shutil.copy2(document, stage / "document.xml") + + +def _place_pages(stage: Path, pages: PagesInput) -> None: + pages_dir = stage / "pages" + if isinstance(pages, Mapping): + for page_number, source in pages.items(): + if not isinstance(page_number, int) or page_number < 1: + raise PackagingError(f"Page numbers must be positive integers, got {page_number!r}") + source_path = Path(source) + _require_file(source_path, label="Page file") + destination = pages_dir / f"{page_number}{source_path.suffix}" + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source_path, destination) + return + + if isinstance(pages, str | Path): + source_dir = Path(pages) + _require_directory(source_dir, label="Pages directory") + _copy_tree_into(source_dir, pages_dir) + return + + for index, source in enumerate(pages, start=1): + source_path = Path(source) + _require_file(source_path, label="Page file") + destination = pages_dir / f"{index}{source_path.suffix}" + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source_path, destination) + + +def _place_assets(stage: Path, assets: AssetsInput) -> None: + assets_dir = stage / "assets" + if isinstance(assets, Mapping): + for archive_path, source in assets.items(): + _validate_archive_relative_path(archive_path, label="asset") + source_path = Path(source) + _require_file(source_path, label="Asset file") + destination = assets_dir / archive_path + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source_path, destination) + return + + source_dir = Path(assets) + _require_directory(source_dir, label="Assets directory") + _copy_tree_into(source_dir, assets_dir) + + +def _write_opc_metadata(stage: Path) -> None: + (stage / "[Content_Types].xml").write_text(_CONTENT_TYPES_XML, encoding="utf-8") + rels_dir = stage / "_rels" + rels_dir.mkdir(parents=True, exist_ok=True) + (rels_dir / ".rels").write_text(_RELS_XML, encoding="utf-8") + + +def _should_exclude_zip_member(arcname: str) -> bool: + parts = arcname.split("/") + if "__MACOSX" in parts: + return True + name = parts[-1] + return name == ".DS_Store" or name.startswith("._") + + +def _create_zip(stage: Path, output: Path) -> None: + output.parent.mkdir(parents=True, exist_ok=True) + if output.exists(): + output.unlink() + with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for path in sorted(stage.rglob("*")): + if not path.is_file(): + continue + arcname = path.relative_to(stage).as_posix() + if _should_exclude_zip_member(arcname): + continue + archive.write(path, arcname) + + +def _pack( + document: Union[str, Path], + *, + output: Union[str, Path, None] = None, + pages: PagesInput | None = None, + assets: AssetsInput | None = None, + validate: bool = False, +) -> Path: + document_path = Path(document) + output_path = Path(output) if output is not None else document_path.with_suffix(".dclx") + + with tempfile.TemporaryDirectory() as temp_dir: + stage = Path(temp_dir) + _place_document(stage, document_path) + if pages is not None: + _place_pages(stage, pages) + if assets is not None: + _place_assets(stage, assets) + _write_opc_metadata(stage) + + if validate: + from doclang.validation import validate as validate_document + + validate_document(stage / "document.xml") + + _create_zip(stage, output_path) + + return output_path.resolve() diff --git a/doclang/cli.py b/doclang/cli.py index 0391131..a8ad1bc 100644 --- a/doclang/cli.py +++ b/doclang/cli.py @@ -1,8 +1,7 @@ """ -DocLang CLI - Command-line interface for XML validation. +DocLang CLI - Command-line interface for the DocLang toolkit. -Provides a user-friendly CLI using Typer for validating DocLang XML documents -against XSD schemas and Schematron rules. +Provides a user-friendly CLI using Typer for working with DocLang documents. """ import json @@ -13,13 +12,15 @@ import typer from doclang._schemas import _bundled_schema_paths +from doclang.packaging import PackagingError +from doclang.packaging import pack as pack_document from doclang.utils import _VERSION from doclang.validation import ValidationError from doclang.validation import validate as validate_document app = typer.Typer( name="doclang", - help="DocLang XML validation tool with XSD and Schematron support", + help="DocLang toolkit", add_completion=False, no_args_is_help=True, ) @@ -145,6 +146,122 @@ def validate( typer.echo("VALIDATION SUCCESSFUL") +def _parse_asset_mapping(value: str) -> tuple[str, Path]: + if "=" not in value: + raise typer.BadParameter(f"Expected ARCHIVE_PATH=SOURCE, got {value!r}") + archive_path, source = value.split("=", 1) + if not archive_path or not source: + raise typer.BadParameter(f"Expected ARCHIVE_PATH=SOURCE, got {value!r}") + return archive_path, Path(source) + + +@app.command( + context_settings={"help_option_names": ["-h", "--help"]}, + no_args_is_help=True, +) +def pack( + document: Path = typer.Argument(..., help="DocLang markup file (.dclg, .xml, …)", exists=True), + output: Path | None = typer.Option( + None, + "-o", + "--output", + help="Output .dclx file (default: same stem as document, .dclx extension)", + ), + pages_dir: Path | None = typer.Option( + None, + "--pages", + help="Directory of page images (1.png, 2.png, …)", + exists=True, + file_okay=False, + dir_okay=True, + ), + page_files: list[Path] | None = typer.Option( + None, + "--page", + help="Page image; repeat to add pages in order (renumbered as 1.ext, 2.ext, …)", + exists=True, + file_okay=True, + dir_okay=False, + ), + asset_mappings: list[str] | None = typer.Option( + None, + "--asset", + help="Asset mapping ARCHIVE_PATH=SOURCE; repeat for multiple", + ), + assets_dir: Path | None = typer.Option( + None, + "--assets", + help="Directory tree copied into assets/", + exists=True, + file_okay=False, + dir_okay=True, + ), + validate_before_pack: bool = typer.Option(False, "--validate", help="Validate document before packing"), + quiet: bool = typer.Option(False, "--quiet", "-q", help="Quiet mode (exit code only)"), +): + """ + Pack a DocLang markup file and optional media into a .dclx archive. + + DOCUMENT is copied to document.xml. Optional page images (--pages, --page) + are placed under pages/. Optional payload files (--assets, --asset) are + placed under assets/ for URIs referenced in the markup. OPC metadata + ([Content_Types].xml, _rels/.rels) is generated automatically. + + By default, writes .dclx next to the input file. + + Examples: + + doclang pack markup.dclg + doclang pack markup.dclg -o report.dclx --pages screenshots/ + doclang pack markup.dclg --page a.png --page b.png + doclang pack markup.dclg --asset chart.svg=exports/diagram.svg + doclang pack markup.dclg --assets payload/ --validate + """ + if pages_dir is not None and page_files: + typer.echo("Error: --pages and --page are mutually exclusive", err=True) + raise typer.Exit(1) + if assets_dir is not None and asset_mappings: + typer.echo("Error: --assets and --asset are mutually exclusive", err=True) + raise typer.Exit(1) + + output_path = output or document.with_suffix(".dclx") + + pages: Path | list[Path] | None + if pages_dir is not None: + pages = pages_dir + elif page_files: + pages = page_files + else: + pages = None + + assets: Path | dict[str, Path] | None + if assets_dir is not None: + assets = assets_dir + elif asset_mappings: + assets = dict(_parse_asset_mapping(value) for value in asset_mappings) + else: + assets = None + + try: + created = pack_document( + document, + output=output_path, + pages=pages, + assets=assets, + validate=validate_before_pack, + ) + except ValidationError as exc: + if not quiet: + typer.echo(str(exc), err=True) + raise typer.Exit(1) + except PackagingError as exc: + typer.echo(f"Error: {exc}", err=True) + raise typer.Exit(1) + + if not quiet: + typer.echo(f"Created {created}") + + def _version_callback(value: bool): """Show version and exit.""" if value: @@ -163,7 +280,7 @@ def main( help="Show version and exit", ), ): - """DocLang XML validation tool.""" + """DocLang toolkit.""" if __name__ == "__main__": diff --git a/doclang/packaging.py b/doclang/packaging.py new file mode 100644 index 0000000..4aebb8d --- /dev/null +++ b/doclang/packaging.py @@ -0,0 +1,49 @@ +"""Public packaging API for DocLang archives.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Union + +from doclang._packaging import PackagingError, _pack + +__all__ = ["PackagingError", "pack"] + + +def pack( + document: Union[str, Path], + *, + output: Union[str, Path, None] = None, + pages: (Union[str, Path] | Sequence[Union[str, Path]] | Mapping[int, Union[str, Path]] | None) = None, + assets: (Union[str, Path] | Mapping[str, Union[str, Path]] | None) = None, + validate: bool = False, +) -> Path: + """Pack a DocLang markup file and optional media into a ``.dclx`` OPC archive. + + ``document`` is copied to ``document.xml`` inside the archive. OPC metadata + (``[Content_Types].xml``, ``_rels/.rels``) is generated automatically. + + By default, writes ``.dclx`` next to the input file. Pass ``output`` + to choose a different path. + + ``pages`` may be a directory (copied into ``pages/``), a sequence of image + paths (renumbered as ``1.ext``, ``2.ext``, …), or a mapping of page number + to image path. + + ``assets`` may be a directory (copied into ``assets/``) or a mapping of + archive-relative asset path to source file. + + Returns the resolved path to the created archive. + + Raises :class:`PackagingError` on packaging failure. + Raises :class:`~doclang.ValidationError` when ``validate=True`` and the + document fails validation. + """ + return _pack( + document, + output=output, + pages=pages, + assets=assets, + validate=validate, + ) diff --git a/pyproject.toml b/pyproject.toml index aba8a91..e7bd9a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "doclang" version = "0.6.0" # DO NOT EDIT MANUALLY, updated automatically -description = "DocLang reference validator" +description = "DocLang reference toolkit" readme = "README.md" requires-python = ">=3.10" license = "Apache-2.0" diff --git a/spec.md b/spec.md index d03e1cc..109d7e5 100644 --- a/spec.md +++ b/spec.md @@ -258,7 +258,7 @@ Non-normative recommendation guidelines are covered in [Recommendations](#recomm Planned extensions are discussed in [Future Extensions](#future-extensions). -Machine-checkable conformance is defined by the [DocLang reference validator](https://github.com/doclang-project). +A reference toolkit for DocLang is provided by the [DocLang Project](https://github.com/doclang-project). ## Usage Examples diff --git a/tests/test_packaging.py b/tests/test_packaging.py new file mode 100644 index 0000000..c5e107b --- /dev/null +++ b/tests/test_packaging.py @@ -0,0 +1,142 @@ +"""Tests for DocLang archive packaging.""" + +from __future__ import annotations + +import zipfile +from pathlib import Path + +import pytest + +from doclang import PackagingError, ValidationError, pack + +REPO_ROOT = Path(__file__).resolve().parents[1] +ARCHIVE_DEMO = REPO_ROOT / "examples" / "archive-demo" +VALID_DIR = Path(__file__).parent / "data" / "valid" + + +def _zip_members(archive_path: Path) -> set[str]: + with zipfile.ZipFile(archive_path) as archive: + return set(archive.namelist()) + + +def test_pack_markup_only_default_output(tmp_path: Path) -> None: + document = tmp_path / "markup.dclg" + document.write_text((ARCHIVE_DEMO / "document.xml").read_text(encoding="utf-8"), encoding="utf-8") + + created = pack(document) + + assert created == document.with_suffix(".dclx").resolve() + members = _zip_members(created) + assert "document.xml" in members + assert "[Content_Types].xml" in members + assert "_rels/.rels" in members + assert not any(member.startswith("pages/") for member in members) + + +def test_pack_markup_only_explicit_output(tmp_path: Path) -> None: + document = ARCHIVE_DEMO / "document.xml" + output = tmp_path / "report.dclx" + + created = pack(document, output=output) + + assert created == output.resolve() + members = _zip_members(created) + assert "document.xml" in members + + +def test_pack_with_pages_directory(tmp_path: Path) -> None: + document = ARCHIVE_DEMO / "document.xml" + output = tmp_path / "demo.dclx" + + pack(document, output=output, pages=ARCHIVE_DEMO / "pages") + + members = _zip_members(output) + assert "pages/1.png" in members + assert "pages/3.png" in members + + +def test_pack_with_pages_sequence(tmp_path: Path) -> None: + document = VALID_DIR / "ok_description_element_head.dclg" + page_one = ARCHIVE_DEMO / "pages" / "1.png" + page_two = ARCHIVE_DEMO / "pages" / "3.png" + output = tmp_path / "ordered.dclx" + + pack(document, output=output, pages=[page_one, page_two]) + + members = _zip_members(output) + assert "pages/1.png" in members + assert "pages/2.png" in members + + +def test_pack_with_pages_mapping(tmp_path: Path) -> None: + document = VALID_DIR / "ok_description_element_head.dclg" + page_one = ARCHIVE_DEMO / "pages" / "1.png" + page_three = ARCHIVE_DEMO / "pages" / "3.png" + output = tmp_path / "mapped.dclx" + + pack(document, output=output, pages={1: page_one, 3: page_three}) + + members = _zip_members(output) + assert "pages/1.png" in members + assert "pages/3.png" in members + assert "pages/2.png" not in members + + +def test_pack_with_asset_mapping(tmp_path: Path) -> None: + document = VALID_DIR / "ok_description_element_head.dclg" + asset_source = ARCHIVE_DEMO / "pages" / "1.png" + output = tmp_path / "assets.dclx" + + pack( + document, + output=output, + assets={"chart.svg": asset_source, "img/sample.png": asset_source}, + ) + + members = _zip_members(output) + assert "assets/chart.svg" in members + assert "assets/img/sample.png" in members + + +def test_pack_with_assets_directory(tmp_path: Path) -> None: + document = VALID_DIR / "ok_description_element_head.dclg" + assets_dir = tmp_path / "payload" + nested = assets_dir / "img" + nested.mkdir(parents=True) + source = ARCHIVE_DEMO / "pages" / "1.png" + (assets_dir / "chart.svg").write_bytes(source.read_bytes()) + (nested / "sample.png").write_bytes(source.read_bytes()) + output = tmp_path / "assets-dir.dclx" + + pack(document, output=output, assets=assets_dir) + + members = _zip_members(output) + assert "assets/chart.svg" in members + assert "assets/img/sample.png" in members + + +def test_pack_validate_invalid_document(tmp_path: Path) -> None: + document = Path(__file__).parent / "data" / "invalid" / "nok_summary_in_doclang.dclg" + output = tmp_path / "invalid.dclx" + + with pytest.raises(ValidationError): + pack(document, output=output, validate=True) + + assert not output.exists() + + +def test_pack_missing_document(tmp_path: Path) -> None: + with pytest.raises(PackagingError, match="Document not found"): + pack(tmp_path / "missing.dclg") + + +def test_pack_invalid_asset_path(tmp_path: Path) -> None: + document = VALID_DIR / "ok_description_element_head.dclg" + asset_source = ARCHIVE_DEMO / "pages" / "1.png" + + with pytest.raises(PackagingError, match="Invalid asset path"): + pack( + document, + output=tmp_path / "bad-asset.dclx", + assets={"../escape.svg": asset_source}, + ) diff --git a/utils/pack-archive.sh b/utils/pack-archive.sh deleted file mode 100755 index 07e45f8..0000000 --- a/utils/pack-archive.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash -# Pack a DocLang archive directory into a .dclx OPC ZIP file. -# Usage: ./utils/pack-archive.sh [output-file] - -set -euo pipefail - -if [[ $# -lt 1 ]]; then - echo "Usage: $0 [output-file]" >&2 - exit 1 -fi - -SRC=$(cd "$1" && pwd) -NAME=$(basename "$SRC") -OUT=${2:-"${NAME}.dclx"} - -if [[ ! -f "$SRC/document.xml" ]]; then - echo "Error: $SRC/document.xml not found" >&2 - exit 1 -fi - -if [[ "$OUT" != /* ]]; then - OUT="$(pwd)/$OUT" -fi - -STAGE=$(mktemp -d) -trap 'rm -rf "$STAGE"' EXIT - -cp -R "$SRC"/. "$STAGE"/ - -if [[ ! -f "$STAGE/[Content_Types].xml" ]]; then - cat > "$STAGE/[Content_Types].xml" <<'EOF' - - - - - - - - - -EOF -fi - -if [[ ! -f "$STAGE/_rels/.rels" ]]; then - mkdir -p "$STAGE/_rels" - cat > "$STAGE/_rels/.rels" <<'EOF' - - - - -EOF -fi - -rm -f "$OUT" -( - cd "$STAGE" - zip -r "$OUT" . \ - -x "*.DS_Store" \ - -x "__MACOSX/*" \ - -x "*/._*" \ - -x "._*" -) -echo "Created $OUT"