diff --git a/.github/workflows/build-placeholder.yml b/.github/workflows/build-catalog.yml similarity index 60% rename from .github/workflows/build-placeholder.yml rename to .github/workflows/build-catalog.yml index 8f8b6fc..86998e8 100644 --- a/.github/workflows/build-placeholder.yml +++ b/.github/workflows/build-catalog.yml @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: 2026-present Brian Wang # SPDX-License-Identifier: LGPL-3.0-or-later -name: Build Placeholder Dist +name: Build Catalog Registry on: workflow_dispatch: @@ -9,8 +9,13 @@ on: - main jobs: - build-placeholder: + build: runs-on: ubuntu-latest + env: + CHOYSUM_NPM_FETCH_TIMEOUT_SECONDS: "10" + CHOYSUM_NPM_FETCH_MAX_RETRIES: "3" + CHOYSUM_NPM_FETCH_BACKOFF_SECONDS: "1.0" + CHOYSUM_BUILD_CONCURRENCY: "5" steps: - name: Checkout uses: actions/checkout@v6 @@ -20,11 +25,11 @@ jobs: with: python-version: "3.12" - - name: Build placeholder dist artifacts - run: python scripts/build_placeholder.py + - name: Build catalog dist artifacts + run: python scripts/build_catalog.py - name: Upload dist artifact uses: actions/upload-artifact@v7 with: name: dist - path: dist \ No newline at end of file + path: dist diff --git a/.github/workflows/validate-catalog.yml b/.github/workflows/validate-catalog.yml index b3f7484..a44a614 100644 --- a/.github/workflows/validate-catalog.yml +++ b/.github/workflows/validate-catalog.yml @@ -11,6 +11,11 @@ on: jobs: validate: runs-on: ubuntu-latest + env: + CHOYSUM_NPM_FETCH_TIMEOUT_SECONDS: "10" + CHOYSUM_NPM_FETCH_MAX_RETRIES: "3" + CHOYSUM_NPM_FETCH_BACKOFF_SECONDS: "1.0" + CHOYSUM_BUILD_CONCURRENCY: "5" steps: - name: Checkout uses: actions/checkout@v6 @@ -35,4 +40,7 @@ jobs: check-jsonschema --schemafile schemas/catalog-entry.schema.json "${files[@]}" - name: Validate catalog pointers and structure - run: python scripts/validate_catalog.py \ No newline at end of file + run: python scripts/validate_catalog.py + + - name: Validate catalog build generation + run: python scripts/build_catalog.py \ No newline at end of file diff --git a/_headers b/_headers new file mode 100644 index 0000000..3f23125 --- /dev/null +++ b/_headers @@ -0,0 +1,15 @@ +/v1/index.json + Access-Control-Allow-Origin: * + Cache-Control: public, max-age=300, stale-while-revalidate=3600 +/v1/index.*.json + Access-Control-Allow-Origin: * + Cache-Control: public, max-age=31536000, immutable +/v1/meta.json + Access-Control-Allow-Origin: * + Cache-Control: public, max-age=60, stale-while-revalidate=300 +/v1/schema/* + Access-Control-Allow-Origin: * + Cache-Control: public, max-age=86400 +/v1/checksums.txt + Access-Control-Allow-Origin: * + Cache-Control: public, max-age=300 diff --git a/_redirects b/_redirects new file mode 100644 index 0000000..c1c9b41 --- /dev/null +++ b/_redirects @@ -0,0 +1 @@ +/ /v1/index.json 302 diff --git a/scripts/build_catalog.py b/scripts/build_catalog.py new file mode 100755 index 0000000..c1e1ca6 --- /dev/null +++ b/scripts/build_catalog.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026-present Brian Wang +# SPDX-License-Identifier: LGPL-3.0-or-later +"""Build the static catalog directory by fetching metadata from NPM.""" + +from __future__ import annotations + +import base64 +import concurrent.futures +import http.client +import hashlib +import json +import os +import shutil +import socket +import time +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +def read_int_env(name: str, default: int, minimum: int = 1) -> int: + raw = os.getenv(name) + if raw is None: + return default + try: + value = int(raw) + except ValueError as exc: + raise RuntimeError(f"{name} must be an integer, got {raw!r}") from exc + if value < minimum: + raise RuntimeError(f"{name} must be >= {minimum}, got {value}") + return value + + +def read_float_env(name: str, default: float, minimum: float = 0.0) -> float: + raw = os.getenv(name) + if raw is None: + return default + try: + value = float(raw) + except ValueError as exc: + raise RuntimeError(f"{name} must be a number, got {raw!r}") from exc + if value < minimum: + raise RuntimeError(f"{name} must be >= {minimum}, got {value}") + return value + + +ROOT = Path(__file__).resolve().parents[1] +TRUST_TIERS = ("official", "verified", "community") +CATALOG_ROOT = ROOT / "modules" +DIST_ROOT = ROOT / "dist" +V1_ROOT = DIST_ROOT / "v1" +SCHEMA_SRC = ROOT / "schemas" +SCHEMA_OUT = V1_ROOT / "schema" +NPM_FETCH_TIMEOUT_SECONDS = read_int_env("CHOYSUM_NPM_FETCH_TIMEOUT_SECONDS", 10) +NPM_FETCH_MAX_RETRIES = read_int_env("CHOYSUM_NPM_FETCH_MAX_RETRIES", 3) +NPM_FETCH_BACKOFF_SECONDS = read_float_env("CHOYSUM_NPM_FETCH_BACKOFF_SECONDS", 1.0) +BUILD_CONCURRENCY = read_int_env("CHOYSUM_BUILD_CONCURRENCY", 5) + +def write_text(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + +def write_json(path: Path, payload: dict) -> None: + text = json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n" + write_text(path, text) + +def load_json(path: Path) -> Any: + with path.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def resolve_integrity(dist_meta: dict[str, Any], package_name: str, version: str) -> str: + integrity = dist_meta.get("integrity") + if isinstance(integrity, str) and integrity.strip(): + return integrity + + shasum = dist_meta.get("shasum") + if isinstance(shasum, str) and shasum.strip(): + shasum_str = shasum.strip() + if len(shasum_str) != 40: + raise ValueError( + f"Invalid shasum length for package '{package_name}' version '{version}'" + ) + try: + digest = bytes.fromhex(shasum_str) + except ValueError as exc: + raise ValueError( + f"Invalid shasum hex for package '{package_name}' version '{version}'" + ) from exc + return "sha1-" + base64.b64encode(digest).decode("ascii") + + raise ValueError( + f"Missing integrity hash for package '{package_name}' version '{version}'" + ) + + +def fetch_npm_meta(package_name: str) -> dict: + quoted_name = urllib.parse.quote(package_name, safe="@") + url = f"https://registry.npmjs.org/{quoted_name}" + req = urllib.request.Request(url, headers={"User-Agent": "Choysum-Catalog-Builder/1.0"}) + last_error: Exception | None = None + + for attempt in range(1, NPM_FETCH_MAX_RETRIES + 1): + try: + with urllib.request.urlopen(req, timeout=NPM_FETCH_TIMEOUT_SECONDS) as response: + payload = json.loads(response.read().decode("utf-8")) + if not isinstance(payload, dict): + raise ValueError("NPM registry response is not a JSON object") + return payload + except urllib.error.HTTPError as exc: + if 400 <= exc.code < 500 and exc.code not in (408, 429): + raise RuntimeError( + f"Package '{package_name}' fetch failed with status {exc.code}." + ) from exc + last_error = exc + except ( + urllib.error.URLError, + http.client.HTTPException, + socket.timeout, + TimeoutError, + ConnectionError, + json.JSONDecodeError, + ValueError, + ) as exc: + last_error = exc + + if attempt < NPM_FETCH_MAX_RETRIES: + backoff_seconds = NPM_FETCH_BACKOFF_SECONDS * (2 ** (attempt - 1)) + time.sleep(backoff_seconds) + + if last_error is None: + raise RuntimeError(f"Failed to fetch metadata for {package_name}: unknown error") + raise RuntimeError( + f"Failed to fetch or parse {package_name} from NPM " + f"after {NPM_FETCH_MAX_RETRIES} attempts: {last_error}" + ) from last_error + +def process_module(entry_file: Path) -> tuple[str, dict[str, Any]]: + entry = load_json(entry_file) + if not isinstance(entry, dict): + raise ValueError(f"Catalog entry must be a JSON object: {entry_file}") + module_id = entry_file.stem + package_name = entry.get("package") + if not isinstance(package_name, str) or not package_name.strip(): + raise ValueError(f"Invalid or missing 'package' field in {entry_file}") + + print(f"Fetching NPM metadata for {package_name} (module: {module_id})...") + npm_data = fetch_npm_meta(package_name) + + versions_out = {} + versions_raw = npm_data.get("versions") + if not isinstance(versions_raw, dict): + versions_raw = {} + for ver, v_data in versions_raw.items(): + if not isinstance(v_data, dict): + continue + choysum_meta = v_data.get("choysum") + if not isinstance(choysum_meta, dict): + choysum_meta = {} + dist_meta = v_data.get("dist") + if not isinstance(dist_meta, dict): + dist_meta = {} + + # Tarball redirect format matching the Phase 4.3 specification + tarball_url = f"https://registry.choysum.dev/v1/tarballs/{package_name}/{ver}.tgz" + + depends = choysum_meta.get("depends") + if not isinstance(depends, list): + depends = [] + + peer_deps = v_data.get("peerDependencies") + if not isinstance(peer_deps, dict): + peer_deps = {} + + integrity = resolve_integrity(dist_meta, package_name, ver) + + v_entry = { + "tarball": tarball_url, + "integrity": integrity, + "depends": depends, + "peerDependencies": peer_deps + } + if "compatibility" in choysum_meta: + v_entry["compatibility"] = choysum_meta["compatibility"] + + versions_out[ver] = v_entry + + if not versions_out: + raise ValueError( + f"No valid versions found for package '{package_name}' (module: {module_id})" + ) + + return module_id, { + "moduleId": module_id, + "package": package_name, + "trust": entry.get("trust"), + "maintainers": entry.get("maintainers", []), + "versions": versions_out + } + +def collect_modules() -> dict[str, dict[str, Any]]: + modules: dict[str, dict[str, Any]] = {} + module_sources: dict[str, Path] = {} + tasks: dict[concurrent.futures.Future[tuple[str, dict[str, Any]]], Path] = {} + + with concurrent.futures.ThreadPoolExecutor(max_workers=BUILD_CONCURRENCY) as executor: + for trust in TRUST_TIERS: + tier_dir = CATALOG_ROOT / trust + if not tier_dir.is_dir(): + continue + + for entry_file in sorted(tier_dir.glob("*.json")): + future = executor.submit(process_module, entry_file) + tasks[future] = entry_file + + errors: list[str] = [] + for future in concurrent.futures.as_completed(tasks): + entry_file = tasks[future] + try: + module_id, mod_payload = future.result() + if module_id in modules: + existing_file = module_sources[module_id] + errors.append( + f" - Duplicate module ID '{module_id}' between " + f"{existing_file.relative_to(ROOT)} and {entry_file.relative_to(ROOT)}" + ) + continue + modules[module_id] = mod_payload + module_sources[module_id] = entry_file + except Exception as e: + errors.append(f" - {entry_file.relative_to(ROOT)}: {e}") + + if errors: + raise RuntimeError("Failed to collect all modules due to the following errors:\n" + "\n".join(errors)) + + return modules + +def utc_now_iso() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + +def generate_checksums(files: list[Path]) -> str: + lines = [] + for f in sorted(files, key=lambda p: p.relative_to(DIST_ROOT).as_posix()): + rel_path = f"/{f.relative_to(DIST_ROOT).as_posix()}" + hasher = hashlib.sha256() + with f.open("rb") as handle: + for chunk in iter(lambda: handle.read(65536), b""): + hasher.update(chunk) + lines.append(f"{hasher.hexdigest()} {rel_path}") + return "\n".join(lines) + "\n" + +def build() -> None: + modules = collect_modules() + generated_at = utc_now_iso() + + if DIST_ROOT.is_symlink(): + DIST_ROOT.unlink() + elif DIST_ROOT.exists() and DIST_ROOT.is_dir(): + shutil.rmtree(DIST_ROOT) + elif DIST_ROOT.exists(): + DIST_ROOT.unlink() + DIST_ROOT.mkdir(parents=True, exist_ok=True) + V1_ROOT.mkdir(parents=True, exist_ok=True) + SCHEMA_OUT.mkdir(parents=True, exist_ok=True) + + index_payload = { + "generatedAt": generated_at, + "modules": modules, + "version": 1, + } + canonical_index = json.dumps(index_payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n" + index_hash = hashlib.sha256(canonical_index.encode("utf-8")).hexdigest() + + index_path = V1_ROOT / "index.json" + index_hashed_path = V1_ROOT / f"index.{index_hash}.json" + write_text(index_path, canonical_index) + write_text(index_hashed_path, canonical_index) + + meta_payload = { + "generatedAt": generated_at, + "indexHash": index_hash, + "indexPath": f"/v1/index.{index_hash}.json", + } + meta_path = V1_ROOT / "meta.json" + write_json(meta_path, meta_payload) + + # Copy schemas + checksum_files = [index_path, index_hashed_path, meta_path] + for schema_file in sorted(SCHEMA_SRC.glob("*.json")): + target = SCHEMA_OUT / schema_file.name + target.write_text(schema_file.read_text(encoding="utf-8"), encoding="utf-8") + checksum_files.append(target) + + headers_src = ROOT / "_headers" + redirects_src = ROOT / "_redirects" + if not headers_src.is_file(): + raise RuntimeError(f"Missing required static file: {headers_src.relative_to(ROOT)}") + if not redirects_src.is_file(): + raise RuntimeError(f"Missing required static file: {redirects_src.relative_to(ROOT)}") + + write_text(DIST_ROOT / "_headers", headers_src.read_text(encoding="utf-8")) + write_text(DIST_ROOT / "_redirects", redirects_src.read_text(encoding="utf-8")) + + # Finally generate checksums + checksums_path = V1_ROOT / "checksums.txt" + checksums_content = generate_checksums(checksum_files) + write_text(checksums_path, checksums_content) + + print(f"Successfully built catalog artifacts under dist/ (total modules: {len(modules)})") + +if __name__ == "__main__": + build() diff --git a/scripts/build_placeholder.py b/scripts/build_placeholder.py deleted file mode 100644 index 8e67989..0000000 --- a/scripts/build_placeholder.py +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: 2026-present Brian Wang -# SPDX-License-Identifier: LGPL-3.0-or-later -"""Build minimal static directory artifacts for bootstrapping.""" - -from __future__ import annotations - -import hashlib -import json -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -ROOT = Path(__file__).resolve().parents[1] -TRUST_TIERS = ("official", "verified", "community") -CATALOG_ROOT = ROOT / "modules" -DIST_ROOT = ROOT / "dist" -V1_ROOT = DIST_ROOT / "v1" -SCHEMA_SRC = ROOT / "schemas" -SCHEMA_OUT = V1_ROOT / "schema" - - -def write_text(path: Path, content: str) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(content, encoding="utf-8") - - -def write_json(path: Path, payload: dict) -> None: - text = json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n" - write_text(path, text) - - -def sha256_of_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(8192), b""): - digest.update(chunk) - return digest.hexdigest() - - -def load_json(path: Path) -> Any: - with path.open("r", encoding="utf-8") as handle: - return json.load(handle) - - -def collect_modules() -> list[dict[str, Any]]: - modules: list[dict[str, Any]] = [] - for trust in TRUST_TIERS: - tier_dir = CATALOG_ROOT / trust - if not tier_dir.is_dir(): - continue - - for entry_file in sorted(tier_dir.glob("*.json")): - entry = load_json(entry_file) - if not isinstance(entry, dict): - raise ValueError(f"Catalog entry must be an object: {entry_file}") - - modules.append( - { - "id": entry_file.stem, - "package": entry.get("package"), - "trust": entry.get("trust"), - "maintainers": entry.get("maintainers", []), - } - ) - return modules - - -def utc_now_iso() -> str: - return ( - datetime.now(timezone.utc) - .replace(microsecond=0) - .isoformat() - .replace("+00:00", "Z") - ) - - -def build() -> None: - V1_ROOT.mkdir(parents=True, exist_ok=True) - SCHEMA_OUT.mkdir(parents=True, exist_ok=True) - - generated_at = utc_now_iso() - modules = collect_modules() - index_payload = { - "generatedAt": generated_at, - "modules": modules, - "version": 1, - } - canonical_index = json.dumps(index_payload, sort_keys=True, separators=(",", ":")) - index_hash = hashlib.sha256(canonical_index.encode("utf-8")).hexdigest() - - index_path = V1_ROOT / "index.json" - index_hashed_path = V1_ROOT / f"index.{index_hash}.json" - write_text(index_path, canonical_index + "\n") - write_text(index_hashed_path, canonical_index + "\n") - - meta_payload = { - "generatedAt": generated_at, - "indexHash": index_hash, - "indexPath": f"/v1/index.{index_hash}.json", - } - meta_path = V1_ROOT / "meta.json" - write_json(meta_path, meta_payload) - - copied_schema_paths: list[Path] = [] - for schema_file in sorted(SCHEMA_SRC.glob("*.json")): - target = SCHEMA_OUT / schema_file.name - target.write_text(schema_file.read_text(encoding="utf-8"), encoding="utf-8") - copied_schema_paths.append(target) - - checksum_targets = [index_path, index_hashed_path, meta_path, *copied_schema_paths] - checksum_lines = [ - f"{sha256_of_file(path)} /{path.relative_to(DIST_ROOT).as_posix()}" - for path in checksum_targets - ] - write_text(V1_ROOT / "checksums.txt", "\n".join(checksum_lines) + "\n") - - headers_content = ( - "/v1/index.json\n" - " Cache-Control: public, max-age=300, stale-while-revalidate=3600\n\n" - "/v1/index.*.json\n" - " Cache-Control: public, max-age=31536000, immutable\n\n" - "/v1/meta.json\n" - " Cache-Control: public, max-age=60, stale-while-revalidate=300\n\n" - "/v1/schema/*\n" - " Cache-Control: public, max-age=86400\n\n" - "/v1/checksums.txt\n" - " Cache-Control: public, max-age=300\n" - ) - write_text(DIST_ROOT / "_headers", headers_content) - write_text(DIST_ROOT / "_redirects", "/ /v1/index.json 302\n") - - print("Built placeholder artifacts under dist/") - - -if __name__ == "__main__": - build() \ No newline at end of file