From eae5261bbcff249c50cf6a588e2518e4cdd98eec Mon Sep 17 00:00:00 2001 From: buke Date: Tue, 9 Jun 2026 21:28:46 +0800 Subject: [PATCH 01/20] chore(infra): add cloudflare pages routing and caching headers --- _headers | 15 +++++++++++++++ _redirects | 1 + 2 files changed, 16 insertions(+) create mode 100644 _headers create mode 100644 _redirects diff --git a/_headers b/_headers new file mode 100644 index 0000000..3f23125 --- /dev/null +++ b/_headers @@ -0,0 +1,15 @@ +/v1/index.json + Access-Control-Allow-Origin: * + Cache-Control: public, max-age=300, stale-while-revalidate=3600 +/v1/index.*.json + Access-Control-Allow-Origin: * + Cache-Control: public, max-age=31536000, immutable +/v1/meta.json + Access-Control-Allow-Origin: * + Cache-Control: public, max-age=60, stale-while-revalidate=300 +/v1/schema/* + Access-Control-Allow-Origin: * + Cache-Control: public, max-age=86400 +/v1/checksums.txt + Access-Control-Allow-Origin: * + Cache-Control: public, max-age=300 diff --git a/_redirects b/_redirects new file mode 100644 index 0000000..c1c9b41 --- /dev/null +++ b/_redirects @@ -0,0 +1 @@ +/ /v1/index.json 302 From 992ac07f03fdccebf4c5804b604a12992fcbd861 Mon Sep 17 00:00:00 2001 From: buke Date: Tue, 9 Jun 2026 21:44:39 +0800 Subject: [PATCH 02/20] feat(registry): implement NPM-based multi-version catalog builder Replaces the placeholder script with a robust Python builder that: - Executes concurrent HTTP requests against registry.npmjs.org - Flattens all valid versions preserving integrity, tarball URLs, and peerDependencies - Strictly adheres to the spec generating choysum module boundaries --- ...uild-placeholder.yml => build-catalog.yml} | 16 +- scripts/build_catalog.py | 160 ++++++++++++++++++ scripts/build_placeholder.py | 137 --------------- 3 files changed, 168 insertions(+), 145 deletions(-) rename .github/workflows/{build-placeholder.yml => build-catalog.yml} (58%) create mode 100755 scripts/build_catalog.py delete mode 100644 scripts/build_placeholder.py diff --git a/.github/workflows/build-placeholder.yml b/.github/workflows/build-catalog.yml similarity index 58% rename from .github/workflows/build-placeholder.yml rename to .github/workflows/build-catalog.yml index 8f8b6fc..a5a8c86 100644 --- a/.github/workflows/build-placeholder.yml +++ b/.github/workflows/build-catalog.yml @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: 2026-present Brian Wang # SPDX-License-Identifier: LGPL-3.0-or-later -name: Build Placeholder Dist +name: Build Catalog Registry on: workflow_dispatch: @@ -9,22 +9,22 @@ on: - main jobs: - build-placeholder: + build: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v6 + uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Build placeholder dist artifacts - run: python scripts/build_placeholder.py + - name: Build catalog dist artifacts + run: python scripts/build_catalog.py - name: Upload dist artifact - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@v4 with: name: dist - path: dist \ No newline at end of file + path: dist diff --git a/scripts/build_catalog.py b/scripts/build_catalog.py new file mode 100755 index 0000000..27d48b7 --- /dev/null +++ b/scripts/build_catalog.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026-present Brian Wang +# SPDX-License-Identifier: LGPL-3.0-or-later +"""Build the static catalog directory by fetching metadata from NPM.""" + +from __future__ import annotations + +import hashlib +import json +import urllib.request +import urllib.error +import concurrent.futures +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +TRUST_TIERS = ("official", "verified", "community") +CATALOG_ROOT = ROOT / "modules" +DIST_ROOT = ROOT / "dist" +V1_ROOT = DIST_ROOT / "v1" +SCHEMA_SRC = ROOT / "schemas" +SCHEMA_OUT = V1_ROOT / "schema" + +def write_text(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + +def write_json(path: Path, payload: dict) -> None: + text = json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n" + write_text(path, text) + +def load_json(path: Path) -> Any: + with path.open("r", encoding="utf-8") as handle: + return json.load(handle) + +def fetch_npm_meta(package_name: str) -> dict: + url = f"https://registry.npmjs.org/{package_name}" + req = urllib.request.Request(url, headers={"User-Agent": "Choysum-Catalog-Builder/1.0"}) + try: + with urllib.request.urlopen(req) as response: + return json.loads(response.read().decode('utf-8')) + except urllib.error.URLError as e: + raise RuntimeError(f"Failed to fetch {package_name} from NPM: {e}") + +def process_module(entry_file: Path) -> tuple[str, dict[str, Any]]: + entry = load_json(entry_file) + module_id = entry_file.stem + package_name = entry.get("package") + + print(f"Fetching NPM metadata for {package_name} (module: {module_id})...") + npm_data = fetch_npm_meta(package_name) + + versions_out = {} + for ver, v_data in npm_data.get("versions", {}).items(): + choysum_meta = v_data.get("choysum", {}) + + # Tarball redirect format matching the Phase 4.3 specification + tarball_url = f"https://registry.choysum.dev/v1/tarballs/{package_name}/{ver}.tgz" + + v_entry = { + "tarball": tarball_url, + "integrity": v_data.get("dist", {}).get("integrity", ""), + "depends": choysum_meta.get("depends", []), + "peerDependencies": v_data.get("peerDependencies", {}) + } + if "compatibility" in choysum_meta: + v_entry["compatibility"] = choysum_meta["compatibility"] + + versions_out[ver] = v_entry + + return module_id, { + "moduleId": module_id, + "package": package_name, + "trust": entry.get("trust"), + "maintainers": entry.get("maintainers", []), + "versions": versions_out + } + +def collect_modules() -> dict[str, dict[str, Any]]: + modules: dict[str, dict[str, Any]] = {} + tasks = [] + + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + for trust in TRUST_TIERS: + tier_dir = CATALOG_ROOT / trust + if not tier_dir.is_dir(): + continue + + for entry_file in sorted(tier_dir.glob("*.json")): + tasks.append(executor.submit(process_module, entry_file)) + + for future in concurrent.futures.as_completed(tasks): + module_id, mod_payload = future.result() + modules[module_id] = mod_payload + + return modules + +def utc_now_iso() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + +def generate_checksums(files: list[Path]) -> str: + lines = [] + for f in sorted(files): + rel_path = f"/{f.relative_to(DIST_ROOT).as_posix()}" + h = hashlib.sha256(f.read_bytes()).hexdigest() + lines.append(f"{h} {rel_path}") + return "\n".join(lines) + "\n" + +def build() -> None: + DIST_ROOT.mkdir(parents=True, exist_ok=True) + V1_ROOT.mkdir(parents=True, exist_ok=True) + SCHEMA_OUT.mkdir(parents=True, exist_ok=True) + + generated_at = utc_now_iso() + modules = collect_modules() + index_payload = { + "generatedAt": generated_at, + "modules": modules, + "version": 1, + } + canonical_index = json.dumps(index_payload, sort_keys=True, separators=(",", ":")) + index_hash = hashlib.sha256(canonical_index.encode("utf-8")).hexdigest() + + index_path = V1_ROOT / "index.json" + index_hashed_path = V1_ROOT / f"index.{index_hash}.json" + write_text(index_path, canonical_index + "\n") + write_text(index_hashed_path, canonical_index + "\n") + + meta_payload = { + "generatedAt": generated_at, + "indexHash": index_hash, + "indexPath": f"/v1/index.{index_hash}.json", + } + meta_path = V1_ROOT / "meta.json" + write_json(meta_path, meta_payload) + + # Copy schemas + checksum_files = [index_path, index_hashed_path, meta_path] + for schema_file in sorted(SCHEMA_SRC.glob("*.json")): + target = SCHEMA_OUT / schema_file.name + target.write_text(schema_file.read_text(encoding="utf-8"), encoding="utf-8") + checksum_files.append(target) + + # Note: _headers and _redirects are managed statically in the Git repository now. + # We will copy them to the dist folder so Cloudflare Pages can use them. + if (ROOT / "_headers").exists(): + write_text(DIST_ROOT / "_headers", (ROOT / "_headers").read_text(encoding="utf-8")) + if (ROOT / "_redirects").exists(): + write_text(DIST_ROOT / "_redirects", (ROOT / "_redirects").read_text(encoding="utf-8")) + + # Finally generate checksums + checksums_path = V1_ROOT / "checksums.txt" + checksums_content = generate_checksums(checksum_files) + write_text(checksums_path, checksums_content) + + print(f"Successfully built catalog artifacts under dist/ (total modules: {len(modules)})") + +if __name__ == "__main__": + build() diff --git a/scripts/build_placeholder.py b/scripts/build_placeholder.py deleted file mode 100644 index 8e67989..0000000 --- a/scripts/build_placeholder.py +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: 2026-present Brian Wang -# SPDX-License-Identifier: LGPL-3.0-or-later -"""Build minimal static directory artifacts for bootstrapping.""" - -from __future__ import annotations - -import hashlib -import json -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -ROOT = Path(__file__).resolve().parents[1] -TRUST_TIERS = ("official", "verified", "community") -CATALOG_ROOT = ROOT / "modules" -DIST_ROOT = ROOT / "dist" -V1_ROOT = DIST_ROOT / "v1" -SCHEMA_SRC = ROOT / "schemas" -SCHEMA_OUT = V1_ROOT / "schema" - - -def write_text(path: Path, content: str) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(content, encoding="utf-8") - - -def write_json(path: Path, payload: dict) -> None: - text = json.dumps(payload, sort_keys=True, separators=(",", ":")) + "\n" - write_text(path, text) - - -def sha256_of_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(8192), b""): - digest.update(chunk) - return digest.hexdigest() - - -def load_json(path: Path) -> Any: - with path.open("r", encoding="utf-8") as handle: - return json.load(handle) - - -def collect_modules() -> list[dict[str, Any]]: - modules: list[dict[str, Any]] = [] - for trust in TRUST_TIERS: - tier_dir = CATALOG_ROOT / trust - if not tier_dir.is_dir(): - continue - - for entry_file in sorted(tier_dir.glob("*.json")): - entry = load_json(entry_file) - if not isinstance(entry, dict): - raise ValueError(f"Catalog entry must be an object: {entry_file}") - - modules.append( - { - "id": entry_file.stem, - "package": entry.get("package"), - "trust": entry.get("trust"), - "maintainers": entry.get("maintainers", []), - } - ) - return modules - - -def utc_now_iso() -> str: - return ( - datetime.now(timezone.utc) - .replace(microsecond=0) - .isoformat() - .replace("+00:00", "Z") - ) - - -def build() -> None: - V1_ROOT.mkdir(parents=True, exist_ok=True) - SCHEMA_OUT.mkdir(parents=True, exist_ok=True) - - generated_at = utc_now_iso() - modules = collect_modules() - index_payload = { - "generatedAt": generated_at, - "modules": modules, - "version": 1, - } - canonical_index = json.dumps(index_payload, sort_keys=True, separators=(",", ":")) - index_hash = hashlib.sha256(canonical_index.encode("utf-8")).hexdigest() - - index_path = V1_ROOT / "index.json" - index_hashed_path = V1_ROOT / f"index.{index_hash}.json" - write_text(index_path, canonical_index + "\n") - write_text(index_hashed_path, canonical_index + "\n") - - meta_payload = { - "generatedAt": generated_at, - "indexHash": index_hash, - "indexPath": f"/v1/index.{index_hash}.json", - } - meta_path = V1_ROOT / "meta.json" - write_json(meta_path, meta_payload) - - copied_schema_paths: list[Path] = [] - for schema_file in sorted(SCHEMA_SRC.glob("*.json")): - target = SCHEMA_OUT / schema_file.name - target.write_text(schema_file.read_text(encoding="utf-8"), encoding="utf-8") - copied_schema_paths.append(target) - - checksum_targets = [index_path, index_hashed_path, meta_path, *copied_schema_paths] - checksum_lines = [ - f"{sha256_of_file(path)} /{path.relative_to(DIST_ROOT).as_posix()}" - for path in checksum_targets - ] - write_text(V1_ROOT / "checksums.txt", "\n".join(checksum_lines) + "\n") - - headers_content = ( - "/v1/index.json\n" - " Cache-Control: public, max-age=300, stale-while-revalidate=3600\n\n" - "/v1/index.*.json\n" - " Cache-Control: public, max-age=31536000, immutable\n\n" - "/v1/meta.json\n" - " Cache-Control: public, max-age=60, stale-while-revalidate=300\n\n" - "/v1/schema/*\n" - " Cache-Control: public, max-age=86400\n\n" - "/v1/checksums.txt\n" - " Cache-Control: public, max-age=300\n" - ) - write_text(DIST_ROOT / "_headers", headers_content) - write_text(DIST_ROOT / "_redirects", "/ /v1/index.json 302\n") - - print("Built placeholder artifacts under dist/") - - -if __name__ == "__main__": - build() \ No newline at end of file From 0a2da88efd81947f17a57e948a8faa1b73489edc Mon Sep 17 00:00:00 2001 From: buke Date: Tue, 9 Jun 2026 21:49:09 +0800 Subject: [PATCH 03/20] chore(build): configure Cloudflare Pages via wrangler.toml Overrides the legacy dashboard settings to automatically point to the new python scripts/build_catalog.py tool. --- wrangler.toml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 wrangler.toml diff --git a/wrangler.toml b/wrangler.toml new file mode 100644 index 0000000..d81dee6 --- /dev/null +++ b/wrangler.toml @@ -0,0 +1,5 @@ +name = "modules-directory" +pages_build_output_dir = "dist" + +[build] +command = "python scripts/build_catalog.py" From accdb8b781640ab917af356137474239ea3d7c42 Mon Sep 17 00:00:00 2001 From: buke Date: Tue, 9 Jun 2026 21:51:05 +0800 Subject: [PATCH 04/20] chore(build): remove unsupported wrangler.toml Cloudflare Pages does not support the [build] configuration block in wrangler.toml. The build command must be updated directly in the Cloudflare Dashboard. --- wrangler.toml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 wrangler.toml diff --git a/wrangler.toml b/wrangler.toml deleted file mode 100644 index d81dee6..0000000 --- a/wrangler.toml +++ /dev/null @@ -1,5 +0,0 @@ -name = "modules-directory" -pages_build_output_dir = "dist" - -[build] -command = "python scripts/build_catalog.py" From 1ed2ddb7252b64cdbc679da2de48b89c38a3d47a Mon Sep 17 00:00:00 2001 From: buke Date: Tue, 9 Jun 2026 21:58:10 +0800 Subject: [PATCH 05/20] chore(ci): restore latest actions versions Restores actions/checkout@v6, setup-python@v6, and upload-artifact@v7 in the build catalog workflow to match project baseline. --- .github/workflows/build-catalog.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-catalog.yml b/.github/workflows/build-catalog.yml index a5a8c86..5ca1ef1 100644 --- a/.github/workflows/build-catalog.yml +++ b/.github/workflows/build-catalog.yml @@ -13,10 +13,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.12" @@ -24,7 +24,7 @@ jobs: run: python scripts/build_catalog.py - name: Upload dist artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: dist path: dist From 287d5627e8b0be9bb9f58385871806a04ef64255 Mon Sep 17 00:00:00 2001 From: buke Date: Tue, 9 Jun 2026 22:01:41 +0800 Subject: [PATCH 06/20] fix(catalog): robust NPM fetching and defensive parsing Incorporates gemini-code-assist review feedback: - Adds a 10s URL timeout to prevent fetch hangs - URL-encodes scoped package names properly - Defensively asserts type dictionary instances for registry payloads --- scripts/build_catalog.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/scripts/build_catalog.py b/scripts/build_catalog.py index 27d48b7..00925e3 100755 --- a/scripts/build_catalog.py +++ b/scripts/build_catalog.py @@ -35,34 +35,47 @@ def load_json(path: Path) -> Any: return json.load(handle) def fetch_npm_meta(package_name: str) -> dict: - url = f"https://registry.npmjs.org/{package_name}" + import urllib.parse + quoted_name = urllib.parse.quote(package_name, safe="@") + url = f"https://registry.npmjs.org/{quoted_name}" req = urllib.request.Request(url, headers={"User-Agent": "Choysum-Catalog-Builder/1.0"}) try: - with urllib.request.urlopen(req) as response: + with urllib.request.urlopen(req, timeout=10) as response: return json.loads(response.read().decode('utf-8')) except urllib.error.URLError as e: raise RuntimeError(f"Failed to fetch {package_name} from NPM: {e}") def process_module(entry_file: Path) -> tuple[str, dict[str, Any]]: entry = load_json(entry_file) + if not isinstance(entry, dict): + raise ValueError(f"Catalog entry must be a JSON object: {entry_file}") module_id = entry_file.stem package_name = entry.get("package") + if not isinstance(package_name, str) or not package_name.strip(): + raise ValueError(f"Invalid or missing 'package' field in {entry_file}") print(f"Fetching NPM metadata for {package_name} (module: {module_id})...") npm_data = fetch_npm_meta(package_name) versions_out = {} - for ver, v_data in npm_data.get("versions", {}).items(): - choysum_meta = v_data.get("choysum", {}) + for ver, v_data in (npm_data.get("versions") or {}).items(): + if not isinstance(v_data, dict): + continue + choysum_meta = v_data.get("choysum") + if not isinstance(choysum_meta, dict): + choysum_meta = {} + dist_meta = v_data.get("dist") + if not isinstance(dist_meta, dict): + dist_meta = {} # Tarball redirect format matching the Phase 4.3 specification tarball_url = f"https://registry.choysum.dev/v1/tarballs/{package_name}/{ver}.tgz" v_entry = { "tarball": tarball_url, - "integrity": v_data.get("dist", {}).get("integrity", ""), + "integrity": dist_meta.get("integrity", ""), "depends": choysum_meta.get("depends", []), - "peerDependencies": v_data.get("peerDependencies", {}) + "peerDependencies": v_data.get("peerDependencies") or {} } if "compatibility" in choysum_meta: v_entry["compatibility"] = choysum_meta["compatibility"] From 3e10b1a552b77d48c508b592949bfb25d71ad8a0 Mon Sep 17 00:00:00 2001 From: buke Date: Tue, 9 Jun 2026 22:04:39 +0800 Subject: [PATCH 07/20] style(catalog): tidy imports based on bot feedback Resolves github-code-quality flagging unused imports by cleanly grouping urllib library components at the top level instead of doing an inline import. --- scripts/build_catalog.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_catalog.py b/scripts/build_catalog.py index 00925e3..fc8b47c 100755 --- a/scripts/build_catalog.py +++ b/scripts/build_catalog.py @@ -7,8 +7,9 @@ import hashlib import json -import urllib.request import urllib.error +import urllib.parse +import urllib.request import concurrent.futures from datetime import datetime, timezone from pathlib import Path @@ -35,7 +36,6 @@ def load_json(path: Path) -> Any: return json.load(handle) def fetch_npm_meta(package_name: str) -> dict: - import urllib.parse quoted_name = urllib.parse.quote(package_name, safe="@") url = f"https://registry.npmjs.org/{quoted_name}" req = urllib.request.Request(url, headers={"User-Agent": "Choysum-Catalog-Builder/1.0"}) From 8e7ee29aebba61e30f7cce8eca93ec2f11c6c6a5 Mon Sep 17 00:00:00 2001 From: buke Date: Tue, 9 Jun 2026 22:07:33 +0800 Subject: [PATCH 08/20] fix(catalog): handle decoding errors and enforce metadata types Incorporates the second round of gemini-code-assist review feedback: - Catches json.JSONDecodeError from NPM registry outages to prevent unhandled crashes - Explicitly enforces list/dict typing for depends/peerDependencies attributes --- scripts/build_catalog.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/build_catalog.py b/scripts/build_catalog.py index fc8b47c..00c1bbb 100755 --- a/scripts/build_catalog.py +++ b/scripts/build_catalog.py @@ -42,8 +42,8 @@ def fetch_npm_meta(package_name: str) -> dict: try: with urllib.request.urlopen(req, timeout=10) as response: return json.loads(response.read().decode('utf-8')) - except urllib.error.URLError as e: - raise RuntimeError(f"Failed to fetch {package_name} from NPM: {e}") + except (urllib.error.URLError, json.JSONDecodeError) as e: + raise RuntimeError(f"Failed to fetch or parse {package_name} from NPM: {e}") def process_module(entry_file: Path) -> tuple[str, dict[str, Any]]: entry = load_json(entry_file) @@ -71,11 +71,19 @@ def process_module(entry_file: Path) -> tuple[str, dict[str, Any]]: # Tarball redirect format matching the Phase 4.3 specification tarball_url = f"https://registry.choysum.dev/v1/tarballs/{package_name}/{ver}.tgz" + depends = choysum_meta.get("depends") + if not isinstance(depends, list): + depends = [] + + peer_deps = v_data.get("peerDependencies") + if not isinstance(peer_deps, dict): + peer_deps = {} + v_entry = { "tarball": tarball_url, "integrity": dist_meta.get("integrity", ""), - "depends": choysum_meta.get("depends", []), - "peerDependencies": v_data.get("peerDependencies") or {} + "depends": depends, + "peerDependencies": peer_deps } if "compatibility" in choysum_meta: v_entry["compatibility"] = choysum_meta["compatibility"] From 06deb4bb782db7896468bc0d184ed3a483accd81 Mon Sep 17 00:00:00 2001 From: buke Date: Tue, 9 Jun 2026 22:13:37 +0800 Subject: [PATCH 09/20] fix(catalog): refine error aggregation and clean dist folder Incorporates the third round of gemini-code-assist review feedback: - Erases the dist/ target folder before building to prevent leftover bloated snapshots - Collects ThreadPool concurrent exceptions instead of short-circuiting so bad module inputs are reported collectively --- scripts/build_catalog.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/scripts/build_catalog.py b/scripts/build_catalog.py index 00c1bbb..a339ce6 100755 --- a/scripts/build_catalog.py +++ b/scripts/build_catalog.py @@ -100,7 +100,7 @@ def process_module(entry_file: Path) -> tuple[str, dict[str, Any]]: def collect_modules() -> dict[str, dict[str, Any]]: modules: dict[str, dict[str, Any]] = {} - tasks = [] + tasks = {} with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: for trust in TRUST_TIERS: @@ -109,11 +109,20 @@ def collect_modules() -> dict[str, dict[str, Any]]: continue for entry_file in sorted(tier_dir.glob("*.json")): - tasks.append(executor.submit(process_module, entry_file)) + future = executor.submit(process_module, entry_file) + tasks[future] = entry_file + errors = [] for future in concurrent.futures.as_completed(tasks): - module_id, mod_payload = future.result() - modules[module_id] = mod_payload + entry_file = tasks[future] + try: + module_id, mod_payload = future.result() + modules[module_id] = mod_payload + except Exception as e: + errors.append(f" - {entry_file.relative_to(ROOT)}: {e}") + + if errors: + raise RuntimeError("Failed to collect all modules due to the following errors:\n" + "\n".join(errors)) return modules @@ -129,6 +138,9 @@ def generate_checksums(files: list[Path]) -> str: return "\n".join(lines) + "\n" def build() -> None: + import shutil + if DIST_ROOT.exists(): + shutil.rmtree(DIST_ROOT) DIST_ROOT.mkdir(parents=True, exist_ok=True) V1_ROOT.mkdir(parents=True, exist_ok=True) SCHEMA_OUT.mkdir(parents=True, exist_ok=True) From 4e76cf73066c5b97221c3f5bc5d927c2aaca1f5f Mon Sep 17 00:00:00 2001 From: buke Date: Tue, 9 Jun 2026 22:25:26 +0800 Subject: [PATCH 10/20] fix(catalog): harden concurrency and configurable retries - Add configurable NPM fetch retry/backoff/timeout via environment variables. - Detect duplicate module IDs across trust tiers and fail with explicit conflict details. - Collect module metadata before deleting dist to avoid destructive cleanup on fetch failures. --- scripts/build_catalog.py | 98 +++++++++++++++++++++++++++++++++------- 1 file changed, 82 insertions(+), 16 deletions(-) diff --git a/scripts/build_catalog.py b/scripts/build_catalog.py index a339ce6..a25cc10 100755 --- a/scripts/build_catalog.py +++ b/scripts/build_catalog.py @@ -5,16 +5,46 @@ from __future__ import annotations +import concurrent.futures import hashlib import json +import os +import shutil +import time import urllib.error import urllib.parse import urllib.request -import concurrent.futures from datetime import datetime, timezone from pathlib import Path from typing import Any + +def read_int_env(name: str, default: int, minimum: int = 1) -> int: + raw = os.getenv(name) + if raw is None: + return default + try: + value = int(raw) + except ValueError as exc: + raise RuntimeError(f"{name} must be an integer, got {raw!r}") from exc + if value < minimum: + raise RuntimeError(f"{name} must be >= {minimum}, got {value}") + return value + + +def read_float_env(name: str, default: float, minimum: float = 0.0) -> float: + raw = os.getenv(name) + if raw is None: + return default + try: + value = float(raw) + except ValueError as exc: + raise RuntimeError(f"{name} must be a number, got {raw!r}") from exc + if value < minimum: + raise RuntimeError(f"{name} must be >= {minimum}, got {value}") + return value + + ROOT = Path(__file__).resolve().parents[1] TRUST_TIERS = ("official", "verified", "community") CATALOG_ROOT = ROOT / "modules" @@ -22,6 +52,9 @@ V1_ROOT = DIST_ROOT / "v1" SCHEMA_SRC = ROOT / "schemas" SCHEMA_OUT = V1_ROOT / "schema" +NPM_FETCH_TIMEOUT_SECONDS = read_int_env("CHOYSUM_NPM_FETCH_TIMEOUT_SECONDS", 10) +NPM_FETCH_MAX_RETRIES = read_int_env("CHOYSUM_NPM_FETCH_MAX_RETRIES", 3) +NPM_FETCH_BACKOFF_SECONDS = read_float_env("CHOYSUM_NPM_FETCH_BACKOFF_SECONDS", 1.0) def write_text(path: Path, content: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) @@ -35,15 +68,39 @@ def load_json(path: Path) -> Any: with path.open("r", encoding="utf-8") as handle: return json.load(handle) + def fetch_npm_meta(package_name: str) -> dict: quoted_name = urllib.parse.quote(package_name, safe="@") url = f"https://registry.npmjs.org/{quoted_name}" req = urllib.request.Request(url, headers={"User-Agent": "Choysum-Catalog-Builder/1.0"}) - try: - with urllib.request.urlopen(req, timeout=10) as response: - return json.loads(response.read().decode('utf-8')) - except (urllib.error.URLError, json.JSONDecodeError) as e: - raise RuntimeError(f"Failed to fetch or parse {package_name} from NPM: {e}") + last_error: Exception | None = None + + for attempt in range(1, NPM_FETCH_MAX_RETRIES + 1): + try: + with urllib.request.urlopen(req, timeout=NPM_FETCH_TIMEOUT_SECONDS) as response: + payload = json.loads(response.read().decode("utf-8")) + if not isinstance(payload, dict): + raise ValueError("NPM registry response is not a JSON object") + return payload + except urllib.error.HTTPError as exc: + if exc.code == 404: + raise RuntimeError( + f"Package '{package_name}' was not found on NPM (404)." + ) from exc + last_error = exc + except (urllib.error.URLError, json.JSONDecodeError, ValueError) as exc: + last_error = exc + + if attempt < NPM_FETCH_MAX_RETRIES: + backoff_seconds = NPM_FETCH_BACKOFF_SECONDS * (2 ** (attempt - 1)) + time.sleep(backoff_seconds) + + if last_error is None: + raise RuntimeError(f"Failed to fetch metadata for {package_name}: unknown error") + raise RuntimeError( + f"Failed to fetch or parse {package_name} from NPM " + f"after {NPM_FETCH_MAX_RETRIES} attempts: {last_error}" + ) from last_error def process_module(entry_file: Path) -> tuple[str, dict[str, Any]]: entry = load_json(entry_file) @@ -100,30 +157,39 @@ def process_module(entry_file: Path) -> tuple[str, dict[str, Any]]: def collect_modules() -> dict[str, dict[str, Any]]: modules: dict[str, dict[str, Any]] = {} - tasks = {} - + module_sources: dict[str, Path] = {} + tasks: dict[concurrent.futures.Future[tuple[str, dict[str, Any]]], Path] = {} + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: for trust in TRUST_TIERS: tier_dir = CATALOG_ROOT / trust if not tier_dir.is_dir(): continue - + for entry_file in sorted(tier_dir.glob("*.json")): future = executor.submit(process_module, entry_file) tasks[future] = entry_file - - errors = [] + + errors: list[str] = [] for future in concurrent.futures.as_completed(tasks): entry_file = tasks[future] try: module_id, mod_payload = future.result() + if module_id in modules: + existing_file = module_sources[module_id] + errors.append( + f" - Duplicate module ID '{module_id}' between " + f"{existing_file.relative_to(ROOT)} and {entry_file.relative_to(ROOT)}" + ) + continue modules[module_id] = mod_payload + module_sources[module_id] = entry_file except Exception as e: errors.append(f" - {entry_file.relative_to(ROOT)}: {e}") - + if errors: raise RuntimeError("Failed to collect all modules due to the following errors:\n" + "\n".join(errors)) - + return modules def utc_now_iso() -> str: @@ -138,15 +204,15 @@ def generate_checksums(files: list[Path]) -> str: return "\n".join(lines) + "\n" def build() -> None: - import shutil + modules = collect_modules() + generated_at = utc_now_iso() + if DIST_ROOT.exists(): shutil.rmtree(DIST_ROOT) DIST_ROOT.mkdir(parents=True, exist_ok=True) V1_ROOT.mkdir(parents=True, exist_ok=True) SCHEMA_OUT.mkdir(parents=True, exist_ok=True) - generated_at = utc_now_iso() - modules = collect_modules() index_payload = { "generatedAt": generated_at, "modules": modules, From 92e1d82a6a5aee08659e655ac5e150430a94e97d Mon Sep 17 00:00:00 2001 From: buke Date: Tue, 9 Jun 2026 22:27:58 +0800 Subject: [PATCH 11/20] chore(ci): configure catalog fetch retry environment - Add CHOYSUM_NPM_FETCH_TIMEOUT_SECONDS for request timeout. - Add CHOYSUM_NPM_FETCH_MAX_RETRIES for transient failure retries. - Add CHOYSUM_NPM_FETCH_BACKOFF_SECONDS for exponential backoff base delay. --- .github/workflows/build-catalog.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build-catalog.yml b/.github/workflows/build-catalog.yml index 5ca1ef1..32f26b3 100644 --- a/.github/workflows/build-catalog.yml +++ b/.github/workflows/build-catalog.yml @@ -11,6 +11,10 @@ on: jobs: build: runs-on: ubuntu-latest + env: + CHOYSUM_NPM_FETCH_TIMEOUT_SECONDS: "10" + CHOYSUM_NPM_FETCH_MAX_RETRIES: "3" + CHOYSUM_NPM_FETCH_BACKOFF_SECONDS: "1.0" steps: - name: Checkout uses: actions/checkout@v6 From 87610a57e54bc81410e467c41e401b77932250b6 Mon Sep 17 00:00:00 2001 From: buke Date: Tue, 9 Jun 2026 22:33:45 +0800 Subject: [PATCH 12/20] fix(catalog): normalize integrity and keep utf8 index output - Use "dist_meta.get('integrity') or ''" to avoid emitting null integrity values. - Set ensure_ascii=False for canonical index JSON to preserve UTF-8 characters. --- scripts/build_catalog.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_catalog.py b/scripts/build_catalog.py index a25cc10..177a2c2 100755 --- a/scripts/build_catalog.py +++ b/scripts/build_catalog.py @@ -138,7 +138,7 @@ def process_module(entry_file: Path) -> tuple[str, dict[str, Any]]: v_entry = { "tarball": tarball_url, - "integrity": dist_meta.get("integrity", ""), + "integrity": dist_meta.get("integrity") or "", "depends": depends, "peerDependencies": peer_deps } @@ -218,7 +218,7 @@ def build() -> None: "modules": modules, "version": 1, } - canonical_index = json.dumps(index_payload, sort_keys=True, separators=(",", ":")) + canonical_index = json.dumps(index_payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) index_hash = hashlib.sha256(canonical_index.encode("utf-8")).hexdigest() index_path = V1_ROOT / "index.json" From 93c559ebe4ff7245c05c7ef8181c39fc2c023970 Mon Sep 17 00:00:00 2001 From: buke Date: Tue, 9 Jun 2026 22:39:34 +0800 Subject: [PATCH 13/20] fix(catalog): tighten response guards and retry rules - Fail fast for non-retryable client HTTP errors during NPM fetches. - Validate that npm_data.versions is a dict before iterating items. - Use strftime for UTC timestamp serialization with Z suffix. --- scripts/build_catalog.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/build_catalog.py b/scripts/build_catalog.py index 177a2c2..348dff7 100755 --- a/scripts/build_catalog.py +++ b/scripts/build_catalog.py @@ -83,9 +83,13 @@ def fetch_npm_meta(package_name: str) -> dict: raise ValueError("NPM registry response is not a JSON object") return payload except urllib.error.HTTPError as exc: - if exc.code == 404: + if exc.code in (400, 401, 403, 404): raise RuntimeError( - f"Package '{package_name}' was not found on NPM (404)." + f"Package '{package_name}' fetch failed with status {exc.code}." + ) from exc + if 400 <= exc.code < 500 and exc.code not in (408, 429): + raise RuntimeError( + f"Package '{package_name}' fetch failed with status {exc.code}." ) from exc last_error = exc except (urllib.error.URLError, json.JSONDecodeError, ValueError) as exc: @@ -115,7 +119,10 @@ def process_module(entry_file: Path) -> tuple[str, dict[str, Any]]: npm_data = fetch_npm_meta(package_name) versions_out = {} - for ver, v_data in (npm_data.get("versions") or {}).items(): + versions_raw = npm_data.get("versions") + if not isinstance(versions_raw, dict): + versions_raw = {} + for ver, v_data in versions_raw.items(): if not isinstance(v_data, dict): continue choysum_meta = v_data.get("choysum") @@ -193,7 +200,7 @@ def collect_modules() -> dict[str, dict[str, Any]]: return modules def utc_now_iso() -> str: - return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") def generate_checksums(files: list[Path]) -> str: lines = [] From e31cc8c0eee9d7d7d3aff8d667815ca685cbe1cd Mon Sep 17 00:00:00 2001 From: buke Date: Tue, 9 Jun 2026 22:57:47 +0800 Subject: [PATCH 14/20] fix(catalog): simplify 4xx checks and narrow retry exceptions - Remove redundant client-status check while preserving non-retryable 4xx fail-fast behavior. - Expand retry handling with explicit network/transport exception types instead of broad catch-all logic. --- scripts/build_catalog.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/scripts/build_catalog.py b/scripts/build_catalog.py index 348dff7..b8a4f2e 100755 --- a/scripts/build_catalog.py +++ b/scripts/build_catalog.py @@ -6,10 +6,12 @@ from __future__ import annotations import concurrent.futures +import http.client import hashlib import json import os import shutil +import socket import time import urllib.error import urllib.parse @@ -83,16 +85,20 @@ def fetch_npm_meta(package_name: str) -> dict: raise ValueError("NPM registry response is not a JSON object") return payload except urllib.error.HTTPError as exc: - if exc.code in (400, 401, 403, 404): - raise RuntimeError( - f"Package '{package_name}' fetch failed with status {exc.code}." - ) from exc if 400 <= exc.code < 500 and exc.code not in (408, 429): raise RuntimeError( f"Package '{package_name}' fetch failed with status {exc.code}." ) from exc last_error = exc - except (urllib.error.URLError, json.JSONDecodeError, ValueError) as exc: + except ( + urllib.error.URLError, + http.client.HTTPException, + socket.timeout, + TimeoutError, + ConnectionError, + json.JSONDecodeError, + ValueError, + ) as exc: last_error = exc if attempt < NPM_FETCH_MAX_RETRIES: From c163ab035846b55258f062cc9971a8f12c4e8eb5 Mon Sep 17 00:00:00 2001 From: buke Date: Wed, 10 Jun 2026 08:11:11 +0800 Subject: [PATCH 15/20] fix(catalog): align hashing and harden build cleanup - Hash the exact newline-terminated index payload written to disk to keep indexHash and file checksums consistent. - Make DIST_ROOT cleanup safe for directory, file, and symlink cases. - Add CHOYSUM_BUILD_CONCURRENCY to configure ThreadPoolExecutor worker count. --- scripts/build_catalog.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/scripts/build_catalog.py b/scripts/build_catalog.py index b8a4f2e..bb6641b 100755 --- a/scripts/build_catalog.py +++ b/scripts/build_catalog.py @@ -57,6 +57,7 @@ def read_float_env(name: str, default: float, minimum: float = 0.0) -> float: NPM_FETCH_TIMEOUT_SECONDS = read_int_env("CHOYSUM_NPM_FETCH_TIMEOUT_SECONDS", 10) NPM_FETCH_MAX_RETRIES = read_int_env("CHOYSUM_NPM_FETCH_MAX_RETRIES", 3) NPM_FETCH_BACKOFF_SECONDS = read_float_env("CHOYSUM_NPM_FETCH_BACKOFF_SECONDS", 1.0) +BUILD_CONCURRENCY = read_int_env("CHOYSUM_BUILD_CONCURRENCY", 5) def write_text(path: Path, content: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) @@ -173,7 +174,7 @@ def collect_modules() -> dict[str, dict[str, Any]]: module_sources: dict[str, Path] = {} tasks: dict[concurrent.futures.Future[tuple[str, dict[str, Any]]], Path] = {} - with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + with concurrent.futures.ThreadPoolExecutor(max_workers=BUILD_CONCURRENCY) as executor: for trust in TRUST_TIERS: tier_dir = CATALOG_ROOT / trust if not tier_dir.is_dir(): @@ -220,8 +221,12 @@ def build() -> None: modules = collect_modules() generated_at = utc_now_iso() - if DIST_ROOT.exists(): + if DIST_ROOT.is_symlink(): + DIST_ROOT.unlink() + elif DIST_ROOT.exists() and DIST_ROOT.is_dir(): shutil.rmtree(DIST_ROOT) + elif DIST_ROOT.exists(): + DIST_ROOT.unlink() DIST_ROOT.mkdir(parents=True, exist_ok=True) V1_ROOT.mkdir(parents=True, exist_ok=True) SCHEMA_OUT.mkdir(parents=True, exist_ok=True) @@ -231,13 +236,13 @@ def build() -> None: "modules": modules, "version": 1, } - canonical_index = json.dumps(index_payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + canonical_index = json.dumps(index_payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n" index_hash = hashlib.sha256(canonical_index.encode("utf-8")).hexdigest() index_path = V1_ROOT / "index.json" index_hashed_path = V1_ROOT / f"index.{index_hash}.json" - write_text(index_path, canonical_index + "\n") - write_text(index_hashed_path, canonical_index + "\n") + write_text(index_path, canonical_index) + write_text(index_hashed_path, canonical_index) meta_payload = { "generatedAt": generated_at, From ac1476e522b7af82994ac76054894c73fa9bf955 Mon Sep 17 00:00:00 2001 From: buke Date: Wed, 10 Jun 2026 08:14:39 +0800 Subject: [PATCH 16/20] chore(ci): add catalog build concurrency env Expose CHOYSUM_BUILD_CONCURRENCY in build-catalog workflow so CI can control ThreadPoolExecutor workers explicitly. --- .github/workflows/build-catalog.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-catalog.yml b/.github/workflows/build-catalog.yml index 32f26b3..86998e8 100644 --- a/.github/workflows/build-catalog.yml +++ b/.github/workflows/build-catalog.yml @@ -15,6 +15,7 @@ jobs: CHOYSUM_NPM_FETCH_TIMEOUT_SECONDS: "10" CHOYSUM_NPM_FETCH_MAX_RETRIES: "3" CHOYSUM_NPM_FETCH_BACKOFF_SECONDS: "1.0" + CHOYSUM_BUILD_CONCURRENCY: "5" steps: - name: Checkout uses: actions/checkout@v6 From 939da504d7323d315c00e14ee709e21b8dfdd4ad Mon Sep 17 00:00:00 2001 From: buke Date: Wed, 10 Jun 2026 08:23:52 +0800 Subject: [PATCH 17/20] fix(catalog): enforce integrity and gate build validation - Enforce non-empty per-version integrity with shasum fallback conversion to sha1 SRI. - Fail module processing when no valid versions are found. - Require _headers and _redirects static files during dist generation. - Run scripts/build_catalog.py in validate workflow with explicit CHOYSUM_* env settings. --- .github/workflows/validate-catalog.yml | 10 +++++- scripts/build_catalog.py | 45 ++++++++++++++++++++++---- 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/.github/workflows/validate-catalog.yml b/.github/workflows/validate-catalog.yml index b3f7484..a44a614 100644 --- a/.github/workflows/validate-catalog.yml +++ b/.github/workflows/validate-catalog.yml @@ -11,6 +11,11 @@ on: jobs: validate: runs-on: ubuntu-latest + env: + CHOYSUM_NPM_FETCH_TIMEOUT_SECONDS: "10" + CHOYSUM_NPM_FETCH_MAX_RETRIES: "3" + CHOYSUM_NPM_FETCH_BACKOFF_SECONDS: "1.0" + CHOYSUM_BUILD_CONCURRENCY: "5" steps: - name: Checkout uses: actions/checkout@v6 @@ -35,4 +40,7 @@ jobs: check-jsonschema --schemafile schemas/catalog-entry.schema.json "${files[@]}" - name: Validate catalog pointers and structure - run: python scripts/validate_catalog.py \ No newline at end of file + run: python scripts/validate_catalog.py + + - name: Validate catalog build generation + run: python scripts/build_catalog.py \ No newline at end of file diff --git a/scripts/build_catalog.py b/scripts/build_catalog.py index bb6641b..6505203 100755 --- a/scripts/build_catalog.py +++ b/scripts/build_catalog.py @@ -5,6 +5,7 @@ from __future__ import annotations +import base64 import concurrent.futures import http.client import hashlib @@ -72,6 +73,26 @@ def load_json(path: Path) -> Any: return json.load(handle) +def resolve_integrity(dist_meta: dict[str, Any], package_name: str, version: str) -> str: + integrity = dist_meta.get("integrity") + if isinstance(integrity, str) and integrity.strip(): + return integrity + + shasum = dist_meta.get("shasum") + if isinstance(shasum, str) and shasum.strip(): + try: + digest = bytes.fromhex(shasum.strip()) + except ValueError as exc: + raise ValueError( + f"Invalid shasum hex for package '{package_name}' version '{version}'" + ) from exc + return "sha1-" + base64.b64encode(digest).decode("ascii") + + raise ValueError( + f"Missing integrity hash for package '{package_name}' version '{version}'" + ) + + def fetch_npm_meta(package_name: str) -> dict: quoted_name = urllib.parse.quote(package_name, safe="@") url = f"https://registry.npmjs.org/{quoted_name}" @@ -150,9 +171,11 @@ def process_module(entry_file: Path) -> tuple[str, dict[str, Any]]: if not isinstance(peer_deps, dict): peer_deps = {} + integrity = resolve_integrity(dist_meta, package_name, ver) + v_entry = { "tarball": tarball_url, - "integrity": dist_meta.get("integrity") or "", + "integrity": integrity, "depends": depends, "peerDependencies": peer_deps } @@ -160,6 +183,11 @@ def process_module(entry_file: Path) -> tuple[str, dict[str, Any]]: v_entry["compatibility"] = choysum_meta["compatibility"] versions_out[ver] = v_entry + + if not versions_out: + raise ValueError( + f"No valid versions found for package '{package_name}' (module: {module_id})" + ) return module_id, { "moduleId": module_id, @@ -259,12 +287,15 @@ def build() -> None: target.write_text(schema_file.read_text(encoding="utf-8"), encoding="utf-8") checksum_files.append(target) - # Note: _headers and _redirects are managed statically in the Git repository now. - # We will copy them to the dist folder so Cloudflare Pages can use them. - if (ROOT / "_headers").exists(): - write_text(DIST_ROOT / "_headers", (ROOT / "_headers").read_text(encoding="utf-8")) - if (ROOT / "_redirects").exists(): - write_text(DIST_ROOT / "_redirects", (ROOT / "_redirects").read_text(encoding="utf-8")) + headers_src = ROOT / "_headers" + redirects_src = ROOT / "_redirects" + if not headers_src.is_file(): + raise RuntimeError(f"Missing required static file: {headers_src.relative_to(ROOT)}") + if not redirects_src.is_file(): + raise RuntimeError(f"Missing required static file: {redirects_src.relative_to(ROOT)}") + + write_text(DIST_ROOT / "_headers", headers_src.read_text(encoding="utf-8")) + write_text(DIST_ROOT / "_redirects", redirects_src.read_text(encoding="utf-8")) # Finally generate checksums checksums_path = V1_ROOT / "checksums.txt" From 6d3275a5125882203aab4de12d6c69d5f812f737 Mon Sep 17 00:00:00 2001 From: buke Date: Wed, 10 Jun 2026 08:38:23 +0800 Subject: [PATCH 18/20] perf(build): stream checksum hashing in catalog build - Replace full-file read_bytes hashing with chunked hashing in generate_checksums. - Keep checksum output format and ordering unchanged while reducing peak memory usage. --- scripts/build_catalog.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/build_catalog.py b/scripts/build_catalog.py index 6505203..6b93296 100755 --- a/scripts/build_catalog.py +++ b/scripts/build_catalog.py @@ -241,8 +241,11 @@ def generate_checksums(files: list[Path]) -> str: lines = [] for f in sorted(files): rel_path = f"/{f.relative_to(DIST_ROOT).as_posix()}" - h = hashlib.sha256(f.read_bytes()).hexdigest() - lines.append(f"{h} {rel_path}") + hasher = hashlib.sha256() + with f.open("rb") as handle: + for chunk in iter(lambda: handle.read(65536), b""): + hasher.update(chunk) + lines.append(f"{hasher.hexdigest()} {rel_path}") return "\n".join(lines) + "\n" def build() -> None: From 688ab5de7743d0f471df237ed9114859e2aba961 Mon Sep 17 00:00:00 2001 From: buke Date: Wed, 10 Jun 2026 08:48:49 +0800 Subject: [PATCH 19/20] fix(build): validate shasum length for integrity fallback - Enforce 40-character SHA-1 hex length before converting shasum to SRI. - Keep fail-fast behavior for malformed integrity metadata. --- scripts/build_catalog.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/build_catalog.py b/scripts/build_catalog.py index 6b93296..889a6f5 100755 --- a/scripts/build_catalog.py +++ b/scripts/build_catalog.py @@ -80,8 +80,13 @@ def resolve_integrity(dist_meta: dict[str, Any], package_name: str, version: str shasum = dist_meta.get("shasum") if isinstance(shasum, str) and shasum.strip(): + shasum_str = shasum.strip() + if len(shasum_str) != 40: + raise ValueError( + f"Invalid shasum length for package '{package_name}' version '{version}'" + ) try: - digest = bytes.fromhex(shasum.strip()) + digest = bytes.fromhex(shasum_str) except ValueError as exc: raise ValueError( f"Invalid shasum hex for package '{package_name}' version '{version}'" From 5de52671ad149b5f523c69104492144bb755349c Mon Sep 17 00:00:00 2001 From: buke Date: Wed, 10 Jun 2026 08:56:23 +0800 Subject: [PATCH 20/20] fix(build): sort checksums by relative posix path - Make checksum file ordering deterministic across operating systems. - Sort paths using DIST_ROOT-relative POSIX string keys in generate_checksums. --- scripts/build_catalog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_catalog.py b/scripts/build_catalog.py index 889a6f5..c1e1ca6 100755 --- a/scripts/build_catalog.py +++ b/scripts/build_catalog.py @@ -244,7 +244,7 @@ def utc_now_iso() -> str: def generate_checksums(files: list[Path]) -> str: lines = [] - for f in sorted(files): + for f in sorted(files, key=lambda p: p.relative_to(DIST_ROOT).as_posix()): rel_path = f"/{f.relative_to(DIST_ROOT).as_posix()}" hasher = hashlib.sha256() with f.open("rb") as handle: