From 9ef48cb7c31950153a4edd3ea05fa73c2f8d7e8f Mon Sep 17 00:00:00 2001 From: Aric Camarata Date: Fri, 4 Sep 2026 06:37:21 -0400 Subject: [PATCH] fix(registry): write, document, and validate the flat checksum field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CLI installer's registry parser (nself-org/cli internal/plugin/registry_parse.go pluginEntry.Checksum, json:"checksum") and installer_locked.go's verifyChecksum() only ever read the FLAT `checksum` field on a plugin entry. build-and-upload-tarballs.sh and registry-schema.json only ever wrote/documented the nested `checksums.sha256` field, which no CLI code path consumes (confirmed via `git grep checksums origin/main -- internal` on nself-org/cli). Result: every plugin without a hand-written flat checksum installs with checksum verification silently skipped by the CLI (a stderr warning only) for any plugin not explicitly status=stable. - build-and-upload-tarballs.sh: write the flat `checksum` field alongside the nested `checksums.sha256` (both raw lowercase hex, no "sha256:" prefix — the previous nested write had the prefix, which would have broken any future checksum == checksums.sha256 consistency check). - registry-schema.json: document `checksum` as the field the CLI reads; require it (non-empty) when a plugin's `status` is the literal string "stable"; document `checksums` as legacy/unread. - shared/validate-registry.sh: new CHECK-15 — errors when a status=stable plugin lacks a valid flat checksum, when a present checksum is not 64 lowercase hex chars, or when checksum and checksums.sha256 disagree. - scripts/verify-published-checksums.sh: new read-only spot-check that downloads a release tag's real tarball assets and hashes them, since the other 121 plugins' checksums were never computed against what the CLI would actually download and recomputing them from the source tree would not verify that. Does not write registry.json. Does not touch the 8 plugin entries PR #81 already populated (both fields, no "sha256:" prefix) for v1.0.1 — rebase-safe with that PR. No plugin in the current registry has an explicit status=stable, so this changes no install-time behavior yet; it fixes the field the installer reads once entries are marked stable. --- registry-schema.json | 20 ++++ scripts/build-and-upload-tarballs.sh | 10 +- scripts/verify-published-checksums.sh | 127 ++++++++++++++++++++++++++ shared/validate-registry.sh | 91 ++++++++++++++++++ 4 files changed, 247 insertions(+), 1 deletion(-) create mode 100755 scripts/verify-published-checksums.sh diff --git a/registry-schema.json b/registry-schema.json index 423153a5..867452d3 100644 --- a/registry-schema.json +++ b/registry-schema.json @@ -58,6 +58,20 @@ "plugin": { "type": "object", "required": ["version", "description"], + "allOf": [ + { + "if": { + "properties": { "status": { "const": "stable" } }, + "required": ["status"] + }, + "then": { + "required": ["checksum"], + "properties": { + "checksum": { "type": "string", "minLength": 64 } + } + } + } + ], "properties": { "name": { "type": "string", @@ -247,8 +261,14 @@ }, "description": "Multi-app isolation support configuration" }, + "checksum": { + "type": "string", + "pattern": "^([a-f0-9]{64})?$", + "description": "SHA-256 checksum of the plugin tarball, lowercase 64-char hex, no 'sha256:' prefix. Empty string means not yet computed (pre-existing placeholder, e.g. ollama). This is the FLAT field the CLI installer actually reads (nself-org/cli internal/plugin/registry_parse.go pluginEntry.Checksum, json:\"checksum\") and enforces in internal/plugin/installer_locked.go's verifyChecksum(). A non-empty value is REQUIRED when status is 'stable' (see the plugin-level conditional below) — a stable plugin with no checksum is refused at install time. Must match checksums.sha256 below when both are non-empty (shared/validate-registry.sh CHECK-15)." + }, "checksums": { "type": "object", + "description": "Legacy/nested checksum record. NOT read by any known CLI code path (verified against nself-org/cli internal/, 2026-09) — kept for tooling that may still reference it and must stay equal to the flat `checksum` field above.", "properties": { "sha256": { "type": "string" } } diff --git a/scripts/build-and-upload-tarballs.sh b/scripts/build-and-upload-tarballs.sh index 1cb23f7b..b10fd288 100755 --- a/scripts/build-and-upload-tarballs.sh +++ b/scripts/build-and-upload-tarballs.sh @@ -133,13 +133,21 @@ for update in "${REGISTRY_UPDATES[@]}"; do sha256="${rest2%%:::*}" release_tag="${rest2##*:::}" + # NOTE (P6 checksum-field fix): the CLI installer's registry parser + # (nself-org/cli internal/plugin/registry_parse.go pluginEntry.Checksum, + # `json:"checksum"`) and installer_locked.go's verifyChecksum() read the + # FLAT `checksum` field, never the nested `checksums.sha256` below. Both + # are written here, raw lowercase hex with NO "sha256:" prefix, so they + # stay in sync (shared/validate-registry.sh CHECK-15 fails the registry if + # they ever disagree). REGISTRY_JSON="$(printf '%s' "$REGISTRY_JSON" | jq \ --arg name "$plugin_name" \ --arg url "$tarball_url" \ - --arg sha "sha256:${sha256}" \ + --arg sha "$sha256" \ --arg tag "$release_tag" \ --arg sig "" \ '(.plugins[$name].tarballUrl) = $url | + (.plugins[$name].checksum) = $sha | (.plugins[$name].checksums.sha256) = $sha | (.plugins[$name].releaseTag) = $tag | (.plugins[$name].signature) = $sig')" diff --git a/scripts/verify-published-checksums.sh b/scripts/verify-published-checksums.sh new file mode 100755 index 00000000..b81b9627 --- /dev/null +++ b/scripts/verify-published-checksums.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# verify-published-checksums.sh +# +# Purpose: spot-check that the checksum registry.json WOULD carry for a +# release tag matches the SHA-256 of the tarball actually served for that +# tag on the GitHub release. This never writes registry.json — it downloads +# each plugin's real tarball asset for TAG and diffs the computed SHA-256 +# against the registry's checksum/checksums.sha256 (if either is already +# populated for that tag) and prints the computed value regardless, so it +# can seed a future registry update by hand or via build-and-upload-tarballs.sh. +# +# Why this exists (P6, registry-flat-checksum-field fix): most of this +# registry's 129 plugins have never had ANY checksum field computed against +# their real published v1.0.0 asset — recomputing all of them from the repo +# source tree would NOT verify what a user's `nself plugin install` actually +# downloads, since the published tarball's bytes depend on the exact `tar` +# invocation/timestamps used at release time, not just file contents. This +# script downloads the real released asset and hashes THAT. +# +# Inputs: TAG (e.g. v1.0.0), PLUGIN_NAMES (space-separated plugin slugs). +# Outputs: a table of plugin / computed sha256 / registry checksum / +# registry checksums.sha256 / match — to stdout. Exit 0 always +# (informational spot-check, not a CI gate); pass --strict to +# exit 1 on any mismatch against a NON-EMPTY registry value. +# Constraints: read-only against registry.json and GitHub; requires gh CLI +# authenticated + jq + sha256sum/shasum. Never edits registry.json. +# +# Usage: +# ./scripts/verify-published-checksums.sh v1.0.0 storage cron notify search maintenance +# ./scripts/verify-published-checksums.sh --strict v1.0.0 ollama + +set -euo pipefail + +REPO="nself-org/plugins" +STRICT=false + +if [ "${1:-}" = "--strict" ]; then + STRICT=true + shift +fi + +TAG="${1:-}" +shift || true +PLUGIN_NAMES=("$@") + +if [ -z "$TAG" ] || [ "${#PLUGIN_NAMES[@]}" -eq 0 ]; then + printf "Usage: %s [--strict] TAG PLUGIN_NAME [PLUGIN_NAME ...]\n" "$0" >&2 + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REGISTRY_FILE="$(cd "${SCRIPT_DIR}/.." && pwd)/registry.json" +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "$WORK_DIR"' EXIT + +log() { printf "[verify-published-checksums] %s\n" "$*"; } +err() { printf "[verify-published-checksums] ERROR: %s\n" "$*" >&2; } + +if ! command -v gh >/dev/null 2>&1; then + err "gh CLI not found." + exit 1 +fi +if ! command -v jq >/dev/null 2>&1; then + err "jq not found." + exit 1 +fi + +sha256_file() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | cut -d' ' -f1 + else + shasum -a 256 "$1" | cut -d' ' -f1 + fi +} + +if ! gh release view "$TAG" --repo "$REPO" >/dev/null 2>&1; then + err "Release $TAG not found in $REPO." + exit 1 +fi + +MISMATCHES=0 + +printf "%-20s %-10s %-64s %-10s %-10s\n" "plugin" "version" "computed_sha256" "reg.flat" "reg.nested" +printf "%-20s %-10s %-64s %-10s %-10s\n" "------" "-------" "---------------" "--------" "----------" + +for plugin_name in "${PLUGIN_NAMES[@]}"; do + version="${TAG#v}" + tarball_name="${plugin_name}-${version}.tar.gz" + asset_path="${WORK_DIR}/${tarball_name}" + + if ! gh release download "$TAG" --repo "$REPO" --pattern "$tarball_name" --dir "$WORK_DIR" --clobber >/dev/null 2>&1; then + printf "%-20s %-10s %-64s %-10s %-10s\n" "$plugin_name" "$version" "(asset not found on $TAG)" "-" "-" + continue + fi + + computed_sha="$(sha256_file "$asset_path")" + + reg_flat="$(jq -r --arg n "$plugin_name" '.plugins[$n].checksum // ""' "$REGISTRY_FILE")" + reg_nested="$(jq -r --arg n "$plugin_name" '.plugins[$n].checksums.sha256 // ""' "$REGISTRY_FILE")" + reg_nested_norm="${reg_nested#sha256:}" + + match_flag="" + if [ -n "$reg_flat" ] && [ "$reg_flat" != "$computed_sha" ]; then + match_flag="${match_flag} FLAT-MISMATCH" + MISMATCHES=$((MISMATCHES + 1)) + fi + if [ -n "$reg_nested_norm" ] && [ "$reg_nested_norm" != "$computed_sha" ]; then + match_flag="${match_flag} NESTED-MISMATCH" + MISMATCHES=$((MISMATCHES + 1)) + fi + [ -z "$match_flag" ] && match_flag="ok-or-unset" + + printf "%-20s %-10s %-64s %-10s %-10s\n" \ + "$plugin_name" "$version" "$computed_sha" "${reg_flat:--}" "${reg_nested:--}" + log " ${plugin_name}: ${match_flag}" +done + +if [ "$MISMATCHES" -gt 0 ]; then + err "$MISMATCHES mismatch(es) between a computed checksum and a NON-EMPTY registry value." + if [ "$STRICT" = "true" ]; then + exit 1 + fi +else + log "No mismatches against any non-empty registry checksum field." +fi + +log "Done. Registry.json was NOT modified by this script." diff --git a/shared/validate-registry.sh b/shared/validate-registry.sh index 4aa561bc..9836a84e 100755 --- a/shared/validate-registry.sh +++ b/shared/validate-registry.sh @@ -287,6 +287,15 @@ _in_list() { return 1 } +_is_valid_sha256() { + local v="$1" + [ "${#v}" -eq 64 ] || return 1 + case "$v" in + *[!0-9a-f]*) return 1 ;; + esac + return 0 +} + # --------------------------------------------------------------------------- # Iterators — produce uniform stream of fields separated by ASCII Unit # Separator (0x1f). Tabs collapse consecutive empties under bash `read` @@ -855,6 +864,88 @@ else warn "CHECK-14" "registry" "python3 not available — skipping duplicate table-prefix check" fi +# ============================================================================= +# CHECK 15 — checksum integrity (added P6, registry-flat-checksum-field fix) +# +# nself-org/cli's installer only ever reads the FLAT `checksum` field +# (internal/plugin/registry_parse.go pluginEntry.Checksum, json:"checksum") +# and its installer_locked.go verifyChecksum() hard-refuses installing a +# status=stable plugin with no checksum. The nested `checksums.sha256` field +# that build-and-upload-tarballs.sh also writes is NOT consumed by any known +# CLI code path (confirmed via `git grep checksums origin/main -- internal` +# on nself-org/cli returning no registry-entry consumer). So: +# - a stable plugin missing a non-empty flat checksum is an ERROR +# - a non-empty checksum must be 64 lowercase hex chars +# - when both checksum and checksums.sha256 are non-empty they must agree +# (a "sha256:" prefix on the legacy nested field is tolerated/stripped +# before comparing, since some historical writes included it) +# ============================================================================= +section "CHECK 15 — checksum integrity (flat field required for status=stable)" +CHECKSUM_VIOLATIONS=0 +CHECKSUM_USV="${TMPDIR_VAL}/checksum.usv" +case "$REGISTRY_FORMAT" in + aggregated) + "$JQ" -r ' + .plugins + | to_entries[] + | [ + .key, + (.value.status // ""), + (.value.checksum // ""), + (.value.checksums.sha256 // "") + ] + | join("'"$US"'") + ' "$REGISTRY_FILE" > "$CHECKSUM_USV" + ;; + array) + "$JQ" -r ' + .[] + | [ + (.name // ""), + (.status // ""), + (.checksum // ""), + (.checksums.sha256 // "") + ] + | join("'"$US"'") + ' "$REGISTRY_FILE" > "$CHECKSUM_USV" + ;; + array-wrapped) + "$JQ" -r ' + .plugins[] + | [ + (.name // ""), + (.status // ""), + (.checksum // ""), + (.checksums.sha256 // "") + ] + | join("'"$US"'") + ' "$REGISTRY_FILE" > "$CHECKSUM_USV" + ;; +esac +while IFS="$US" read -r name status checksum checksums_sha; do + [ -z "$name" ] && continue + if [ "$status" = "stable" ] && [ -z "$checksum" ]; then + err "CHECK-15" "$name" "status=stable but flat 'checksum' field is missing/empty — cli's verifyChecksum() hard-refuses installing a stable plugin with no checksum (it reads .checksum, NOT .checksums.sha256)" + CHECKSUM_VIOLATIONS=$((CHECKSUM_VIOLATIONS + 1)) + fi + if [ -n "$checksum" ] && ! _is_valid_sha256 "$checksum"; then + err "CHECK-15" "$name" "checksum '$checksum' is not a lowercase 64-char hex sha256" + CHECKSUM_VIOLATIONS=$((CHECKSUM_VIOLATIONS + 1)) + fi + if [ -n "$checksum" ] && [ -n "$checksums_sha" ]; then + checksums_sha_norm="${checksums_sha#sha256:}" + checksum_lc="$(printf '%s' "$checksum" | tr 'A-F' 'a-f')" + checksums_sha_lc="$(printf '%s' "$checksums_sha_norm" | tr 'A-F' 'a-f')" + if [ "$checksum_lc" != "$checksums_sha_lc" ]; then + err "CHECK-15" "$name" "checksum ('$checksum') and checksums.sha256 ('$checksums_sha') disagree" + CHECKSUM_VIOLATIONS=$((CHECKSUM_VIOLATIONS + 1)) + fi + fi +done < "$CHECKSUM_USV" +if [ "$CHECKSUM_VIOLATIONS" -eq 0 ]; then + ok "CHECK-15" "checksum fields present/consistent for all status=stable plugins" +fi + # ============================================================================= # Optional: --strict mode runs the legacy per-plugin field validator # (kept for backward compatibility with the original validator behavior)