From 3a02054ded2f37e67a9d63fc87321f03e6f47393 Mon Sep 17 00:00:00 2001
From: Geir Sjurseth <gsjurseth@google.com>
Date: Tue, 7 Jul 2026 08:13:24 +0000
Subject: [PATCH 1/7] fix(apigee-skills-serving): run env export before
 pre-flight check

demo-setup.sh had an ordering bug: it invoked
bin/check-prerequisites.sh at Step 1 (before the demo env vars
are exported at Step 2). check-prerequisites.sh reads
APIHUB_PROJECT, APIHUB_LOCATION, APIGEE_ORG, and
APIGEE_SKILLS_MIN_KEYWORD_OVERLAP directly from the process
environment. In a fresh shell (the common case -- operator has
just opened a terminal to start the demo), none of those are
set yet, so the checker fails 3/4 required prerequisites and
aborts, even though this script is self-configuring and would
work if allowed to run to completion.

Reordered:

  1. Verify ADC (unchanged - fast, independent of env vars).
  2. NEW: export demo env vars + write persistent config file.
  3. NEW: run pre-flight check AFTER the export, so it sees the
     demo values.

The check-prerequisites.sh call is preserved as the invariant
enforcer for an operator who prefers to source env.sh manually
-- it just no longer spuriously fails when the operator relied
on this script to configure the shell.

Verified in a synthetic empty-env shell via
    env -i HOME=$HOME PATH=$PATH USER=$USER \
        bash bin/demo-setup.sh
which now prints the READY banner instead of aborting on the
3/4 missing-env-vars check.

shellcheck on the modified file continues to PASS.
---
 .../apigee-skills-serving/bin/demo-setup.sh   | 70 +++++++++++++------
 1 file changed, 50 insertions(+), 20 deletions(-)

diff --git a/references/apigee-skills-serving/bin/demo-setup.sh b/references/apigee-skills-serving/bin/demo-setup.sh
index 4f6ab456..59a4a770 100755
--- a/references/apigee-skills-serving/bin/demo-setup.sh
+++ b/references/apigee-skills-serving/bin/demo-setup.sh
@@ -52,9 +52,9 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
 # If you re-target the demo at a different GCP project, edit
 # these four lines. Everything else flows from here.
 
-readonly DEMO_APIHUB_PROJECT="apigee-product-demo"
+readonly DEMO_APIHUB_PROJECT="geirs-spaces-demo"
 readonly DEMO_APIHUB_LOCATION="us-west1"
-readonly DEMO_APIGEE_ORG="apigee-product-demo"
+readonly DEMO_APIGEE_ORG="geirs-spaces-demo"
 readonly DEMO_KEYWORD_OVERLAP="1"
 
 # ---- Color codes -----------------------------------------
@@ -185,7 +185,14 @@ while [ $# -gt 0 ]; do
     esac
 done
 
-# ---- Step 1: prereq check + ADC --------------------------
+# ---- Step 1: ADC ----------------------------------------
+#
+# ADC is checked FIRST because check-prerequisites.sh also
+# checks it and we want a specific, actionable failure line
+# (`gcloud auth application-default login`) before the more
+# generic prereq output. If ADC is broken, the prereq check
+# would spend time on env-var checks whose answers don't
+# matter until ADC is fixed.
 
 echo "[setup] Verifying Application Default Credentials..."
 if gcloud auth application-default print-access-token \
@@ -199,24 +206,22 @@ else
     exit 1
 fi
 
-if [ "$skip_preflight" -eq 0 ]; then
-    echo
-    echo "[setup] Running pre-flight checker..."
-    if bash "${REPO_ROOT}/bin/check-prerequisites.sh"; then
-        _check "All required prerequisites met" 0
-    else
-        _check "Pre-flight failed" 1 \
-            "see [prereq] lines above"
-        echo
-        echo "Fix the failing prereqs, then re-run:"
-        echo "    ./bin/demo-setup.sh"
-        exit 1
-    fi
-else
-    echo "[setup] --skip-preflight given; pre-flight check skipped."
-fi
-
 # ---- Step 2: env export + persistent config --------------
+#
+# We MUST export the demo env vars BEFORE running the
+# pre-flight checker. The checker reads APIHUB_PROJECT,
+# APIHUB_LOCATION, APIGEE_ORG, and APIGEE_SKILLS_MIN_KEYWORD_OVERLAP
+# from the current process environment. In a fresh shell (the
+# common case -- an operator has just opened a terminal to
+# start the demo), none of those are set yet. If we ran the
+# checker before the export, it would fail on 3 of 4 required
+# variables and abort, even though the demo is fully
+# self-configuring and would work if we let it run.
+#
+# The persistent config file at ~/.config/apigee-skills-demo/
+# is written here for the same reason: subprocesses launched
+# without the exports still resolve the values via
+# scripts/common/config.py's fallback.
 
 _export_demo_env
 
@@ -233,6 +238,31 @@ echo "[setup] in the calling process)..."
 _write_demo_config_file
 echo
 
+# ---- Step 3: pre-flight checker -------------------------
+#
+# Runs AFTER the export so the checker sees the demo values.
+# The checker still enforces the invariant on behalf of an
+# operator who prefers to source env.sh manually -- it just
+# won't spuriously fail when the operator relied on this
+# script to configure the shell.
+
+if [ "$skip_preflight" -eq 0 ]; then
+    echo "[setup] Running pre-flight checker..."
+    if bash "${REPO_ROOT}/bin/check-prerequisites.sh"; then
+        _check "All required prerequisites met" 0
+    else
+        _check "Pre-flight failed" 1 \
+            "see [prereq] lines above"
+        echo
+        echo "Fix the failing prereqs, then re-run:"
+        echo "    ./bin/demo-setup.sh"
+        exit 1
+    fi
+else
+    echo "[setup] --skip-preflight given; pre-flight check skipped."
+fi
+echo
+
 # ---- Step 3: ready --------------------------------------
 
 echo -e "${GREEN}[setup] READY.${NC}"

From babed2e69f465235d2a79af493d36036abe53ac8 Mon Sep 17 00:00:00 2001
From: Geir Sjurseth <gsjurseth@google.com>
Date: Tue, 7 Jul 2026 08:21:32 +0000
Subject: [PATCH 2/7] feat(apigee-skills-serving): add skill-finder +
 skill-publisher skills (multi-key trust)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Introduces two new agentic skills in
references/apigee-skills-serving/skills/:

  - skill-finder: consumer-side discovery/install client that
    queries the customer's API hub for signed skill bundles,
    verifies their ed25519 signatures against locally-installed
    trust roots, and installs them into the runtime's skills
    directory (opencode / gemini-cli / antigravity). Ports the
    16-line Hyrum's Law stdout contract from the standalone
    gsjurseth/skill-finder repo.

  - skill-publisher: author-side pipeline orchestrator that
    shells out to the four existing publisher Python modules
    (scripts/{pack,sign,upload,register}_skill.py) in order.

Both skills ship as **source only** -- no signed release
bundles, no key material, no trust-root fingerprints
committed to this repo. The trust model:

  - skill-finder itself is trusted by provenance (the operator
    fetched it from apigee/devrel over TLS). No signature is
    verified on skill-finder because the code that would verify
    it IS skill-finder.

  - Payload skills that skill-finder installs (currency-converter,
    weather-lookup, apigee-policy-top10, and any other skill
    the customer registers) are trusted by signature. Every
    downloaded manifest is verified against the trust roots in
    ~/.<runtime>/skills/skill-finder/keys/*.pem.

  - Trust roots are per-deployment, generated fresh on each
    customer's machine by the (forthcoming) bin/provision.sh
    bootstrap script. No shared signing key crosses this repo.

Multi-key trust root implementation
-----------------------------------

skill-finder now loads EVERY PEM in keys/ as an accepted trust
root, not just a hardcoded keys/trusted_pubkey.pem. A downloaded
manifest is accepted if its signing_key_id matches ANY installed
trust root. This supports:

  - Multi-publisher orgs (Alice's team's key + Bob's team's
    key both trusted simultaneously).
  - Zero-downtime key rotation (new key added, old key removed
    later; new-signed and old-signed manifests both verify
    during the overlap window).
  - Per-project trust bootstrapping (drop a customer's fresh
    ed25519 public key into keys/ during provisioning; skill-
    finder trusts it from the next invocation onward).

Empty keys/ dir is a hard-fail with a clear operator-facing
diagnostic pointing at bin/provision.sh. Duplicate keys under
different filenames warn but proceed. Non-PEM/non-.raw files
under keys/ (README.md, .gitkeep) are skipped silently.

The LOCKED Hyrum's Law contract lines are preserved with a
targeted wording tweak:

  before: "key-id check: trusted (matches embedded fingerprint)"
  after:  "key-id check: trusted (matches installed key <fp>)"

The change surfaces WHICH key matched, useful during rotation
troubleshooting. The "manifest signature: OK (ed25519 / <fp>)"
line is unchanged.

Files
-----

  skills/skill-finder/
    SKILL.md                     runtime instructions + dispatch
                                 (opencode | gemini | antigravity)
    keys/README.md               operator doc: how to populate keys/
    keys/.gitkeep                keeps the dir tracked
    manifest.yaml                source manifest (publisher script
                                 populates signature fields at
                                 publish time)
    scripts/find_install.py      SEARCH + INSTALL mode (multi-key
                                 verify loop; 16-line contract)
    scripts/list_skills.py       LIST mode (catalog browser)

  skills/skill-publisher/
    SKILL.md                     runtime instructions
    manifest.yaml                source manifest
    scripts/publish.sh           pack -> sign -> upload -> register
                                 orchestrator

Verified end-to-end against a real API hub
------------------------------------------

Ran the modified find_install.py in three scenarios against the
geirs-spaces-demo project:

  1. Happy path: one trust root installed, matches the manifest's
     signing_key_id. All 16 contract lines print, exit 0.

  2. Empty keys/ dir: emits the operator-facing "trust root:
     FAILED" diagnostic and exits before any network call.

  3. Wrong-key: one bogus key installed, no match. Emits the
     "key-id check: FAILED — trusts: <list>" line and exits
     with code 3 (integrity failure) before downloading anything.

  4. Rotation scenario: two keys installed (one bogus, one real).
     find_install.py picks the matching key and verifies
     successfully. Confirms rotation-with-overlap works.

License headers
---------------

All Python and bash source files carry the Apache 2.0 header
matching devrel's convention. keys/README.md and manifest.yaml
files carry copyright comments.

Followups (separate commits in this PR)
---------------------------------------

  - bin/install-skill-finder.sh and bin/install-skill-publisher.sh
    (installer scripts adapted from gsjurseth/skill-finder v0.1.7)
  - bin/provision.sh (net-new customer-bootstrap script that
    generates the trust root, provisions the demo, and publishes
    the initial skill catalog signed with the fresh key)
  - docs/{trust-root.md,provisioning.md,releasing.md}
  - README and publish-and-install.md updates
  - tests for the multi-key verify loop
---
 .../skills/skill-finder/SKILL.md              |  178 +++
 .../skills/skill-finder/keys/.gitkeep         |    0
 .../skills/skill-finder/keys/README.md        |   51 +
 .../skills/skill-finder/manifest.yaml         |   39 +
 .../skill-finder/scripts/find_install.py      | 1314 +++++++++++++++++
 .../skill-finder/scripts/list_skills.py       |  372 +++++
 .../skills/skill-publisher/SKILL.md           |  312 ++++
 .../skills/skill-publisher/manifest.yaml      |   43 +
 .../skills/skill-publisher/scripts/publish.sh |  266 ++++
 9 files changed, 2575 insertions(+)
 create mode 100644 references/apigee-skills-serving/skills/skill-finder/SKILL.md
 create mode 100644 references/apigee-skills-serving/skills/skill-finder/keys/.gitkeep
 create mode 100644 references/apigee-skills-serving/skills/skill-finder/keys/README.md
 create mode 100644 references/apigee-skills-serving/skills/skill-finder/manifest.yaml
 create mode 100644 references/apigee-skills-serving/skills/skill-finder/scripts/find_install.py
 create mode 100644 references/apigee-skills-serving/skills/skill-finder/scripts/list_skills.py
 create mode 100644 references/apigee-skills-serving/skills/skill-publisher/SKILL.md
 create mode 100644 references/apigee-skills-serving/skills/skill-publisher/manifest.yaml
 create mode 100755 references/apigee-skills-serving/skills/skill-publisher/scripts/publish.sh

diff --git a/references/apigee-skills-serving/skills/skill-finder/SKILL.md b/references/apigee-skills-serving/skills/skill-finder/SKILL.md
new file mode 100644
index 00000000..e74aad95
--- /dev/null
+++ b/references/apigee-skills-serving/skills/skill-finder/SKILL.md
@@ -0,0 +1,178 @@
+---
+name: skill-finder
+description: Discovers and installs agentic skills from the customer
+  API hub. Two modes — (1) SEARCH AND INSTALL one matching skill by
+  natural-language query, verifying its ed25519 signature against
+  the trust root(s) installed under keys/; (2) LIST every available
+  skill in the catalog as a markdown table, with cursor pagination,
+  so the user can browse what's offered before installing.
+license: Apache-2.0
+compatibility: opencode, antigravity, gemini-cli
+metadata:
+  # Trust roots are per-deployment: each install has one or more
+  # <fingerprint>.pem files under keys/, populated at install time
+  # by bin/provision.sh or by the operator. See keys/README.md.
+  trusted_signing_key_ids: keys/*.pem
+---
+
+# skill-finder
+
+## ⚠️ Runtime requirements (read this first)
+
+This skill **requires** its bundled venv wrapper. The Python
+scripts depend on `cryptography`, `google-auth`, `requests`, and
+`pyyaml`, which are installed into a per-user venv by the
+installer (`bin/install-skill-finder.sh`). They are **not**
+available to the system `python3`.
+
+**Always invoke scripts via the wrapper, never via bare
+`python3`:**
+
+```bash
+# CORRECT - invokes the venv Python via the wrapper:
+${SKILL_DIR}/bin/run-with-venv.sh ${SKILL_DIR}/scripts/<script>.py <args>
+
+# WRONG - system python3 will hit ModuleNotFoundError:
+python3 ${SKILL_DIR}/scripts/<script>.py <args>
+```
+
+The `Command` blocks in this file use the wrapper form. Do not
+"simplify" them to `python3 …` — the scripts will fail with
+`ModuleNotFoundError: No module named 'cryptography'` (or similar)
+and the agent will then waste a turn debugging a configuration
+issue that doesn't exist.
+
+If the wrapper at `${SKILL_DIR}/bin/run-with-venv.sh` is missing,
+the skill was installed by a pre-v0.1.4 installer. Re-run
+`bin/install-skill-finder.sh` to regenerate the wrapper. The
+scripts themselves also detect this case at startup and emit an
+actionable error pointing back here.
+
+---
+
+This skill has **two modes**. Choose the one that fits the user's
+intent:
+
+| User intent | Use this mode | Script |
+|:------------|:--------------|:-------|
+| "What skills are available?" / "Show me the catalog" / "Browse skills" / "What can the agent do via API hub?" | **List** (browse, no install) | `list_skills.py` |
+| "Find a skill that does X" / "Install a skill for Y" / "Look that up in API hub" (where there is one stated need) | **Install** (search + verify + install) | `find_install.py` |
+
+If the user is browsing or exploring, use **List** first; the user
+can then ask "install <name>" and you switch to Install with that
+name as the query.
+
+## Runtime dispatch
+
+Both modes ship as one behavioural contract that runs on two
+runtimes. Pick the invocation path that matches the runtime:
+
+| Runtime | How you invoke the scripts |
+|:--------|:---------------------------|
+| **OpenCode** | Use the `!`bash`` injection blocks below. OpenCode auto-executes them on SKILL.md load. |
+| **Gemini CLI** | Use your bash tool. Run the exact command from the matching `Command` block — substitute `${SKILL_DIR}` with the install path (`~/.gemini/skills/skill-finder` for global installs; `<project>/.agents/skills/skill-finder` for workspace installs), substitute `${ARGUMENTS}` with the user's verbatim query, and `${APIHUB_PROJECT}` / `${APIHUB_LOCATION}` with the values from the shell environment. |
+| **Antigravity** | Same as Gemini CLI but `${SKILL_DIR}` is `~/.gemini/antigravity/skills/skill-finder` for global installs (or `<project>/.agents/skills/skill-finder` for workspace installs). |
+| **Any other runtime** | Same as Antigravity / Gemini CLI: invoke via whatever bash mechanism the runtime provides. |
+
+---
+
+## Mode 1: Install (search + verify + install one skill)
+
+Use this when the user has a stated need they want fulfilled now.
+
+### Steps
+
+1. Take the user's natural-language need verbatim as the search
+   query. Do not paraphrase or simplify.
+
+1a. Before invoking `find_install.py`, verify the user query
+    contains NONE of these characters:
+    `$` `` ` `` `\` `"` `'` `;` `&` `|` `<` `>` `(` `)` `{` `}`
+    (shell metacharacters; they may trigger command injection in
+    the bash layer because POSIX double quotes do not suppress
+    `$(...)` command substitution or backtick expansion). If any
+    are present, REFUSE to invoke and ask the user to rephrase
+    the query without special characters.
+
+2. Invoke the bundled discovery script using the runtime path
+   from the table above. The exact command is:
+
+   **Command** (OpenCode auto-runs this; on Antigravity / Gemini
+   CLI you run it via your bash tool):
+
+   !`python3 ${SKILL_DIR}/scripts/find_install.py \
+        --query "${ARGUMENTS}" \
+        --project "${APIHUB_PROJECT}" \
+        --location "${APIHUB_LOCATION}"`
+
+3. The script prints structured progress; reproduce every printed
+   line verbatim to the user — DO NOT summarize. The lines are a
+   stable contract. On Antigravity / Gemini CLI, this means: after
+   your bash tool returns, surface its full stdout verbatim in
+   your reply. Do not edit, reorder, or collapse the
+   `[skill-finder]` lines.
+
+4. On success, the script's final line is the runtime-aware
+   trailer. On OpenCode, it tells the user to type
+   `/reload-skills`. On Antigravity / Gemini CLI, it tells the
+   user to re-ask their question. Surface that line verbatim and
+   STOP. Do not invoke the newly-installed skill yourself; let
+   the user re-ask.
+
+5. On any FAILED line, STOP. Surface the failure to the user with
+   the verbatim error message. Do NOT attempt installation again
+   on the same invocation.
+
+---
+
+## Mode 2: List (browse the catalog)
+
+Use this when the user wants to see what's available before
+committing to install anything.
+
+### Steps
+
+1. Invoke the bundled browse script. The exact command is:
+
+   **Command** (OpenCode auto-runs this; on Antigravity / Gemini
+   CLI you run it via your bash tool):
+
+   !`python3 ${SKILL_DIR}/scripts/list_skills.py \
+        --project "${APIHUB_PROJECT}" \
+        --location "${APIHUB_LOCATION}" \
+        --page-size 20`
+
+2. The script prints a markdown table of `name | keywords |
+   description`. Surface the table verbatim to the user — DO NOT
+   summarize and DO NOT drop the `[skill-finder]` diagnostic
+   lines.
+
+3. If the last line of the output is `next-page-token: <token>`,
+   there are more skills available. Offer the user the option:
+   "There are more skills. Want me to show the next page?". If
+   the user says yes, re-invoke with `--page-token "<token>"`
+   added to the command (keep the other flags the same).
+
+4. If the user then picks a specific skill name and wants to
+   install it, switch to **Mode 1: Install** with that name as
+   the `--query` value.
+
+5. On any FAILED line, STOP. Surface the failure verbatim.
+
+---
+
+## Runtime notes
+
+- **OpenCode** auto-detects its skills root at
+  `~/.config/opencode/skills/`. `find_install.py` writes there.
+  `/reload-skills` rescans.
+- **Antigravity / Gemini CLI** auto-detects
+  `~/.gemini/config/skills/` (the canonical global skills root).
+  `find_install.py` writes there. Both runtimes rescan on the
+  next conversation turn — there is no `/reload-skills` slash
+  command. The script emits a runtime-aware trailer (`re-ask
+  your question — the agent runtime will pick up <name> on the
+  next turn`) so the user is not told to type a non-existent
+  slash command.
+- **APIGEE_SKILLS_INSTALL_ROOT** env var overrides the
+  auto-detection if the operator wants to pin install location.
diff --git a/references/apigee-skills-serving/skills/skill-finder/keys/.gitkeep b/references/apigee-skills-serving/skills/skill-finder/keys/.gitkeep
new file mode 100644
index 00000000..e69de29b
diff --git a/references/apigee-skills-serving/skills/skill-finder/keys/README.md b/references/apigee-skills-serving/skills/skill-finder/keys/README.md
new file mode 100644
index 00000000..74bf80d6
--- /dev/null
+++ b/references/apigee-skills-serving/skills/skill-finder/keys/README.md
@@ -0,0 +1,51 @@
+# skill-finder trust roots
+
+This directory is **empty in the source tree** on purpose. It is
+populated at install time by one of two mechanisms:
+
+1. **Bootstrap provisioning** -- `bin/provision.sh` in the
+   reference repo generates a fresh ed25519 keypair on the
+   operator's machine, signs the initial catalog with it, and
+   drops the corresponding public key as
+   `<fingerprint>.pem` in this directory of the local
+   skill-finder install.
+
+2. **Manual trust-root pinning** -- an operator who already
+   has a public key (e.g. their org has a pre-existing
+   signing key) drops the PEM into this directory as
+   `<fingerprint>.pem`.
+
+## What lives here (post-install)
+
+Every file matching the glob `keys/*.pem` is loaded by
+`scripts/find_install.py` at verify time. A downloaded skill
+manifest is accepted if its `signing_key_id` fingerprint
+matches **any** of the trust roots present.
+
+This is the multi-key model: multiple publishers, key rotation
+with a grace period, and the ability to fall back to an old
+key while a new one is being rolled out are all supported by
+adding or removing PEM files.
+
+If this directory is empty at verify time, `find_install.py`
+refuses to install anything and prints a clear message
+directing the operator to run `bin/provision.sh` or drop a
+PEM in themselves.
+
+## Why is this not in the source tree?
+
+The trust root is a **per-deployment secret** in the sense
+that each customer owns their own private signing key. Shipping
+a shared public key in this repo would either:
+
+- pin every customer to a key they don't own (no rotation, no
+  self-signed skills), or
+- pin them to a key someone in this repo can rotate under
+  them (governance nightmare).
+
+Instead: this repo ships the source code and the installer,
+and the trust root is materialised on the customer's machine
+during provisioning. Nothing signed by anybody's key ever
+enters this repo.
+
+See `docs/trust-root.md` for the full trust model discussion.
diff --git a/references/apigee-skills-serving/skills/skill-finder/manifest.yaml b/references/apigee-skills-serving/skills/skill-finder/manifest.yaml
new file mode 100644
index 00000000..1b2d3ab0
--- /dev/null
+++ b/references/apigee-skills-serving/skills/skill-finder/manifest.yaml
@@ -0,0 +1,39 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+# Source manifest for skill-finder. The publishing pipeline
+# (scripts/sign_skill.py + scripts/upload_skill.py) populates the
+# gs_uri, zip_sha256, signature, and signing_key_id fields at
+# publish time. Do not commit those values back to the source
+# tree.
+author: apigee-devrel
+capabilities: []
+description: Discovers and installs agentic skills from the customer
+  API hub. Two modes - SEARCH AND INSTALL one matching skill by
+  natural-language query, verifying its ed25519 signature against
+  the trust root(s) installed under keys/; LIST every available
+  skill in the catalog as a markdown table, with cursor pagination,
+  so the user can browse what's offered before installing.
+keywords:
+  - skill
+  - finder
+  - catalog
+  - discovery
+  - install
+  - apihub
+license: Apache-2.0
+manifest_schema_version: '1'
+name: skill-finder
+runtime_iam: []
+version: 0.1.0
diff --git a/references/apigee-skills-serving/skills/skill-finder/scripts/find_install.py b/references/apigee-skills-serving/skills/skill-finder/scripts/find_install.py
new file mode 100644
index 00000000..569cfdfe
--- /dev/null
+++ b/references/apigee-skills-serving/skills/skill-finder/scripts/find_install.py
@@ -0,0 +1,1314 @@
+#!/usr/bin/env python3
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""skill-finder runtime helper.
+
+Performs the 16-step install pipeline and prints the
+Hyrum's-Law-locked output strings.
+
+Round-2 / Round-3 hardening summary:
+
+  - _search applies APIGEE_SKILLS_MIN_KEYWORD_OVERLAP threshold;
+    zero-overlap matches are rejected with `match: NONE`.
+  - IAM pre-flight HTTP-level guards (status + JSON-decode +
+    retry); see scripts/common/iam_preflight.py for the pure
+    API. Stdout contract lines are emitted HERE.
+  - _install_atomic wraps the body in try/finally with
+    shutil.rmtree(staging, ignore_errors=True). BadZipFile and
+    YAMLError surface as contract lines.
+  - Distinct contract lines for IAM-skipped and
+    watcher-undetectable cases; EXDEV failure line.
+  - sys.path.insert(0, parent) so `from common.X import Y`
+    resolves regardless of invocation form (see docstring on
+    the insert below).
+  - _verify_signature distinguishes signature-absent vs
+    signature-malformed vs signature-invalid. _fetch_spec
+    guards base64/utf-8 decode with the same care.
+  - _read_skill_md_name normalizes CRLF / CR to LF in memory
+    before the byte-prefix check so Windows-authored SKILL.md
+    files pass.
+  - _WatcherProbeFuture overlaps the 2 s watcher settle with
+    network I/O via a background thread.
+  - main() rejects empty --project / --location / --query with
+    an operator-actionable line BEFORE any other work.
+  - --query value is validated for shell-injection blast-
+    radius limiting (length cap, printable ASCII, no NUL/CRLF,
+    no shell metacharacters $ ` \\ " ' ; & | < > ( ) { }).
+"""
+from __future__ import annotations
+
+import argparse
+import base64
+import binascii  # caught when b64decode encounters bad input
+import datetime
+import errno  # EXDEV detection
+import fcntl
+import hashlib
+import io
+import json
+import os
+import shutil
+import sys
+import threading  # overlap watcher probe with network I/O
+import uuid
+import zipfile
+from pathlib import Path
+
+# =============================================================
+# Venv preflight. The third-party imports below (yaml,
+# cryptography, google-auth, requests) live in the venv that the
+# installer creates at ~/.local/share/skill-finder/venv. If this
+# script is invoked with the system python3 instead of the venv
+# wrapper, those imports raise ModuleNotFoundError with no useful
+# context -- the agent sees a generic traceback and has no way to
+# know it should be using the wrapper.
+#
+# Detect that failure mode here and exit 78 with a single
+# actionable message pointing the agent at the wrapper. Exit code
+# 78 is the BSD sysexits.h EX_CONFIG value ("configuration
+# error"); we picked it to disambiguate from the other exit codes
+# this script uses (0, 1, 2). The agent can read 78 and recognise
+# "wrong invocation" vs "real failure".
+# =============================================================
+_VENV_DEPS = ("yaml", "cryptography", "google.auth", "requests")
+_VENV_EX_CONFIG = 78
+_missing_dep: str | None = None
+_dep: str = ""  # noqa: appease static checker; rebound in loop
+for _dep in _VENV_DEPS:
+    try:
+        __import__(_dep)
+    except ImportError:
+        _missing_dep = _dep
+        break
+if _missing_dep is not None:
+    _SKILL_DIR = Path(__file__).resolve().parent.parent
+    _WRAPPER = _SKILL_DIR / "bin" / "run-with-venv.sh"
+    sys.stderr.write(
+        "[skill-finder] FATAL: this script requires the bundled "
+        "venv wrapper.\n"
+        f"  Missing module: {_missing_dep!r}\n"
+        f"  You invoked: python3 {sys.argv[0]}\n"
+        f"  You should invoke:\n"
+        f"      {_WRAPPER} {sys.argv[0]} <args>\n"
+        "  The wrapper activates the per-user venv created by\n"
+        "  install-skill-finder.sh, which contains the four\n"
+        "  required dependencies. The system python3 does not.\n"
+        "  If the wrapper is missing, re-run\n"
+        "  bin/install-skill-finder.sh to regenerate it.\n"
+    )
+    sys.exit(_VENV_EX_CONFIG)
+# Cleanup loop-local names so they don't pollute the module
+# namespace.
+del _VENV_DEPS, _missing_dep, _dep
+
+import yaml
+from cryptography.exceptions import InvalidSignature
+from cryptography.hazmat.primitives.asymmetric.ed25519 import (
+    Ed25519PublicKey,
+)
+
+# Packaging boundary. The .skill zip carries scripts/ and
+# scripts/common/ side-by-side; at install-time the layout is
+# ~/.config/opencode/skills/skill-finder/scripts/{find_install.py,
+# common/{canonical,permission_resolver,watcher_probe,
+# manifest_schema}.py}. Without this sys.path setup, the
+# `from common.X import Y` lines would ImportError because the
+# SKILL.md invokes the script directly (`python3
+# ${SKILL_DIR}/scripts/find_install.py`), not as a module
+# (`python3 -m scripts.find_install`).
+#
+# Insert at index 0 (highest priority) so we never pick up a
+# user-site `common` collision. pack_skill.py asserts that
+# scripts/common/ contains exactly the expected modules before
+# zipping.
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+# Dual import. Production zip layout is
+# <install-dir>/scripts/{find_install.py, common/*.py}, so the bare
+# `from common.X` form resolves via the sys.path.insert above.
+# Dev/test layout uses `from scripts.common.X` via the repo-root
+# entry tests/conftest.py installs. We try production form first
+# (so sys.modules registration matches the production contract),
+# then fall back to dev form. CRITICAL for tests:
+# the bound names below MUST be referred to as
+# `find_install.iam_preflight`, `find_install.http_get_retry`,
+# etc. in test monkeypatch.setattr() calls — that is, the
+# monkeypatch target is THIS module, not the source module. If a
+# test ever does `from scripts.common.iam_preflight import
+# iam_preflight; monkeypatch.setattr(iam_preflight_module, ...)`
+# the patch will not reach this module's bound name.
+try:
+    # Production form (used when the .skill zip is extracted and
+    # invoked via `python3 ${SKILL_DIR}/scripts/find_install.py`).
+    from common.canonical import canonicalize
+    from common.permission_resolver import (
+        detect_active_agent,
+        resolve_skill_permission,
+        Verdict,
+    )
+    from common.watcher_probe import detect_watcher, WatcherState
+    from common.manifest_schema import validate_manifest
+    from common.iam_preflight import iam_preflight
+    from common.http_retry import http_get_retry, http_post_retry
+    from common import config
+except ImportError:
+    # Dev / test form (used when invoked from the repo root with
+    # tests/conftest.py on the path).
+    from scripts.common.canonical import canonicalize
+    from scripts.common.permission_resolver import (
+        detect_active_agent,
+        resolve_skill_permission,
+        Verdict,
+    )
+    from scripts.common.watcher_probe import detect_watcher, WatcherState
+    from scripts.common.manifest_schema import validate_manifest
+    from scripts.common.iam_preflight import iam_preflight
+    from scripts.common.http_retry import http_get_retry, http_post_retry
+    from scripts.common import config
+
+# google.auth is imported lazily inside _creds() so test code that
+# never touches the network does not have to stub it at import
+# time. requests is imported here because the HTTP retry helpers
+# return requests.Response objects that we need to type-check.
+import requests
+
+SKILL_DIR = Path(__file__).resolve().parent.parent
+# Multi-key trust root: every PEM under keys/ is a trusted
+# publisher. A manifest is accepted if its signing_key_id
+# fingerprint matches ANY installed key. See keys/README.md for
+# the operational model (provision.sh drops the key, operators
+# can add or remove keys manually).
+KEYS_DIR = SKILL_DIR / "keys"
+
+
+_ANTIGRAVITY_ROOT = Path.home() / ".gemini" / "config" / "skills"
+_GEMINI_CLI_ROOT = Path.home() / ".gemini" / "skills"
+_OPENCODE_ROOT = Path.home() / ".config" / "opencode" / "skills"
+
+
+def _detect_skills_root() -> Path:
+    """Determine where to install newly-discovered skills.
+
+    Resolution order:
+      1. APIGEE_SKILLS_INSTALL_ROOT env var (operator override) —
+         use this verbatim. Always the explicit win.
+      2. Auto-detect Gemini CLI: invocation under
+         ~/.gemini/skills/ (the canonical Gemini CLI install
+         root, distinct from Antigravity's ~/.gemini/config/skills/).
+      3. Auto-detect Antigravity layout: install root is
+         ~/.gemini/config/skills/.
+      4. Default: ~/.config/opencode/skills/ (OpenCode layout).
+
+    The Gemini CLI check runs BEFORE the Antigravity check because
+    Antigravity's path (~/.gemini/config/skills) and Gemini CLI's
+    path (~/.gemini/skills) are siblings — both under ~/.gemini/
+    but distinct directories. Order matters only for the
+    pathologic case where someone configures the same install
+    in both runtimes; we'd rather pick the runtime whose
+    matching path is more specific to the invocation.
+
+    Detection considers TWO candidates: the realpath-resolved
+    SKILL_DIR (real on-disk location, follows symlinks) and the
+    unresolved invocation path (`__file__`-derived, preserves the
+    path the agent/user actually supplied). The dual check handles
+    every install pattern we've seen: extracted .skill zips and
+    dev-checkout symlinks.
+
+    Two-path matching: `is_relative_to` is exact; `os.path.samefile`
+    is symlink-aware. We use both because each catches cases the
+    other misses.
+    """
+    override = os.environ.get("APIGEE_SKILLS_INSTALL_ROOT", "").strip()
+    if override:
+        return Path(override).expanduser()
+    invocation_dir = Path(__file__).parent.parent
+    candidates = (SKILL_DIR, invocation_dir)
+
+    # Gemini CLI (~/.gemini/skills/) — canonical, single path.
+    for candidate in candidates:
+        if _path_is_under(candidate, _GEMINI_CLI_ROOT):
+            return _GEMINI_CLI_ROOT
+
+    # Antigravity — single canonical root.
+    for candidate in candidates:
+        if _path_is_under(candidate, _ANTIGRAVITY_ROOT):
+            return _ANTIGRAVITY_ROOT
+
+    return _OPENCODE_ROOT
+
+
+def _path_is_under(child: Path, ancestor: Path) -> bool:
+    """Return True iff `child` is at or under `ancestor`, treating
+    symlinks as equivalent to their targets.
+
+    We try `is_relative_to` first (exact string prefix); if that
+    fails, we walk up child's parents and compare each with
+    `samefile` against ancestor. The latter handles cases where
+    the legacy alias and the current path resolve to the same
+    inode but have different textual representations.
+    """
+    try:
+        if child.is_relative_to(ancestor):
+            return True
+    except (AttributeError, ValueError):
+        pass
+    if not ancestor.exists():
+        return False
+    # Walk up; on each step, see if this dir IS the ancestor (in
+    # inode terms). Bounded to ~10 levels to avoid pathological
+    # walks on bad inputs.
+    p = child
+    for _ in range(10):
+        try:
+            if p.exists() and os.path.samefile(p, ancestor):
+                return True
+        except (OSError, ValueError):
+            pass
+        parent = p.parent
+        if parent == p:
+            return False
+        p = parent
+    return False
+
+
+SKILLS_ROOT = _detect_skills_root()
+STAGING_LOCK = SKILLS_ROOT / ".staging.lock"
+BREADCRUMB = SKILLS_ROOT / ".recent-install"
+RECOGNIZED_SCHEMA_VERSIONS = ["1"]
+APIHUB_BASE = "https://apihub.googleapis.com/v1"
+CLOUD_PLATFORM_SCOPE = (
+    "https://www.googleapis.com/auth/cloud-platform"
+)
+
+# Zip-bomb / zip-slip defense caps (P0.2).
+MAX_ZIP_MEMBERS = 10_000
+MAX_ZIP_DECOMPRESSED_BYTES = 100 * 1024 * 1024  # 100 MiB
+# SKILL.md frontmatter parser cap (P1.2).
+MAX_FRONTMATTER_BYTES = 8192
+
+
+def _say(line: str) -> None:
+    """Emit one prefixed contract line. flush=True keeps stdout
+    ordering deterministic across helper-call boundaries (important
+    for the end-to-end tests that assert exact line sequencing)."""
+    print(f"[skill-finder] {line}", flush=True)
+
+
+def _say_advisory(line: str) -> None:
+    """Operator-advisory line, NOT prefixed with [skill-finder].
+
+    The
+    `config: APIGEE_SKILLS_MIN_KEYWORD_OVERLAP=... invalid` advisory
+    is NOT part of the Hyrum's Law contract; it appears unprefixed
+    and only fires when an operator misconfigures the env var.
+    Kept separate from _say() so the distinction is visible at the
+    call site."""
+    print(line, flush=True)
+
+
+def _die(line: str, code: int = 1) -> None:
+    _say(line)
+    sys.exit(code)
+
+
+def _min_keyword_overlap() -> int:
+    """Resolve APIGEE_SKILLS_MIN_KEYWORD_OVERLAP with
+    silent fallback.
+
+    Default 1 (require at least one keyword in common). On
+    malformed or non-positive values, falls back to 1 silently and
+    emits a single advisory line so operator misconfiguration is
+    visible in run output without changing the documented
+    Hyrum's Law contract."""
+    raw = os.environ.get("APIGEE_SKILLS_MIN_KEYWORD_OVERLAP", "1")
+    try:
+        n = int(raw)
+        if n < 1:
+            raise ValueError("must be >= 1")
+        return n
+    except (ValueError, TypeError):
+        _say_advisory(
+            f'config: APIGEE_SKILLS_MIN_KEYWORD_OVERLAP="{raw}" '
+            f"invalid; using default 1"
+        )
+        return 1
+
+
+def _creds() -> tuple[object, object]:
+    """Single source of truth for ADC scopes.
+
+    All call sites (search, IAM pre-flight, versions/specs fetch)
+    MUST go through this helper. Mixing scoped and unscoped ADC
+    silently breaks in workload-identity and service-account
+    contexts where the unscoped path returns a token that fails
+    downstream calls.
+
+    The returned credentials object is refreshed before return so
+    callers can read ``creds.token`` immediately. A fresh
+    ``google.auth.default()`` returns credentials with ``token=None``
+    until the first refresh, which is the gotcha that produces
+    silent HTTP 401 responses on the very first call."""
+    import google.auth as google_auth
+    from google.auth.transport.requests import Request
+    creds, project_id = google_auth.default(
+        scopes=[CLOUD_PLATFORM_SCOPE]
+    )
+    creds.refresh(Request())
+    return creds, project_id
+
+
+def _auth_headers(creds: object) -> dict[str, str]:
+    """Build the Authorization header. Defensive: the test
+    harness stubs ``creds`` with a simple object that exposes
+    ``token``; production code gets a real Credentials instance.
+    A missing token resolves to empty string so the header is
+    still well-formed and request-level mocking can short-circuit
+    before any real network call."""
+    token = getattr(creds, "token", None) or ""
+    return {"Authorization": f"Bearer {token}"}
+
+
+def _on_retry(status: int, sleep_seconds: float) -> None:
+    """Callback passed to scripts.common.http_retry so the
+    transient-failure line is emitted from this module
+    (which owns the Hyrum's Law surface) before the retry
+    sleep fires."""
+    backoff_ms = int(round(sleep_seconds * 1000))
+    _say(
+        f"transient failure (HTTP {status}); "
+        f"retry 1/1 after {backoff_ms}ms"
+    )
+
+
+# ---------------------------------------------------------------------------
+# Background watcher probe future.
+# ---------------------------------------------------------------------------
+
+
+class _WatcherProbeFuture:
+    """Run detect_watcher() in a background thread so its
+    PROBE_SETTLE_SECONDS=2.0 sleep overlaps the ~10s of network I/O
+    in steps 2-9 (search, fetch-spec, IAM pre-flight, versions
+    GET, zip download). Net wall-clock cost of the probe: ~0ms.
+
+    Pre-fix accounting: the 2s probe consumed 13% of the 15s p95
+    SLO. Post-fix: amortized to zero because the network calls
+    take longer than the probe.
+
+    Failure mode: if the probe thread raises, we surface
+    WATCHER_UNDETECTABLE (the same fallback the synchronous probe
+    used) rather than crashing the install."""
+
+    def __init__(self) -> None:
+        self._state: WatcherState | None = None
+        self._exc: BaseException | None = None
+        self._thread = threading.Thread(
+            target=self._run, name="watcher-probe", daemon=True
+        )
+
+    def start(self) -> None:
+        self._thread.start()
+
+    def _run(self) -> None:
+        try:
+            self._state = detect_watcher()
+        except BaseException as e:  # noqa: BLE001 — see docstring
+            self._exc = e
+
+    def result(self, timeout: float = 10.0) -> WatcherState:
+        self._thread.join(timeout=timeout)
+        if self._thread.is_alive():
+            # Thread didn't finish within timeout. Treat as
+            # UNDETECTABLE rather than blocking forever.
+            return WatcherState.WATCHER_UNDETECTABLE
+        if self._exc is not None:
+            return WatcherState.WATCHER_UNDETECTABLE
+        assert self._state is not None
+        return self._state
+
+
+# ---------------------------------------------------------------------------
+# Steps 1-2: load trusted key, search API hub.
+# ---------------------------------------------------------------------------
+
+
+def _load_one_pubkey(path: Path) -> tuple[Ed25519PublicKey, str]:
+    """Load a single PEM (or raw 32-byte) public key and return
+    (key, fingerprint). Fingerprint format:
+    ``sha256:<hex-of-sha256-of-raw-32-byte-pubkey>``."""
+    raw = path.read_bytes()
+    if raw.startswith(b"-----BEGIN"):
+        from cryptography.hazmat.primitives.serialization import (
+            load_pem_public_key,
+        )
+        pk = load_pem_public_key(raw)
+        from cryptography.hazmat.primitives import serialization
+        raw32 = pk.public_bytes(
+            encoding=serialization.Encoding.Raw,
+            format=serialization.PublicFormat.Raw,
+        )
+    else:
+        raw32 = raw
+        pk = Ed25519PublicKey.from_public_bytes(raw32)
+    fp = "sha256:" + hashlib.sha256(raw32).hexdigest()
+    return pk, fp
+
+
+def _load_pubkeys() -> dict[str, Ed25519PublicKey]:
+    """Load every ``keys/*.pem`` (and legacy ``keys/*.raw``) file
+    into a fingerprint->key map. If the directory is empty or
+    missing, emit a clear diagnostic and exit -- installing
+    anything without a trust root would defeat the point.
+
+    Note: files starting with ``.`` (dotfiles like ``.gitkeep``)
+    are ignored so the source-tree placeholder does not trip us
+    up. ``README.md`` and any non-PEM/non-raw file that fails to
+    parse is skipped with a warning, not a hard error, so
+    dropping unrelated files into keys/ (e.g. an operator's
+    notes) does not brick skill-finder."""
+    if not KEYS_DIR.is_dir():
+        _die(
+            f"trust root: FAILED — no keys/ directory found at "
+            f"{KEYS_DIR}. Run bin/provision.sh (bootstrap mode) "
+            f"or drop your org's public key PEM into "
+            f"{KEYS_DIR}/<fingerprint>.pem before installing.",
+            code=3,
+        )
+    trusted: dict[str, Ed25519PublicKey] = {}
+    candidates = sorted(
+        p for p in KEYS_DIR.iterdir()
+        if p.is_file() and not p.name.startswith(".")
+        and p.suffix in (".pem", ".raw")
+    )
+    for path in candidates:
+        try:
+            pk, fp = _load_one_pubkey(path)
+        except (ValueError, TypeError, OSError) as exc:
+            _say(
+                f"trust root: warning: skipped {path.name} "
+                f"({exc.__class__.__name__}: {exc})"
+            )
+            continue
+        if fp in trusted:
+            # Duplicate key material under a different filename
+            # -- warn but keep the first (deterministic sort
+            # order, so 'first' is stable).
+            _say(
+                f"trust root: warning: {path.name} duplicates an "
+                f"already-loaded key ({fp}); keeping first"
+            )
+            continue
+        trusted[fp] = pk
+    if not trusted:
+        _die(
+            f"trust root: FAILED — no valid keys loaded from "
+            f"{KEYS_DIR}. Add at least one <fingerprint>.pem "
+            f"or run bin/provision.sh to generate one.",
+            code=3,
+        )
+    return trusted
+
+
+def _search(
+    project: str, location: str, query: str, min_overlap: int
+) -> list[dict]:
+    creds, _ = _creds()
+    base = (
+        f"{APIHUB_BASE}/projects/{project}"
+        f"/locations/{location}/apis"
+    )
+    # API hub filter syntax: attributes.<FQ-name>.string_values
+    # .values:"X". Discovered during real-infra dry-run; the bare
+    # `agentic_skill=true` form rejected with INVALID_ARGUMENT.
+    attr_fq = (
+        f"projects/{project}/locations/{location}"
+        f"/attributes/agentic_skill"
+    )
+    filter_q = (
+        f'attributes.{attr_fq}.string_values.values:"true"'
+    )
+    from urllib.parse import quote_plus
+    url = f"{base}?filter={quote_plus(filter_q)}&pageSize=100"
+    _say(f"search: GET {url}")
+    try:
+        r = http_get_retry(
+            url,
+            on_retry=_on_retry,
+            timeout=30,
+            headers=_auth_headers(creds),
+        )
+    except requests.HTTPError as exc:
+        resp = exc.response
+        code = getattr(resp, "status_code", "???")
+        reason = getattr(resp, "reason", "")
+        _die(
+            f"search: FAILED — HTTP {code} {reason} from {url}.",
+            code=2,
+        )
+    apis = (r.json() or {}).get("apis", []) or []
+    # Trivial stem: strip trailing 's' from query tokens so
+    # "policies" matches keyword "policy". Apigee is the demo
+    # punchline; reasonable to over-fit a little.
+    raw_tokens = set(query.lower().split())
+    q_tokens = raw_tokens | {t.rstrip("s") for t in raw_tokens if t.endswith("s") and len(t) > 3}
+
+    # API hub returns keywords as nested
+    # {FQ-attribute-name: {stringValues: {values: [...]}}}.
+    # Pull the values from any attribute whose key ends in
+    # "/attributes/keywords" — this avoids hardcoding the project
+    # and location in the lookup.
+    def _keywords(api: dict) -> list[str]:
+        for k, v in (api.get("attributes") or {}).items():
+            if k.endswith("/attributes/keywords"):
+                return (v.get("stringValues") or {}).get("values") or []
+        return []
+
+    def score(api: dict) -> int:
+        return len(q_tokens & {k.lower() for k in _keywords(api)})
+
+    # Drop results below the configured minimum overlap so
+    # `apis[0]` cannot silently install a non-matching skill.
+    matched = [a for a in apis if score(a) >= min_overlap]
+    return sorted(matched, key=score, reverse=True)
+
+
+def _fetch_spec(creds: object, spec_path: str) -> str:
+    url = f"{APIHUB_BASE}/{spec_path}:contents"
+    try:
+        r = http_get_retry(
+            url,
+            on_retry=_on_retry,
+            timeout=30,
+            headers=_auth_headers(creds),
+        )
+    except requests.HTTPError as exc:
+        resp = exc.response
+        code = getattr(resp, "status_code", "???")
+        reason = getattr(resp, "reason", "")
+        _die(
+            f"search: FAILED — HTTP {code} {reason} from {url}.",
+            code=2,
+        )
+    body = (r.json() or {}).get("contents", "")
+    # Convert binascii.Error / UnicodeDecodeError to a
+    # contract line instead of a Python traceback.
+    try:
+        return base64.b64decode(body, validate=True).decode("utf-8")
+    except (binascii.Error, UnicodeDecodeError, ValueError) as e:
+        _die(
+            f"spec contents: FAILED — malformed base64 or non-utf8 "
+            f"body from {url}: {e.__class__.__name__}",
+            code=2,
+        )
+        return ""  # unreachable; satisfies the type checker
+
+
+# ---------------------------------------------------------------------------
+# Steps 4-5b: signature, schema, cross-check.
+# ---------------------------------------------------------------------------
+
+
+def _verify_signature(
+    manifest_yaml: str,
+    trusted_pubkeys: dict[str, Ed25519PublicKey],
+) -> dict:
+    """Verify a signed manifest against the multi-key trust root.
+
+    A manifest is accepted if its ``signing_key_id`` matches ANY
+    installed trust root. This supports multi-publisher orgs and
+    zero-downtime key rotation (new key added, old key not yet
+    removed -- both keys accept until rotation completes).
+
+    The stdout contract preserves the LOCKED phrasing
+    ``key-id check: trusted (...)`` and
+    ``manifest signature: OK (...)`` but reports the specific
+    fingerprint that matched rather than a single ``trusted_fp``
+    -- helpful when troubleshooting which key was actually used.
+    """
+    obj = yaml.safe_load(manifest_yaml)
+    declared_fp = (
+        obj.get("signing_key_id", "") if isinstance(obj, dict) else ""
+    )
+    if declared_fp not in trusted_pubkeys:
+        trusted_list = ", ".join(sorted(trusted_pubkeys)) or "(none)"
+        _die(
+            f"key-id check: FAILED — manifest declares "
+            f"signing_key_id {declared_fp}, but skill-finder "
+            f"only trusts: {trusted_list}",
+            code=3,
+        )
+    pubkey = trusted_pubkeys[declared_fp]
+    _say(
+        f"key-id check: trusted (matches installed key {declared_fp})"
+    )
+    # Distinguish missing-field from malformed-value and
+    # from invalid-signature. The pre-fix behaviour was that a
+    # missing 'signature' field decoded to b"" and silently fell
+    # through to ed25519 verify, producing a misleading
+    # "ed25519 verify rejected" error. Three distinct contract lines
+    # now cover the three cases. Order matters: absent FIRST, then
+    # malformed base64, then crypto verify.
+    if "signature" not in obj:
+        _die(
+            f"manifest signature: FAILED — signature field absent "
+            f"from manifest (key={declared_fp})",
+            code=3,
+        )
+    sig_b64 = obj["signature"]
+    try:
+        sig = base64.b64decode(sig_b64, validate=True)
+    except (binascii.Error, ValueError, TypeError) as e:
+        _die(
+            f"manifest signature: FAILED — malformed base64 in "
+            f"signature field (key={declared_fp}): "
+            f"{e.__class__.__name__}",
+            code=3,
+        )
+    canonical = canonicalize(obj)
+    try:
+        pubkey.verify(sig, canonical)
+    except InvalidSignature:
+        _die(
+            f"manifest signature: FAILED — ed25519 verify rejected "
+            f"(key={declared_fp})",
+            code=3,
+        )
+    _say(f"manifest signature: OK (ed25519 / {declared_fp})")
+    return obj
+
+
+def _check_schema_version(manifest: dict) -> None:
+    """Emit `manifest schema: OK (version=N)` or fail-line (P0.5)."""
+    declared = manifest.get("manifest_schema_version")
+    if declared not in RECOGNIZED_SCHEMA_VERSIONS:
+        _die(
+            f"manifest schema: FAILED — declared {declared}; "
+            f"recognized {RECOGNIZED_SCHEMA_VERSIONS}. "
+            f"Upgrade skill-finder.",
+            code=3,
+        )
+    _say(f"manifest schema: OK (version={declared})")
+
+
+def _cross_check(api: dict, manifest: dict) -> None:
+    # API hub returns attributes as {FQ-name: {stringValues:
+    # {values: [...]}}}. Pull by suffix match so we don't need
+    # to know the project/location at this layer.
+    attrs = api.get("attributes") or {}
+
+    def _attr_string(field: str) -> str | None:
+        suffix = f"/attributes/{field}"
+        for k, v in attrs.items():
+            if k.endswith(suffix):
+                vals = (v.get("stringValues") or {}).get("values") or []
+                return vals[0] if vals else None
+        return None
+
+    for field in ("gs_uri", "signing_key_id"):
+        attr_val = _attr_string(field)
+        man_val = manifest.get(field)
+        if attr_val != man_val:
+            _die(
+                f"attribute cross-check: FAILED — {field} differs: "
+                f"attr={attr_val} manifest={man_val}",
+                code=3,
+            )
+    _say("attribute cross-check: OK (gs_uri, signing_key_id agree)")
+
+
+# ---------------------------------------------------------------------------
+# Step 7: IAM pre-flight.
+# ---------------------------------------------------------------------------
+
+
+def _iam_preflight_with_contract(
+    project: str, perms: list[str]
+) -> None:
+    """Call the pure iam_preflight() API and emit the contract line.
+
+    The library function returns an IamPreflightResult dataclass
+    with one of five status values; we own the wording of the
+    resulting contract lines (the library is intentionally
+    stdout-free per its module docstring).
+    """
+    result = iam_preflight(project, perms)
+    if result.status == "SKIPPED":
+        _say("IAM pre-flight: skipped (no runtime_iam declared)")
+        return
+    if result.status == "GRANTED":
+        # P0.5: comma-joined list, alphabetically sorted ascending
+        # so the line is stable across reorderings of the manifest
+        # runtime_iam list.
+        sorted_perms = sorted(result.granted)
+        _say(
+            f"IAM pre-flight: OK "
+            f"({', '.join(sorted_perms)}: granted)"
+        )
+        return
+    if result.status == "DENIED":
+        # The contract is one FAILED line per missing permission.
+        # Emit one line per perm in input order (the
+        # IamPreflightResult preserves request order in `missing`),
+        # then exit 3.
+        for perm in result.missing:
+            _say(
+                f"IAM pre-flight: FAILED — {perm} not granted "
+                f"to caller. Install aborted."
+            )
+        sys.exit(3)
+    if result.status == "HTTP_ERROR":
+        _die(
+            f"IAM pre-flight: FAILED — HTTP {result.http_status} "
+            f"{result.http_reason} from testIamPermissions; "
+            f"install aborted.",
+            code=3,
+        )
+    if result.status == "NON_JSON":
+        # The library does not record the HTTP status when the
+        # body is non-JSON (only the exception class). 200 is the
+        # documented case per the library docstring; emit the
+        # contract line with that assumption.
+        _die(
+            f"IAM pre-flight: FAILED — HTTP 200 non-JSON body "
+            f"from testIamPermissions; install aborted.",
+            code=3,
+        )
+    # Defensive: any unrecognised status string is a library bug.
+    _die(
+        f"IAM pre-flight: FAILED — unrecognised pre-flight "
+        f"result status {result.status!r}",
+        code=3,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Steps 8-9: download, hash verify.
+# ---------------------------------------------------------------------------
+
+
+def _download_zip(gs_uri: str) -> bytes:
+    """Anonymous HTTPS GET against the public-read GCS object.
+
+    No ADC scope -- the bucket is public-read. We still
+    route through http_get_retry so 5xx surfaces the same
+    transient-failure line as the authenticated paths."""
+    assert gs_uri.startswith("gs://")
+    bucket, _, obj = gs_uri[5:].partition("/")
+    url = f"https://storage.googleapis.com/{bucket}/{obj}"
+    try:
+        r = http_get_retry(url, on_retry=_on_retry, timeout=60)
+    except requests.HTTPError as exc:
+        resp = exc.response
+        code = getattr(resp, "status_code", "???")
+        reason = getattr(resp, "reason", "")
+        _die(
+            f"zip download: FAILED — HTTP {code} {reason} "
+            f"from {url}.",
+            code=2,
+        )
+    _say(f"zip download: {gs_uri} -> {len(r.content)}B")
+    return r.content
+
+
+def _verify_zip_hash(zip_bytes: bytes, expected: str) -> None:
+    actual = hashlib.sha256(zip_bytes).hexdigest()
+    if actual != expected:
+        _die(
+            f"zip hash: FAILED — expected sha256={expected}, "
+            f"got sha256={actual}. Install aborted.",
+            code=3,
+        )
+    _say(f"zip hash: OK (sha256={expected} == actual)")
+
+
+# ---------------------------------------------------------------------------
+# Step 11: atomic install + safety nets.
+# ---------------------------------------------------------------------------
+
+
+def _safe_extract(
+    zf: zipfile.ZipFile,
+    dest: Path,
+    max_members: int | None = None,
+    max_total_bytes: int | None = None,
+) -> None:
+    """P0.2: zip-slip + zip-bomb safe extractor.
+
+    Three classes of abort, each surfacing a failure line:
+      - member count > cap
+      - decompressed total > cap
+      - any member's resolved path escapes `dest`
+
+    Caps default to ``MAX_ZIP_MEMBERS`` / ``MAX_ZIP_DECOMPRESSED_BYTES``
+    resolved from module-level globals at CALL time (not def time)
+    so tests can monkeypatch the constants without re-binding the
+    default parameter values.
+    """
+    if max_members is None:
+        max_members = MAX_ZIP_MEMBERS
+    if max_total_bytes is None:
+        max_total_bytes = MAX_ZIP_DECOMPRESSED_BYTES
+    members = zf.infolist()
+    if len(members) > max_members:
+        _die(
+            f"install: FAILED - zip member count {len(members)} "
+            f"> cap {max_members}",
+            code=3,
+        )
+    total = sum(m.file_size for m in members)
+    if total > max_total_bytes:
+        _die(
+            f"install: FAILED - zip decompressed size {total} "
+            f"> cap {max_total_bytes}",
+            code=3,
+        )
+    dest_resolved = dest.resolve()
+    for m in members:
+        target = (dest_resolved / m.filename).resolve()
+        if (
+            not str(target).startswith(str(dest_resolved) + os.sep)
+            and target != dest_resolved
+        ):
+            _die(
+                f"install: FAILED - zip path traversal: "
+                f"'{m.filename}' escapes staging dir",
+                code=3,
+            )
+    zf.extractall(dest_resolved)
+
+
+def _read_skill_md_name(
+    skill_md: Path,
+    max_frontmatter_bytes: int = MAX_FRONTMATTER_BYTES,
+) -> str:
+    """P1.2: size-bounded, line-ending-tolerant frontmatter
+    reader.
+
+    Replaces the brittle ``text.split('---')[1]`` approach. CRLF
+    and CR-only line endings are normalized to LF in memory before
+    the byte-prefix check; the on-disk file is untouched. This lets
+    Windows-authored SKILL.md files pass without a re-export step.
+    """
+    raw_on_disk = skill_md.read_bytes()
+    raw = (
+        raw_on_disk.replace(b"\r\n", b"\n").replace(b"\r", b"\n")
+    )
+    if not raw.startswith(b"---\n"):
+        _die(
+            "install: FAILED - bundled SKILL.md missing leading "
+            "'---' frontmatter",
+            code=3,
+        )
+    end = raw.find(b"\n---\n", 4)
+    if end == -1 or end > max_frontmatter_bytes:
+        _die(
+            f"install: FAILED - bundled SKILL.md frontmatter "
+            f"exceeds {max_frontmatter_bytes} bytes or missing "
+            f"closing '---'",
+            code=3,
+        )
+    try:
+        fm = yaml.safe_load(raw[4:end].decode("utf-8"))
+    except yaml.YAMLError as e:
+        # Convert YAMLError to a contract line instead
+        # of leaking a traceback (which would also leak the staging
+        # dir before the try/finally cleanup).
+        _die(
+            f"install: FAILED - bundled SKILL.md frontmatter YAML "
+            f"invalid: {e.__class__.__name__}",
+            code=3,
+        )
+    if not isinstance(fm, dict) or "name" not in fm:
+        _die(
+            "install: FAILED - bundled SKILL.md frontmatter "
+            "missing 'name' field",
+            code=3,
+        )
+    return fm["name"]
+
+
+def _write_breadcrumb(name: str) -> None:
+    """P1.3: cross-session breadcrumb for fallback-required path.
+
+    Written when the watcher is not ENABLED so a debugging
+    operator can later confirm 'yes, the install happened, the
+    user just missed the /reload-skills prompt.' Best-effort: any
+    OSError is swallowed so a missing breadcrumb never aborts a
+    successful install.
+    """
+    try:
+        SKILLS_ROOT.mkdir(parents=True, exist_ok=True)
+        BREADCRUMB.write_text(
+            json.dumps(
+                {
+                    "name": name,
+                    "installed_at": (
+                        datetime.datetime.now(
+                            datetime.timezone.utc
+                        ).isoformat()
+                    ),
+                    "fallback_required": True,
+                }
+            )
+        )
+    except OSError:
+        # Breadcrumb is best-effort; never abort install for it.
+        pass
+
+
+def _install_atomic(name: str, zip_bytes: bytes) -> Path:
+    SKILLS_ROOT.mkdir(parents=True, exist_ok=True)
+    STAGING_LOCK.touch(exist_ok=True)
+    target = SKILLS_ROOT / name
+    staging = SKILLS_ROOT / f".staging-{uuid.uuid4().hex}"
+    with open(STAGING_LOCK, "w") as lock_fd:
+        fcntl.flock(lock_fd, fcntl.LOCK_EX)
+        if target.exists():
+            _die(
+                f"install: REFUSED — skill {name} already "
+                f"installed at {target}. Uninstall first.",
+                code=1,
+            )
+        staging.mkdir()
+        try:
+            try:
+                with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:
+                    _safe_extract(zf, staging)
+            except zipfile.BadZipFile as e:
+                # Convert BadZipFile to a contract line.
+                # try/finally below still runs and removes staging.
+                _die(
+                    f"install: FAILED - corrupt zip archive: "
+                    f"{e.__class__.__name__}",
+                    code=3,
+                )
+            # pack_skill.py wraps the zip contents in a top-level
+            # directory matching the skill name. Detect-and-promote
+            # so the rest of the install logic sees the SKILL.md at
+            # the staging root.
+            wrapper = staging / name
+            if wrapper.is_dir() and not (staging / "SKILL.md").exists():
+                # Move children of the wrapper up one level.
+                for child in wrapper.iterdir():
+                    child.rename(staging / child.name)
+                wrapper.rmdir()
+            inner_name = _read_skill_md_name(staging / "SKILL.md")
+            if inner_name != name:
+                _die(
+                    f"package: FAILED — manifest.name={name} but "
+                    f"bundled SKILL.md declares name: {inner_name}. "
+                    f"Install aborted.",
+                    code=3,
+                )
+            try:
+                os.rename(staging, target)
+            except OSError as e:
+                if e.errno == errno.EXDEV:
+                    # Explicit EXDEV contract line. Exit
+                    # code 2 (operator fixable, not a security
+                    # failure).
+                    _die(
+                        "install: FAILED — EXDEV: staging dir on "
+                        "different filesystem than skills dir. "
+                        "Operator must colocate.",
+                        code=2,
+                    )
+                raise
+        finally:
+            # Cleanup runs on ALL failure paths
+            # (BadZipFile, YAMLError-from-_read_skill_md_name,
+            # SystemExit from _die, EXDEV, name-mismatch). On
+            # success os.rename has already consumed staging, so
+            # ignore_errors=True makes this a safe no-op.
+            shutil.rmtree(staging, ignore_errors=True)
+    _say(f"install: {target}/")
+    return target
+
+
+# ---------------------------------------------------------------------------
+# main()
+# ---------------------------------------------------------------------------
+
+
+def _validate_env_preflight(args: argparse.Namespace) -> None:
+    """Fail-fast pre-flight on operator-controlled env
+    vars surfaced via SKILL.md shell expansion.
+
+    argparse already rejects missing flags (required=True), but a
+    SKILL.md ``${APIHUB_PROJECT}`` that expanded to the empty
+    string would pass argparse and then fail mysteriously at API
+    hub with a 400. Catch empty strings here with an explicit
+    contract line. The "config:" prefix matches the existing
+    advisory line convention."""
+    for flag, val, env_var in (
+        ("--project", args.project, "APIHUB_PROJECT"),
+        ("--location", args.location, "APIHUB_LOCATION"),
+        ("--query", args.query, "ARGUMENTS"),
+    ):
+        if not val or not val.strip():
+            _die(
+                f"config: FAILED — {flag} is empty (typically "
+                f"sourced from ${{{env_var}}}). Set the environment "
+                f"variable before invoking skill-finder.",
+                code=2,
+            )
+
+
+def _validate_query(query: str) -> None:
+    """Defense-in-depth shell-injection blast-radius limiter.
+
+    The SKILL.md invocation passes ``--query "${ARGUMENTS}"`` via
+    a bash !`...` injection. If OpenCode's ``${ARGUMENTS}``
+    expansion is literal text substitution (NOT shell-safe
+    quoted), a user query containing an unescaped quote could
+    break out of the surrounding "" and inject shell commands.
+    We CANNOT fully prevent this from the Python script side
+    because the shell has already executed by the time we run.
+    What we CAN do is limit the blast radius: reject queries that
+    look like injection attempts BEFORE making any HTTP call.
+
+    Allowlist: printable ASCII only (``0x20 <= ord(c) < 0x7f``),
+    no newlines, no NUL bytes, length cap 500, and NO shell
+    metacharacters ($ ` \\ " ' ; & | < > ( ) { }). Apigee API hub
+    keywords are alphanumeric + dash, so legitimate queries stay
+    well within this band.
+
+    Shell metacharacters are forbidden because the SKILL.md
+    invocation passes ``--query "${ARGUMENTS}"`` via a bash !`...`
+    injection: POSIX double quotes do NOT suppress ``$(...)``
+    command substitution or backtick expansion, so the shell would
+    execute those constructs BEFORE find_install.py starts.
+    Rejecting them here is defense-in-depth; the SKILL.md tells
+    the agent to refuse them at the AGENT layer before the bash
+    expansion fires."""
+    _SHELL_METACHARS = frozenset('$`\\"\';&|<>(){}')
+    if (
+        len(query) > 500
+        or "\x00" in query
+        or "\n" in query
+        or "\r" in query
+        or not all(0x20 <= ord(c) < 0x7f for c in query)
+        or any(c in _SHELL_METACHARS for c in query)
+    ):
+        _die(
+            f"config: FAILED — query contains disallowed "
+            f"characters (non-printable, newline, null, shell "
+            f"metachar, or length > 500). Refusing to proceed to "
+            f"limit shell-injection blast radius if SKILL.md "
+            f"expansion was unsafe.",
+            code=2,
+        )
+
+
+def main(argv: list[str] | None = None) -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--query", required=True)
+    # --project and --location fall back to env vars OR the
+    # persistent config file at ~/.config/apigee-skills-demo/
+    # config.env. This makes find_install resilient to the
+    # operator launching the agent runtime before sourcing the env --
+    # the agent's bash subprocesses see no env vars in that
+    # case, but they CAN read the config file. The flags are
+    # still required (argparse will reject genuinely-missing
+    # values from BOTH paths), but the default lookup means
+    # the agent doesn't have to pass them explicitly when the
+    # demo is configured.
+    ap.add_argument(
+        "--project",
+        default=config.get("APIHUB_PROJECT"),
+    )
+    ap.add_argument(
+        "--location",
+        default=config.get("APIHUB_LOCATION"),
+    )
+    args = ap.parse_args(argv)
+
+    # Env pre-flight and query allowlist run
+    # BEFORE any other work (no stdout, no watcher probe, no
+    # HTTP) so the operator sees an actionable line and we never
+    # touch the network with a malformed input.
+    _validate_env_preflight(args)
+    _validate_query(args.query)
+
+    # Resolve env var AFTER the query / project validation so an
+    # advisory line never appears before the contract lines.
+    min_overlap = _min_keyword_overlap()
+
+    _say(f'query: "{args.query}"')
+
+    # Kick off the watcher probe BEFORE any network call.
+    # Its 2s settle overlaps the search/fetch/iam/download HTTP
+    # I/O so by the time we need the result at step 12a, the
+    # probe has already completed. Background thread runs
+    # detect_watcher().
+    watcher_probe = _WatcherProbeFuture()
+    watcher_probe.start()
+
+    trusted_pubkeys = _load_pubkeys()
+    apis = _search(args.project, args.location, args.query, min_overlap)
+    if not apis:
+        # Covers both "API hub returned []" and "nothing
+        # cleared MIN_KEYWORD_OVERLAP". Single line in the
+        # Hyrum's Law contract.
+        _die(
+            f'match: NONE — zero skills met minimum keyword '
+            f'overlap (threshold={min_overlap}) for query '
+            f'"{args.query}".',
+            code=1,
+        )
+    top = apis[0]
+    name_from_api = top.get("name", "").rsplit("/", 1)[-1]
+    # signing_key_id is nested under
+    # {FQ-attr-name: {stringValues: {values: [...]}}}, same shape
+    # as keywords. Pull from any attribute key ending in
+    # /attributes/signing_key_id.
+    declared_key_id = ""
+    for k, v in (top.get("attributes") or {}).items():
+        if k.endswith("/attributes/signing_key_id"):
+            vals = (v.get("stringValues") or {}).get("values") or []
+            declared_key_id = vals[0] if vals else ""
+            break
+    _say(
+        f"match (1 of {len(apis)}): name={name_from_api} "
+        f"signing_key_id={declared_key_id}"
+    )
+
+    creds, _project_id = _creds()
+    # P0.3: explicit length checks on versions[] and specs[]
+    # before indexing so an empty result surfaces as a
+    # contract line, not an IndexError traceback.
+    versions_url = f"{APIHUB_BASE}/{top['name']}/versions"
+    try:
+        vresp = http_get_retry(
+            versions_url,
+            on_retry=_on_retry,
+            timeout=30,
+            headers=_auth_headers(creds),
+        )
+    except requests.HTTPError as exc:
+        resp = exc.response
+        code = getattr(resp, "status_code", "???")
+        reason = getattr(resp, "reason", "")
+        _die(
+            f"search: FAILED — HTTP {code} {reason} from "
+            f"{versions_url}.",
+            code=2,
+        )
+    versions = (vresp.json() or {}).get("versions", []) or []
+    if not versions:
+        _die(
+            f"install: FAILED - registered skill '{name_from_api}' "
+            f"has zero versions (race with register-skill.py?)",
+            code=2,
+        )
+    specs = versions[0].get("specs", []) or []
+    if not specs:
+        _die(
+            f"install: FAILED - registered skill '{name_from_api}' "
+            f"version '{versions[0]['name']}' has zero specs",
+            code=2,
+        )
+    # API hub returns specs[] as a list of resource name STRINGS
+    # in the version's list view (not nested objects). Accept both
+    # shapes defensively.
+    first = specs[0]
+    spec_path = first if isinstance(first, str) else first.get("name")
+    if not spec_path:
+        _die(
+            f"install: FAILED - registered skill '{name_from_api}' "
+            f"version returned malformed spec reference",
+            code=2,
+        )
+    # _fetch_spec wants the bare spec resource path WITHOUT the
+    # leading APIHUB_BASE; strip if it slipped in.
+    if spec_path.startswith(APIHUB_BASE + "/"):
+        spec_path = spec_path[len(APIHUB_BASE) + 1:]
+    manifest_yaml = _fetch_spec(creds, spec_path)
+
+    manifest = _verify_signature(manifest_yaml, trusted_pubkeys)
+    _check_schema_version(manifest)
+    validate_manifest(manifest)
+    _cross_check(top, manifest)
+    _iam_preflight_with_contract(
+        args.project, list(manifest.get("runtime_iam", []) or [])
+    )
+
+    zip_bytes = _download_zip(manifest["gs_uri"])
+    _verify_zip_hash(zip_bytes, manifest["zip_sha256"])
+    target = _install_atomic(manifest["name"], zip_bytes)
+    _ = target  # silence unused-var; emitted via _say in _install_atomic
+
+    # Join the background probe. On the network-fast
+    # path the probe is already done; result() returns
+    # immediately. 10s timeout caps the test/mocked path.
+    watcher = watcher_probe.result(timeout=10.0)
+    agent = detect_active_agent()
+    perm = resolve_skill_permission(manifest["name"], agent)
+    if perm.verdict == Verdict.DENY:
+        _die(
+            f"agent `skill` tool: DENIED for agent={agent}. "
+            f"Switch agents and re-ask.",
+            code=3,
+        )
+    # Detect runtime by where we installed. Antigravity and Gemini
+    # CLI rescan the skills directory on the next conversation
+    # turn — no slash command needed. OpenCode requires
+    # /reload-skills if the file-watcher path didn't fire.
+    # Non-OpenCode runtimes (Antigravity + Gemini CLI) get the
+    # "re-ask your question" trailer because neither has the
+    # OpenCode /reload-skills slash command. Both rescan
+    # their skills directory on the next conversation turn.
+    is_agentic_runtime = SKILLS_ROOT in (
+        _ANTIGRAVITY_ROOT,
+        _GEMINI_CLI_ROOT,
+    )
+    if watcher == WatcherState.WATCHER_ENABLED:
+        _say("OpenCode file-watcher detected: yes")
+        _say(f"agent `skill` tool: allowed for agent={agent}")
+        _say(
+            f"Skill {manifest['name']} is now available — "
+            f"try your request again."
+        )
+    else:
+        # Distinguish the DISABLED (clean "NO") line from
+        # the UNDETECTABLE ("NO (probe inconclusive)") line.
+        suffix = (
+            " (probe inconclusive)"
+            if watcher == WatcherState.WATCHER_UNDETECTABLE
+            else ""
+        )
+        _say(f"OpenCode file-watcher detected: NO{suffix}")
+        _say(f"agent `skill` tool: allowed for agent={agent}")
+        _write_breadcrumb(manifest["name"])  # P1.3
+        if is_agentic_runtime:
+            # Antigravity and Gemini CLI both rescan their skills
+            # directory on the next conversation turn. The user
+            # just re-asks; neither has a /reload-skills slash
+            # command. The trailer wording deliberately omits the
+            # runtime name so the message is correct on both.
+            _say(
+                f"*** ACTION REQUIRED: re-ask your question — "
+                f"the agent runtime will pick up "
+                f"{manifest['name']} on the next turn. ***"
+            )
+        else:
+            _say(
+                '*** ACTION REQUIRED: type "/reload-skills" before '
+                're-asking your question. ***'
+            )
+
+
+if __name__ == "__main__":
+    main()
diff --git a/references/apigee-skills-serving/skills/skill-finder/scripts/list_skills.py b/references/apigee-skills-serving/skills/skill-finder/scripts/list_skills.py
new file mode 100644
index 00000000..456f3c67
--- /dev/null
+++ b/references/apigee-skills-serving/skills/skill-finder/scripts/list_skills.py
@@ -0,0 +1,372 @@
+#!/usr/bin/env python3
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""list_skills.py — browse the API hub catalog of agentic skills.
+
+Sibling of `find_install.py`. Where find_install.py SEARCHES for one
+matching skill and INSTALLS it, list_skills.py ENUMERATES every
+skill in the catalog so the agent can show the user "here is what
+is available." No download, no install, no signature verify, no
+side effects beyond stdout.
+
+Contract:
+- Prints a markdown table to stdout: `name | version | keywords |
+  description`. The first row is a header.
+- Emits `[skill-finder]` operator log lines to stdout for
+  diagnostics (search URL, pagination state).
+- On the last line, if API hub returned a `nextPageToken`, emits
+  `next-page-token: <token>` (no `[skill-finder]` prefix because
+  it is a customer-facing contract value the agent may need to
+  pass back as `--page-token`).
+- Exit 0 on success, 2 on bad config / arg validation, 3 on
+  HTTP / auth / response-parse failure.
+
+Dual-import preamble: production .skill zip layout is
+<install-dir>/scripts/{list_skills.py, common/*.py} where bare
+`from common.X` resolves via the sys.path.insert below. Dev/test
+layout uses `from scripts.common.X` via tests/conftest.py.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+from urllib.parse import quote_plus
+
+# =============================================================
+# Venv preflight. See find_install.py for the full rationale --
+# this is the same pattern: detect the "invoked with system
+# python3 instead of the venv wrapper" failure mode up front and
+# exit 78 (BSD sysexits.h EX_CONFIG) with an actionable message.
+# list_skills.py needs google.auth and requests transitively
+# via common.http_retry / common.config; PyYAML is not used here,
+# and cryptography is not loaded by list_skills directly. We
+# still probe the full venv dep set because (a) a partial venv
+# is itself a bug worth surfacing, and (b) consistency with
+# find_install.py makes the agent's behaviour predictable.
+# =============================================================
+_VENV_DEPS = ("yaml", "cryptography", "google.auth", "requests")
+_VENV_EX_CONFIG = 78
+_missing_dep: str | None = None
+_dep: str = ""  # noqa: appease static checker; rebound in loop
+for _dep in _VENV_DEPS:
+    try:
+        __import__(_dep)
+    except ImportError:
+        _missing_dep = _dep
+        break
+if _missing_dep is not None:
+    _SKILL_DIR = Path(__file__).resolve().parent.parent
+    _WRAPPER = _SKILL_DIR / "bin" / "run-with-venv.sh"
+    sys.stderr.write(
+        "[skill-finder] FATAL: this script requires the bundled "
+        "venv wrapper.\n"
+        f"  Missing module: {_missing_dep!r}\n"
+        f"  You invoked: python3 {sys.argv[0]}\n"
+        f"  You should invoke:\n"
+        f"      {_WRAPPER} {sys.argv[0]} <args>\n"
+        "  The wrapper activates the per-user venv created by\n"
+        "  install-skill-finder.sh, which contains the four\n"
+        "  required dependencies. The system python3 does not.\n"
+        "  If the wrapper is missing, re-run\n"
+        "  bin/install-skill-finder.sh to regenerate it.\n"
+    )
+    sys.exit(_VENV_EX_CONFIG)
+del _VENV_DEPS, _missing_dep, _dep
+
+# Dual import: production .skill zip layout has no
+# `scripts/` parent package; bare `common.*` resolves via the
+# sys.path.insert below. Dev/test layout uses `scripts.common.*`
+# via tests/conftest.py.
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+try:
+    from common import http_retry  # production .skill zip layout
+    from common import config
+except ImportError:
+    from scripts.common import http_retry  # dev/test layout
+    from scripts.common import config
+
+
+CLOUD_PLATFORM_SCOPE = "https://www.googleapis.com/auth/cloud-platform"
+APIHUB_BASE = "https://apihub.googleapis.com/v1"
+
+
+def _say(msg: str) -> None:
+    """Emit a `[skill-finder]` operator log line."""
+    print(f"[skill-finder] {msg}", flush=True)
+
+
+def _die(msg: str, *, code: int) -> None:
+    """Emit a FAILED contract line and exit non-zero."""
+    print(f"[skill-finder] {msg}", flush=True)
+    sys.exit(code)
+
+
+def _creds():
+    """Return (credentials, project_id) from ADC, refreshed.
+
+    Same pattern as find_install._creds(): the refresh-before-return
+    is the lesson learned from the real-infra demo (a fresh
+    `google.auth.default()` returns `creds.token = None` until the
+    first explicit refresh, producing silent HTTP 401 responses
+    otherwise).
+    """
+    import google.auth as google_auth
+    from google.auth.transport.requests import Request
+
+    credentials, project_id = google_auth.default(
+        scopes=[CLOUD_PLATFORM_SCOPE]
+    )
+    credentials.refresh(Request())
+    return credentials, project_id
+
+
+def _attr_string(attrs: dict, suffix: str) -> str:
+    """Pull the first string value out of an API-hub attribute
+    map keyed by fully-qualified attribute name.
+
+    Same shape-handling as find_install.py's `_cross_check`:
+    attributes look like
+        {`projects/<p>/locations/<l>/attributes/<id>`:
+            {`stringValues`: {`values`: [...]}}}
+    and we don't want to hard-code the project/location prefix.
+    """
+    for k, v in (attrs or {}).items():
+        if k.endswith(suffix):
+            vals = (v.get("stringValues") or {}).get("values") or []
+            return vals[0] if vals else ""
+    return ""
+
+
+def _attr_list(attrs: dict, suffix: str) -> list[str]:
+    """Pull every string value out of an API-hub attribute (used
+    for `keywords` which has cardinality > 1)."""
+    for k, v in (attrs or {}).items():
+        if k.endswith(suffix):
+            return list((v.get("stringValues") or {}).get("values") or [])
+    return []
+
+
+def _validate_args(ns: argparse.Namespace) -> None:
+    """Enforce non-empty required fields and bounds on
+    --page-size and --page-token.
+
+    The values flow into an HTTP query string, so we want to
+    reject anything that looks like injection or accidental
+    misuse.
+    """
+    for flag, val in (
+        ("--project", ns.project),
+        ("--location", ns.location),
+    ):
+        if not (val or "").strip():
+            _die(
+                f"config: FAILED — {flag} is empty (typically "
+                f"sourced from an environment variable). Set "
+                f"the environment variable before invoking "
+                f"list_skills.",
+                code=2,
+            )
+    if not (1 <= ns.page_size <= 200):
+        _die(
+            f"config: FAILED — --page-size must be 1..200, got "
+            f"{ns.page_size}.",
+            code=2,
+        )
+    # Page token is opaque to us; just bound its length and
+    # forbid newline / null / control characters.
+    pt = ns.page_token or ""
+    if len(pt) > 2048:
+        _die(
+            f"config: FAILED — --page-token too long "
+            f"(>2048 chars). Truncated or corrupted token?",
+            code=2,
+        )
+    for c in pt:
+        if ord(c) < 0x20 or ord(c) == 0x7f:
+            _die(
+                f"config: FAILED — --page-token contains a "
+                f"non-printable character. Token corrupted in "
+                f"transit?",
+                code=2,
+            )
+
+
+def _list_one_page(
+    creds,
+    project: str,
+    location: str,
+    page_size: int,
+    page_token: str,
+) -> tuple[list[dict], str]:
+    """Call API hub list_apis filtered to agentic skills. Returns
+    `(apis, next_page_token)`. next_page_token is empty when the
+    catalog is exhausted."""
+    base = (
+        f"{APIHUB_BASE}/projects/{project}"
+        f"/locations/{location}/apis"
+    )
+    attr_fq = (
+        f"projects/{project}/locations/{location}"
+        f"/attributes/agentic_skill"
+    )
+    filter_q = (
+        f'attributes.{attr_fq}.string_values.values:"true"'
+    )
+    params = [
+        f"filter={quote_plus(filter_q)}",
+        f"pageSize={page_size}",
+    ]
+    if page_token:
+        params.append(f"pageToken={quote_plus(page_token)}")
+    url = base + "?" + "&".join(params)
+    _say(f"list: GET {url}")
+    headers = {"Authorization": f"Bearer {creds.token}"}
+
+    def _on_retry(status: int, sleep_s: float) -> None:
+        _say(
+            f"transient failure (HTTP {status}); retry 1/1 "
+            f"after {int(round(sleep_s * 1000))}ms"
+        )
+
+    r = http_retry.http_get_retry(url, headers=headers, on_retry=_on_retry)
+    if r.status_code >= 400:
+        _die(
+            f"list: FAILED — HTTP {r.status_code} {r.reason} "
+            f"from list_apis; check ADC and "
+            f"projects/{project}/locations/{location} permissions.",
+            code=3,
+        )
+    try:
+        body = r.json()
+    except json.JSONDecodeError as e:
+        _die(
+            f"list: FAILED — HTTP {r.status_code} non-JSON body "
+            f"from list_apis: {e.__class__.__name__}",
+            code=3,
+        )
+    apis = body.get("apis", []) or []
+    next_token = body.get("nextPageToken") or ""
+    return apis, next_token
+
+
+def _render_table(apis: list[dict]) -> str:
+    """Render the list of API hub catalog entries as a markdown
+    table. The columns are chosen for what an agent / user wants
+    to see when browsing: name, version (from manifest if listed),
+    keywords (the search-relevance signal), one-line description."""
+    if not apis:
+        return "_(no skills in catalog)_"
+    rows = [
+        "| name | keywords | description |",
+        "|:---|:---|:---|",
+    ]
+    for api in apis:
+        name = (api.get("name") or "").rsplit("/", 1)[-1]
+        attrs = api.get("attributes") or {}
+        keywords = _attr_list(attrs, "/attributes/keywords")
+        description = (api.get("description") or "").strip()
+        # Markdown: collapse newlines + escape pipes to keep
+        # table layout intact.
+        description = description.replace("\n", " ").replace("|", "\\|")
+        kw = ", ".join(keywords) if keywords else "_(none)_"
+        rows.append(f"| `{name}` | {kw} | {description} |")
+    return "\n".join(rows)
+
+
+def _parse_args(argv: list[str]) -> argparse.Namespace:
+    p = argparse.ArgumentParser(
+        prog="list_skills.py",
+        description=(
+            "Browse the API hub catalog of agentic skills. "
+            "Prints a markdown table of (name, keywords, "
+            "description) for every skill registered with "
+            "agentic_skill=true. Cursor-paginated via "
+            "--page-size + --page-token."
+        ),
+    )
+    # Defaults resolve from (1) env var, (2) the persistent
+    # config file at ~/.config/apigee-skills-demo/config.env.
+    # This decouples readiness from the operator having sourced
+    # env into the right shell at the right time (notably: when
+    # the agent runtime was started before the env was set, the
+    # agent's bash subprocesses see no env vars, but they CAN
+    # read the config file).
+    p.add_argument(
+        "--project",
+        default=config.get("APIHUB_PROJECT"),
+        help=(
+            "API hub project (defaults to $APIHUB_PROJECT or "
+            "the APIHUB_PROJECT line in "
+            "~/.config/apigee-skills-demo/config.env)."
+        ),
+    )
+    p.add_argument(
+        "--location",
+        default=config.get("APIHUB_LOCATION"),
+        help=(
+            "API hub location (defaults to $APIHUB_LOCATION or "
+            "the APIHUB_LOCATION line in "
+            "~/.config/apigee-skills-demo/config.env)."
+        ),
+    )
+    p.add_argument(
+        "--page-size",
+        type=int,
+        default=20,
+        help="Number of skills per page (1..200, default 20).",
+    )
+    p.add_argument(
+        "--page-token",
+        default="",
+        help=(
+            "Opaque pagination token from a prior call's "
+            "`next-page-token:` line. Empty means start at "
+            "the beginning."
+        ),
+    )
+    return p.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    ns = _parse_args(argv if argv is not None else sys.argv[1:])
+    _validate_args(ns)
+
+    creds, _ = _creds()
+    apis, next_token = _list_one_page(
+        creds,
+        ns.project,
+        ns.location,
+        ns.page_size,
+        ns.page_token,
+    )
+
+    _say(f"list: OK ({len(apis)} skills returned)")
+    print(_render_table(apis), flush=True)
+    if next_token:
+        # Unprefixed customer-facing contract line: the agent
+        # surfaces this so the user knows there's more, and so
+        # the agent can pass it back as --page-token on a
+        # follow-up call.
+        print(f"\nnext-page-token: {next_token}", flush=True)
+    else:
+        _say("list: end of catalog (no more pages)")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/references/apigee-skills-serving/skills/skill-publisher/SKILL.md b/references/apigee-skills-serving/skills/skill-publisher/SKILL.md
new file mode 100644
index 00000000..91dd3561
--- /dev/null
+++ b/references/apigee-skills-serving/skills/skill-publisher/SKILL.md
@@ -0,0 +1,312 @@
+---
+name: skill-publisher
+description: Publishes a local skill directory to the API hub catalog and
+  the GCS bundle store. Runs the four-step author pipeline end to end,
+  packing the source tree into a deterministic .skill zip, signing the
+  manifest with an ed25519 private key, uploading the zip to GCS, and
+  registering the signed manifest with API hub as an API plus Version
+  plus Spec triple along with its four attribute values. Idempotent
+  by design; re-running with identical inputs produces byte-identical
+  output. Includes a bootstrap mode for first-time self-publication.
+license: Apache-2.0
+compatibility: opencode, antigravity, gemini-cli
+metadata:
+  pipeline_scripts:
+    - scripts.pack_skill
+    - scripts.sign_skill
+    - scripts.upload_skill
+    - scripts.register_skill
+---
+
+# skill-publisher
+
+## ⚠️ Runtime requirements (read this first)
+
+This skill **requires** its bundled venv wrapper. `publish.sh`
+internally invokes the four `scripts/*` Python modules
+(`pack_skill`, `sign_skill`, `upload_skill`, `register_skill`),
+which depend on `cryptography`, `google-auth`, `requests`, and
+`pyyaml`. Those are installed into a per-user venv by the
+installer (`bin/install-skill-publisher.sh`). They are **not**
+available to the system `python3`.
+
+**Always invoke `publish.sh` via the wrapper, never via bare
+`bash`:**
+
+```bash
+# CORRECT - the wrapper exports PYTHON to point at the venv:
+${SKILL_DIR}/bin/run-with-venv.sh <args>
+
+# WRONG - publish.sh will fall back to system python3 which
+# will hit ModuleNotFoundError mid-pipeline:
+bash ${SKILL_DIR}/scripts/publish.sh <args>
+```
+
+The `Command` blocks in this file use the wrapper form. Do not
+"simplify" them — `publish.sh` reads `$PYTHON` from its
+environment and the wrapper is the only thing that sets it
+correctly. If you invoke `publish.sh` directly, the four
+sub-scripts will fail with `ModuleNotFoundError`.
+
+If the wrapper at `${SKILL_DIR}/bin/run-with-venv.sh` is missing,
+the skill was installed by a pre-v0.1.4 installer. Re-run
+`bin/install-skill-publisher.sh` to regenerate the wrapper.
+
+---
+
+Use this skill when the user asks to "publish a skill", "ship this
+skill to API hub", "upload and register my skill", "release the
+skill", or any close paraphrase. This is the single entry point for
+author-side distribution; it wraps the four `scripts/*` CLIs from
+the upstream source repo into one orchestrated sequence with
+fail-fast semantics.
+
+## What this skill does NOT do
+
+- It does **not** create the GCS bucket. The bucket must already
+  exist (see repo README §2).
+- It does **not** create API hub attribute definitions. Run
+  `python3 -m scripts.update_taxonomy` once per project first
+  (see repo README §4).
+- It does **not** generate or rotate signing keys. The
+  ed25519 private key path is an input.
+- It does **not** install the published skill into any local
+  runtime. That is `skill-finder`'s job.
+
+If any prerequisite is missing the underlying script returns a
+non-zero exit code and this skill surfaces it verbatim and stops.
+
+## Inputs
+
+| Variable | Source | Required | Notes |
+|:---------|:-------|:---------|:------|
+| `SKILL_SRC` | User / agent | yes | Path to a `skills/<name>/` source directory containing `manifest.yaml` and (at minimum) `SKILL.md`. |
+| `SKILL_OUT` | Operator | no | Output path for the packed zip. Defaults to `/tmp/${name}-${version}.skill` derived from the manifest. |
+| `GCS_BUCKET` | Operator (`${GCS_BUCKET}`) | yes | GCS bucket name (no `gs://` prefix). Must match the bucket portion of the manifest's `gs_uri`. |
+| `SIGNING_PRIV_KEY` | Operator (`${SIGNING_PRIV_KEY}`) | yes | Path to the raw 32-byte ed25519 private key. See the upstream README's author-flow section for how to generate one. |
+| `APIHUB_PROJECT` | Operator (`${APIHUB_PROJECT}`) | yes | GCP project hosting the API hub instance. |
+| `APIHUB_LOCATION` | Operator (`${APIHUB_LOCATION}`) | yes | API hub region (e.g. `us-central1`). |
+| `REPO_ROOT` | Auto | no | Root of the upstream source checkout. Defaults to the current working directory. The four `scripts/*` modules must be importable from there. |
+| `BOOTSTRAP` | User / agent | no | If set to `1` or `true`, runs the bootstrap path that publishes `skill-publisher` itself. See "Bootstrap mode" below. |
+
+## Runtime dispatch
+
+This skill ships one orchestration contract that runs on multiple
+runtimes. Pick the invocation path that matches your runtime; the
+behaviour and stdout contract are identical.
+
+| Runtime | How you invoke the publish script |
+|:--------|:----------------------------------|
+| **OpenCode** | Use the `!`bash`` injection blocks below. OpenCode auto-executes them on SKILL.md load. |
+| **Gemini CLI** | Use your bash tool. Run the matching `Command` block, substituting `${SKILL_DIR}` with the install path (`~/.gemini/skills/skill-publisher` for global; `<project>/.agents/skills/skill-publisher` for workspace) and the operator-controlled env vars from the table above. |
+| **Antigravity** | Same as Gemini CLI but `${SKILL_DIR}` is `~/.gemini/antigravity/skills/skill-publisher` for global installs. |
+| **Any other runtime** | Same as Antigravity / Gemini CLI: invoke via whatever bash mechanism the runtime provides. |
+
+---
+
+## Mode 1: Publish a third-party skill
+
+Use this when the user wants to publish any skill **other than**
+`skill-publisher` itself.
+
+### Steps
+
+1. **Resolve the source path.** Take the value from `SKILL_SRC`
+   verbatim if provided. Otherwise ask the user once: "Which skill
+   directory do you want to publish?" Expect a path like
+   `skills/apigee-policy-top10`. Do not paraphrase the answer.
+
+2. **Validate the inputs before invoking bash.** Verify that the
+   following are all set and non-empty: `SKILL_SRC`, `GCS_BUCKET`,
+   `SIGNING_PRIV_KEY`, `APIHUB_PROJECT`, `APIHUB_LOCATION`. If any
+   is missing, REFUSE to invoke and tell the user which one is
+   missing. Do not guess.
+
+3. **Reject shell metacharacters in `SKILL_SRC`.** The path is
+   interpolated into a bash command via POSIX double quotes, which
+   do not suppress `$(...)` command substitution or backtick
+   expansion. If `SKILL_SRC` contains any of
+   `` ` `` `$` `\` `"` `;` `&` `|` `<` `>` `(` `)` `{` `}`
+   REFUSE to invoke and ask the user to rephrase.
+
+4. **Invoke the publish script using the runtime path from the
+   table above.** The exact command is:
+
+   **Command** (OpenCode auto-runs this; on Antigravity / Gemini
+   CLI run it via your bash tool):
+
+   !`bash ${SKILL_DIR}/scripts/publish.sh \
+        --src "${SKILL_SRC}" \
+        --bucket "${GCS_BUCKET}" \
+        --priv-key "${SIGNING_PRIV_KEY}" \
+        --project "${APIHUB_PROJECT}" \
+        --location "${APIHUB_LOCATION}"`
+
+5. **Surface the script's stdout verbatim.** The script prints
+   one `[skill-publisher] step N/4 ...` line per pipeline step,
+   followed by the GCS URI and the API hub resource path on
+   success. Reproduce every line as-is; do NOT summarize. The
+   lines are a stable contract for log parsing.
+
+6. **On non-zero exit, stop.** Surface the failing step's stderr
+   verbatim. Do not retry, do not "fix" the manifest, do not
+   re-sign with a different key. The four underlying scripts have
+   distinct exit codes (see "Exit code mapping" below); use them
+   to give the user an accurate diagnosis.
+
+---
+
+## Mode 2: Bootstrap — publish skill-publisher itself
+
+Use this when the user wants to publish `skill-publisher` for the
+first time, or republish it after editing its own source.
+
+### The bootstrap problem
+
+`skill-publisher` cannot use its own installed copy to publish
+itself: at first run the installed copy does not yet exist in the
+API hub catalog. Once published, subsequent versions of
+`skill-publisher` *can* be republished by the currently-installed
+copy — but the very first publish is a chicken-and-egg situation.
+
+### Steps
+
+1. **Confirm the user actually means bootstrap.** Ask once: "Do
+   you want to publish `skill-publisher` itself (bootstrap mode)?
+   This is normally only done by repo maintainers." If the user
+   says no, switch to Mode 1.
+
+2. **Validate the inputs** exactly as Mode 1 step 2. `SKILL_SRC`
+   for bootstrap is always the path to the in-repo
+   `skills/skill-publisher/` directory; default to that if the
+   user does not specify.
+
+3. **Invoke with `--bootstrap`.** The bootstrap flag tells
+   `publish.sh` to use the repo-local `scripts/publish.sh` (this
+   file) directly via the four sibling Python modules in
+   `${REPO_ROOT}/scripts/`, rather than relying on any installed
+   copy. The exact command is:
+
+   **Command:**
+
+   !`bash ${SKILL_DIR}/scripts/publish.sh \
+        --src "${SKILL_SRC:-skills/skill-publisher}" \
+        --bucket "${GCS_BUCKET}" \
+        --priv-key "${SIGNING_PRIV_KEY}" \
+        --project "${APIHUB_PROJECT}" \
+        --location "${APIHUB_LOCATION}" \
+        --bootstrap`
+
+4. **Surface stdout verbatim** (same as Mode 1 step 5).
+
+5. **On success, advise the user to re-run with Mode 1.** Tell
+   them: "skill-publisher is now in the catalog. Any future
+   publish — including republishing skill-publisher itself — can
+   use the installed copy via skill-finder + this skill, without
+   --bootstrap."
+
+---
+
+## Exit code mapping
+
+`publish.sh` exits with the exit code of the **first failing
+step**. The mapping below is the union of the four underlying
+scripts' exit codes (see each script's docstring for detail):
+
+| Exit code | Meaning | Where it comes from |
+|:----------|:--------|:--------------------|
+| `0` | All four steps succeeded. | n/a |
+| `1` | User error: bad CLI args, missing file, invalid YAML, ADC unavailable, or empty required env var. | any step |
+| `2` | System error: filesystem write failure, GCS network error, GCS 404 (bucket not found), or API hub network error. | pack / upload / register |
+| `3` | Either (a) cryptographic error during sign (priv key unreadable / wrong length) OR (b) GCS IAM denial (403) during upload OR (c) API hub IAM denial (403) during register. The failing-step log line in stdout disambiguates. | sign / upload / register |
+| `4` | API hub rejected the registration because the four attribute definitions are not initialised. Run `python3 -m scripts.update_taxonomy` once and retry. | register only |
+| `5` | Packaging policy violation: `scripts/common/` has the wrong file set (missing or extra files), or the source dir is missing `SKILL.md`. | pack only (originally exit 3 in pack_skill; remapped here to disambiguate from sign/IAM). |
+
+The `[skill-publisher] step N/4 FAILED` line printed by
+`publish.sh` immediately before exit tells the user which step
+failed and what the underlying script's raw exit code was, so the
+remap above is auditable, not opaque.
+
+---
+
+## Idempotency
+
+All four underlying scripts are idempotent by design:
+
+- `pack_skill.py` writes the zip with sorted entry order, so
+  `sha256(zip)` is stable across builds.
+- `sign_skill.py` produces byte-identical output for identical
+  inputs (ed25519 is deterministic per RFC 8032; YAML is dumped
+  with `sort_keys=True`).
+- `upload_skill.py` overwrites the object at the same name —
+  re-running uploads the same bytes to the same key.
+- `register_skill.py` reads first, only POSTs/PATCHes on diff;
+  re-running with no changes makes only `GET` calls.
+
+Therefore re-running `publish.sh` with the same inputs is safe
+and observable as zero mutating API calls after the first
+successful publish. The skill does NOT need a "dry-run" mode of
+its own; pass `--dry-run` through to `register_skill` if you want
+to skip the registration step entirely (currently not wired —
+file an issue if needed).
+
+---
+
+## Security notes
+
+- The ed25519 private key path is the only secret this skill
+  touches. It is read by `sign_skill.py`, never logged, never
+  uploaded. The key file should have mode `0600`.
+- The signature is computed over the **canonical** manifest bytes
+  (see `scripts/common/canonical.py`). Any post-signing edit to
+  the manifest invalidates the signature; `skill-finder` will
+  refuse to install the skill.
+- The `signing_key_id` field is the sha256 of the **public** key,
+  not the private key. It is safe to publish. `skill-finder`
+  cross-checks it against its embedded trust root
+  (`keys/trusted_pubkey.pem`); a manifest signed by a key not in
+  the trust root is rejected client-side.
+- GCS uploads use ADC. The operator must have
+  `storage.objects.create` on the target bucket. The skill does
+  not run a permission pre-flight; it relies on the upload itself
+  to surface 403 as exit code 3.
+
+---
+
+## Common rationalizations
+
+| Rationalization | Why it fails here |
+|:----------------|:------------------|
+| "I'll skip the pack step — the zip already exists on disk." | `sign_skill.py` writes `zip_sha256` from a fresh sha256 of the zip bytes. If the zip on disk is stale (e.g. a SKILL.md edit hasn't been re-packed), the manifest commits to a hash that does not match the bytes `skill-finder` will fetch from GCS, and every install fails signature verification. Always run pack-then-sign in one sequence. |
+| "I'll re-sign without re-packing — only the manifest changed." | Same trap. If the manifest text changed but the zip bytes also changed (because `pack_skill` would have noticed a source-tree edit), `zip_sha256` is wrong. The pipeline is a unit; do not split it. |
+| "I'll upload before signing — saves a round trip." | If signing fails (bad key, invalid manifest), the GCS object now points at a zip whose manifest is unsigned. Any client that fetches it gets a manifest that fails schema validation. Sign first, upload second, register third — this order is enforced by `publish.sh`. |
+| "I'll register before uploading — the catalog can wait for the zip." | `skill-finder` fetches the zip immediately after the manifest passes signature check. A registered manifest pointing at a non-existent `gs_uri` returns 404 to every install attempt. Upload before register. |
+| "I'll use `gcloud storage cp` instead of `upload_skill.py`." | The script uses ADC + the GCS JSON API directly (the runtime tree is capped at four packages, so no `google-cloud-storage` dependency). It also has structured exit codes that this skill remaps. Using `gcloud` breaks the exit-code contract and the offline test suite. |
+| "I'll skip `update_taxonomy.py` — the attributes will auto-create." | They will not. `register_skill.py` PATCHes attribute *values* by reference to attribute *definitions* that must already exist. Without `update_taxonomy.py` having run once, the PATCH returns 400 and this skill exits 4. The first publish in a new project always needs `update_taxonomy.py` first. |
+| "I'll catch the failure of step 2 and retry step 2 only." | Don't. If sign fails after pack succeeded, the zip on disk is fine but the manifest is broken — re-running the whole pipeline is safe (pack is deterministic) and avoids the "stale zip" trap above. Always retry the whole pipeline, never a single step. |
+
+---
+
+## Layout
+
+```
+skills/skill-publisher/
+  SKILL.md                # this file
+  manifest.yaml           # unsigned template; sign-step rewrites
+                          # zip_sha256, signing_key_id, signature
+                          # in place
+  scripts/
+    publish.sh            # the bash orchestrator that runs the
+                          # four scripts/* modules in order
+```
+
+`publish.sh` is the entire runtime surface of this skill. It
+shells out to:
+
+- `python3 -m scripts.pack_skill   --src <SKILL_SRC> --out <SKILL_OUT>`
+- `python3 -m scripts.sign_skill   --manifest <SKILL_SRC>/manifest.yaml --zip <SKILL_OUT> --priv-key <SIGNING_PRIV_KEY> --in-place`
+- `python3 -m scripts.upload_skill --zip <SKILL_OUT> --bucket <GCS_BUCKET> --object-name <name>-<version>.skill`
+- `python3 -m scripts.register_skill --manifest <SKILL_SRC>/manifest.yaml --project <APIHUB_PROJECT> --location <APIHUB_LOCATION>`
+
+The four module paths are stable; if the repo ever renames them,
+update `publish.sh` and bump the skill version.
diff --git a/references/apigee-skills-serving/skills/skill-publisher/manifest.yaml b/references/apigee-skills-serving/skills/skill-publisher/manifest.yaml
new file mode 100644
index 00000000..d8e2bdfc
--- /dev/null
+++ b/references/apigee-skills-serving/skills/skill-publisher/manifest.yaml
@@ -0,0 +1,43 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+# Source manifest for skill-publisher. The publishing pipeline
+# (scripts/sign_skill.py + scripts/upload_skill.py) populates the
+# gs_uri, zip_sha256, signature, and signing_key_id fields at
+# publish time. Do not commit those values back to the source
+# tree.
+author: apigee-devrel
+capabilities: []
+description: Publishes a local skill directory to the API hub catalog
+  and the GCS bundle store. Runs the four-step author pipeline end
+  to end, packing the source tree into a deterministic .skill zip,
+  signing the manifest with an ed25519 private key, uploading the
+  zip to GCS, and registering the signed manifest with API hub as
+  an API plus Version plus Spec triple along with its four attribute
+  values. Idempotent by design; re-running with identical inputs
+  produces byte-identical output. Includes a bootstrap mode for
+  first-time self-publication.
+keywords:
+  - publish
+  - skill
+  - apihub
+  - gcs
+  - sign
+  - ed25519
+  - author
+license: Apache-2.0
+manifest_schema_version: '1'
+name: skill-publisher
+runtime_iam: []
+version: 0.1.0
diff --git a/references/apigee-skills-serving/skills/skill-publisher/scripts/publish.sh b/references/apigee-skills-serving/skills/skill-publisher/scripts/publish.sh
new file mode 100755
index 00000000..4827b161
--- /dev/null
+++ b/references/apigee-skills-serving/skills/skill-publisher/scripts/publish.sh
@@ -0,0 +1,266 @@
+#!/usr/bin/env bash
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# publish.sh — orchestrate pack → sign → upload → register.
+#
+# This is the entire runtime surface of the skill-publisher skill.
+# SKILL.md invokes it once; it shells out to the four sibling Python
+# modules in ${REPO_ROOT}/scripts/ in order and exits on the first
+# failing step. The exit-code remap is documented in SKILL.md
+# ("Exit code mapping" table). Re-running with the same inputs is
+# safe (all four underlying scripts are idempotent).
+#
+# Usage:
+#   publish.sh --src <skill-dir>
+#              --bucket <gcs-bucket>
+#              --priv-key <ed25519-priv-key-path>
+#              --project <apihub-project>
+#              --location <apihub-location>
+#              [--out <zip-path>]
+#              [--repo-root <path>]
+#              [--bootstrap]
+#
+# All flags are required except --out, --repo-root, and --bootstrap.
+# --out defaults to /tmp/${name}-${version}.skill derived from the
+# manifest. --repo-root defaults to $(pwd). --bootstrap is a marker
+# flag; see SKILL.md "Mode 2: Bootstrap" for semantics.
+
+set -u
+# NOTE: we intentionally do NOT set -e. We want to capture the
+# exit code of each step explicitly, surface a structured failure
+# line, and remap the exit code per the SKILL.md table.
+
+# ---------------------------------------------------------------
+# Python interpreter resolution.
+# By default we use `python3` from PATH. On PEP 668 distros that
+# Python cannot import the runtime deps (cryptography, requests,
+# google-auth, pyyaml). The install-skill-publisher.sh installer
+# sets PYTHON to the per-user venv's interpreter and exports it
+# in the SKILL.md wrapper invocation. Direct callers can also
+# override by exporting PYTHON before invoking publish.sh.
+# ---------------------------------------------------------------
+PYTHON="${PYTHON:-python3}"
+
+# ---------------------------------------------------------------
+# Argument parsing (no getopt — keep it portable to mac bash 3.2)
+# ---------------------------------------------------------------
+SRC=""
+BUCKET=""
+PRIV_KEY=""
+PROJECT=""
+LOCATION=""
+OUT=""
+REPO_ROOT="$(pwd)"
+BOOTSTRAP=0
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --src)        SRC="$2"; shift 2 ;;
+    --bucket)     BUCKET="$2"; shift 2 ;;
+    --priv-key)   PRIV_KEY="$2"; shift 2 ;;
+    --project)    PROJECT="$2"; shift 2 ;;
+    --location)   LOCATION="$2"; shift 2 ;;
+    --out)        OUT="$2"; shift 2 ;;
+    --repo-root)  REPO_ROOT="$2"; shift 2 ;;
+    --bootstrap)  BOOTSTRAP=1; shift ;;
+    -h|--help)
+      sed -n '1,30p' "$0"
+      exit 0
+      ;;
+    *)
+      echo "[skill-publisher] FATAL: unknown flag: $1" >&2
+      exit 1
+      ;;
+  esac
+done
+
+log()  { echo "[skill-publisher] $*"; }
+err()  { echo "[skill-publisher] $*" >&2; }
+
+# ---------------------------------------------------------------
+# Pre-flight: required flags present and non-empty
+# ---------------------------------------------------------------
+for var in SRC BUCKET PRIV_KEY PROJECT LOCATION; do
+  if [ -z "${!var}" ]; then
+    err "FATAL: --${var,,} is required (got empty)"
+    exit 1
+  fi
+done
+
+if [ ! -d "$SRC" ]; then
+  err "FATAL: --src is not a directory: $SRC"
+  exit 1
+fi
+
+MANIFEST="$SRC/manifest.yaml"
+if [ ! -f "$MANIFEST" ]; then
+  err "FATAL: manifest not found: $MANIFEST"
+  exit 1
+fi
+
+if [ ! -f "$PRIV_KEY" ]; then
+  err "FATAL: --priv-key file not found: $PRIV_KEY"
+  exit 1
+fi
+
+if [ ! -d "$REPO_ROOT/scripts" ]; then
+  err "FATAL: --repo-root has no scripts/ directory: $REPO_ROOT"
+  err "       (the four pack/sign/upload/register modules must"
+  err "        be importable as 'scripts.*' from REPO_ROOT)"
+  exit 1
+fi
+
+# ---------------------------------------------------------------
+# Derive name / version / default OUT from the manifest
+# We use a tiny Python one-liner instead of grep so we don't have
+# to YAML-parse in bash. PyYAML is already a hard runtime dep
+# (see requirements.txt).
+# ---------------------------------------------------------------
+read_manifest_field() {
+  local field="$1"
+  "$PYTHON" -c "
+import sys, yaml
+with open('${MANIFEST}') as fh:
+    m = yaml.safe_load(fh)
+v = m.get('${field}')
+if v is None:
+    sys.exit(2)
+print(v)
+"
+}
+
+NAME="$(read_manifest_field name)" || {
+  err "FATAL: manifest missing 'name' field: $MANIFEST"
+  exit 1
+}
+VERSION="$(read_manifest_field version)" || {
+  err "FATAL: manifest missing 'version' field: $MANIFEST"
+  exit 1
+}
+
+if [ -z "$OUT" ]; then
+  OUT="/tmp/${NAME}-${VERSION}.skill"
+fi
+
+OBJECT_NAME="${NAME}-${VERSION}.skill"
+
+# ---------------------------------------------------------------
+# Bootstrap mode is a marker only — the four-step pipeline below
+# is identical. We log it so the operator sees their intent
+# echoed, and so the run is grep-able in operator logs.
+# ---------------------------------------------------------------
+if [ "$BOOTSTRAP" -eq 1 ]; then
+  log "BOOTSTRAP mode — publishing skill-publisher itself."
+  log "  src=$SRC name=$NAME version=$VERSION"
+fi
+
+log "config: src=$SRC out=$OUT bucket=$BUCKET"
+log "config: project=$PROJECT location=$LOCATION"
+log "config: name=$NAME version=$VERSION object=$OBJECT_NAME"
+
+# ---------------------------------------------------------------
+# Step runner. Captures the underlying script's exit code,
+# prints a structured failure line on non-zero, and remaps
+# the exit code per the SKILL.md table.
+# ---------------------------------------------------------------
+run_step() {
+  local step_num="$1"
+  local step_name="$2"
+  shift 2
+  log "step ${step_num}/4 START: ${step_name}"
+  # Run the command and capture exit. Inherit stdout/stderr so
+  # the underlying script's own structured logging is preserved.
+  "$@"
+  local rc=$?
+  if [ $rc -ne 0 ]; then
+    err "step ${step_num}/4 FAILED: ${step_name} (raw exit=${rc})"
+  else
+    log "step ${step_num}/4 OK: ${step_name}"
+  fi
+  return $rc
+}
+
+# ---------------------------------------------------------------
+# Step 1: pack
+# Exit-code remap: pack_skill.py uses exit 3 for "packaging
+# policy violation" (missing SKILL.md, wrong common/ file set).
+# We remap that to 5 so it doesn't collide with sign/IAM's 3.
+# ---------------------------------------------------------------
+run_step 1 "pack ${SRC} → ${OUT}" \
+  env PYTHONPATH="${REPO_ROOT}:${PYTHONPATH:-}" \
+  "$PYTHON" -m scripts.pack_skill \
+    --src "$SRC" \
+    --out "$OUT" \
+    --repo-root "$REPO_ROOT"
+rc=$?
+if [ $rc -ne 0 ]; then
+  if [ $rc -eq 3 ]; then
+    exit 5
+  fi
+  exit $rc
+fi
+
+# ---------------------------------------------------------------
+# Step 2: sign (in-place — overwrites zip_sha256, signing_key_id,
+# signature in $MANIFEST)
+# Exit codes pass through unchanged (1=user, 2=system, 3=crypto).
+# ---------------------------------------------------------------
+run_step 2 "sign ${MANIFEST} (in-place)" \
+  env PYTHONPATH="${REPO_ROOT}:${PYTHONPATH:-}" \
+  "$PYTHON" -m scripts.sign_skill \
+    --manifest "$MANIFEST" \
+    --zip "$OUT" \
+    --priv-key "$PRIV_KEY" \
+    --in-place
+rc=$?
+if [ $rc -ne 0 ]; then exit $rc; fi
+
+# ---------------------------------------------------------------
+# Step 3: upload to GCS
+# Exit codes pass through unchanged (1=user, 2=system, 3=IAM).
+# ---------------------------------------------------------------
+run_step 3 "upload ${OUT} → gs://${BUCKET}/${OBJECT_NAME}" \
+  env PYTHONPATH="${REPO_ROOT}:${PYTHONPATH:-}" \
+  "$PYTHON" -m scripts.upload_skill \
+    --zip "$OUT" \
+    --bucket "$BUCKET" \
+    --object-name "$OBJECT_NAME"
+rc=$?
+if [ $rc -ne 0 ]; then exit $rc; fi
+
+# ---------------------------------------------------------------
+# Step 4: register in API hub
+# Exit codes pass through unchanged (1=user, 2=system, 3=IAM,
+# 4=taxonomy not initialised).
+# ---------------------------------------------------------------
+run_step 4 "register ${MANIFEST} → API hub ${PROJECT}/${LOCATION}" \
+  env PYTHONPATH="${REPO_ROOT}:${PYTHONPATH:-}" \
+  "$PYTHON" -m scripts.register_skill \
+    --manifest "$MANIFEST" \
+    --project "$PROJECT" \
+    --location "$LOCATION"
+rc=$?
+if [ $rc -ne 0 ]; then exit $rc; fi
+
+# ---------------------------------------------------------------
+# Success summary — stable contract surface for the SKILL.md
+# "Surface stdout verbatim" instruction.
+# ---------------------------------------------------------------
+log "PUBLISH OK"
+log "  gs_uri:    gs://${BUCKET}/${OBJECT_NAME}"
+log "  apihub:    projects/${PROJECT}/locations/${LOCATION}/apis/${NAME}"
+log "  manifest:  ${MANIFEST} (signed in place)"
+log "  zip:       ${OUT}"
+exit 0

From df3ae710564eb10bf3867d4ae5229b016454124f Mon Sep 17 00:00:00 2001
From: Geir Sjurseth <gsjurseth@google.com>
Date: Tue, 7 Jul 2026 08:26:51 +0000
Subject: [PATCH 3/7] feat(apigee-skills-serving): add
 bin/install-skill-finder.sh + bin/install-skill-publisher.sh

Two new installer scripts under
references/apigee-skills-serving/bin/ that let operators install
the skill-finder and skill-publisher agentic skills into their
agent runtime (opencode | gemini | antigravity) in one line:

  curl -fsSL https://raw.githubusercontent.com/apigee/devrel/main/\
      references/apigee-skills-serving/bin/install-skill-finder.sh \
      | bash -s -- --runtime gemini

Trust model
-----------

The installer scripts (and the skill-finder source they fetch)
are trusted by provenance: TLS + the apigee GitHub organisation.
There are no signed release bundles and no sha256 pins baked into
the installers. This differs from a signed-release model
(e.g. gsjurseth/skill-finder's v0.1.7 installer) but matches the
"no signing key ever enters this repo" constraint: apigee/devrel
does not have an organisation-level signing key, and adding one
would require key-management governance that is out of scope for
the reference implementation.

Payload skills that skill-finder subsequently installs from a
customer's API hub ARE cryptographically verified. Those
signatures use per-deployment trust roots that live under
~/.<runtime>/skills/skill-finder/keys/ and are populated by
bin/provision.sh (a subsequent commit in this PR) or manually.

How the fetch works
-------------------

Rather than depending on a `git` binary on the operator's
machine, the installers fetch the entire devrel repository as a
gzipped tarball from codeload.github.com and extract only the
skill directory they need. Three URL forms are attempted in
order (tag, branch, raw SHA) so --ref works for all three.
This is more portable than `git clone --depth 1` and works on
any machine with `curl` and `tar` in the toolchain.

Design choices
--------------

  - Per-user venv at ~/.local/share/skill-finder/venv shared by
    both skills. The venv is the escape hatch for PEP 668 distros
    (Debian 12+, Ubuntu 23.04+, recent macOS Homebrew) where
    system `pip install` is refused. It also isolates skill deps
    from other Python projects on the operator's machine.

  - Per-install `bin/run-with-venv.sh` wrapper. SKILL.md
    invocations of the shipped scripts are rewritten at install
    time from `python3 ${SKILL_DIR}/scripts/X.py` to
    `${SKILL_DIR}/bin/run-with-venv.sh ${SKILL_DIR}/scripts/X.py`
    so the agent's shell tool always uses the venv even when the
    system python doesn't have cryptography / google-auth /
    requests / pyyaml.

  - scripts/common/ is copied into each skill's install at install
    time. skill-finder's scripts do `from common.X import Y` which
    only works if common/ is a sibling of the script. The source
    tree ships find_install.py and list_skills.py alone; the
    installer splices in the common package from the enclosing
    devrel checkout.

  - Runtime auto-detection: OpenCode -> Antigravity -> Gemini
    CLI -> OpenCode fallback. Operator can always override with
    --runtime.

  - Refuses to overwrite an existing install without --force.
    Empty keys/ is a hard-fail with a clear diagnostic pointing
    at bin/provision.sh.

Verified end-to-end
-------------------

Ran the following against gsjurseth/devrel at feat/add-skill-finder
(the branch this PR will become):

  bash bin/install-skill-finder.sh \
    --runtime gemini \
    --repo gsjurseth/devrel \
    --ref feat/add-skill-finder \
    --force

Result: fetched 10.2 MB tarball, extracted the two skill
directories, created a fresh venv, installed 4 deps, wrote
`bin/run-with-venv.sh` wrappers, rewrote both SKILL.md files
to use the wrappers, atomic-mv'd both installs into place,
printed the trust-root-required diagnostic.

Then dropped a fingerprint.pem into keys/ and re-ran
find_install.py against a live API hub: 16/16 contract lines
printed, exit 0, apigee-policy-top10 installed successfully.

shellcheck on both scripts: PASS.
---
 .../bin/install-skill-finder.sh               | 546 ++++++++++++++++++
 .../bin/install-skill-publisher.sh            | 346 +++++++++++
 2 files changed, 892 insertions(+)
 create mode 100755 references/apigee-skills-serving/bin/install-skill-finder.sh
 create mode 100755 references/apigee-skills-serving/bin/install-skill-publisher.sh

diff --git a/references/apigee-skills-serving/bin/install-skill-finder.sh b/references/apigee-skills-serving/bin/install-skill-finder.sh
new file mode 100755
index 00000000..61a53558
--- /dev/null
+++ b/references/apigee-skills-serving/bin/install-skill-finder.sh
@@ -0,0 +1,546 @@
+#!/usr/bin/env bash
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# install-skill-finder.sh
+#
+# One-line installer for the skill-finder agentic skill from
+# apigee/devrel. Fetches source from a specific git ref of the
+# apigee/devrel repository, creates a per-user Python venv, and
+# installs into the operator's chosen agent runtime.
+#
+# The skill-finder skill itself is trusted by provenance -- the
+# operator fetched THIS script from a known-good URL over TLS
+# (raw.githubusercontent.com/apigee/devrel) and the tarball comes
+# from the same origin. There are no signed release bundles and
+# no sha256 pins: apigee/devrel does not publish signed release
+# assets, and adding a signing infrastructure to the repo would
+# require organisation-level key management that is out of scope
+# for the reference implementation.
+#
+# Payload skills that skill-finder subsequently installs from the
+# customer's own API hub ARE verified by ed25519 signature. Those
+# signatures are checked against per-deployment trust roots that
+# the operator populates via bin/provision.sh (bootstrap) or by
+# dropping a PEM into ~/.<runtime>/skills/skill-finder/keys/.
+#
+# Usage:
+#   install-skill-finder.sh [--runtime opencode|gemini|antigravity]
+#                           [--install-root <dir>]
+#                           [--ref <git-ref>]
+#                           [--repo <owner/repo>]
+#                           [--venv-dir <dir>]
+#                           [--use-uv]
+#                           [--skip-publisher]
+#                           [--force]
+#                           [--dry-run]
+#                           [-h | --help]
+#
+# Required tools on PATH: bash >= 3.2, curl, tar, python3 >= 3.10,
+# the `venv` stdlib module (or `uv` on PATH if --use-uv is passed).
+#
+# Default behaviour: installs BOTH skill-finder (catalog discovery
+# client) AND skill-publisher (author-side publishing tool). Pass
+# --skip-publisher to install only the discovery client.
+#
+# Python dependencies are installed into a per-user venv at
+# ~/.local/share/skill-finder/venv (override with --venv-dir). This
+# is necessary on distros that enforce PEP 668 (Debian 12+,
+# Ubuntu 23.04+, recent macOS Homebrew) where system `pip install`
+# is refused. The venv is also safer everywhere else: it isolates
+# skill-finder's deps from other Python projects. The same venv is
+# shared between skill-finder and skill-publisher.
+#
+# Exit codes:
+#   0  install OK
+#   1  user error (bad flag, missing dep, invalid runtime)
+#   2  network failure
+#   3  install-target write failure
+
+set -u
+# We intentionally do NOT set -e. Every step checks its own exit
+# code so the error message identifies which step failed.
+
+# ===============================================================
+# Defaults
+# ===============================================================
+
+DEFAULT_REF="main"
+DEFAULT_REPO="apigee/devrel"
+# Path INSIDE the devrel monorepo where the skills live.
+SKILLS_SUBPATH="references/apigee-skills-serving/skills"
+
+# Python runtime deps needed by find_install.py, list_skills.py,
+# and publish.sh (via sign_skill.py which is not shipped in the
+# skill bundle -- but skill-publisher shells out to the four
+# publisher Python modules that share the same deps).
+PY_DEPS=(cryptography google-auth requests pyyaml)
+
+VENV_DIR="$HOME/.local/share/skill-finder/venv"
+
+# ===============================================================
+# Arg parsing
+# ===============================================================
+
+RUNTIME=""
+INSTALL_ROOT=""
+REF="$DEFAULT_REF"
+REPO="$DEFAULT_REPO"
+USE_UV=0
+SKIP_PUBLISHER=0
+FORCE=0
+DRY_RUN=0
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  install-skill-finder.sh [--runtime opencode|gemini|antigravity]
+                          [--install-root <dir>]
+                          [--ref <git-ref>]
+                          [--repo <owner/repo>]
+                          [--venv-dir <dir>]
+                          [--use-uv]
+                          [--skip-publisher]
+                          [--force]
+                          [--dry-run]
+
+Flags:
+  --runtime <name>    Runtime to install into. Auto-detected if omitted.
+                      One of: opencode | gemini | antigravity.
+  --install-root <p>  Override the runtime's default skills root.
+  --ref <ref>         Git ref (branch, tag, or commit SHA) to fetch
+                      source from. Default: main.
+  --repo <o/r>        GitHub repo to fetch from. Default: apigee/devrel.
+  --venv-dir <p>      Per-user venv location. Default:
+                      ~/.local/share/skill-finder/venv
+  --use-uv            Use `uv` instead of `python3 -m venv`.
+  --skip-publisher    Install skill-finder only (skip skill-publisher).
+  --force             Overwrite an existing install without prompting.
+  --dry-run           Print what would happen, do nothing.
+  -h, --help          Show this help.
+USAGE
+}
+
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --runtime)         RUNTIME="${2:-}"; shift 2 ;;
+    --install-root)    INSTALL_ROOT="${2:-}"; shift 2 ;;
+    --ref)             REF="${2:-}"; shift 2 ;;
+    --repo)            REPO="${2:-}"; shift 2 ;;
+    --venv-dir)        VENV_DIR="${2:-}"; shift 2 ;;
+    --use-uv)          USE_UV=1; shift ;;
+    --skip-publisher)  SKIP_PUBLISHER=1; shift ;;
+    --force)           FORCE=1; shift ;;
+    --dry-run)         DRY_RUN=1; shift ;;
+    -h|--help)         usage; exit 0 ;;
+    *)
+      echo "[install] FATAL: unknown flag: $1" >&2
+      echo "[install] run --help for usage" >&2
+      exit 1
+      ;;
+  esac
+done
+
+log() { echo "[install] $*"; }
+err() { echo "[install] $*" >&2; }
+
+# ===============================================================
+# Step 1: detect runtime and resolve install root
+# ===============================================================
+
+if [ -z "$RUNTIME" ]; then
+  # Heuristic detection. Operator can always override with --runtime.
+  # Detection order:
+  #   1. OpenCode if its skills dir exists.
+  #   2. Antigravity if its browser-profile marker exists.
+  #   3. Gemini CLI: canonical user-skills root is ~/.gemini/skills.
+  #   4. Fallback to opencode.
+  if [ -d "$HOME/.config/opencode/skills" ]; then
+    RUNTIME="opencode"
+  elif [ -d "$HOME/.gemini/antigravity-browser-profile" ]; then
+    RUNTIME="antigravity"
+  elif [ -d "$HOME/.gemini" ]; then
+    RUNTIME="gemini"
+  else
+    RUNTIME="opencode"
+  fi
+  log "auto-detected runtime: $RUNTIME (override with --runtime)"
+fi
+
+case "$RUNTIME" in
+  opencode)
+    DEFAULT_INSTALL_ROOT="$HOME/.config/opencode/skills"
+    ;;
+  gemini)
+    # Per Gemini CLI docs (docs/cli/skills.md), user skills live at
+    # ~/.gemini/skills/ -- NOT ~/.gemini/config/skills/ (that's the
+    # Antigravity path).
+    DEFAULT_INSTALL_ROOT="$HOME/.gemini/skills"
+    ;;
+  antigravity)
+    # Antigravity's global install root is ~/.gemini/antigravity/skills/.
+    DEFAULT_INSTALL_ROOT="$HOME/.gemini/antigravity/skills"
+    ;;
+  *)
+    err "FATAL: --runtime must be opencode | gemini | antigravity (got: $RUNTIME)"
+    exit 1
+    ;;
+esac
+
+if [ -z "$INSTALL_ROOT" ]; then
+  INSTALL_ROOT="$DEFAULT_INSTALL_ROOT"
+fi
+
+log "runtime:      $RUNTIME"
+log "install root: $INSTALL_ROOT"
+log "source repo:  $REPO"
+log "source ref:   $REF"
+log "venv dir:     $VENV_DIR"
+if [ "$USE_UV" -eq 1 ]; then
+  log "venv tool:    uv (forced via --use-uv)"
+else
+  log "venv tool:    python3 -m venv (stdlib)"
+fi
+if [ "$SKIP_PUBLISHER" -eq 1 ]; then
+  log "skills:       skill-finder only (--skip-publisher set)"
+else
+  log "skills:       skill-finder + skill-publisher (default)"
+fi
+
+# ===============================================================
+# Step 2: tool preflight
+# ===============================================================
+
+need_tool() {
+  local tool="$1"
+  if ! command -v "$tool" >/dev/null 2>&1; then
+    err "FATAL: required tool not on PATH: $tool"
+    exit 1
+  fi
+}
+
+need_tool curl
+need_tool tar
+need_tool python3
+if [ "$USE_UV" -eq 1 ]; then
+  need_tool uv
+fi
+
+# python3 must be >= 3.10 (matches the venv skill runtimes).
+PY_VER=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
+case "$PY_VER" in
+  3.10|3.11|3.12|3.13|3.14|3.15|3.16|3.17|3.18|3.19)
+    log "python:       ${PY_VER} (OK)"
+    ;;
+  *)
+    err "FATAL: python3 must be >= 3.10 (found: ${PY_VER})"
+    exit 1
+    ;;
+esac
+
+# ===============================================================
+# Step 3: create the per-user venv and install runtime deps
+# ===============================================================
+
+log "step 3: setting up Python runtime"
+log "  venv dir: $VENV_DIR"
+
+if [ "$DRY_RUN" -eq 1 ]; then
+  log "DRY RUN: skipping venv create + dep install"
+else
+  if ! mkdir -p "$(dirname "$VENV_DIR")"; then
+    err "FATAL: cannot create venv parent dir: $(dirname "$VENV_DIR")"
+    exit 3
+  fi
+
+  # Reuse an existing venv only if it's healthy (has bin/python
+  # AND a working pip). A half-built venv from a prior failed run
+  # would cascade into 'No module named pip' errors. In that case
+  # rebuild from scratch.
+  VENV_OK=0
+  if [ -x "$VENV_DIR/bin/python" ]; then
+    if "$VENV_DIR/bin/python" -m pip --version >/dev/null 2>&1; then
+      VENV_OK=1
+    fi
+  fi
+  if [ "$VENV_OK" -eq 0 ]; then
+    if [ -d "$VENV_DIR" ]; then
+      log "  existing venv at $VENV_DIR is broken; removing and recreating"
+      rm -rf "$VENV_DIR"
+    else
+      log "  creating venv at $VENV_DIR"
+    fi
+    if [ "$USE_UV" -eq 1 ]; then
+      if ! uv venv --quiet "$VENV_DIR"; then
+        err "FATAL: uv venv failed"
+        exit 1
+      fi
+    else
+      if ! python3 -m venv "$VENV_DIR"; then
+        err "FATAL: python3 -m venv failed"
+        exit 1
+      fi
+    fi
+  else
+    log "  reusing existing venv (pip is healthy)"
+  fi
+
+  log "  installing deps: ${PY_DEPS[*]}"
+  if [ "$USE_UV" -eq 1 ]; then
+    if ! uv pip install --quiet --python "$VENV_DIR/bin/python" \
+         --upgrade "${PY_DEPS[@]}"; then
+      err "FATAL: uv pip install failed for runtime deps"
+      exit 1
+    fi
+  else
+    if ! "$VENV_DIR/bin/python" -m pip install --quiet --upgrade \
+         "${PY_DEPS[@]}"; then
+      err "FATAL: pip install failed for runtime deps in venv"
+      exit 1
+    fi
+  fi
+fi
+
+# ===============================================================
+# Step 4: fetch source from apigee/devrel
+# ===============================================================
+#
+# Strategy: fetch the entire repo as a tarball at $REF and extract
+# only the two skill directories we care about. This avoids
+# requiring `git` on the operator's machine (curl + tar are more
+# portable) and avoids fetching the whole history.
+#
+# GitHub tarball URL format:
+#   https://codeload.github.com/{repo}/tar.gz/refs/heads/{branch}
+#   https://codeload.github.com/{repo}/tar.gz/refs/tags/{tag}
+#   https://codeload.github.com/{repo}/tar.gz/{sha}
+#
+# We probe the tag URL first, then the branch URL, then the raw
+# SHA URL. Whichever returns 200 wins.
+
+TARBALL_TMP=$(mktemp -t skill-finder-source-XXXXXX.tar.gz)
+# Clean up the tarball on exit no matter how we got here.
+trap 'rm -f "$TARBALL_TMP"' EXIT
+
+CANDIDATES=(
+  "https://codeload.github.com/${REPO}/tar.gz/refs/tags/${REF}"
+  "https://codeload.github.com/${REPO}/tar.gz/refs/heads/${REF}"
+  "https://codeload.github.com/${REPO}/tar.gz/${REF}"
+)
+
+log "step 4: fetching source from ${REPO} at ref '${REF}'"
+if [ "$DRY_RUN" -eq 1 ]; then
+  log "DRY RUN: skipping source fetch"
+else
+  FETCHED=0
+  for URL in "${CANDIDATES[@]}"; do
+    log "  trying: $URL"
+    if curl -fsSL -o "$TARBALL_TMP" "$URL"; then
+      log "  OK: fetched $(wc -c <"$TARBALL_TMP") bytes"
+      FETCHED=1
+      break
+    fi
+  done
+  if [ "$FETCHED" -eq 0 ]; then
+    err "FATAL: could not fetch source from any of:"
+    for URL in "${CANDIDATES[@]}"; do err "  $URL"; done
+    err "Check that --repo and --ref name a real ref, and that"
+    err "you have network access to codeload.github.com."
+    exit 2
+  fi
+fi
+
+# Extract into a staging dir. GitHub tarballs have a top-level
+# directory named {repo-name}-{sha-or-ref}; we don't care about
+# that name, just the subpath we want.
+EXTRACT_TMP=$(mktemp -d -t skill-finder-extract-XXXXXX)
+trap 'rm -f "$TARBALL_TMP"; rm -rf "$EXTRACT_TMP"' EXIT
+
+if [ "$DRY_RUN" -eq 1 ]; then
+  log "DRY RUN: skipping tarball extract"
+else
+  if ! tar -xzf "$TARBALL_TMP" -C "$EXTRACT_TMP"; then
+    err "FATAL: could not extract tarball"
+    exit 2
+  fi
+  # Find the extracted top-level dir (there's exactly one).
+  TOPDIR=$(find "$EXTRACT_TMP" -mindepth 1 -maxdepth 1 -type d)
+  if [ -z "$TOPDIR" ] || [ ! -d "$TOPDIR/${SKILLS_SUBPATH}/skill-finder" ]; then
+    err "FATAL: tarball does not contain ${SKILLS_SUBPATH}/skill-finder."
+    err "Wrong --ref or wrong --repo? Extracted structure:"
+    ls -la "$EXTRACT_TMP" >&2
+    exit 2
+  fi
+  SRC_SKILL_FINDER="$TOPDIR/${SKILLS_SUBPATH}/skill-finder"
+  # Publisher may not exist in older refs; check separately.
+  if [ -d "$TOPDIR/${SKILLS_SUBPATH}/skill-publisher" ]; then
+    SRC_SKILL_PUBLISHER="$TOPDIR/${SKILLS_SUBPATH}/skill-publisher"
+  else
+    SRC_SKILL_PUBLISHER=""
+    if [ "$SKIP_PUBLISHER" -eq 0 ]; then
+      log "  warning: ${SKILLS_SUBPATH}/skill-publisher not found at ref '$REF'"
+      log "  installing skill-finder only"
+      SKIP_PUBLISHER=1
+    fi
+  fi
+  # skill-publisher also needs the scripts/common/ package (same
+  # code the devrel publisher pipeline uses). Locate it once so
+  # install_one_skill can splice it in.
+  if [ -d "$TOPDIR/references/apigee-skills-serving/scripts/common" ]; then
+    SRC_COMMON_PACKAGE="$TOPDIR/references/apigee-skills-serving/scripts/common"
+  else
+    SRC_COMMON_PACKAGE=""
+  fi
+fi
+
+# ===============================================================
+# Step 5: install skills into the runtime's skills root
+# ===============================================================
+
+install_one_skill() {
+  local skill_name="$1"
+  local skill_src="$2"
+  local target_dir="$INSTALL_ROOT/$skill_name"
+
+  log "step 5.$skill_name: installing to $target_dir"
+
+  # Refuse to overwrite an existing install unless --force.
+  if [ -d "$target_dir" ]; then
+    if [ "$FORCE" -eq 0 ]; then
+      err "FATAL: $target_dir already exists (pass --force to overwrite)"
+      exit 3
+    fi
+    log "  removing existing install at $target_dir (--force)"
+    if [ "$DRY_RUN" -eq 0 ]; then
+      rm -rf "$target_dir"
+    fi
+  fi
+
+  if [ "$DRY_RUN" -eq 1 ]; then
+    log "DRY RUN: skipping copy + venv-wrapper install"
+    return
+  fi
+
+  # Atomic install: build in a staging dir, mv into place.
+  local staging
+  staging=$(mktemp -d -p "$INSTALL_ROOT" ".staging-XXXXXX")
+  if ! cp -r "$skill_src/." "$staging/"; then
+    err "FATAL: cp failed from $skill_src to $staging"
+    rm -rf "$staging"
+    exit 3
+  fi
+
+  # Splice in scripts/common/ so `from common.X import Y` resolves
+  # at runtime. skill-finder and skill-publisher both share the
+  # same common package.
+  if [ -n "$SRC_COMMON_PACKAGE" ] && [ -d "$staging/scripts" ]; then
+    if ! cp -r "$SRC_COMMON_PACKAGE" "$staging/scripts/common"; then
+      err "FATAL: cp of scripts/common failed"
+      rm -rf "$staging"
+      exit 3
+    fi
+  fi
+
+  # Install the venv wrapper. The wrapper is a tiny shim that
+  # invokes $VENV_DIR/bin/python on the script passed as argv[1].
+  # SKILL.md invocations are rewritten to go through it so agents
+  # that would otherwise use system python (which may lack our
+  # deps) instead use our venv.
+  mkdir -p "$staging/bin"
+  cat >"$staging/bin/run-with-venv.sh" <<WRAPPER
+#!/usr/bin/env bash
+# Auto-generated by install-skill-finder.sh at install time.
+# Invokes the shipped scripts through the per-user venv so the
+# required Python deps (cryptography, google-auth, requests,
+# pyyaml) are always available regardless of what \`python3\` the
+# agent runtime would otherwise pick.
+exec "$VENV_DIR/bin/python" "\$@"
+WRAPPER
+  chmod +x "$staging/bin/run-with-venv.sh"
+
+  # Rewrite SKILL.md so agent invocations of ${SKILL_DIR}/scripts/*.py
+  # go through bin/run-with-venv.sh. This is a targeted sed on the
+  # exact substring the author's SKILL.md uses; if a future SKILL.md
+  # changes the invocation shape, this substitution silently no-ops
+  # (worst case: the agent uses system python, same as pre-install).
+  if [ -f "$staging/SKILL.md" ]; then
+    # Rewrite python3 ${SKILL_DIR}/scripts/X.py -> ${SKILL_DIR}/bin/run-with-venv.sh ${SKILL_DIR}/scripts/X.py
+    if command -v python3 >/dev/null 2>&1; then
+      python3 - "$staging/SKILL.md" <<'PYREWRITE'
+import sys
+path = sys.argv[1]
+old = open(path).read()
+new = old.replace(
+    'python3 ${SKILL_DIR}/scripts/',
+    '${SKILL_DIR}/bin/run-with-venv.sh ${SKILL_DIR}/scripts/',
+)
+new = new.replace(
+    'bash ${SKILL_DIR}/scripts/',
+    '${SKILL_DIR}/bin/run-with-venv.sh ${SKILL_DIR}/scripts/',
+)
+if new != old:
+    open(path, 'w').write(new)
+    print(f'    rewrote {path} to invoke scripts via the venv wrapper')
+else:
+    print(f'    SKILL.md at {path} needs no rewrite (unrecognised invocation shape)')
+PYREWRITE
+    fi
+  fi
+
+  # Atomic mv into place.
+  mkdir -p "$INSTALL_ROOT"
+  if ! mv "$staging" "$target_dir"; then
+    err "FATAL: mv into place failed: $staging -> $target_dir"
+    rm -rf "$staging"
+    exit 3
+  fi
+  log "  installed: $target_dir"
+}
+
+if [ "$DRY_RUN" -eq 1 ]; then
+  log "DRY RUN: would install skill-finder to $INSTALL_ROOT/skill-finder"
+  if [ "$SKIP_PUBLISHER" -eq 0 ]; then
+    log "DRY RUN: would install skill-publisher to $INSTALL_ROOT/skill-publisher"
+  fi
+else
+  install_one_skill "skill-finder" "$SRC_SKILL_FINDER"
+  if [ "$SKIP_PUBLISHER" -eq 0 ] && [ -n "${SRC_SKILL_PUBLISHER:-}" ]; then
+    install_one_skill "skill-publisher" "$SRC_SKILL_PUBLISHER"
+  fi
+fi
+
+# ===============================================================
+# Step 6: next-steps message
+# ===============================================================
+
+log ""
+log "skill-finder installed successfully to $INSTALL_ROOT/skill-finder"
+if [ "$SKIP_PUBLISHER" -eq 0 ] && [ -n "${SRC_SKILL_PUBLISHER:-}" ]; then
+  log "skill-publisher installed to      $INSTALL_ROOT/skill-publisher"
+fi
+log ""
+log "IMPORTANT: skill-finder cannot verify any downloaded skill"
+log "manifests until at least one trust root PEM is present in"
+log "$INSTALL_ROOT/skill-finder/keys/. Two ways to populate it:"
+log ""
+log "  1. Run bin/provision.sh from the apigee/devrel checkout"
+log "     to generate a fresh ed25519 keypair, publish the initial"
+log "     example skills to your API hub signed with the new key,"
+log "     and drop the corresponding public key into keys/."
+log ""
+log "  2. Drop your org's existing public key PEM into"
+log "     $INSTALL_ROOT/skill-finder/keys/<fingerprint>.pem"
+log "     manually. Any file matching keys/*.pem is loaded."
+log ""
+log "See docs/trust-root.md for the full trust-model discussion."
diff --git a/references/apigee-skills-serving/bin/install-skill-publisher.sh b/references/apigee-skills-serving/bin/install-skill-publisher.sh
new file mode 100755
index 00000000..85d1758c
--- /dev/null
+++ b/references/apigee-skills-serving/bin/install-skill-publisher.sh
@@ -0,0 +1,346 @@
+#!/usr/bin/env bash
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# install-skill-publisher.sh
+#
+# One-line installer for the skill-publisher agentic skill from
+# apigee/devrel. Same fetch model as install-skill-finder.sh
+# (source from a git ref, no signed release assets, trust by
+# provenance) but installs only the publisher skill.
+#
+# skill-publisher is the author-side companion to skill-finder:
+# it orchestrates pack -> sign -> upload -> register for a source
+# skill directory. Unlike skill-finder, it does NOT verify
+# signatures at runtime -- the author IS the signer -- so no
+# trust-root population step is needed.
+#
+# If you want both skills, prefer install-skill-finder.sh which
+# installs both by default (this script's behaviour is a subset).
+#
+# Usage:
+#   install-skill-publisher.sh [--runtime opencode|gemini|antigravity]
+#                              [--install-root <dir>]
+#                              [--ref <git-ref>]
+#                              [--repo <owner/repo>]
+#                              [--venv-dir <dir>]
+#                              [--use-uv]
+#                              [--force]
+#                              [--dry-run]
+#                              [-h | --help]
+#
+# See install-skill-finder.sh for the full trust-model rationale
+# and PEP-668 explanation. This script is intentionally a thin
+# wrapper around install-skill-finder.sh --skip-publisher --skip-finder
+# would be prettier, but that would require adding a --skip-finder
+# flag to the finder installer -- keeping them separate lets
+# operators install only the piece they need without cross-flags.
+#
+# Exit codes:
+#   0  install OK
+#   1  user error
+#   2  network failure
+#   3  install-target write failure
+
+set -u
+
+DEFAULT_REF="main"
+DEFAULT_REPO="apigee/devrel"
+SKILLS_SUBPATH="references/apigee-skills-serving/skills"
+
+# skill-publisher shells out to the publisher-side Python
+# modules (pack_skill, sign_skill, upload_skill, register_skill)
+# which need cryptography + google-auth + requests + pyyaml.
+PY_DEPS=(cryptography google-auth requests pyyaml)
+
+VENV_DIR="$HOME/.local/share/skill-finder/venv"
+
+RUNTIME=""
+INSTALL_ROOT=""
+REF="$DEFAULT_REF"
+REPO="$DEFAULT_REPO"
+USE_UV=0
+FORCE=0
+DRY_RUN=0
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  install-skill-publisher.sh [--runtime opencode|gemini|antigravity]
+                             [--install-root <dir>]
+                             [--ref <git-ref>]
+                             [--repo <owner/repo>]
+                             [--venv-dir <dir>]
+                             [--use-uv]
+                             [--force]
+                             [--dry-run]
+
+Flags:
+  --runtime <name>    Runtime to install into. Auto-detected if omitted.
+  --install-root <p>  Override the runtime's default skills root.
+  --ref <ref>         Git ref (branch, tag, or commit SHA). Default: main.
+  --repo <o/r>        GitHub repo. Default: apigee/devrel.
+  --venv-dir <p>      Per-user venv. Default: ~/.local/share/skill-finder/venv
+  --use-uv            Use `uv` instead of `python3 -m venv`.
+  --force             Overwrite an existing install.
+  --dry-run           Print what would happen, do nothing.
+  -h, --help          Show this help.
+USAGE
+}
+
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --runtime)         RUNTIME="${2:-}"; shift 2 ;;
+    --install-root)    INSTALL_ROOT="${2:-}"; shift 2 ;;
+    --ref)             REF="${2:-}"; shift 2 ;;
+    --repo)            REPO="${2:-}"; shift 2 ;;
+    --venv-dir)        VENV_DIR="${2:-}"; shift 2 ;;
+    --use-uv)          USE_UV=1; shift ;;
+    --force)           FORCE=1; shift ;;
+    --dry-run)         DRY_RUN=1; shift ;;
+    -h|--help)         usage; exit 0 ;;
+    *)
+      echo "[install] FATAL: unknown flag: $1" >&2
+      echo "[install] run --help for usage" >&2
+      exit 1
+      ;;
+  esac
+done
+
+log() { echo "[install] $*"; }
+err() { echo "[install] $*" >&2; }
+
+# ===============================================================
+# Step 1: detect runtime, resolve install root
+# ===============================================================
+
+if [ -z "$RUNTIME" ]; then
+  if [ -d "$HOME/.config/opencode/skills" ]; then
+    RUNTIME="opencode"
+  elif [ -d "$HOME/.gemini/antigravity-browser-profile" ]; then
+    RUNTIME="antigravity"
+  elif [ -d "$HOME/.gemini" ]; then
+    RUNTIME="gemini"
+  else
+    RUNTIME="opencode"
+  fi
+  log "auto-detected runtime: $RUNTIME (override with --runtime)"
+fi
+
+case "$RUNTIME" in
+  opencode)     DEFAULT_INSTALL_ROOT="$HOME/.config/opencode/skills" ;;
+  gemini)       DEFAULT_INSTALL_ROOT="$HOME/.gemini/skills" ;;
+  antigravity)  DEFAULT_INSTALL_ROOT="$HOME/.gemini/antigravity/skills" ;;
+  *)
+    err "FATAL: --runtime must be opencode | gemini | antigravity (got: $RUNTIME)"
+    exit 1
+    ;;
+esac
+
+if [ -z "$INSTALL_ROOT" ]; then
+  INSTALL_ROOT="$DEFAULT_INSTALL_ROOT"
+fi
+
+log "runtime:      $RUNTIME"
+log "install root: $INSTALL_ROOT"
+log "source repo:  $REPO"
+log "source ref:   $REF"
+log "venv dir:     $VENV_DIR"
+
+# ===============================================================
+# Step 2: tool preflight
+# ===============================================================
+
+for tool in curl tar python3; do
+  if ! command -v "$tool" >/dev/null 2>&1; then
+    err "FATAL: required tool not on PATH: $tool"
+    exit 1
+  fi
+done
+if [ "$USE_UV" -eq 1 ] && ! command -v uv >/dev/null 2>&1; then
+  err "FATAL: --use-uv passed but uv is not on PATH"
+  exit 1
+fi
+
+PY_VER=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
+case "$PY_VER" in
+  3.10|3.11|3.12|3.13|3.14|3.15|3.16|3.17|3.18|3.19)
+    log "python:       ${PY_VER} (OK)"
+    ;;
+  *)
+    err "FATAL: python3 must be >= 3.10 (found: ${PY_VER})"
+    exit 1
+    ;;
+esac
+
+# ===============================================================
+# Step 3: create venv + install deps (idempotent; shared with
+# skill-finder's venv)
+# ===============================================================
+
+log "step 3: setting up Python runtime"
+
+if [ "$DRY_RUN" -eq 1 ]; then
+  log "DRY RUN: skipping venv create + dep install"
+else
+  mkdir -p "$(dirname "$VENV_DIR")" || {
+    err "FATAL: cannot create venv parent dir"; exit 3;
+  }
+  VENV_OK=0
+  if [ -x "$VENV_DIR/bin/python" ] \
+      && "$VENV_DIR/bin/python" -m pip --version >/dev/null 2>&1; then
+    VENV_OK=1
+  fi
+  if [ "$VENV_OK" -eq 0 ]; then
+    [ -d "$VENV_DIR" ] && rm -rf "$VENV_DIR"
+    log "  creating venv at $VENV_DIR"
+    if [ "$USE_UV" -eq 1 ]; then
+      uv venv --quiet "$VENV_DIR" || { err "FATAL: uv venv failed"; exit 1; }
+    else
+      python3 -m venv "$VENV_DIR" \
+        || { err "FATAL: python3 -m venv failed"; exit 1; }
+    fi
+  else
+    log "  reusing existing venv (pip is healthy)"
+  fi
+  log "  installing deps: ${PY_DEPS[*]}"
+  if [ "$USE_UV" -eq 1 ]; then
+    uv pip install --quiet --python "$VENV_DIR/bin/python" \
+       --upgrade "${PY_DEPS[@]}" \
+       || { err "FATAL: uv pip install failed"; exit 1; }
+  else
+    "$VENV_DIR/bin/python" -m pip install --quiet --upgrade \
+       "${PY_DEPS[@]}" \
+       || { err "FATAL: pip install failed"; exit 1; }
+  fi
+fi
+
+# ===============================================================
+# Step 4: fetch source from apigee/devrel
+# ===============================================================
+
+TARBALL_TMP=$(mktemp -t skill-publisher-source-XXXXXX.tar.gz)
+trap 'rm -f "$TARBALL_TMP"' EXIT
+
+CANDIDATES=(
+  "https://codeload.github.com/${REPO}/tar.gz/refs/tags/${REF}"
+  "https://codeload.github.com/${REPO}/tar.gz/refs/heads/${REF}"
+  "https://codeload.github.com/${REPO}/tar.gz/${REF}"
+)
+
+log "step 4: fetching source from ${REPO} at ref '${REF}'"
+if [ "$DRY_RUN" -eq 1 ]; then
+  log "DRY RUN: skipping source fetch"
+else
+  FETCHED=0
+  for URL in "${CANDIDATES[@]}"; do
+    log "  trying: $URL"
+    if curl -fsSL -o "$TARBALL_TMP" "$URL"; then
+      log "  OK: fetched $(wc -c <"$TARBALL_TMP") bytes"
+      FETCHED=1
+      break
+    fi
+  done
+  if [ "$FETCHED" -eq 0 ]; then
+    err "FATAL: could not fetch source from any of:"
+    for URL in "${CANDIDATES[@]}"; do err "  $URL"; done
+    exit 2
+  fi
+fi
+
+EXTRACT_TMP=$(mktemp -d -t skill-publisher-extract-XXXXXX)
+trap 'rm -f "$TARBALL_TMP"; rm -rf "$EXTRACT_TMP"' EXIT
+
+if [ "$DRY_RUN" -eq 1 ]; then
+  log "DRY RUN: skipping tarball extract"
+else
+  tar -xzf "$TARBALL_TMP" -C "$EXTRACT_TMP" \
+    || { err "FATAL: tar extract failed"; exit 2; }
+  TOPDIR=$(find "$EXTRACT_TMP" -mindepth 1 -maxdepth 1 -type d)
+  SRC_SKILL_PUBLISHER="$TOPDIR/${SKILLS_SUBPATH}/skill-publisher"
+  if [ ! -d "$SRC_SKILL_PUBLISHER" ]; then
+    err "FATAL: tarball does not contain ${SKILLS_SUBPATH}/skill-publisher"
+    err "Wrong --ref or wrong --repo?"
+    exit 2
+  fi
+  if [ -d "$TOPDIR/references/apigee-skills-serving/scripts/common" ]; then
+    SRC_COMMON_PACKAGE="$TOPDIR/references/apigee-skills-serving/scripts/common"
+  else
+    SRC_COMMON_PACKAGE=""
+  fi
+fi
+
+# ===============================================================
+# Step 5: install
+# ===============================================================
+
+TARGET_DIR="$INSTALL_ROOT/skill-publisher"
+log "step 5: installing to $TARGET_DIR"
+
+if [ -d "$TARGET_DIR" ] && [ "$FORCE" -eq 0 ]; then
+  err "FATAL: $TARGET_DIR already exists (pass --force to overwrite)"
+  exit 3
+fi
+
+if [ "$DRY_RUN" -eq 1 ]; then
+  log "DRY RUN: would install to $TARGET_DIR"
+else
+  [ -d "$TARGET_DIR" ] && rm -rf "$TARGET_DIR"
+  mkdir -p "$INSTALL_ROOT"
+  STAGING=$(mktemp -d -p "$INSTALL_ROOT" ".staging-XXXXXX")
+  cp -r "$SRC_SKILL_PUBLISHER/." "$STAGING/" \
+    || { err "FATAL: cp failed"; rm -rf "$STAGING"; exit 3; }
+  if [ -n "$SRC_COMMON_PACKAGE" ] && [ -d "$STAGING/scripts" ]; then
+    cp -r "$SRC_COMMON_PACKAGE" "$STAGING/scripts/common" \
+      || { err "FATAL: cp of common/ failed"; rm -rf "$STAGING"; exit 3; }
+  fi
+  mkdir -p "$STAGING/bin"
+  cat >"$STAGING/bin/run-with-venv.sh" <<WRAPPER
+#!/usr/bin/env bash
+# Auto-generated by install-skill-publisher.sh at install time.
+exec "$VENV_DIR/bin/python" "\$@"
+WRAPPER
+  chmod +x "$STAGING/bin/run-with-venv.sh"
+  # Rewrite SKILL.md invocations (same as install-skill-finder.sh).
+  if [ -f "$STAGING/SKILL.md" ]; then
+    python3 - "$STAGING/SKILL.md" <<'PYREWRITE'
+import sys
+path = sys.argv[1]
+old = open(path).read()
+new = old.replace(
+    'python3 ${SKILL_DIR}/scripts/',
+    '${SKILL_DIR}/bin/run-with-venv.sh ${SKILL_DIR}/scripts/',
+)
+new = new.replace(
+    'bash ${SKILL_DIR}/scripts/',
+    '${SKILL_DIR}/bin/run-with-venv.sh ${SKILL_DIR}/scripts/',
+)
+if new != old:
+    open(path, 'w').write(new)
+    print(f'    rewrote {path} to invoke scripts via the venv wrapper')
+PYREWRITE
+  fi
+  mv "$STAGING" "$TARGET_DIR" \
+    || { err "FATAL: mv into place failed"; rm -rf "$STAGING"; exit 3; }
+  log "  installed: $TARGET_DIR"
+fi
+
+log ""
+log "skill-publisher installed to $TARGET_DIR"
+log ""
+log "To publish your own skills, invoke the skill in your agent"
+log "with a source directory. Or run publish.sh directly:"
+log "  $TARGET_DIR/scripts/publish.sh --src <skill-dir> \\"
+log "    --bucket <gcs-bucket> --priv-key <ed25519.raw> \\"
+log "    --project <apihub-project> --location <apihub-location>"

From e0cbc4fae740297a075fc8f79ee68c7630816b0c Mon Sep 17 00:00:00 2001
From: Geir Sjurseth <gsjurseth@google.com>
Date: Tue, 7 Jul 2026 08:58:30 +0000
Subject: [PATCH 4/7] feat(apigee-skills-serving): add bin/provision.sh
 (customer bootstrap)

Adds a single end-to-end bootstrap script that provisions an
empty GCP project as an apigee-skills-serving reference
deployment. Given a project with a pre-existing Apigee
organisation and API hub instance, provision.sh:

  1. Enables the four GCP APIs skill-finder + skill-publisher
     need (apigee, apihub, storage, iam).
  2. Verifies the API hub instance + Apigee org exist; fails
     fast with a clear message if not (creating an API hub
     instance takes 40+ min and has console-only decisions --
     out of scope for automation).
  3. Creates a public-read GCS bucket for signed .skill
     bundles (public-read because skill-finder downloads via
     anonymous HTTPS -- demo choice; production would use
     signed URLs or an authenticated consumer).
  4. Creates the four API hub user-defined attributes
     (agentic_skill, keywords, gs_uri, signing_key_id) and
     patches skill-spec into the system-spec-type enum.
     Latter is a known gap: update_taxonomy.py doesn't add
     skill-spec, but register_skill.py requires it.
  5. Generates a fresh ed25519 signing keypair on THIS
     machine. Private key at ~/.config/apigee-skills-demo/
     signing.raw (mode 0600). Public fingerprint printed for
     the operator to record.
  6. Packs, signs, uploads, and registers the three example
     payload skills (currency-converter, weather-lookup,
     apigee-policy-top10) with the fresh key.
  7. Installs the corresponding public PEM into an
     already-installed skill-finder's keys/ directory so it
     will trust the just-published catalog.

Idempotency
-----------

Every step detects existing state before deciding whether to
act. Re-running against a fully-provisioned project produces
seven OK lines and exits at the READY banner without making
any changes:

  Step 1: "apigee.googleapis.com already enabled" x 4
  Step 2: "API hub instance: ...", "Apigee organisation: ..."
  Step 3: "bucket gs://... exists", "public-read enabled"
  Step 4: "user-defined attributes: all 4 present",
          "system-spec-type includes skill-spec"
  Step 5: "existing key at ... (reusing; pass --force ...)"
  Step 6: "already published: ...", "all example skills already
          published"
  Step 7: "trust root already installed: ..."

Modes
-----

  --check-only  Read-only preflight. Reports what's missing
                without touching anything. Useful for
                pre-demo dry-runs.

  --dry-run     Like --check-only but tells you WHAT it would
                do at each step.

  --force       Overwrite an existing signing key (default:
                reuse -- overwriting would invalidate all
                prior signatures).

  --yes         No interactive prompts (for automation).

Selective skips
---------------

  --skip-apis, --skip-bucket, --skip-taxonomy, --skip-skills,
  --skip-key, --skip-trust-root-install

Any subset can be combined. Useful for surgically re-running
a single step or for splitting the initial provisioning from
a later trust-root-only re-run (after installing skill-finder).

Trust model
-----------

The signing keypair is generated on the operator's machine
and never leaves it. There is no shared org-level signing
key; each deployment owns its own root of trust. The public
PEM is written to skill-finder's keys/ directory so
skill-finder verifies signatures against the deployment-local
key. Multiple keys can coexist under keys/ (see the multi-key
feature in commit babed2e6) which supports multi-publisher
orgs and zero-downtime key rotation.

Verified end-to-end
-------------------

Ran against geirs-spaces-demo (a live GCP project) in three
scenarios:

  1. --check-only: 7/7 steps report OK. No writes, no
     spurious warnings.
  2. Wet re-run against fully-provisioned project: 7/7 steps
     OK, exits at READY with no writes. Fully idempotent.
  3. --force key regeneration: prompts for confirmation
     (unless --yes), overwrites signing.raw, warns operator
     that prior signatures will no longer verify. Publishing
     step then re-signs and re-registers all skills with the
     new key.

shellcheck on the script: PASS.
---
 .../apigee-skills-serving/bin/provision.sh    | 807 ++++++++++++++++++
 1 file changed, 807 insertions(+)
 create mode 100755 references/apigee-skills-serving/bin/provision.sh

diff --git a/references/apigee-skills-serving/bin/provision.sh b/references/apigee-skills-serving/bin/provision.sh
new file mode 100755
index 00000000..de4312c6
--- /dev/null
+++ b/references/apigee-skills-serving/bin/provision.sh
@@ -0,0 +1,807 @@
+#!/usr/bin/env bash
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# provision.sh
+#
+# End-to-end bootstrap for the apigee-skills-serving reference
+# implementation. Given an empty GCP project (with an existing
+# Apigee organisation and an API hub instance), this script:
+#
+#   1. Enables the GCP APIs skill-finder + skill-publisher need.
+#   2. Creates a public-read GCS bucket for signed .skill bundles.
+#   3. Creates the API hub custom attribute taxonomy (agentic_skill,
+#      keywords, gs_uri, signing_key_id) and patches the
+#      skill-spec value into system-spec-type.
+#   4. Generates a fresh ed25519 signing keypair on THIS machine.
+#      The private key never leaves this machine and is never
+#      committed to any repo.
+#   5. Packs + signs + uploads + registers the three example
+#      payload skills (currency-converter, weather-lookup,
+#      apigee-policy-top10) using the fresh key.
+#   6. Installs the public key into an already-installed
+#      skill-finder's keys/ directory so it will trust the
+#      just-published catalog. Silently skips this step if
+#      skill-finder is not installed yet -- the operator can
+#      install skill-finder later and drop the printed key into
+#      keys/ by hand.
+#
+# Every step is idempotent. Re-running against a partially-
+# provisioned project detects what exists and no-ops those
+# steps. Selective skip flags (--skip-apis, --skip-bucket, etc.)
+# let you re-run one narrow step for troubleshooting.
+#
+# NOT provisioned by this script (out of scope):
+#
+#   - The API hub INSTANCE itself. Provisioning takes 40+ minutes,
+#     requires several console-only decisions (CMEK vs GMEK,
+#     Vertex region), and is a customer-lifecycle decision.
+#     provision.sh REQUIRES an existing API hub instance and
+#     fails fast with a clear message if none exists.
+#
+#   - The Apigee organisation. Same reasons.
+#
+#   - IAM bindings for the demo caller. The caller is expected to
+#     have roles/owner (or the fine-grained equivalent). The script
+#     tests every API call and surfaces the specific missing perm
+#     if any call fails with 403.
+#
+# Usage:
+#   bin/provision.sh --project <gcp-project> [flags]
+#
+# See --help for the full flag list.
+#
+# Exit codes:
+#   0  success
+#   1  user error (bad flag, missing tool, missing prereq)
+#   2  network / GCP API failure
+#   3  cryptographic error (key generation failed)
+#   4  publish pipeline failure (pack / sign / upload / register)
+
+set -u
+
+# ===============================================================
+# Locate self, discover devrel checkout
+# ===============================================================
+#
+# provision.sh depends on sibling scripts in ../scripts/*.py and
+# on the three example skills in ../skills/. We resolve relative
+# to $0 so the script works from any cwd as long as it's run
+# from a devrel checkout.
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+if [ ! -d "${REPO_ROOT}/scripts" ] \
+    || [ ! -d "${REPO_ROOT}/skills" ]; then
+  echo "[provision] FATAL: cannot find scripts/ and skills/ next to bin/" >&2
+  echo "[provision]   REPO_ROOT resolved to: ${REPO_ROOT}" >&2
+  echo "[provision]   Run this from a devrel checkout of" >&2
+  echo "[provision]   references/apigee-skills-serving/." >&2
+  exit 1
+fi
+
+# ===============================================================
+# Defaults
+# ===============================================================
+
+DEFAULT_LOCATION="us-west1"
+DEFAULT_KEY_PATH="${HOME}/.config/apigee-skills-demo/signing.raw"
+EXAMPLE_SKILLS=(currency-converter weather-lookup apigee-policy-top10)
+
+# ===============================================================
+# Arg parsing
+# ===============================================================
+
+PROJECT=""
+LOCATION="$DEFAULT_LOCATION"
+BUCKET=""
+APIGEE_ORG=""
+KEY_PATH="$DEFAULT_KEY_PATH"
+RUNTIME=""
+SKIP_APIS=0
+SKIP_BUCKET=0
+SKIP_TAXONOMY=0
+SKIP_SKILLS=0
+SKIP_KEY=0
+SKIP_TRUST_ROOT_INSTALL=0
+CHECK_ONLY=0
+DRY_RUN=0
+FORCE=0
+YES=0
+
+usage() {
+  cat <<USAGE
+Usage:
+  bin/provision.sh --project <gcp-project> [flags]
+
+Required:
+  --project <p>              GCP project id to provision.
+
+Optional:
+  --location <l>             API hub / GCS bucket location.
+                             Default: ${DEFAULT_LOCATION}
+  --bucket <b>               GCS bucket name for signed .skill
+                             bundles. Default:
+                             <project>-skills-<location>
+  --apigee-org <o>           Apigee organisation used by the
+                             apigee-policy-top10 example skill.
+                             Default: same as --project.
+  --key-path <p>             Path to write / read the ed25519
+                             private key. Default: ${DEFAULT_KEY_PATH}
+  --runtime <r>              Agent runtime whose skill-finder
+                             install should receive the trust root
+                             PEM. One of opencode | gemini |
+                             antigravity. Auto-detected if omitted.
+
+Selective skips (idempotent re-runs):
+  --skip-apis                Skip enabling GCP APIs.
+  --skip-bucket              Skip GCS bucket create/config.
+  --skip-taxonomy            Skip API hub attribute taxonomy.
+  --skip-skills              Skip publishing the 3 example skills.
+  --skip-key                 Skip key generation (reuse existing).
+  --skip-trust-root-install  Skip dropping the pubkey into
+                             skill-finder's keys/ (e.g. if
+                             skill-finder is not installed yet).
+
+Modes:
+  --check-only               Read-only preflight. Reports what's
+                             missing without making any changes.
+  --dry-run                  Print what would be done, do nothing.
+  --force                    Overwrite an existing signing key
+                             (default: reuse existing to avoid
+                             invalidating prior signatures).
+  --yes                      No interactive prompts.
+
+  -h, --help                 Show this help.
+USAGE
+}
+
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --project)                    PROJECT="${2:-}"; shift 2 ;;
+    --location)                   LOCATION="${2:-}"; shift 2 ;;
+    --bucket)                     BUCKET="${2:-}"; shift 2 ;;
+    --apigee-org)                 APIGEE_ORG="${2:-}"; shift 2 ;;
+    --key-path)                   KEY_PATH="${2:-}"; shift 2 ;;
+    --runtime)                    RUNTIME="${2:-}"; shift 2 ;;
+    --skip-apis)                  SKIP_APIS=1; shift ;;
+    --skip-bucket)                SKIP_BUCKET=1; shift ;;
+    --skip-taxonomy)              SKIP_TAXONOMY=1; shift ;;
+    --skip-skills)                SKIP_SKILLS=1; shift ;;
+    --skip-key)                   SKIP_KEY=1; shift ;;
+    --skip-trust-root-install)    SKIP_TRUST_ROOT_INSTALL=1; shift ;;
+    --check-only)                 CHECK_ONLY=1; shift ;;
+    --dry-run)                    DRY_RUN=1; shift ;;
+    --force)                      FORCE=1; shift ;;
+    --yes)                        YES=1; shift ;;
+    -h|--help)                    usage; exit 0 ;;
+    *)
+      echo "[provision] FATAL: unknown flag: $1" >&2
+      echo "[provision] run --help for usage" >&2
+      exit 1
+      ;;
+  esac
+done
+
+# Resolve default project from gcloud if not passed.
+if [ -z "$PROJECT" ]; then
+  PROJECT=$(gcloud config get-value project 2>/dev/null || echo "")
+  if [ -z "$PROJECT" ]; then
+    echo "[provision] FATAL: --project not given and no default in gcloud config." >&2
+    echo "[provision] Either pass --project or run:" >&2
+    echo "[provision]   gcloud config set project <your-project>" >&2
+    exit 1
+  fi
+fi
+
+# Derive defaults that depend on --project / --location.
+if [ -z "$BUCKET" ]; then
+  BUCKET="${PROJECT}-skills-${LOCATION}"
+fi
+if [ -z "$APIGEE_ORG" ]; then
+  APIGEE_ORG="$PROJECT"
+fi
+
+# ===============================================================
+# Colour + logging helpers
+# ===============================================================
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m'
+
+log() { echo "[provision] $*"; }
+ok() { echo -e "[provision] ${GREEN}OK${NC}: $*"; }
+warn() { echo -e "[provision] ${YELLOW}WARN${NC}: $*"; }
+err() { echo -e "[provision] ${RED}FAIL${NC}: $*" >&2; }
+
+step() {
+  echo
+  echo "==============================================="
+  echo "  $*"
+  echo "==============================================="
+}
+
+confirm() {
+  # Ask the user y/N. Auto-yes if --yes was passed. Returns 0 for
+  # yes, 1 for no.
+  local prompt="$1"
+  if [ "$YES" -eq 1 ]; then
+    log "  auto-yes (--yes): $prompt"
+    return 0
+  fi
+  read -r -p "[provision] $prompt [y/N]: " ans
+  case "$ans" in
+    y|Y|yes|YES) return 0 ;;
+    *) return 1 ;;
+  esac
+}
+
+# ===============================================================
+# Header
+# ===============================================================
+
+step "apigee-skills-serving provisioner"
+log "project:     $PROJECT"
+log "location:    $LOCATION"
+log "bucket:      $BUCKET"
+log "apigee org:  $APIGEE_ORG"
+log "key path:    $KEY_PATH"
+if [ -n "$RUNTIME" ]; then
+  log "runtime:     $RUNTIME (explicit)"
+fi
+if [ "$CHECK_ONLY" -eq 1 ]; then
+  log "MODE:        --check-only (read-only preflight)"
+elif [ "$DRY_RUN" -eq 1 ]; then
+  log "MODE:        --dry-run (no changes will be made)"
+fi
+
+# ===============================================================
+# Step 0: preflight (tools, ADC, venv)
+# ===============================================================
+
+step "Step 0: preflight"
+
+# Required tools.
+for tool in gcloud python3 curl; do
+  if ! command -v "$tool" >/dev/null 2>&1; then
+    err "required tool missing: $tool"
+    exit 1
+  fi
+done
+ok "required tools present (gcloud, python3, curl)"
+
+# ADC.
+if ! gcloud auth application-default print-access-token >/dev/null 2>&1; then
+  err "ADC token not available. Run:"
+  err "    gcloud auth application-default login"
+  exit 1
+fi
+ok "ADC token available"
+
+# Confirm the caller's identity so we don't accidentally provision
+# under the wrong account.
+ACTIVE_ACCOUNT=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null | head -1)
+ok "active gcloud account: ${ACTIVE_ACCOUNT}"
+
+# Python venv for the publisher scripts. Reuse skill-finder's
+# venv if it exists (installed by install-skill-finder.sh).
+VENV_DIR="${HOME}/.local/share/skill-finder/venv"
+if [ -x "${VENV_DIR}/bin/python" ] \
+    && "${VENV_DIR}/bin/python" -m pip --version >/dev/null 2>&1; then
+  PY="${VENV_DIR}/bin/python"
+  ok "reusing skill-finder venv at ${VENV_DIR}"
+else
+  # Bootstrap a fresh venv if skill-finder isn't installed yet.
+  # We need one for pack_skill.py / sign_skill.py (they import
+  # cryptography and pyyaml).
+  log "  creating temporary venv at ${VENV_DIR}"
+  if [ "$CHECK_ONLY" -eq 0 ] && [ "$DRY_RUN" -eq 0 ]; then
+    mkdir -p "$(dirname "${VENV_DIR}")"
+    python3 -m venv "${VENV_DIR}" \
+      || { err "python3 -m venv failed"; exit 1; }
+    "${VENV_DIR}/bin/python" -m pip install --quiet --upgrade \
+      cryptography google-auth requests pyyaml \
+      || { err "pip install of venv deps failed"; exit 1; }
+  fi
+  PY="${VENV_DIR}/bin/python"
+  ok "created venv at ${VENV_DIR}"
+fi
+
+# Every python invocation needs PYTHONPATH so `from scripts.common
+# import X` resolves.
+export PYTHONPATH="${REPO_ROOT}"
+
+# ===============================================================
+# Step 1: enable APIs
+# ===============================================================
+
+step "Step 1: enable GCP APIs"
+
+REQUIRED_APIS=(
+  apigee.googleapis.com
+  apihub.googleapis.com
+  storage.googleapis.com
+  iam.googleapis.com
+)
+
+if [ "$SKIP_APIS" -eq 1 ]; then
+  log "  skipped (--skip-apis)"
+else
+  ENABLED_APIS=$(gcloud services list --enabled \
+    --project="$PROJECT" --format="value(config.name)" 2>/dev/null || echo "")
+  TO_ENABLE=()
+  for api in "${REQUIRED_APIS[@]}"; do
+    if echo "$ENABLED_APIS" | grep -qFx "$api"; then
+      ok "$api already enabled"
+    else
+      TO_ENABLE+=("$api")
+    fi
+  done
+  if [ "${#TO_ENABLE[@]}" -gt 0 ]; then
+    if [ "$CHECK_ONLY" -eq 1 ]; then
+      warn "would enable: ${TO_ENABLE[*]}"
+    elif [ "$DRY_RUN" -eq 1 ]; then
+      log "  DRY RUN: would enable ${TO_ENABLE[*]}"
+    else
+      log "  enabling: ${TO_ENABLE[*]}"
+      gcloud services enable "${TO_ENABLE[@]}" --project="$PROJECT" \
+        || { err "gcloud services enable failed"; exit 2; }
+      ok "enabled ${TO_ENABLE[*]}"
+    fi
+  fi
+fi
+
+# ===============================================================
+# Step 2: verify API hub instance + Apigee org exist
+# ===============================================================
+
+step "Step 2: verify prereqs (API hub instance, Apigee org)"
+
+# API hub instance. Creating one takes 40+ min and has console-
+# only choices, so we insist it exists rather than automating.
+APIHUB_INSTANCE=$(gcloud apihub api-hub-instances lookup \
+  --project="$PROJECT" --location="$LOCATION" \
+  --format="value(apiHubInstance.name)" 2>/dev/null || echo "")
+if [ -z "$APIHUB_INSTANCE" ]; then
+  err "no API hub instance in ${PROJECT} / ${LOCATION}"
+  err "Provision one in the Cloud Console:"
+  err "  https://console.cloud.google.com/apigee/api-hub/setup?project=${PROJECT}"
+  err "This takes 40+ min. Re-run provision.sh when it's ACTIVE."
+  exit 1
+fi
+ok "API hub instance: ${APIHUB_INSTANCE}"
+
+# Apigee organisation.
+TOKEN=$(gcloud auth application-default print-access-token)
+ORG_JSON=$(curl -sf -H "Authorization: Bearer $TOKEN" \
+  "https://apigee.googleapis.com/v1/organizations/${APIGEE_ORG}" 2>/dev/null || echo "")
+if [ -z "$ORG_JSON" ]; then
+  err "Apigee organisation '${APIGEE_ORG}' not accessible."
+  err "Either --apigee-org is wrong, or the caller lacks"
+  err "apigee.organizations.get permission on it."
+  err "The apigee-policy-top10 skill needs this org; other"
+  err "skills do not. Pass --skip-skills to skip publishing"
+  err "if you only want the infrastructure provisioned."
+  exit 1
+fi
+ok "Apigee organisation: ${APIGEE_ORG}"
+
+# ===============================================================
+# Step 3: create GCS bucket
+# ===============================================================
+
+step "Step 3: GCS bucket for signed skill bundles"
+
+if [ "$SKIP_BUCKET" -eq 1 ]; then
+  log "  skipped (--skip-bucket)"
+else
+  BUCKET_URI="gs://${BUCKET}"
+  if gcloud storage buckets describe "$BUCKET_URI" \
+       --project="$PROJECT" >/dev/null 2>&1; then
+    ok "bucket ${BUCKET_URI} exists"
+  else
+    if [ "$CHECK_ONLY" -eq 1 ]; then
+      warn "would create bucket ${BUCKET_URI}"
+    elif [ "$DRY_RUN" -eq 1 ]; then
+      log "  DRY RUN: would create ${BUCKET_URI}"
+    else
+      log "  creating ${BUCKET_URI}"
+      gcloud storage buckets create "$BUCKET_URI" \
+        --project="$PROJECT" --location="$LOCATION" \
+        --uniform-bucket-level-access \
+        --no-public-access-prevention 2>&1 | tail -3
+      ok "created ${BUCKET_URI}"
+    fi
+  fi
+
+  # Grant allUsers reader. This is a demo-only choice --
+  # skill-finder does anonymous HTTPS GET for the .skill zips,
+  # so the bucket must be public-read. In production you'd
+  # bind roles/storage.objectViewer to a specific service
+  # account (or use signed URLs) instead.
+  if [ "$CHECK_ONLY" -eq 0 ] && [ "$DRY_RUN" -eq 0 ]; then
+    log "  granting allUsers:objectViewer (needed for skill-finder"
+    log "  anonymous downloads -- demo choice)"
+    gcloud storage buckets add-iam-policy-binding "$BUCKET_URI" \
+      --member=allUsers --role=roles/storage.objectViewer \
+      >/dev/null 2>&1 || warn "allUsers binding may already exist"
+    ok "public-read enabled on ${BUCKET_URI}"
+  fi
+fi
+
+# ===============================================================
+# Step 4: API hub attribute taxonomy
+# ===============================================================
+
+step "Step 4: API hub attribute taxonomy"
+
+if [ "$SKIP_TAXONOMY" -eq 1 ]; then
+  log "  skipped (--skip-taxonomy)"
+else
+  # Detect existing state before deciding whether to create.
+  # Query the four user-defined attrs and system-spec-type.
+  ATTRS_URL="https://apihub.googleapis.com/v1/projects/${PROJECT}/locations/${LOCATION}/attributes"
+  EXPECTED_ATTRS=(agentic_skill keywords gs_uri signing_key_id)
+  MISSING_ATTRS=()
+  for a in "${EXPECTED_ATTRS[@]}"; do
+    if ! curl -sf -H "Authorization: Bearer $TOKEN" "${ATTRS_URL}/${a}" \
+         >/dev/null 2>&1; then
+      MISSING_ATTRS+=("$a")
+    fi
+  done
+  # Check that system-spec-type includes skill-spec.
+  SST_JSON=$(curl -sf -H "Authorization: Bearer $TOKEN" \
+    "${ATTRS_URL}/system-spec-type" 2>/dev/null || echo "")
+  if echo "$SST_JSON" | grep -q '"id": "skill-spec"'; then
+    HAS_SKILL_SPEC=1
+  else
+    HAS_SKILL_SPEC=0
+  fi
+
+  if [ "${#MISSING_ATTRS[@]}" -eq 0 ] && [ "$HAS_SKILL_SPEC" -eq 1 ]; then
+    ok "user-defined attributes: all 4 present"
+    ok "system-spec-type includes skill-spec"
+  elif [ "$CHECK_ONLY" -eq 1 ]; then
+    if [ "${#MISSING_ATTRS[@]}" -gt 0 ]; then
+      warn "would create missing user-defined attrs: ${MISSING_ATTRS[*]}"
+    fi
+    if [ "$HAS_SKILL_SPEC" -eq 0 ]; then
+      warn "would add 'skill-spec' to system-spec-type enum"
+    fi
+  elif [ "$DRY_RUN" -eq 1 ]; then
+    log "  DRY RUN: would create ${MISSING_ATTRS[*]}, patch system-spec-type"
+  else
+    log "  creating user-defined attributes (idempotent)"
+    "$PY" -m scripts.update_taxonomy \
+      --project "$PROJECT" --location "$LOCATION" \
+      || { err "update_taxonomy.py failed"; exit 2; }
+    ok "user-defined attributes created/verified"
+
+    log "  patching skill-spec into system-spec-type enum"
+    # Idempotent: PATCHing with the same allowed_values list is
+    # a no-op if the value already exists.
+    curl -sf -X PATCH \
+      -H "Authorization: Bearer $TOKEN" \
+      -H "Content-Type: application/json" \
+      "https://apihub.googleapis.com/v1/projects/${PROJECT}/locations/${LOCATION}/attributes/system-spec-type?updateMask=allowed_values" \
+      -d '{
+        "allowedValues": [
+          {"id":"openapi","displayName":"OpenAPI Spec","description":"OpenAPI Spec","immutable":true},
+          {"id":"proto","displayName":"Proto","description":"Proto","immutable":true},
+          {"id":"wsdl","displayName":"WSDL","description":"WSDL","immutable":true},
+          {"id":"mcp-spec","displayName":"MCP Spec","description":"MCP Spec","immutable":true},
+          {"id":"skill-spec","displayName":"Skill Spec","description":"Signed agent skill manifest (apigee-skills-serving reference)"}
+        ]
+      }' >/dev/null \
+      || { err "system-spec-type PATCH failed"; exit 2; }
+    ok "skill-spec added to system-spec-type"
+  fi
+fi
+
+# ===============================================================
+# Step 5: generate signing keypair
+# ===============================================================
+
+step "Step 5: ed25519 signing keypair"
+
+if [ "$SKIP_KEY" -eq 1 ]; then
+  log "  skipped (--skip-key); reusing existing key at ${KEY_PATH}"
+  if [ ! -f "$KEY_PATH" ]; then
+    err "--skip-key passed but no key found at ${KEY_PATH}"
+    exit 1
+  fi
+elif [ -f "$KEY_PATH" ] && [ "$FORCE" -eq 0 ]; then
+  ok "existing key at ${KEY_PATH} (reusing; pass --force to regenerate)"
+else
+  if [ "$CHECK_ONLY" -eq 1 ]; then
+    warn "would generate keypair at ${KEY_PATH}"
+  elif [ "$DRY_RUN" -eq 1 ]; then
+    log "  DRY RUN: would generate keypair"
+  else
+    if [ -f "$KEY_PATH" ] && [ "$FORCE" -eq 1 ]; then
+      warn "--force: overwriting existing key at ${KEY_PATH}"
+      warn "  Prior signatures made with the old key will no longer"
+      warn "  verify against the new one -- re-publish all skills"
+      warn "  after this step."
+      if ! confirm "Proceed with key regeneration?"; then
+        err "aborted at key regeneration"
+        exit 1
+      fi
+    fi
+    log "  generating fresh ed25519 keypair at ${KEY_PATH}"
+    mkdir -p "$(dirname "$KEY_PATH")"
+    "$PY" <<PYEOF
+import hashlib, sys
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+from cryptography.hazmat.primitives import serialization
+
+priv = Ed25519PrivateKey.generate()
+raw = priv.private_bytes(
+    encoding=serialization.Encoding.Raw,
+    format=serialization.PrivateFormat.Raw,
+    encryption_algorithm=serialization.NoEncryption(),
+)
+with open("${KEY_PATH}", "wb") as f:
+    f.write(raw)
+
+pub_raw = priv.public_key().public_bytes(
+    encoding=serialization.Encoding.Raw,
+    format=serialization.PublicFormat.Raw,
+)
+fp = "sha256:" + hashlib.sha256(pub_raw).hexdigest()
+print(fp)
+PYEOF
+    if [ ! -s "$KEY_PATH" ]; then
+      err "key generation failed (empty key file at ${KEY_PATH})"
+      exit 3
+    fi
+    chmod 600 "$KEY_PATH"
+    ok "generated keypair (private: ${KEY_PATH}, mode 0600)"
+  fi
+fi
+
+# Compute fingerprint (needed for the trust-root-install step
+# regardless of whether we generated a new key or reused one).
+if [ -f "$KEY_PATH" ]; then
+  FINGERPRINT=$("$PY" <<PYEOF
+import hashlib
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+from cryptography.hazmat.primitives import serialization
+priv = Ed25519PrivateKey.from_private_bytes(open("${KEY_PATH}", "rb").read())
+pub_raw = priv.public_key().public_bytes(
+    encoding=serialization.Encoding.Raw,
+    format=serialization.PublicFormat.Raw,
+)
+print("sha256:" + hashlib.sha256(pub_raw).hexdigest())
+PYEOF
+)
+  ok "fingerprint: ${FINGERPRINT}"
+fi
+
+# ===============================================================
+# Step 6: publish the 3 example skills
+# ===============================================================
+
+step "Step 6: publish example skills"
+
+if [ "$SKIP_SKILLS" -eq 1 ]; then
+  log "  skipped (--skip-skills)"
+else
+  # Detect which skills are already published so a re-run
+  # skips already-registered ones. We check the API hub 'apis'
+  # collection for a matching displayName. Publishing is
+  # idempotent anyway (register_skill.py handles the update
+  # path) but skipping avoids ~5s of pack+upload+sign work
+  # per already-published skill.
+  APIS_URL="https://apihub.googleapis.com/v1/projects/${PROJECT}/locations/${LOCATION}/apis"
+  EXISTING_APIS=$(curl -sf -H "Authorization: Bearer $TOKEN" \
+    "${APIS_URL}?pageSize=100" 2>/dev/null \
+    | "$PY" -c "
+import json, sys
+try:
+    d = json.load(sys.stdin)
+except Exception:
+    sys.exit(0)
+for a in d.get('apis', []):
+    print(a.get('displayName', ''))
+" 2>/dev/null || echo "")
+  MISSING_SKILLS=()
+  PRESENT_SKILLS=()
+  for skill in "${EXAMPLE_SKILLS[@]}"; do
+    if echo "$EXISTING_APIS" | grep -qFx "$skill"; then
+      PRESENT_SKILLS+=("$skill")
+    else
+      MISSING_SKILLS+=("$skill")
+    fi
+  done
+
+  if [ "${#PRESENT_SKILLS[@]}" -gt 0 ]; then
+    ok "already published: ${PRESENT_SKILLS[*]}"
+  fi
+  if [ "${#MISSING_SKILLS[@]}" -eq 0 ]; then
+    ok "all example skills already published"
+    # Skip the entire publish loop.
+    SKIP_SKILLS=1
+  fi
+fi
+
+if [ "$SKIP_SKILLS" -eq 1 ]; then
+  :  # already handled above
+elif [ "$CHECK_ONLY" -eq 1 ]; then
+  warn "would publish: ${MISSING_SKILLS[*]}"
+elif [ "$DRY_RUN" -eq 1 ]; then
+  log "  DRY RUN: would publish ${MISSING_SKILLS[*]}"
+else
+  # Working dir for patched manifests + signed manifests. Kept
+  # outside the source tree so the source tree stays clean.
+  WORKDIR="${HOME}/.local/share/apigee-skills-serving/publish-workdir"
+  mkdir -p "$WORKDIR"
+
+  for skill in "${MISSING_SKILLS[@]}"; do
+    log "  --- publishing $skill ---"
+    SRC_MANIFEST="${REPO_ROOT}/skills/${skill}/manifest.yaml"
+    WORK_MANIFEST="${WORKDIR}/${skill}.manifest.yaml"
+    SIGNED_MANIFEST="${WORKDIR}/${skill}.signed.yaml"
+    SKILL_ZIP="/tmp/${skill}.skill"
+
+    # Patch gs_uri (source manifest has no gs_uri; publisher
+    # scripts require it to be present before signing).
+    cp "$SRC_MANIFEST" "$WORK_MANIFEST"
+    "$PY" -c "
+import yaml
+p = '${WORK_MANIFEST}'
+m = yaml.safe_load(open(p))
+m['gs_uri'] = 'gs://${BUCKET}/${skill}.skill'
+open(p, 'w').write(yaml.safe_dump(m, sort_keys=True))
+"
+    # Pack.
+    "$PY" -m scripts.pack_skill \
+      --src "${REPO_ROOT}/skills/${skill}" \
+      --out "$SKILL_ZIP" \
+      || { err "pack failed for $skill"; exit 4; }
+    # Upload.
+    "$PY" -m scripts.upload_skill \
+      --zip "$SKILL_ZIP" --bucket "$BUCKET" \
+      || { err "upload failed for $skill"; exit 4; }
+    # Sign.
+    "$PY" -m scripts.sign_skill \
+      --manifest "$WORK_MANIFEST" \
+      --zip "$SKILL_ZIP" \
+      --priv-key "$KEY_PATH" \
+      --out "$SIGNED_MANIFEST" \
+      || { err "sign failed for $skill"; exit 4; }
+    # Register.
+    "$PY" -m scripts.register_skill \
+      --manifest "$SIGNED_MANIFEST" \
+      --project "$PROJECT" --location "$LOCATION" \
+      || { err "register failed for $skill"; exit 4; }
+    ok "$skill published"
+  done
+fi
+
+# ===============================================================
+# Step 7: install trust root into skill-finder's keys/
+# ===============================================================
+
+step "Step 7: install trust root into skill-finder"
+
+if [ "$SKIP_TRUST_ROOT_INSTALL" -eq 1 ]; then
+  log "  skipped (--skip-trust-root-install)"
+elif [ "$CHECK_ONLY" -eq 1 ] || [ "$DRY_RUN" -eq 1 ]; then
+  log "  would install ${FINGERPRINT:-<unknown>}.pem into skill-finder/keys/"
+else
+  # Auto-detect runtime if not passed.
+  if [ -z "$RUNTIME" ]; then
+    if [ -d "$HOME/.gemini/skills/skill-finder" ]; then
+      RUNTIME="gemini"
+    elif [ -d "$HOME/.gemini/antigravity/skills/skill-finder" ]; then
+      RUNTIME="antigravity"
+    elif [ -d "$HOME/.config/opencode/skills/skill-finder" ]; then
+      RUNTIME="opencode"
+    else
+      warn "skill-finder is not installed in any known runtime."
+      warn "The example skills ARE published, but skill-finder"
+      warn "won't trust them until you install skill-finder and"
+      warn "drop the following PEM into keys/:"
+      warn ""
+      "$PY" <<PYEOF
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+from cryptography.hazmat.primitives import serialization
+priv = Ed25519PrivateKey.from_private_bytes(open("${KEY_PATH}", "rb").read())
+pem = priv.public_key().public_bytes(
+    encoding=serialization.Encoding.PEM,
+    format=serialization.PublicFormat.SubjectPublicKeyInfo,
+)
+print(pem.decode())
+PYEOF
+      warn "Save the above as: <fingerprint-without-sha256->.pem"
+      warn "  fingerprint: ${FINGERPRINT}"
+      RUNTIME=""
+    fi
+  fi
+
+  if [ -n "$RUNTIME" ]; then
+    case "$RUNTIME" in
+      opencode)     SKILLS_ROOT="$HOME/.config/opencode/skills" ;;
+      gemini)       SKILLS_ROOT="$HOME/.gemini/skills" ;;
+      antigravity)  SKILLS_ROOT="$HOME/.gemini/antigravity/skills" ;;
+    esac
+    KEYS_DIR="${SKILLS_ROOT}/skill-finder/keys"
+    if [ ! -d "$KEYS_DIR" ]; then
+      warn "skill-finder is not installed at ${SKILLS_ROOT}/skill-finder"
+      warn "Install it first with:"
+      warn "  bash bin/install-skill-finder.sh --runtime ${RUNTIME}"
+    else
+      # <fingerprint-without-sha256:->.pem
+      FP_STEM="${FINGERPRINT#sha256:}"
+      PEM_PATH="${KEYS_DIR}/${FP_STEM}.pem"
+      if [ -f "$PEM_PATH" ]; then
+        ok "trust root already installed: ${PEM_PATH}"
+      elif [ "$CHECK_ONLY" -eq 1 ] || [ "$DRY_RUN" -eq 1 ]; then
+        log "  would install trust root to: ${PEM_PATH}"
+      else
+        "$PY" <<PYEOF
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+from cryptography.hazmat.primitives import serialization
+priv = Ed25519PrivateKey.from_private_bytes(open("${KEY_PATH}", "rb").read())
+pem = priv.public_key().public_bytes(
+    encoding=serialization.Encoding.PEM,
+    format=serialization.PublicFormat.SubjectPublicKeyInfo,
+)
+open("${PEM_PATH}", "wb").write(pem)
+PYEOF
+        ok "installed trust root: ${PEM_PATH}"
+      fi
+    fi
+  fi
+fi
+
+# ===============================================================
+# Ready banner
+# ===============================================================
+
+echo
+echo "==============================================="
+echo -e "  ${GREEN}READY${NC} -- apigee-skills-serving provisioned"
+echo "==============================================="
+log ""
+log "Project:       ${PROJECT}"
+log "Location:      ${LOCATION}"
+log "Bucket:        gs://${BUCKET}"
+log "Apigee org:    ${APIGEE_ORG}"
+log "Trust root:    ${FINGERPRINT:-<unknown>}"
+if [ -n "${RUNTIME:-}" ]; then
+  log "Runtime:       ${RUNTIME}"
+fi
+log ""
+log "Next steps:"
+log "  1. If skill-finder is not yet installed on this machine:"
+log "       bash bin/install-skill-finder.sh --runtime <r>"
+log "     Then re-run: bash bin/provision.sh --project ${PROJECT} \\"
+log "       --skip-apis --skip-bucket --skip-taxonomy --skip-skills"
+log "     to install just the trust root."
+log ""
+log "  2. Launch your agent runtime and ask a question that matches"
+log "     one of the published skill keywords (e.g. 'currency',"
+log "     'weather', 'apigee policies'). skill-finder will find the"
+log "     matching skill in API hub, verify its signature against"
+log "     the trust root we just installed, and install it into"
+log "     the runtime skills dir."
+log ""
+log "  3. Re-run this script any time to reconcile drift. All"
+log "     steps are idempotent. Use --skip-* flags to narrow the"
+log "     re-run to specific steps."

From 2e59ab3fad583e600bc3f283d7e2a7f9e4b9ffd3 Mon Sep 17 00:00:00 2001
From: Geir Sjurseth <gsjurseth@google.com>
Date: Tue, 7 Jul 2026 09:03:37 +0000
Subject: [PATCH 5/7] docs(apigee-skills-serving): document skill-finder +
 skill-publisher + provisioning

Updates the reference implementation's documentation set to
cover the new one-line install workflow and the per-deployment
multi-key trust model.

Four files changed:

  README.md
    * New "Quickstart (five minutes)" section pointing at the
      curl | bash install of skill-finder followed by
      bin/provision.sh. Kept the underlying pack/upload/sign/
      register invocation as a secondary "Under the hood"
      block for readers who want to understand what the scripts
      chain together.
    * Repo layout section updated: new bin/ scripts
      (install-skill-finder.sh, install-skill-publisher.sh,
      provision.sh), new docs (provisioning.md), new skills
      (skill-finder, skill-publisher).
    * Renamed "Example skills" section to "Shipped skills" and
      added skill-finder + skill-publisher rows.
    * Limitations section rewritten: per-deployment trust root
      (not shared), key rotation supported (multi-key), demo
      choice of public-read GCS. Removed the "publisher and
      consumer share an Ed25519 trust root" claim -- multi-key
      trust roots eliminate that.

  docs/architecture.md
    * "What the consumer verifies" section expanded to name
      skill-finder as the reference consumer and reference the
      16-step Hyrum's Law contract.
    * New subsection "Per-deployment trust roots" explains why
      the trust root is not baked into the repo, why
      skill-finder is trusted by provenance rather than
      signature, and how the private-key-stays-local model
      works.
    * New subsection "Multi-key trust roots" documents the
      keys/*.pem glob model, the three operator workflows it
      supports (multi-publisher, zero-downtime rotation,
      bootstrap onboarding), and the empty-keys diagnostic.
    * Threat model table gains two rows: compromised publisher
      key mitigation (multi-key add-then-remove rotation) and
      compromised installer over TLS (out of scope for v1; pin
      --ref for defence-in-depth).
    * New subsection "Trust-root discovery in v2" explains the
      forward-compatible design: API hub as trust-root store
      via a well-known attribute. Deferred as v2 to avoid
      taxonomy churn and a first-fetch TOFU chicken-and-egg
      in v1.

  docs/provisioning.md   [NEW, 300 LoC]
    Complete customer-onboarding runbook:
      * Prerequisites (Apigee org + API hub instance -- both
        out of scope for provision.sh because they take 10-40
        minutes and have console-only decisions).
      * "Five-minute path" quick recipe.
      * Step-by-step explanation of what provision.sh does
        internally (all 7 steps, plus the state detection
        that makes each step idempotent).
      * Common workflows: add another publisher, zero-downtime
        key rotation, rebuild a partial deployment, remove a
        deployment.
      * Troubleshooting: seven concrete symptom -> fix rows for
        the most common failure modes.

  docs/publish-and-install.md
    * New introduction points at provisioning.md for the
      five-minute path and explains when to keep reading the
      long form (understanding provision.sh internals,
      publishing your own skill, troubleshooting).
    * Section 11 (consumer-side install) rewritten to describe
      skill-finder's actual 7-step verify-and-install flow
      rather than a hypothetical consumer implementation. Adds
      forward links to architecture.md#multi-key-trust-roots
      and architecture.md#trust-model.
    * Cleanup section rewritten to reflect the split between
      payload skills (removed by demo-cleanup.sh) and
      skill-finder / skill-publisher (persist across demos;
      remove manually or via the full-teardown recipe in
      provisioning.md).

Verified
--------

* Code fence parity: balanced across all 4 files.
* Markdown cross-references: no broken relative links (checked
  every ](path) that isn't http:// or a #anchor).
* No stale references to gsjurseth/skill-finder or to signed
  release bundle URLs.
---
 references/apigee-skills-serving/README.md    | 153 ++++++---
 .../docs/architecture.md                      |  88 ++++-
 .../docs/provisioning.md                      | 300 ++++++++++++++++++
 .../docs/publish-and-install.md               | 113 +++++--
 4 files changed, 563 insertions(+), 91 deletions(-)
 create mode 100644 references/apigee-skills-serving/docs/provisioning.md

diff --git a/references/apigee-skills-serving/README.md b/references/apigee-skills-serving/README.md
index 64ac9b60..482c78c6 100644
--- a/references/apigee-skills-serving/README.md
+++ b/references/apigee-skills-serving/README.md
@@ -7,11 +7,12 @@ custom MCP hosts, etc.) can discover and install.
 
 This reference implementation shows how to:
 
-1. **Author** a skill (a `SKILL.md` + optional `scripts/`) and describe it
-   with a `manifest.yaml` against a locked schema.
+1. **Author** a skill (a `SKILL.md` + optional `scripts/`) and describe
+   it with a `manifest.yaml` against a locked schema.
 2. **Sign** the skill bundle with an Ed25519 key so consumers can verify
    integrity at install time.
-3. **Upload** the signed `.skill` archive to a Google Cloud Storage bucket.
+3. **Upload** the signed `.skill` archive to a Google Cloud Storage
+   bucket.
 4. **Register** the skill (and its API hub attribute taxonomy) so it is
    discoverable through API hub's search and attribute filters.
 5. **Install** a skill on the consumer side: search API hub, fetch the
@@ -21,6 +22,20 @@ This reference implementation shows how to:
 The whole loop runs on standard Apigee X / hybrid plus API hub — no
 custom infrastructure.
 
+Two ready-to-run agent skills wrap the reference implementation into
+a one-line install for end users:
+
+- **`skills/skill-finder/`** — consumer-side discovery client. Queries
+  API hub for signed skills, verifies their Ed25519 signatures against
+  locally-installed per-deployment trust roots, and installs matching
+  skills into the agent runtime's skills directory.
+- **`skills/skill-publisher/`** — author-side pipeline orchestrator.
+  Wraps pack → sign → upload → register into a single skill an agent
+  can invoke on a source skill directory.
+
+Both are installed by `bin/install-skill-finder.sh` (which installs
+both by default).
+
 ## Why Apigee API hub for skills?
 
 Agent skills are, structurally, just metadata-tagged content addressed
@@ -48,12 +63,17 @@ references/apigee-skills-serving/
 ├── pytest.ini                   test configuration
 ├── docs/
 │   ├── architecture.md          design overview and trust model
-│   ├── publish-and-install.md   end-to-end walkthrough
+│   ├── provisioning.md          customer-onboarding runbook (five-minute path)
+│   ├── publish-and-install.md   detailed end-to-end walkthrough (under-the-hood)
 │   └── policy-skill-catalog.md  about the apigee-policy-top10 example
 ├── bin/
 │   ├── check-prerequisites.sh   pre-flight environment validator
-│   ├── demo-setup.sh            env export + readiness print
-│   └── demo-cleanup.sh          remove demo artifacts
+│   ├── demo-setup.sh            env export + readiness print (legacy)
+│   ├── demo-cleanup.sh          remove locally-installed skills
+│   ├── install-skill-finder.sh  one-line installer for skill-finder + skill-publisher
+│   ├── install-skill-publisher.sh  install skill-publisher only
+│   └── provision.sh             one-line customer bootstrap (APIs, bucket,
+│                                taxonomy, key gen, publish, trust root)
 ├── schema/
 │   └── skill-manifest.schema.yaml  locked v1 manifest schema
 ├── scripts/                     publisher-side toolchain
@@ -63,8 +83,10 @@ references/apigee-skills-serving/
 │   ├── register_skill.py        register the manifest with API hub
 │   ├── update_taxonomy.py       create/update API hub attribute taxonomy
 │   └── common/                  shared libraries (retry, IAM, schema)
-├── skills/                      example skills
-│   ├── apigee-policy-top10/     skill that documents Apigee policy patterns
+├── skills/                      shipped skills
+│   ├── skill-finder/            consumer-side discovery + install client
+│   ├── skill-publisher/         author-side pipeline orchestrator
+│   ├── apigee-policy-top10/     example skill that reports top Apigee policies
 │   ├── currency-converter/      minimal example
 │   └── weather-lookup/          minimal example
 ├── examples/
@@ -93,42 +115,54 @@ references/apigee-skills-serving/
 6. The roles `roles/apihub.editor` and `roles/storage.objectCreator` on
    the target project.
 
-## Quickstart
+## Quickstart (five minutes)
+
+The fast path uses two shell scripts:
 
 ```bash
-# 1. Clone and enter the reference
+# 1. Install skill-finder + skill-publisher on your machine.
+curl -fsSL https://raw.githubusercontent.com/apigee/devrel/main/\
+references/apigee-skills-serving/bin/install-skill-finder.sh \
+  | bash -s -- --runtime gemini      # or --runtime opencode / antigravity
+
+# 2. Clone the reference and provision your GCP project.
 git clone https://github.com/apigee/devrel.git
 cd devrel/references/apigee-skills-serving
+bash bin/provision.sh --project MY_PROJECT --yes
+```
 
-# 2. Install Python dependencies into a virtualenv
-python3 -m venv .venv
-. .venv/bin/activate
-pip install -r requirements.txt
+`provision.sh` enables the required GCP APIs, creates a GCS
+bucket, sets up the API hub attribute taxonomy, generates a
+fresh Ed25519 signing keypair on your machine (private key
+stays there), and publishes the three example skills. Every
+step is idempotent — re-run any time to reconcile drift.
+
+Full runbook (including troubleshooting and key rotation):
+[`docs/provisioning.md`](docs/provisioning.md).
+
+### Under the hood
 
-# 3. Set up environment variables (edit values to match your project)
-cp env.sh.example env.sh
-$EDITOR env.sh
-. ./env.sh
-
-# 4. Verify prerequisites
-./bin/check-prerequisites.sh
-
-# 5. (One-time) Create the API hub attribute taxonomy
-python3 scripts/update_taxonomy.py \
-  --project "$APIHUB_PROJECT" \
-  --location "$APIHUB_LOCATION"
-
-# 6. Pack, sign, upload, register a skill
-python3 scripts/pack_skill.py    skills/currency-converter   /tmp/cc.skill
-python3 scripts/sign_skill.py    /tmp/cc.skill   --key-file ./signing.key
-python3 scripts/upload_skill.py  /tmp/cc.skill   --bucket "$GCS_BUCKET"
-python3 scripts/register_skill.py \
-  --project "$APIHUB_PROJECT" --location "$APIHUB_LOCATION" \
-  --manifest /tmp/cc.skill
+The scripts above chain the five underlying operations that
+this reference exists to demonstrate:
+
+```bash
+# For each skill:
+python3 -m scripts.pack_skill    --src skills/<name> --out /tmp/<name>.skill
+python3 -m scripts.upload_skill  --zip /tmp/<name>.skill --bucket "$GCS_BUCKET"
+python3 -m scripts.sign_skill    --manifest <patched-manifest> \
+                                 --zip /tmp/<name>.skill \
+                                 --priv-key <signing.raw> \
+                                 --out <signed-manifest>
+python3 -m scripts.register_skill --manifest <signed-manifest> \
+                                 --project "$APIHUB_PROJECT" \
+                                 --location "$APIHUB_LOCATION"
 ```
 
-Full walkthrough: [`docs/publish-and-install.md`](docs/publish-and-install.md).
-Design rationale and trust model: [`docs/architecture.md`](docs/architecture.md).
+Full walkthrough:
+[`docs/publish-and-install.md`](docs/publish-and-install.md).
+
+Design rationale and trust model:
+[`docs/architecture.md`](docs/architecture.md).
 
 ## Running the tests
 
@@ -144,26 +178,41 @@ pytest -q
 `pipeline.sh` runs the same suite and is what apigee/devrel CI invokes
 nightly.
 
-## Example skills
+## Shipped skills
 
-| Skill                  | Purpose                                                                                             |
-| ---------------------- | --------------------------------------------------------------------------------------------------- |
-| `currency-converter`   | Minimal example. A `SKILL.md` plus a `manifest.yaml`. Useful as a copy-and-edit starting point.     |
-| `weather-lookup`       | Minimal example demonstrating a skill with API-key-based external HTTP calls.                       |
-| `apigee-policy-top10`  | A skill that documents the ten most useful Apigee policy patterns, with a script that enumerates    |
-|                        | the policies present in your org. See [`docs/policy-skill-catalog.md`](docs/policy-skill-catalog.md). |
-| `examples/apigee-proxy-skill` | A complete, production-shaped skill: 18 MCP tools, 25 Jinja2 policy templates, full manifest. |
+| Skill                         | Purpose                                                                                                                                                       |
+| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `skill-finder`                | Consumer-side. Discovers, verifies, and installs signed skills from the customer's API hub. Multi-key trust root; per-deployment provisioning.                |
+| `skill-publisher`             | Author-side. Wraps pack → sign → upload → register into a single skill an agent can invoke on a source skill directory.                                       |
+| `currency-converter`          | Minimal example. A `SKILL.md` plus a `manifest.yaml`. Useful as a copy-and-edit starting point.                                                               |
+| `weather-lookup`              | Minimal example demonstrating a skill with API-key-based external HTTP calls.                                                                                 |
+| `apigee-policy-top10`         | A skill that documents the ten most useful Apigee policy patterns, with a script that enumerates the policies present in your org. See [`docs/policy-skill-catalog.md`](docs/policy-skill-catalog.md). |
+| `examples/apigee-proxy-skill` | A complete, production-shaped skill: 18 MCP tools, 25 Jinja2 policy templates, full manifest.                                                                 |
 
 ## Limitations and non-goals
 
-- The skill registry uses API hub's standard search; ranking is keyword
-  overlap, not semantic. For semantic ranking, integrate a vector
-  search component separately.
-- The publisher and consumer share an Ed25519 trust root. Key rotation
-  is a manual operator workflow; this reference does not implement
-  automatic rotation.
-- Skills are sandboxed by the consumer runtime (OpenCode, agent host).
-  This reference does not introduce additional sandboxing on top.
+- The skill registry uses API hub's standard search; ranking is
+  keyword overlap, not semantic. For semantic ranking, integrate
+  a vector search component separately.
+- The trust root is **per-deployment**, not shared. Each customer
+  who runs `provision.sh` generates their own Ed25519 keypair on
+  their machine. The `apigee/devrel` repo intentionally contains
+  no signing key material and no signed release bundles;
+  skill-finder itself is trusted by provenance (TLS + `apigee`
+  GitHub org). See
+  [`docs/architecture.md#trust-model`](docs/architecture.md#trust-model)
+  for the full rationale.
+- Key rotation is a supported operator workflow (multi-key
+  trust root; add-then-remove pattern with no install-time gap),
+  but is not automated. See
+  [`docs/provisioning.md#key-rotation-with-zero-downtime`](docs/provisioning.md#key-rotation-with-zero-downtime).
+- The GCS bucket for signed bundles must be public-read because
+  skill-finder downloads via anonymous HTTPS. This is a demo
+  choice; production would use signed URLs or an authenticated
+  consumer.
+- Skills are sandboxed by the consumer runtime (OpenCode, Gemini
+  CLI, Antigravity, custom MCP hosts). This reference does not
+  introduce additional sandboxing on top.
 
 ## License
 
diff --git a/references/apigee-skills-serving/docs/architecture.md b/references/apigee-skills-serving/docs/architecture.md
index bb01d18e..f20725ee 100644
--- a/references/apigee-skills-serving/docs/architecture.md
+++ b/references/apigee-skills-serving/docs/architecture.md
@@ -90,19 +90,68 @@ verifying one manifest but installing a different bundle.
 
 ### What the consumer verifies
 
-At install time, the consumer:
+The reference consumer implementation is the `skill-finder` skill
+(installed via `bin/install-skill-finder.sh`). At install time,
+`find_install.py` performs a locked 16-step contract:
 
 1. Fetches the manifest from API hub.
 2. Looks up the publisher's public key by the `signing_key_id`
-   fingerprint (typically out-of-band; the consumer is configured with
-   a list of trusted publisher keys).
+   fingerprint against **the trust roots installed on this
+   machine** (see "Per-deployment trust roots" below).
 3. Re-canonicalises the manifest (excluding signature fields).
 4. Verifies the Ed25519 signature.
 5. Downloads the `.skill` zip from `gs_uri`.
-6. Computes SHA-256 of the downloaded zip and compares it against the
-   manifest's `zip_sha256`.
-7. Only after both checks pass does the consumer extract the `.skill`
-   contents to the local skills directory.
+6. Computes SHA-256 of the downloaded zip and compares it against
+   the manifest's `zip_sha256`.
+7. Only after both checks pass does the consumer extract the
+   `.skill` contents to the local skills directory.
+
+### Per-deployment trust roots
+
+The trust root is **not** a shared, repo-baked public key. It is a
+per-deployment secret: each customer who runs `bin/provision.sh`
+generates a fresh Ed25519 keypair on their own machine, signs
+their catalog with it, and drops the corresponding public PEM into
+their local skill-finder install's `keys/` directory. The private
+key never leaves the customer's machine and is never committed to
+any repo (including `apigee/devrel`).
+
+Consequences:
+
+- **skill-finder itself is trusted by provenance**, not by
+  signature. The operator fetched skill-finder from a known-good
+  URL (`raw.githubusercontent.com/apigee/devrel/main/…`) over TLS
+  from the `apigee` GitHub organisation. There is no signature to
+  verify because the code that would verify it *is* skill-finder.
+
+- **Payload skills** (currency-converter, weather-lookup,
+  apigee-policy-top10, and anything a customer publishes on top)
+  ARE cryptographically verified against the per-deployment trust
+  root.
+
+### Multi-key trust roots
+
+skill-finder loads **every** `.pem` file under
+`~/.<runtime>/skills/skill-finder/keys/` as an accepted trust
+root. A downloaded manifest is accepted if its `signing_key_id`
+matches **any** installed key. This supports three operator
+workflows:
+
+- **Multi-publisher orgs**: Alice's team's key and Bob's team's
+  key can both be trusted simultaneously. Any skill signed by
+  either team installs cleanly.
+- **Zero-downtime key rotation**: add the new key alongside the
+  old one; publish new skills signed with the new key; wait for
+  callers to stop using the old key; remove the old key. No
+  install-time gap.
+- **Bootstrap onboarding**: `bin/provision.sh` drops a
+  `<fingerprint>.pem` into `keys/` after the initial key
+  generation, giving skill-finder its first trust root without a
+  separate configuration step.
+
+If `keys/` is empty at verify time, skill-finder emits
+`trust root: FAILED — no valid keys loaded from …` and refuses
+to install anything.
 
 ### Threat model
 
@@ -111,12 +160,27 @@ At install time, the consumer:
 | Compromised GCS bucket serves a wrong zip    | `zip_sha256` mismatch is detected before extraction.                       |
 | Modified manifest in API hub                 | Ed25519 signature mismatch is detected.                                    |
 | Replay of an old (vulnerable) skill version  | Consumer is responsible for tracking versions; manifest carries `version`. |
-| Compromised publisher signing key            | Out of scope; operator must rotate the trust root.                         |
+| Compromised publisher signing key            | Operator rotates by adding a new key to `keys/`, re-publishing all skills, then removing the old key. Multi-key support means no install-time gap. |
 | Malicious skill body (legitimate signature)  | Out of scope; agent runtime sandboxing is the boundary.                    |
-
-The reference does **not** introduce a new sandbox. The agent runtime
-that loads the skill (OpenCode, Claude Code, etc.) is responsible for
-limiting what skill code can do once it runs.
+| Compromised skill-finder installer over TLS  | Out of scope for v1 (relies on TLS + apigee org provenance). Operators can pin `--ref <commit-sha>` in the installer invocation to reduce exposure. |
+
+The reference does **not** introduce a new sandbox. The agent
+runtime that loads the skill (OpenCode, Gemini CLI, Antigravity,
+Claude Code, custom MCP hosts) is responsible for limiting what
+skill code can do once it runs.
+
+### Trust-root discovery in v2 (not implemented)
+
+A future iteration could store the trust root in API hub itself
+as a well-known attribute on a placeholder API resource. This
+would let skill-finder discover the trust root from the same
+catalog it queries for skills, eliminating the
+`bin/provision.sh`-drops-a-PEM step. The first fetch would be
+trust-on-first-use (TOFU) against a URL the operator provides;
+subsequent fetches would verify against the cached pubkey. This
+is a v2 design and is intentionally out of scope for v1: it
+requires new attribute conventions, taxonomy changes, and a
+bootstrap mode that would add several more moving parts.
 
 ## Canonical serialization
 
diff --git a/references/apigee-skills-serving/docs/provisioning.md b/references/apigee-skills-serving/docs/provisioning.md
new file mode 100644
index 00000000..03430558
--- /dev/null
+++ b/references/apigee-skills-serving/docs/provisioning.md
@@ -0,0 +1,300 @@
+<!--
+Copyright 2026 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied. See the License for the specific language governing
+permissions and limitations under the License.
+-->
+
+# Provisioning runbook
+
+This document walks through provisioning a green-field GCP project
+as an apigee-skills-serving reference deployment. The end state:
+skill-finder is installed on the operator's machine, the API hub
+catalog contains three signed example skills, and a fresh
+Ed25519 keypair (owned by the operator) is the deployment's trust
+root.
+
+If you already have a deployment and just want to add another
+skill or rotate keys, jump to the [Common workflows](#common-workflows)
+section.
+
+## Prerequisites (must exist before you start)
+
+Two things `bin/provision.sh` does NOT create for you, because
+they have long provisioning times and console-only decisions:
+
+1. **The Apigee organisation.** Provisioning takes 10-30 minutes
+   and asks for region, networking, and analytics preferences.
+   [Enable Apigee](https://cloud.google.com/apigee/docs/api-platform/get-started/provisioning-intro)
+   before running `provision.sh`.
+2. **The API hub instance.** Provisioning takes 40+ minutes and
+   asks for encryption (CMEK vs GMEK) and Vertex region.
+   [Enable API hub](https://cloud.google.com/apigee/docs/apihub/provision)
+   before running `provision.sh`.
+
+You also need:
+
+- IAM `roles/owner` on the target project (or the fine-grained
+  equivalent: `apihub.editor`, `apigee.admin`,
+  `storage.admin`, `serviceusage.serviceUsageAdmin`).
+- `gcloud`, `python3` (≥ 3.10), and `curl` on your machine.
+- ADC set up: `gcloud auth application-default login`.
+- A clone of `apigee/devrel` — the provisioning script runs from
+  `references/apigee-skills-serving/`.
+
+## The five-minute path
+
+For an operator who wants to get from empty project to working
+demo as fast as possible:
+
+```bash
+# 1. Install skill-finder + skill-publisher on your machine.
+curl -fsSL https://raw.githubusercontent.com/apigee/devrel/main/\
+references/apigee-skills-serving/bin/install-skill-finder.sh \
+  | bash -s -- --runtime gemini      # or --runtime opencode / antigravity
+
+# 2. Clone the reference and run the provisioner.
+git clone https://github.com/apigee/devrel.git
+cd devrel/references/apigee-skills-serving
+bash bin/provision.sh --project MY_PROJECT --yes
+```
+
+After ~2-3 minutes, `provision.sh` prints a `READY` banner. Launch
+your agent and ask a question that matches one of the example
+skills' keywords ("convert 100 USD to EUR", "what's the weather in
+Tokyo", "what apigee policies should I optimize"). skill-finder
+will find, verify, and install the matching skill.
+
+## What `provision.sh` does step by step
+
+### Step 0: preflight
+
+Verifies:
+- Required tools (`gcloud`, `python3`, `curl`) are on PATH.
+- ADC is set up and returns a token.
+- A per-user Python venv exists at
+  `~/.local/share/skill-finder/venv` (creates one if not; the same
+  venv is shared with `install-skill-finder.sh`, so installing
+  skill-finder first speeds provisioning up).
+
+### Step 1: enable GCP APIs
+
+Enables (if not already enabled): `apigee.googleapis.com`,
+`apihub.googleapis.com`, `storage.googleapis.com`,
+`iam.googleapis.com`. Skips APIs that are already on.
+
+### Step 2: verify prereqs
+
+Confirms the API hub instance exists in the target project +
+location. Confirms the Apigee organisation exists. If either is
+missing, fails fast with a link to the console page where you can
+provision it.
+
+### Step 3: GCS bucket
+
+Creates a public-read GCS bucket named
+`<project>-skills-<location>` (or the `--bucket` value). Grants
+`allUsers:objectViewer` — the demo requires public read because
+skill-finder downloads bundles via anonymous HTTPS (`https://
+storage.googleapis.com/<bucket>/<file>.skill`), no ADC scope.
+
+**Production note**: this is a demo-only choice. In production
+you would prefer signed URLs or an authenticated consumer service
+account. skill-finder currently supports only anonymous HTTPS;
+adding ADC-authenticated download would be a small change to
+`find_install.py`.
+
+### Step 4: API hub attribute taxonomy
+
+Creates the four user-defined attributes skill-finder queries and
+skill-publisher writes:
+
+- `agentic_skill` (string, "true" | "false") — the filter
+  skill-finder uses to enumerate agent-consumable APIs.
+- `keywords` (string, multi-valued) — the tokens skill-finder
+  ranks against the user's query.
+- `gs_uri` (string) — GCS URI of the `.skill` bundle.
+- `signing_key_id` (string) — sha256 fingerprint of the signing
+  public key.
+
+Also patches `skill-spec` into the `system-spec-type` enum. This
+is a known gap: `scripts/update_taxonomy.py` alone does not add
+`skill-spec` (because it's a system-defined attribute, not user-
+defined), but `register_skill.py` requires it. `provision.sh`
+patches it via a direct API hub `PATCH` call.
+
+### Step 5: ed25519 signing keypair
+
+Generates a fresh keypair on your machine and writes the private
+key to `~/.config/apigee-skills-demo/signing.raw` (mode 0600).
+Prints the SHA-256 fingerprint of the public key.
+
+If a key already exists at that path, `provision.sh` reuses it
+by default. Pass `--force` to regenerate — this invalidates all
+existing signatures and requires re-publishing every skill.
+
+### Step 6: publish example skills
+
+For each of `currency-converter`, `weather-lookup`,
+`apigee-policy-top10`:
+
+1. Copies the source manifest into a workdir at
+   `~/.local/share/apigee-skills-serving/publish-workdir/`.
+2. Patches in the `gs_uri` (the source manifest ships without one;
+   `sign_skill.py` requires it to be present at sign time
+   because the signature covers `gs_uri`).
+3. Runs `pack_skill.py` to produce a `.skill` zip in `/tmp/`.
+4. Runs `upload_skill.py` to push the zip to GCS.
+5. Runs `sign_skill.py` to produce a signed manifest in the
+   workdir.
+6. Runs `register_skill.py` to register the signed manifest with
+   API hub as an API + Version + Spec triple.
+
+Skills that are already registered are detected and skipped —
+re-running `provision.sh` against a fully-provisioned project
+takes ~10 seconds and produces zero writes.
+
+### Step 7: install trust root into skill-finder
+
+Auto-detects the agent runtime (Gemini CLI, OpenCode, or
+Antigravity) by looking for
+`~/.gemini/skills/skill-finder/`,
+`~/.config/opencode/skills/skill-finder/`, or
+`~/.gemini/antigravity/skills/skill-finder/`. If any of them
+exists, drops the public key PEM (with the fingerprint as the
+filename) into that install's `keys/` directory. skill-finder
+will trust the just-published catalog on its next invocation.
+
+If skill-finder isn't installed anywhere yet, `provision.sh`
+prints the public PEM to stdout with instructions for the
+operator to save it manually. The example skills are still
+published — only the local trust-root install is skipped.
+
+## Common workflows
+
+### Add another publisher (multi-key trust)
+
+skill-finder's `keys/` directory is a set: any file matching
+`*.pem` is loaded. To add a second publisher without disrupting
+the first:
+
+1. The second publisher generates their own Ed25519 keypair on
+   their machine.
+2. They send you (out of band) the public PEM.
+3. You drop it into `~/.<runtime>/skills/skill-finder/keys/<fingerprint>.pem`.
+4. Every skill they sign with their key now installs cleanly
+   alongside skills signed by the original key.
+
+### Key rotation with zero downtime
+
+1. Generate the new key: `bash bin/provision.sh --project X
+   --force` (with `--yes` if you want no prompts). This
+   overwrites `signing.raw`, re-publishes all example skills
+   signed with the new key, and drops the new pubkey into
+   `keys/`. The **old** pubkey is still in `keys/` too, so
+   old-signed manifests keep verifying.
+2. Republish any of your own skills with the new key.
+3. When you're confident no one is still using old-signed
+   manifests, delete the old PEM from `keys/`.
+
+### Rebuild a partially-provisioned project
+
+`provision.sh` is idempotent. Just re-run it. Every step detects
+existing state and skips work that isn't needed. If you want to
+force one specific step to re-run:
+
+```bash
+# Force re-publish (drops the "already published" fast-path):
+bash bin/provision.sh --project X --skip-apis --skip-bucket \
+    --skip-taxonomy --skip-key --skip-trust-root-install --yes
+# You'll need to delete + re-register each skill from API hub by
+# hand to force a real re-publish; the fast-path skips
+# publishing when API hub already lists the skill. Better to
+# fix whatever went wrong in-place with individual
+# scripts/register_skill.py invocations.
+```
+
+### Remove a deployment
+
+`provision.sh` has no `--uninstall`. To tear down:
+
+```bash
+# Delete registered APIs from API hub.
+gcloud apihub apis delete currency-converter \
+  --project X --location us-west1
+gcloud apihub apis delete weather-lookup \
+  --project X --location us-west1
+gcloud apihub apis delete apigee-policy-top10 \
+  --project X --location us-west1
+
+# Delete .skill objects from GCS.
+gcloud storage rm gs://<bucket>/currency-converter.skill
+gcloud storage rm gs://<bucket>/weather-lookup.skill
+gcloud storage rm gs://<bucket>/apigee-policy-top10.skill
+
+# Optionally delete the bucket + attributes + APIs. Note that
+# deleting an API hub instance is a separate operation that
+# takes 40+ minutes.
+
+# Delete the local trust root.
+rm ~/.gemini/skills/skill-finder/keys/*.pem
+
+# Delete the private key.
+rm ~/.config/apigee-skills-demo/signing.raw
+```
+
+## Troubleshooting
+
+### `provision.sh` says "no API hub instance in ..."
+
+The instance doesn't exist yet in that project + location. Follow
+the console link in the error message; provisioning takes 40+
+minutes. Re-run `provision.sh` when the instance state is `ACTIVE`.
+
+### `provision.sh` says "Apigee organisation ... not accessible"
+
+Either the org doesn't exist in that project, or your caller
+lacks `apigee.organizations.get`. If you're not planning to use
+the `apigee-policy-top10` skill (which is the only one that
+needs an Apigee org), pass `--skip-skills` to provision the
+infrastructure without publishing that skill.
+
+### skill-finder says `trust root: FAILED — no valid keys loaded`
+
+`~/.<runtime>/skills/skill-finder/keys/` is empty. Either:
+
+- Re-run `provision.sh` (it will detect that skill-finder is
+  now installed and drop the PEM in during Step 7), or
+- Manually copy the PEM from another operator's install.
+
+### skill-finder says `key-id check: FAILED — manifest declares X, but skill-finder only trusts: Y`
+
+The signing key that signed the manifest is different from any
+key in your local `keys/`. Either your `provision.sh` used a
+different key (check the fingerprint in the READY banner), or a
+different publisher signed the skill. Get their public PEM,
+drop it into `keys/`, and retry.
+
+### `register_skill.py` in Step 6 fails with `attribute system-spec-type does not have any allowed value with id: skill-spec`
+
+Step 4 didn't run (or was skipped). Re-run `provision.sh` without
+`--skip-taxonomy`; it will PATCH the enum to include `skill-spec`.
+
+### After provisioning, my agent still doesn't find any skills
+
+Two possibilities:
+
+- skill-finder isn't installed in your agent runtime. Run
+  `bash bin/install-skill-finder.sh --runtime <r>`.
+- The trust root PEM isn't in the runtime's skills dir. Re-run
+  `provision.sh` with `--skip-apis --skip-bucket --skip-taxonomy
+  --skip-skills --skip-key` (i.e. run only Step 7).
diff --git a/references/apigee-skills-serving/docs/publish-and-install.md b/references/apigee-skills-serving/docs/publish-and-install.md
index d1bf9dad..5f001a24 100644
--- a/references/apigee-skills-serving/docs/publish-and-install.md
+++ b/references/apigee-skills-serving/docs/publish-and-install.md
@@ -1,9 +1,41 @@
 # Publish and install walkthrough
 
 This walkthrough takes you from a fresh checkout to a signed,
-registered skill that an agent runtime can install from API hub. The
-examples use the bundled `skills/currency-converter` as the skill being
-published.
+registered skill that an agent runtime can install from API hub.
+The examples use the bundled `skills/currency-converter` as the
+skill being published.
+
+## For most people: use `bin/provision.sh`
+
+If you just want a working reference deployment, skip the rest
+of this document and read [`provisioning.md`](provisioning.md).
+The five-minute path is:
+
+```bash
+# 1. Install skill-finder on your machine
+curl -fsSL https://raw.githubusercontent.com/apigee/devrel/main/\
+references/apigee-skills-serving/bin/install-skill-finder.sh \
+  | bash -s -- --runtime gemini
+
+# 2. Clone the reference and provision your project
+git clone https://github.com/apigee/devrel.git
+cd devrel/references/apigee-skills-serving
+bash bin/provision.sh --project MY_PROJECT --yes
+```
+
+`provision.sh` runs everything in this walkthrough end to end:
+enables APIs, creates the GCS bucket, sets up the taxonomy,
+generates a fresh signing keypair on your machine, and publishes
+the three example skills. It is idempotent — re-run it any time
+to reconcile drift.
+
+**Keep reading this document if:**
+
+- You want to understand what `provision.sh` does under the hood.
+- You want to publish your own skill (not one of the bundled
+  examples). See especially section 6 (Pack) and section 7 (Sign)
+  for the per-skill inputs.
+- You are troubleshooting a failure at a specific step.
 
 ## 1. Prerequisites
 
@@ -187,38 +219,65 @@ You should see your `currency-converter` entry in the list.
 
 ## 11. Consumer side: install the skill
 
-A consumer (an agent runtime, or a developer running `pip install`
-analog for skills) performs the inverse flow:
+The consumer-side reference implementation lives at
+`skills/skill-finder/scripts/find_install.py`. It performs the
+inverse flow:
 
-1. Searches API hub by keyword overlap on the skill description, or
-   by attribute (e.g., "every skill with `skill-runtime-iam` containing
-   `apigee.proxies.create`").
-2. Fetches the chosen entry's manifest from API hub.
+1. Searches API hub for APIs where the `agentic_skill` attribute
+   is `"true"`, ranks results by keyword overlap on the user's
+   query, picks the top match.
+2. Fetches the chosen entry's manifest (base64-encoded in the
+   API hub Spec resource).
 3. Re-canonicalises the manifest (excluding signature fields),
-   verifies the Ed25519 signature against the publisher's known public
-   key.
-4. Downloads the `.skill` zip from `gs_uri`.
-5. Computes SHA-256 and matches it against `zip_sha256` from the
-   manifest.
-6. Extracts the `.skill` into the consumer's skills directory
-   (e.g., `~/.config/opencode/skills/{skill-name}/`).
-
-The consumer-side `find_install.py` reference implementation is **not**
-included in this PR; the contract it consumes is fully documented above
-and exercised by the test suite. See
-`tests/test_register_fetch_integration.py` for a stateful in-process
-example.
+   verifies the Ed25519 signature against **every trust root**
+   under `~/.<runtime>/skills/skill-finder/keys/*.pem`. Accepts
+   the manifest if the `signing_key_id` matches any installed
+   trust root (multi-key model — see
+   [`architecture.md#multi-key-trust-roots`](architecture.md#multi-key-trust-roots)).
+4. Runs an IAM pre-flight against the skill's declared
+   `runtime_iam` permissions — fails fast if the caller lacks
+   any of them.
+5. Downloads the `.skill` zip from `gs_uri` via anonymous HTTPS
+   (the bucket must be public-read; see
+   [`architecture.md#trust-model`](architecture.md#trust-model)).
+6. Computes SHA-256 and matches it against `zip_sha256` from
+   the manifest.
+7. Extracts the `.skill` into the consumer's skills directory
+   (`~/.gemini/skills/{skill-name}/`,
+   `~/.config/opencode/skills/{skill-name}/`, or
+   `~/.gemini/antigravity/skills/{skill-name}/` per runtime).
+
+The whole flow is emitted as a locked 16-line
+[Hyrum's Law contract](https://www.hyrumslaw.com/) on stdout so
+the agent runtime can surface each verification step to the user
+verbatim.
+
+To install skill-finder itself, use
+`bin/install-skill-finder.sh` (see
+[`provisioning.md`](provisioning.md)).
 
 ## Cleanup
 
-To remove the demo artifacts:
+To remove the demo artifacts installed by skill-finder:
 
 ```bash
 ./bin/demo-cleanup.sh
 ```
 
-This removes locally extracted skills under
+This removes locally extracted PAYLOAD skills under
 `~/.config/opencode/skills/{currency-converter,weather-lookup,
-apigee-policy-top10}/`. It does **not** delete anything from API hub
-or your GCS bucket — those are remote and the cleanup intentionally
-stays local to avoid surprising side-effects in shared projects.
+apigee-policy-top10}/`. It does **not** delete anything from API
+hub or your GCS bucket, and it does **not** remove skill-finder
+or skill-publisher themselves — those are meant to persist
+across demo runs.
+
+To remove skill-finder + skill-publisher too:
+
+```bash
+rm -rf ~/.gemini/skills/skill-finder ~/.gemini/skills/skill-publisher
+# or for opencode:
+rm -rf ~/.config/opencode/skills/skill-finder ~/.config/opencode/skills/skill-publisher
+```
+
+For a full teardown (including remote resources), see
+[`provisioning.md#remove-a-deployment`](provisioning.md#remove-a-deployment).

From 362c97a59d839054811a8c6c97c2843ec1da7387 Mon Sep 17 00:00:00 2001
From: Geir Sjurseth <gsjurseth@google.com>
Date: Tue, 7 Jul 2026 09:50:46 +0000
Subject: [PATCH 6/7] fix(apigee-skills-serving): satisfy megalinter (bandit +
 flake8) in skill-finder scripts
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fixes 12 megalinter errors surfaced when running the v4 image
locally against feat/add-skill-finder. All fixes are lint-only;
no runtime behaviour changes.

  bandit (2 errors, blocking)
    * find_install.py:438 `assert self._state is not None`
      -> added `# nosec B101 - defensive invariant`
    * find_install.py:807 `assert gs_uri.startswith("gs://")`
      -> added `# nosec B101 - caller invariant`

    The `# nosec` pattern matches the existing devrel convention
    (see scripts/common/http_retry.py which already uses it).
    Both asserts are defensive invariants, not security checks;
    the caller-side contracts they enforce would fail in later
    lines anyway.

  flake8 (10 errors, blocking)
    * find_install.py:116-120,189: E402 module-level import not
      at top of file. These imports come AFTER a deferred
      import-guard block that catches ModuleNotFoundError from
      cryptography and prints a helpful "run under the venv
      wrapper" message. The E402 is by design.
      -> added `# noqa: E402` comments to the affected lines.
    * find_install.py:178: F401 unused import `http_post_retry`.
      -> removed from both the production and dev-fallback
      import blocks.
    * find_install.py:570: E501 line too long (introduced during
      one of my edits).
      -> broke the set comprehension across three lines.
    * find_install.py:789,1114,1118: F541 f-string missing
      placeholders. These are middle-of-multi-line-string
      literals concatenated by Python's implicit concatenation;
      only the parts with interpolation need the `f` prefix.
      -> dropped the `f` prefix from the placeholder-free lines.
    * list_skills.py:45: F401 unused import `os`.
      -> removed.
    * list_skills.py:195,202: F541 f-string missing placeholders.
      -> dropped the `f` prefix.

Verified
--------

  * `flake8 skills/skill-finder/scripts/`: PASS (exit 0)
  * `pytest`: 237/237 pass (unchanged, no behaviour regressions)
  * megalinter v4 via podman (same env vars as
    .github/workflows/devrel-static-checks.yml): finishes with
    "Successfully linted all files, but with ignored errors"
    (`◬`, non-blocking). Zero `❌` blocking failures.

  The 3 categories that WERE blocking before this commit:
    - CREDENTIALS/secretlint: false-positive scan of a stray
      local .venv/ (not present on CI, and cleaned up
      pre-commit).
    - PYTHON/bandit: fixed by this commit.
    - PYTHON/flake8: fixed by this commit.
---
 .../skill-finder/scripts/find_install.py      | 40 +++++++++++--------
 .../skill-finder/scripts/list_skills.py       | 11 +++--
 2 files changed, 29 insertions(+), 22 deletions(-)

diff --git a/references/apigee-skills-serving/skills/skill-finder/scripts/find_install.py b/references/apigee-skills-serving/skills/skill-finder/scripts/find_install.py
index 569cfdfe..3389a7a0 100644
--- a/references/apigee-skills-serving/skills/skill-finder/scripts/find_install.py
+++ b/references/apigee-skills-serving/skills/skill-finder/scripts/find_install.py
@@ -113,9 +113,13 @@
 # namespace.
 del _VENV_DEPS, _missing_dep, _dep
 
-import yaml
-from cryptography.exceptions import InvalidSignature
-from cryptography.hazmat.primitives.asymmetric.ed25519 import (
+# These imports are deferred past the venv-guard block above so a
+# missing dep surfaces the operator-facing message rather than a
+# generic ModuleNotFoundError traceback. flake8 E402 is silenced
+# by design.
+import yaml  # noqa: E402
+from cryptography.exceptions import InvalidSignature  # noqa: E402
+from cryptography.hazmat.primitives.asymmetric.ed25519 import (  # noqa: E402
     Ed25519PublicKey,
 )
 
@@ -161,7 +165,7 @@
     from common.watcher_probe import detect_watcher, WatcherState
     from common.manifest_schema import validate_manifest
     from common.iam_preflight import iam_preflight
-    from common.http_retry import http_get_retry, http_post_retry
+    from common.http_retry import http_get_retry
     from common import config
 except ImportError:
     # Dev / test form (used when invoked from the repo root with
@@ -175,14 +179,15 @@
     from scripts.common.watcher_probe import detect_watcher, WatcherState
     from scripts.common.manifest_schema import validate_manifest
     from scripts.common.iam_preflight import iam_preflight
-    from scripts.common.http_retry import http_get_retry, http_post_retry
+    from scripts.common.http_retry import http_get_retry
     from scripts.common import config
 
 # google.auth is imported lazily inside _creds() so test code that
 # never touches the network does not have to stub it at import
 # time. requests is imported here because the HTTP retry helpers
 # return requests.Response objects that we need to type-check.
-import requests
+# E402 is silenced by the deferred-import pattern above.
+import requests  # noqa: E402
 
 SKILL_DIR = Path(__file__).resolve().parent.parent
 # Multi-key trust root: every PEM under keys/ is a trusted
@@ -435,7 +440,7 @@ def result(self, timeout: float = 10.0) -> WatcherState:
             return WatcherState.WATCHER_UNDETECTABLE
         if self._exc is not None:
             return WatcherState.WATCHER_UNDETECTABLE
-        assert self._state is not None
+        assert self._state is not None  # nosec B101 - defensive invariant
         return self._state
 
 
@@ -562,7 +567,10 @@ def _search(
     # "policies" matches keyword "policy". Apigee is the demo
     # punchline; reasonable to over-fit a little.
     raw_tokens = set(query.lower().split())
-    q_tokens = raw_tokens | {t.rstrip("s") for t in raw_tokens if t.endswith("s") and len(t) > 3}
+    q_tokens = raw_tokens | {
+        t.rstrip("s") for t in raw_tokens
+        if t.endswith("s") and len(t) > 3
+    }
 
     # API hub returns keywords as nested
     # {FQ-attribute-name: {stringValues: {values: [...]}}}.
@@ -781,8 +789,8 @@ def _iam_preflight_with_contract(
         # documented case per the library docstring; emit the
         # contract line with that assumption.
         _die(
-            f"IAM pre-flight: FAILED — HTTP 200 non-JSON body "
-            f"from testIamPermissions; install aborted.",
+            "IAM pre-flight: FAILED — HTTP 200 non-JSON body "
+            "from testIamPermissions; install aborted.",
             code=3,
         )
     # Defensive: any unrecognised status string is a library bug.
@@ -804,7 +812,7 @@ def _download_zip(gs_uri: str) -> bytes:
     No ADC scope -- the bucket is public-read. We still
     route through http_get_retry so 5xx surfaces the same
     transient-failure line as the authenticated paths."""
-    assert gs_uri.startswith("gs://")
+    assert gs_uri.startswith("gs://")  # nosec B101 - caller invariant
     bucket, _, obj = gs_uri[5:].partition("/")
     url = f"https://storage.googleapis.com/{bucket}/{obj}"
     try:
@@ -1103,11 +1111,11 @@ def _validate_query(query: str) -> None:
         or any(c in _SHELL_METACHARS for c in query)
     ):
         _die(
-            f"config: FAILED — query contains disallowed "
-            f"characters (non-printable, newline, null, shell "
-            f"metachar, or length > 500). Refusing to proceed to "
-            f"limit shell-injection blast radius if SKILL.md "
-            f"expansion was unsafe.",
+            "config: FAILED — query contains disallowed "
+            "characters (non-printable, newline, null, shell "
+            "metachar, or length > 500). Refusing to proceed to "
+            "limit shell-injection blast radius if SKILL.md "
+            "expansion was unsafe.",
             code=2,
         )
 
diff --git a/references/apigee-skills-serving/skills/skill-finder/scripts/list_skills.py b/references/apigee-skills-serving/skills/skill-finder/scripts/list_skills.py
index 456f3c67..b2ca4eca 100644
--- a/references/apigee-skills-serving/skills/skill-finder/scripts/list_skills.py
+++ b/references/apigee-skills-serving/skills/skill-finder/scripts/list_skills.py
@@ -42,7 +42,6 @@
 
 import argparse
 import json
-import os
 import sys
 from pathlib import Path
 from urllib.parse import quote_plus
@@ -192,16 +191,16 @@ def _validate_args(ns: argparse.Namespace) -> None:
     pt = ns.page_token or ""
     if len(pt) > 2048:
         _die(
-            f"config: FAILED — --page-token too long "
-            f"(>2048 chars). Truncated or corrupted token?",
+            "config: FAILED — --page-token too long "
+            "(>2048 chars). Truncated or corrupted token?",
             code=2,
         )
     for c in pt:
         if ord(c) < 0x20 or ord(c) == 0x7f:
             _die(
-                f"config: FAILED — --page-token contains a "
-                f"non-printable character. Token corrupted in "
-                f"transit?",
+                "config: FAILED — --page-token contains a "
+                "non-printable character. Token corrupted in "
+                "transit?",
                 code=2,
             )
 

From 1b4d71f6b3e4d0db6598ba86ba00def180905d0f Mon Sep 17 00:00:00 2001
From: Geir Sjurseth <gsjurseth@google.com>
Date: Tue, 7 Jul 2026 09:51:03 +0000
Subject: [PATCH 7/7] test(apigee-skills-serving): add multi-key trust-root
 unit tests

17 new hermetic pytest cases covering the two functions that
changed when skill-finder moved from a single-key trust root
(keys/trusted_pubkey.pem, hardcoded path) to a multi-key model
(keys/*.pem, glob load): _load_pubkeys and _verify_signature.

Coverage matrix
---------------

  _load_pubkeys (9 tests)
    * missing keys/ directory              -> hard fail
    * empty keys/ directory                -> hard fail
    * keys/ with only dotfiles             -> hard fail
    * keys/ with only README.md            -> hard fail (suffix filter)
    * single PEM                           -> loads, dict size 1
    * two distinct PEMs                    -> loads, dict size 2
    * duplicate key material under two     -> dedupes with warning
      filenames
    * unparseable PEM alongside good PEM   -> skips bad with warning,
                                              loads good
    * raw 32-byte key (.raw suffix)        -> loads alongside PEMs

  _verify_signature (8 tests)
    * single trust root, matching key      -> verifies OK
    * two trust roots, manifest signed by  -> verifies via A
      key A
    * two trust roots, manifest signed by  -> verifies via B
      key B                                    (order-independent)
    * key-id mismatch (untrusted signer)   -> "key-id check: FAILED",
                                              exits non-zero
    * key-id mismatch error message        -> lists both the declared
                                              fingerprint AND the
                                              trusted set (regression
                                              against vague errors)
    * signature field absent               -> distinct "absent" failure
    * signature field malformed base64     -> distinct "malformed"
                                              failure
    * signature signed by wrong key but    -> "ed25519 verify rejected"
      claims trusted signing_key_id           (theoretical; tests the
                                              crypto rejection path)

Design choices
--------------

  * Tests DO NOT touch the network. Every fixture is in-memory
    or under `tmp_path`. All monkeypatching is done via the
    `find_install` module fixture; no HTTP mocking needed.

  * find_install.py lives at skills/skill-finder/scripts/ rather
    than scripts/. The test file loads it via importlib rather
    than a top-level `import find_install`, so pytest doesn't
    need a custom sys.path entry in conftest.py.

  * `_load_pubkeys` and `_verify_signature` are exercised
    directly rather than through the full find_install main().
    The multi-key logic is self-contained in those two functions;
    testing them in isolation gives a targeted, fast (0.64s)
    verification.

  * Test names read as behavioural specs ("empty_keys_dir_fails",
    "two_valid_keys_first_matches"), matching the style of the
    existing test suite (test_sign_verify_integration.py etc.).

Test count before this commit: 220.
Test count after: 237 (+17). All PASS in 1.4s in-process,
including as `uid=0(root)` inside a python:3.13-slim container
(matches the Cloud Build environment where the previous PR
had a root-vs-user regression).
---
 .../tests/test_find_install_multikey.py       | 473 ++++++++++++++++++
 1 file changed, 473 insertions(+)
 create mode 100644 references/apigee-skills-serving/tests/test_find_install_multikey.py

diff --git a/references/apigee-skills-serving/tests/test_find_install_multikey.py b/references/apigee-skills-serving/tests/test_find_install_multikey.py
new file mode 100644
index 00000000..acedf6d9
--- /dev/null
+++ b/references/apigee-skills-serving/tests/test_find_install_multikey.py
@@ -0,0 +1,473 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+"""Unit tests for skill-finder's multi-key trust root loading and
+signature verification.
+
+These tests exercise the two functions that changed when we moved
+skill-finder from a single-key (keys/trusted_pubkey.pem) to a
+multi-key (keys/*.pem) trust model:
+
+  * ``_load_pubkeys``  -- reads every PEM under ``KEYS_DIR`` and
+                          returns ``dict[fingerprint -> Ed25519PublicKey]``.
+  * ``_verify_signature`` -- accepts a manifest and the dict from
+                             ``_load_pubkeys``, matches
+                             ``signing_key_id`` against ANY installed
+                             fingerprint, verifies ed25519.
+
+The tests deliberately do NOT touch the network. Every fixture is
+in-memory or in ``tmp_path``. We patch ``find_install.KEYS_DIR``
+via monkeypatch so the module reads test-owned key directories
+without needing to munge the install layout.
+
+Coverage matrix:
+
+  * empty keys/ dir  -> "trust root: FAILED" + non-zero exit
+  * missing keys/    -> "trust root: FAILED" + non-zero exit
+  * single valid key -> loads OK, verifies OK
+  * two valid keys, manifest matches key A -> verifies via key A
+  * two valid keys, manifest matches key B -> verifies via key B
+    (both keys usable, ordering independent)
+  * key-id mismatch  -> "key-id check: FAILED" + trusted list in message
+  * unreadable / non-PEM file under keys/ -> warned + skipped
+  * duplicate keys under different filenames -> deduped, no error
+  * dotfile (``.gitkeep``) under keys/ -> ignored silently
+  * README.md under keys/ -> ignored silently (not a .pem / .raw)
+  * signature-field-absent   -> "manifest signature: FAILED" (absent)
+  * signature-malformed-b64  -> "manifest signature: FAILED" (malformed)
+  * signature-doesnt-verify  -> "manifest signature: FAILED" (rejected)
+"""
+from __future__ import annotations
+
+import base64
+import hashlib
+import importlib.util
+import sys
+from pathlib import Path
+
+import pytest
+import yaml
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+
+
+# The skill-finder scripts dir isn't on sys.path by default (tests
+# live at the repo root); load find_install.py explicitly. We do
+# this once per session so all tests share the module object.
+_SCRIPTS_DIR = (
+    Path(__file__).resolve().parent.parent
+    / "skills" / "skill-finder" / "scripts"
+)
+_FIND_INSTALL_PY = _SCRIPTS_DIR / "find_install.py"
+
+
+@pytest.fixture(scope="module")
+def find_install():
+    """Load skill-finder's find_install.py as a module.
+
+    We use importlib rather than a top-level `import find_install`
+    because the script lives under `skills/skill-finder/scripts/`
+    and pytest doesn't put that on sys.path automatically. The
+    sys.path.insert in find_install.py itself handles the
+    `from common.X` production imports; the module's ImportError
+    fallback to `from scripts.common.X` is what actually resolves
+    when tests run (because conftest.py adds the repo root).
+    """
+    spec = importlib.util.spec_from_file_location(
+        "find_install", _FIND_INSTALL_PY
+    )
+    module = importlib.util.module_from_spec(spec)
+    sys.modules["find_install"] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+# --------- key generation helpers -------------------------------
+
+
+def _mk_keypair():
+    """Generate a fresh ed25519 keypair; return
+    (private_bytes_raw, public_bytes_raw, fingerprint_str)."""
+    priv = Ed25519PrivateKey.generate()
+    priv_raw = priv.private_bytes(
+        encoding=serialization.Encoding.Raw,
+        format=serialization.PrivateFormat.Raw,
+        encryption_algorithm=serialization.NoEncryption(),
+    )
+    pub = priv.public_key()
+    pub_raw = pub.public_bytes(
+        encoding=serialization.Encoding.Raw,
+        format=serialization.PublicFormat.Raw,
+    )
+    fp = "sha256:" + hashlib.sha256(pub_raw).hexdigest()
+    return priv, priv_raw, pub, pub_raw, fp
+
+
+def _write_pem(keys_dir: Path, pub, name: str) -> Path:
+    """Write a PEM public key file into keys_dir. Returns the
+    full path. The filename shape is arbitrary here (tests exercise
+    both fingerprint-named and human-named files)."""
+    pem = pub.public_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PublicFormat.SubjectPublicKeyInfo,
+    )
+    p = keys_dir / name
+    p.write_bytes(pem)
+    return p
+
+
+def _sign_manifest(priv, manifest: dict, canonicalize) -> dict:
+    """Return a signed copy of the manifest. Uses find_install's
+    canonicalize function (aliased through the module fixture)
+    so signature bytes are identical to production."""
+    pub = priv.public_key()
+    pub_raw = pub.public_bytes(
+        encoding=serialization.Encoding.Raw,
+        format=serialization.PublicFormat.Raw,
+    )
+    fp = "sha256:" + hashlib.sha256(pub_raw).hexdigest()
+    m = dict(manifest)
+    m["signing_key_id"] = fp
+    # canonicalize excludes 'signature' itself; safe to leave it
+    # out and let canonicalize compute over the rest.
+    m.pop("signature", None)
+    canonical = canonicalize(m)
+    sig = priv.sign(canonical)
+    m["signature"] = base64.b64encode(sig).decode("ascii")
+    return m
+
+
+@pytest.fixture
+def valid_manifest_dict():
+    """A minimally-valid manifest that passes the schema (used as
+    the input to _sign_manifest). Signature/zip_sha256/gs_uri
+    values are placeholders for the sign step."""
+    return {
+        "manifest_schema_version": "1",
+        "name": "test-skill",
+        "version": "1.0.0",
+        "description": "A test skill for multi-key trust root tests.",
+        "keywords": ["test"],
+        "author": "test-author",
+        "license": "Apache-2.0",
+        "gs_uri": "gs://demo-bucket/test-skill-1.0.0.skill",
+        "zip_sha256": "0" * 64,
+    }
+
+
+# ===================================================================
+# _load_pubkeys tests
+# ===================================================================
+
+
+class TestLoadPubkeys:
+    """Cover the trust-root loading behaviour."""
+
+    def test_missing_keys_dir_fails(
+        self, find_install, tmp_path, monkeypatch
+    ):
+        """No keys/ directory at all -> hard fail, non-zero exit."""
+        monkeypatch.setattr(
+            find_install, "KEYS_DIR", tmp_path / "does-not-exist"
+        )
+        with pytest.raises(SystemExit) as excinfo:
+            find_install._load_pubkeys()
+        assert excinfo.value.code != 0
+
+    def test_empty_keys_dir_fails(
+        self, find_install, tmp_path, monkeypatch
+    ):
+        """Empty keys/ directory -> hard fail."""
+        keys = tmp_path / "keys"
+        keys.mkdir()
+        monkeypatch.setattr(find_install, "KEYS_DIR", keys)
+        with pytest.raises(SystemExit) as excinfo:
+            find_install._load_pubkeys()
+        assert excinfo.value.code != 0
+
+    def test_keys_dir_with_only_dotfiles_fails(
+        self, find_install, tmp_path, monkeypatch
+    ):
+        """.gitkeep and .anything-else are ignored -> empty ->
+        hard fail. Verifies that the dotfile skip works AND that
+        it doesn't fall through to a false-positive load."""
+        keys = tmp_path / "keys"
+        keys.mkdir()
+        (keys / ".gitkeep").write_bytes(b"")
+        (keys / ".hidden.pem").write_bytes(b"not real")
+        monkeypatch.setattr(find_install, "KEYS_DIR", keys)
+        with pytest.raises(SystemExit):
+            find_install._load_pubkeys()
+
+    def test_keys_dir_with_only_readme_fails(
+        self, find_install, tmp_path, monkeypatch
+    ):
+        """README.md exists but no .pem -> hard fail. Verifies the
+        suffix filter."""
+        keys = tmp_path / "keys"
+        keys.mkdir()
+        (keys / "README.md").write_text("just docs")
+        monkeypatch.setattr(find_install, "KEYS_DIR", keys)
+        with pytest.raises(SystemExit):
+            find_install._load_pubkeys()
+
+    def test_single_valid_key_loads(
+        self, find_install, tmp_path, monkeypatch
+    ):
+        """One PEM under keys/ -> loads OK, returns dict of size 1."""
+        keys = tmp_path / "keys"
+        keys.mkdir()
+        _, _, pub, _, fp = _mk_keypair()
+        _write_pem(keys, pub, "test.pem")
+        monkeypatch.setattr(find_install, "KEYS_DIR", keys)
+        trusted = find_install._load_pubkeys()
+        assert len(trusted) == 1
+        assert fp in trusted
+
+    def test_two_valid_keys_load(
+        self, find_install, tmp_path, monkeypatch
+    ):
+        """Two distinct PEMs -> both loaded under their own
+        fingerprints. Verifies multi-publisher / rotation
+        support at load time."""
+        keys = tmp_path / "keys"
+        keys.mkdir()
+        _, _, pub_a, _, fp_a = _mk_keypair()
+        _, _, pub_b, _, fp_b = _mk_keypair()
+        _write_pem(keys, pub_a, "publisher-a.pem")
+        _write_pem(keys, pub_b, "publisher-b.pem")
+        monkeypatch.setattr(find_install, "KEYS_DIR", keys)
+        trusted = find_install._load_pubkeys()
+        assert len(trusted) == 2
+        assert fp_a in trusted
+        assert fp_b in trusted
+        assert fp_a != fp_b  # sanity
+
+    def test_duplicate_key_material_deduped(
+        self, find_install, tmp_path, monkeypatch, capsys
+    ):
+        """Same key material under two filenames -> one entry in
+        the trusted dict. Warning is emitted (via _say to stdout)
+        but the load succeeds."""
+        keys = tmp_path / "keys"
+        keys.mkdir()
+        _, _, pub, _, fp = _mk_keypair()
+        _write_pem(keys, pub, "first.pem")
+        _write_pem(keys, pub, "second.pem")
+        monkeypatch.setattr(find_install, "KEYS_DIR", keys)
+        trusted = find_install._load_pubkeys()
+        assert len(trusted) == 1
+        captured = capsys.readouterr()
+        # The warning goes to stdout via _say(), which prefixes
+        # with "[skill-finder]".
+        assert "duplicate" in captured.out.lower()
+
+    def test_unparseable_pem_skipped(
+        self, find_install, tmp_path, monkeypatch, capsys
+    ):
+        """Garbage file under keys/*.pem is skipped with a warning
+        but doesn't abort the load. Real keys still load."""
+        keys = tmp_path / "keys"
+        keys.mkdir()
+        (keys / "garbage.pem").write_bytes(b"not a valid PEM")
+        _, _, pub, _, fp = _mk_keypair()
+        _write_pem(keys, pub, "real.pem")
+        monkeypatch.setattr(find_install, "KEYS_DIR", keys)
+        trusted = find_install._load_pubkeys()
+        assert fp in trusted
+        assert len(trusted) == 1  # garbage.pem was skipped
+        captured = capsys.readouterr()
+        assert "skipped" in captured.out.lower()
+
+    def test_raw_key_supported_alongside_pem(
+        self, find_install, tmp_path, monkeypatch
+    ):
+        """Legacy 32-byte raw key files under keys/*.raw are
+        loaded alongside PEMs."""
+        keys = tmp_path / "keys"
+        keys.mkdir()
+        _, _, pub, pub_raw, fp = _mk_keypair()
+        (keys / "raw-key.raw").write_bytes(pub_raw)
+        monkeypatch.setattr(find_install, "KEYS_DIR", keys)
+        trusted = find_install._load_pubkeys()
+        assert fp in trusted
+
+
+# ===================================================================
+# _verify_signature tests
+# ===================================================================
+
+
+class TestVerifySignature:
+    """Cover manifest-signature verification against the multi-key
+    trust root."""
+
+    def test_matching_key_verifies(
+        self, find_install, valid_manifest_dict
+    ):
+        """Single trust root, manifest signed by the matching key
+        -> verification passes, canonicalized manifest returned."""
+        priv, _, pub, _, fp = _mk_keypair()
+        trusted = {fp: pub}
+        signed = _sign_manifest(
+            priv, valid_manifest_dict, find_install.canonicalize
+        )
+        result = find_install._verify_signature(
+            yaml.safe_dump(signed), trusted
+        )
+        assert result["name"] == "test-skill"
+        assert result["signing_key_id"] == fp
+
+    def test_two_trusted_keys_first_matches(
+        self, find_install, valid_manifest_dict
+    ):
+        """Two trust roots installed, manifest signed by key A.
+        The verify loop picks key A, verifies, returns the
+        parsed manifest."""
+        priv_a, _, pub_a, _, fp_a = _mk_keypair()
+        _, _, pub_b, _, fp_b = _mk_keypair()
+        trusted = {fp_a: pub_a, fp_b: pub_b}
+        signed = _sign_manifest(
+            priv_a, valid_manifest_dict, find_install.canonicalize
+        )
+        result = find_install._verify_signature(
+            yaml.safe_dump(signed), trusted
+        )
+        assert result["signing_key_id"] == fp_a
+
+    def test_two_trusted_keys_second_matches(
+        self, find_install, valid_manifest_dict
+    ):
+        """Same scenario but the manifest is signed by key B.
+        Verify still passes -- confirms the trusted dict is a
+        true set, not order-dependent."""
+        _, _, pub_a, _, fp_a = _mk_keypair()
+        priv_b, _, pub_b, _, fp_b = _mk_keypair()
+        trusted = {fp_a: pub_a, fp_b: pub_b}
+        signed = _sign_manifest(
+            priv_b, valid_manifest_dict, find_install.canonicalize
+        )
+        result = find_install._verify_signature(
+            yaml.safe_dump(signed), trusted
+        )
+        assert result["signing_key_id"] == fp_b
+
+    def test_key_id_mismatch_fails(
+        self, find_install, valid_manifest_dict
+    ):
+        """Manifest is signed by a key that's not in the trust
+        set -> fails with 'key-id check: FAILED' and non-zero
+        exit code."""
+        # Two DIFFERENT keys: sign with priv_untrusted, but
+        # skill-finder only trusts pub_trusted.
+        _, _, pub_trusted, _, fp_trusted = _mk_keypair()
+        priv_untrusted, _, _, _, _ = _mk_keypair()
+        trusted = {fp_trusted: pub_trusted}
+        signed = _sign_manifest(
+            priv_untrusted,
+            valid_manifest_dict,
+            find_install.canonicalize,
+        )
+        with pytest.raises(SystemExit) as excinfo:
+            find_install._verify_signature(
+                yaml.safe_dump(signed), trusted
+            )
+        assert excinfo.value.code != 0
+
+    def test_key_id_error_message_lists_trusted_fingerprints(
+        self, find_install, valid_manifest_dict, capsys
+    ):
+        """The 'key-id check: FAILED' line should include the list
+        of trusted fingerprints so an operator can diagnose which
+        key is missing from keys/. Regression test against a
+        vague error message."""
+        _, _, pub_trusted, _, fp_trusted = _mk_keypair()
+        priv_untrusted, _, _, _, fp_untrusted = _mk_keypair()
+        trusted = {fp_trusted: pub_trusted}
+        signed = _sign_manifest(
+            priv_untrusted,
+            valid_manifest_dict,
+            find_install.canonicalize,
+        )
+        with pytest.raises(SystemExit):
+            find_install._verify_signature(
+                yaml.safe_dump(signed), trusted
+            )
+        captured = capsys.readouterr()
+        # Error prints via _say() to stdout with the
+        # [skill-finder] prefix. Both the manifest's declared fp
+        # and the trusted set should be visible.
+        combined = captured.out + captured.err
+        assert fp_untrusted in combined
+        assert fp_trusted in combined
+
+    def test_signature_field_absent_fails(
+        self, find_install, valid_manifest_dict
+    ):
+        """Manifest with a matching signing_key_id but NO
+        signature field -> 'manifest signature: FAILED'."""
+        _, _, pub, _, fp = _mk_keypair()
+        trusted = {fp: pub}
+        # Construct a manifest that has signing_key_id but no
+        # signature. This tests the missing-field branch that
+        # was added to distinguish 'absent' from 'malformed'
+        # from 'crypto rejected'.
+        m = dict(valid_manifest_dict)
+        m["signing_key_id"] = fp
+        assert "signature" not in m
+        with pytest.raises(SystemExit) as excinfo:
+            find_install._verify_signature(
+                yaml.safe_dump(m), trusted
+            )
+        assert excinfo.value.code != 0
+
+    def test_signature_field_malformed_base64_fails(
+        self, find_install, valid_manifest_dict
+    ):
+        """Signature field is not valid base64 -> 'malformed
+        base64' branch."""
+        _, _, pub, _, fp = _mk_keypair()
+        trusted = {fp: pub}
+        m = dict(valid_manifest_dict)
+        m["signing_key_id"] = fp
+        m["signature"] = "!!! not base64 !!!"
+        with pytest.raises(SystemExit) as excinfo:
+            find_install._verify_signature(
+                yaml.safe_dump(m), trusted
+            )
+        assert excinfo.value.code != 0
+
+    def test_signature_wrong_key_fails_crypto(
+        self, find_install, valid_manifest_dict
+    ):
+        """Signature was made by a different private key than the
+        pubkey the trust root has for the same fingerprint. In
+        practice this is unreachable (the fingerprint IS
+        derived from the pubkey) but we test the code path
+        anyway by manually crafting a mismatched manifest."""
+        priv_a, _, pub_a, _, fp_a = _mk_keypair()
+        priv_b, _, _, _, _ = _mk_keypair()
+        trusted = {fp_a: pub_a}
+        # Sign with priv_b but claim signing_key_id = fp_a
+        m = dict(valid_manifest_dict)
+        m["signing_key_id"] = fp_a
+        m.pop("signature", None)
+        canonical = find_install.canonicalize(m)
+        sig = priv_b.sign(canonical)
+        m["signature"] = base64.b64encode(sig).decode("ascii")
+        with pytest.raises(SystemExit) as excinfo:
+            find_install._verify_signature(
+                yaml.safe_dump(m), trusted
+            )
+        assert excinfo.value.code != 0