From a5ac5115d09fc5b46f57a3d7b7124845cd5b8615 Mon Sep 17 00:00:00 2001 From: Offending Commit Date: Fri, 12 Jun 2026 16:04:19 -0500 Subject: [PATCH] test: contract tests against real hermes-agent source Keeps the kit from silently drifting from upstream. tests/test_hermes_contract.py registers a kit-built tool into the REAL hermes registry and asserts: - registry.get_definitions() yields a function with populated parameters (the empty-{} failure mode, checked against the actual conversion); - registry.dispatch() runs the handler and the kit's validation in-band; - register_all's calls bind to the real PluginContext.register_tool signature. Skip-guarded, so it's green standalone. Runs against upstream via a staging clone: 'make test-contract' (clones NousResearch/hermes-agent into .hermes-agent) and a CI 'hermes-contract' job that checks out upstream main. PyYAML added as a dev-only dep (runtime stays dependency-free). --- .github/workflows/test.yml | 21 +++++ .gitignore | 1 + Makefile | 12 ++- pyproject.toml | 5 ++ tests/test_hermes_contract.py | 149 ++++++++++++++++++++++++++++++++++ uv.lock | 46 +++++++++++ 6 files changed, 233 insertions(+), 1 deletion(-) create mode 100644 tests/test_hermes_contract.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bdbcfd1..16b9ffb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,3 +15,24 @@ jobs: enable-cache: true - name: Run unittest suite run: make test + + # Validate the kit against the *real* hermes-agent source so it can't silently + # drift from upstream. Checks out NousResearch/hermes-agent and points the + # contract tests at it; they skip if the import fails, so this never blocks on + # an upstream layout change — it goes red only on a genuine contract break. + hermes-contract: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/checkout@v6 + with: + repository: NousResearch/hermes-agent + path: .hermes-agent + - uses: astral-sh/setup-uv@v8.2.0 + with: + python-version: "3.13" + enable-cache: true + - name: Run hermes contract tests against upstream main + env: + HERMES_AGENT_PATH: ${{ github.workspace }}/.hermes-agent + run: uv run python -m unittest tests.test_hermes_contract -v diff --git a/.gitignore b/.gitignore index 60adc2a..036c367 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ build/ dist/ .venv/ .eggs/ +.hermes-agent/ diff --git a/Makefile b/Makefile index 43026b5..1b81e53 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,9 @@ .DEFAULT_GOAL := help UV ?= uv +HERMES_AGENT_REPO ?= https://github.com/NousResearch/hermes-agent.git +HERMES_AGENT_DIR ?= .hermes-agent -.PHONY: help install test test-one build clean +.PHONY: help install test test-one test-contract build clean help: ## Show available targets @grep -E '^[a-zA-Z_-]+:.*?## ' $(MAKEFILE_LIST) | \ @@ -16,6 +18,14 @@ test: ## Run the full unittest suite test-one: ## Run a single test: make test-one T=tests.test_kit.Class.method $(UV) run python -m unittest $(T) +test-contract: ## Clone hermes-agent into a staging dir and run the contract tests against it + @if [ -d "$(HERMES_AGENT_DIR)/.git" ]; then \ + echo "Updating $(HERMES_AGENT_DIR)"; git -C "$(HERMES_AGENT_DIR)" pull --ff-only -q || true; \ + else \ + echo "Cloning hermes-agent into $(HERMES_AGENT_DIR)"; git clone --depth 1 "$(HERMES_AGENT_REPO)" "$(HERMES_AGENT_DIR)"; \ + fi + HERMES_AGENT_PATH="$(abspath $(HERMES_AGENT_DIR))" $(UV) run python -m unittest tests.test_hermes_contract -v + build: ## Build the wheel/sdist distribution $(UV) build diff --git a/pyproject.toml b/pyproject.toml index e4b790a..1cc6a80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,5 +20,10 @@ dependencies = [] [project.urls] Repository = "https://github.com/offendingcommit/hermes-plugin-kit" +# Runtime stays dependency-free. PyYAML is dev-only: the hermes contract tests +# import hermes_cli.plugins, which transitively needs yaml. Skipped without it. +[dependency-groups] +dev = ["pyyaml"] + [tool.setuptools] packages = ["hermes_plugin_kit"] diff --git a/tests/test_hermes_contract.py b/tests/test_hermes_contract.py new file mode 100644 index 0000000..caf98b9 --- /dev/null +++ b/tests/test_hermes_contract.py @@ -0,0 +1,149 @@ +"""Contract tests against the *real* hermes-agent source. + +These keep the kit relevant: they import the genuine ``PluginContext`` and tool +``registry`` from a hermes-agent checkout and prove that what the kit emits still +satisfies the runtime contract — most importantly that a kit-built schema, run +through the registry's real OpenAI-tool conversion, yields a function whose +``parameters`` are populated (the empty-``{}`` failure mode this kit exists to +prevent). + +The whole module is skipped when hermes-agent is not importable, so the suite +stays green standalone and in public CI. Point it at a checkout with +``HERMES_AGENT_PATH`` (a CI job can ``actions/checkout`` NousResearch/hermes-agent +and set it); locally it discovers ``~/hermes-agent`` automatically. +""" + +from __future__ import annotations + +import inspect +import json +import os +import sys +import types +import unittest +from pathlib import Path + +import hermes_plugin_kit as hpk + + +def _import_real_hermes(): + """Return a namespace with the real PluginContext / VALID_HOOKS / registry, or None.""" + candidates: list[Path] = [] + env_path = os.environ.get("HERMES_AGENT_PATH") + if env_path: + candidates.append(Path(env_path)) + candidates.append(Path.home() / "hermes-agent") + candidates.append(Path.home() / ".hermes" / "hermes-agent") + + def _try(): + from hermes_cli.plugins import PluginContext, VALID_HOOKS # type: ignore + from tools.registry import registry # type: ignore + + return types.SimpleNamespace( + PluginContext=PluginContext, VALID_HOOKS=set(VALID_HOOKS), registry=registry + ) + + try: + return _try() + except Exception: + pass + + for root in candidates: + if not (root / "hermes_cli" / "plugins.py").exists(): + continue + sys.path.insert(0, str(root)) + try: + return _try() + except Exception: + continue + return None + + +_REAL = _import_real_hermes() + + +class _RecordingCtx: + def __init__(self) -> None: + self.calls: list[dict] = [] + + def register_tool(self, **kwargs) -> None: + self.calls.append(kwargs) + + +@hpk.tool( + toolset="probe", + requires_env=["PROBE_KEY"], + params={ + "channel_id_or_url": hpk.str_arg( + "A Discord channel id or URL", required=True, example="123456789012345678" + ), + "limit": hpk.int_arg("How many", minimum=1, maximum=100), + }, +) +def hpk_contract_probe(args, **kwargs): + """Probe tool used only by the hermes contract tests.""" + return {"echo": args.get("channel_id_or_url")} + + +_SPEC = getattr(hpk_contract_probe, "_hpk_tool_spec") + + +@unittest.skipUnless(_REAL is not None, "hermes-agent source not importable") +class HermesContractTests(unittest.TestCase): + """Validate the kit's output against genuine hermes-agent runtime APIs.""" + + def _register_probe(self): + reg = _REAL.registry + reg.register( + name=_SPEC["name"], + toolset=_SPEC["toolset"], + schema=_SPEC["schema"], + handler=hpk_contract_probe, + requires_env=_SPEC["requires_env"], + description=_SPEC["schema"]["description"], + emoji=_SPEC["emoji"], + ) + self.addCleanup(reg.deregister, _SPEC["name"]) + return reg + + def test_kit_schema_survives_real_registry_conversion(self) -> None: + # registry.get_definitions does the exact {**schema, "name": ...} spread the + # model receives. The kit's parameters wrapper must survive it populated. + reg = self._register_probe() + defs = reg.get_definitions({_SPEC["name"]}) + fn = next(d["function"] for d in defs if d["function"]["name"] == _SPEC["name"]) + + self.assertIn("parameters", fn, "model would receive a tool with no parameters") + props = fn["parameters"]["properties"] + self.assertTrue(props, "parameters.properties is empty — the empty-{} failure mode") + self.assertIn("channel_id_or_url", props) + self.assertEqual(fn["parameters"]["required"], ["channel_id_or_url"]) + + def test_kit_handler_dispatches_through_real_registry(self) -> None: + # The runtime calls handler(args, **kwargs) and expects a JSON string. + reg = self._register_probe() + + ok = json.loads(reg.dispatch(_SPEC["name"], {"channel_id_or_url": "999"})) + self.assertTrue(ok["success"]) + self.assertEqual(ok["data"]["echo"], "999") + + # A missing required arg fails in-band (no exception) with an instructive error. + missing = json.loads(reg.dispatch(_SPEC["name"], {})) + self.assertFalse(missing["success"]) + self.assertIn("channel_id_or_url", missing["error"]) + + def test_register_all_binds_to_real_plugincontext_signature(self) -> None: + # If hermes renames/removes a register_tool parameter the kit passes, this fails. + ctx = _RecordingCtx() + hpk.register_all(ctx, __name__) + self.assertTrue(ctx.calls) + sig = inspect.signature(_REAL.PluginContext.register_tool) + for call in ctx.calls: + try: + sig.bind(None, **call) # None stands in for self + except TypeError as exc: # pragma: no cover - failure path + self.fail(f"register_all call does not match PluginContext.register_tool: {exc}") + + +if __name__ == "__main__": + unittest.main() diff --git a/uv.lock b/uv.lock index 00416c5..f256b50 100644 --- a/uv.lock +++ b/uv.lock @@ -6,3 +6,49 @@ requires-python = ">=3.13" name = "hermes-plugin-kit" version = "0.1.0" source = { editable = "." } + +[package.dev-dependencies] +dev = [ + { name = "pyyaml" }, +] + +[package.metadata] + +[package.metadata.requires-dev] +dev = [{ name = "pyyaml" }] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +]