From 0f967b7f9896fbedc108ac64147db7ee0c5ebc28 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 22 Jul 2026 21:12:56 -0600 Subject: [PATCH 1/4] feat(skills): add knowledge adoption workflow --- package.json | 4 +- scripts/check-skills.mjs | 57 +++++++++++++ scripts/verify-package.mjs | 9 +- skills/build-with-agent-knowledge/SKILL.md | 97 ++++++++++++++++++++++ 4 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 scripts/check-skills.mjs create mode 100644 skills/build-with-agent-knowledge/SKILL.md diff --git a/package.json b/package.json index d4f9c32..26b09b3 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,7 @@ "files": [ "dist", "docs", + "skills", "AGENTS.md", "CHANGELOG.md", "README.md" @@ -67,7 +68,8 @@ "typecheck": "tsc --noEmit", "lint": "biome check src tests", "format": "biome format --write src tests", - "verify:package": "node scripts/verify-package.mjs" + "check:skills": "node scripts/check-skills.mjs", + "verify:package": "pnpm run check:skills && node scripts/verify-package.mjs" }, "dependencies": { "@tangle-network/agent-eval": "^0.122.8", diff --git a/scripts/check-skills.mjs b/scripts/check-skills.mjs new file mode 100644 index 0000000..132f0f6 --- /dev/null +++ b/scripts/check-skills.mjs @@ -0,0 +1,57 @@ +import { readdirSync, readFileSync, statSync } from 'node:fs' +import { join } from 'node:path' + +const root = new URL('../skills/', import.meta.url) +const maxDescriptionChars = 96 +const maxSkillBytes = 20_000 +const errors = [] +let descriptionChars = 0 + +for (const entry of readdirSync(root, { withFileTypes: true })) { + if (!entry.isDirectory()) continue + + const path = join(root.pathname, entry.name, 'SKILL.md') + if (!statSync(path, { throwIfNoEntry: false })?.isFile()) continue + + const content = readFileSync(path, 'utf8') + const frontmatter = content.match(/^---\n([\s\S]*?)\n---(?:\n|$)/)?.[1] + if (!frontmatter) { + errors.push(`${entry.name}: missing YAML frontmatter`) + continue + } + + const name = frontmatter.match(/^name:\s*["']?([^"'\n]+)["']?$/m)?.[1] + const rawDescription = frontmatter.match(/^description:\s*(.+)$/m)?.[1] + const description = rawDescription?.replace(/^["']|["']$/g, '') + + if (name !== entry.name) { + errors.push(`${entry.name}: frontmatter name is ${JSON.stringify(name)}`) + } + if (!description) { + errors.push(`${entry.name}: description is missing`) + } else { + descriptionChars += description.length + if (description.length > maxDescriptionChars) { + errors.push( + `${entry.name}: description has ${description.length} chars; max is ${maxDescriptionChars}`, + ) + } + } + if (Buffer.byteLength(content) > maxSkillBytes) { + errors.push( + `${entry.name}: SKILL.md has ${Buffer.byteLength(content)} bytes; max is ${maxSkillBytes}`, + ) + } + + const footer = content.lastIndexOf('\n## Then consider\n') + if (footer === -1 || content.indexOf('\n## ', footer + 1) !== -1) { + errors.push(`${entry.name}: ## Then consider must be the final level-two section`) + } +} + +if (errors.length > 0) { + for (const error of errors) console.error(error) + process.exitCode = 1 +} else { + console.log(`skills valid: ${descriptionChars} description chars`) +} diff --git a/scripts/verify-package.mjs b/scripts/verify-package.mjs index 5bc8b9d..f5c5a9e 100644 --- a/scripts/verify-package.mjs +++ b/scripts/verify-package.mjs @@ -64,6 +64,13 @@ try { const installedPackage = JSON.parse( readFileSync(join(installedPackageDir, 'package.json'), 'utf8'), ) + const installedSkill = readFileSync( + join(installedPackageDir, 'skills', 'build-with-agent-knowledge', 'SKILL.md'), + 'utf8', + ) + if (!installedSkill.includes('name: build-with-agent-knowledge')) { + throw new Error('published package is missing the build-with-agent-knowledge skill') + } run( process.execPath, @@ -97,7 +104,7 @@ try { onlyTarball(repackDir) process.stdout.write( - `Verified ${packageName}@${installedPackage.version}: clean install, ${publicImports.length} imports, CLI version, and re-pack.\n`, + `Verified ${packageName}@${installedPackage.version}: clean install, ${publicImports.length} imports, skill, CLI version, and re-pack.\n`, ) } finally { rmSync(tempRoot, { recursive: true, force: true }) diff --git a/skills/build-with-agent-knowledge/SKILL.md b/skills/build-with-agent-knowledge/SKILL.md new file mode 100644 index 0000000..43b1ef7 --- /dev/null +++ b/skills/build-with-agent-knowledge/SKILL.md @@ -0,0 +1,97 @@ +--- +name: build-with-agent-knowledge +description: Build, evaluate, and improve source-backed knowledge, retrieval, and memory systems. +--- + +# Build With Agent Knowledge + +Use this when a product needs a knowledge base, retrieval system, RAG improvement process, or agent memory provider. +Read the installed package README, exports, types, and nearest tests before choosing an API. +Do not copy signatures from this skill. + +## Choose The Job + +| Product need | Package capability | +|---|---| +| Start a source-backed Markdown knowledge base | File knowledge base and source registry | +| Search an existing package knowledge base | File search provider or a product search adapter | +| Improve knowledge without editing live files | Isolated knowledge candidates | +| Tune retrieval on labeled questions | Retrieval improvement loop | +| Diagnose and repair retrieval, sources, pages, and answers together | RAG knowledge improvement loop | +| Compare memory systems | Memory adapter and experiment APIs | +| Run retrieval, answer, knowledge, or memory cases | Knowledge benchmark APIs | +| Let agents research or edit candidates | Runtime knowledge integration | + +Use the narrowest capability that solves the product problem. +Do not add an agent loop when deterministic ingestion or indexing is enough. + +## Define Truth And Success + +Record: + +- the user task the knowledge should improve; +- authoritative and disallowed sources; +- tenant, user, agent, and sharing scope; +- freshness and deletion requirements; +- the current retrieval, answer, or memory baseline; +- objective checks, semantic checks, cost, and latency limits; +- who may approve and apply a candidate. + +Use independent labels and source evidence. +The agent's current answer is not a gold answer. + +## Keep The Boundary Clean + +`agent-knowledge` owns source records, indexes, retrieval tests, memory contracts, isolated candidates, and exact promotion. +It does not own model choice, prompts, browsing, agent scheduling, product authorization, or product storage transactions. + +Supply callbacks for research, retrieval, answer generation, and scoring. +Use `@tangle-network/agent-runtime/knowledge` when those callbacks should run agents. +Use existing vector, graph, search, and memory systems through adapters instead of rebuilding their databases here. + +## Build The Smallest Complete Path + +1. Ingest one real source with provenance and tenant scope. +2. Build or connect the index used by the product. +3. Run one representative query or memory sequence through the production path. +4. Capture retrieved items, final answer or action, citations, errors, tokens, cost, and latency. +5. Prove a known good case passes and a realistic unsupported or missed case fails. +6. Add only the missing improvement step: retrieval search, source acquisition, page update, answer repair, or memory policy. +7. Write changes to an isolated candidate with a stable run identity. +8. Compare baseline and candidate on the same development cases, then on unseen cases. +9. Apply only an approved candidate whose base identity is still current. + +Use separate run IDs for parallel branches and the same run ID to resume interrupted work. +Reject stale promotion rather than replacing newer knowledge. + +## Evaluate The Right Layer + +| Layer | Minimum evidence | +|---|---| +| Retrieval | Labeled relevant items, ranking measures, misses, latency, and cost | +| Answer | Claim support, relevance, citation correctness, abstention, and final text | +| Knowledge base | Source coverage, provenance, freshness, structure, conflicts, and validation findings | +| Memory | Multi-turn task outcome, correct recall, harmful recall, isolation, writes, latency, and cost | + +Report service and measurement failures separately from product failures. +Keep candidate-generation cases separate from the final decision set. +Bundled benchmark samples prove adapter wiring only; use complete external datasets for benchmark claims. + +## Completion + +One customer-like path must prove: + +```text +source or memory event -> production retrieval -> observable answer or action +-> isolated candidate -> baseline comparison -> unseen comparison +-> approved promotion or correctly rejected change -> reproducible rerun +``` + +Report installed versions, exact imports, provider adapters, scope policy, case counts, baseline and candidate results, cost, latency, candidate identity, promotion result, and artifact paths. + +## Then consider + +- `eval-engineering` when new production-derived cases are needed. +- `build-with-agent-runtime` when agents should research, edit, or compare candidates. +- `agent-eval-adoption` when the product needs shared comparison and release records. +- `harden` when changing tenant isolation, source trust, deletion, or promotion authority. From 6fc150a6b41543f5935bc1217d31293eecfb670c Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 22 Jul 2026 21:32:00 -0600 Subject: [PATCH 2/4] fix(skills): harden package checks --- scripts/check-skills.mjs | 20 +++++--- tests/skill-check.test.ts | 96 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 6 deletions(-) create mode 100644 tests/skill-check.test.ts diff --git a/scripts/check-skills.mjs b/scripts/check-skills.mjs index 132f0f6..4c5cafb 100644 --- a/scripts/check-skills.mjs +++ b/scripts/check-skills.mjs @@ -1,19 +1,25 @@ import { readdirSync, readFileSync, statSync } from 'node:fs' -import { join } from 'node:path' +import { join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' -const root = new URL('../skills/', import.meta.url) +const root = process.env.AGENT_KNOWLEDGE_SKILLS_ROOT + ? resolve(process.env.AGENT_KNOWLEDGE_SKILLS_ROOT) + : fileURLToPath(new URL('../skills/', import.meta.url)) const maxDescriptionChars = 96 const maxSkillBytes = 20_000 const errors = [] let descriptionChars = 0 +let skillCount = 0 for (const entry of readdirSync(root, { withFileTypes: true })) { if (!entry.isDirectory()) continue - const path = join(root.pathname, entry.name, 'SKILL.md') + const path = join(root, entry.name, 'SKILL.md') if (!statSync(path, { throwIfNoEntry: false })?.isFile()) continue + skillCount += 1 - const content = readFileSync(path, 'utf8') + const rawContent = readFileSync(path, 'utf8') + const content = rawContent.replace(/\r\n?/g, '\n') const frontmatter = content.match(/^---\n([\s\S]*?)\n---(?:\n|$)/)?.[1] if (!frontmatter) { errors.push(`${entry.name}: missing YAML frontmatter`) @@ -37,9 +43,9 @@ for (const entry of readdirSync(root, { withFileTypes: true })) { ) } } - if (Buffer.byteLength(content) > maxSkillBytes) { + if (Buffer.byteLength(rawContent) > maxSkillBytes) { errors.push( - `${entry.name}: SKILL.md has ${Buffer.byteLength(content)} bytes; max is ${maxSkillBytes}`, + `${entry.name}: SKILL.md has ${Buffer.byteLength(rawContent)} bytes; max is ${maxSkillBytes}`, ) } @@ -49,6 +55,8 @@ for (const entry of readdirSync(root, { withFileTypes: true })) { } } +if (skillCount === 0) errors.push('no skills found') + if (errors.length > 0) { for (const error of errors) console.error(error) process.exitCode = 1 diff --git a/tests/skill-check.test.ts b/tests/skill-check.test.ts new file mode 100644 index 0000000..23e70b9 --- /dev/null +++ b/tests/skill-check.test.ts @@ -0,0 +1,96 @@ +import { execFile } from 'node:child_process' +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { promisify } from 'node:util' +import { afterEach, describe, expect, it } from 'vitest' + +const execFileAsync = promisify(execFile) +const roots: string[] = [] + +async function createRoot(): Promise { + const root = await mkdtemp(join(tmpdir(), 'agent-knowledge-skills-')) + roots.push(root) + return root +} + +async function writeSkill(root: string, name: string, content: string): Promise { + const directory = join(root, name) + await mkdir(directory) + await writeFile(join(directory, 'SKILL.md'), content) +} + +async function check(root: string) { + return execFileAsync(process.execPath, ['scripts/check-skills.mjs'], { + cwd: process.cwd(), + env: { ...process.env, AGENT_KNOWLEDGE_SKILLS_ROOT: root }, + }) +} + +afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))) +}) + +describe('skill package check', () => { + it('accepts valid CRLF frontmatter', async () => { + const root = await createRoot() + await writeSkill( + root, + 'valid', + [ + '---', + 'name: valid', + 'description: A compact test skill.', + '---', + '', + '# Valid', + '', + '## Then consider', + '', + '- `verify` before release.', + '', + ].join('\r\n'), + ) + + await expect(check(root)).resolves.toMatchObject({ stderr: '' }) + }) + + it('rejects an empty skills directory', async () => { + const root = await createRoot() + await expect(check(root)).rejects.toMatchObject({ + stderr: expect.stringContaining('no skills'), + }) + }) + + it.each([ + ['wrong name', 'name: different', 'frontmatter name'], + ['long description', `description: ${'x'.repeat(97)}`, 'description has 97 chars'], + [ + 'misplaced footer', + '## Then consider\n\n- `verify` before release.\n\n## Later', + 'final level-two section', + ], + ])('rejects %s', async (_label, mutation, expected) => { + const root = await createRoot() + let content = [ + '---', + 'name: invalid', + 'description: A compact test skill.', + '---', + '', + '# Invalid', + '', + '## Then consider', + '', + '- `verify` before release.', + '', + ].join('\n') + if (mutation.startsWith('name:')) content = content.replace('name: invalid', mutation) + else if (mutation.startsWith('description:')) { + content = content.replace('description: A compact test skill.', mutation) + } else content = content.replace('## Then consider\n\n- `verify` before release.', mutation) + await writeSkill(root, 'invalid', content) + + await expect(check(root)).rejects.toMatchObject({ stderr: expect.stringContaining(expected) }) + }) +}) From e698c21807ed5cbfa0fab8d5835526723f2f8b99 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 22 Jul 2026 21:35:44 -0600 Subject: [PATCH 3/4] fix(skills): validate folded descriptions --- scripts/check-skills.mjs | 21 ++++++++++++++++++--- tests/skill-check.test.ts | 1 + 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/scripts/check-skills.mjs b/scripts/check-skills.mjs index 4c5cafb..9cf1fa5 100644 --- a/scripts/check-skills.mjs +++ b/scripts/check-skills.mjs @@ -11,6 +11,22 @@ const errors = [] let descriptionChars = 0 let skillCount = 0 +function frontmatterField(frontmatter, key) { + const lines = frontmatter.split('\n') + const index = lines.findIndex((line) => line.startsWith(`${key}:`)) + if (index === -1) return undefined + + const value = lines[index].slice(key.length + 1).trim() + if (!/^[>|][0-9+-]*$/.test(value)) return value.replace(/^["']|["']$/g, '') + + const continuation = [] + for (const line of lines.slice(index + 1)) { + if (line && !/^\s/.test(line)) break + continuation.push(line.trim()) + } + return (value.startsWith('>') ? continuation.join(' ') : continuation.join('\n')).trim() +} + for (const entry of readdirSync(root, { withFileTypes: true })) { if (!entry.isDirectory()) continue @@ -26,9 +42,8 @@ for (const entry of readdirSync(root, { withFileTypes: true })) { continue } - const name = frontmatter.match(/^name:\s*["']?([^"'\n]+)["']?$/m)?.[1] - const rawDescription = frontmatter.match(/^description:\s*(.+)$/m)?.[1] - const description = rawDescription?.replace(/^["']|["']$/g, '') + const name = frontmatterField(frontmatter, 'name') + const description = frontmatterField(frontmatter, 'description') if (name !== entry.name) { errors.push(`${entry.name}: frontmatter name is ${JSON.stringify(name)}`) diff --git a/tests/skill-check.test.ts b/tests/skill-check.test.ts index 23e70b9..e743c3d 100644 --- a/tests/skill-check.test.ts +++ b/tests/skill-check.test.ts @@ -65,6 +65,7 @@ describe('skill package check', () => { it.each([ ['wrong name', 'name: different', 'frontmatter name'], ['long description', `description: ${'x'.repeat(97)}`, 'description has 97 chars'], + ['folded long description', `description: >-\n ${'x'.repeat(97)}`, 'description has 97 chars'], [ 'misplaced footer', '## Then consider\n\n- `verify` before release.\n\n## Later', From b098eddffe4bf5e1446a8dc0bfdf4ad051fc96dd Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 22 Jul 2026 21:47:27 -0600 Subject: [PATCH 4/4] fix(skills): harden installed skill verification --- scripts/check-skills.mjs | 5 +++++ scripts/verify-package.mjs | 12 ++++++++++-- tests/skill-check.test.ts | 9 +++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/scripts/check-skills.mjs b/scripts/check-skills.mjs index 9cf1fa5..7ce25f7 100644 --- a/scripts/check-skills.mjs +++ b/scripts/check-skills.mjs @@ -11,6 +11,11 @@ const errors = [] let descriptionChars = 0 let skillCount = 0 +if (!statSync(root, { throwIfNoEntry: false })?.isDirectory()) { + console.error(`skills directory is missing: ${root}`) + process.exit(1) +} + function frontmatterField(frontmatter, key) { const lines = frontmatter.split('\n') const index = lines.findIndex((line) => line.startsWith(`${key}:`)) diff --git a/scripts/verify-package.mjs b/scripts/verify-package.mjs index f5c5a9e..3664b7a 100644 --- a/scripts/verify-package.mjs +++ b/scripts/verify-package.mjs @@ -68,8 +68,16 @@ try { join(installedPackageDir, 'skills', 'build-with-agent-knowledge', 'SKILL.md'), 'utf8', ) - if (!installedSkill.includes('name: build-with-agent-knowledge')) { - throw new Error('published package is missing the build-with-agent-knowledge skill') + const installedFrontmatter = installedSkill + .replace(/\r\n?/g, '\n') + .match(/^---\n([\s\S]*?)\n---(?:\n|$)/)?.[1] + const installedSkillName = installedFrontmatter + ?.match(/^name:\s*["']?([^"'\n]+)["']?$/m)?.[1] + ?.trim() + if (installedSkillName !== 'build-with-agent-knowledge') { + throw new Error( + `published package has an invalid skill name: ${JSON.stringify(installedSkillName)}`, + ) } run( diff --git a/tests/skill-check.test.ts b/tests/skill-check.test.ts index e743c3d..6350ab7 100644 --- a/tests/skill-check.test.ts +++ b/tests/skill-check.test.ts @@ -62,6 +62,15 @@ describe('skill package check', () => { }) }) + it('rejects a missing skills directory', async () => { + const root = await createRoot() + await rm(root, { recursive: true }) + + await expect(check(root)).rejects.toMatchObject({ + stderr: expect.stringContaining('skills directory is missing'), + }) + }) + it.each([ ['wrong name', 'name: different', 'frontmatter name'], ['long description', `description: ${'x'.repeat(97)}`, 'description has 97 chars'],