From b5020dd5912fa715a9f5cc81a59aed849fd0451c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=84=EC=88=98?= Date: Thu, 23 Jul 2026 19:08:57 +0900 Subject: [PATCH 1/9] =?UTF-8?q?harness:=20skill=5Fforge=20=ED=9D=A1?= =?UTF-8?q?=EC=88=98(Jermes)=20+=20skill=5Fregistry=20=EC=9B=90=EC=9E=A5?= =?UTF-8?q?=20=EC=86=8C=EC=8A=A4=20seam?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - harness/skill_forge/ 13모듈: 메모리→스킬 자가승격 루프 — signals(후보신호)→curator(안티학습·안전필터·patch우선)→synthesis(guide/ config/tool 3형)→gate(SelfForge promote 대수+결정적 재현벤치, LLM-judge 0) →ledger(semver·provenance·lineage·wilson)→recall(PD 2단)+drafter(3층복구· 앙상블·예시유출가드)+host(SpineStore 어댑터)+bench(재현벤치·자동캡처). 의존성 0, 원본 레포 engine/skill-forge-engineering(테스트 50 green). - tools/skill_registry.py: register_ledger_source — 빌트인/코드등록/ entry_points에 이은 4번째 회상 소스(원장 스킬을 get_skill_body/ list_skill_names에 폴백 병합, staged 노출은 소스 정책) 검증: 원장 스킬 회상 노출 통합테스트 + 라이브 스택 E2E(실 PG skill_def 영속, qwen-turbo/qwen3-8b/qwen3.6-27b/claude-sonnet-4-6 동일 산출) --- src/xgen_sdk/harness/skill_forge/__init__.py | 82 +++++ src/xgen_sdk/harness/skill_forge/bench.py | 130 ++++++++ src/xgen_sdk/harness/skill_forge/curator.py | 185 +++++++++++ src/xgen_sdk/harness/skill_forge/drafter.py | 299 ++++++++++++++++++ src/xgen_sdk/harness/skill_forge/gate.py | 113 +++++++ src/xgen_sdk/harness/skill_forge/host.py | 129 ++++++++ src/xgen_sdk/harness/skill_forge/ledger.py | 169 ++++++++++ src/xgen_sdk/harness/skill_forge/loop.py | 100 ++++++ src/xgen_sdk/harness/skill_forge/model.py | 186 +++++++++++ src/xgen_sdk/harness/skill_forge/recall.py | 70 ++++ src/xgen_sdk/harness/skill_forge/registry.py | 60 ++++ src/xgen_sdk/harness/skill_forge/signals.py | 122 +++++++ src/xgen_sdk/harness/skill_forge/synthesis.py | 120 +++++++ src/xgen_sdk/harness/tools/skill_registry.py | 36 ++- 14 files changed, 1799 insertions(+), 2 deletions(-) create mode 100644 src/xgen_sdk/harness/skill_forge/__init__.py create mode 100644 src/xgen_sdk/harness/skill_forge/bench.py create mode 100644 src/xgen_sdk/harness/skill_forge/curator.py create mode 100644 src/xgen_sdk/harness/skill_forge/drafter.py create mode 100644 src/xgen_sdk/harness/skill_forge/gate.py create mode 100644 src/xgen_sdk/harness/skill_forge/host.py create mode 100644 src/xgen_sdk/harness/skill_forge/ledger.py create mode 100644 src/xgen_sdk/harness/skill_forge/loop.py create mode 100644 src/xgen_sdk/harness/skill_forge/model.py create mode 100644 src/xgen_sdk/harness/skill_forge/recall.py create mode 100644 src/xgen_sdk/harness/skill_forge/registry.py create mode 100644 src/xgen_sdk/harness/skill_forge/signals.py create mode 100644 src/xgen_sdk/harness/skill_forge/synthesis.py diff --git a/src/xgen_sdk/harness/skill_forge/__init__.py b/src/xgen_sdk/harness/skill_forge/__init__.py new file mode 100644 index 0000000..228b2e6 --- /dev/null +++ b/src/xgen_sdk/harness/skill_forge/__init__.py @@ -0,0 +1,82 @@ +"""xgen-skill-forge — verified memory-to-skill promotion loop. + +SF1 signals -> SF2 curator -> SF3 synthesis -> SF4 forge gate -> SF5 ledger/recall. +Design: D:\\harness-engineering\\SKILL-FORGE-PLAN.md +""" + +from .bench import Expectation, ReplayCase, ReproReplayRunner, cases_from_repro_rows +from .curator import Curator, CurationResult, Rejection +from .drafter import ( + EnsembleDrafter, + LLMDrafter, + anthropic_completer, + openai_chat_completer, +) +from .gate import BenchCase, BenchRunner, ForgeGate, GateConfig +from .host import ( + InMemorySpineStore, + SpineSkillLedger, + SpineStore, + signature_counts, + trace_from_spine, +) +from .ledger import InMemorySkillLedger, JsonlSkillLedger, SkillLedger, SkillRecord +from .loop import ApprovalPolicy, ForgeEpisode, SkillForge +from .model import ( + GateResult, + Provenance, + RunTrace, + SkillCandidate, + SkillDef, + TraceEvent, + UsageStats, +) +from .recall import LedgerSkillSource, SkillListing +from .signals import SIGNAL_EXTRACTORS, SignalHit, extract_signals +from .synthesis import SYNTHESIZERS, synthesize + +__version__ = "0.1.0" + +__all__ = [ + "ApprovalPolicy", + "BenchCase", + "BenchRunner", + "Curator", + "CurationResult", + "EnsembleDrafter", + "Expectation", + "ForgeEpisode", + "ForgeGate", + "GateConfig", + "GateResult", + "InMemorySkillLedger", + "InMemorySpineStore", + "JsonlSkillLedger", + "LLMDrafter", + "LedgerSkillSource", + "Provenance", + "Rejection", + "ReplayCase", + "ReproReplayRunner", + "RunTrace", + "SIGNAL_EXTRACTORS", + "SYNTHESIZERS", + "SignalHit", + "SkillCandidate", + "SkillDef", + "SkillForge", + "SkillLedger", + "SkillListing", + "SkillRecord", + "SpineSkillLedger", + "SpineStore", + "TraceEvent", + "UsageStats", + "anthropic_completer", + "cases_from_repro_rows", + "extract_signals", + "openai_chat_completer", + "signature_counts", + "synthesize", + "trace_from_spine", +] diff --git a/src/xgen_sdk/harness/skill_forge/bench.py b/src/xgen_sdk/harness/skill_forge/bench.py new file mode 100644 index 0000000..ce684b1 --- /dev/null +++ b/src/xgen_sdk/harness/skill_forge/bench.py @@ -0,0 +1,130 @@ +"""Deterministic replay bench — the weak-model equalizer. + +The gate's verdict must not depend on model intelligence, so the bench judges +with code, not an LLM: each ReplayCase carries machine-checkable expectations +(substrings, regexes, forbidden markers). A weak drafter's skill survives only +if replaying real historic cases with the skill injected measurably helps. +Quality comes from selection pressure, not from the drafting model. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Callable + +from .gate import BenchCase +from .model import SkillDef + + +@dataclass +class Expectation: + require: list[str] = field(default_factory=list) # substrings that must appear + require_regex: list[str] = field(default_factory=list) + forbid: list[str] = field(default_factory=list) # markers that must not appear + + def score(self, output: str) -> float: + checks: list[bool] = [] + low = output.lower() + checks += [needle.lower() in low for needle in self.require] + checks += [re.search(pattern, output, re.IGNORECASE) is not None + for pattern in self.require_regex] + checks += [marker.lower() not in low for marker in self.forbid] + if not checks: + return 0.0 + return sum(checks) / len(checks) + + +@dataclass +class ReplayCase: + """One historic interaction worth re-running (built from a ReproBundle).""" + + case_id: str + payload: dict + expect: Expectation + + def as_bench_case(self) -> BenchCase: + return BenchCase(case_id=self.case_id, payload=self.payload) + + +RunFn = Callable[[dict, SkillDef | None], str] +"""(case payload, candidate skill or None) -> the run's final output text. +Hosts back this with a real pipeline run (skill injected via loaded_skills); +tests back it with a scripted function.""" + + +class ReproReplayRunner: + """BenchRunner over replay cases. Deterministic given a deterministic RunFn: + zero LLM-judge dependence, so verification quality is model-independent.""" + + def __init__(self, run_fn: RunFn, cases: list[ReplayCase]) -> None: + self.run_fn = run_fn + self._by_id = {case.case_id: case for case in cases} + + def bench_cases(self) -> list[BenchCase]: + return [case.as_bench_case() for case in self._by_id.values()] + + def score(self, case: BenchCase, skill: SkillDef | None) -> float: + replay = self._by_id.get(case.case_id) + if replay is None: + return 0.0 + try: + output = self.run_fn(replay.payload, skill) + except Exception: + return 0.0 + return replay.expect.score(output) + + +_ERROR_MARKER = re.compile(r"\b([45]\d\d)\b|\b(timeout|refused|denied|not found|failed)\b", + re.IGNORECASE) + + +def capture_repro_rows(trace) -> list[dict]: + """Auto-capture replay rows from a finished trace's error->recovery pairs. + + Conservative heuristic: forbid the error's distinctive marker (status code + or failure word), require the recovery detail's distinctive tokens. Rows + are tagged auto_captured so hosts can review before trusting them as gate + evidence. Returns [] when the trace has no usable error/recovery signal.""" + rows: list[dict] = [] + recovery_details = [e.detail for e in trace.events + if e.type == "recovery" and e.detail] + for index, event in enumerate(trace.events): + if event.type != "error" or not event.detail: + continue + marker = _ERROR_MARKER.search(event.detail) + if not marker: + continue + require = [] + if recovery_details: + tokens = re.findall(r"[a-zA-Z0-9?=_-]{4,}", recovery_details[0]) + require = tokens[:2] + rows.append({ + "case_id": f"{trace.run_id}-repro-{index}", + "payload": {"error_detail": event.detail, "tool": event.name, + "run_id": trace.run_id}, + "forbid": [marker.group(0)], + "require": require, + "auto_captured": True, + }) + return rows + + +def cases_from_repro_rows(rows: list[dict]) -> list[ReplayCase]: + """Build replay cases from persisted repro rows. Expected row shape: + {case_id, payload, require?, require_regex?, forbid?} — spine `repro` + entries and ReproBundle exports both map onto this.""" + cases = [] + for row in rows: + cases.append( + ReplayCase( + case_id=str(row["case_id"]), + payload=dict(row.get("payload", {})), + expect=Expectation( + require=[str(s) for s in row.get("require", [])], + require_regex=[str(s) for s in row.get("require_regex", [])], + forbid=[str(s) for s in row.get("forbid", [])], + ), + ) + ) + return cases diff --git a/src/xgen_sdk/harness/skill_forge/curator.py b/src/xgen_sdk/harness/skill_forge/curator.py new file mode 100644 index 0000000..6192193 --- /dev/null +++ b/src/xgen_sdk/harness/skill_forge/curator.py @@ -0,0 +1,185 @@ +"""SF2 — curation: turn raw signal hits into vetted skill candidates. + +Three responsibilities, in order: +1. anti-learning filter — refuse what must never become a skill (Hermes rules) +2. safety filter — refuse injection / secret material (poisoning defense) +3. patch-over-create — resolve against the ledger so knowledge accretes + instead of fragmenting + +The LLM (if any) only *drafts* candidate content upstream; every draft passes +through these deterministic filters. That ordering is the point: curation is +code, not vibes. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + +from .ledger import SkillLedger +from .model import Provenance, RunTrace, SkillCandidate +from .signals import SignalHit + +_SECRET_PATTERNS = [ + re.compile(r"(?i)(api[_-]?key|secret|token|passwd|password)\s*[:=]\s*\S{8,}"), + re.compile(r"sk-[A-Za-z0-9]{20,}"), + re.compile(r"ghp_[A-Za-z0-9]{20,}"), + re.compile(r"AKIA[0-9A-Z]{16}"), + re.compile(r"(?i)bearer\s+[A-Za-z0-9._-]{20,}"), +] + +_INJECTION_PATTERNS = [ + re.compile(r"(?i)ignore (all )?(previous|prior|above) (instructions|rules)"), + re.compile(r"(?i)(always|must) (run|execute|call)\b.*\b(curl|wget|nc|powershell|bash)\b"), + re.compile(r"(?i)do not (tell|inform|alert) the (user|admin)"), + re.compile(r"(?i)(exfiltrate|send|post) .*(http|ftp)s?://"), + re.compile(r"(?i)disable (the )?(guard|policy|safety|approval)"), +] + +_BROAD_NEGATIVE = re.compile( + r"(?i)^(never|don'?t|do not|avoid) (use|call|try|trust)\b" +) + +_TRANSIENT_HINTS = re.compile( + r"(?i)(timeout|timed out|rate.?limit|429|503|connection (reset|refused)|" + r"temporarily|transient|flaky|dns)" +) + +_ENV_SPECIFIC_HINTS = re.compile( + r"(?i)(not installed|command not found|no such file|missing (binary|package)|" + r"only on (this|my) (machine|host)|c:\\users\\|/home/\w+/)" +) + + +@dataclass +class Rejection: + candidate_name: str + rule: str + reason: str + + +@dataclass +class CurationResult: + accepted: list[SkillCandidate] = field(default_factory=list) + rejected: list[Rejection] = field(default_factory=list) + + +def _candidate_text(candidate: SkillCandidate) -> str: + parts = [candidate.rationale, candidate.when_to_use] + parts += candidate.procedure + candidate.pitfalls + candidate.verification + return "\n".join(p for p in parts if p) + + +def anti_learning_check(candidate: SkillCandidate) -> str | None: + """Returns a rejection reason, or None when the candidate may proceed.""" + text = _candidate_text(candidate) + if len(candidate.procedure) < 2 and candidate.action == "create": + return "one-off narrative: fewer than 2 reusable procedure steps" + if _TRANSIENT_HINTS.search(text) and not candidate.verification: + return "transient failure with no verification recipe — likely self-resolving" + if _ENV_SPECIFIC_HINTS.search(text): + return "environment-specific detail — not portable knowledge" + negatives = [s for s in candidate.procedure if _BROAD_NEGATIVE.match(s.strip())] + if candidate.procedure and len(negatives) == len(candidate.procedure): + return "broad negative claims only — bans are not procedures" + return None + + +def safety_check(candidate: SkillCandidate) -> str | None: + text = _candidate_text(candidate) + "\n" + str(candidate.payload) + for pattern in _SECRET_PATTERNS: + if pattern.search(text): + return f"secret material matched {pattern.pattern!r}" + for pattern in _INJECTION_PATTERNS: + if pattern.search(text): + return f"injection pattern matched {pattern.pattern!r}" + return None + + +def _token_set(text: str) -> set[str]: + return {t for t in re.findall(r"[a-z0-9가-힣]{2,}", text.lower())} + + +def resolve_patch_over_create(candidate: SkillCandidate, ledger: SkillLedger, + overlap_threshold: float = 0.5) -> SkillCandidate: + """If a live skill already covers this ground, convert `create` to `patch`.""" + if candidate.action == "patch": + return candidate + existing = ledger.get(candidate.name) + if existing is not None: + candidate.action = "patch" + candidate.target_skill = existing.name + return candidate + cand_tokens = _token_set(candidate.rationale + " " + candidate.when_to_use) + if not cand_tokens: + return candidate + best_name, best_overlap = "", 0.0 + for record in ledger.list(scope=candidate.scope): + if record.status not in ("active", "staged"): + continue + rec_tokens = _token_set(record.description) + if not rec_tokens: + continue + overlap = len(cand_tokens & rec_tokens) / min(len(cand_tokens), len(rec_tokens)) + if overlap > best_overlap: + best_name, best_overlap = record.name, overlap + if best_overlap >= overlap_threshold: + candidate.action = "patch" + candidate.target_skill = best_name + return candidate + + +class Curator: + def __init__(self, ledger: SkillLedger, curator_id: str = "background_curator") -> None: + self.ledger = ledger + self.curator_id = curator_id + + def curate(self, candidates: list[SkillCandidate]) -> CurationResult: + result = CurationResult() + for candidate in candidates: + reason = anti_learning_check(candidate) + if reason: + result.rejected.append(Rejection(candidate.name, "anti_learning", reason)) + continue + reason = safety_check(candidate) + if reason: + result.rejected.append(Rejection(candidate.name, "safety", reason)) + continue + candidate = resolve_patch_over_create(candidate, self.ledger) + result.accepted.append(candidate) + return result + + def draft_from_signals(self, trace: RunTrace, + hits: list[SignalHit]) -> list[SkillCandidate]: + """Deterministic fallback drafter (no LLM): one candidate per strong hit. + Hosts normally replace this with an LLM drafter run inside a + tool-whitelisted harness; the filters above still apply either way.""" + candidates: list[SkillCandidate] = [] + for hit in hits: + if hit.strength < 0.5: + continue + steps = [e.name for e in trace.tool_calls()] + if not steps: + continue + name = f"auto-{hit.signal.replace('_', '-')}-{trace.signature()[:8]}" + candidates.append( + SkillCandidate( + name=name, + kind="guide", + scope=trace.scope, + action="create", + rationale=hit.evidence, + when_to_use=f"When a task resembles run {trace.run_id} ({hit.signal})", + procedure=[f"Use `{s}`" for s in steps], + pitfalls=[e.detail for e in trace.events + if e.type == "error" and e.detail][:3], + verification=trace.lessons[:3] or ["Re-run and compare the outcome."], + provenance=Provenance( + origin="background_curator", + source_run_ids=[trace.run_id], + curator_id=self.curator_id, + signal=hit.signal, + ), + ) + ) + return candidates diff --git a/src/xgen_sdk/harness/skill_forge/drafter.py b/src/xgen_sdk/harness/skill_forge/drafter.py new file mode 100644 index 0000000..2559471 --- /dev/null +++ b/src/xgen_sdk/harness/skill_forge/drafter.py @@ -0,0 +1,299 @@ +"""Jermes drafter — LLM-backed candidate drafting from a run trace. + +The LLM only *proposes*; every proposal still passes the deterministic +curator filters and the forge gate. Provider-agnostic: `complete` is any +(prompt -> text) callable. A stdlib OpenAI-compatible completer is included +so the package stays dependency-free. +""" + +from __future__ import annotations + +import json +import re +import urllib.request +from typing import Callable + +from .model import Provenance, RunTrace, SkillCandidate +from .signals import SignalHit + +Completer = Callable[[str], str] + +_PROMPT = """You are Jermes, the skill smith of the XGEN harness. You review a \ +finished agent run and propose at most {max_candidates} reusable skills. + +Rules (violations are discarded by a deterministic filter, so obey them): +- Only durable, portable procedures. NO environment-specific paths, NO \ +transient failures (timeouts, rate limits), NO broad bans ("never use X"), \ +NO secrets or credentials. +- Prefer nothing over something marginal. Return [] when the run taught \ +nothing reusable. +- Procedures need >= 2 concrete steps and a verification recipe. + +Run trace: +{trace} + +Detected signals: +{signals} + +Lessons recorded by the agent: +{lessons} + +Respond with ONLY a JSON array (no prose). Each item: +{{"name": "kebab-case-name", "when_to_use": "...", "rationale": "...", + "procedure": ["step", ...], "pitfalls": ["...", ...], + "verification": ["...", ...]}} + +Example of a good response (format reference only — do not copy content): +[{{"name": "paginate-with-cursor", "when_to_use": "when listing more than one \ +page from the orders API", "rationale": "offset pagination silently drops rows \ +under concurrent writes", "procedure": ["Request the first page with limit and \ +no cursor", "Loop passing next_cursor until it returns empty"], "pitfalls": \ +["Reusing an expired cursor returns 410"], "verification": ["Total fetched \ +count equals the summary endpoint count"]}}]""" + +_REPAIR_PROMPT = """The following text was supposed to be a JSON array of skill \ +objects but is malformed. Output ONLY the corrected JSON array, nothing else. \ +If no skill objects can be recovered, output []. + +{raw}""" + +_RETRY_SUFFIX = """ + +Your previous attempt was invalid: {feedback} +Respond again with ONLY a valid JSON array.""" + + +def _render_trace(trace: RunTrace, cap: int = 40) -> str: + lines = [] + for event in trace.events[:cap]: + status = "" if event.ok else " FAILED" + detail = f" — {event.detail}" if event.detail else "" + lines.append(f"- {event.type}: {event.name}{status}{detail}") + lines.append(f"- outcome: {'success' if trace.success else 'failure'}") + return "\n".join(lines) + + +def build_prompt(trace: RunTrace, hits: list[SignalHit], + max_candidates: int = 2) -> str: + signals = "\n".join(f"- {h.signal} ({h.strength:.2f}): {h.evidence}" + for h in hits) or "- none" + lessons = "\n".join(f"- {l}" for l in trace.lessons) or "- none" + return _PROMPT.format(max_candidates=max_candidates, + trace=_render_trace(trace), + signals=signals, lessons=lessons) + + +_EXAMPLE_NAME = "paginate-with-cursor" +_EXAMPLE_MARKERS = ("orders api", "next_cursor", "offset pagination") + + +def _is_example_leak(item: dict) -> bool: + """Weak models sometimes copy the few-shot example despite instructions. + Drop any candidate that reproduces the example's name or content.""" + name = str(item.get("name", "")).strip().lower() + if name == _EXAMPLE_NAME: + return True + text = " ".join(str(item.get(k, "")) for k in + ("when_to_use", "rationale")).lower() + return sum(marker in text for marker in _EXAMPLE_MARKERS) >= 2 + + +def _sanitize_name(raw: str) -> str: + name = re.sub(r"[^a-z0-9-]", "-", raw.strip().lower().replace("_", "-").replace(" ", "-")) + name = re.sub(r"-{2,}", "-", name).strip("-") + return name[:64] or "unnamed-skill" + + +def _extract_json_array(text: str) -> list: + text = re.sub(r"^```(?:json)?|```$", "", text.strip(), flags=re.MULTILINE).strip() + start = text.find("[") + if start == -1: + return [] + depth = 0 + for i in range(start, len(text)): + if text[i] == "[": + depth += 1 + elif text[i] == "]": + depth -= 1 + if depth == 0: + try: + parsed = json.loads(text[start:i + 1]) + return parsed if isinstance(parsed, list) else [] + except json.JSONDecodeError: + return [] + return [] + + +class LLMDrafter: + """Weak-model-hardened drafter. Three recovery layers before giving up: + (1) tolerant extraction (fences, surrounding prose), (2) an LLM repair pass + on malformed output, (3) a bounded retry with explicit failure feedback. + A weak model that CAN emit JSON some of the time therefore still drafts; + a weak model that drafts nonsense is stopped later by curator + gate.""" + + def __init__(self, complete: Completer, max_candidates: int = 2, + drafter_id: str = "jermes", repair: bool = True, + max_retries: int = 1) -> None: + self.complete = complete + self.max_candidates = max_candidates + self.drafter_id = drafter_id + self.repair = repair + self.max_retries = max_retries + + def _complete_array(self, prompt: str) -> list: + raw = self.complete(prompt) + items = _extract_json_array(raw) + if not items and self.repair and raw.strip() and raw.strip() != "[]": + repaired = self.complete(_REPAIR_PROMPT.format(raw=raw[:4000])) + items = _extract_json_array(repaired) + return items + + def draft(self, trace: RunTrace, hits: list[SignalHit]) -> list[SkillCandidate]: + prompt = build_prompt(trace, hits, self.max_candidates) + items: list = [] + feedback = "" + for attempt in range(self.max_retries + 1): + attempt_prompt = prompt if not feedback else ( + prompt + _RETRY_SUFFIX.format(feedback=feedback)) + try: + items = self._complete_array(attempt_prompt) + except Exception: + return [] + if items: + break + feedback = "no parsable JSON array of skill objects was found" + candidates: list[SkillCandidate] = [] + strongest = max((h.signal for h in sorted(hits, key=lambda h: -h.strength)), + default="") + for item in items[: self.max_candidates]: + if not isinstance(item, dict) or _is_example_leak(item): + continue + try: + candidates.append( + SkillCandidate( + name=_sanitize_name(str(item.get("name", ""))), + kind="guide", + scope=trace.scope, + action="create", + rationale=str(item.get("rationale", ""))[:500], + when_to_use=str(item.get("when_to_use", ""))[:300], + procedure=[str(s)[:300] for s in item.get("procedure", [])][:12], + pitfalls=[str(s)[:300] for s in item.get("pitfalls", [])][:6], + verification=[str(s)[:300] for s in item.get("verification", [])][:6], + provenance=Provenance( + origin="llm_drafter", + source_run_ids=[trace.run_id], + curator_id=self.drafter_id, + signal=strongest, + ), + ) + ) + except ValueError: + continue + return candidates + + +class EnsembleDrafter: + """Self-consistency for weak models: sample the drafter k times, pool the + candidates, and collapse near-duplicates. Selection among survivors is NOT + done here — the curator (patch-over-create) and the gate (replay bench) do + it deterministically, which is exactly what makes a weak drafter viable: + breadth from sampling, quality from selection pressure.""" + + def __init__(self, drafter: LLMDrafter, samples: int = 3, + max_candidates: int = 4) -> None: + self.drafter = drafter + self.samples = samples + self.max_candidates = max_candidates + + def draft(self, trace: RunTrace, hits: list[SignalHit]) -> list[SkillCandidate]: + pool: list[SkillCandidate] = [] + for _ in range(self.samples): + pool.extend(self.drafter.draft(trace, hits)) + deduped: list[SkillCandidate] = [] + seen_names: set[str] = set() + for candidate in pool: + if candidate.name in seen_names: + continue + if any(_near_duplicate(candidate, kept) for kept in deduped): + continue + seen_names.add(candidate.name) + deduped.append(candidate) + return deduped[: self.max_candidates] + + +def _near_duplicate(a: "SkillCandidate", b: "SkillCandidate") -> bool: + ta = set(re.findall(r"[a-z0-9가-힣]{2,}", (a.when_to_use + " " + a.rationale).lower())) + tb = set(re.findall(r"[a-z0-9가-힣]{2,}", (b.when_to_use + " " + b.rationale).lower())) + if not ta or not tb: + return False + return len(ta & tb) / min(len(ta), len(tb)) >= 0.7 + + +def anthropic_completer(model: str, api_key: str, + max_tokens: int = 2048, + timeout: float = 120.0) -> Completer: + """Stdlib completer for the Anthropic Messages API. Raw HTTP is deliberate: + this package ships with dependencies=[] (engine ethos), so the official SDK + is not available here — hosts with the SDK installed should pass their own + Completer instead.""" + url = "https://api.anthropic.com/v1/messages" + + def complete(prompt: str) -> str: + payload = { + "model": model, + "max_tokens": max_tokens, + "messages": [{"role": "user", "content": prompt}], + } + request = urllib.request.Request( + url, + data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), + headers={ + "Content-Type": "application/json", + "x-api-key": api_key, + "anthropic-version": "2023-06-01", + }, + method="POST", + ) + with urllib.request.urlopen(request, timeout=timeout) as response: + body = json.loads(response.read().decode("utf-8")) + if body.get("stop_reason") == "refusal": + return "[]" + return "".join(block.get("text", "") for block in body.get("content", []) + if block.get("type") == "text") + + return complete + + +def openai_chat_completer(base_url: str, model: str, + temperature: float = 0.2, + timeout: float = 120.0, + api_key: str = "", + extra: dict | None = None) -> Completer: + """Stdlib completer for any OpenAI-compatible /chat/completions endpoint. + + `extra` merges provider-specific body params (e.g. DashScope Qwen3 needs + {"enable_thinking": False} on non-streaming calls).""" + url = base_url.rstrip("/") + "/chat/completions" + + def complete(prompt: str) -> str: + payload = { + "model": model, + "temperature": temperature, + "messages": [{"role": "user", "content": prompt}], + **(extra or {}), + } + request = urllib.request.Request( + url, + data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), + headers={ + "Content-Type": "application/json", + **({"Authorization": f"Bearer {api_key}"} if api_key else {}), + }, + method="POST", + ) + with urllib.request.urlopen(request, timeout=timeout) as response: + body = json.loads(response.read().decode("utf-8")) + return body["choices"][0]["message"]["content"] + + return complete diff --git a/src/xgen_sdk/harness/skill_forge/gate.py b/src/xgen_sdk/harness/skill_forge/gate.py new file mode 100644 index 0000000..f76359b --- /dev/null +++ b/src/xgen_sdk/harness/skill_forge/gate.py @@ -0,0 +1,113 @@ +"""SF4 — the forge verification gate. + +Same promote algebra as SelfForge: promote = dev_up AND held_ok AND sec_ok +AND NOT overopt. Bench cases are split deterministically by id hash into +dev/holdout (the Synapse/forge discipline: holdout never trains, only judges). + +With no bench cases available the gate is honest: verdict "staged" +(unverified 2nd-track), never a fake pass. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass, field +from typing import Callable, Protocol, Sequence + +from .curator import safety_check +from .model import GateResult, SkillCandidate, SkillDef + + +@dataclass +class BenchCase: + case_id: str + payload: dict = field(default_factory=dict) + + def is_holdout(self, ratio: float = 0.25) -> bool: + digest = hashlib.sha256(self.case_id.encode("utf-8")).digest() + return (digest[0] / 255.0) < ratio + + +class BenchRunner(Protocol): + """Scores one bench case, optionally with the candidate skill installed. + Hosts back this with PipelineRunner + ReproBundle replays.""" + + def score(self, case: BenchCase, skill: SkillDef | None) -> float: ... + + +ScoreFn = Callable[[BenchCase, SkillDef | None], float] + + +@dataclass +class GateConfig: + holdout_ratio: float = 0.25 + min_gain: float = 0.0 # dev must strictly beat baseline by more than this + max_holdout_drop: float = 0.02 + overopt_gap: float = 0.25 # dev-holdout gain divergence alarm + min_cases: int = 4 + + +class ForgeGate: + def __init__(self, runner: BenchRunner | ScoreFn, + config: GateConfig | None = None) -> None: + self._score: ScoreFn = runner.score if hasattr(runner, "score") else runner # type: ignore[union-attr] + self.config = config or GateConfig() + + def verify(self, candidate: SkillCandidate, skill: SkillDef, + cases: Sequence[BenchCase]) -> GateResult: + reason = safety_check(candidate) + if reason: + return GateResult(verdict="rejected", reasons=[f"sec: {reason}"]) + + if len(cases) < self.config.min_cases: + return GateResult( + verdict="staged", + reasons=[ + f"unverified: {len(cases)} bench case(s) < min {self.config.min_cases}" + ], + ) + + dev = [c for c in cases if not c.is_holdout(self.config.holdout_ratio)] + holdout = [c for c in cases if c.is_holdout(self.config.holdout_ratio)] + if not dev or not holdout: + return GateResult(verdict="staged", + reasons=["unverified: degenerate dev/holdout split"]) + + def mean(cs: Sequence[BenchCase], skill_def: SkillDef | None) -> float: + return sum(self._score(c, skill_def) for c in cs) / len(cs) + + baseline_dev = mean(dev, None) + baseline_holdout = mean(holdout, None) + dev_score = mean(dev, skill) + holdout_score = mean(holdout, skill) + + dev_gain = dev_score - baseline_dev + holdout_gain = holdout_score - baseline_holdout + + dev_up = dev_gain > self.config.min_gain + held_ok = holdout_gain >= -self.config.max_holdout_drop + overopt = (dev_gain - holdout_gain) > self.config.overopt_gap + + reasons = [ + f"dev {baseline_dev:.3f}->{dev_score:.3f} ({dev_gain:+.3f})", + f"holdout {baseline_holdout:.3f}->{holdout_score:.3f} ({holdout_gain:+.3f})", + ] + if dev_up and held_ok and not overopt: + verdict = "promoted" + elif not dev_up: + verdict = "rejected" + reasons.append("no dev gain — the skill does not help") + elif not held_ok: + verdict = "rejected" + reasons.append("holdout regressed — memorization, not knowledge") + else: + verdict = "rejected" + reasons.append("over-optimization gap — dev gain does not generalize") + return GateResult( + verdict=verdict, + reasons=reasons, + dev_score=dev_score, + holdout_score=holdout_score, + baseline_dev=baseline_dev, + baseline_holdout=baseline_holdout, + ) diff --git a/src/xgen_sdk/harness/skill_forge/host.py b/src/xgen_sdk/harness/skill_forge/host.py new file mode 100644 index 0000000..5f044d8 --- /dev/null +++ b/src/xgen_sdk/harness/skill_forge/host.py @@ -0,0 +1,129 @@ +"""Host adapters — how the XGEN platform plugs the loop into the State Spine. + +The platform's `harness_spine_service` satisfies `SpineStore` with a thin +wrapper (append -> write_spine, query -> read_spine). Everything here is +row-dict based so the engine keeps zero knowledge of the DB layer. + +Row conventions (mirrors xgen_harness_spine): +- spine_type "activity" payload {type,name,ok,detail,run_id} +- spine_type "lesson" payload {text,run_id} +- spine_type "refined_memory" payload {text,run_id} +- spine_type "judge_score" payload {score,run_id} +- spine_type "skill_def" payload {event: commit|status|outcome, ...} +""" + +from __future__ import annotations + +from collections import Counter +from typing import Iterable, Protocol + +from .ledger import InMemorySkillLedger, SkillRecord +from .model import Provenance, RunTrace, SkillDef, TraceEvent + + +class SpineStore(Protocol): + def append(self, spine_type: str, key_id: str, payload: dict) -> None: ... + def query(self, spine_type: str, + key_id: str | None = None) -> list[dict]: ... + + +class InMemorySpineStore: + def __init__(self) -> None: + self._rows: list[tuple[str, str, dict]] = [] + + def append(self, spine_type: str, key_id: str, payload: dict) -> None: + self._rows.append((spine_type, key_id, dict(payload))) + + def query(self, spine_type: str, key_id: str | None = None) -> list[dict]: + return [dict(p) for t, k, p in self._rows + if t == spine_type and (key_id is None or k == key_id)] + + +def trace_from_spine(store: SpineStore, run_id: str, + scope: str = "user", scope_key: str = "") -> RunTrace: + events = [ + TraceEvent( + type=row.get("type", "note"), + name=row.get("name", ""), + ok=bool(row.get("ok", True)), + detail=row.get("detail", ""), + ) + for row in store.query("activity", run_id) + ] + lessons = [row.get("text", "") for row in store.query("lesson", run_id)] + refined = "\n".join(row.get("text", "") + for row in store.query("refined_memory", run_id)) + judge_rows = store.query("judge_score", run_id) + judge = float(judge_rows[-1]["score"]) if judge_rows else None + outcome_ok = all(e.ok for e in events if e.type == "outcome") if events else True + return RunTrace( + run_id=run_id, + scope=scope, + scope_key=scope_key, + events=events, + lessons=[l for l in lessons if l], + refined_memory=refined, + judge_score=judge, + success=outcome_ok, + ) + + +def signature_counts(traces: Iterable[RunTrace]) -> dict[str, int]: + """Historical tool-sequence counts feeding the repetition signal.""" + return dict(Counter(t.signature() for t in traces)) + + +class SpineSkillLedger(InMemorySkillLedger): + """SkillLedger persisted as append-only `skill_def` spine rows. + + Same fold discipline as JsonlSkillLedger; the spine's own versioning and + provenance columns come for free on the platform side. + """ + + SPINE_TYPE = "skill_def" + + def __init__(self, store: SpineStore) -> None: + super().__init__() + self.store = store + self._replay() + + def _replay(self) -> None: + for row in self.store.query(self.SPINE_TYPE): + event = row.get("event") + if event == "commit": + data = dict(row["skill"]) + provenance = data.pop("provenance", None) + skill = SkillDef(**{**data, "provenance": None}) + if provenance: + skill.provenance = Provenance(**provenance) + super().commit(skill, note=row.get("note", "")) + record = super().get(skill.name) + if record is not None: + record.skill.version = data.get("version", record.skill.version) + elif event == "status": + super().set_status(row["name"], row["status"], row.get("note", "")) + elif event == "outcome": + super().record_outcome([row["name"]], row["success"]) + + def commit(self, skill: SkillDef, note: str = "") -> SkillRecord: + record = super().commit(skill, note) + self.store.append(self.SPINE_TYPE, skill.name, + {"event": "commit", "skill": record.skill.to_dict(), + "note": note}) + return record + + def set_status(self, name: str, status: str, note: str = "") -> SkillRecord: + record = super().set_status(name, status, note) + self.store.append(self.SPINE_TYPE, name, + {"event": "status", "name": name, "status": status, + "note": note}) + return record + + def record_outcome(self, names, success: bool) -> None: + names = list(names) + super().record_outcome(names, success) + for name in names: + if super().get(name) is not None: + self.store.append(self.SPINE_TYPE, name, + {"event": "outcome", "name": name, + "success": success}) diff --git a/src/xgen_sdk/harness/skill_forge/ledger.py b/src/xgen_sdk/harness/skill_forge/ledger.py new file mode 100644 index 0000000..0a49435 --- /dev/null +++ b/src/xgen_sdk/harness/skill_forge/ledger.py @@ -0,0 +1,169 @@ +"""SF5 — the skill ledger: versioned, provenance-carrying skill records. + +Protocol + two reference backends (in-memory, JSONL append-only). The XGEN +host maps this onto the State Spine (`spine_type=skill_def`) — same shape: +every mutation is an append with version + provenance, state is a fold. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Iterable, Protocol + +from .model import ( + SkillDef, + UsageStats, + bump_patch, +) + + +@dataclass +class SkillRecord: + skill: SkillDef + usage: UsageStats = field(default_factory=UsageStats) + history: list[str] = field(default_factory=list) # ": " + + @property + def name(self) -> str: + return self.skill.name + + @property + def status(self) -> str: + return self.skill.status + + @property + def description(self) -> str: + return self.skill.description + + def rank_score(self) -> float: + """Recall ordering: verified beats unverified, then Wilson lower bound. + Loading is never counted — only run outcomes feed `usage`.""" + base = 0.5 if self.skill.verified else 0.0 + return base + self.usage.wilson_lower() + + +class SkillLedger(Protocol): + def get(self, name: str) -> SkillRecord | None: ... + def list(self, scope: str | None = None, + status: str | None = None) -> list[SkillRecord]: ... + def commit(self, skill: SkillDef, note: str = "") -> SkillRecord: ... + def set_status(self, name: str, status: str, note: str = "") -> SkillRecord: ... + def record_outcome(self, names: Iterable[str], success: bool) -> None: ... + + +class InMemorySkillLedger: + def __init__(self) -> None: + self._records: dict[str, SkillRecord] = {} + + def get(self, name: str) -> SkillRecord | None: + return self._records.get(name) + + def list(self, scope: str | None = None, + status: str | None = None) -> list[SkillRecord]: + records = self._records.values() + if scope is not None: + records = [r for r in records if r.skill.scope == scope] + if status is not None: + records = [r for r in records if r.status == status] + return sorted(records, key=lambda r: (-r.rank_score(), r.name)) + + def commit(self, skill: SkillDef, note: str = "") -> SkillRecord: + existing = self._records.get(skill.name) + if existing is not None: + prior = existing.skill + skill.version = bump_patch(prior.version) + skill.supersedes = f"{prior.name}@{prior.version}" + existing.skill = skill + existing.history.append(f"{skill.version}: {note or 'update'}") + return existing + record = SkillRecord(skill=skill, history=[f"{skill.version}: {note or 'create'}"]) + self._records[skill.name] = record + return record + + def set_status(self, name: str, status: str, note: str = "") -> SkillRecord: + record = self._records[name] + record.skill.status = status + record.history.append(f"{record.skill.version}: status={status} {note}".rstrip()) + return record + + def record_outcome(self, names: Iterable[str], success: bool) -> None: + for name in names: + record = self._records.get(name) + if record is None: + continue + if success: + record.usage.successes += 1 + else: + record.usage.failures += 1 + + def sweep_deprecate(self, min_uses: int = 5, max_wilson: float = 0.2) -> list[str]: + """Low-signal prune (4-scope routine #5): enough real outcomes, and the + Wilson lower bound still under the floor -> deprecate, never delete.""" + deprecated = [] + for record in self._records.values(): + if record.status != "active": + continue + if record.usage.total >= min_uses and record.usage.wilson_lower() <= max_wilson: + self.set_status(record.name, "deprecated", "low-signal sweep") + deprecated.append(record.name) + return deprecated + + +class JsonlSkillLedger(InMemorySkillLedger): + """Append-only JSONL journal + in-memory fold. Crash-safe enough for the + research phase; the production backend is the State Spine.""" + + def __init__(self, path: str | Path) -> None: + super().__init__() + self.path = Path(path) + if self.path.exists(): + self._replay() + + def _append(self, kind: str, payload: dict) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps({"kind": kind, **payload}, ensure_ascii=False) + "\n") + + def _replay(self) -> None: + with self.path.open("r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + event = json.loads(line) + kind = event.pop("kind") + if kind == "commit": + data = event["skill"] + provenance = data.pop("provenance", None) + skill = SkillDef(**{**data, "provenance": None}) + if provenance: + from .model import Provenance + skill.provenance = Provenance(**provenance) + super().commit(skill, note=event.get("note", "")) + # replay keeps journal authority over derived version fields + record = super().get(skill.name) + if record is not None: + record.skill.version = data.get("version", record.skill.version) + elif kind == "status": + super().set_status(event["name"], event["status"], event.get("note", "")) + elif kind == "outcome": + super().record_outcome([event["name"]], event["success"]) + + def commit(self, skill: SkillDef, note: str = "") -> SkillRecord: + record = super().commit(skill, note) + self._append("commit", {"skill": record.skill.to_dict(), "note": note}) + return record + + def set_status(self, name: str, status: str, note: str = "") -> SkillRecord: + record = super().set_status(name, status, note) + self._append("status", {"name": name, "status": status, "note": note}) + return record + + def record_outcome(self, names: Iterable[str], success: bool) -> None: + names = list(names) + super().record_outcome(names, success) + for name in names: + if super().get(name) is not None: + self._append("outcome", {"name": name, "success": success}) diff --git a/src/xgen_sdk/harness/skill_forge/loop.py b/src/xgen_sdk/harness/skill_forge/loop.py new file mode 100644 index 0000000..3641804 --- /dev/null +++ b/src/xgen_sdk/harness/skill_forge/loop.py @@ -0,0 +1,100 @@ +"""The Skill-Forge orchestrator: observe -> curate -> synthesize -> verify -> commit. + +Pure and synchronous; hosts decide scheduling (post-run async job, batch cron). +Every stage's output is returned in the report so the host can journal the +entire episode to the spine. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Sequence + +from .curator import Curator, Rejection +from .gate import BenchCase, ForgeGate +from .ledger import SkillLedger +from .model import GateResult, RunTrace, SkillCandidate, SkillDef +from .signals import SignalHit, extract_signals +from .synthesis import Reflector, synthesize + + +@dataclass +class ApprovalPolicy: + """Scope-aware activation policy. Verified user-scope skills may go live + automatically; broader scopes stage for human approval by default.""" + + auto_activate_scopes: tuple[str, ...] = ("session", "user") + + def on_promoted(self, skill: SkillDef) -> str: + return "active" if skill.scope in self.auto_activate_scopes else "staged" + + +@dataclass +class ForgeEpisode: + run_id: str + signals: list[SignalHit] = field(default_factory=list) + drafted: list[SkillCandidate] = field(default_factory=list) + rejected: list[Rejection] = field(default_factory=list) + results: list[tuple[SkillDef, GateResult]] = field(default_factory=list) + + def summary(self) -> str: + lines = [f"run={self.run_id} signals={len(self.signals)} " + f"drafted={len(self.drafted)} curator_rejects={len(self.rejected)}"] + for skill, gate in self.results: + lines.append(f" {skill.name} [{skill.kind}/{skill.scope}] -> " + f"{gate.verdict} ({'; '.join(gate.reasons)})") + return "\n".join(lines) + + +class SkillForge: + def __init__(self, ledger: SkillLedger, gate: ForgeGate, + curator: Curator | None = None, + reflector: Reflector | None = None, + approval: ApprovalPolicy | None = None) -> None: + self.ledger = ledger + self.gate = gate + self.curator = curator or Curator(ledger) + self.reflector = reflector + self.approval = approval or ApprovalPolicy() + + def process_trace(self, trace: RunTrace, + bench_cases: Sequence[BenchCase] = (), + prior_signatures: dict[str, int] | None = None, + drafted: list[SkillCandidate] | None = None) -> ForgeEpisode: + episode = ForgeEpisode(run_id=trace.run_id) + episode.signals = extract_signals(trace, prior_signatures=prior_signatures) + if not episode.signals and drafted is None: + return episode + + candidates = drafted if drafted is not None else self.curator.draft_from_signals( + trace, episode.signals) + episode.drafted = candidates + + curation = self.curator.curate(candidates) + episode.rejected = curation.rejected + + for candidate in curation.accepted: + skill = self._synthesize_resolved(candidate) + gate_result = self.gate.verify(candidate, skill, bench_cases) + if gate_result.verdict == "promoted": + skill.verified = True + skill.status = self.approval.on_promoted(skill) + self.ledger.commit(skill, note=f"promoted: {gate_result.reasons[0]}") + elif gate_result.verdict == "staged": + skill.verified = False + skill.status = "staged" + self.ledger.commit(skill, note=f"staged: {'; '.join(gate_result.reasons)}") + # rejected candidates are journaled in the episode, not the ledger + episode.results.append((skill, gate_result)) + return episode + + def _synthesize_resolved(self, candidate: SkillCandidate) -> SkillDef: + if candidate.action == "patch": + target = self.ledger.get(candidate.target_skill) + if target is not None: + merged = candidate + merged.name = target.name + skill = synthesize(merged, reflector=self.reflector) + skill.meta["patched_from"] = f"{target.name}@{target.skill.version}" + return skill + return synthesize(candidate, reflector=self.reflector) diff --git a/src/xgen_sdk/harness/skill_forge/model.py b/src/xgen_sdk/harness/skill_forge/model.py new file mode 100644 index 0000000..8bdae65 --- /dev/null +++ b/src/xgen_sdk/harness/skill_forge/model.py @@ -0,0 +1,186 @@ +"""Core data model for the Skill-Forge loop. + +Everything is a plain dataclass with dict round-trip so hosts can persist +records in any store (State Spine, SQL, files) without importing this package +at the storage layer. +""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass, field +from typing import Any + +SKILL_KINDS = ("guide", "config", "tool") +SKILL_SCOPES = ("session", "workflow", "user", "platform") +CANDIDATE_ACTIONS = ("create", "patch") +GATE_VERDICTS = ("promoted", "staged", "rejected") +SKILL_STATUSES = ("staged", "active", "deprecated", "rejected") + + +def _require(value: str, allowed: tuple[str, ...], label: str) -> str: + if value not in allowed: + raise ValueError(f"{label} must be one of {allowed}, got {value!r}") + return value + + +@dataclass +class TraceEvent: + """One observed event of a finished run. Hosts map their own trace + (spine activity, RunEvent, tool journal) into this neutral shape.""" + + type: str # tool_call | error | recovery | user_correction | outcome | note + name: str = "" + ok: bool = True + detail: str = "" + meta: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class RunTrace: + run_id: str + scope: str = "user" + scope_key: str = "" + events: list[TraceEvent] = field(default_factory=list) + lessons: list[str] = field(default_factory=list) + refined_memory: str = "" + judge_score: float | None = None + success: bool = True + + def __post_init__(self) -> None: + _require(self.scope, SKILL_SCOPES, "scope") + + def tool_calls(self) -> list[TraceEvent]: + return [e for e in self.events if e.type == "tool_call"] + + def signature(self) -> str: + """Stable signature of the tool sequence, for repetition detection.""" + seq = "|".join(e.name for e in self.tool_calls()) + return hashlib.sha256(seq.encode("utf-8")).hexdigest()[:16] + + +@dataclass +class Provenance: + origin: str # e.g. "background_curator", "manual", "import" + source_run_ids: list[str] = field(default_factory=list) + curator_id: str = "" + signal: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "origin": self.origin, + "source_run_ids": list(self.source_run_ids), + "curator_id": self.curator_id, + "signal": self.signal, + } + + +@dataclass +class SkillCandidate: + """What the curator emits: not yet a skill, just an argued proposal.""" + + name: str + kind: str + scope: str + action: str # create | patch + rationale: str + procedure: list[str] = field(default_factory=list) + pitfalls: list[str] = field(default_factory=list) + verification: list[str] = field(default_factory=list) + when_to_use: str = "" + target_skill: str = "" # set when action == "patch" + provenance: Provenance | None = None + payload: dict[str, Any] = field(default_factory=dict) # kind-specific extras + + def __post_init__(self) -> None: + _require(self.kind, SKILL_KINDS, "kind") + _require(self.scope, SKILL_SCOPES, "scope") + _require(self.action, CANDIDATE_ACTIONS, "action") + if not _NAME_RE.match(self.name): + raise ValueError(f"skill name must be kebab-case, got {self.name!r}") + if self.action == "patch" and not self.target_skill: + raise ValueError("patch candidate requires target_skill") + + +_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]{1,63}$") + + +@dataclass +class SkillDef: + """A synthesized skill artifact ready for gating / the ledger.""" + + name: str + kind: str + scope: str + description: str + body: str # guide: SKILL.md text | config: json text | tool: manifest text + version: str = "0.1.0" + status: str = "staged" + provenance: Provenance | None = None + supersedes: str = "" # "@" lineage pointer + verified: bool = False + meta: dict[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + _require(self.kind, SKILL_KINDS, "kind") + _require(self.scope, SKILL_SCOPES, "scope") + _require(self.status, SKILL_STATUSES, "status") + + def to_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "kind": self.kind, + "scope": self.scope, + "description": self.description, + "body": self.body, + "version": self.version, + "status": self.status, + "provenance": self.provenance.to_dict() if self.provenance else None, + "supersedes": self.supersedes, + "verified": self.verified, + "meta": dict(self.meta), + } + + +@dataclass +class GateResult: + verdict: str # promoted | staged | rejected + reasons: list[str] = field(default_factory=list) + dev_score: float | None = None + holdout_score: float | None = None + baseline_dev: float | None = None + baseline_holdout: float | None = None + + def __post_init__(self) -> None: + _require(self.verdict, GATE_VERDICTS, "verdict") + + +@dataclass +class UsageStats: + """Run-outcome-only feedback. Loading a skill is NOT a signal — + only the outcome of a run that had it loaded counts (anti self-reinforcement, + same discipline as forge/Synapse).""" + + successes: int = 0 + failures: int = 0 + + @property + def total(self) -> int: + return self.successes + self.failures + + def wilson_lower(self, z: float = 1.96) -> float: + """Wilson score lower bound of the success rate; 0.0 when unused.""" + n = self.total + if n == 0: + return 0.0 + p = self.successes / n + denom = 1 + z * z / n + centre = p + z * z / (2 * n) + margin = z * ((p * (1 - p) + z * z / (4 * n)) / n) ** 0.5 + return max(0.0, (centre - margin) / denom) + + +def bump_patch(version: str) -> str: + major, minor, patch = (int(x) for x in version.split(".")) + return f"{major}.{minor}.{patch + 1}" diff --git a/src/xgen_sdk/harness/skill_forge/recall.py b/src/xgen_sdk/harness/skill_forge/recall.py new file mode 100644 index 0000000..c714178 --- /dev/null +++ b/src/xgen_sdk/harness/skill_forge/recall.py @@ -0,0 +1,70 @@ +"""SF5 recall side — progressive disclosure over the ledger. + +Mirrors the engine skill_registry contract (name -> short description, body on +demand) so absorption is a thin `LedgerSkillSource` added to +tools/skill_registry.py. Two-step PD: list (metadata only) -> view (body). + +Discipline: viewing/loading is NEVER recorded as a positive signal. Hosts call +`record_run_outcome` once per finished run with the set of skills that were +loaded and whether the run succeeded. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from .ledger import SkillLedger + + +@dataclass +class SkillListing: + name: str + kind: str + description: str + version: str + verified: bool + rank: float + + +class LedgerSkillSource: + def __init__(self, ledger: SkillLedger, scope: str | None = None, + include_staged: bool = False) -> None: + self.ledger = ledger + self.scope = scope + self.include_staged = include_staged + + def list(self) -> list[SkillListing]: + listings: list[SkillListing] = [] + statuses = ("active", "staged") if self.include_staged else ("active",) + for status in statuses: + for record in self.ledger.list(scope=self.scope, status=status): + listings.append( + SkillListing( + name=record.name, + kind=record.skill.kind, + description=record.description, + version=record.skill.version, + verified=record.skill.verified, + rank=record.rank_score(), + ) + ) + return sorted(listings, key=lambda item: (-item.rank, item.name)) + + def view(self, name: str) -> str: + record = self.ledger.get(name) + if record is None: + raise KeyError(f"unknown skill {name!r}") + label = "verified" if record.skill.verified else "UNVERIFIED" + header = f"[{label}] {record.name} v{record.skill.version} ({record.skill.kind})\n" + return header + record.skill.body + + def render_index(self, limit: int = 20) -> str: + """Compact index for prompt injection (s03) — metadata only.""" + lines = [] + for item in self.list()[:limit]: + flag = "✔" if item.verified else "•" + lines.append(f"{flag} {item.name} v{item.version} — {item.description}") + return "\n".join(lines) + + def record_run_outcome(self, loaded_skills: list[str], success: bool) -> None: + self.ledger.record_outcome(loaded_skills, success) diff --git a/src/xgen_sdk/harness/skill_forge/registry.py b/src/xgen_sdk/harness/skill_forge/registry.py new file mode 100644 index 0000000..1befeeb --- /dev/null +++ b/src/xgen_sdk/harness/skill_forge/registry.py @@ -0,0 +1,60 @@ +"""Tiny plugin registry with optional entry_points loading. + +Groups follow the engine convention (`xgen_harness.skill_*`) so absorption +into xgen_sdk.harness keeps external contracts unchanged. +""" + +from __future__ import annotations + +from importlib import metadata +from typing import Any, Callable + +GROUP_SIGNAL_EXTRACTORS = "xgen_harness.skill_signal_extractors" +GROUP_SYNTHESIZERS = "xgen_harness.skill_synthesizers" +GROUP_GATES = "xgen_harness.skill_gates" +GROUP_LEDGERS = "xgen_harness.skill_ledgers" + + +class Registry: + def __init__(self, group: str) -> None: + self.group = group + self._items: dict[str, Any] = {} + + def register(self, name: str, item: Any) -> None: + self._items[name] = item + + def get(self, name: str) -> Any: + if name not in self._items: + raise KeyError(f"{self.group}: unknown entry {name!r}") + return self._items[name] + + def names(self) -> list[str]: + return sorted(self._items) + + def items(self) -> list[tuple[str, Any]]: + return sorted(self._items.items()) + + def load_entry_points(self) -> int: + loaded = 0 + try: + eps = metadata.entry_points(group=self.group) + except Exception: + return 0 + for ep in eps: + try: + self.register(ep.name, ep.load()) + loaded += 1 + except Exception: + continue + return loaded + + +def registry_decorator(reg: Registry) -> Callable[[str], Callable[[Any], Any]]: + def outer(name: str) -> Callable[[Any], Any]: + def inner(obj: Any) -> Any: + reg.register(name, obj) + return obj + + return inner + + return outer diff --git a/src/xgen_sdk/harness/skill_forge/signals.py b/src/xgen_sdk/harness/skill_forge/signals.py new file mode 100644 index 0000000..576d4e2 --- /dev/null +++ b/src/xgen_sdk/harness/skill_forge/signals.py @@ -0,0 +1,122 @@ +"""SF1 — candidate signal extraction from finished run traces. + +Extractors are pure functions RunTrace -> list[SignalHit]; hosts add their own +via the registry / entry_points. Defaults mirror the Hermes trigger set but are +tunable and engine-neutral. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Callable + +from .model import RunTrace +from .registry import GROUP_SIGNAL_EXTRACTORS, Registry, registry_decorator + + +@dataclass +class SignalHit: + signal: str + strength: float # 0..1, extractor-relative + evidence: str + meta: dict = field(default_factory=dict) + + +SIGNAL_EXTRACTORS = Registry(GROUP_SIGNAL_EXTRACTORS) +signal_extractor = registry_decorator(SIGNAL_EXTRACTORS) + +Extractor = Callable[[RunTrace], list[SignalHit]] + + +@signal_extractor("complex_success") +def complex_success(trace: RunTrace, min_tool_calls: int = 5) -> list[SignalHit]: + calls = trace.tool_calls() + if trace.success and len(calls) >= min_tool_calls: + return [ + SignalHit( + signal="complex_success", + strength=min(1.0, len(calls) / (min_tool_calls * 2)), + evidence=f"{len(calls)} tool calls succeeded end-to-end", + meta={"tool_sequence": [c.name for c in calls]}, + ) + ] + return [] + + +@signal_extractor("recovery") +def recovery(trace: RunTrace) -> list[SignalHit]: + """Error followed later by a successful outcome — a workaround was found.""" + saw_error = False + error_detail = "" + for event in trace.events: + if event.type == "error" or (event.type == "tool_call" and not event.ok): + saw_error = True + error_detail = event.detail or event.name + if event.type == "recovery" and saw_error: + return [ + SignalHit( + signal="recovery", + strength=0.9, + evidence=f"recovered from: {error_detail}", + meta={"error": error_detail, "recovery": event.detail}, + ) + ] + if saw_error and trace.success: + return [ + SignalHit( + signal="recovery", + strength=0.6, + evidence=f"run succeeded despite error: {error_detail}", + meta={"error": error_detail}, + ) + ] + return [] + + +@signal_extractor("user_correction") +def user_correction(trace: RunTrace) -> list[SignalHit]: + hits = [] + for event in trace.events: + if event.type == "user_correction": + hits.append( + SignalHit( + signal="user_correction", + strength=1.0, + evidence=event.detail or "user corrected the approach", + meta={"detail": event.detail}, + ) + ) + return hits + + +@signal_extractor("repetition") +def repetition(trace: RunTrace, prior_signatures: dict[str, int] | None = None, + min_repeats: int = 3) -> list[SignalHit]: + """Same tool-sequence signature seen across runs. The host passes the + historical signature counts (e.g. aggregated from the spine).""" + prior = trace.events and (prior_signatures or trace.__dict__.get("_prior_signatures")) + if not prior: + return [] + sig = trace.signature() + seen = prior.get(sig, 0) + if trace.success and seen + 1 >= min_repeats: + return [ + SignalHit( + signal="repetition", + strength=min(1.0, (seen + 1) / (min_repeats * 2)), + evidence=f"tool sequence repeated {seen + 1} times", + meta={"signature": sig, "count": seen + 1}, + ) + ] + return [] + + +def extract_signals(trace: RunTrace, + prior_signatures: dict[str, int] | None = None) -> list[SignalHit]: + hits: list[SignalHit] = [] + for name, extractor in SIGNAL_EXTRACTORS.items(): + if name == "repetition": + hits.extend(extractor(trace, prior_signatures=prior_signatures)) + else: + hits.extend(extractor(trace)) + return hits diff --git a/src/xgen_sdk/harness/skill_forge/synthesis.py b/src/xgen_sdk/harness/skill_forge/synthesis.py new file mode 100644 index 0000000..204c648 --- /dev/null +++ b/src/xgen_sdk/harness/skill_forge/synthesis.py @@ -0,0 +1,120 @@ +"""SF3 — synthesizers: SkillCandidate -> SkillDef artifact. + +Three kinds (the ladder Hermes/Geny don't have): +- guide : SKILL.md-style markdown (agentskills.io-compatible sections) +- config : a HarnessConfig fragment (stage_params / criteria preset) as JSON +- tool : a compile manifest pointing at the workflow->npm->MCP path + +An optional `reflector` callable (LLM seam, same shape as forge's GEPA seam) +may polish the drafted body; synthesis must still work with reflector=None. +""" + +from __future__ import annotations + +import json +from typing import Callable + +from .model import SkillCandidate, SkillDef +from .registry import GROUP_SYNTHESIZERS, Registry, registry_decorator + +SYNTHESIZERS = Registry(GROUP_SYNTHESIZERS) +synthesizer = registry_decorator(SYNTHESIZERS) + +Reflector = Callable[[str, str], str] # (purpose, draft) -> improved draft + + +def _frontmatter(candidate: SkillCandidate, description: str) -> str: + return ( + "---\n" + f"name: {candidate.name}\n" + f"description: {description[:60]}\n" + "version: 0.1.0\n" + f"kind: {candidate.kind}\n" + f"scope: {candidate.scope}\n" + f"origin: {candidate.provenance.origin if candidate.provenance else 'manual'}\n" + "---\n" + ) + + +def _section(title: str, lines: list[str]) -> str: + if not lines: + return "" + body = "\n".join(f"- {line}" for line in lines) + return f"\n## {title}\n{body}\n" + + +@synthesizer("guide") +def synthesize_guide(candidate: SkillCandidate, + reflector: Reflector | None = None) -> SkillDef: + description = candidate.when_to_use or candidate.rationale + body = _frontmatter(candidate, description) + body += f"\n# {candidate.name}\n\n{candidate.rationale}\n" + body += _section("When to Use", [candidate.when_to_use] if candidate.when_to_use else []) + body += _section("Procedure", candidate.procedure) + body += _section("Pitfalls", candidate.pitfalls) + body += _section("Verification", candidate.verification) + if reflector is not None: + body = reflector("guide-skill", body) + return SkillDef( + name=candidate.name, + kind="guide", + scope=candidate.scope, + description=description[:200], + body=body, + provenance=candidate.provenance, + ) + + +@synthesizer("config") +def synthesize_config(candidate: SkillCandidate, + reflector: Reflector | None = None) -> SkillDef: + fragment = candidate.payload.get("config_fragment") + if not isinstance(fragment, dict) or not fragment: + raise ValueError("config candidate requires payload['config_fragment'] dict") + body = json.dumps( + { + "name": candidate.name, + "when_to_use": candidate.when_to_use, + "fragment": fragment, + }, + ensure_ascii=False, + indent=2, + ) + return SkillDef( + name=candidate.name, + kind="config", + scope=candidate.scope, + description=(candidate.when_to_use or candidate.rationale)[:200], + body=body, + provenance=candidate.provenance, + meta={"fragment_keys": sorted(fragment)}, + ) + + +@synthesizer("tool") +def synthesize_tool(candidate: SkillCandidate, + reflector: Reflector | None = None) -> SkillDef: + workflow = candidate.payload.get("workflow_ref") + if not workflow: + raise ValueError("tool candidate requires payload['workflow_ref']") + manifest = { + "name": candidate.name, + "when_to_use": candidate.when_to_use, + "workflow_ref": workflow, + "compile": {"target": "npm-mcp", "entry": "compile_workflow_to_npm"}, + } + return SkillDef( + name=candidate.name, + kind="tool", + scope=candidate.scope, + description=(candidate.when_to_use or candidate.rationale)[:200], + body=json.dumps(manifest, ensure_ascii=False, indent=2), + provenance=candidate.provenance, + meta={"workflow_ref": workflow}, + ) + + +def synthesize(candidate: SkillCandidate, + reflector: Reflector | None = None) -> SkillDef: + fn = SYNTHESIZERS.get(candidate.kind) + return fn(candidate, reflector=reflector) diff --git a/src/xgen_sdk/harness/tools/skill_registry.py b/src/xgen_sdk/harness/tools/skill_registry.py index e533206..9518a1c 100644 --- a/src/xgen_sdk/harness/tools/skill_registry.py +++ b/src/xgen_sdk/harness/tools/skill_registry.py @@ -287,12 +287,28 @@ def get_skill_body(name: str) -> Optional[str]: """Skill body lookup. None 이면 등록 안 됨.""" - return _BUILTIN_SKILL_BODIES.get(name) + body = _BUILTIN_SKILL_BODIES.get(name) + if body is not None: + return body + for source in _LEDGER_SOURCES: + try: + return source.view(name) + except KeyError: + continue + except Exception as e: + logger.warning("[skills] ledger source view(%s) 실패: %s", name, e) + return None def list_skill_names() -> list[str]: """등록된 skill 이름 list. discover 용.""" - return sorted(_BUILTIN_SKILL_BODIES.keys()) + names = set(_BUILTIN_SKILL_BODIES.keys()) + for source in _LEDGER_SOURCES: + try: + names.update(item.name for item in source.list()) + except Exception as e: + logger.warning("[skills] ledger source list 실패: %s", e) + return sorted(names) def register_skill_body(name: str, body: str) -> None: @@ -307,6 +323,22 @@ def register_skill_body(name: str, body: str) -> None: logger.debug("[skills] registered skill body: %s (%d chars)", name, len(body)) +_LEDGER_SOURCES: list = [] + + +def register_ledger_source(source) -> None: + """Jermes skill-forge 원장(spine 등)을 회상 소스로 주입 — 빌트인/코드등록/ + entry_points 에 이은 4번째 소스. + + source 계약 = ``xgen_sdk.harness.skill_forge.recall.LedgerSkillSource``: + list() -> 메타데이터(name/description), view(name) -> 본문(미존재시 KeyError). + staged(미검증) 스킬 노출 여부는 source 생성 시 정책으로 결정. + """ + if source is not None and source not in _LEDGER_SOURCES: + _LEDGER_SOURCES.append(source) + logger.debug("[skills] ledger source registered: %r", type(source).__name__) + + def _discover_from_entry_points() -> None: """entry_points 그룹 ``xgen_harness.skill_bodies`` 자동 발견. From 6bf41c7bbe9c0dfa9f5899c84130e500fc553a1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=84=EC=88=98?= Date: Fri, 24 Jul 2026 14:14:07 +0900 Subject: [PATCH 2/9] =?UTF-8?q?harness/skill=5Fforge:=20=EA=B2=B0=ED=95=A8?= =?UTF-8?q?=EC=88=98=EC=A0=95=205=EA=B1=B4=20=ED=9D=A1=EC=88=98=20(?= =?UTF-8?q?=EC=9B=90=EB=B3=B8=20v0.6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit D1 검증 active 스킬의 미검증 강등 차단 / D2 drafted=[] 폴백 금지 의미론 / D3 view() 순수 본문(frontmatter 오염 제거, 콘솔용 view_annotated 분리) / D4 repetition __dict__ 백도어 제거 / D5 앙상블 샘플별 관점 변주. 원본 engine/skill-forge-engineering 54 green, 라이브 스택 스모크 통과. --- src/xgen_sdk/harness/skill_forge/drafter.py | 18 +++++++++++++++--- src/xgen_sdk/harness/skill_forge/loop.py | 15 ++++++++++++--- src/xgen_sdk/harness/skill_forge/recall.py | 9 +++++++++ src/xgen_sdk/harness/skill_forge/signals.py | 5 ++--- 4 files changed, 38 insertions(+), 9 deletions(-) diff --git a/src/xgen_sdk/harness/skill_forge/drafter.py b/src/xgen_sdk/harness/skill_forge/drafter.py index 2559471..a81d69d 100644 --- a/src/xgen_sdk/harness/skill_forge/drafter.py +++ b/src/xgen_sdk/harness/skill_forge/drafter.py @@ -148,8 +148,11 @@ def _complete_array(self, prompt: str) -> list: items = _extract_json_array(repaired) return items - def draft(self, trace: RunTrace, hits: list[SignalHit]) -> list[SkillCandidate]: + def draft(self, trace: RunTrace, hits: list[SignalHit], + variant: str = "") -> list[SkillCandidate]: prompt = build_prompt(trace, hits, self.max_candidates) + if variant: + prompt += f"\n\nAttempt focus: {variant}" items: list = [] feedback = "" for attempt in range(self.max_retries + 1): @@ -200,6 +203,14 @@ class EnsembleDrafter: it deterministically, which is exactly what makes a weak drafter viable: breadth from sampling, quality from selection pressure.""" + VARIANTS = ( + "", + "propose a different angle than the most obvious one", + "focus on prevention and verification steps rather than the happy path", + "focus on what the user corrected or what almost went wrong", + "focus on ordering constraints between the tools used", + ) + def __init__(self, drafter: LLMDrafter, samples: int = 3, max_candidates: int = 4) -> None: self.drafter = drafter @@ -208,8 +219,9 @@ def __init__(self, drafter: LLMDrafter, samples: int = 3, def draft(self, trace: RunTrace, hits: list[SignalHit]) -> list[SkillCandidate]: pool: list[SkillCandidate] = [] - for _ in range(self.samples): - pool.extend(self.drafter.draft(trace, hits)) + for index in range(self.samples): + variant = self.VARIANTS[index % len(self.VARIANTS)] + pool.extend(self.drafter.draft(trace, hits, variant=variant)) deduped: list[SkillCandidate] = [] seen_names: set[str] = set() for candidate in pool: diff --git a/src/xgen_sdk/harness/skill_forge/loop.py b/src/xgen_sdk/harness/skill_forge/loop.py index 3641804..dc4fce3 100644 --- a/src/xgen_sdk/harness/skill_forge/loop.py +++ b/src/xgen_sdk/harness/skill_forge/loop.py @@ -81,9 +81,18 @@ def process_trace(self, trace: RunTrace, skill.status = self.approval.on_promoted(skill) self.ledger.commit(skill, note=f"promoted: {gate_result.reasons[0]}") elif gate_result.verdict == "staged": - skill.verified = False - skill.status = "staged" - self.ledger.commit(skill, note=f"staged: {'; '.join(gate_result.reasons)}") + existing = self.ledger.get(skill.name) + if (existing is not None and existing.skill.verified + and existing.status == "active"): + # never let an unverified redraft downgrade a verified + # active skill — keep the proven version untouched + gate_result.reasons.append( + "kept existing verified active version; unverified draft dropped") + else: + skill.verified = False + skill.status = "staged" + self.ledger.commit( + skill, note=f"staged: {'; '.join(gate_result.reasons)}") # rejected candidates are journaled in the episode, not the ledger episode.results.append((skill, gate_result)) return episode diff --git a/src/xgen_sdk/harness/skill_forge/recall.py b/src/xgen_sdk/harness/skill_forge/recall.py index c714178..977e55f 100644 --- a/src/xgen_sdk/harness/skill_forge/recall.py +++ b/src/xgen_sdk/harness/skill_forge/recall.py @@ -51,6 +51,15 @@ def list(self) -> list[SkillListing]: return sorted(listings, key=lambda item: (-item.rank, item.name)) def view(self, name: str) -> str: + """Raw body only — this feeds the prompt-injection path (skill_registry + get_skill_body), so no annotation header may precede the frontmatter.""" + record = self.ledger.get(name) + if record is None: + raise KeyError(f"unknown skill {name!r}") + return record.skill.body + + def view_annotated(self, name: str) -> str: + """Human-facing variant with the verification label (console use).""" record = self.ledger.get(name) if record is None: raise KeyError(f"unknown skill {name!r}") diff --git a/src/xgen_sdk/harness/skill_forge/signals.py b/src/xgen_sdk/harness/skill_forge/signals.py index 576d4e2..7a441e0 100644 --- a/src/xgen_sdk/harness/skill_forge/signals.py +++ b/src/xgen_sdk/harness/skill_forge/signals.py @@ -94,11 +94,10 @@ def repetition(trace: RunTrace, prior_signatures: dict[str, int] | None = None, min_repeats: int = 3) -> list[SignalHit]: """Same tool-sequence signature seen across runs. The host passes the historical signature counts (e.g. aggregated from the spine).""" - prior = trace.events and (prior_signatures or trace.__dict__.get("_prior_signatures")) - if not prior: + if not trace.events or not prior_signatures: return [] sig = trace.signature() - seen = prior.get(sig, 0) + seen = prior_signatures.get(sig, 0) if trace.success and seen + 1 >= min_repeats: return [ SignalHit( From edc89a0459c03f0a836ec1306e693b2514a90985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=84=EC=88=98?= Date: Mon, 27 Jul 2026 08:37:58 +0900 Subject: [PATCH 3/9] =?UTF-8?q?harness/skill=5Fforge:=20=EB=93=9C=EB=9E=98?= =?UTF-8?q?=ED=94=84=ED=84=B0=20=EC=A1=B0=EC=9A=A9=ED=95=9C=20=EC=8B=A4?= =?UTF-8?q?=ED=8C=A8=20=EC=A0=9C=EA=B1=B0=20=ED=9D=A1=EC=88=98=20(?= =?UTF-8?q?=EC=9B=90=EB=B3=B8=20v0.7)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit completer 예외(HTTP 본문 포함)·파싱 실패·수리 실패를 WARNING 으로 노출. 원본 engine/skill-forge-engineering 56 green. --- src/xgen_sdk/harness/skill_forge/drafter.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/xgen_sdk/harness/skill_forge/drafter.py b/src/xgen_sdk/harness/skill_forge/drafter.py index a81d69d..93efe25 100644 --- a/src/xgen_sdk/harness/skill_forge/drafter.py +++ b/src/xgen_sdk/harness/skill_forge/drafter.py @@ -9,10 +9,13 @@ from __future__ import annotations import json +import logging import re import urllib.request from typing import Callable +logger = logging.getLogger("xgen_skill_forge.drafter") + from .model import Provenance, RunTrace, SkillCandidate from .signals import SignalHit @@ -144,8 +147,12 @@ def _complete_array(self, prompt: str) -> list: raw = self.complete(prompt) items = _extract_json_array(raw) if not items and self.repair and raw.strip() and raw.strip() != "[]": + logger.warning("[drafter] unparsable output, repairing. head=%r", + raw[:200]) repaired = self.complete(_REPAIR_PROMPT.format(raw=raw[:4000])) items = _extract_json_array(repaired) + if not items: + logger.warning("[drafter] repair failed. head=%r", repaired[:200]) return items def draft(self, trace: RunTrace, hits: list[SignalHit], @@ -160,7 +167,16 @@ def draft(self, trace: RunTrace, hits: list[SignalHit], prompt + _RETRY_SUFFIX.format(feedback=feedback)) try: items = self._complete_array(attempt_prompt) - except Exception: + except Exception as exc: + detail = "" + read = getattr(exc, "read", None) + if callable(read): + try: + detail = read().decode("utf-8", "replace")[:300] + except Exception: + detail = "" + logger.warning("[drafter] completion failed: %s: %s %s", + type(exc).__name__, str(exc)[:200], detail) return [] if items: break From 9ed5c821cad614b2ea6a8e193df049015f3e3ea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=84=EC=88=98?= Date: Mon, 27 Jul 2026 13:29:30 +0900 Subject: [PATCH 4/9] =?UTF-8?q?harness/skill=5Fforge:=20=EA=B2=8C=EC=9D=B4?= =?UTF-8?q?=ED=8A=B8=20=EC=97=84=EA=B2=A9=ED=99=94=20=ED=9D=A1=EC=88=98=20?= =?UTF-8?q?(=EC=9B=90=EB=B3=B8=20v0.8)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 승격에 holdout 이득>0 요구(require_holdout_gain). dev 만 오른 스킬은 staged 로 강등 — 일반화 증거 없는 자동 verified 를 구조적으로 차단. --- src/xgen_sdk/harness/skill_forge/gate.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/xgen_sdk/harness/skill_forge/gate.py b/src/xgen_sdk/harness/skill_forge/gate.py index f76359b..808a960 100644 --- a/src/xgen_sdk/harness/skill_forge/gate.py +++ b/src/xgen_sdk/harness/skill_forge/gate.py @@ -45,6 +45,12 @@ class GateConfig: max_holdout_drop: float = 0.02 overopt_gap: float = 0.25 # dev-holdout gain divergence alarm min_cases: int = 4 + require_holdout_gain: bool = True + """A dev-only gain is not evidence of a reusable skill — it is evidence of + fitting the cases the skill was written from. Verified promotion therefore + requires the gain to reproduce on held-out cases. When dev improves but + holdout stays flat the verdict is `staged` (unproven, human may approve), + not `promoted` and not `rejected`.""" class ForgeGate: @@ -93,7 +99,12 @@ def mean(cs: Sequence[BenchCase], skill_def: SkillDef | None) -> float: f"holdout {baseline_holdout:.3f}->{holdout_score:.3f} ({holdout_gain:+.3f})", ] if dev_up and held_ok and not overopt: - verdict = "promoted" + if self.config.require_holdout_gain and holdout_gain <= 0: + verdict = "staged" + reasons.append("dev gain did not reproduce on held-out cases — " + "unproven, not auto-verified") + else: + verdict = "promoted" elif not dev_up: verdict = "rejected" reasons.append("no dev gain — the skill does not help") From 2bb6a6a08bf46324d7568c97d0ba931da39f0759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=84=EC=88=98?= Date: Thu, 30 Jul 2026 01:33:08 +0900 Subject: [PATCH 5/9] =?UTF-8?q?absorb:=20skill=5Fforge=20=ED=9D=A1?= =?UTF-8?q?=EC=88=98=EB=B3=B8=20=EA=B0=B1=EC=8B=A0=20=E2=80=94=20drafter?= =?UTF-8?q?=20=EA=B4=80=EC=B8=A1=EC=84=B1=20=EB=A1=9C=EA=B7=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 엔진(jinsoo96/jermes `e1a0bff`)의 변경을 흡수본에 반영한다. "0건 초안"이 모델 빈배열 / 예시유출 가드 / 필드 파손 중 무엇인지 한 줄로 구분해 남긴다. 드리프트 테스트(test_absorption_drift)가 원본-흡수본 불일치를 실제로 잡아 `absorb.py --force` 로 재흡수한 결과다. Co-Authored-By: Claude Opus 5 (1M context) --- src/xgen_sdk/harness/skill_forge/drafter.py | 47 +++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/xgen_sdk/harness/skill_forge/drafter.py b/src/xgen_sdk/harness/skill_forge/drafter.py index 93efe25..6ec1206 100644 --- a/src/xgen_sdk/harness/skill_forge/drafter.py +++ b/src/xgen_sdk/harness/skill_forge/drafter.py @@ -184,8 +184,10 @@ def draft(self, trace: RunTrace, hits: list[SignalHit], candidates: list[SkillCandidate] = [] strongest = max((h.signal for h in sorted(hits, key=lambda h: -h.strength)), default="") + dropped_leak = dropped_invalid = 0 for item in items[: self.max_candidates]: if not isinstance(item, dict) or _is_example_leak(item): + dropped_leak += 1 continue try: candidates.append( @@ -208,7 +210,15 @@ def draft(self, trace: RunTrace, hits: list[SignalHit], ) ) except ValueError: + dropped_invalid += 1 continue + if hits: + # "0건"이 세 가지 다른 사건일 수 있다 — 모델이 빈 배열을 줬거나, + # 예시 유출 가드가 전부 걷어냈거나, 필드가 깨져 버려졌거나. + # 숫자로 나뉘지 않으면 약한 모델을 튜닝할 근거가 없다. + logger.info("[drafter] hits=%d · model=%d · leak=%d · invalid=%d · kept=%d", + len(hits), len(items), dropped_leak, dropped_invalid, + len(candidates)) return candidates @@ -258,6 +268,43 @@ def _near_duplicate(a: "SkillCandidate", b: "SkillCandidate") -> bool: return len(ta & tb) / min(len(ta), len(tb)) >= 0.7 +def failover_completer(completers: list[Completer]) -> Completer: + """Try each completer in order; remember the one that works. + + Observed failure this exists for: the box serving the primary model went + down and curation stopped silently — the loop kept "running" while every + draft returned nothing. With a second endpoint listed, learning continues. + + A completer that raises is treated as unhealthy and the next one is tried. + The last known-good index is tried first next time, so the healthy path + costs no extra calls. + """ + live = [c for c in completers if c is not None] + if not live: + raise ValueError("failover_completer needs at least one completer") + state = {"index": 0} + + def complete(prompt: str) -> str: + order = list(range(state["index"], len(live))) + \ + list(range(0, state["index"])) + last: Exception | None = None + for i in order: + try: + result = live[i](prompt) + except Exception as exc: + last = exc + logger.warning("[drafter] endpoint %d unhealthy: %s: %s", + i, type(exc).__name__, str(exc)[:120]) + continue + if i != state["index"]: + logger.warning("[drafter] failed over to endpoint %d", i) + state["index"] = i + return result + raise last if last else RuntimeError("no endpoint answered") + + return complete + + def anthropic_completer(model: str, api_key: str, max_tokens: int = 2048, timeout: float = 120.0) -> Completer: From db323d8b54deb4cf9cd30d550dab7548511553d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=84=EC=88=98?= Date: Thu, 30 Jul 2026 10:36:45 +0900 Subject: [PATCH 6/9] =?UTF-8?q?absorb:=20skill=5Fforge=20=E2=80=94=20agent?= =?UTF-8?q?skills.io=20=EC=83=81=ED=98=B8=EC=9A=B4=EC=9A=A9=20=EA=B3=84?= =?UTF-8?q?=EC=B8=B5=20=ED=9D=A1=EC=88=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 엔진(jinsoo96/jermes `20c468f`)의 portable 모듈과 스펙 위반 수정을 흡수한다. 플랫폼이 `xgen_sdk.harness.skill_forge.portable` 로 표준 SKILL.md 를 내보내고 들여올 수 있다. 검증 증거는 스펙이 허용한 metadata 로 싣는다. 같이 온 수정: description 60자 절단 제거(스펙 1024·발견 품질의 핵심), YAML 이 깨질 수 있던 날값 기록, 스펙에 없는 최상위 필드를 metadata 로 이동, 이름 절단 후 말미 하이픈 잔존. Co-Authored-By: Claude Opus 5 (1M context) --- src/xgen_sdk/harness/skill_forge/drafter.py | 7 +- src/xgen_sdk/harness/skill_forge/portable.py | 283 ++++++++++++++++++ src/xgen_sdk/harness/skill_forge/synthesis.py | 23 +- 3 files changed, 305 insertions(+), 8 deletions(-) create mode 100644 src/xgen_sdk/harness/skill_forge/portable.py diff --git a/src/xgen_sdk/harness/skill_forge/drafter.py b/src/xgen_sdk/harness/skill_forge/drafter.py index 6ec1206..b2b1ed4 100644 --- a/src/xgen_sdk/harness/skill_forge/drafter.py +++ b/src/xgen_sdk/harness/skill_forge/drafter.py @@ -102,9 +102,10 @@ def _is_example_leak(item: dict) -> bool: def _sanitize_name(raw: str) -> str: - name = re.sub(r"[^a-z0-9-]", "-", raw.strip().lower().replace("_", "-").replace(" ", "-")) - name = re.sub(r"-{2,}", "-", name).strip("-") - return name[:64] or "unnamed-skill" + # agentskills.io 이름 규칙을 그대로 따른다(64자·소문자 영숫자·하이픈, 앞뒤와 + # 연속 하이픈 금지). 규칙을 어기면 다른 에이전트가 스킬을 조용히 무시한다. + from .portable import spec_name + return spec_name(raw) def _extract_json_array(text: str) -> list: diff --git a/src/xgen_sdk/harness/skill_forge/portable.py b/src/xgen_sdk/harness/skill_forge/portable.py new file mode 100644 index 0000000..dc1982e --- /dev/null +++ b/src/xgen_sdk/harness/skill_forge/portable.py @@ -0,0 +1,283 @@ +"""SF6 — agentskills.io interop: 원장 스킬 <-> `SKILL.md` 패키지. + +**왜 이게 전략적으로 중요한가.** Agent Skills 는 앤트로픽이 만들어 공개 표준으로 +풀었고 Claude Code·Cursor·Copilot·VS Code·Gemini CLI·Codex·OpenHands·Goose·Letta +등 수십 개 제품이 채택했다. 그런데 표준이 정의한 검사는 `skills-ref validate` — +**프론트매터 문법과 이름 규칙만** 본다. "이 스킬이 실제로 효과가 있나"는 표준에 +없다. Hermes 도 자동 생성만 하고 검증은 없다. + +그래서 이 모듈의 역할은 두 방향이다. +- **내보내기**: 홀드아웃 게이트를 통과한 스킬을 스펙 호환 `SKILL.md` 로 내보낸다. + → 검증된 스킬이 45개 제품 어디서나 쓰인다. 증거는 스펙이 허용한 자유 필드 + `metadata` 에 실어 보내므로 표준을 깨지 않으면서 출처·이득이 따라간다. +- **들여오기**: 남이 만든 `SKILL.md` 를 후보로 받아 재현벤치에 태운다. + → 생태계의 어떤 스킬이든 "효과가 실측된 것"과 아닌 것을 가를 수 있다. + 아무도 안 하는 일이고, 우리 게이트가 이미 그걸 한다. + +스펙 준수는 타협 대상이 아니다 — 규칙을 어긴 프론트매터는 다른 에이전트가 조용히 +무시하거나 거부한다. 그래서 내보내기 전에 스스로 검증하고, 위반이면 예외를 던진다. + +스펙 출처: https://agentskills.io/specification (2026-07 확인) +""" + +from __future__ import annotations + +import json +import re +from typing import Any + +from .model import Provenance, SkillCandidate, SkillDef + +# 스펙 상한 — 넘으면 다른 클라이언트가 거부한다. +MAX_NAME = 64 +MAX_DESCRIPTION = 1024 +MAX_COMPATIBILITY = 500 + +# name: 소문자 영숫자와 하이픈, 앞뒤 하이픈 금지, 연속 하이픈 금지. +_NAME_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") + +# metadata 키는 충돌을 피하라고 스펙이 권고한다 — 우리 것은 전부 이 접두를 쓴다. +META_PREFIX = "xgen-jermes-" + + +def validate_name(name: str) -> list[str]: + problems: list[str] = [] + if not name: + problems.append("name 이 비어 있다") + return problems + if len(name) > MAX_NAME: + problems.append(f"name 이 {MAX_NAME}자를 넘는다({len(name)})") + if not _NAME_RE.match(name): + problems.append( + "name 은 소문자 영숫자와 하이픈만 쓰고, 앞뒤 하이픈·연속 하이픈이 없어야 한다" + f" (받은 값: {name!r})") + return problems + + +def validate_description(description: str) -> list[str]: + problems: list[str] = [] + text = (description or "").strip() + if not text: + problems.append("description 이 비어 있다(스펙상 필수·비어있으면 안 됨)") + if len(text) > MAX_DESCRIPTION: + problems.append(f"description 이 {MAX_DESCRIPTION}자를 넘는다({len(text)})") + return problems + + +def spec_name(raw: str) -> str: + """임의 문자열을 스펙에 맞는 name 으로 접는다. + + `_sanitize_name` 이 64자로 자르면서 하이픈으로 끝나 스펙을 어길 수 있었다 — + 자른 다음에 다시 깎아야 한다(순서가 중요). + """ + name = re.sub(r"[^a-z0-9-]", "-", raw.strip().lower().replace("_", "-")) + name = re.sub(r"-{2,}", "-", name).strip("-") + if len(name) > MAX_NAME: + name = name[:MAX_NAME].rstrip("-") # 자른 뒤 재정리 + return name or "unnamed-skill" + + +def _yaml_scalar(value: str) -> str: + """의존성 없이 YAML 스칼라를 안전하게 쓴다. + + 콜론·해시·따옴표·개행이 들어간 description 을 날값으로 적으면 프론트매터가 + 깨져 스킬이 통째로 무시된다. 여러 줄은 한 줄로 접는다(프론트매터는 단행이 + 안전하고, 본문에 이미 전체 서술이 있다). + """ + text = re.sub(r"\s+", " ", str(value)).strip() + if not text: + return '""' + if re.search(r'[:#\-\[\]{}&*!|>%@`"\']', text) or text[0] in "?,": + return '"' + text.replace("\\", "\\\\").replace('"', '\\"') + '"' + return text + + +def _yaml_string(value: Any) -> str: + """항상 따옴표로 감싼 YAML 문자열 — metadata 값 전용.""" + text = re.sub(r"\s+", " ", str(value)).strip() + return '"' + text.replace("\\", "\\\\").replace('"', '\\"') + '"' + + +def _body_without_frontmatter(body: str) -> str: + """우리 내부 body 에 붙어 있던 예전 프론트매터를 떼어낸다. + + 안 떼면 내보낸 파일에 프론트매터가 두 번 들어가고, 두 번째 것은 본문 + 텍스트로 읽혀 프롬프트를 오염시킨다.""" + text = (body or "").lstrip() + if not text.startswith("---"): + return body or "" + end = text.find("\n---", 3) + if end == -1: + return body or "" + return text[end + 4:].lstrip("\n") + + +def to_skill_md(skill: SkillDef, *, evidence: dict[str, Any] | None = None, + license_name: str = "") -> str: + """원장 스킬 -> 스펙 호환 `SKILL.md` 텍스트. + + 검증 증거는 `metadata` 로 간다 — 스펙이 정의하지 않은 필드를 최상위에 쓰면 + 호환이 깨지지만, `metadata` 는 자유 키/값 맵이라 표준을 지키면서 실을 수 있다. + """ + problems = validate_name(skill.name) + validate_description(skill.description) + if problems: + raise ValueError("스펙 위반으로 내보낼 수 없다: " + "; ".join(problems)) + + meta: dict[str, str] = { + f"{META_PREFIX}kind": skill.kind, + f"{META_PREFIX}scope": skill.scope, + f"{META_PREFIX}version": skill.version, + f"{META_PREFIX}status": skill.status, + # 이 표준에 없는 단 하나의 정보 = 효과가 실측됐는지. + f"{META_PREFIX}verified": "true" if skill.verified else "false", + } + if skill.supersedes: + meta[f"{META_PREFIX}supersedes"] = skill.supersedes + if skill.provenance: + prov = skill.provenance + if getattr(prov, "origin", ""): + meta[f"{META_PREFIX}origin"] = prov.origin + runs = list(getattr(prov, "source_run_ids", []) or []) + if runs: + meta[f"{META_PREFIX}source-runs"] = ",".join(runs[:8]) + if getattr(prov, "signal", ""): + meta[f"{META_PREFIX}signal"] = prov.signal + for key, value in (evidence or {}).items(): + meta[f"{META_PREFIX}{key}"] = str(value) + + lines = ["---", f"name: {skill.name}", + f"description: {_yaml_scalar(skill.description)}"] + if license_name: + lines.append(f"license: {_yaml_scalar(license_name)}") + lines.append("metadata:") + for key in sorted(meta): + # 스펙: metadata 는 "문자열 키 -> 문자열 값" 맵. 날값으로 적으면 true/0.34 가 + # 불리언·숫자로 파싱돼 타입 계약이 깨진다 — 항상 따옴표로 감싼다. + lines.append(f" {key}: {_yaml_string(meta[key])}") + lines.append("---") + + body = _body_without_frontmatter(skill.body).rstrip() + if skill.kind != "guide": + # config/tool 은 마크다운이 아니라 JSON/매니페스트다. 그대로 흘리면 다른 + # 에이전트가 지시문으로 읽으므로 코드블록으로 감싸 설명을 붙인다. + body = (f"# {skill.name}\n\n{skill.description}\n\n" + f"## Payload ({skill.kind})\n\n```json\n{body}\n```\n") + elif not body: + body = f"# {skill.name}\n\n{skill.description}\n" + return "\n".join(lines) + "\n\n" + body + "\n" + + +def parse_skill_md(text: str) -> tuple[dict[str, Any], str]: + """`SKILL.md` -> (프론트매터 dict, 본문). 의존성 0으로 필요한 만큼만 읽는다. + + 중첩은 `metadata:` 한 단계만 지원한다 — 스펙이 정의한 구조가 그것뿐이다. + """ + raw = (text or "").lstrip() + if not raw.startswith("---"): + raise ValueError("프론트매터가 없다(`---` 로 시작해야 한다)") + end = raw.find("\n---", 3) + if end == -1: + raise ValueError("프론트매터가 닫히지 않았다") + head, body = raw[3:end], raw[end + 4:].lstrip("\n") + + front: dict[str, Any] = {} + current: dict[str, Any] | None = None + for line in head.splitlines(): + if not line.strip() or line.lstrip().startswith("#"): + continue + indented = line[:1] in (" ", "\t") + if ":" not in line: + continue + key, _, value = line.strip().partition(":") + key, value = key.strip(), value.strip() + if indented and current is not None: + current[key] = _unquote(value) + continue + if not value: + current = {} + front[key] = current + continue + current = None + front[key] = _unquote(value) + return front, body + + +def _unquote(value: str) -> str: + text = value.strip() + if len(text) >= 2 and text[0] == text[-1] and text[0] in "\"'": + inner = text[1:-1] + return inner.replace('\\"', '"').replace("\\\\", "\\") + return text + + +def validate_skill_md(text: str) -> list[str]: + """스펙 위반 목록. 빈 리스트면 다른 에이전트가 받아준다는 뜻.""" + try: + front, _ = parse_skill_md(text) + except ValueError as exc: + return [str(exc)] + problems = validate_name(str(front.get("name", ""))) + problems += validate_description(str(front.get("description", ""))) + compatibility = str(front.get("compatibility", "") or "") + if len(compatibility) > MAX_COMPATIBILITY: + problems.append(f"compatibility 가 {MAX_COMPATIBILITY}자를 넘는다") + metadata = front.get("metadata") + if metadata is not None and not isinstance(metadata, dict): + problems.append("metadata 는 키/값 맵이어야 한다") + return problems + + +def candidate_from_skill_md(text: str, *, scope: str = "user", + curator_id: str = "skill-md-import") -> SkillCandidate: + """남의 `SKILL.md` 를 우리 후보로 들여온다 — 게이트에 태우기 위한 입구. + + 들여올 때 `verified` 는 절대 믿지 않는다. 남이 스스로 붙인 표시이고, + 검증은 우리 벤치가 이 환경에서 다시 해야 의미가 있다(그게 차별점이다). + """ + problems = validate_skill_md(text) + if problems: + raise ValueError("들여올 수 없는 SKILL.md: " + "; ".join(problems)) + front, body = parse_skill_md(text) + metadata = front.get("metadata") if isinstance(front.get("metadata"), dict) else {} + return SkillCandidate( + name=spec_name(str(front["name"])), + kind="guide", + scope=scope, + action="create", + rationale=str(front.get("description", ""))[:500], + when_to_use=str(front.get("description", ""))[:300], + procedure=_body_steps(body) or ["(본문 참조)"], + verification=[], + payload={"imported_body": body, + "imported_metadata": dict(metadata), + "claimed_verified": str(metadata.get(f"{META_PREFIX}verified", "")), + "license": str(front.get("license", ""))}, + provenance=Provenance(origin="skill_md_import", curator_id=curator_id), + ) + + +def _body_steps(body: str) -> list[str]: + """본문에서 절차로 보이는 줄만 뽑는다(불릿·번호). 없으면 빈 목록.""" + steps: list[str] = [] + for line in (body or "").splitlines(): + stripped = line.strip() + if re.match(r"^(?:[-*+]\s+|\d+[.)]\s+)", stripped): + steps.append(re.sub(r"^(?:[-*+]\s+|\d+[.)]\s+)", "", stripped)[:300]) + if len(steps) >= 12: + break + return steps + + +def skill_package(skill: SkillDef, *, evidence: dict[str, Any] | None = None, + license_name: str = "") -> dict[str, str]: + """{상대경로: 내용} — 파일시스템에 쓰든 zip 으로 묶든 호출측이 결정한다. + + 디렉토리 이름은 `name` 과 같아야 한다는 스펙 규칙을 여기서 지킨다. + """ + text = to_skill_md(skill, evidence=evidence, license_name=license_name) + files = {f"{skill.name}/SKILL.md": text} + if skill.kind != "guide": + # 원본 payload 도 같이 실어야 기계가 다시 쓸 수 있다(본문 코드블록은 사람용). + files[f"{skill.name}/assets/payload.json"] = _body_without_frontmatter( + skill.body).strip() or json.dumps({}) + return files diff --git a/src/xgen_sdk/harness/skill_forge/synthesis.py b/src/xgen_sdk/harness/skill_forge/synthesis.py index 204c648..675600b 100644 --- a/src/xgen_sdk/harness/skill_forge/synthesis.py +++ b/src/xgen_sdk/harness/skill_forge/synthesis.py @@ -24,14 +24,27 @@ def _frontmatter(candidate: SkillCandidate, description: str) -> str: + """내부 body 의 프론트매터. + + 두 가지를 고쳤다. ①description 을 60자로 자르고 있었는데 스펙 상한은 1024 이고 + 이 필드가 곧 발견 품질이다(에이전트는 name+description 만 보고 활성화를 + 결정한다) — 자르면 안 뜬다. ②콜론이 들어간 description 을 날값으로 적으면 + YAML 이 깨져 스킬이 통째로 무시된다. + + 스펙에 없는 kind/scope/origin 은 최상위에 두면 호환이 깨지므로 `metadata` + 아래로 내렸다. 내보내기 전용 정본은 portable.to_skill_md 다. + """ + from .portable import META_PREFIX, MAX_DESCRIPTION, _yaml_scalar + origin = candidate.provenance.origin if candidate.provenance else "manual" return ( "---\n" f"name: {candidate.name}\n" - f"description: {description[:60]}\n" - "version: 0.1.0\n" - f"kind: {candidate.kind}\n" - f"scope: {candidate.scope}\n" - f"origin: {candidate.provenance.origin if candidate.provenance else 'manual'}\n" + f"description: {_yaml_scalar(description[:MAX_DESCRIPTION])}\n" + "metadata:\n" + f" {META_PREFIX}version: 0.1.0\n" + f" {META_PREFIX}kind: {candidate.kind}\n" + f" {META_PREFIX}scope: {candidate.scope}\n" + f" {META_PREFIX}origin: {origin}\n" "---\n" ) From 640da9dd39133c55e57155abf04c1cac2335dafb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=84=EC=88=98?= Date: Tue, 4 Aug 2026 18:42:54 +0900 Subject: [PATCH 7/9] =?UTF-8?q?absorb:=20skill=5Fforge=20=E2=80=94=20?= =?UTF-8?q?=EC=97=90=EC=9D=B4=EC=A0=84=ED=8A=B8=20fa=C3=A7ade=20=C2=B7=20?= =?UTF-8?q?=EA=B8=B0=EC=96=B5=20=EA=B1=B0=EB=B2=84=EB=84=8C=EC=8A=A4=20?= =?UTF-8?q?=C2=B7=20=EA=B7=9C=EC=95=BD=20=EC=A7=91=ED=96=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 엔진(jinsoo96/jermes `d29df3b`)을 흡수한다. - agent: JermesAgent — remember→학습→측정→화해→보고를 한 사이클로 - memory: 증거로 등급을 매기는 기억(측정만이 trust 를 움직임, 모순은 벤치로 판정, 자동 삭제 없음, 중립 감쇠로 경화 방지) - constitution: never_learn 을 게이트가 집행. 에이전트는 자기 규약을 못 고친다 - gate: 규약 검사를 벤치 앞단에 배치(선택 인자라 기존 호출부 무영향) Co-Authored-By: Claude Opus 5 (1M context) --- src/xgen_sdk/harness/skill_forge/__init__.py | 34 ++- src/xgen_sdk/harness/skill_forge/agent.py | 256 ++++++++++++++++ .../harness/skill_forge/constitution.py | 170 +++++++++++ src/xgen_sdk/harness/skill_forge/gate.py | 12 +- src/xgen_sdk/harness/skill_forge/memory.py | 282 ++++++++++++++++++ 5 files changed, 752 insertions(+), 2 deletions(-) create mode 100644 src/xgen_sdk/harness/skill_forge/agent.py create mode 100644 src/xgen_sdk/harness/skill_forge/constitution.py create mode 100644 src/xgen_sdk/harness/skill_forge/memory.py diff --git a/src/xgen_sdk/harness/skill_forge/__init__.py b/src/xgen_sdk/harness/skill_forge/__init__.py index 228b2e6..31504a7 100644 --- a/src/xgen_sdk/harness/skill_forge/__init__.py +++ b/src/xgen_sdk/harness/skill_forge/__init__.py @@ -4,12 +4,15 @@ Design: D:\\harness-engineering\\SKILL-FORGE-PLAN.md """ +from .agent import ContextPack, CycleReport, JermesAgent, RecalledSkill from .bench import Expectation, ReplayCase, ReproReplayRunner, cases_from_repro_rows +from .constitution import Constitution from .curator import Curator, CurationResult, Rejection from .drafter import ( EnsembleDrafter, LLMDrafter, anthropic_completer, + failover_completer, openai_chat_completer, ) from .gate import BenchCase, BenchRunner, ForgeGate, GateConfig @@ -22,6 +25,14 @@ ) from .ledger import InMemorySkillLedger, JsonlSkillLedger, SkillLedger, SkillRecord from .loop import ApprovalPolicy, ForgeEpisode, SkillForge +from .memory import ( + Contradiction, + MemoryItem, + MemoryPolicy, + detect_contradictions, + measure, + resolve, +) from .model import ( GateResult, Provenance, @@ -31,6 +42,12 @@ TraceEvent, UsageStats, ) +from .portable import ( + candidate_from_skill_md, + skill_package, + to_skill_md, + validate_skill_md, +) from .recall import LedgerSkillSource, SkillListing from .signals import SIGNAL_EXTRACTORS, SignalHit, extract_signals from .synthesis import SYNTHESIZERS, synthesize @@ -41,8 +58,12 @@ "ApprovalPolicy", "BenchCase", "BenchRunner", - "Curator", + "Constitution", + "ContextPack", + "Contradiction", "CurationResult", + "Curator", + "CycleReport", "EnsembleDrafter", "Expectation", "ForgeEpisode", @@ -51,10 +72,14 @@ "GateResult", "InMemorySkillLedger", "InMemorySpineStore", + "JermesAgent", "JsonlSkillLedger", "LLMDrafter", "LedgerSkillSource", + "MemoryItem", + "MemoryPolicy", "Provenance", + "RecalledSkill", "Rejection", "ReplayCase", "ReproReplayRunner", @@ -73,10 +98,17 @@ "TraceEvent", "UsageStats", "anthropic_completer", + "candidate_from_skill_md", "cases_from_repro_rows", + "detect_contradictions", "extract_signals", + "measure", "openai_chat_completer", + "resolve", "signature_counts", + "skill_package", "synthesize", + "to_skill_md", "trace_from_spine", + "validate_skill_md", ] diff --git a/src/xgen_sdk/harness/skill_forge/agent.py b/src/xgen_sdk/harness/skill_forge/agent.py new file mode 100644 index 0000000..c083a50 --- /dev/null +++ b/src/xgen_sdk/harness/skill_forge/agent.py @@ -0,0 +1,256 @@ +"""Jermes — 하나의 에이전트. 흩어진 계층을 한 사이클로 묶는다. + +지금까지 모듈은 다 있었지만 "에이전트"는 없었다. 호스트가 신호·초안·게이트·원장· +회상을 매번 손으로 이어붙였다. 여기가 그 조립을 한 자리로 모은다. + +**Hermes / Geny 와 다른 로직 네 가지** — 말이 아니라 이 파일의 코드가 지키는 것: + +1. **기억과 스킬에 같은 잣대.** 둘 다 `BenchCase` + 점수 함수로 잰다. Hermes 는 스킬을 + 자동 생성하지만 검증이 없고, Geny 는 기억을 대리 신호(조회·편집)로 등급 매기며 + 스킬 학습이 없다. 여기서는 **둘 다 "빼고 재생해서 나빠지면 값어치가 있다"** 로 잰다. +2. **딱지 없이는 컨텍스트에 못 들어간다.** 회상 결과는 검증됨/미검증이 표시된 채로 + 나간다. 라벨 없는 주입은 모델이 추측을 사실로 믿게 만든다. +3. **자동 삭제 없음.** 스킬도 기억도 내려갈 때 `disputed`/`staged` 로 남고 이력이 붙는다. + 되돌릴 수 없는 자동 조치는 하지 않는다. +4. **모든 0 에는 이유가 있다.** 사이클 보고는 숫자로 말한다 — 신호 0인지, 초안 0인지, + 케이스가 모자라 못 잰 건지. (라이브에서 "성공인데 아무것도 안 배움"을 세 번 겪고 얻은 규칙.) + +순수·동기 함수다. 일정과 영속은 호스트가 정한다. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Sequence + +from .constitution import Constitution +from .gate import BenchCase, ForgeGate +from .ledger import SkillLedger +from .loop import ApprovalPolicy, ForgeEpisode, SkillForge +from .memory import ( + Contradiction, + MemoryItem, + MemoryPolicy, + MemoryScoreFn, + Resolution, + apply_measurement, + decay_unmeasured, + detect_contradictions, + measure, + recall as recall_memory, + resolve, +) +from .model import RunTrace, SkillCandidate + + +@dataclass +class RecalledSkill: + name: str + body: str + verified: bool + + +@dataclass +class ContextPack: + """다음 실행에 넣을 것들 — 검증 여부가 붙은 채로.""" + + skills: list[RecalledSkill] = field(default_factory=list) + memory: list[MemoryItem] = field(default_factory=list) + + def render(self) -> str: + """프롬프트 조각. **라벨을 지우지 않는다** — 미검증을 검증된 것처럼 보이게 + 만드는 순간 이 시스템의 의미가 사라진다.""" + blocks: list[str] = [] + for skill in self.skills: + mark = "검증됨" if skill.verified else "미검증(참고)" + blocks.append(f"\n" + f"{skill.body.strip()}\n") + for item in self.memory: + mark = "측정됨" if item.measured else "미측정" + blocks.append(f"{item.text.strip()}") + return "\n".join(blocks) + + +@dataclass +class CycleReport: + run_id: str + signals: int = 0 + drafted: int = 0 + promoted: list[str] = field(default_factory=list) + staged: list[str] = field(default_factory=list) + rejected: list[str] = field(default_factory=list) + memory_added: list[str] = field(default_factory=list) + memory_measured: int = 0 + memory_up: int = 0 + memory_down: int = 0 + contradictions: int = 0 + resolved: int = 0 + disputed: list[str] = field(default_factory=list) + notes: list[str] = field(default_factory=list) + + def summary(self) -> str: + parts = [f"run={self.run_id}", + f"신호 {self.signals} · 초안 {self.drafted}", + f"스킬 검증 {len(self.promoted)} / 대기 {len(self.staged)} / 거절 {len(self.rejected)}", + f"기억 +{len(self.memory_added)} · 측정 {self.memory_measured}" + f"(↑{self.memory_up} ↓{self.memory_down})", + f"모순 {self.contradictions} → 판정 {self.resolved} · 보류 {len(self.disputed)}"] + if self.notes: + parts.append("· " + " · ".join(self.notes)) + return " | ".join(parts) + + +class JermesAgent: + """스킬 원장과 기억을 하나의 규율 아래 두는 에이전트. + + `memory` 는 호스트가 넘겨준 리스트를 **그대로 들고** 변경한다(영속은 호스트 몫). + """ + + def __init__(self, ledger: SkillLedger, gate: ForgeGate, + memory: list[MemoryItem] | None = None, + memory_policy: MemoryPolicy | None = None, + approval: ApprovalPolicy | None = None, + forge: SkillForge | None = None, + constitution: Constitution | None = None) -> None: + self.ledger = ledger + self.gate = gate + self.memory: list[MemoryItem] = memory if memory is not None else [] + self.memory_policy = memory_policy or MemoryPolicy() + self.constitution = constitution or Constitution() + # 규약은 게이트가 집행한다 — 에이전트가 "지키겠다"고 약속하는 구조가 아니다. + if getattr(gate, "constitution", None) is None: + gate.constitution = self.constitution + self.forge = forge or SkillForge(ledger, gate, approval=approval) + + # ------------------------------------------------------------ 기억 + + def remember(self, trace: RunTrace) -> list[MemoryItem]: + """런에서 기억 후보를 뽑는다 — 교훈과 정제 기억만. + + 도구 출력 원문은 담지 않는다. 그건 사실이 아니라 그때의 상황이고, 기억으로 + 굳으면 다음 실행을 과거에 묶는다. + """ + added: list[MemoryItem] = [] + seen = {item.text.strip() for item in self.memory} + texts = [t for t in list(trace.lessons) + [trace.refined_memory] if t and t.strip()] + for index, text in enumerate(texts): + text = text.strip() + if text in seen: + continue # 같은 사실을 두 번 적지 않는다(멱등) + item = MemoryItem(item_id=f"{trace.run_id}#m{index}", text=text, + scope=trace.scope, source_run_ids=[trace.run_id]) + self.memory.append(item) + added.append(item) + seen.add(text) + return added + + def measure_memory(self, score: MemoryScoreFn, cases: Sequence[BenchCase], + items: Sequence[MemoryItem] | None = None) -> tuple[int, int, int]: + """(잰 개수, 오른 개수, 내린 개수). 못 재면 0 을 돌려주고 조용히 넘어가지 않는다.""" + measured = up = down = 0 + for item in (items if items is not None else self.memory): + if item.status == "retired": + continue + result = measure(item, score, cases, self.memory_policy) + if result is None: + continue + before = item.trust + apply_measurement(item, result, self.memory_policy) + measured += 1 + up += item.trust > before + down += item.trust < before + return measured, up, down + + def reconcile(self, score: MemoryScoreFn | None = None, + cases: Sequence[BenchCase] = ()) -> tuple[list[Contradiction], + list[Resolution]]: + """모순을 찾고, 잴 수 있으면 증거로 판정한다. + + 점수 함수가 없으면 판정하지 않고 **드러내기만** 한다(Geny 와 같은 수준). + 있으면 한 걸음 더 간다 — 재현벤치가 이긴 쪽을 정한다. + """ + found = detect_contradictions(self.memory) + if not found or score is None: + return found, [] + index = {item.item_id: item for item in self.memory} + resolutions = [] + for contradiction in found: + left, right = index.get(contradiction.left), index.get(contradiction.right) + if left is None or right is None: + continue + resolutions.append( + resolve(contradiction, left, right, score, cases, self.memory_policy)) + return found, resolutions + + # ------------------------------------------------------------ 회상 + + def recall(self, skill_limit: int = 5, memory_limit: int = 5, + include_unverified: bool = False) -> ContextPack: + """다음 실행에 넣을 묶음. 기본은 **검증된 스킬만**. + + 미검증 포함은 호출측이 명시적으로 켜야 하고, 켜도 라벨은 남는다. + """ + records = [r for r in self.ledger.list() if r.status == "active"] + if not include_unverified: + records = [r for r in records if r.skill.verified] + records.sort(key=lambda r: (-int(r.skill.verified), r.name)) + skills = [RecalledSkill(name=r.name, body=r.skill.body, + verified=bool(r.skill.verified)) + for r in records[:skill_limit]] + return ContextPack(skills=skills, + memory=recall_memory(self.memory, limit=memory_limit)) + + # ------------------------------------------------------------ 한 사이클 + + def cycle(self, trace: RunTrace, + bench_cases: Sequence[BenchCase] = (), + drafted: list[SkillCandidate] | None = None, + memory_score: MemoryScoreFn | None = None, + prior_signatures: dict[str, int] | None = None, + decay: bool = True) -> CycleReport: + """관찰 → 기억 → 학습 → 화해 → 보고. 이 순서가 곧 에이전트의 정의다.""" + report = CycleReport(run_id=trace.run_id) + + added = self.remember(trace) + report.memory_added = [item.item_id for item in added] + + episode: ForgeEpisode = self.forge.process_trace( + trace, bench_cases=bench_cases, prior_signatures=prior_signatures, + drafted=drafted) + report.signals = len(episode.signals) + report.drafted = len(episode.drafted) + # 거절은 두 곳에서 나온다 — 큐레이터(중복·안전)와 게이트(규약 위반). + # 한쪽만 보면 규약으로 막힌 후보가 보고에서 사라진다. + report.rejected = [r.candidate_name for r in episode.rejected] + for skill, result in episode.results: + if result.verdict == "promoted": + report.promoted.append(skill.name) + elif result.verdict == "rejected": + report.rejected.append(skill.name) + report.notes.append(f"거절 {skill.name}: {'; '.join(result.reasons)[:120]}") + else: + report.staged.append(skill.name) + + if memory_score is not None: + measured, up, down = self.measure_memory(memory_score, bench_cases) + report.memory_measured, report.memory_up, report.memory_down = measured, up, down + if measured == 0 and self.memory: + # 오늘 세 번 당한 실패 방식: 0인데 이유를 안 말하면 멈춘 것과 구분이 안 된다. + report.notes.append( + f"기억 측정 0 — 케이스 {len(bench_cases)}개 < 최소 " + f"{self.memory_policy.min_cases}") + elif self.memory: + report.notes.append("기억 측정 안 함 — 점수 함수 미지정") + + found, resolutions = self.reconcile(memory_score, bench_cases) + report.contradictions = len(found) + report.resolved = sum(1 for r in resolutions if r.decided) + report.disputed = [item.item_id for item in self.memory + if item.status == "disputed"] + if found and memory_score is None: + report.notes.append("모순 판정 안 함 — 점수 함수가 없어 드러내기만 함") + + if decay: + decay_unmeasured(self.memory, self.memory_policy) + return report diff --git a/src/xgen_sdk/harness/skill_forge/constitution.py b/src/xgen_sdk/harness/skill_forge/constitution.py new file mode 100644 index 0000000..a883b9f --- /dev/null +++ b/src/xgen_sdk/harness/skill_forge/constitution.py @@ -0,0 +1,170 @@ +"""SF8 — 규약(constitution). Hermes 의 SOUL.md 에 대응하되, 텍스트가 아니라 **집행**이다. + +Hermes 는 페르소나를 `SOUL.md` 로 이어가고 "학습하지 말 것" 규칙을 스킬 프롬프트 안에 +문장으로 둔다. 문장으로 둔 규칙은 모델이 지키면 지켜지고 안 지키면 안 지켜진다 — +그리고 페르소나 파일은 에이전트가 스스로 고치므로 **조용히 표류**한다(그 표류를 알 +방법이 없다는 게 진짜 문제다). + +여기서는 세 가지를 다르게 한다. + +1. **금지는 게이트가 집행한다.** `never_learn` 은 프롬프트가 아니라 `check_candidate()` + 로 걸러진다. 모델의 선의에 기대지 않는다. +2. **규약은 에이전트가 못 고친다.** `propose()` 는 **차이만** 돌려준다. 적용은 사람이 + `adopt()` 를 부르는 것으로만 일어나고 이력이 남는다. +3. **표류가 보인다.** `diff()` 가 무엇이 언제 바뀌었는지 줄 단위로 말한다. + +파일 형식은 agentskills 프론트매터와 같은 모양이라(`---` YAML + 본문) 다른 도구가 +읽어도 깨지지 않는다. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Sequence + +from .model import SkillCandidate + +DEFAULT_IDENTITY = "Jermes" +DEFAULT_ROLE = "끝난 실행에서 재사용 가능한 절차를 배우고, 검증된 것만 남기는 큐레이터" + + +@dataclass +class Constitution: + """에이전트가 스스로 바꿀 수 없는 부분.""" + + identity: str = DEFAULT_IDENTITY + role: str = DEFAULT_ROLE + principles: list[str] = field(default_factory=lambda: [ + "검증되지 않은 것을 검증된 것처럼 제시하지 않는다.", + "증거 없이 지우지 않는다 — 내려갈 때도 이력과 함께 남긴다.", + "0건일 때는 왜 0건인지 말한다.", + ]) + never_learn: list[str] = field(default_factory=lambda: [ + # 배우면 안 되는 것 = 오래가지 않거나, 배우는 순간 위험해지는 것. + r"비밀번호|password|api[_-]?key|secret|token", + r"특정 날짜에만 맞는|only on \d{4}-\d{2}-\d{2}", + r"검증을 건너뛰|skip (?:the )?(?:verification|gate|bench)", + r"사람 승인 없이|without (?:human )?approval", + ]) + approval_required_scopes: list[str] = field(default_factory=lambda: ["project", "org"]) + version: str = "1.0.0" + history: list[str] = field(default_factory=list) + + # ------------------------------------------------------------ 집행 + + def check_candidate(self, candidate: SkillCandidate) -> str | None: + """규약 위반이면 이유, 아니면 None. `safety_check` 와 같은 계약이라 + 게이트에 그대로 꽂힌다.""" + text = " ".join([ + candidate.name or "", candidate.rationale or "", candidate.when_to_use or "", + " ".join(candidate.procedure or []), " ".join(candidate.pitfalls or []), + " ".join(candidate.verification or []), str(candidate.payload or {}), + ]) + for pattern in self.never_learn: + try: + match = re.search(pattern, text, re.IGNORECASE) + except re.error: + continue # 잘못된 규칙 하나가 집행 전체를 멈추게 두지 않는다 + if match: + return f"규약 위반(never_learn): {pattern!r} 이 {match.group(0)!r} 에 걸림" + return None + + def needs_human_approval(self, scope: str) -> bool: + return scope in self.approval_required_scopes + + # ------------------------------------------------------------ 변경 통제 + + def propose(self, **changes) -> list[str]: + """제안만 한다 — **적용하지 않는다**. 에이전트가 자기 규약을 바꾸는 경로는 없다.""" + lines: list[str] = [] + for key, value in changes.items(): + if not hasattr(self, key) or key in ("history", "version"): + lines.append(f"거부: {key} 는 제안 대상이 아니다") + continue + current = getattr(self, key) + if current == value: + continue + lines.append(f"{key}: {current!r} -> {value!r}") + return lines + + def adopt(self, changes: dict, approved_by: str) -> list[str]: + """사람이 승인했을 때만 적용된다. 승인자 없이 부르면 거부.""" + if not approved_by.strip(): + raise ValueError("규약 변경에는 승인자가 필요하다") + applied = self.propose(**changes) + applied = [line for line in applied if not line.startswith("거부:")] + for key, value in changes.items(): + if hasattr(self, key) and key not in ("history", "version"): + setattr(self, key, value) + if applied: + major, minor, patch = (self.version.split(".") + ["0", "0"])[:3] + self.version = f"{major}.{int(minor) + 1}.0" + self.history.append(f"{self.version} by {approved_by}: " + "; ".join(applied)) + return applied + + # ------------------------------------------------------------ 표류 감시 + + def diff(self, other: "Constitution") -> list[str]: + """무엇이 달라졌는지 줄 단위로. 표류를 눈에 보이게 하는 것이 목적이다.""" + lines: list[str] = [] + for field_name in ("identity", "role", "version"): + mine, theirs = getattr(self, field_name), getattr(other, field_name) + if mine != theirs: + lines.append(f"{field_name}: {mine!r} -> {theirs!r}") + for field_name in ("principles", "never_learn", "approval_required_scopes"): + mine, theirs = set(getattr(self, field_name)), set(getattr(other, field_name)) + for gone in sorted(mine - theirs): + lines.append(f"{field_name} 삭제: {gone}") + for added in sorted(theirs - mine): + lines.append(f"{field_name} 추가: {added}") + return lines + + # ------------------------------------------------------------ 직렬화 + + def to_markdown(self) -> str: + """agentskills 프론트매터와 같은 모양 — 다른 도구가 읽어도 안 깨진다.""" + def block(name: str, values: Sequence[str]) -> str: + return f"{name}:\n" + "".join(f' - "{v}"\n' for v in values) + + return ( + "---\n" + f'name: {self.identity.lower()}-constitution\n' + f'description: "{self.role}"\n' + f'metadata:\n version: "{self.version}"\n' + "---\n\n" + f"# {self.identity}\n\n{self.role}\n\n" + "## Principles\n" + "".join(f"- {p}\n" for p in self.principles) + + "\n## Never learn\n" + "".join(f"- `{p}`\n" for p in self.never_learn) + + "\n## Approval required\n" + + "".join(f"- {s}\n" for s in self.approval_required_scopes) + + ("\n## History\n" + "".join(f"- {h}\n" for h in self.history) + if self.history else "") + ) + + @classmethod + def from_markdown(cls, text: str) -> "Constitution": + """to_markdown 의 역. 못 읽는 줄은 조용히 버리지 않고 기본값으로 남긴다.""" + def section(title: str) -> list[str]: + match = re.search(rf"^## {re.escape(title)}\n(.*?)(?=^## |\Z)", + text, re.MULTILINE | re.DOTALL) + if not match: + return [] + return [re.sub(r"^[-*]\s*", "", line).strip().strip("`") + for line in match.group(1).splitlines() if line.strip().startswith(("-", "*"))] + + identity = re.search(r"^# (.+)$", text, re.MULTILINE) + version = re.search(r'version:\s*"?([0-9.]+)"?', text) + role = re.search(r'description:\s*"([^"]*)"', text) + constitution = cls( + identity=(identity.group(1).strip() if identity else DEFAULT_IDENTITY), + role=(role.group(1) if role else DEFAULT_ROLE), + version=(version.group(1) if version else "1.0.0"), + ) + for name, values in (("principles", section("Principles")), + ("never_learn", section("Never learn")), + ("approval_required_scopes", section("Approval required"))): + if values: + setattr(constitution, name, values) + constitution.history = section("History") + return constitution diff --git a/src/xgen_sdk/harness/skill_forge/gate.py b/src/xgen_sdk/harness/skill_forge/gate.py index 808a960..70810ef 100644 --- a/src/xgen_sdk/harness/skill_forge/gate.py +++ b/src/xgen_sdk/harness/skill_forge/gate.py @@ -55,9 +55,13 @@ class GateConfig: class ForgeGate: def __init__(self, runner: BenchRunner | ScoreFn, - config: GateConfig | None = None) -> None: + config: GateConfig | None = None, + constitution=None) -> None: self._score: ScoreFn = runner.score if hasattr(runner, "score") else runner # type: ignore[union-attr] self.config = config or GateConfig() + # 규약(constitution.py). Hermes 는 "배우지 말 것"을 프롬프트 문장으로 두지만 + # 문장은 모델이 지키면 지켜지고 안 지키면 안 지켜진다 — 여기서 집행한다. + self.constitution = constitution def verify(self, candidate: SkillCandidate, skill: SkillDef, cases: Sequence[BenchCase]) -> GateResult: @@ -65,6 +69,12 @@ def verify(self, candidate: SkillCandidate, skill: SkillDef, if reason: return GateResult(verdict="rejected", reasons=[f"sec: {reason}"]) + if self.constitution is not None: + violation = self.constitution.check_candidate(candidate) + if violation: + # 벤치를 돌려보기 전에 막는다 — 배우면 안 되는 것은 성능이 좋아도 안 된다. + return GateResult(verdict="rejected", reasons=[violation]) + if len(cases) < self.config.min_cases: return GateResult( verdict="staged", diff --git a/src/xgen_sdk/harness/skill_forge/memory.py b/src/xgen_sdk/harness/skill_forge/memory.py new file mode 100644 index 0000000..bcae985 --- /dev/null +++ b/src/xgen_sdk/harness/skill_forge/memory.py @@ -0,0 +1,282 @@ +"""SF7 — 증거로 등급을 매기는 기억(evidence-graded memory). + +**왜 다르게 만드는가.** Geny 는 2026-07-23 에 메모리 거버넌스를 붙였다(항목별 trust, +모순 감지). 규율이 좋다 — 특히 "보였는데 안 썼다"를 음의 신호로 자동 배선하지 않은 +판단은 옳다. 그건 나쁜 질의를 애먼 노트 탓으로 돌리는 신호이기 때문이다. + +그런데 그 판단은 **신호가 대리(proxy)라서** 나온 제약이다. 편집·사용 여부는 유용함의 +그림자일 뿐이다. 우리에겐 스킬을 검증하는 재현벤치가 이미 있고, **기억 항목의 값어치는 +스킬과 똑같은 방법으로 직접 잴 수 있다** — 그 항목을 빼고 재생해서 점수가 떨어지면 +그 항목이 일한 것이다. 대리 신호가 아니라 측정이다. 그래서 여기서는: + +- **trust 는 측정으로만 움직인다.** 조회·노출·편집 같은 대리 신호로는 절대 안 움직인다. +- **모순은 드러내는 데서 끝내지 않고 증거로 판정한다.** 충돌하는 두 항목을 각각 넣고 + 재생해 이긴 쪽을 남긴다. 변별이 안 되면 둘 다 `disputed` 로 두고 사람에게 넘긴다. +- **자동 삭제는 없다.** 은퇴는 측정된 해악이나 사람의 결정으로만. 되돌릴 수 있어야 한다. +- **감쇠는 중립으로.** 오래 재측정되지 않은 확신은 중립으로 흘러 경화(ossification)를 막는다. + 0 으로 떨어뜨리지 않는 이유: 안 재봤다는 것은 나쁘다는 뜻이 아니다. + +계약은 게이트와 같은 것을 쓴다(`BenchCase` + score 함수) — 기억과 스킬이 같은 잣대를 +쓰는 것이 이 설계의 요점이다. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Callable, Iterable, Sequence + +from .gate import BenchCase + +NEUTRAL = 0.5 +MEMORY_STATUSES = ("active", "disputed", "retired") + +# (기억 텍스트, 케이스) -> 점수. 기억을 넣고 재생했을 때의 점수를 준다. +# skill 게이트의 ScoreFn 과 같은 모양이라 호스트가 하나의 러너를 둘 다에 쓸 수 있다. +MemoryScoreFn = Callable[[BenchCase, "MemoryItem | None"], float] + + +@dataclass +class MemoryItem: + """기억 한 항목. 값어치는 주장이 아니라 측정으로 붙는다.""" + + item_id: str + text: str + scope: str = "user" + trust: float = NEUTRAL + status: str = "active" + source_run_ids: list[str] = field(default_factory=list) + evidence: dict = field(default_factory=dict) + history: list[str] = field(default_factory=list) + + def __post_init__(self) -> None: + if self.status not in MEMORY_STATUSES: + raise ValueError(f"unknown memory status: {self.status}") + self.trust = min(1.0, max(0.0, float(self.trust))) + + @property + def measured(self) -> bool: + """한 번이라도 재현벤치로 재본 적이 있는가. 없으면 trust 는 그냥 중립이다.""" + return bool(self.evidence.get("measurements")) + + +@dataclass +class Measurement: + item_id: str + cases: int + with_item: float + without_item: float + + @property + def gain(self) -> float: + return self.with_item - self.without_item + + def verdict(self, min_gain: float, harm_threshold: float) -> str: + if self.gain > min_gain: + return "helpful" + if self.gain < -abs(harm_threshold): + return "harmful" + return "neutral" + + +@dataclass +class MemoryPolicy: + min_cases: int = 4 + min_gain: float = 0.0 + harm_threshold: float = 0.05 + """이 이상 점수를 떨어뜨리면 해롭다고 본다 — 잡음과 구분하려고 0 이 아니다.""" + step: float = 0.15 + """한 번의 측정이 trust 를 옮기는 폭. 한 방에 확신하지 않는다.""" + decay: float = 0.05 + """재측정 없이 흐르면 중립으로 끌어당기는 폭(경화 방지).""" + + +def measure(item: MemoryItem, score: MemoryScoreFn, cases: Sequence[BenchCase], + policy: MemoryPolicy | None = None) -> Measurement | None: + """항목을 넣고/빼고 재생해 실제 기여를 잰다. 케이스가 모자라면 재지 않는다. + + 측정할 수 없을 때 추측으로 채우지 않는 것이 요점이다 — 스킬 게이트가 + 케이스 부족을 `staged` 로 정직하게 처리하는 것과 같은 규율이다. + """ + policy = policy or MemoryPolicy() + if len(cases) < policy.min_cases: + return None + with_item = sum(score(case, item) for case in cases) / len(cases) + without_item = sum(score(case, None) for case in cases) / len(cases) + return Measurement(item_id=item.item_id, cases=len(cases), + with_item=with_item, without_item=without_item) + + +def apply_measurement(item: MemoryItem, measurement: Measurement, + policy: MemoryPolicy | None = None) -> MemoryItem: + """측정 결과만이 trust 를 움직인다.""" + policy = policy or MemoryPolicy() + verdict = measurement.verdict(policy.min_gain, policy.harm_threshold) + before = item.trust + if verdict == "helpful": + item.trust = min(1.0, item.trust + policy.step) + elif verdict == "harmful": + item.trust = max(0.0, item.trust - policy.step) + # neutral 은 움직이지 않는다 — "차이가 없다"는 "나쁘다"가 아니다. + item.evidence.setdefault("measurements", []).append({ + "cases": measurement.cases, + "gain": round(measurement.gain, 4), + "verdict": verdict, + }) + item.history.append( + f"measure: {verdict} gain={measurement.gain:+.3f} " + f"trust {before:.2f}->{item.trust:.2f} (n={measurement.cases})") + return item + + +def decay_unmeasured(items: Iterable[MemoryItem], + policy: MemoryPolicy | None = None) -> list[MemoryItem]: + """재측정되지 않은 확신을 중립으로 끌어당긴다. + + 0 으로 보내지 않는다 — 안 재봤다는 것이 나쁘다는 뜻은 아니기 때문이다. + 측정 이력이 있어도 계속 감쇠시키는 이유: 세상이 변하면 옛 측정은 낡는다. + """ + policy = policy or MemoryPolicy() + moved = [] + for item in items: + if item.status == "retired": + continue + if abs(item.trust - NEUTRAL) < 1e-9: + continue + direction = -1.0 if item.trust > NEUTRAL else 1.0 + stepped = item.trust + direction * policy.decay + item.trust = NEUTRAL if (item.trust - NEUTRAL) * (stepped - NEUTRAL) <= 0 else stepped + moved.append(item) + return moved + + +# --------------------------------------------------------------- 모순 + +# 한국어는 조사가 붙어 단어 경계(`\b`)가 생기지 않는다 — "하지 않는다" 에서 `\b않\b` +# 는 절대 안 맞는다(실제로 안 맞아서 감지가 0건이었다). 그래서 언어별로 나눈다. +_NEGATION_EN = re.compile(r"\b(not|never|no longer|isn't|aren't|doesn't|don't|won't)\b", + re.IGNORECASE) +_NEGATION_KO = re.compile(r"(않|없|아니|못)") + +_NUMBER = re.compile(r"(-?\d+(?:\.\d+)?)") +_STOP = {"the", "a", "an", "is", "are", "was", "were", "to", "of", "in", "on", + "and", "or", "for", "with", "that", "this", "it", "be", "as", "at", + "은", "는", "이", "가", "을", "를", "에", "의", "로", "와", "과", "도"} + + +def _tokens(text: str) -> set[str]: + """형태소 분석기 없이 자르므로 한국어는 조사가 붙은 채로 남는다("배포는" ≠ + "배포가"). 그래서 모순 감지는 **재현율보다 정밀도**를 택한 장치다 — 놓치는 + 모순은 있어도, 엉뚱한 쌍을 모순이라 우기지는 않는다.""" + words = re.findall(r"[a-z0-9]+|[가-힣]+", (text or "").lower()) + return {w for w in words if w not in _STOP and len(w) > 1} + + +def _has_negation(text: str) -> bool: + return bool(_NEGATION_EN.search(text or "") or _NEGATION_KO.search(text or "")) + + +def _strip_negation(text: str) -> str: + return _NEGATION_KO.sub(" ", _NEGATION_EN.sub(" ", text or "")) + + +@dataclass +class Contradiction: + left: str + right: str + kind: str # negation_flip | numeric_conflict + overlap: float + detail: str = "" + + +def detect_contradictions(items: Sequence[MemoryItem], + min_overlap: float = 0.5) -> list[Contradiction]: + """같은 것을 말하는데 반대로 말하는 쌍을 찾는다. + + 두 가지만 본다 — 부정 뒤집힘과 같은 자리의 숫자 충돌. 의미 모순 전반을 + LLM 으로 판정하지 않는 이유: 판정에 LLM 을 쓰면 이 계층이 모델 품질에 + 끌려간다(우리 벤치가 LLM-judge 를 안 쓰는 것과 같은 이유). + """ + found: list[Contradiction] = [] + active = [i for i in items if i.status != "retired"] + for index, left in enumerate(active): + for right in active[index + 1:]: + if left.scope != right.scope: + continue + left_core, right_core = _tokens(_strip_negation(left.text)), _tokens( + _strip_negation(right.text)) + if not left_core or not right_core: + continue + overlap = len(left_core & right_core) / min(len(left_core), len(right_core)) + if overlap < min_overlap: + continue + left_neg = _has_negation(left.text) + right_neg = _has_negation(right.text) + if left_neg != right_neg: + found.append(Contradiction(left.item_id, right.item_id, + "negation_flip", round(overlap, 3), + "한쪽만 부정형")) + continue + left_nums, right_nums = _NUMBER.findall(left.text), _NUMBER.findall(right.text) + if left_nums and right_nums and left_nums != right_nums: + found.append(Contradiction(left.item_id, right.item_id, + "numeric_conflict", round(overlap, 3), + f"{left_nums} vs {right_nums}")) + return found + + +@dataclass +class Resolution: + contradiction: Contradiction + winner: str = "" + loser: str = "" + decided: bool = False + reason: str = "" + + +def resolve(contradiction: Contradiction, left: MemoryItem, right: MemoryItem, + score: MemoryScoreFn, cases: Sequence[BenchCase], + policy: MemoryPolicy | None = None) -> Resolution: + """모순을 증거로 판정한다 — 드러내고 끝내지 않는다. + + 둘을 각각 넣고 재생해 더 나은 쪽을 남긴다. 차이가 잡음 수준이면 판정하지 + 않고 **둘 다 `disputed`** 로 둔다. 자동 삭제는 없다 — 진 쪽도 은퇴가 아니라 + `disputed` 다. 우리가 틀렸을 때 되돌릴 수 있어야 하기 때문이다. + """ + policy = policy or MemoryPolicy() + left_measurement = measure(left, score, cases, policy) + right_measurement = measure(right, score, cases, policy) + if left_measurement is None or right_measurement is None: + left.status = right.status = "disputed" + return Resolution(contradiction, reason="케이스 부족 — 판정 불가, 사람 확인 필요") + + apply_measurement(left, left_measurement, policy) + apply_measurement(right, right_measurement, policy) + margin = left_measurement.gain - right_measurement.gain + if abs(margin) <= policy.harm_threshold: + left.status = right.status = "disputed" + return Resolution(contradiction, + reason=f"변별 없음(차이 {margin:+.3f}) — 사람 확인 필요") + + winner, loser = (left, right) if margin > 0 else (right, left) + winner.status = "active" + loser.status = "disputed" # 삭제하지 않는다 + loser.history.append(f"disputed: {winner.item_id} 이 재현벤치에서 {abs(margin):.3f} 앞섬") + return Resolution(contradiction, winner=winner.item_id, loser=loser.item_id, + decided=True, + reason=f"재현벤치 판정: {winner.item_id} 우세({abs(margin):+.3f})") + + +# --------------------------------------------------------------- 회상 + +def recall(items: Sequence[MemoryItem], limit: int = 5, + min_trust: float = NEUTRAL) -> list[MemoryItem]: + """프롬프트에 넣을 항목 고르기. + + `disputed` 는 절대 넣지 않는다 — 모순이 해결되기 전에 주입하면 에이전트가 + 서로 반대되는 두 사실을 동시에 믿게 된다. 미측정 항목은 중립이라 기본 + 문턱(0.5)에 걸려 들어오지만, 측정으로 해롭다고 나온 것은 걸러진다. + """ + usable = [i for i in items if i.status == "active" and i.trust >= min_trust] + usable.sort(key=lambda i: (-i.trust, i.item_id)) + return usable[:limit] From ca023ab13b74af867010ac0d0706b7125cf2e292 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=84=EC=88=98?= Date: Tue, 4 Aug 2026 19:04:30 +0900 Subject: [PATCH 8/9] =?UTF-8?q?absorb:=20skill=5Fforge=20=E2=80=94=20?= =?UTF-8?q?=EC=9E=90=EC=B2=B4=20=EA=B2=80=EC=88=98=20=EC=88=98=EC=A0=95=20?= =?UTF-8?q?=EB=B0=98=EC=98=81(=EB=9D=BC=EB=B2=A8=20=EC=9C=84=EC=A1=B0=20?= =?UTF-8?q?=EC=B0=A8=EB=8B=A8=C2=B7=EC=8B=A0=EB=A2=B0=20=ED=8E=B8=ED=96=A5?= =?UTF-8?q?=C2=B7=EC=98=A4=ED=83=90=C2=B7=ED=8E=9C=EC=8A=A4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 엔진(jinsoo96/jermes `78a59be`)의 검수 수정을 흡수한다. - agent: 회상 렌더에서 본문 escape + 속성 전용 `_attr()`. stdlib quoteattr 는 값에 큰따옴표가 있으면 작은따옴표로 감싸 `status="검증됨"` 이 문자 그대로 남아 못 막는다(읽는 건 XML 파서가 아니라 LLM 이다). - memory: 잡음 구간 대칭(min_gain 0.0→0.05), 수치 비교를 float 으로(0.8 vs 0.80 오탐), 무판정 시 양쪽 이력에 사유 기록. - portable: payload 안의 ``` 로 펜스가 조기 종료되던 것 — 내용에 맞춰 펜스 확장. Co-Authored-By: Claude Opus 5 (1M context) --- src/xgen_sdk/harness/skill_forge/agent.py | 72 +++++++++++++++----- src/xgen_sdk/harness/skill_forge/memory.py | 30 ++++++-- src/xgen_sdk/harness/skill_forge/portable.py | 12 +++- 3 files changed, 92 insertions(+), 22 deletions(-) diff --git a/src/xgen_sdk/harness/skill_forge/agent.py b/src/xgen_sdk/harness/skill_forge/agent.py index c083a50..0e78ec3 100644 --- a/src/xgen_sdk/harness/skill_forge/agent.py +++ b/src/xgen_sdk/harness/skill_forge/agent.py @@ -22,6 +22,7 @@ from dataclasses import dataclass, field from typing import Sequence +from xml.sax.saxutils import escape from .constitution import Constitution from .gate import BenchCase, ForgeGate @@ -43,6 +44,17 @@ from .model import RunTrace, SkillCandidate +def _attr(value: str) -> str: + """속성값을 **항상 큰따옴표**로 감싸고 내부 따옴표는 실체참조로 바꾼다. + + stdlib `quoteattr` 를 쓰면 안 된다 — 값에 큰따옴표가 있으면 작은따옴표로 감싸므로 + `name='a" status="검증됨'` 같은 결과가 나오고, 그 안의 `status="검증됨"` 이 **문자 + 그대로 남는다**. XML 파서에는 안전하지만 이 문자열을 읽는 건 파서가 아니라 LLM 이다. + (검수에서 실제로 이 형태가 나왔다.) + """ + return '"' + escape(str(value), {'"': """, "'": "'"}) + '"' + + @dataclass class RecalledSkill: name: str @@ -59,16 +71,23 @@ class ContextPack: def render(self) -> str: """프롬프트 조각. **라벨을 지우지 않는다** — 미검증을 검증된 것처럼 보이게 - 만드는 순간 이 시스템의 의미가 사라진다.""" + 만드는 순간 이 시스템의 의미가 사라진다. + + 그래서 본문과 속성을 반드시 이스케이프한다. 검수에서 실제로 뚫렸다: + 기억 텍스트에 `…` 를 넣으면 태그를 닫고 + **검증된 스킬 블록을 위조**할 수 있었다. 기억과 스킬 본문은 런 트레이스와 + 모델 출력에서 오므로 적대적일 수 있다 — 경계는 내용이 못 넘는다. + """ blocks: list[str] = [] for skill in self.skills: mark = "검증됨" if skill.verified else "미검증(참고)" - blocks.append(f"\n" - f"{skill.body.strip()}\n") + blocks.append(f"\n" + f"{escape(skill.body.strip())}\n") for item in self.memory: mark = "측정됨" if item.measured else "미측정" - blocks.append(f"{item.text.strip()}") + blocks.append(f"" + f"{escape(item.text.strip())}") return "\n".join(blocks) @@ -146,10 +165,21 @@ def remember(self, trace: RunTrace) -> list[MemoryItem]: return added def measure_memory(self, score: MemoryScoreFn, cases: Sequence[BenchCase], - items: Sequence[MemoryItem] | None = None) -> tuple[int, int, int]: - """(잰 개수, 오른 개수, 내린 개수). 못 재면 0 을 돌려주고 조용히 넘어가지 않는다.""" + items: Sequence[MemoryItem] | None = None, + limit: int | None = None) -> tuple[int, int, int]: + """(잰 개수, 오른 개수, 내린 개수). 못 재면 0 을 돌려주고 조용히 넘어가지 않는다. + + `limit` 이 필요한 이유: 한 번 재는 데 케이스 수 × 2 번의 채점이 든다. + 기억이 200개면 한 사이클에 수천 번이고, 채점이 LLM 이면 그대로 멈춘 것처럼 + 보인다. 그래서 **미측정 항목을 먼저** 재고 나머지는 다음 사이클로 넘긴다. + """ + pool = list(items if items is not None else self.memory) + if limit is not None: + # 아직 안 재본 것 우선 — 새 기억이 영영 순번을 못 받는 걸 막는다. + pool.sort(key=lambda i: (i.measured, i.item_id)) + pool = pool[:max(0, limit)] measured = up = down = 0 - for item in (items if items is not None else self.memory): + for item in pool: if item.status == "retired": continue result = measure(item, score, cases, self.memory_policy) @@ -208,7 +238,8 @@ def cycle(self, trace: RunTrace, drafted: list[SkillCandidate] | None = None, memory_score: MemoryScoreFn | None = None, prior_signatures: dict[str, int] | None = None, - decay: bool = True) -> CycleReport: + decay: bool = True, + memory_measure_limit: int | None = 20) -> CycleReport: """관찰 → 기억 → 학습 → 화해 → 보고. 이 순서가 곧 에이전트의 정의다.""" report = CycleReport(run_id=trace.run_id) @@ -232,24 +263,33 @@ def cycle(self, trace: RunTrace, else: report.staged.append(skill.name) + # 화해를 먼저 한다. `resolve` 가 충돌 쌍을 이미 재기 때문에, 뒤에 일괄 측정을 + # 돌리면 같은 항목이 한 사이클에 두 번 측정돼 trust 가 이중으로 움직인다. + found, resolutions = self.reconcile(memory_score, bench_cases) + report.contradictions = len(found) + report.resolved = sum(1 for r in resolutions if r.decided) + adjudicated = {side for c in found for side in (c.left, c.right)} + if found and memory_score is None: + report.notes.append("모순 판정 안 함 — 점수 함수가 없어 드러내기만 함") + if memory_score is not None: - measured, up, down = self.measure_memory(memory_score, bench_cases) + rest = [i for i in self.memory if i.item_id not in adjudicated] + measured, up, down = self.measure_memory( + memory_score, bench_cases, items=rest, limit=memory_measure_limit) report.memory_measured, report.memory_up, report.memory_down = measured, up, down - if measured == 0 and self.memory: + if measured == 0 and rest: # 오늘 세 번 당한 실패 방식: 0인데 이유를 안 말하면 멈춘 것과 구분이 안 된다. report.notes.append( f"기억 측정 0 — 케이스 {len(bench_cases)}개 < 최소 " f"{self.memory_policy.min_cases}") + if memory_measure_limit is not None and len(rest) > memory_measure_limit: + report.notes.append( + f"기억 측정 {memory_measure_limit}/{len(rest)} — 나머지는 다음 사이클") elif self.memory: report.notes.append("기억 측정 안 함 — 점수 함수 미지정") - found, resolutions = self.reconcile(memory_score, bench_cases) - report.contradictions = len(found) - report.resolved = sum(1 for r in resolutions if r.decided) report.disputed = [item.item_id for item in self.memory if item.status == "disputed"] - if found and memory_score is None: - report.notes.append("모순 판정 안 함 — 점수 함수가 없어 드러내기만 함") if decay: decay_unmeasured(self.memory, self.memory_policy) diff --git a/src/xgen_sdk/harness/skill_forge/memory.py b/src/xgen_sdk/harness/skill_forge/memory.py index bcae985..c3a7860 100644 --- a/src/xgen_sdk/harness/skill_forge/memory.py +++ b/src/xgen_sdk/harness/skill_forge/memory.py @@ -82,7 +82,13 @@ def verdict(self, min_gain: float, harm_threshold: float) -> str: @dataclass class MemoryPolicy: min_cases: int = 4 - min_gain: float = 0.0 + min_gain: float = 0.05 + """이 이상 올려야 도움이 됐다고 본다. + + 0 이면 안 된다 — 잡음 수준의 +0.001 도 helpful 로 세어 **trust 가 한쪽으로만 + 흐른다**(검수에서 실제로 잡힌 편향: +0.001 은 trust 를 올리는데 -0.001 은 + 중립이었다). 해악 문턱과 같은 값으로 둬서 잡음 구간을 대칭으로 만든다. + """ harm_threshold: float = 0.05 """이 이상 점수를 떨어뜨리면 해롭다고 본다 — 잡음과 구분하려고 0 이 아니다.""" step: float = 0.15 @@ -190,7 +196,8 @@ class Contradiction: def detect_contradictions(items: Sequence[MemoryItem], - min_overlap: float = 0.5) -> list[Contradiction]: + min_overlap: float = 0.5, + min_tokens: int = 1) -> list[Contradiction]: """같은 것을 말하는데 반대로 말하는 쌍을 찾는다. 두 가지만 본다 — 부정 뒤집힘과 같은 자리의 숫자 충돌. 의미 모순 전반을 @@ -205,7 +212,11 @@ def detect_contradictions(items: Sequence[MemoryItem], continue left_core, right_core = _tokens(_strip_negation(left.text)), _tokens( _strip_negation(right.text)) - if not left_core or not right_core: + # `min_tokens` 로 얇은 근거를 걸러낼 수 있다(기본 1 = 안 거름). + # 2 로 올리면 "threshold is 0.8" 처럼 핵심 토큰이 하나인 **정당한** 모순도 + # 같이 죽는다 — 검수에서 확인했다. 그래서 기본값은 민감하게 두고, + # 오탐의 대가는 `resolve` 가 감당한다(둘 다 disputed → 사람이 본다). + if min(len(left_core), len(right_core)) < min_tokens: continue overlap = len(left_core & right_core) / min(len(left_core), len(right_core)) if overlap < min_overlap: @@ -217,7 +228,10 @@ def detect_contradictions(items: Sequence[MemoryItem], "negation_flip", round(overlap, 3), "한쪽만 부정형")) continue - left_nums, right_nums = _NUMBER.findall(left.text), _NUMBER.findall(right.text) + # 문자열로 비교하면 "0.8" 과 "0.80" 이 충돌로 잡힌다(검수에서 확인). + # 같은 값의 다른 표기는 모순이 아니다. + left_nums = [float(n) for n in _NUMBER.findall(left.text)] + right_nums = [float(n) for n in _NUMBER.findall(right.text)] if left_nums and right_nums and left_nums != right_nums: found.append(Contradiction(left.item_id, right.item_id, "numeric_conflict", round(overlap, 3), @@ -254,7 +268,15 @@ def resolve(contradiction: Contradiction, left: MemoryItem, right: MemoryItem, apply_measurement(right, right_measurement, policy) margin = left_measurement.gain - right_measurement.gain if abs(margin) <= policy.harm_threshold: + # 둘 다 보류한다 — 서로 반대되는 두 사실을 동시에 믿는 것이 더 나쁘다. + # 대신 **왜 보류됐는지**를 양쪽 이력에 남긴다. 오탐이면 사람이 여기서 되돌린다. + def note(other: MemoryItem) -> str: + return (f"disputed: {contradiction.kind}({contradiction.detail}) " + f"상대={other.item_id} — 재현벤치가 변별 못 함(차이 {margin:+.3f})") + left.status = right.status = "disputed" + left.history.append(note(right)) + right.history.append(note(left)) return Resolution(contradiction, reason=f"변별 없음(차이 {margin:+.3f}) — 사람 확인 필요") diff --git a/src/xgen_sdk/harness/skill_forge/portable.py b/src/xgen_sdk/harness/skill_forge/portable.py index dc1982e..a921882 100644 --- a/src/xgen_sdk/harness/skill_forge/portable.py +++ b/src/xgen_sdk/harness/skill_forge/portable.py @@ -159,9 +159,12 @@ def to_skill_md(skill: SkillDef, *, evidence: dict[str, Any] | None = None, body = _body_without_frontmatter(skill.body).rstrip() if skill.kind != "guide": # config/tool 은 마크다운이 아니라 JSON/매니페스트다. 그대로 흘리면 다른 - # 에이전트가 지시문으로 읽으므로 코드블록으로 감싸 설명을 붙인다. + # 에이전트가 지시문으로 읽으므로 코드블록으로 감싼다. + # 펜스 길이는 내용에 맞춰 늘린다 — payload 안에 ``` 가 있으면 3중 백틱이 + # 거기서 닫혀 뒷부분이 본문으로 새어 나온다(검수에서 확인). + fence = "`" * max(3, max((len(m) for m in re.findall(r"`+", body)), default=0) + 1) body = (f"# {skill.name}\n\n{skill.description}\n\n" - f"## Payload ({skill.kind})\n\n```json\n{body}\n```\n") + f"## Payload ({skill.kind})\n\n{fence}json\n{body}\n{fence}\n") elif not body: body = f"# {skill.name}\n\n{skill.description}\n" return "\n".join(lines) + "\n\n" + body + "\n" @@ -233,6 +236,11 @@ def candidate_from_skill_md(text: str, *, scope: str = "user", 들여올 때 `verified` 는 절대 믿지 않는다. 남이 스스로 붙인 표시이고, 검증은 우리 벤치가 이 환경에서 다시 해야 의미가 있다(그게 차별점이다). + + **kind 는 항상 `guide`.** 파일에 `xgen-jermes-kind: config|tool` 이 적혀 있어도 + 따르지 않는다 — config 는 하네스 설정을 바꾸고 tool 은 실행 가능한 산출물로 + 컴파일되므로, 남의 파일이 그걸 정하게 두면 안 된다. 원래 kind 는 payload 의 + `imported_metadata` 에 그대로 남아 사람이 보고 승격시킬 수 있다. """ problems = validate_skill_md(text) if problems: From e7d71530a8a1f1f01a610aa5b7c5f97c4adda48b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A7=84=EC=88=98?= Date: Wed, 5 Aug 2026 16:23:33 +0900 Subject: [PATCH 9/9] =?UTF-8?q?fix(logging):=20metadata=20=EC=A7=81?= =?UTF-8?q?=EB=A0=AC=ED=99=94=20=EC=8B=A4=ED=8C=A8=EB=A1=9C=20backend=5Flo?= =?UTF-8?q?gs=20=ED=96=89=EC=9D=B4=20=EC=9C=A0=EC=8B=A4=EB=90=98=EB=8D=98?= =?UTF-8?q?=20=EB=AC=B8=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 무엇/왜: BackendLogger._log 가 json.dumps(metadata) 를 default 없이 호출해, metadata 에 DB row 가 그대로 실려 오는 호출부(예: 배포상태 조회의 metadata=deploy_meta) 에서 datetime 을 만나면 TypeError 로 터졌다. except 가 이를 삼켜 로그만 남기고 넘어가므로 서비스는 안 죽지만 backend_logs 행이 통째로 사라져, 관리자 화면에서 해당 이력이 비어 보였다 (운영 로그에서 확인: 'Error logging backend data: Object of type datetime is not JSON serializable'). default=str 로 직렬화 불가 값(datetime/Decimal/UUID 등)을 문자열로 보존하고, ensure_ascii=False 로 한글 metadata 가 유니코드 이스케이프로 저장되지 않게 한다. DB 호출 이전 단계라 트랜잭션 영향은 없고, 성공 경로 동작도 그대로다. 회귀 테스트 3건 추가 (패치 전 datetime·한글 케이스 실패 확인). --- src/xgen_sdk/logging/backend_logger.py | 5 +- tests/test_backend_logger_metadata.py | 68 ++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 tests/test_backend_logger_metadata.py diff --git a/src/xgen_sdk/logging/backend_logger.py b/src/xgen_sdk/logging/backend_logger.py index f8d84fa..a80fa10 100644 --- a/src/xgen_sdk/logging/backend_logger.py +++ b/src/xgen_sdk/logging/backend_logger.py @@ -48,7 +48,10 @@ def _log(self, level: str, message: str, metadata: Optional[Dict] = None, 'message': message, 'function_name': func_name, 'api_endpoint': endpoint, - 'metadata': json.dumps(metadata) if metadata else '{}' + # default=str — metadata 에 DB row(datetime/Decimal/UUID 등)가 그대로 + # 섞여 들어오는 호출부가 있다. 직렬화 실패 시 로그 행이 통째로 유실되므로 + # 값 보존을 우선한다. + 'metadata': json.dumps(metadata, ensure_ascii=False, default=str) if metadata else '{}' } self.app_db.insert_record('backend_logs', log_data) logger.info(f"Logged backend data with log_id: {log_id}") diff --git a/tests/test_backend_logger_metadata.py b/tests/test_backend_logger_metadata.py new file mode 100644 index 0000000..c5ea9ea --- /dev/null +++ b/tests/test_backend_logger_metadata.py @@ -0,0 +1,68 @@ +import json +import logging +from datetime import datetime, timezone +from decimal import Decimal +from uuid import UUID + +from xgen_sdk.logging.backend_logger import BackendLogger + + +class _DBStub: + """insert_record 로 들어온 payload 를 그대로 붙잡아 두는 스텁.""" + + def __init__(self): + self.rows = [] + + def insert_record(self, table, data): + self.rows.append((table, data)) + return True + + +def test_metadata_with_datetime_is_persisted(caplog): + """DB row 를 metadata 로 그대로 넘겨도 로그 행이 유실되지 않아야 한다. + + 회귀: json.dumps 가 datetime 에서 TypeError 를 내면 _log 의 except 가 삼켜 + backend_logs 행이 통째로 누락됐다 (배포상태 조회 경로에서 실제 발생). + """ + db = _DBStub() + caplog.set_level(logging.ERROR, logger="backend-logger") + + BackendLogger(db, user_id=41380).success( + "Deploy status retrieved successfully", + metadata={ + "workflow_id": "wf_1785285623355_x73gftt", + "created_at": datetime(2026, 8, 5, 6, 16, 34, tzinfo=timezone.utc), + "score": Decimal("1.5"), + "trace_id": UUID("12345678-1234-5678-1234-567812345678"), + }, + function_name="get_deploy_status", + ) + + assert len(db.rows) == 1, "직렬화 실패로 행이 유실되면 안 된다" + assert "Error logging backend data" not in caplog.text + + table, row = db.rows[0] + assert table == "backend_logs" + parsed = json.loads(row["metadata"]) + assert parsed["workflow_id"] == "wf_1785285623355_x73gftt" + assert parsed["created_at"].startswith("2026-08-05") + assert parsed["score"] == "1.5" + + +def test_metadata_keeps_korean_readable(): + """ensure_ascii=False — 관리자 화면에서 한글 metadata 가 \\uXXXX 로 보이지 않도록.""" + db = _DBStub() + + BackendLogger(db, user_id=1).info("작업 기록", metadata={"이름": "김진수"}) + + _, row = db.rows[0] + assert "김진수" in row["metadata"] + + +def test_no_metadata_stores_empty_object(): + db = _DBStub() + + BackendLogger(db, user_id=1).info("no metadata") + + _, row = db.rows[0] + assert row["metadata"] == "{}"