diff --git a/.claude/skills/add-ave-record/SKILL.md b/.claude/skills/add-ave-record/SKILL.md index b6c64ab..a034546 100644 --- a/.claude/skills/add-ave-record/SKILL.md +++ b/.claude/skills/add-ave-record/SKILL.md @@ -45,6 +45,24 @@ python scripts/validate_records.py pytest tests/ -x -q ``` +If validate_records.py fails on AIVSS arithmetic, fix the record's own +aarf/cvss_base/thm/mitigation_factor values or the stated aivss_score, +don't just adjust one to match the other without checking which one is +actually wrong; a computed mismatch usually means the record was drafted +against a different set of factors than what got written down. + +### 7. Publish +A record passing validation is not yet a published one. Update: +- dist/ave-records-latest.json — add or replace this record's entry, + keeping the array sorted by ave_id. +- CHANGELOG.md — one line under Unreleased/Added: the ave_id, title, + severity, and aivss_score. + +Do not bump schema_version or create a new versioned dist snapshot +(dist/ave-records-vX.Y.Z.json) as part of this step. That's a separate, +deliberate decision tied to an actual schema change, not something that +happens automatically because one record got added. + ## Severity / AIVSS consistency CRITICAL → aivss_score >= 9.0 @@ -54,9 +72,30 @@ LOW → < 4.0 If severity and aivss_score disagree, the record fails validation. +A mechanism that reads as severe in plain English can still land MEDIUM, +correctly, if it's narrow and single-vector — AARF's ten factors reward +breadth of amplification, not just raw impact. cvss_base alone carries +the severity of the underlying impact. Don't inflate AARF factors to +force a record into a more severe-sounding band; if the honestly computed +score feels low relative to the mechanism's intuitive severity, say so in +the record's own aivss.notes field rather than adjusting the inputs to +hit a target. See references/aivss-scoring.md for the full formula and +worked examples of this exact situation. + ## confidence_baseline guide High-signal AVE (hardcoded AWS key, explicit external fetch): 0.85-0.95 Medium-signal (suspicious instruction phrasing): 0.55-0.75 Low-signal (vague, needs corroboration): 0.40-0.55 The scanner adjusts from this baseline via the FP pipeline. + +## Reference files + +- references/aivss-scoring.md — the AARF formula, how aars is computed + from the ten factors, and worked examples spanning MEDIUM through HIGH, + including the specific trap of inflating factors to chase a severity + band. +- references/schema-fields.md — the provenance_vector.entry_class enum + (confirmed live against the corpus) and escalation values, distinct + from grill-with-docs Q7's detection_layer, a coarser, separate field; + don't conflate the two when writing provenance_vector. \ No newline at end of file diff --git a/.claude/skills/grill-with-docs/SKILL.md b/.claude/skills/grill-with-docs/SKILL.md index 349dc22..4174b3f 100644 --- a/.claude/skills/grill-with-docs/SKILL.md +++ b/.claude/skills/grill-with-docs/SKILL.md @@ -7,7 +7,14 @@ Grill before defining a vulnerability class. No record until complete. Q1: In one sentence, what does a vulnerable component DO? (This becomes behavioral_fingerprint — must be behavioral, not a string.) Q2: Is this a new attack_class or a variant of an existing one? - (Check records/ for similar attack_class values first.) + Don't check attack_class label similarity alone, that's not + reliable, a genuinely distinct mechanism can have a similar-sounding + name, and a genuine duplicate can have a completely different one. + Pull any plausible match's real provenance_vector fields + (entry_class, payload_surface, escalation) and the full description, + compare directly against this candidate's actual mechanism. Only + call it a variant if the entry_class and payload_surface genuinely + match, not if the label or general topic sounds similar. Q3: What is the worst realistic impact? (drives cvss_base and severity) Q4: How much does agent autonomy amplify it? (drives aars) Q5: Which engines can detect it? pattern/yara/semgrep/llm/sandbox/magika diff --git a/records/AVE-2026-00048.json b/records/AVE-2026-00048.json index 1a5d421..04374b0 100644 --- a/records/AVE-2026-00048.json +++ b/records/AVE-2026-00048.json @@ -83,27 +83,27 @@ "references": [ { "tag": "CWE-269", - "text": "CWE-269: Improper Privilege Management \u2014 MITRE Common Weakness Enumeration", + "text": "CWE-269: Improper Privilege Management — MITRE Common Weakness Enumeration", "url": "https://cwe.mitre.org/data/definitions/269.html" }, { "tag": "OWASP LLM Excessive Agency", - "text": "OWASP Top 10 for LLM Applications \u2014 Excessive Agency", + "text": "OWASP Top 10 for LLM Applications — Excessive Agency", "url": "https://owasp.org/www-project-top-10-for-large-language-model-applications/" }, { "tag": "Cohen 2024", - "text": "Cohen et al. \u2014 Here Comes The AI Worm: Unleashing Zero-click Worms that Target GenAI-Powered Applications (arXiv 2403.02817)", + "text": "Cohen et al. — Here Comes The AI Worm: Unleashing Zero-click Worms that Target GenAI-Powered Applications (arXiv 2403.02817)", "url": "https://arxiv.org/abs/2403.02817" }, { "tag": "CWE-284", - "text": "CWE-284: Improper Access Control \u2014 MITRE Common Weakness Enumeration", + "text": "CWE-284: Improper Access Control — MITRE Common Weakness Enumeration", "url": "https://cwe.mitre.org/data/definitions/284.html" }, { "tag": "AVE Registry", - "text": "AVE-2026-00048 \u2014 AVE behavioral vulnerability registry", + "text": "AVE-2026-00048 — AVE behavioral vulnerability registry", "url": "https://github.com/aveproject/ave/blob/main/records/AVE-2026-00048.json" } ], @@ -125,7 +125,7 @@ "data_access": 1, "external_dependencies": 0.5 }, - "aars": 7.5, + "aars": 8.0, "thm": 0.9, "mitigation_factor": 1, "aivss_score": 7.7, @@ -145,4 +145,4 @@ "derivable_into": [ "privilege-escalation-chain" ] -} +} \ No newline at end of file diff --git a/references/aivss-scoring.md b/references/aivss-scoring.md new file mode 100644 index 0000000..2035bb5 --- /dev/null +++ b/references/aivss-scoring.md @@ -0,0 +1,106 @@ +# AIVSS scoring reference + +The formula: `aivss_score = round(((cvss_base + aars) / 2) * thm * mitigation_factor, 1)` + +Where `aars` is the sum of ten AARF (Agentic AI Risk Factor) values, each +scored 0, 0.5, or 1: + +- `autonomy`: does this fire without further attacker interaction once + triggered +- `tool_use`: does it require the agent's own tool-calling capability +- `multi_agent`: does it require or specifically involve more than one + agent +- `non_determinism`: does exploitation reliability vary run to run +- `self_modification`: does the component modify its own behavior or the + agent's runtime +- `dynamic_identity`: does it involve impersonation, identity claims, or + trust-anchor confusion +- `persistent_memory`: does the effect persist beyond the current session +- `natural_language_input`: is the exploit mechanism itself natural + language, versus a structural/syntactic mechanism that merely + originates from an NL-driven call +- `data_access`: does exploitation grant or require broad data access +- `external_dependencies`: does severity depend on which specific SDK, + library, or third-party service is in use + +`thm` (technique has model): 1 if the mechanism is real and demonstrated +(a disclosed CVE, published research, a working exploit description), 1 +means no discount; lower values exist for less-verified mechanisms but +every record in this corpus so far has scored 1, since the evidentiary +bar in Step 3 of the main workflow already requires real sourcing. + +`mitigation_factor`: 1 if no broadly effective, ecosystem-wide mitigation +exists yet (don't discount just because a fix is theoretically possible); +0.83 if a simple, well-known, standard mitigation exists and is +reasonably expected to be applied (pinning a dependency, re-enabling TLS +verification). Never invent a value outside this observed 1/0.83 range +without a clearly stated reason, since every record so far has used one +of these two. + +## Severity bands + +- CRITICAL: 9.0 to 10.0 +- HIGH: 7.0 to 8.9 +- MEDIUM: 4.0 to 6.9 +- LOW: below 4.0 + +## The counterintuitive part, worth internalizing before scoring anything + +A mechanism that sounds severe in plain English can still land MEDIUM, +correctly, if it's narrow and single-vector. AARF's ten factors reward +*breadth* of amplification, not just raw impact. `cvss_base` alone +carries the raw severity of the underlying impact; a near-maximum +`cvss_base` (9.0+) on a narrow, single-mechanism class (no multi-agent +involvement, no persistence, no self-modification) will still average +down to MEDIUM once combined with a modest `aars`. This already happened +correctly for the zero-click auto-run record (`cvss_base` 9.0, landed at +5.2 MEDIUM) and the STDIO shell injection record landing HIGH rather than +CRITICAL despite being RCE (`cvss_base` 9.8, `aars` only 4.5). Do not +inflate AARF factors to force a class into a "more severe-sounding" band; +report the honest computed result and explain why in the record's own +`aivss.notes` field, the same way every record in this corpus already +does. + +## Worked examples + +**STDIO transport shell injection** (HIGH, 7.2): `cvss_base` 9.8 (near-max, +this is RCE), `aarf` sums to 4.5 (autonomy 1, tool_use 1, natural_language_input +0.5, data_access 1, external_dependencies 1, everything else 0), `thm` 1, +`mitigation_factor` 1 (patches exist but ecosystem-wide exposure wasn't +resolved at time of writing). `((9.8+4.5)/2)*1*1 = 7.15` rounds to 7.2. + +**TLS verification disabled** (MEDIUM, 4.1): `cvss_base` 7.5, `aarf` sums +to 2.5 (autonomy 0.5, tool_use 0.5, data_access 1, external_dependencies +0.5), `thm` 1, `mitigation_factor` 0.83 (a simple, standard fix exists). +`((7.5+2.5)/2)*1*0.83 = 4.15` rounds to 4.1. + +**A2A agent card poisoning** (HIGH, 7.1): `cvss_base` 8.7, `aarf` sums to +5.5 (autonomy 1, tool_use 0.5, multi_agent 1 at genuine maximum since +this is definitionally a two-agent mechanism, non_determinism 0.5, +dynamic_identity 0.5, natural_language_input 1, data_access 0.5, +external_dependencies 0.5), `thm` 1, `mitigation_factor` 1. `((8.7+5.5)/2)*1*1 += 7.1`. + +**Zero-click IDE auto-run** (MEDIUM, 5.2, despite sounding severe): +`cvss_base` 9.0 (near-max, zero-click RCE-adjacent), `aarf` sums to only +3.5 (autonomy 1, tool_use 0.5, persistent_memory 0.5, data_access 1, +external_dependencies 0.5, no multi-agent, no self-modification), `thm` +1, `mitigation_factor` 0.83 (disabling auto-run is a known, standard +fix). `((9.0+3.5)/2)*1*0.83 = 5.1875` rounds to 5.2. This is the record +worth re-reading if a future score feels wrong, it's the clearest example +of a severe-sounding mechanism correctly landing MEDIUM. + +**Unpinned dependency substitution** (MEDIUM, 4.4): `cvss_base` 7.0, +`aarf` sums to 3.5 (autonomy 0.5, tool_use 0.5, non_determinism 0.5, +persistent_memory 0.5, data_access 0.5, external_dependencies 1 at +maximum since this class is definitionally about dependency behavior), +`thm` 1, `mitigation_factor` 0.83. `((7.0+3.5)/2)*1*0.83 = 4.3575` rounds +to 4.4. + +## Always independently re-verify + +Compute the score by hand or by reasoning, then run the actual arithmetic +in `scripts/verify_and_publish.py` before treating it as final. This +reference and the worked examples are for building the right intuition +while drafting, not a substitute for the script actually re-running the +sum and the formula against the record as written. \ No newline at end of file diff --git a/references/schema-fields.md b/references/schema-fields.md new file mode 100644 index 0000000..e78a449 --- /dev/null +++ b/references/schema-fields.md @@ -0,0 +1,105 @@ +# Schema fields reference + +## `entry_class`, confirmed live against the corpus, not assumed + +This list was pulled directly from the live `dist/ave-records-latest.json` +during this workflow's own development, not reconstructed from memory. +Re-run the query below periodically, since new records may introduce +values not listed here yet: + +```bash +curl -s https://raw.githubusercontent.com/aveproject/ave/main/dist/ave-records-latest.json | python3 -c " +import json, sys +records = json.load(sys.stdin) +classes = set() +for r in records: + ec = r.get('provenance_vector', {}).get('entry_class') + if ec: classes.add(ec) +print(sorted(classes)) +" +``` + +Confirmed values and what each actually means, with a real example +record for each: + +- **`content`**: instruction text embedded in a skill's own body. + Example: `AVE-2026-00048`, sub-agent delegation instructions written + directly into skill instruction text. +- **`memory`**: an agent's persistent memory store. Example: + `AVE-2026-00019`, planted false beliefs or instructions written into + memory, executed in a future session. +- **`model_generated`**: content the agent itself produces, not something + it reads. Example: `AVE-2026-00056`, a markdown image URL the agent's + own generated response embeds. +- **`registry_metadata`**: a declarative configuration or registry entry, + not instruction text. Example: `AVE-2026-00061` through `00064` + (TLS verification, dependency pinning, approval bypass, auto-run + configuration), all static config states, not content an agent reads + and interprets as an instruction. +- **`retrieved_document`**: content pulled in via RAG or similar retrieval, + distinct from a skill file's own body. +- **`runtime`**: something happening at execution time, not at a fixed + location in a file. Example: `AVE-2026-00050`, tool registration + happening during session initialization. +- **`server_card_document`**: a trusted capability-declaration document + read before interaction begins, regardless of which protocol produces + it. Example: `AVE-2026-00041` (MCP server-card injection) and + `AVE-2026-00065` (A2A agent card poisoning), same entry_class, + genuinely different protocols and payload surfaces, see the note in + `00065` for the reasoning behind reusing rather than forking this + value. +- **`skill_file`**: the skill file itself as a static artifact, distinct + from `content` (the instruction text within it). Example: + `AVE-2026-00024`, a file whose actual bytes don't match its declared + extension. +- **`tool_response`**: a tool call's return value, not the request. + Example: `AVE-2026-00018`, tool result manipulation. +- **`tool_schema`**: an MCP tool's own description or parameter schema + field. Example: `AVE-2026-00002`, `AVE-2026-00059` (ShareLock). +- **`transport`**: the protocol/transport layer itself, not content + carried over it. Example: `AVE-2026-00049` (HTTP header injection), + `AVE-2026-00060` (STDIO shell injection), genuinely different + mechanisms sharing this value the same way `server_card_document` is + shared, confirmed by direct comparison before assuming overlap. +- **`user_input`**: content the user directly supplies, not something the + agent fetches or reads from a component. + +**When deciding whether a new candidate needs a new `entry_class` value or +can reuse an existing one**: ask whether the *role* the content plays is +the same as an existing value, even if the protocol or format differs. +`server_card_document` covers "trusted capability metadata read before +interaction" across two different protocols already. Reuse before +forking, and state the reasoning in the record's own `aivss.notes` field +either way, the same way `00065` did. + +## `escalation`, the values seen so far + +- `data_to_instruction`: passive content gets treated as an active + directive. The most common value in the corpus. +- `instruction_to_capability`: an instruction is followed and grants or + exercises a capability (a tool call, a permission grant). +- `capability_to_identity`: exercising a capability results in an + identity or trust claim being accepted (impersonation, spoofing). + +Not every record needs this field; omit rather than force a fit if none +of these describes the actual mechanism. + +## Required fields, minimum viable record + +`ave_id`, `schema_version`, `status`, `component_type`, `title`, +`attack_class`, `severity`, `description`, `aivss_score`, +`behavioral_fingerprint`, `provenance_vector` (at least `entry_class`), +`mitigation`, `detection_methodology`, `indicators_of_compromise`, +`remediation`, `researcher`, `published`, `references` (at least one, with +a real, working URL), `aivss` (the full scoring object, not just the +top-level `aivss_score` summary). + +## `status` values + +Currently only `active` is implemented in the schema. `deprecated`, +`merged`, and `rejected` are policy (see +`docs/specs/scaling-and-governance.md` Section 3) but not yet schema +fields, don't use them on a record until the schema change implementing +them has actually shipped, using them prematurely would produce a record +that fails validation or silently means nothing to any tooling reading +it. \ No newline at end of file diff --git a/scripts/validate_records.py b/scripts/validate_records.py index b200bc2..a760cbf 100644 --- a/scripts/validate_records.py +++ b/scripts/validate_records.py @@ -1,14 +1,23 @@ # What: validates every AVE record against the current schema plus the Section 8 # invariants from the v1.1.0 migration (no stale field names, no leaked -# enforcement config, no dual-empty behavioral_vector/example_patterns) +# enforcement config, no dual-empty behavioral_vector/example_patterns), +# plus AIVSS score arithmetic and vendor-neutral language, added after a +# hand-drafted batch of records caught real instances of exactly these +# problems that nothing here checked # Why: a malformed or drifted record breaks every downstream scanner that loads it, # and a free-text value in `mitigation` would let vendor-specific config -# leak back into a standard that is supposed to stay vendor-neutral +# leak back into a standard that is supposed to stay vendor-neutral. +# A stated aivss_score that doesn't match the record's own aarf/cvss_base/ +# thm/mitigation_factor is silently wrong severity data shipped to every +# consumer of the corpus. A stray vendor product name is a neutrality +# violation this project enforces everywhere else; records shouldn't be +# the one place it's unchecked. # How: jsonschema.Draft202012Validator against schema/ave-record-1.1.0.schema.json # (handles the draft-vs-active conditional required set natively), plus a # handful of checks the schema's additionalProperties:false already implies # but which deserve a readable, named failure message of their own import json +import re import sys from pathlib import Path @@ -33,6 +42,13 @@ }, } +VENDOR_BOILERPLATE_PATTERNS = [ + r"bawbel-scanner", + r"bawbel-gate", + r"bawbel\s+scan\b", + r"piranha", +] + def check_schema(record: dict, validator: jsonschema.Draft202012Validator) -> list[str]: return [f"schema: {e.message} (at {'/'.join(str(p) for p in e.path) or ''})" @@ -74,6 +90,49 @@ def check_mitigation_enums_only(record: dict) -> list[str]: return errors +def check_aivss_arithmetic(record: dict) -> list[str]: + """Recomputes aars and aivss_score from the record's own aarf, cvss_base, + thm, and mitigation_factor fields, and confirms both the nested + aivss.aivss_score and the top-level aivss_score field agree with it. + A record that drifts here is shipping a severity number nobody + actually derived from its own stated inputs.""" + aivss = record.get("aivss") + if not isinstance(aivss, dict): + return [] + aarf = aivss.get("aarf") + if not isinstance(aarf, dict) or not aarf: + return [] + + errors = [] + aars = round(sum(aarf.values()), 4) + stated_aars = aivss.get("aars") + if aars != stated_aars: + errors.append(f"aivss.aars mismatch: computed {aars}, record states {stated_aars}") + + required = ("cvss_base", "thm", "mitigation_factor") + missing = [f for f in required if f not in aivss] + if missing: + errors.append(f"aivss missing scoring field(s): {', '.join(missing)}") + return errors + + computed_score = round(((aivss["cvss_base"] + aars) / 2) * aivss["thm"] * aivss["mitigation_factor"], 1) + stated_score = aivss.get("aivss_score") + if computed_score != stated_score: + errors.append(f"aivss.aivss_score mismatch: computed {computed_score}, record states {stated_score}") + + top_level_score = record.get("aivss_score") + if top_level_score != stated_score: + errors.append(f"top-level aivss_score ({top_level_score}) does not match aivss.aivss_score ({stated_score})") + + return errors + + +def check_no_vendor_boilerplate(raw_text: str) -> list[str]: + lower = raw_text.lower() + return [f"vendor-specific reference found: '{pattern}'" + for pattern in VENDOR_BOILERPLATE_PATTERNS if re.search(pattern, lower)] + + def main() -> int: schema = json.loads(SCHEMA_PATH.read_text()) jsonschema.Draft202012Validator.check_schema(schema) @@ -86,7 +145,8 @@ def main() -> int: total_errors = 0 for path in paths: - record = json.loads(path.read_text()) + raw_text = path.read_text() + record = json.loads(raw_text) rid = record.get("ave_id", path.name) errors = ( check_schema(record, validator) @@ -94,6 +154,8 @@ def main() -> int: + check_no_nested_owasp_mcp_mapping(record) + check_behavioral_vector_or_example_patterns(record) + check_mitigation_enums_only(record) + + check_aivss_arithmetic(record) + + check_no_vendor_boilerplate(raw_text) ) for e in errors: print(f"{rid}: {e}") @@ -107,4 +169,4 @@ def main() -> int: if __name__ == "__main__": - raise SystemExit(main()) + raise SystemExit(main()) \ No newline at end of file