diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 39a7be1..692f729 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "prman", - "version": "0.4.0", - "description": "Human-confirmed, end-to-end Draft PR automation for Codex", + "version": "0.5.0", + "description": "Help developers contribute to well-known open-source projects with Codex", "author": { "name": "PRman contributors" }, @@ -17,8 +17,8 @@ "skills": "./skills/", "interface": { "displayName": "PRman", - "shortDescription": "Find, fix, and draft PRs.", - "longDescription": "PRman guides Codex from read-only repository discovery through implementation, deterministic quality checks, a content-bound human confirmation, Draft PR creation, and validated CI repair transitions. Codex supplies the coding and connected GitHub tools; PRman supplies the workflow and fail-closed local safety state.", + "shortDescription": "Contribute to popular repos.", + "longDescription": "PRman helps an ordinary developer make one useful contribution to a well-known open-source project. It finds a suitable issue, lets Codex implement and test the change, shows a simple preview, asks for one short confirmation, creates a Draft PR, and follows CI. Detailed quality and safety checks stay in the background.", "developerName": "PRman contributors", "category": "Productivity", "capabilities": [ @@ -26,8 +26,8 @@ "Write" ], "defaultPrompt": [ - "Use PRman to find a suitable Python issue, implement and verify it, then ask before opening a Draft PR.", - "Use PRman to fix this GitHub issue, assess the final diff, and prepare an exact Draft PR confirmation." + "Help me find a small issue in a well-known Python project, fix it, test it, and ask me before creating a Draft PR.", + "Help me contribute this fix to the project, show me a simple preview, then ask before creating a Draft PR." ] } } diff --git a/README.md b/README.md index fe33d85..a84c0b3 100644 --- a/README.md +++ b/README.md @@ -1,131 +1,134 @@ # PRman -![PRman social cover showing the diff, evidence, scorer, and decision flow](docs/assets/prman-social-preview.png) +![PRman social cover](docs/assets/prman-social-preview.png) -PRman is a pre-alpha Codex Skill and Plugin for one complete pull-request workflow: +PRman helps ordinary developers make useful pull requests to well-known open-source projects. + +You tell Codex what kind of project you want to contribute to. PRman finds one suitable issue, +Codex makes and tests the change, and you review a simple preview. Nothing is sent to GitHub until +you confirm it. + +## What it does ~~~text -User goal - | - v -Read-only GitHub search -> choose one suitable repository and issue - | - v -Codex reads the rules -> implements the change -> runs the repository's checks - | - v -PRman binds the diff and evidence -> ready / revise / abstain - | - v -Exact packet: repository + branches + diff + tests + assessment + PR text - | - v -User confirms - | - v -Create Draft PR -> follow CI -> make bounded in-scope repairs +You: “Help me contribute to a well-known Python project.” + | + v +PRman finds one active project and a small, unclaimed issue + | + v +Codex reads the project rules, makes the change, and runs the tests + | + v +PRman shows a simple contribution preview and the full diff + | + v +You confirm: CREATE DRAFT PR owner/repo + | + v +PRman creates a Draft PR, follows CI, and reports the result ~~~ -PRman is not another coding model. Codex does the searching, reading, editing, command execution, and -connected GitHub operations. PRman supplies the state machine, quality gate, confirmation boundary, -and safe order of operations. It stores no GitHub token and never auto-merges. +You do not need to understand GitHub search syntax, scoring systems, JSON packets, or PR workflow +states. PRman keeps those details in the background. -## What a user can ask +## What you can ask ~~~text -Use $prman to find an active Python repository with a suitable bug, -implement and test the fix, then ask me before opening a Draft PR. +Help me find a small issue in a well-known Python project and make a PR. ~~~ -PRman searches read-only, selects one contribution-friendly target, checks its AGENTS.md, -CONTRIBUTING, SECURITY, issue state, and CI rules, then lets Codex implement and verify the smallest -useful change. It shows the exact proposed write before any GitHub mutation. A reply such as -“确认” or “yes” is rejected: the user must repeat the displayed target-specific phrase exactly. -For a non-ready assessment, that phrase also names the result being acknowledged. - -After confirmation, PRman may create the listed fork or branch, push the assessed commits, create a -Draft PR, and follow CI. It can make up to two directly related CI repairs by default. A new -repository, changed base, changed PR plan, or material scope expansion requires a new confirmation. - -## What version 0.4.0 implements - -- An installable Plugin manifest and an implicitly discoverable prman Skill. -- A declared GitHub MCP dependency; PRman reuses the connection managed by Codex. -- Read-only repository and issue discovery with contribution-fit and anti-spam checks. -- Repository-instruction, security-policy, existing-PR, base-commit, and CI inspection. -- Codex-native local implementation and execution of the target repository's own checks. -- A strict confirmation packet for the exact repository, branch route, diff, verification, - assessment, Draft PR text, external writes, and CI budget. -- A deterministic confirmation helper that hashes the exact packet, rejects stale or inexact user - responses, and emits a scoped write authorization only after the displayed phrase is repeated. -- Draft-only publication, no default-branch write, no force-push, and no merge or auto-merge. -- A local workflow state machine that accepts only Draft PRs, binds CI to the current commit, counts - the confirmed repair rounds, checks the observed base, head route, diff, URL, and PR number, - rejects out-of-scope updates, and completes only after passing CI. -- The existing assessment 1.1 core: strict JSON contracts, repository/base/task/diff bindings, typed - evidence, required and advisory gates, HMAC evidence attestation, optional authenticated scoring, - uncertainty-aware aggregation, and deterministic ready / revise / abstain results. -- Test-only scorers that always force abstain, plus fail-closed scorer errors. - -The confirmation, authorization, and run-state contracts are available under [schemas](schemas), -with an illustrative [examples/confirmation-packet.json](examples/confirmation-packet.json). +~~~text +I want to contribute to the React ecosystem. Find something useful that is small enough to finish. +~~~ + +~~~text +Help me fix this GitHub issue and prepare the PR. +~~~ + +If the request is broad, PRman chooses one good target automatically. It prefers a recognised, +active project with a contribution guide, an open and unclaimed issue, no competing PR, a manageable +scope, and tests that can verify the change. It avoids abandoned projects, public security fixes, +bulk PRs, and cosmetic changes with little value. + +## What you see before anything is sent + +PRman shows a short contribution preview: + +~~~text +Repository: owner/project +Task: Fix empty configuration handling (#42) +Why this one: Active project, clear unclaimed issue, and a focused testable fix. +Changed: 2 files — handle empty input and add a regression test. +Tests: 12 passed. +Risk: No known failing checks. Extra automatic quality score is unavailable. +Draft PR: Fix empty configuration handling + +Review the full diff, then reply: +CREATE DRAFT PR owner/project +~~~ + +That short phrase authorizes only the previewed Draft PR. Internally, PRman binds it to the exact +repository, base commit, branch, diff, PR text, and CI repair limit. If any of those materially +changes before publication, PRman asks again. + +## What version 0.5.0 implements + +- A Codex Skill and installable Plugin for one end-to-end open-source contribution. +- Read-only discovery of one well-known, active, contribution-friendly project and suitable issue. +- Inspection of project instructions, contribution and security policies, duplicate work, CI rules, + and the exact base commit. +- Codex-native implementation and execution of the project's own tests and checks. +- Internal diff, evidence, and quality binding without exposing technical packet details by default. +- A simple user-facing preview with the selected task, changes, tests, risks, Draft PR title, and + reviewable diff. +- The short repository-bound confirmation phrase `CREATE DRAFT PR owner/repo`. +- Draft-only publication. PRman never merges, enables auto-merge, force-pushes, or writes directly + to the default branch. +- CI monitoring and at most two same-task repair rounds by default. +- Executable local contracts that reject stale confirmation, changed targets, unsafe writes, + out-of-scope repairs, and CI results from the wrong commit. + +PRman uses Codex's existing coding, shell, repository, and connected GitHub tools. It does not store +a GitHub token or create a second coding agent. ## Current limits -- Local Plugin installation, explicit and implicit fresh-task discovery, unrelated-question - non-triggering, installed helper resolution, and cache-busted reinstall have passed validation. - See the [2026-08-30 installation record](docs/plugin-installation-validation-2026-08-30.md). - A controlled real-repository Draft PR and CI run is still release work. Treat this as pre-alpha. -- No production scorer or trusted evidence executor is shipped. The checked-in research profile - therefore cannot honestly return production ready. -- When required gates pass but production scoring or attestation is missing, PRman reports abstain. - The user may still explicitly confirm a Draft PR while acknowledging that uncertainty; PRman must - never rename the result to ready. -- PRman is not a background bot or hosted service. It runs inside the active Codex task and depends - on the GitHub tools and permissions available there. -- The local state machine verifies order, content binding, and budgets; it cannot prove that a - confirmation response truly came from the user, that assessment, GitHub, or CI observations are - truthful, or that an `in_scope` repair claim is correct. Its locally writable JSON is a workflow - record, not a hostile-host security boundary. Codex and its connected tools remain responsible - for those observations and the actual writes. -- It does not do bulk outreach, public vulnerability disclosure, reviewer assignment, comments, - approval, merge, auto-merge, or repository administration. -- Thresholds remain research defaults and are not calibrated for production gating. - -## Quality gate inside the workflow - -![PRman quality pipeline from an untrusted target repository through evidence collection, deterministic assessment, authenticated scoring, and human-confirmed external mutation](docs/assets/prman-pipeline-trust-boundaries.png) - -The Python helper validates an exact UTF-8 diff and its evidence. Required gates run before scoring -and cannot be overridden by a model. A production ready additionally requires an exact scorer -binding, an absolute lower-confidence-bound floor, and a verified evidence attestation. - -The preferred scorer boundary is an HMAC-authenticated loopback HTTP service. Fully trusted Python -entry-point scorers are available only through explicit opt-in. Fixture and static providers exist -for tests and demos and can never issue a readiness claim. See -[docs/scorer-protocol.md](docs/scorer-protocol.md). - -Ready means “eligible to ask the user,” not “correct,” “approved,” or “authorized to publish.” -Every assessment result keeps external_write_authorized false; authority comes only from the later -human confirmation packet. The separate write-authorization artifact is content-bound to that -packet and still permits only the listed Draft PR operations. - -## Skill and Plugin shape - -The Skill contains the workflow and its progressively loaded references. The Plugin makes that Skill -installable and declares the connected GitHub tool it needs. This follows the official OpenAI -documentation for [building skills](https://learn.chatgpt.com/docs/build-skills), +- Version 0.5.0 is pre-alpha. The Skill, local safety contracts, and automated tests work, but this + simplified flow still needs a fresh installed-Plugin test and a complete contribution run against + a controlled real repository. +- PRman runs inside the active Codex task; it is not a hosted service or background bot. +- It handles one contribution at a time, not bulk outreach or mass PR creation. +- It does not assign reviewers, post comments, approve, merge, change repository settings, or make a + private security report public. +- No production quality scorer is bundled. Repository checks still run, and the preview plainly says + when the optional extra quality score is unavailable. +- A maintainer may still request changes, reject the PR, or require a CLA or other human action. + +## How safety works in the background + +PRman keeps an internal packet containing the exact target, base commit, diff, test results, Draft PR +text, planned GitHub writes, and CI repair budget. A local helper hashes and validates that packet. +The user sees a plain-language preview, while the helper ensures the later GitHub write still matches +what was previewed. + +The assessment core checks required gates such as scope, secrets, and tests before optional scoring. +A test scorer can never claim production readiness, and a missing production scorer is reported as +an uncertainty rather than silently treated as a pass. See [architecture](docs/architecture.md), +[threat model](docs/threat-model.md), and [implementation status](docs/IMPLEMENTATION_STATUS.md) for +the technical boundary. + +## Skill and Plugin + +The Skill contains the contribution workflow. The Plugin makes the Skill installable and declares +the connected GitHub tool it needs. The Python distribution and command are named `prman-codex` to +avoid the unrelated existing PyPI project named `prman`. + +PRman follows the official OpenAI documentation for +[building skills](https://learn.chatgpt.com/docs/build-skills), [building plugins](https://learn.chatgpt.com/docs/build-plugins), and [agent approvals](https://learn.chatgpt.com/docs/agent-approvals-security). -The Python distribution and installed command are both named prman-codex, avoiding the unrelated -existing PyPI prman project. - -The local personal-marketplace install and fresh-task invocation checks are recorded in -[docs/plugin-installation-validation-2026-08-30.md](docs/plugin-installation-validation-2026-08-30.md). -That record validates Plugin discovery and routing; it is not evidence of a completed live GitHub -Draft PR. - ## Development PRman supports Python 3.11 and 3.12 and has no runtime dependencies. @@ -138,7 +141,7 @@ make check PYTHON=python make demo PYTHON=python ~~~ -The demo intentionally uses a fixture scorer and returns abstain: +The demo uses a test-only scorer and cannot claim production readiness: ~~~bash python skills/prman/scripts/assess.py \ @@ -147,38 +150,28 @@ python skills/prman/scripts/assess.py \ --allow-test-scorer ~~~ -For a fail-closed run without any scorer: - -~~~bash -python skills/prman/scripts/assess.py \ - --input examples/assessment.json -~~~ - -To validate a confirmation packet before showing it to the user: +To validate an internal confirmation packet: ~~~bash python skills/prman/scripts/workflow.py confirmation prepare \ --input examples/confirmation-packet.json ~~~ -See [docs/architecture.md](docs/architecture.md), -[docs/threat-model.md](docs/threat-model.md), and -[docs/IMPLEMENTATION_STATUS.md](docs/IMPLEMENTATION_STATUS.md) for the exact implementation -boundary. Visuals and mockups are cataloged in +Visual assets and their intended use are listed in [docs/visual-assets.md](docs/visual-assets.md). ## Repository map -- .codex-plugin/: installable Plugin metadata. -- skills/prman/: full Codex workflow, safety rules, references, and bundled assessment helper. -- src/prman/: deterministic assessment, authorization, workflow-state, and scorer code. -- schemas/: assessment, confirmation, authorization, and run-state JSON contracts. -- configs/: decision thresholds and scorer configuration examples. -- examples/: fixture-only assessment, scorer, diff, and confirmation-packet examples. -- docs/assets/: public diagrams, brand references, and clearly separated mockups. -- tests/core/: unit, safety, distribution, CLI, schema, and Skill contract tests. +- `.codex-plugin/`: installable Plugin metadata. +- `skills/prman/`: contribution workflow, safety rules, and bundled helpers. +- `src/prman/`: deterministic assessment, authorization, and workflow-state code. +- `schemas/`: machine-readable assessment and workflow contracts. +- `configs/`: quality-decision and scorer examples. +- `examples/`: assessment and internal confirmation examples. +- `docs/assets/`: public diagrams, brand assets, and separated mockups. +- `tests/core/`: unit, safety, distribution, CLI, Schema, and Skill tests. -## License and repository +## License PRman is available under the [Apache License 2.0](LICENSE). The canonical repository is [primorLee/PRman](https://github.com/primorLee/PRman). diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md index 98aee46..85cd73c 100644 --- a/docs/IMPLEMENTATION_STATUS.md +++ b/docs/IMPLEMENTATION_STATUS.md @@ -1,33 +1,40 @@ # Implementation status -PRman 0.4.0 is a pre-alpha, end-to-end Draft PR orchestration Skill for Codex. The repository is -public at primorLee/PRman and licensed under Apache-2.0. The workflow is implemented in the Skill and -in executable local contracts. Local personal-marketplace installation, fresh-task discovery, -negative routing, installed helper resolution, and cache-busted reinstall passed on 2026-08-30; -live GitHub end-to-end validation is still required before a production claim. +PRman 0.5.0 is a pre-alpha Codex Skill for ordinary developers who want to contribute to well-known +open-source projects. Its default flow is: find one suitable issue, implement and test it, show a +simple contribution preview, wait for a short repository-bound confirmation, create a Draft PR, and +follow CI. Detailed quality and authorization artifacts remain internal. The repository is public +at primorLee/PRman and licensed under Apache-2.0. + +The 0.4.0 Plugin installation and routing checks passed on 2026-08-30. The new 0.5.0 simplified +experience is implemented in source and local contracts but still needs fresh installed-Plugin and +live GitHub end-to-end validation before a production claim. ## Implemented workflow - Installable Codex Plugin and implicitly discoverable prman Skill. - GitHub MCP dependency declared in the Skill metadata; no PRman-owned token or credential store. -- Read-only repository and issue discovery, small-target comparison, contribution-fit selection, - duplicate-work checks, and anti-spam rules. +- Read-only repository and issue discovery focused by default on one recognised, active, + contribution-friendly project, with automatic selection when there is a clear best target. +- Suitability checks for issue clarity, manageable scope, outside-contributor acceptance, + duplicate work, verification feasibility, usefulness, and anti-spam behavior. - Inspection of agent instructions, README, contribution and security policies, license, issue context, pull-request template, default branch, full base commit, and CI configuration. - Codex-native local implementation and execution of repository-prescribed verification. - Existing deterministic assessment after the final diff, with two revision rounds by default. -- Exact confirmation packet covering repository, task, branch and fork route, base commit, diff, +- A simple user-facing preview covering the selected task, why it was chosen, changed files, tests, + risks or unknowns, Draft PR title, and a reviewable full diff. +- An internal exact packet covering repository, task, branch and fork route, base commit, diff, verification, assessment, Draft PR text, all writes, and CI repair budget. - Runtime validation of the exact patch digest, target and fork route, Draft-only plan, structural ready-state prerequisites, allowed writes, confirmation text, and CI budget. -- Exact target-specific confirmation phrase. Plain yes/confirm responses and whitespace-normalized - variants are rejected; non-ready publication requires the phrase to acknowledge the decision and - the prompt to show its exact reason. +- Short target-specific confirmation phrase: `CREATE DRAFT PR OWNER/REPO`. Plain yes/confirm, + a different repository, and whitespace-normalized variants are rejected. - A content-bound write-authorization artifact limited to the confirmed repository, base commit, head route, initial diff, Draft PR operations, and repair budget. -- Explicit treatment of ready, revise, and abstain. Missing production scoring can never be - represented as ready, although a user may acknowledge the uncertainty and still confirm a Draft - PR. +- Internal treatment of ready, revise, and abstain. These labels are not shown by default. Missing + production scoring can never be represented as ready; the preview instead says plainly that the + optional extra quality score is unavailable. - Draft-only GitHub publication after confirmation, using Codex's connected GitHub tools. - Default maximum of two same-scope CI repair rounds, with new verification and assessment after each edit. @@ -73,9 +80,9 @@ is skills/prman/scripts/workflow.py. The archived review that motivated the assessment hardening is [security-review-2026-08-29.md](security-review-2026-08-29.md). -## Validated Codex installation +## Previously validated Codex installation -The PRM-008 acceptance run installed PRman from the default local `personal` marketplace and +The PRM-008 acceptance run validated version 0.4.0 from the default local `personal` marketplace and verified all of the following in ephemeral, read-only Codex tasks started outside this repository: - Explicit `$prman` invocation loaded version `0.4.0` from the Plugin cache. @@ -85,7 +92,8 @@ verified all of the following in ephemeral, read-only Codex tasks started outsid - Uninstall and reinstall with `0.4.0+codex.validation-20260830` produced a new cache entry, and a subsequent fresh task reported that exact installed version. - Both before and after reinstall, the task rejected a plain yes as write authorization and allowed - only an explicitly confirmed Draft PR. + only an explicitly confirmed Draft PR. This confirms the safety boundary, not the new 0.5.0 + wording or preview experience. The redacted command transcript and exact validation boundary are in [plugin-installation-validation-2026-08-30.md](plugin-installation-validation-2026-08-30.md). @@ -113,11 +121,18 @@ them; they are workflow records, not a hostile-host security boundary. - Run representative repository-discovery tasks and record target selection quality, duplicate-work avoidance, and refusal behavior. -- Exercise the exact confirmation, denial, stale-packet, abstain-acknowledgement, fork, Draft PR, CI - success, CI failure, repair-budget, and material-scope-change paths against controlled repositories. +- Install version 0.5.0 through the Plugin flow and verify implicit invocation, the simple preview, + hidden internal artifacts, and the short `CREATE DRAFT PR OWNER/REPO` confirmation in a fresh + Codex task. +- Complete the live Draft PR and CI portion of the controlled + [confirmation-path validation](confirmation-path-validation-2026-08-30.md). Local denial, + stale-packet, Draft-only, and prohibited-operation checks now pass; the first GitHub write still + requires the packet-bound user response. +- Exercise the missing-extra-score disclosure, fork, CI failure, repair-budget, and + material-scope-change paths against controlled repositories. - Integrate and calibrate one external production scorer and trusted evidence executor. - Publish scorer conformance tests and adversarial false-ready and fabricated-evidence evaluation. - Add a hash-locked, platform-specific dependency lockfile and reproducible runner image. -Until those checks are complete, PRman is a working workflow contract and quality core, not a -validated autonomous production PR service. +Until those checks are complete, PRman is a working Skill, workflow contract, and quality core, not +a validated autonomous production PR service. diff --git a/docs/architecture.md b/docs/architecture.md index f243144..f50aacc 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2,8 +2,9 @@ ## Decision -PRman is a full pull-request orchestration Skill distributed as a Plugin. Codex remains the only -coding and tool-execution harness. +PRman is a contribution workflow Skill distributed as a Plugin for ordinary developers who want to +make a useful pull request to a well-known open-source project. Codex remains the only coding and +tool-execution harness. The product boundary is “PRman directs; Codex acts.” PRman defines repository selection, workflow state, evidence requirements, human confirmation, allowed GitHub writes, and CI stopping rules. @@ -15,16 +16,18 @@ store, or background worker. ### User goal and authorization -The first request defines repository and task criteria. It authorizes read-only discovery and local +The first request defines repository and task criteria. A broad request uses safe defaults rather +than requiring the developer to supply search filters. It authorizes read-only discovery and local implementation, but never serves as the final GitHub-write approval. Publication authority is -granted only after the user sees and confirms the exact post-implementation packet. +granted only after the user sees and confirms the post-implementation contribution preview. ### Read-only discovery The Skill directs Codex to search for a small number of relevant repositories and issues, reject -unsuitable or duplicate work, and select one contribution-friendly target. It checks activity, -issue clarity, assignment and existing-PR state, contribution rules, security policy, licensing, -verification feasibility, default branch, and full base commit. +unsuitable or duplicate work, and select one contribution-friendly target. By default it prefers a +recognised, actively maintained project with evidence that outside PRs are reviewed. It checks +activity, issue clarity, assignment and existing-PR state, contribution rules, security policy, +licensing, verification feasibility, default branch, and full base commit. Discovery uses the GitHub MCP dependency declared in skills/prman/agents/openai.yaml. No fork, branch, comment, assignment, or other GitHub mutation occurs in this stage. @@ -47,7 +50,7 @@ default required gates are scope, secrets, and tests. Extra gates are advisory. readiness requires criterion minima, raw and lower-confidence-bound thresholds, acceptable uncertainty and truncation, exact scorer metadata binding, and a verified canonical evidence HMAC. -### Confirmation packet +### Contribution preview and internal confirmation packet After the final diff is verified and assessed, the Skill prepares a packet containing: @@ -61,14 +64,20 @@ After the final diff is verified and assessed, the Skill prepares a packet conta schemas/confirmation_packet.schema.json defines the machine-readable contract. Initial publication is valid only for the unchanged packet; later diff updates are limited to its explicit CI repair envelope. The assessment result itself always denies external-write authority. The local workflow -helper validates the packet, returns its canonical digest and target-specific confirmation phrase, -and creates a scoped authorization only when the response matches that phrase byte-for-byte. A -non-ready phrase explicitly acknowledges revise or abstain, and its prompt must include the exact -assessment reason. +helper validates the packet, returns its canonical digest and the short phrase +`CREATE DRAFT PR OWNER/REPO`, and creates a scoped authorization only when the response matches that +phrase byte-for-byte. + +The packet and digest stay internal by default. The user sees a contribution preview containing the +repository and task, selection reason, change summary and files, test results, risks or unknowns, +Draft PR title, and a reviewable full diff. Internal non-ready states are translated into plain +language; a known required-check failure is not offered for publication in the default flow. The authorization artifact and run state have separate schemas. Their parsers recheck the target, -base, branch and fork route, allowed writes, Draft-only policy, confirmation-phrase digest, and CI -budget instead of trusting serialized state. +base, branch and fork route, allowed writes, Draft-only policy, confirmation-scope digest, and CI +budget instead of trusting serialized state. That internal digest combines the short user phrase +with the complete packet-bound authorization scope, so shortening the phrase does not drop the +branch, diff, assessment, or repair-budget binding. ### GitHub publication and CI follow-up @@ -96,7 +105,7 @@ passing CI reaches complete. 4. Inspect: load repository rules, task context, base commit, and CI expectations. 5. Implement: let Codex edit locally and collect observed verification evidence. 6. Assess: run deterministic gates and optional scoring; revise at most twice by default. -7. Confirm: display the exact packet and stop for the user's answer. +7. Confirm: display the simple contribution preview and stop for the user's short answer. 8. Publish: perform only the confirmed fork, branch, push, and Draft PR operations. 9. Follow CI: inspect checks and perform bounded in-scope repairs. 10. Report: return the Draft PR URL, head commit, CI state, assessment, and remaining human work. @@ -107,9 +116,10 @@ change returns to Confirm. A denied or missing confirmation terminates with a lo ## Abstention behavior Without a production scorer or trusted evidence attestation, required gates can pass while the final -result remains abstain. PRman must show that uncertainty. The user may still explicitly confirm a -Draft PR when the abstention is due only to missing production quality infrastructure. A known failed -required gate is not equivalent and requires a separate, failure-specific override. +internal result remains abstain. PRman translates that into a plain note that the optional extra +quality score is unavailable. The user may still explicitly confirm a Draft PR when repository +checks passed and this is the only uncertainty. A known failed required gate is not equivalent and +is not offered in the default flow. ## Package boundary diff --git a/docs/confirmation-path-validation-2026-08-30.md b/docs/confirmation-path-validation-2026-08-30.md new file mode 100644 index 0000000..031d215 --- /dev/null +++ b/docs/confirmation-path-validation-2026-08-30.md @@ -0,0 +1,85 @@ +# Confirmation-path validation — 2026-08-30 + +## Current result + +The local and fresh-task parts of PRM-009 pass. PRman refuses explicit denial, ambiguous responses, +and stale packets without creating a write authorization; its authorization contract permits the +listed Draft PR operations and rejects merge, auto-merge, force-push, default-branch writes, and +mark-ready operations. + +No GitHub mutation was performed during these checks. The final live step is a controlled Draft PR +from `codex/prm-009-controlled-validation` to `primorLee/PRman:main`, and it remains pending the +packet-bound user confirmation described below. + +## Controlled target selection + +- Target: `primorLee/PRman` +- Repository state: public, active, Apache-2.0, default branch `main` +- Permission observed read-only through GitHub CLI: `ADMIN` +- Base commit: `bb3363872b6659048bae442e10627896cdc9aa82` +- Duplicate-work check: no open pull requests and no open issues at selection time +- Reason: this is the project owner's controlled repository, so it can validate the write boundary + without sending unsolicited work to a third party + +The proposed change adds focused denial/staleness coverage and makes the local-handoff behavior +explicit in the Skill safety reference. It does not add a GitHub client or a second execution +harness. + +## Deterministic contract checks + +The focused tests exercise these cases: + +| Case | Expected result | +| --- | --- | +| Exact unchanged phrase | A content-bound authorization may be emitted for only the packet's operations | +| `no, do not publish` | CLI exits with status 2 and writes no authorization file | +| Correct phrase with a stale packet digest | CLI exits with status 2 and writes no authorization file | +| Normal rather than Draft PR | Contract rejects the observation | +| Merge, auto-merge, force-push, mark-ready, or default-branch write | Authorization returns false | +| Draft PR route, repository, base, branch, or diff mismatch | Contract rejects the observation | + +The complete project suite must pass on Python 3.11 and 3.12 before the confirmation packet is +prepared. Skill and Plugin validation are also required because the safety reference changes. + +Completed verification: + +- Python 3.11: 74 tests passed, 2 loopback-socket tests skipped by the sandbox, 80% branch coverage. +- Python 3.12: 74 tests passed, 2 loopback-socket tests skipped by the sandbox, 80% branch coverage. +- Ruff lint and format, strict mypy, compileall, Skill validation, Plugin validation, and JSONL + parsing all passed. + +## Fresh Codex task result + +An ephemeral, read-only Codex task started outside the authoring repository and explicitly invoked +the installed `prman:prman` Skill. It read version +`0.4.0+codex.validation-20260830` from the local Plugin cache and evaluated two cases without +accessing GitHub: + +```text +Case A — "no, do not publish" +Authorization artifact: No. +GitHub writes: None, including no fork, branch creation, push, Draft PR, or smaller write. +Handoff: Stop with the local diff and verification results; do not retry automatically. + +Case B — diff changed after the exact response +Authorization artifact: No valid artifact for the changed diff. +GitHub writes: None, not even the first write. +Next: rerun verification and assessment, prepare a new packet and digest, and wait for a fresh +byte-for-byte response. +``` + +The task also retained the prohibitions on merge, approval, auto-merge, mark-ready, force-push, +default-branch writes, normal PR creation, repository administration, and unlisted GitHub actions. +The public record omits the Codex task ID and user-specific cache path. + +## Live step still pending + +Before any remote branch, push, or Draft PR is created, PRman must prepare a packet containing the +exact final patch, test evidence, assessment result, base and head route, Draft PR title and body, +write list, and CI budget. The preparation result must still say +`external_write_authorized: false`. + +Only the byte-for-byte phrase emitted for that unchanged packet may create the local authorization. +After that response, the allowed live sequence is limited to pushing the confirmed head branch, +creating the confirmed Draft PR, recording its returned identity, and reading its CI. The Draft PR +must not be merged or marked ready as part of this validation. diff --git a/docs/threat-model.md b/docs/threat-model.md index f5a55bc..7c14100 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -111,10 +111,11 @@ test_only true. ### Stale or ambiguous write confirmation The assessment result always sets external_write_authorized false. Before any GitHub mutation, the -Skill presents the exact packet and stops. A short affirmative response such as yes or confirm is -never sufficient. The response must exactly match a phrase naming the repository and head branch; -for revise or abstain it also names the decision being acknowledged, while the prompt shows the -exact non-ready reason. Leading or trailing whitespace is not normalized. +Skill prepares the exact packet, presents a plain-language contribution preview, and stops. A short +affirmative response such as yes or confirm is never sufficient. The response must exactly match +`CREATE DRAFT PR OWNER/REPO`. The unchanged internal packet binds the base, branch route, diff, PR +text, writes, assessment, and CI budget. Any non-ready reason is disclosed in the preview in plain +language. Leading or trailing whitespace is not normalized. Changing the repository, base commit, branch route, initial diff, PR title or body, operation list, or CI budget before publication invalidates confirmation. After publication, only a repair within diff --git a/examples/assessment.json b/examples/assessment.json index 2b6a03b..912ce2c 100644 --- a/examples/assessment.json +++ b/examples/assessment.json @@ -25,7 +25,7 @@ "candidate_id": "4332ae57e04a07cdc1a796642b5de1f6367a30487e465f4bfea2757a19d156a7", "observed_at": "2026-08-29T00:00:00Z", "producer": "prman-example", - "producer_version": "0.4.0", + "producer_version": "0.5.0", "summary": "The fixture changes only example.txt.", "log_digest": "f0b8ab916e84ea561bfefddd6ebd509deef30cc9d5c90fa15ff8804ef56ea481", "command": null, @@ -43,7 +43,7 @@ "candidate_id": "4332ae57e04a07cdc1a796642b5de1f6367a30487e465f4bfea2757a19d156a7", "observed_at": "2026-08-29T00:00:00Z", "producer": "prman-example", - "producer_version": "0.4.0", + "producer_version": "0.5.0", "summary": "The fixture diff contains no credential-like value.", "log_digest": "427eb6524b1a1b6585711d12ac601c8fde770495fab2fb68c69ae4f3deac35b3", "command": null, diff --git a/examples/confirmation-packet.json b/examples/confirmation-packet.json index cfaf2d2..ecf0064 100644 --- a/examples/confirmation-packet.json +++ b/examples/confirmation-packet.json @@ -1,5 +1,5 @@ { - "schema_version": "prman-confirmation-packet/1.1", + "schema_version": "prman-confirmation-packet/1.2", "repository": { "full_name": "octo-org/widget", "url": "https://github.com/octo-org/widget", @@ -39,7 +39,7 @@ ], "assessment": { "decision": "abstain", - "reason": "Required gates passed, but no production scorer or trusted evidence attestation is configured.", + "reason": "Repository checks passed, but PRman's optional extra quality score is unavailable.", "scorer": null, "test_only": false, "attestation_verified": false, @@ -67,7 +67,7 @@ }, "approval": { "status": "pending", - "prompt": "Assessment result: Required gates passed, but no production scorer or trusted evidence attestation is configured. Reply exactly ‘CONFIRM DRAFT PR octo-org/widget codex/handle-empty-config ACKNOWLEDGE ABSTAIN’ to acknowledge that result, create this Draft PR, and allow up to two in-scope CI repair updates.", - "confirmation_phrase": "CONFIRM DRAFT PR octo-org/widget codex/handle-empty-config ACKNOWLEDGE ABSTAIN" + "prompt": "Repository checks passed, but PRman's optional extra quality score is unavailable. Reply exactly ‘CREATE DRAFT PR octo-org/widget’ to create this Draft PR and allow up to two same-task CI repair updates.", + "confirmation_phrase": "CREATE DRAFT PR octo-org/widget" } } diff --git a/github_issues.jsonl b/github_issues.jsonl index 076d632..aca0cb9 100644 --- a/github_issues.jsonl +++ b/github_issues.jsonl @@ -1,5 +1,5 @@ {"external_id":"PRM-001","title":"[PRM-001] Freeze the Codex-native architecture boundary","milestone":"M0 Codex-native MVP","labels":["type:architecture","area:skill","priority:P0"],"depends_on":[],"body_markdown":"## Goal\nRecord that Codex is the only coding and execution harness.\n\n## Acceptance criteria\n- [x] ADR selects Skill-first, Plugin-distributed architecture\n- [x] Coding-agent, worktree, sandbox, and GitHub harnesses are outside the published package\n- [x] Evidence-integrity limitations are explicit\n\n## Deliverables\n- docs/adr/0003-codex-native.md\n- docs/architecture.md"} -{"external_id":"PRM-002","title":"[PRM-002] Create the focused PRman Skill","milestone":"M0 Codex-native MVP","labels":["type:engineering","area:skill","priority:P0"],"depends_on":["PRM-001"],"body_markdown":"## Goal\nDefine the reusable Codex workflow without repeating generic coding guidance.\n\n## Acceptance criteria\n- [x] Skill routes assessment, scorer, and write-safety details progressively\n- [x] Review-only requests do not authorize edits\n- [x] GitHub writes require exact human confirmation and remain Draft-only\n- [ ] Representative fresh-task behavior is forward-tested in Codex\n\n## Deliverables\n- skills/prman/"} +{"external_id":"PRM-002","title":"[PRM-002] Create the focused PRman Skill","milestone":"M0 Codex-native MVP","labels":["type:engineering","area:skill","priority:P0"],"depends_on":["PRM-001"],"body_markdown":"## Goal\nDefine the reusable Codex workflow without repeating generic coding guidance.\n\n## Acceptance criteria\n- [x] Skill routes assessment, scorer, and write-safety details progressively\n- [x] Review-only requests do not authorize edits\n- [x] GitHub writes require exact human confirmation and remain Draft-only\n- [x] Representative fresh-task behavior is forward-tested in Codex\n\n## Deliverables\n- skills/prman/\n- docs/plugin-installation-validation-2026-08-30.md\n- docs/confirmation-path-validation-2026-08-30.md"} {"external_id":"PRM-003","title":"[PRM-003] Ship deterministic assessment contracts and CLI","milestone":"M0 Codex-native MVP","labels":["type:engineering","area:core","priority:P0"],"depends_on":["PRM-001"],"body_markdown":"## Goal\nConsume Codex-supplied evidence and return a strict ready/revise/abstain result.\n\n## Acceptance criteria\n- [x] Duplicate and unknown contract fields fail closed\n- [x] Missing, unknown, and fatal gates dominate scorer output\n- [x] Single and comparison modes are deterministic\n- [x] Result never authorizes an external write\n\n## Deliverables\n- src/prman/assessment.py\n- src/prman/decision.py\n- schemas/"} {"external_id":"PRM-004","title":"[PRM-004] Package PRman as a validated Codex Plugin","milestone":"M0 Codex-native MVP","labels":["type:engineering","area:plugin","priority:P0"],"depends_on":["PRM-002","PRM-003"],"body_markdown":"## Goal\nMake the Skill and deterministic helper installable as one plugin bundle.\n\n## Acceptance criteria\n- [x] Manifest contains valid metadata and the Skill path\n- [x] Bundled helper resolves paths relative to the installed plugin\n- [x] Internal distribution smoke tests pass\n- [x] Official Plugin and Skill validators pass\n\n## Deliverables\n- .codex-plugin/plugin.json\n- skills/prman/scripts/assess.py"} {"external_id":"PRM-005","title":"[PRM-005] Publish scorer provider conformance tests","milestone":"M1 Production scorer boundary","labels":["type:engineering","area:scorer","priority:P0"],"depends_on":["PRM-003"],"body_markdown":"## Goal\nLet independently packaged scorers prove protocol compatibility.\n\n## Acceptance criteria\n- [ ] Extract reusable provider conformance tests for metadata stability, digest echo, canonical criteria, finite ranges, OOD, authentication, and error handling\n- [x] Test providers are core-classified and always force the final selection to abstain\n- [x] Provider failures produce a structured fail-closed handoff\n\n## Deliverables\n- scorer conformance test module\n- provider author documentation"} diff --git a/pyproject.toml b/pyproject.toml index fabeb02..4dcfd5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta" [project] name = "prman-codex" -version = "0.4.0" -description = "Pre-alpha end-to-end Draft PR automation for Codex" +version = "0.5.0" +description = "Help developers contribute to well-known open-source projects with Codex" readme = "README.md" license = "Apache-2.0" license-files = ["LICENSE"] diff --git a/schemas/confirmation_check.schema.json b/schemas/confirmation_check.schema.json index ec40a22..4923022 100644 --- a/schemas/confirmation_check.schema.json +++ b/schemas/confirmation_check.schema.json @@ -13,7 +13,7 @@ ], "properties": { "schema_version": { - "const": "prman-confirmation-check/1.0" + "const": "prman-confirmation-check/1.1" }, "packet_digest": { "type": "string", @@ -27,7 +27,7 @@ "type": "string", "minLength": 1, "maxLength": 200, - "pattern": "^CONFIRM DRAFT PR [A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+ \\S+(?: ACKNOWLEDGE (?:REVISE|ABSTAIN))?$" + "pattern": "^CREATE DRAFT PR [A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$" }, "policy": { "const": { diff --git a/schemas/confirmation_packet.schema.json b/schemas/confirmation_packet.schema.json index bb68e6c..c57777f 100644 --- a/schemas/confirmation_packet.schema.json +++ b/schemas/confirmation_packet.schema.json @@ -20,7 +20,7 @@ ], "properties": { "schema_version": { - "const": "prman-confirmation-packet/1.1" + "const": "prman-confirmation-packet/1.2" }, "repository": { "type": "object", @@ -294,7 +294,7 @@ "type": "string", "minLength": 1, "maxLength": 200, - "pattern": "^CONFIRM DRAFT PR [A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+ \\S+(?: ACKNOWLEDGE (?:REVISE|ABSTAIN))?$" + "pattern": "^CREATE DRAFT PR [A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$" } } } @@ -328,62 +328,6 @@ } } } - }, - { - "if": { - "properties": { - "assessment": { - "properties": {"decision": {"const": "ready"}} - } - } - }, - "then": { - "properties": { - "approval": { - "properties": { - "confirmation_phrase": { - "pattern": "^CONFIRM DRAFT PR [A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+ \\S+$" - } - } - } - } - } - }, - { - "if": { - "properties": { - "assessment": { - "properties": {"decision": {"const": "revise"}} - } - } - }, - "then": { - "properties": { - "approval": { - "properties": { - "confirmation_phrase": {"pattern": " ACKNOWLEDGE REVISE$"} - } - } - } - } - }, - { - "if": { - "properties": { - "assessment": { - "properties": {"decision": {"const": "abstain"}} - } - } - }, - "then": { - "properties": { - "approval": { - "properties": { - "confirmation_phrase": {"pattern": " ACKNOWLEDGE ABSTAIN$"} - } - } - } - } } ], "$defs": { diff --git a/schemas/write_authorization.schema.json b/schemas/write_authorization.schema.json index ce2f3b9..7fa2402 100644 --- a/schemas/write_authorization.schema.json +++ b/schemas/write_authorization.schema.json @@ -19,7 +19,7 @@ ], "properties": { "schema_version": { - "const": "prman-write-authorization/1.0" + "const": "prman-write-authorization/1.1" }, "packet_digest": { "$ref": "#/$defs/sha256" diff --git a/skills/prman/SKILL.md b/skills/prman/SKILL.md index 0dfde04..af67cf6 100644 --- a/skills/prman/SKILL.md +++ b/skills/prman/SKILL.md @@ -1,22 +1,29 @@ --- name: prman -description: "Run a human-confirmed GitHub pull-request workflow with Codex: discover a suitable repository or issue, implement and verify a focused change, assess it with evidence gates, prepare an exact confirmation packet, create a Draft PR, and follow CI. Use when the user asks Codex to find or complete contribution work and prepare or send a PR; do not use for generic code questions, review-only requests, or bulk outreach." +description: "Help ordinary developers contribute to well-known open-source projects with Codex: find one suitable repository issue, implement and test a focused change, show a simple contribution preview, create a human-confirmed Draft PR, and follow CI. Use when the user wants to find an open-source contribution or complete and send a PR; do not use for generic code questions, review-only requests, or bulk outreach." --- # PRman -PRman is the workflow layer; Codex is the coding and tool-execution layer. Use Codex's existing -repository, shell, and GitHub tools. Do not create a second coding agent, credential store, GitHub -client, worktree manager, or command sandbox. +PRman helps a developer complete one useful open-source contribution. Codex performs the search, +coding, tests, and connected GitHub operations; PRman keeps the steps safe and understandable. Use +Codex's existing repository, shell, and GitHub tools. Do not create a second coding agent, +credential store, GitHub client, worktree manager, or command sandbox. + +Keep the default user experience simple. Do not expose JSON packets, digests, state-machine names, +scorer terminology, or attestation details unless the user asks. Store those internal artifacts in +temporary files. Speak in plain language about what was selected, changed, tested, and still +uncertain. ## End-to-end workflow -1. Establish the requested repository criteria, task type, and any fixed target. The initial request - authorizes read-only discovery and local implementation, not an external GitHub write. Even if - the user asks to "send a PR automatically," require the confirmation in step 7. -2. Read [references/orchestration.md](references/orchestration.md). Search GitHub read-only, select - one contribution-friendly target, and inspect its instructions before editing. If the user names - a repository or issue, start from that target but still verify that the contribution is suitable. +1. Understand what the developer wants to contribute. If the request is broad, default to one + small, well-defined issue in a well-known, active, contribution-friendly project; do not make the + user design search filters. The initial request authorizes read-only discovery and local work, + not a GitHub write. Even if the user asks to send a PR automatically, require step 7. +2. Read [references/orchestration.md](references/orchestration.md). Search GitHub read-only, choose + one target, and inspect its instructions before editing. If the user names a repository or issue, + start there but still verify that an outside contribution is suitable. 3. Let Codex make the smallest task-complete local change and run the target repository's own relevant tests, lint, type checks, or builds. Preserve exact command results and the final diff. 4. Read [references/assessment-contract.md](references/assessment-contract.md), prepare a bound @@ -28,21 +35,19 @@ client, worktree manager, or command sandbox. Use any available Python 3.11+ interpreter. Resolve the script relative to this `SKILL.md`. The helper evaluates supplied evidence; it does not run commands or call GitHub. -5. Interpret the quality result literally: - - `ready`: eligible to ask for confirmation, not permission to publish or proof of correctness. - - `revise`: address the reported, actionable problem and reassess. - - `abstain`: disclose the uncertainty or missing production scorer/attestation. Never relabel it - as `ready`. A Draft PR may still be offered only through the explicit acknowledgement path in - the GitHub workflow reference. -6. Use at most two assessment-guided revision rounds by default. If a required gate still has a - known failure, stop unless the user separately and explicitly accepts that exact failure. +5. Treat assessment as an internal quality check. Fix actionable problems and reassess, for at most + two rounds by default. Do not offer a Draft PR with a known required-check failure. If repository + tests pass but optional production scoring is unavailable, say only that the automatic quality + score is unavailable; do not ask the user to understand `abstain`, scorers, or attestations. +6. Keep the exact diff, evidence, assessment, and planned GitHub writes bound in temporary internal + artifacts. Any edit after assessment requires fresh verification and assessment. 7. Read [references/github-workflow.md](references/github-workflow.md) and - [references/safety.md](references/safety.md). Show one exact confirmation packet containing the - target, branches, base commit, diff, verification, assessment, Draft PR text, planned writes, and - CI repair budget. Validate it with the bundled workflow helper, show its digest and exact - target-specific confirmation phrase, then stop and wait. For `revise` or `abstain`, show the - exact reason and require the helper's acknowledgement phrase. Create a scoped write authorization - only from the user's byte-for-byte response to that unchanged packet; a plain yes is invalid. + [references/safety.md](references/safety.md). Build and validate the complete internal packet, + then show a short contribution preview: repository and issue, why it was selected, what changed, + test results, risks or unknowns, Draft PR title, and an accessible full diff. State that PRman + will create a Draft PR and follow CI. Show the helper's short target-specific confirmation phrase + and stop. Do not display the packet JSON or digest unless asked. Create scoped write authorization + only from the user's exact response to that unchanged preview; a plain yes is invalid. 8. After authorization, use Codex's GitHub tools to perform only the listed writes: create or use the confirmed contribution branch, publish the exact assessed change, and open a Draft PR. Record the returned Draft PR in the helper state. Never merge, enable auto-merge, force-push, or write @@ -51,15 +56,15 @@ client, worktree manager, or command sandbox. repairs, rerun verification and assessment, and update the same Draft PR without force-pushing. Advance the helper state for each CI result and repair round. A material scope change requires a refreshed packet and new confirmation. -10. Return the Draft PR URL, CI state, assessment decision, changes made after confirmation, and any - remaining human action. +10. Return the Draft PR URL, CI result, any repairs made, and any remaining human action in plain + language. Keep internal decision labels out of the default report. -## Scorer boundary +## Internal scorer boundary The scorer is optional and replaceable. Without an authenticated production scorer, an exact decision-profile binding, and a trusted-executor evidence attestation, passing hard gates still -results in `abstain`. Test scorers always force `abstain`; never use a fixture or static scorer to -claim readiness. +results internally in `abstain`. Test scorers always force `abstain`; never use a fixture or static +scorer to claim readiness. Translate this state into a short user-facing uncertainty note. Read [references/scorer-contract.md](references/scorer-contract.md) only when configuring, implementing, or diagnosing a scorer. Treat Python entry-point scorers as fully trusted in-process diff --git a/skills/prman/agents/openai.yaml b/skills/prman/agents/openai.yaml index 836d512..606b299 100644 --- a/skills/prman/agents/openai.yaml +++ b/skills/prman/agents/openai.yaml @@ -1,7 +1,7 @@ interface: display_name: "PRman" - short_description: "Find, fix, validate, and draft GitHub PRs" - default_prompt: "Use $prman to find a suitable Python issue, implement and validate the fix, then ask before creating a Draft PR." + short_description: "Contribute to well-known open-source projects" + default_prompt: "Use $prman to find a small issue in a well-known project, implement and test the fix, show a simple preview, then ask before creating a Draft PR." dependencies: tools: - type: "mcp" diff --git a/skills/prman/references/assessment-contract.md b/skills/prman/references/assessment-contract.md index 93d0c47..7cd0a19 100644 --- a/skills/prman/references/assessment-contract.md +++ b/skills/prman/references/assessment-contract.md @@ -144,5 +144,5 @@ so it abstains until exact calibrated and attestation bindings are explicitly co Copy the exact decision, reason, test-only state, scorer identity, and attestation state into the confirmation packet. Do not treat the assessment result as publication authority. When the result -abstains only because production scoring or attestation is unavailable, follow the explicit -uncertainty-acknowledgement path in `github-workflow.md`. +abstains only because production scoring or attestation is unavailable, follow the plain-language +uncertainty disclosure in `github-workflow.md`; keep the internal labels out of the default preview. diff --git a/skills/prman/references/github-workflow.md b/skills/prman/references/github-workflow.md index 71e3b6d..5c68bdc 100644 --- a/skills/prman/references/github-workflow.md +++ b/skills/prman/references/github-workflow.md @@ -1,9 +1,8 @@ # Confirmation, Draft PR, and CI workflow -Read this reference before presenting a GitHub confirmation packet and again after the user -confirms it. +Read this reference before presenting a contribution preview and again after the user confirms it. -## Confirmation packet +## Internal confirmation packet Before any GitHub mutation, create one self-contained packet matching the [confirmation-packet schema](../../../schemas/confirmation_packet.schema.json). It must identify: @@ -19,7 +18,9 @@ Before any GitHub mutation, create one self-contained packet matching the - whether CI will be monitored and the maximum number of in-scope repair rounds (default: two). Embed the exact UTF-8 patch in the packet. Set its SHA-256 to the digest of those exact bytes. The -approval object must show a short, target-specific `confirmation_phrase` verbatim inside its prompt. +approval object must show the short phrase `CREATE DRAFT PR OWNER/REPO` verbatim inside its prompt. +The packet binds the branch, diff, assessment, PR text, writes, and CI budget even though the user +does not need to repeat them. Validate and bind the packet from the skill directory, using temporary output paths: @@ -29,19 +30,32 @@ python3.11 scripts/workflow.py confirmation prepare \ --output ``` -Show the complete packet plus the returned packet digest and confirmation phrase. The check output -must still say `external_write_authorized: false`. - -Ask for confirmation only after displaying these fields. The response must exactly match the shown -confirmation phrase. Confirmation expires if the target, base commit, initial diff, branch route, PR -text, planned writes, or CI budget changes before initial publication. After the Draft PR exists, -only the explicitly confirmed CI repair envelope may change the diff without a new packet. - -`ready` is never write authorization. If the decision is `abstain` only because a production scorer -or trusted attestation is unavailable, the packet may still offer a Draft PR, but it must say that -quality readiness is unknown and ask the user to acknowledge that uncertainty. Do not present a -known required-gate failure as this missing-scorer exception. A remaining `revise` or failed required -gate needs a separate explicit override naming the failure. +Keep the packet, check output, and digest in temporary storage. The check output must still say +`external_write_authorized: false`. + +Before asking, show a compact contribution preview with: + +- repository and selected issue or task; +- one sentence explaining why it is a good contribution target; +- changed files and a plain-language summary; +- tests run and their result; +- any risk, unverified item, or automatic-quality-score limitation; +- proposed Draft PR title and a reviewable full diff; +- a short statement that confirmation will allow a fork if needed, a contribution branch, a Draft + PR, CI monitoring, and only the stated number of same-task CI repairs. + +Do not paste the JSON packet, digest, raw scorer output, or state-machine fields by default. Provide +them only if the user asks. The response must exactly match the displayed short phrase. Confirmation +expires if the target, base commit, initial diff, branch route, PR text, planned writes, or CI budget +changes before initial publication. After the Draft PR exists, only the confirmed CI repair envelope +may change the diff without a new packet. + +`ready` is never write authorization. If the decision is `abstain` only because optional production +scoring or trusted attestation is unavailable, the preview may still offer a Draft PR, but say in +plain language that the repository tests passed while an extra automatic quality score is +unavailable. The confirmation phrase remains the same short repository-bound phrase. Do not offer a +Draft PR with a known required-gate failure in the default workflow; a separate user-requested +override must name that exact failure. If confirmation is denied, ambiguous, or absent, stop with the local diff and test results. Do not retry the prompt or perform a smaller write. @@ -62,10 +76,10 @@ python3.11 scripts/workflow.py workflow begin \ The helper rejects a changed packet, an inexact response, a normal PR, inconsistent branches, a `ready` claim missing its required production-scorer or attestation fields, unsupported writes, and -mismatched CI repair authority. The exact phrase names the repository and head branch; non-ready -phrases also acknowledge the decision and the prompt must show its exact reason. It records -user-derived authorization but cannot prove the response actually came from the user or that the -packet was displayed; Codex must never invent or reuse it. +mismatched CI repair authority. The exact phrase names the repository; the prompt must also show any +non-ready reason in understandable language. It records user-derived authorization but cannot prove +the response actually came from the user or that the preview was displayed; Codex must never invent +or reuse it. ## Publish only what was confirmed diff --git a/skills/prman/references/orchestration.md b/skills/prman/references/orchestration.md index 7c27532..75f8eed 100644 --- a/skills/prman/references/orchestration.md +++ b/skills/prman/references/orchestration.md @@ -5,27 +5,35 @@ a user-selected target. ## Discovery is read-only -Translate the user's goal into repository, language, activity, and task criteria. Search with -Codex's existing GitHub tools and build a small shortlist; do not fork, branch, comment, assign an -issue, or otherwise mutate GitHub during discovery. +Translate the user's goal into repository, language, activity, and task criteria. When the user only +says they want to contribute, choose sensible defaults instead of asking them to understand GitHub +search filters. Search with Codex's existing GitHub tools and build a small internal shortlist; do +not fork, branch, comment, assign an issue, or otherwise mutate GitHub during discovery. -Choose one target, not a batch. Prefer work with all of the following: +Choose one target, not a batch. By default, look for a well-known project: one that is recognisable +in its ecosystem, actively maintained, used by a real developer community, and has a visible path +for outside contributions. Stars and forks are useful signals, not a hard cutoff. Prefer work with +all of the following: - a clear, open issue or maintainer-requested task that is not already assigned or covered by an active pull request; -- recent project activity and a visible contribution path; -- a scope that can be understood, implemented, and verified in the current task; +- recent maintainer activity, a contribution guide, and evidence that outside PRs are reviewed; +- a small scope that an ordinary developer can understand, implement, and verify in the current + task; - repository instructions and licensing that do not conflict with the proposed contribution; -- tests or another concrete way to check the change. +- tests or another concrete way to check the change; +- an outcome that is useful to the project, not merely a cosmetic change made to create a PR. Avoid speculative drive-by changes, generated bulk PRs, abandoned repositories, dependency churn without a stated need, and cosmetic edits presented as substantive work. Never turn a suspected vulnerability into a public issue or PR; follow the repository's private security-reporting policy instead and stop the public contribution workflow. -When several targets are plausible, compare at most three using issue clarity, contribution fit, -maintainer activity, verification feasibility, and expected scope. Record why the selected target is -the best fit so the confirmation packet can explain the choice. +When several targets are plausible, compare at most three internally using project recognition, +issue clarity, contribution fit, maintainer activity, verification feasibility, and expected scope. +Select the best one automatically when there is a clear winner. Ask the user to choose only when the +options differ materially in language, effort, or risk. Record a one-sentence plain-language reason +for the contribution preview. ## Inspect before editing diff --git a/skills/prman/references/safety.md b/skills/prman/references/safety.md index e56bc92..4119d70 100644 --- a/skills/prman/references/safety.md +++ b/skills/prman/references/safety.md @@ -3,23 +3,38 @@ Read this reference immediately before any proposed GitHub mutation. - Discovery, cloning, local edits, tests, and assessment do not authorize an external write. -- Present the exact confirmation packet defined in `github-workflow.md`, then wait. Confirmation - applies only to that repository, base, branch route, initial diff, PR text, write list, and the - explicitly bounded CI repair envelope. +- Prepare the exact internal packet defined in `github-workflow.md`, present its simple contribution + preview, then wait. Confirmation applies only to that repository, base, branch route, initial + diff, PR text, write list, and the explicitly bounded CI repair envelope. - Run the bundled confirmation helper before asking and after the exact response. No authorization - artifact means no GitHub write. A plain yes/confirm is invalid; repeat only the target-specific - phrase displayed by the helper. Never invent, normalize, trim, or reuse the user's response. + artifact means no GitHub write. A plain yes/confirm is invalid; require only the short phrase + `CREATE DRAFT PR OWNER/REPO` displayed by the helper. Never invent, normalize, trim, or reuse the + user's response. - Only create or update the confirmed Draft PR. Never merge, approve, enable auto-merge, mark ready for review, force-push, write the default branch, change repository administration, or expose secrets. - Recompute the exact diff/candidate ID and rerun bound verification after every edit. A `ready` result is eligibility to ask, not approval or a correctness proof. -- An `abstain` caused by missing production scoring or attestation may be published only after the - user acknowledges that uncertainty. Never hide a known failing gate behind `abstain`. +- An internal `abstain` caused only by missing optional production scoring or attestation may be + published after the preview plainly discloses that limitation. Never hide a known failing gate + behind `abstain`, and do not make internal assessment vocabulary part of the default interaction. - CI repair authority is bounded by the confirmed task and round limit. Material scope changes, review responses, and new GitHub operations require new confirmation. - If confirmation is missing, ambiguous, denied, or stale, stop with a local handoff. - Never place tokens, private payloads, model weights, or raw scorer training data in artifacts. +## Denied, ambiguous, or stale confirmation + +Do not run `confirmation authorize` unless the response is the exact short phrase for the unchanged +packet. If an inexact response or stale digest is passed to the helper, it exits unsuccessfully +before emitting a write-authorization artifact. The absence of an artifact is a hard stop, not a +reason to retry with normalized text or a reduced write. + +Return a plain-language local-only handoff containing the selected target, change summary, test +results, reviewable diff, and the reason publication stopped. Keep the local branch, commit, digest, +and assessment record available if the user asks. An explicit denial ends the attempt without +another confirmation prompt. A stale packet requires updated verification and assessment, a newly +prepared complete packet, and a fresh exact response before the first GitHub write. + Use Codex's connected GitHub tools and approval surfaces. PRman stores no GitHub credential and provides no separate network mutation layer. diff --git a/src/prman/__init__.py b/src/prman/__init__.py index 98b2163..efc9072 100644 --- a/src/prman/__init__.py +++ b/src/prman/__init__.py @@ -4,4 +4,4 @@ __all__ = ["__version__"] -__version__ = "0.4.0" +__version__ = "0.5.0" diff --git a/src/prman/workflow.py b/src/prman/workflow.py index 073ea3f..77f5c6b 100644 --- a/src/prman/workflow.py +++ b/src/prman/workflow.py @@ -130,19 +130,48 @@ def _require_unique_strings(value: Any, *, path: str) -> tuple[str, ...]: return result -def _confirmation_phrase( - repository: str, - head_branch: str, - decision: AssessmentDecision, -) -> str: - phrase = f"CONFIRM DRAFT PR {repository} {head_branch}" - if decision != "ready": - phrase += f" ACKNOWLEDGE {decision.upper()}" +def _confirmation_phrase(repository: str) -> str: + phrase = f"CREATE DRAFT PR {repository}" if len(phrase) > 200: raise ContractError("confirmation.approval.confirmation_phrase exceeds 200 characters") return phrase +def _confirmation_scope_digest( + *, + phrase: str, + packet_digest: str, + repository: str, + base_branch: str, + base_commit: str, + head_repository: str, + head_branch: str, + diff_sha256: str, + assessment_decision: AssessmentDecision, + external_writes: tuple[WriteOperation, ...], + ci_max_fix_rounds: int, + ci_publish_repairs: bool, + ci_scope: str, +) -> str: + return canonical_digest( + { + "phrase": phrase, + "packet_digest": packet_digest, + "repository": repository, + "base": {"branch": base_branch, "commit": base_commit}, + "head": {"repository": head_repository, "branch": head_branch}, + "diff_sha256": diff_sha256, + "assessment_decision": assessment_decision, + "external_writes": list(external_writes), + "ci_followup": { + "max_fix_rounds": ci_max_fix_rounds, + "publish_repairs": ci_publish_repairs, + "scope": ci_scope, + }, + } + ) + + def _validate_verification(value: Any) -> None: records = require_list(value, path="confirmation.verification") if not records: @@ -188,7 +217,7 @@ class ConfirmationPacket: ci_publish_repairs: bool ci_scope: str confirmation_phrase: str - schema_version: str = "prman-confirmation-packet/1.1" + schema_version: str = "prman-confirmation-packet/1.2" @classmethod def from_dict(cls, value: Any) -> ConfirmationPacket: @@ -211,7 +240,7 @@ def from_dict(cls, value: Any) -> ConfirmationPacket: }, path="confirmation", ) - if item["schema_version"] != "prman-confirmation-packet/1.1": + if item["schema_version"] != "prman-confirmation-packet/1.2": raise ContractError(f"unsupported confirmation packet {item['schema_version']!r}") repository_value = require_object(item["repository"], path="confirmation.repository") @@ -394,17 +423,14 @@ def from_dict(cls, value: Any) -> ConfirmationPacket: confirmation_phrase = require_string( approval["confirmation_phrase"], path="confirmation.approval.confirmation_phrase" ) - expected_phrase = _confirmation_phrase(repository, head_branch, assessment_decision) + expected_phrase = _confirmation_phrase(repository) if confirmation_phrase != expected_phrase: raise ContractError( "confirmation.approval.confirmation_phrase must name the exact Draft PR target" ) if confirmation_phrase not in prompt: raise ContractError("confirmation.approval.prompt must show the confirmation phrase") - if ( - assessment_decision != "ready" - and f"Assessment result: {assessment_reason}" not in prompt - ): + if assessment_decision != "ready" and assessment_reason not in prompt: raise ContractError( "confirmation.approval.prompt must show the exact non-ready assessment reason" ) @@ -429,7 +455,7 @@ def from_dict(cls, value: Any) -> ConfirmationPacket: def preparation(self) -> dict[str, Any]: return { - "schema_version": "prman-confirmation-check/1.0", + "schema_version": "prman-confirmation-check/1.1", "packet_digest": self.packet_digest, "repository": self.repository, "confirmation_phrase": self.confirmation_phrase, @@ -456,7 +482,7 @@ class WriteAuthorization: ci_max_fix_rounds: int ci_publish_repairs: bool ci_scope: str - schema_version: str = "prman-write-authorization/1.0" + schema_version: str = "prman-write-authorization/1.1" @classmethod def from_dict(cls, value: Any) -> WriteAuthorization: @@ -478,7 +504,7 @@ def from_dict(cls, value: Any) -> WriteAuthorization: }, path="authorization", ) - if item["schema_version"] != "prman-write-authorization/1.0": + if item["schema_version"] != "prman-write-authorization/1.1": raise ContractError(f"unsupported write authorization {item['schema_version']!r}") base = require_object(item["base"], path="authorization.base") exact_fields(base, {"branch", "commit"}, path="authorization.base") @@ -552,8 +578,24 @@ def from_dict(cls, value: Any) -> WriteAuthorization: raise ContractError( "authorization.external_writes: update_draft_pr must match the CI repair policy" ) - expected_phrase_digest = sha256_text( - _confirmation_phrase(repository, head_branch, assessment_decision) + initial_diff_sha256 = require_sha256( + item["initial_diff_sha256"], path="authorization.initial_diff_sha256" + ) + ci_scope = require_string(ci["scope"], path="authorization.ci_followup.scope") + expected_phrase_digest = _confirmation_scope_digest( + phrase=_confirmation_phrase(repository), + packet_digest=packet_digest, + repository=repository, + base_branch=base_branch, + base_commit=base_commit, + head_repository=head_repository, + head_branch=head_branch, + diff_sha256=initial_diff_sha256, + assessment_decision=assessment_decision, + external_writes=writes, + ci_max_fix_rounds=ci_max_fix_rounds, + ci_publish_repairs=ci_publish_repairs, + ci_scope=ci_scope, ) if confirmation_phrase_digest != expected_phrase_digest: raise ContractError( @@ -568,14 +610,12 @@ def from_dict(cls, value: Any) -> WriteAuthorization: base_commit=base_commit, head_repository=head_repository, head_branch=head_branch, - initial_diff_sha256=require_sha256( - item["initial_diff_sha256"], path="authorization.initial_diff_sha256" - ), + initial_diff_sha256=initial_diff_sha256, assessment_decision=assessment_decision, external_writes=writes, ci_max_fix_rounds=ci_max_fix_rounds, ci_publish_repairs=ci_publish_repairs, - ci_scope=require_string(ci["scope"], path="authorization.ci_followup.scope"), + ci_scope=ci_scope, ) def as_dict(self) -> dict[str, Any]: @@ -656,7 +696,21 @@ def authorize_confirmation( raise ContractError("response does not exactly match the confirmation phrase") return WriteAuthorization( packet_digest=packet.packet_digest, - confirmation_phrase_digest=sha256_text(response), + confirmation_phrase_digest=_confirmation_scope_digest( + phrase=response, + packet_digest=packet.packet_digest, + repository=packet.repository, + base_branch=packet.base_branch, + base_commit=packet.base_commit, + head_repository=packet.head_repository, + head_branch=packet.head_branch, + diff_sha256=packet.diff_sha256, + assessment_decision=packet.assessment_decision, + external_writes=packet.external_writes, + ci_max_fix_rounds=packet.ci_max_fix_rounds, + ci_publish_repairs=packet.ci_publish_repairs, + ci_scope=packet.ci_scope, + ), repository=packet.repository, base_branch=packet.base_branch, base_commit=packet.base_commit, diff --git a/tests/core/test_distribution.py b/tests/core/test_distribution.py index 7166433..8bb5e41 100644 --- a/tests/core/test_distribution.py +++ b/tests/core/test_distribution.py @@ -37,7 +37,7 @@ def test_plugin_manifest_names_skill_directory(self) -> None: manifest = json.loads((ROOT / ".codex-plugin" / "plugin.json").read_text(encoding="utf-8")) self.assertEqual(manifest["name"], "prman") self.assertEqual(manifest["skills"], "./skills/") - self.assertEqual(manifest["version"], "0.4.0") + self.assertEqual(manifest["version"], "0.5.0") self.assertIn("Write", manifest["interface"]["capabilities"]) self.assertLessEqual(len(manifest["interface"]["shortDescription"]), 30) prompts = manifest["interface"]["defaultPrompt"] @@ -112,11 +112,8 @@ def test_confirmation_packet_contract_is_draft_only_and_bounded(self) -> None: } ready_packet["approval"] = { "status": "pending", - "prompt": ( - "Reply exactly ‘CONFIRM DRAFT PR octo-org/widget " - "codex/handle-empty-config’ to create this Draft PR." - ), - "confirmation_phrase": ("CONFIRM DRAFT PR octo-org/widget codex/handle-empty-config"), + "prompt": "Reply exactly ‘CREATE DRAFT PR octo-org/widget’ to create this Draft PR.", + "confirmation_phrase": "CREATE DRAFT PR octo-org/widget", } validator.validate(ready_packet) @@ -162,11 +159,11 @@ def test_confirmation_packet_contract_is_draft_only_and_bounded(self) -> None: inexact_confirmation["approval"]["confirmation_phrase"] = "yes" invalid_packets.append(inexact_confirmation) - missing_abstain_acknowledgement = copy.deepcopy(packet) - missing_abstain_acknowledgement["approval"]["confirmation_phrase"] = ( - "CONFIRM DRAFT PR octo-org/widget codex/handle-empty-config" + verbose_confirmation = copy.deepcopy(packet) + verbose_confirmation["approval"]["confirmation_phrase"] = ( + "CONFIRM DRAFT PR octo-org/widget codex/handle-empty-config ACKNOWLEDGE ABSTAIN" ) - invalid_packets.append(missing_abstain_acknowledgement) + invalid_packets.append(verbose_confirmation) for invalid_packet in invalid_packets: with self.subTest(invalid_packet=invalid_packet), self.assertRaises(ValidationError): diff --git a/tests/core/test_workflow.py b/tests/core/test_workflow.py index 76e3105..dacbf64 100644 --- a/tests/core/test_workflow.py +++ b/tests/core/test_workflow.py @@ -59,19 +59,27 @@ def test_exact_response_creates_a_scoped_roundtrippable_authorization(self) -> N diff_sha256=grant.initial_diff_sha256, ) ) - self.assertFalse( - grant.allows_initial_write( - "merge", - repository=grant.repository, - base_branch=grant.base_branch, - base_commit=grant.base_commit, - head_repository=grant.head_repository, - head_branch=grant.head_branch, - diff_sha256=grant.initial_diff_sha256, - ) - ) + for operation in ( + "merge", + "auto_merge", + "force_push", + "mark_ready_for_review", + "write_default_branch", + ): + with self.subTest(operation=operation): + self.assertFalse( + grant.allows_initial_write( + operation, + repository=grant.repository, + base_branch=grant.base_branch, + base_commit=grant.base_commit, + head_repository=grant.head_repository, + head_branch=grant.head_branch, + diff_sha256=grant.initial_diff_sha256, + ) + ) - def test_ready_packet_uses_a_target_phrase_without_an_acknowledgement_suffix(self) -> None: + def test_ready_packet_uses_the_same_short_repository_phrase(self) -> None: value = confirmation_value() value["assessment"] = { "decision": "ready", @@ -81,7 +89,7 @@ def test_ready_packet_uses_a_target_phrase_without_an_acknowledgement_suffix(sel "attestation_verified": True, "override_acknowledgement_required": False, } - phrase = "CONFIRM DRAFT PR octo-org/widget codex/handle-empty-config" + phrase = "CREATE DRAFT PR octo-org/widget" value["approval"] = { "status": "pending", "prompt": f"Reply exactly ‘{phrase}’ to create this Draft PR.", @@ -199,10 +207,15 @@ def test_packet_contract_rejects_unsafe_or_inconsistent_plans(self) -> None: hidden_phrase["approval"]["prompt"] = "Confirm this write." invalid_values.append(hidden_phrase) - short_phrase = copy.deepcopy(confirmation_value()) - short_phrase["approval"]["prompt"] = "Reply exactly yes." - short_phrase["approval"]["confirmation_phrase"] = "yes" - invalid_values.append(short_phrase) + generic_phrase = copy.deepcopy(confirmation_value()) + generic_phrase["approval"]["prompt"] = "Reply exactly yes." + generic_phrase["approval"]["confirmation_phrase"] = "yes" + invalid_values.append(generic_phrase) + + wrong_repository = copy.deepcopy(confirmation_value()) + wrong_repository["approval"]["prompt"] = "Reply exactly CREATE DRAFT PR other/widget." + wrong_repository["approval"]["confirmation_phrase"] = "CREATE DRAFT PR other/widget" + invalid_values.append(wrong_repository) hidden_non_ready_reason = copy.deepcopy(confirmation_value()) hidden_non_ready_reason["approval"]["prompt"] = ( @@ -230,6 +243,14 @@ def test_tampered_authorization_cross_fields_are_rejected(self) -> None: default_branch["head"]["branch"] = default_branch["base"]["branch"] invalid_values.append(default_branch) + changed_branch = copy.deepcopy(value) + changed_branch["head"]["branch"] = "codex/different-change" + invalid_values.append(changed_branch) + + changed_base = copy.deepcopy(value) + changed_base["base"]["commit"] = "b" * 40 + invalid_values.append(changed_base) + missing_fork = copy.deepcopy(value) missing_fork["external_writes"].remove("create_fork") invalid_values.append(missing_fork) @@ -430,6 +451,38 @@ def test_cli_prepares_and_authorizes_exact_packet(self) -> None: grant = json.loads(authorized_output.getvalue()) self.assertTrue(grant["policy"]["external_write_authorized"]) + def test_denied_or_stale_confirmation_writes_no_authorization(self) -> None: + packet = ConfirmationPacket.from_dict(confirmation_value()) + packet_path = ROOT / "examples" / "confirmation-packet.json" + cases = ( + ("no, do not publish", packet.packet_digest, "exactly match"), + (packet.confirmation_phrase, "0" * 64, "packet changed"), + ) + with tempfile.TemporaryDirectory() as directory: + authorization_path = Path(directory) / "authorization.json" + for response, expected_digest, expected_error in cases: + with ( + self.subTest(response=response), + contextlib.redirect_stderr(io.StringIO()) as err, + ): + status = main( + [ + "confirmation", + "authorize", + "--input", + str(packet_path), + "--expected-packet-digest", + expected_digest, + "--response", + response, + "--output", + str(authorization_path), + ] + ) + self.assertEqual(status, 2) + self.assertIn(expected_error, err.getvalue()) + self.assertFalse(authorization_path.exists()) + def test_cli_persists_a_complete_workflow_run(self) -> None: grant = authorization() with tempfile.TemporaryDirectory() as directory: