From 645d7552d27a33146365b348ee474be6f795bbeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 02:35:05 +0300 Subject: [PATCH 001/119] docs: lock Go v2 rewrite contract --- docs/V2_CONTRACT.md | 555 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 555 insertions(+) create mode 100644 docs/V2_CONTRACT.md diff --git a/docs/V2_CONTRACT.md b/docs/V2_CONTRACT.md new file mode 100644 index 0000000..f868db4 --- /dev/null +++ b/docs/V2_CONTRACT.md @@ -0,0 +1,555 @@ +# Mattermost CLI v2 Contract + +Status: locked for implementation on `feat/go-v2-rewrite` + +This document defines the Go v2 product, safety, storage, compatibility, testing, and release contract. Implementation may refine internal structure, but changing a behavioral invariant in this document requires an explicit contract update and regression coverage. + +## 1. Product identity + +`mm` is a Mattermost CLI for humans and agents. It provides honest, bounded reads and real-time watch output, plus deliberately staged remote mutations. + +The central v2 rule is: + +> No command that merely describes or prepares an operation may mutate Mattermost. Every remote mutation must first become a persisted, inspectable stage and then be applied by a separate command. + +`mm apply @` is the only general remote-mutation boundary. The explicit revision binds apply to what the caller inspected. There are no direct-send, direct-edit, direct-delete, direct-react, or bypass aliases. + +## 2. Cutover and compatibility + +- TypeScript `v1.6.0` at commit `eccfc50` is the behavioral oracle during development. +- Go is developed beside the TypeScript implementation on the rewrite branch. +- The public `mm` binary remains v1 until Go reaches full read, watch, security, formatting, and current-send feature parity and satisfies this contract's expanded messaging surface. +- Public v2 is a clean break. Command output and JSON compatibility with v1 are not required. +- The internal dual implementation is temporary verification scaffolding, not a supported hybrid product. +- At cutover, the TypeScript implementation and Vitest harness are removed. Language-neutral fixtures, scenarios, schemas, and Docker acceptance tests remain. +- The first public Go release is `v2.0.0`. + +## 3. Required read and watch parity + +Go v2 must preserve every user-visible capability and negative guarantee currently covered by v1 for: + +- `whoami` +- `teams` +- `users` +- `channels` +- `dms` +- `group-dms` +- `channel` +- `thread` +- `search` +- `mentions` +- `unread` +- `watch` +- `config` +- `doctor` + +Parity means semantic parity, not a line-for-line port. In particular, v2 must preserve: + +- bounded HTTP requests, pagination, concurrency, retries, and backoff; +- deterministic global selection and ordering; +- versioned opaque cursor validation and stale-boundary recovery; +- tri-state retrieval completeness and fail-closed incomplete-empty behavior; +- bounded thread hydration, missing-root tolerance, and honest partial metadata; +- deleted-content suppression and latest-visible-state fidelity; +- safe permalink construction; +- authenticated WebSocket handshake, heartbeat, bounded reconnect, sequence-gap diagnostics, and no false REST-backfill claims; +- strict stdout/stderr separation, especially JSONL watch output; +- hostile terminal, control, Unicode bidi, Markdown, URL, and remote-value handling; +- secret masking without retaining original secret values; +- exact active Mattermost credential masking even when heuristic redaction is disabled. + +## 4. Go architecture and dependencies + +The implementation targets Go 1.26 or newer and prefers small, explicit packages. + +Approved foundations: + +- CLI: `github.com/spf13/cobra` +- HTTP: standard library `net/http` +- database API: standard library `database/sql` +- SQLite driver: `modernc.org/sqlite` +- WebSocket: `github.com/coder/websocket` +- TOML: `github.com/pelletier/go-toml/v2` + +The Mattermost REST client remains local rather than importing the large Mattermost server SDK. All dependencies require a health and license check before landing. + +Expected package boundaries are configuration, transport, Mattermost API, retrieval, preprocessing, formatting, schema, stage store, mutation planning, mutation execution, and command orchestration. Package boundaries may change without altering this contract. + +## 5. Configuration and local state + +Configuration precedence remains: + +1. CLI flags +2. environment variables +3. TOML configuration +4. hardcoded defaults + +Configuration path: + +- `$XDG_CONFIG_HOME/mattermost-cli/config.toml` when `XDG_CONFIG_HOME` is set and absolute; +- otherwise `~/.config/mattermost-cli/config.toml` on every operating system, including macOS. + +When the selected XDG path differs from the v1 path, migration behavior is mandatory and deterministic: + +- if the selected path exists, it is authoritative; an additional legacy file is ignored with a narrow warning; +- if the selected path is absent and the legacy path exists, v2 reads the legacy file as a read-only fallback and warns with both paths; +- if neither exists, configuration is absent normally. + +v2 never silently moves, rewrites, merges, or deletes the legacy file. `config --init` and all explicit writes target the selected v2 path. + +State path: + +- `$XDG_STATE_HOME/mattermost-cli` when `XDG_STATE_HOME` is set and absolute; +- otherwise `~/.local/state/mattermost-cli` on every operating system, including macOS. + +The state directory is created with mode `0700`. The SQLite database and other private state files are created with mode `0600`. v2 refuses unsafe ownership, symlink, non-regular-file, or permission conditions when they could expose credentials or staged content. Unsupported permission semantics must be diagnosed honestly rather than claimed secure. + +The database uses schema migrations, foreign keys, bounded busy handling, WAL where supported, integrity diagnostics, and best-effort secure deletion. Filesystem snapshots, backups, swap, and privileged local access are explicitly outside the confidentiality guarantee. + +No token, token digest suitable for offline guessing, original secret captured by redaction provenance, remote error body, or shell command is stored in the stage database. Staged outbound bodies may contain caller-supplied heuristic secrets because redaction is display-only; they are protected as plaintext staged content under the retention boundary in section 14. The active Mattermost credential remains forbidden. + +## 6. Machine contracts + +Every structured input, JSON output, and JSONL event is self-identifying with a checked-in schema identifier under `mm/v2`. + +Examples: + +- `mm/v2/stage-request` +- `mm/v2/stage` +- `mm/v2/apply-receipt` +- `mm/v2/error` +- `mm/v2/watch-event` +- command-specific read envelopes under `mm/v2/` + +Checked-in JSON Schemas and golden examples are part of the release contract. Unknown fields may be added only where the schema explicitly permits them. Field removal, meaning changes, or type changes require a new schema identifier. + +`--json`, `--from-json`, and JSONL watch mode imply machine mode. Machine mode never prompts, launches an editor, emits ANSI, or mixes diagnostics into stdout. For a one-shot command that fails before any successful envelope emission, stdout is empty and one schema-valid `mm/v2/error` object is written to stderr. Watch diagnostics are JSONL error/diagnostic envelopes on stderr after any prior events. A low-level partial stream write can make byte-perfect recovery impossible; it is classified honestly and never followed by another object pretending the stream remained valid. Human and machine surfaces use the same underlying operations and validation. + +Exit classes are stable: + +- `0`: completed successfully, including an already-satisfied idempotent intent; +- `2`: invalid invocation or local input; +- `3`: configuration, authentication, authorization, or read failure before a mutation request is dispatched; +- `4`: a dispatched remote mutation was definitively rejected, including 401/403 from that mutation endpoint; +- `5`: remote mutation is partial or unknown and must not be ordinarily retried; +- `6`: local state, revision, claim, attachment-drift, or target-drift conflict; +- `7`: intended remote effect was confirmed but local receipt output or persistence failed; do not retry. + +## 7. Staging grammar + +Human-oriented creation commands follow one grammar: + +```text +mm stage send dm +mm stage send group +mm stage send channel [--team ] +mm stage reply +mm stage post-edit +mm stage post-delete +mm stage react +mm stage unreact +mm stage dm-create +mm stage group-create ... + +mm stage list +mm stage show +mm stage revise +mm stage revise --revive +mm stage cancel +mm stage prune + +mm apply @ +mm apply @ --resume-partial +mm apply @ --force-unknown +``` + +Exact flags and aliases may be added where they do not weaken the grammar. A top-level `send`, `edit`, `delete`, `react`, or conversation-creation command that bypasses staging is forbidden. + +Every stage-creation action supports `--dry-run`. It performs the same read-only resolution, validation, plan construction, and safe preview but does not persist local state and cannot be applied. Structured requests express this as `persist: false`. This is v2's non-persisting replacement for v1 send dry-run. + +Dry-run does not require or read message content, launch an editor, hash attachments, or persist a caller request ID. It previews target resolution and the possible remote-step shape without claiming content validation. This preserves v1's bodyless destination preview. + +Operational inspection surfaces are also required: + +```text +mm store doctor +mm store migrations +mm schema list +mm schema show +mm schema validate +``` + +They are read-only unless a future contract explicitly stages a repair operation. + +Top-level posts may target any exact joined conversation: direct, group direct, public channel, or private channel. Name-based public/private channel resolution is team-aware and fails on ambiguity. Exact channel IDs remain supported. + +## 8. Content input + +Message, reply, and post-edit stages accept exactly one content source: + +- piped stdin, preserved as exact valid UTF-8 bytes; +- `$VISUAL`, then `$EDITOR`, only when stdin is a TTY and human mode is active; +- explicit `--message `, documented as visible to shell history and process inspection. + +Structured agent creation is also supported: + +```text +mm stage --from-json +``` + +It consumes one versioned `mm/v2/stage-request` object from stdin. It is the canonical structured agent input, while human subcommands remain first-class rather than wrappers with weaker behavior. Persisted structured requests require a caller-generated `requestId`. Uniqueness is scoped to the normalized server base URL and authenticated user. An identical replay returns the existing stage and receipt; reuse with different semantic content is a local conflict. Human commands may supply the same behavior through `--request-id`. + +Structured apply is also first-class: + +```text +mm apply --from-json +``` + +It consumes one `mm/v2/apply-request` containing `requestId`, `stageId`, `revision`, expected semantic digest, and exactly one `recoveryMode`: `ordinary`, `resume_partial`, or `force_unknown`. Identical replay returns the existing attempt receipt without dispatch. Reuse of a request ID with different content conflicts. Flags and structured recovery mode cannot be combined. + +Every machine-issued local state mutation, including revise, cancel, revive, and destructive prune, has a versioned request schema and required caller request ID with the same identical-replay/conflicting-reuse semantics. Human subcommands may opt into those semantics with `--request-id`. + +Input methods are mutually exclusive. Empty or whitespace-only content is rejected. UTF-8 failure is fatal. The current Mattermost limits remain locally enforced before any remote mutation: at most 16,383 Unicode code points and 65,535 UTF-8 bytes, unless a verified server capability establishes a lower bound. + +The active Mattermost credential is rejected in outbound text, stored paths, filenames, attachment metadata supplied by the caller, structured mutation fields, and attachment bytes before persistence or remote dispatch. Attachment scanning is exact-byte streaming with boundary-safe chunk handling and is repeated against the apply-time spool. + +## 9. Stage identity and revision binding + +Every stage has: + +- an opaque high-entropy stage ID; +- an immutable creation timestamp; +- a monotonically increasing revision; +- an operation type; +- the complete normalized Mattermost API base URL, including any base path, and authenticated Mattermost user ID; +- canonical destination, participant, post, and channel IDs as applicable; +- a normalized operation plan; +- a digest of every semantically relevant field; +- lifecycle state and timestamps; +- zero or more independently journaled apply attempts. + +Stage creation is online when remote identity or target resolution is required. Resolution is read-only. The exact target and authenticated account are bound at creation. Apply revalidates that same identity and refuses if credentials now represent another user, the complete API base URL changed, a verified stable server identifier changed when available, access changed, or the target's identity/type no longer matches. + +Revising a stage creates a new revision and marks older revisions superseded. Apply requires `@` and transactionally compare-and-swaps that exact current revision plus its semantic digest into an applying claim. Structured apply requests also carry the expected digest. Human `stage show` prints the exact apply reference. A concurrent revise produces a local conflict rather than applying unreviewed content. Concurrent processes cannot both ordinarily apply one revision. + +`stage list` and ordinary receipts omit message bodies and attachment contents. `stage show` is an explicit content-revealing operation and may display the staged body. Human output warns accordingly; machine output includes content only for that explicit command. + +## 10. Attachment binding + +Attachments are path-backed, not copied into SQLite or managed blob storage. + +At staging time v2 records, at minimum: + +- caller-provided path; +- canonical path information needed for later safe reopening; +- filename used remotely; +- byte length; +- detected or explicit media type; +- cryptographic content digest. + +Apply securely reopens and rehashes the file. Missing, inaccessible, non-regular, replaced, symlink-swapped, resized, or digest-mismatched files cause a local conflict before upload. v2 never silently uploads bytes different from the staged revision. + +To close the hash-to-upload race without retaining attachment copies between commands, apply copies each securely opened source into a private `0600` spool file while hashing it. Only a complete spool whose digest and length match the staged revision may be uploaded. Upload reads the spool, not the original path. Thus path-backed staging remains low-storage at rest while dispatched bytes are an immutable snapshot of the reviewed file. + +Spools live under a private per-attempt directory. A spool is deleted after validated success or definitive rejection once the journal commit is durable. It is retained for partial/unknown recovery when it represents bytes that may need exact reuse. On startup, unreferenced spools and spools whose journal proves no dispatch are removed; spools referenced by a stale applying claim are retained while that claim becomes unknown. Explicit prune removes retained spools only under the recovery-destructive rules below. + +Server upload limits are checked before upload when discoverable. Upload and post creation are separate remote substeps and therefore use the compound-operation journal. + +## 11. Supported remote mutation lifecycle + +The first complete v2 supports the Mattermost message lifecycle, not server administration: + +- create top-level posts in all joined conversation types; +- create replies; +- upload and attach files; +- edit the authenticated user's posts; +- delete the authenticated user's posts; +- add and remove the authenticated user's reactions; +- create or resolve direct conversations; +- create group direct conversations from an exact participant set. + +Explicitly excluded from v2.0: + +- public/private channel creation, archival, membership, or moderation; +- team, user, role, permission, bot, webhook, or server administration; +- scheduled server-side sending; +- automatic content generation or autonomous recipient selection. + +Stage creation captures enough current remote state to make later drift visible: + +- edit/delete bind author, post ID, channel ID, update timestamp, and relevant content digest; +- reply binds root/channel identity; +- react/unreact bind post/channel/emoji and current-user reaction state; +- group creation binds a deduplicated canonical participant-ID set; +- channel sends bind channel ID, type, team identity where applicable, and membership/access. + +Apply re-fetches relevant state. Changed, deleted, re-authored, moved, inaccessible, or ambiguously resolved targets fail closed. Already-satisfied reaction state succeeds without a write and is reported as such. + +## 12. Compound operation semantics + +One stage may represent multiple remote API steps required by one reviewed user intent, including conversation creation, file upload, and post creation. + +`stage show` exposes the ordered plan and conditional steps. Apply journals before and after every substep. A later failure never erases known earlier effects. + +Rules: + +- read-only preparation may use bounded retry policy; +- a remote mutation is never automatically replayed after dispatch; +- redirects are rejected for mutation requests; +- timeout, transport failure, malformed success, unexpected identity, and unvalidated success are `unknown`; +- a known response proving rejection is `rejected`; +- known completed early steps followed by rejection are `partial` if externally visible residue remains; +- any unknown substep stops the plan immediately; +- safe known completed results, such as a validated created channel or uploaded file ID, may be reused by an explicit partial resume or uncertainty override; +- uncertain non-idempotent substeps are never silently reused or replayed. + +The CLI never claims distributed exactly-once delivery. Under normal process control it makes at most one dispatch attempt for each substep attempt. The journal records `dispatch_intent` before handing work to the transport, then `response_validated`, `rejected`, or `outcome_unknown` when observation permits. A crash between those records is recovered conservatively and never converted into proof that a request was or was not dispatched. + +## 13. Stage and attempt states + +Stage lifecycle and remote attempt outcome are separate axes. + +Stage lifecycle states: + +- `open`: the current revision may be eligible for apply or explicit recovery; +- `applying`: one process holds the transactional claim for one exact revision; +- `completed`: the intended effect is confirmed or already satisfied; +- `canceled`: local eligibility was explicitly revoked; +- `expired`: TTL policy revoked eligibility; +- `pruned`: sensitive local content was removed while required audit tombstones remain. + +Revision states are `current` or `superseded`. Apply attempt outcomes are `succeeded`, `already_satisfied`, `rejected`, `partial`, or `unknown`. Cancel, expiry, and prune never overwrite or reinterpret an attempt outcome. + +Each apply attempt has its own ID, pending-post ID where applicable, plan snapshot, substep journal, start/end timestamps, and outcome. + +Every stage also has an aggregate recovery requirement derived monotonically from its complete attempt history: + +- `none`: ordinary apply is safe if all other eligibility checks pass; +- `resume_partial`: confirmed reusable effects exist and every remaining effect is proven not applied; +- `force_unknown`: at least one prior effect may have occurred and duplicate risk remains; +- `forbidden`: the stage is completed or its lifecycle no longer permits application. + +`force_unknown` dominates `resume_partial`, which dominates `none`. A later rejected attempt never erases uncertainty from an earlier attempt. Only confirmed completion or explicit lifecycle closure changes recovery to `forbidden`. + +Normal transitions are: + +| Event | Lifecycle after event | Attempt outcome | Recovery requirement | +| --- | --- | --- | --- | +| create or safe revise | `open` | unchanged | inherited from complete history | +| ordinary claim | `applying` | pending | unchanged | +| local failure proven before dispatch | `open` | no attempt or `rejected` | prior requirement | +| all intended effects validated | `completed` | `succeeded` or `already_satisfied` | `forbidden` | +| definitive rejection with no externally visible residue | `open` | `rejected` | prior requirement | +| confirmed residue, all remaining effects proven not applied | `open` | `partial` | max(prior, `resume_partial`) | +| any uncertain effect | `open` | `unknown` or uncertainty-bearing `partial` | `force_unknown` | +| cancel eligible open stage | `canceled` | unchanged | `forbidden` | +| expire eligible inactive stage | `expired` | unchanged | `forbidden` | +| prune eligible inactive stage | `pruned` | unchanged | `forbidden` | + +An interrupted or stale `applying` claim creates an `unknown` attempt outcome unless the journal proves no remote mutation dispatch was handed to the transport. The stage returns to `open` with aggregate `force_unknown`. Lease expiry never makes a mutation ordinarily replayable. + +Revise is refused while `applying` or after `completed`, `canceled`, or `pruned`. An expired stage may be revised only with `stage revise --revive`, which atomically returns it to `open` with a new revision. Revision never clears recovery history: revising after partial or unknown carries `resume_partial` or `force_unknown` to the new revision. Revised content is therefore still gated by the unresolved risk of prior effects. + +Ordinary `mm apply @` requires the exact current revision, lifecycle `open`, and recovery requirement `none`. + +`mm apply @ --resume-partial` requires recovery `resume_partial`. It preserves the prior attempt, assigns fresh idempotency/pending IDs to new substep attempts where applicable, revalidates reusable effects, and continues only effects proven not applied. A definitively rejected request is proven not applied and may be retried this way. Resume is forbidden when any attempt in the stage's history remains uncertain. + +`mm apply @ --force-unknown` requires aggregate recovery `force_unknown`. It: + +- emits an unmistakable duplicate/side-effect warning in human mode; +- requires the flag in machine mode without prompting; +- preserves all earlier attempts immutably; +- creates a new attempt and new pending-post ID where applicable; +- reuses only effects that were previously validated and remain safe; +- records that the caller knowingly accepted duplicate risk. + +## 14. Retention and confidentiality + +Staged bodies are plaintext SQLite data protected by local filesystem permissions. This is a deliberate portability choice, not encryption at rest. + +After confirmed success: + +- message/edit body content is cleared from active stage storage; +- local attachment paths and caller-only sensitive composition metadata are cleared when no longer needed; +- destination IDs, digests, timestamps, attempt journal, and narrow Mattermost receipts remain for audit; +- confirmed remote content is never copied back into the receipt record. + +Rejected, partial, unknown, and unapplied stages retain the minimum content required for inspection and deliberate recovery until eligible cleanup. + +TTL is configurable. Its default is `0`, meaning stages never expire automatically. Enabling TTL is explicit. Age is measured from the latest revision or attempt activity. Automatic expiry applies only to inactive open stages with recovery `none`; it never races an applying claim or expires a recovery-eligible stage. Expiry changes lifecycle eligibility but preserves prior attempt outcomes and does not pretend secure erasure. + +Cancel is refused while applying. On an open stage it revokes future apply without rewriting history. `stage prune` defaults to completed, canceled, and expired stages older than an explicit or configured age. It refuses applying and recovery-eligible stages. Removing content or retained spools from a `resume_partial` or `force_unknown` stage requires an exact stage reference plus `--abandon-recovery`; this makes the stage `pruned`, recovery `forbidden`, and preserves an audit tombstone stating that recovery material was deliberately destroyed. + +## 15. Receipts, errors, and output failure + +Mutation receipts are narrow local projections. They never include full message bodies, attachment bytes, raw Mattermost objects, arbitrary response fields, remote response bodies, credentials, or original secret values. + +Receipts include only fields needed to establish: + +- schema and operation type; +- stage ID, revision, and attempt ID; +- destination and authenticated identity in sanitized narrow form; +- each planned substep's known state; +- canonical post/channel/file identifiers when validated; +- server creation/update timestamps when validated; +- overall `succeeded`, `rejected`, `partial`, `unknown`, or `already_satisfied` outcome; +- stable `recovery` value: `none`, `resume_partial`, `force_unknown`, or `forbidden`. + +`stage list`, `stage show`, apply receipts, and structured errors expose the same recovery enum. Human output renders the exact next safe command when recovery is available. + +If the intended remote effect is confirmed but database receipt persistence or stdout emission fails, v2 reports a distinct confirmed-effect failure and instructs the caller not to retry. It must be testable when stdout closes or disk persistence fails. + +Remote error bodies, reason phrases, headers not explicitly allowlisted, and unsanitized caller-controlled values are never reflected in errors. Explicit content-revealing commands such as `stage show` and narrow sanitized destination receipts are the documented exceptions to general output minimization. + +## 16. Security invariants + +The v1 security boundary survives the rewrite: + +- HTTPS is required except explicitly allowed loopback HTTP for local testing. +- URLs with credentials, query strings, fragments, unsafe schemes, or ambiguous normalization fail closed. +- User-controlled path components are encoded. +- Mutation requests do not follow redirects. +- Active credentials are ownership-scoped and fully masked everywhere. +- Heuristic secret redaction is display-only and may be disabled; credential masking and terminal/control sanitization may not. +- Original secret values and unredacted originals never appear in redaction provenance. +- Canonical unsanitized IDs drive identity, ordering, grouping, and deduplication; sanitized copies are presentation-only. +- Go RE2 incompatibilities are implemented with candidate matching plus explicit boundary validation, not weaker regex substitutions. +- Byte, rune, and structured-redaction offsets are defined explicitly and tested across non-ASCII text. +- Files, config, database, editor temp handling, and subprocess invocation resist symlink and permission attacks within the documented local-user threat model. + +## 17. Test and conformance contract + +The rewrite is accepted by evidence, not source resemblance. + +### 17.1 Frozen oracle + +- Tag `v1.6.0` and commit `eccfc50` are immutable oracle inputs. +- Current evidence baseline: 33 Vitest files, 517 tests, shared-state `--no-isolate` pass, Biome/typecheck/build pass, dependency audit pass, exact npm tarball smoke, and disposable Mattermost 11.8.3 E2E pass. +- The existing tests are inventoried into a language-neutral parity matrix before removal. +- The matrix explicitly disposes every current command, flag, environment variable, TOML key, output mode, exit behavior, warning, agent-detected default, and release smoke. It must include `mention_names`, `MM_REDACT`, relative-time agent detection, `--dry-run`, no-color format selection, version checks, and config permission behavior even when they lack broad E2E coverage. + +### 17.2 Language-neutral fake server + +A fake Mattermost server and scenario manifests capture: + +- exact request method, URL, headers, body bytes, count, and order; +- response, disconnect, timeout, malformed payload, redirect, and rate-limit behavior; +- stdout bytes, stderr bytes, exit class, and machine-schema validity; +- resulting server and local-store state. + +Where v1 and v2 semantics intentionally match, both binaries run the same scenario. Where v2 intentionally changes output, scenarios compare normalized semantic results and enforce the new schema. + +### 17.3 Go test layers + +Required layers: + +- table-driven unit tests for validation, formatting, preprocessing, state transitions, planners, and receipts; +- golden tests for every human format and checked-in machine schema example; +- fuzz tests for URL parsing, cursors, UTF-8/input bounds, JSON/TOML decoding, secret detection, sanitization, pagination payloads, and stage requests; +- HTTP contract tests for all Mattermost endpoints and mutation no-replay behavior; +- SQLite migration, permissions, integrity, multi-process claim, busy, and recovery tests; +- `go test -race ./...` for concurrency-sensitive packages and the full suite where practical; +- subprocess tests for signals, editor behavior, stdin, closed stdout/stderr, exit classes, and exact byte preservation; +- fault-injection tests for crash-before-dispatch, crash-after-journal, crash-after-server-acceptance, disk-full, failed commit, stale claim, changed attachment, changed destination, and failed receipt output; +- transition and race tests for show-revise-apply, revise/cancel/prune versus applying, known-partial resume, uncertainty-bearing expiry/cancel, and stale revisions; +- recovery-history tests for unknown then forced rejection, revise after partial/unknown, definitively rejected suffix resume, and the impossibility of clearing aggregate uncertainty through a later outcome; +- adversarial attachment tests for in-place writes during spool creation, inode/path replacement, truncation, symlink swaps, and active-credential bytes; +- spool recovery tests for pre-dispatch crash cleanup, stale applying retention, successful cleanup, and explicit recovery abandonment; +- lost-output idempotency tests for stage creation and revision as well as apply receipts; +- security fixture parity for every v1 secret pattern and hostile presentation input; +- artifact tests against the exact released archives and npm platform packages. + +### 17.4 Disposable Mattermost acceptance + +Docker E2E remains isolated from real users and creates unique fake accounts and conversations. It must prove at least: + +- exact short and near-limit long Markdown storage/readback; +- final-newline and Unicode fidelity; +- stage creation causes no Mattermost mutation; +- apply creates exactly the intended post under normal success; +- DM and group-DM creation and exact participant binding; +- public/private channel sends with team-aware resolution; +- replies, edits, deletes, reactions, unreact, files, and compound plans; +- unknown-outcome behavior without automatic replay; +- explicit `--force-unknown` audit behavior; +- concurrent apply claim exclusion; +- body clearing and receipt retention after confirmed success; +- read, search, thread, cursor, and watch behavior against a real server. + +No configured workplace server or real account is used by automated tests. + +### 17.5 Release gates + +Before v2 cutover: + +- all unit, integration, conformance, fuzz-seed, race, fault, and Docker E2E gates pass; +- `go vet`, `staticcheck`, `govulncheck`, formatting, module verification, and dependency license checks pass; +- darwin arm64/amd64 and linux arm64/amd64 artifacts build reproducibly enough for checksum verification; +- each exact archive installs and runs version, help, doctor, `store doctor`, and schema smokes; +- Homebrew installation and upgrade are tested; +- npm optional-platform packages install the correct binary and verify checksum/version; +- the working tree and generated schema artifacts are clean. + +## 18. Implementation sequence + +Implementation proceeds in reviewable conventional commits: + +1. freeze and inventory the v1 behavioral oracle; +2. add Go module, command skeleton, build metadata, and baseline gates; +3. add machine schemas and the language-neutral scenario harness; +4. port config, URL policy, transport, identity, doctor, sanitization, and redaction; +5. port read and formatting surfaces in vertical slices; +6. port cursors, pagination, thread hydration, and completeness semantics; +7. port watch and reconnect/gap behavior; +8. add SQLite migrations and offline stage management; +9. add target-bound stage creation and revision handling; +10. add apply claiming, single-step mutations, receipts, and unknown outcomes; +11. add compound conversations/files/posts and recovery journal; +12. add full message lifecycle operations; +13. complete differential, race, fault, and Docker acceptance; +14. add GitHub, Homebrew, install-script, `go install`, and npm-shim distribution; +15. remove the TypeScript implementation after every gate passes; +16. cut `v2.0.0`, verify live artifacts, and synchronize release receipts. + +## 19. Distribution and migration + +Primary Go v2 distribution: + +- GitHub release archives for darwin arm64/amd64 and linux arm64/amd64; +- checksums and verifiable build provenance; +- Homebrew using the established patterns in Arda's current Go projects; +- install script with checksum verification; +- `go install` for source-based installation. + +The existing unscoped npm package remains an upgrade path for prior npm users. Its v2 package becomes a small launcher backed by platform-specific optional packages containing the exact Go release binaries. It must not fetch and execute an unverified binary during `postinstall`. npm installation continuity does not imply v1 command or schema compatibility. + +Windows is not a supported v2.0 release target. The code should avoid gratuitous portability barriers, but no Windows guarantee exists until its filesystem, signal, terminal, SQLite, and artifact behavior receive native testing. + +## 20. Rollback + +- Before public cutover, rollback is simply continued use of tagged TypeScript `v1.6.0`. +- v2 never mutates or deletes v1 configuration during migration. +- The initial v2 database is new state and has no automatic downgrade promise. +- Release documentation preserves explicit v1.6.0 reinstall instructions during the v2 migration window. +- A failed v2 release is fixed forward or withdrawn without rewriting existing tags or release assets. + +## 21. Interview decisions captured + +- full public feature parity before cutover; +- no behavioral backwards-compatibility requirement; +- universal `stage` / `apply` mutation grammar; +- target binding during stage creation; +- private plaintext SQLite storage; +- confirmed-send body clearing with receipt retention; +- force of unknown outcomes only through `--force-unknown`; +- full message lifecycle, excluding administration; +- all joined conversations as top-level post targets; +- path-plus-digest attachment binding; +- explicit compound plans; +- XDG state path with no macOS application-support fallback; +- stdin, editor, and explicit `--message` human input; +- additional structured JSON agent input; +- versioned schemas for every machine surface; +- darwin/linux arm64/amd64 release matrix; +- Go-native release plus Homebrew and npm migration shim; +- configurable TTL with never-expire as the default. From c633922622657e4da74e67822f1c222830528602 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 02:38:53 +0300 Subject: [PATCH 002/119] docs: inventory v1 parity gates --- docs/V1_PARITY_MATRIX.md | 208 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 docs/V1_PARITY_MATRIX.md diff --git a/docs/V1_PARITY_MATRIX.md b/docs/V1_PARITY_MATRIX.md new file mode 100644 index 0000000..0944f69 --- /dev/null +++ b/docs/V1_PARITY_MATRIX.md @@ -0,0 +1,208 @@ +# v1.6.0 to Go v2 Parity Matrix + +Oracle: tag `v1.6.0`, commit `eccfc5029cc1a51514873b5cd5d7a4d3ded8d5cd` + +This is the removal gate for the TypeScript implementation. A row may become `verified`, `intentionally_changed`, or `removed_by_contract`. Anything else blocks cutover. + +## Status vocabulary + +- `oracle`: behavior exists in v1 and still needs Go evidence +- `contracted`: new v2 behavior is locked but not implemented +- `scaffolded`: Go surface exists but parity evidence is incomplete +- `verified`: Go unit/conformance/E2E evidence satisfies the row +- `intentionally_changed`: v2 contract defines replacement behavior and tests prove it +- `removed_by_contract`: behavior is explicitly forbidden or superseded by v2 + +## Executable and global behavior + +| Surface | v1 behavior | v2 disposition | Status | +| --- | --- | --- | --- | +| binary | `mm` | same public name at cutover | oracle | +| `--help` | Commander help | Cobra help, artifact-smoked | oracle | +| `--version` | package version | Go build metadata, artifact-smoked | oracle | +| `-t`, `--token` | credential CLI override | preserve | oracle | +| `--url` | server URL CLI override | preserve | oracle | +| `--json` | command JSON; watch JSONL | replace with schema-identified `mm/v2` JSON/JSONL | intentionally_changed | +| `--no-color` | disables ANSI, not output-format selection | preserve | oracle | +| `-r`, `--relative` | relative timestamps | preserve | oracle | +| `--no-relative` | absolute timestamps | preserve | oracle | +| agent relative default | `is-ai-agent` enables relative output | preserve semantically with Go detection | oracle | +| `--redact` | enable heuristic redaction | preserve | oracle | +| `--no-redact` | disable heuristic redaction, never active-token masking | preserve | oracle | +| `--threads` | hydrate complete visible threads | preserve | oracle | +| `--no-threads` | selected seeds only except `thread` | preserve | oracle | +| numeric validation | canonical positive safe integer only | preserve bounded canonical integer validation | oracle | +| duration validation | `^\d+[hdwm]$` | preserve | oracle | + +## Configuration + +| Surface | v1 behavior | v2 disposition | Status | +| --- | --- | --- | --- | +| default path | `~/.config/mattermost-cli/config.toml` | preserve on every OS | oracle | +| XDG path | ignored | use absolute `$XDG_CONFIG_HOME`, mandatory read-only v1 fallback | intentionally_changed | +| `url` | TOML server URL | preserve | oracle | +| `token` | TOML PAT | preserve | oracle | +| `redact` | TOML default | preserve | oracle | +| `mention_names` | trimmed non-empty string array | preserve | oracle | +| `MM_URL` | URL env override | preserve | oracle | +| `MM_TOKEN` | token env override | preserve | oracle | +| `MM_REDACT` | `false` disables; other defined values enable | preserve | oracle | +| precedence | CLI, env, file, defaults | preserve | oracle | +| init | non-overwriting, mode `0600` | preserve at selected v2 path | oracle | +| permissions | diagnose group/other access; token exposure can be fatal | preserve/fail closed | oracle | +| state path | none | XDG state with `~/.local/state` fallback | intentionally_changed | + +## Read, diagnostic, and watch commands + +| Command | Flags/defaults | Required v2 behavior | Status | +| --- | --- | --- | --- | +| `doctor` | global flags | same read-only readiness checks; `mm/v2/doctor` | oracle | +| `config` | `--path`, `--init` | preserve plus deterministic XDG migration warning | oracle | +| `whoami` | global flags | narrow validated identity | oracle | +| `teams` | global flags | validated deterministic teams | oracle | +| `users [query]` | `--team`, `-l`/`--limit 20` | preserve exact directory semantics | oracle | +| `channels` | `--type all`; `dm/public/private/group/all` | preserve account-wide dedupe/team identity | oracle | +| `dms` | repeated `-u`/`--user`; `-l`/`--limit 50`; `-s`/`--since 7d`; `-c`/`--channel`; `--cursor` | preserve exact D-channel and aggregate semantics | oracle | +| `group-dms` | `-l`/`--limit 50`; `-s`/`--since 7d`; `-c`/`--channel`; `--cursor` | preserve exact G-channel and aggregate semantics | oracle | +| `channel ` | `--team`; `-l`/`--limit 50`; `-s`/`--since 7d`; `--cursor` | preserve team-aware resolution | oracle | +| `thread ` | always hydrate | preserve | oracle | +| `search ` | `--team`; `-l`/`--limit 50` | preserve bounded search/completeness | oracle | +| `mentions` | `--team`; `-l`/`--limit 50`; optional `-s`/`--since`; `--channel` | preserve aliases and resolution | oracle | +| `unread` | `--team`; `--peek` | preserve metrics/sorting/fail-closed empty | oracle | +| `watch [channel]` | `--team`; `--dm` | preserve auth, heartbeat, reconnect, gap diagnostics | oracle | + +Bare `mm config` must also preserve the human status surface: selected path, file existence, URL configured, token configured, and permission warning without exposing values. + +## v1 write surface and v2 replacement + +| v1 surface | v1 behavior | v2 disposition | Status | +| --- | --- | --- | --- | +| `send dm ` | stdin then immediate remote write | forbidden; replace with `stage send dm`, then revision-bound `apply` | removed_by_contract | +| `send group ` | stdin then immediate remote write | forbidden; replace with `stage send group`, then revision-bound `apply` | removed_by_contract | +| `--dry-run` | bodyless read-only destination preview | `stage ... --dry-run`, unpersisted and bodyless | intentionally_changed | +| message stdin | exact UTF-8, non-TTY | preserve plus editor/`--message` human options | intentionally_changed | +| bounds | 16,383 code points and 65,535 bytes | preserve before persistence/dispatch | oracle | +| DM creation | conditional direct-channel creation then post | explicit compound staged plan | intentionally_changed | +| group target | exact existing type-G ID | preserve plus explicit group-create stages | intentionally_changed | +| post attempt | one client dispatch, no automatic replay | preserve per attempt; explicit recovery modes only | oracle | +| receipt | narrow, no message body | preserve under `mm/v2/apply-receipt` | oracle | + +## New v2 mutation surface + +| Surface | Contract gate | Status | +| --- | --- | --- | +| `stage send dm/group/channel` | exact online binding, no remote mutation | contracted | +| `stage reply` | bind exact root/channel | contracted | +| `stage post-edit` | own-post/state binding | contracted | +| `stage post-delete` | own-post/state binding | contracted | +| `stage react/unreact` | exact post/emoji/current-user state | contracted | +| `stage dm-create/group-create` | exact canonical participant set | contracted | +| attachments | path + digest, private apply-time spool | contracted | +| `stage list/show/revise/cancel/prune` | revision/history/privacy contract | contracted | +| `apply @` | CAS claim and ordinary recovery `none` | contracted | +| `--resume-partial` | only proven-not-applied suffix | contracted | +| `--force-unknown` | explicit duplicate-risk attempt | contracted | +| structured requests | versioned JSON plus idempotency key | contracted | +| SQLite store | migrations, WAL, permissions, recovery journal | contracted | + +## Output and presentation + +| Behavior | v1 oracle | v2 disposition | Status | +| --- | --- | --- | --- | +| TTY human output | pretty/color-capable | preserve semantic content | oracle | +| non-TTY human output | Markdown independent of color | preserve | oracle | +| explicit JSON | unversioned command shape | schema-identified v2 envelope | intentionally_changed | +| watch JSON | JSONL stdout | schema-identified JSONL stdout | intentionally_changed | +| diagnostics | stderr | preserve; schema errors in machine mode | oracle | +| empty messages | `No messages found.` | preserve human meaning | oracle | +| disabled-redaction warning | stderr | preserve | oracle | +| latest visible post state | edits/deletes/system/pin/files/attachments/reactions | preserve | oracle | +| Markdown safety | escape remote structure and unsafe links | preserve | oracle | +| timestamps | absolute/relative and edit metadata | preserve | oracle | +| retrieval metadata | selected/visible counts and completeness | preserve under v2 schema | oracle | + +## Critical negative guarantees + +Every row requires a named Go regression test and, where applicable, a language-neutral scenario. + +| Guarantee | Status | +| --- | --- | +| active Mattermost credential is never emitted, even with `--no-redact` | oracle | +| outbound staged content containing the active credential is rejected before persistence/network | oracle | +| attachment paths, metadata, and bytes containing the active credential are rejected | contracted | +| outbound bodies never appear in ordinary receipts/errors | oracle | +| exact username/current-user/channel-type/participant validation completes before post dispatch | oracle | +| remote response bodies/reason phrases/parser details are never reflected | oracle | +| mutation redirects are rejected | oracle | +| uncertain mutation requests are never automatically replayed | oracle | +| confirmed effect plus local receipt/output failure says do not retry | oracle | +| reads retry only bounded safe failures | oracle | +| HTTP is allowed only for loopback; unsafe URL components fail closed | oracle | +| complete normalized base path is preserved and stage-bound | oracle | +| read commands never create DMs or perform POST fallback | oracle | +| dry-run performs no local persistence or remote mutation | oracle | +| malformed identity/team/channel/retrieval payloads fail closed | oracle | +| unknown completeness never becomes confirmed empty/exhausted | oracle | +| malformed/context-mismatched cursors fail before fetching | oracle | +| explicit cursor plus `--since` conflicts | oracle | +| deleted posts never leak stale content | oracle | +| terminal/control/bidi hazards are visible or removed safely | oracle | +| overlapping redaction matches never re-append plaintext | oracle | +| truncated private-key blocks fail closed without plaintext leakage | oracle | +| invalid UTF-8, whitespace-only, oversized, and unintended TTY input fail before work | oracle | +| watch validates events/sequences and bounds reconnect | oracle | +| watch auth failure stops reconnect and releases credentials/timers | oracle | +| thread hydration concurrency is at most four | oracle | +| `--no-threads` avoids hydration except explicit thread | oracle | +| stage apply is exact-revision CAS-bound | contracted | +| concurrent apply cannot double-claim | contracted | +| prior uncertainty cannot be erased by rejection or revision | contracted | +| stale applying claim cannot become ordinary replay | contracted | +| path attachment mutation cannot change uploaded bytes | contracted | +| lifecycle cleanup cannot erase attempt truth | contracted | +| Docker harness refuses non-loopback and always tears down | oracle | + +## Test-file disposition + +| v1 domain | Vitest files | Go destination | +| --- | --- | --- | +| API | `tests/api/{channels,client,messages,paths,posts,retrieval,url,websocket}.test.ts` | transport/API/retrieval/websocket unit + conformance | +| CLI | `tests/{channels-type-validation,cli-empty-reads,cli-failure-propagation,cli-metadata,cli-output,cli-retrieval,config-doctor,cursor-cli,cursor-commander,cursor,group-dms,identity-teams,input,send-commander,send,users,validation}.test.ts` | command subprocess + package tests | +| preprocessing | `tests/preprocessing/{post,sanitize,secrets}.test.ts` | preprocessing fixture parity + fuzz | +| formatters | `tests/formatters/{headers,watch}.test.ts` | golden human/schema tests | +| utilities | `tests/utils/{date,threading,unread}.test.ts` | Go unit tests | +| E2E | `tests/e2e/send-live.e2e.ts` | parameterized language-neutral Docker E2E, then Go-native runner | + +The E2E disposition specifically preserves verbatim short Markdown DM and near-limit long Markdown group storage/readback, final-newline and Unicode fidelity, and exactly one resulting post. + +Removal requires every v1 test file to link to one or more verified Go tests/scenarios in this table or a finer generated inventory. + +## Build, CI, and release disposition + +| v1 gate | Go v2 replacement | Status | +| --- | --- | --- | +| Biome check | `gofmt`/`goimports` plus static analysis | oracle | +| TypeScript typecheck | Go compile and `go vet` | oracle | +| 517 Vitest tests | Go unit/conformance/fault/race/E2E matrix | oracle | +| `bun audit` | `govulncheck` plus module/license checks | oracle | +| version invariant | Go build metadata, tag, schemas, npm shim invariant | oracle | +| Node bundle smoke | exact native archive smoke | oracle | +| exact npm tarball | exact launcher + platform package smoke | intentionally_changed | +| npm global install | npm migration-shim `mm` smoke | intentionally_changed | +| OIDC npm provenance | preserve for shim/platform packages | oracle | +| Docker Mattermost 11.8.3 | preserve and expand full lifecycle coverage | oracle | +| `RELEASE_TAG=v` | preserve tag/version gate | oracle | +| release exact-tag checkout | preserve | oracle | +| already-published guard | preserve across native/npm release surfaces | oracle | + +The v1 package receipt is an exact four-file allowlist: `LICENSE`, `README.md`, `dist/index.js`, and `package.json`. During migration, `prepack` still gates build plus version invariants and `prepublishOnly` still gates the full verification suite until the native/npm release pipeline replaces them with equivalent exact-artifact gates. + +## Cutover proof + +TypeScript removal is allowed only when: + +1. no row remains `oracle`, `contracted`, or `scaffolded`; +2. every intentional change cites `docs/V2_CONTRACT.md` and passing v2 evidence; +3. every Vitest file has a recorded Go/conformance disposition; +4. the full Go gate, race gate, fault suite, disposable Mattermost E2E, archive smokes, Homebrew smoke, and npm migration smoke pass; +5. release artifacts are generated from the reviewed commit and their schemas/checksums are clean. From 7b7f30436c9b7213271b1df0ef8e11b121c1d24c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 02:41:27 +0300 Subject: [PATCH 003/119] build: scaffold Go v2 command --- .gitignore | 1 + cmd/mm/main.go | 12 +++++++++ conformance/README.md | 7 +++++ go.mod | 10 +++++++ go.sum | 10 +++++++ internal/buildinfo/buildinfo.go | 6 +++++ internal/cli/root.go | 48 +++++++++++++++++++++++++++++++++ internal/cli/root_test.go | 45 +++++++++++++++++++++++++++++++ justfile | 30 +++++++++++++++++++++ 9 files changed, 169 insertions(+) create mode 100644 cmd/mm/main.go create mode 100644 conformance/README.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/buildinfo/buildinfo.go create mode 100644 internal/cli/root.go create mode 100644 internal/cli/root_test.go create mode 100644 justfile diff --git a/.gitignore b/.gitignore index c4ebbfd..84222e9 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ node_modules # output out dist +bin *.tgz # code coverage diff --git a/cmd/mm/main.go b/cmd/mm/main.go new file mode 100644 index 0000000..32f6cad --- /dev/null +++ b/cmd/mm/main.go @@ -0,0 +1,12 @@ +package main + +import ( + "context" + "os" + + "github.com/ardasevinc/mattermost-cli/internal/cli" +) + +func main() { + os.Exit(cli.Execute(context.Background(), os.Args[1:], os.Stdin, os.Stdout, os.Stderr)) +} diff --git a/conformance/README.md b/conformance/README.md new file mode 100644 index 0000000..93c198e --- /dev/null +++ b/conformance/README.md @@ -0,0 +1,7 @@ +# Conformance harness + +This directory owns language-neutral Mattermost scenarios used to compare the frozen TypeScript v1.6.0 oracle with Go v2. + +Scenarios will describe fake-server behavior, expected request order and bytes, subprocess stdout/stderr, exit classes, and resulting server/local state. Go tests execute the scenarios against a selected binary. Fixtures remain after the TypeScript implementation is removed. + +The scenario schema and runner land before feature ports. See `docs/V2_CONTRACT.md` and `docs/V1_PARITY_MATRIX.md` for the acceptance boundary. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..2612d55 --- /dev/null +++ b/go.mod @@ -0,0 +1,10 @@ +module github.com/ardasevinc/mattermost-cli + +go 1.26.5 + +require github.com/spf13/cobra v1.10.2 + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.9 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..a6ee3e0 --- /dev/null +++ b/go.sum @@ -0,0 +1,10 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/buildinfo/buildinfo.go b/internal/buildinfo/buildinfo.go new file mode 100644 index 0000000..d1f2b01 --- /dev/null +++ b/internal/buildinfo/buildinfo.go @@ -0,0 +1,6 @@ +package buildinfo + +var ( + Version = "2.0.0-dev" + Commit = "dev" +) diff --git a/internal/cli/root.go b/internal/cli/root.go new file mode 100644 index 0000000..b955728 --- /dev/null +++ b/internal/cli/root.go @@ -0,0 +1,48 @@ +package cli + +import ( + "context" + "fmt" + "io" + + "github.com/spf13/cobra" + + "github.com/ardasevinc/mattermost-cli/internal/buildinfo" +) + +type streams struct { + in io.Reader + out io.Writer + err io.Writer +} + +func Execute(ctx context.Context, args []string, in io.Reader, out, errOut io.Writer) int { + cmd := newRoot(streams{in: in, out: out, err: errOut}) + cmd.SetArgs(args) + if err := cmd.ExecuteContext(ctx); err != nil { + _, _ = fmt.Fprintf(errOut, "error: %s\n", err) + return 2 + } + return 0 +} + +func newRoot(s streams) *cobra.Command { + cmd := &cobra.Command{ + Use: "mm", + Short: "Mattermost CLI for agents and humans", + SilenceUsage: true, + SilenceErrors: true, + Version: buildinfo.Version + " (" + buildinfo.Commit + ")", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return cmd.Help() + }, + CompletionOptions: cobra.CompletionOptions{ + DisableDefaultCmd: true, + }, + } + cmd.SetIn(s.in) + cmd.SetOut(s.out) + cmd.SetErr(s.err) + return cmd +} diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go new file mode 100644 index 0000000..1c754e2 --- /dev/null +++ b/internal/cli/root_test.go @@ -0,0 +1,45 @@ +package cli + +import ( + "bytes" + "context" + "strings" + "testing" +) + +func TestExecuteVersion(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + code := Execute(context.Background(), []string{"--version"}, strings.NewReader(""), &stdout, &stderr) + + if code != 0 { + t.Fatalf("exit code = %d, want 0", code) + } + if got, want := stdout.String(), "mm version 2.0.0-dev (dev)\n"; got != want { + t.Fatalf("stdout = %q, want %q", got, want) + } + if stderr.Len() != 0 { + t.Fatalf("stderr = %q, want empty", stderr.String()) + } +} + +func TestExecuteRejectsUnknownCommandWithoutUsage(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + code := Execute(context.Background(), []string{"nope"}, strings.NewReader(""), &stdout, &stderr) + + if code != 2 { + t.Fatalf("exit code = %d, want 2", code) + } + if stdout.Len() != 0 { + t.Fatalf("stdout = %q, want empty", stdout.String()) + } + if !strings.Contains(stderr.String(), "unknown command") { + t.Fatalf("stderr = %q, want unknown-command error", stderr.String()) + } + if strings.Contains(stderr.String(), "Usage:") { + t.Fatalf("stderr = %q, must not contain usage", stderr.String()) + } +} diff --git a/justfile b/justfile new file mode 100644 index 0000000..cdf7129 --- /dev/null +++ b/justfile @@ -0,0 +1,30 @@ +set shell := ["zsh", "-eu", "-o", "pipefail", "-c"] + +default: + @just --list + +go-format-check: + @unformatted="$(gofmt -l cmd internal)"; if [[ -n "$unformatted" ]]; then print -r -- "$unformatted"; exit 1; fi + +go-test: + go test ./... + +go-race: + go test -race ./... + +go-vet: + go vet ./... + +go-modules: + go mod verify + +go-build: + go build -o "${TMPDIR:-/tmp}/mattermost-cli-mm" ./cmd/mm + +go-gate: go-format-check go-test go-race go-vet go-modules go-build + git diff --check + +legacy-gate: + bun run verify + +gate: go-gate legacy-gate From 7686dac85da12567dc01c61d66ec9fdb07a0ccb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 03:08:54 +0300 Subject: [PATCH 004/119] test: add cross-runtime conformance harness --- cmd/conformance/main.go | 42 ++++++ conformance/README.md | 2 +- conformance/scenario.schema.json | 73 ++++++++++ conformance/scenarios/v1/whoami.json | 40 ++++++ internal/conformance/buffer.go | 36 +++++ internal/conformance/process_other.go | 9 ++ internal/conformance/process_unix.go | 32 +++++ internal/conformance/process_unix_test.go | 105 ++++++++++++++ internal/conformance/runner.go | 160 ++++++++++++++++++++++ internal/conformance/runner_test.go | 152 ++++++++++++++++++++ internal/conformance/scenario.go | 111 +++++++++++++++ internal/conformance/scenario_test.go | 59 ++++++++ internal/conformance/server.go | 135 ++++++++++++++++++ justfile | 6 +- 14 files changed, 960 insertions(+), 2 deletions(-) create mode 100644 cmd/conformance/main.go create mode 100644 conformance/scenario.schema.json create mode 100644 conformance/scenarios/v1/whoami.json create mode 100644 internal/conformance/buffer.go create mode 100644 internal/conformance/process_other.go create mode 100644 internal/conformance/process_unix.go create mode 100644 internal/conformance/process_unix_test.go create mode 100644 internal/conformance/runner.go create mode 100644 internal/conformance/runner_test.go create mode 100644 internal/conformance/scenario.go create mode 100644 internal/conformance/scenario_test.go create mode 100644 internal/conformance/server.go diff --git a/cmd/conformance/main.go b/cmd/conformance/main.go new file mode 100644 index 0000000..77969bd --- /dev/null +++ b/cmd/conformance/main.go @@ -0,0 +1,42 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/ardasevinc/mattermost-cli/internal/conformance" +) + +func main() { + flags := flag.NewFlagSet("conformance", flag.ContinueOnError) + flags.SetOutput(os.Stderr) + scenarioPath := flags.String("scenario", "", "path to a conformance scenario") + cwd := flags.String("cwd", "", "working directory for the command under test") + if err := flags.Parse(os.Args[1:]); err != nil { + os.Exit(2) + } + command := flags.Args() + if *scenarioPath == "" || len(command) == 0 { + _, _ = fmt.Fprintln(os.Stderr, "usage: conformance --scenario FILE [--cwd DIR] -- COMMAND [PREFIX_ARGS...]") + os.Exit(2) + } + scenario, err := conformance.Load(*scenarioPath) + if err == nil { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + err = conformance.Run(ctx, conformance.Command{ + Path: command[0], + PrefixArgs: command[1:], + Dir: *cwd, + }, scenario) + } + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, "conformance:", err) + os.Exit(1) + } + _, _ = fmt.Fprintln(os.Stdout, "passed:", scenario.Name) +} diff --git a/conformance/README.md b/conformance/README.md index 93c198e..c19cca7 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -2,6 +2,6 @@ This directory owns language-neutral Mattermost scenarios used to compare the frozen TypeScript v1.6.0 oracle with Go v2. -Scenarios will describe fake-server behavior, expected request order and bytes, subprocess stdout/stderr, exit classes, and resulting server/local state. Go tests execute the scenarios against a selected binary. Fixtures remain after the TypeScript implementation is removed. +Scenarios describe fake-server behavior, expected request order and bytes, subprocess stdout/stderr, and exit classes. `scenario.schema.json` is the language-neutral manifest contract. The Go runner executes scenarios against a selected binary. Fixtures remain after the TypeScript implementation is removed. The scenario schema and runner land before feature ports. See `docs/V2_CONTRACT.md` and `docs/V1_PARITY_MATRIX.md` for the acceptance boundary. diff --git a/conformance/scenario.schema.json b/conformance/scenario.schema.json new file mode 100644 index 0000000..6fd6d36 --- /dev/null +++ b/conformance/scenario.schema.json @@ -0,0 +1,73 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "mm/conformance/v1", + "title": "mattermost-cli conformance scenario", + "type": "object", + "additionalProperties": false, + "required": ["schema", "name", "args", "expected"], + "properties": { + "schema": { "const": "mm/conformance/v1" }, + "name": { "type": "string", "minLength": 1 }, + "args": { "type": "array", "items": { "type": "string" } }, + "stdin": { "type": "string" }, + "timeoutMs": { "type": "integer", "minimum": 1, "maximum": 300000 }, + "env": { + "type": "object", + "additionalProperties": { "type": "string" } + }, + "http": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["request", "response"], + "properties": { + "request": { "$ref": "#/$defs/request" }, + "response": { "$ref": "#/$defs/response" } + } + } + }, + "expected": { + "type": "object", + "additionalProperties": false, + "required": ["exitCode", "stdout", "stderr"], + "properties": { + "exitCode": { "type": "integer", "minimum": 0, "maximum": 255 }, + "stdout": { "type": "string" }, + "stderr": { "type": "string" } + } + } + }, + "$defs": { + "stringMap": { + "type": "object", + "additionalProperties": { "type": "string" } + }, + "request": { + "type": "object", + "additionalProperties": false, + "required": ["method", "uri"], + "properties": { + "method": { "type": "string", "minLength": 1 }, + "uri": { "type": "string", "pattern": "^/" }, + "headers": { "$ref": "#/$defs/stringMap" }, + "ignoreHeaders": { + "type": "array", + "uniqueItems": true, + "items": { "type": "string", "minLength": 1 } + }, + "body": { "type": "string" } + } + }, + "response": { + "type": "object", + "additionalProperties": false, + "required": ["status"], + "properties": { + "status": { "type": "integer", "minimum": 100, "maximum": 599 }, + "headers": { "$ref": "#/$defs/stringMap" }, + "body": { "type": "string" } + } + } + } +} diff --git a/conformance/scenarios/v1/whoami.json b/conformance/scenarios/v1/whoami.json new file mode 100644 index 0000000..ed1e703 --- /dev/null +++ b/conformance/scenarios/v1/whoami.json @@ -0,0 +1,40 @@ +{ + "schema": "mm/conformance/v1", + "name": "v1 whoami emits a narrow identity", + "args": ["--json", "whoami"], + "env": { + "MM_TOKEN": "fixture-mattermost-token" + }, + "http": [ + { + "request": { + "method": "GET", + "uri": "/api/v4/users/me", + "headers": { + "Authorization": "Bearer fixture-mattermost-token" + }, + "ignoreHeaders": [ + "Accept", + "Accept-Encoding", + "Accept-Language", + "Connection", + "Content-Type", + "Sec-Fetch-Mode", + "User-Agent" + ] + }, + "response": { + "status": 200, + "headers": { + "Content-Type": "application/json" + }, + "body": "{\"id\":\"user-1\",\"username\":\"alice\",\"first_name\":\"Alice\",\"last_name\":\"Agent\",\"nickname\":\"ali\",\"roles\":\"system_user team_user\",\"email\":\"must-not-leak@example.test\"}" + } + } + ], + "expected": { + "exitCode": 0, + "stdout": "{\n \"id\": \"user-1\",\n \"username\": \"alice\",\n \"displayName\": \"Alice Agent\",\n \"nickname\": \"ali\",\n \"roles\": [\n \"system_user\",\n \"team_user\"\n ]\n}\n", + "stderr": "" + } +} diff --git a/internal/conformance/buffer.go b/internal/conformance/buffer.go new file mode 100644 index 0000000..93ccfc7 --- /dev/null +++ b/internal/conformance/buffer.go @@ -0,0 +1,36 @@ +package conformance + +import "bytes" + +type limitedBuffer struct { + buffer bytes.Buffer + limit int + exceeded bool +} + +func newLimitedBuffer(limit int) *limitedBuffer { + return &limitedBuffer{limit: limit} +} + +func (b *limitedBuffer) Write(p []byte) (int, error) { + remaining := b.limit - b.buffer.Len() + if remaining > 0 { + write := len(p) + if write > remaining { + write = remaining + } + _, _ = b.buffer.Write(p[:write]) + } + if len(p) > remaining { + b.exceeded = true + } + return len(p), nil +} + +func (b *limitedBuffer) String() string { + return b.buffer.String() +} + +func (b *limitedBuffer) Exceeded() bool { + return b.exceeded +} diff --git a/internal/conformance/process_other.go b/internal/conformance/process_other.go new file mode 100644 index 0000000..e4ef982 --- /dev/null +++ b/internal/conformance/process_other.go @@ -0,0 +1,9 @@ +//go:build !darwin && !linux + +package conformance + +import "os/exec" + +func configureProcessGroup(_ *exec.Cmd) {} + +func cleanupProcessGroup(_ *exec.Cmd) {} diff --git a/internal/conformance/process_unix.go b/internal/conformance/process_unix.go new file mode 100644 index 0000000..5cfae6a --- /dev/null +++ b/internal/conformance/process_unix.go @@ -0,0 +1,32 @@ +//go:build darwin || linux + +package conformance + +import ( + "errors" + "os" + "os/exec" + "syscall" +) + +func configureProcessGroup(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Cancel = func() error { + return killProcessGroup(cmd) + } +} + +func cleanupProcessGroup(cmd *exec.Cmd) { + _ = killProcessGroup(cmd) +} + +func killProcessGroup(cmd *exec.Cmd) error { + if cmd.Process == nil { + return os.ErrProcessDone + } + err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + if errors.Is(err, syscall.ESRCH) { + return os.ErrProcessDone + } + return err +} diff --git a/internal/conformance/process_unix_test.go b/internal/conformance/process_unix_test.go new file mode 100644 index 0000000..7cbda0e --- /dev/null +++ b/internal/conformance/process_unix_test.go @@ -0,0 +1,105 @@ +//go:build darwin || linux + +package conformance + +import ( + "context" + "errors" + "os" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +func TestRunTimeoutKillsDescendantProcess(t *testing.T) { + pidFile := t.TempDir() + "/child.pid" + timeout := 250 + scenario := processScenario(timeout, pidFile) + + err := Run(context.Background(), Command{ + Path: "/bin/sh", + }, scenario) + + if err == nil || !strings.Contains(err.Error(), "scenario timeout") { + t.Fatalf("Run() error = %v, want scenario timeout", err) + } + assertRecordedProcessGone(t, pidFile) +} + +func TestRunCleansDescendantAfterLeaderExits(t *testing.T) { + pidFile := t.TempDir() + "/child.pid" + exitCode, stdout, stderr := 0, "", "" + scenario := Scenario{ + Args: []string{"-c", `sleep 30 & child=$!; echo "$child" > "$PIDFILE"; exit 0`}, + Env: map[string]string{"PIDFILE": pidFile}, + Expected: &ProcessExpected{ + ExitCode: &exitCode, + Stdout: &stdout, + Stderr: &stderr, + }, + } + + _ = Run(context.Background(), Command{Path: "/bin/sh"}, scenario) + + assertRecordedProcessGone(t, pidFile) +} + +func TestRunCancellationKillsDescendantProcess(t *testing.T) { + pidFile := t.TempDir() + "/child.pid" + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(pidFile); err == nil { + cancel() + return + } + time.Sleep(10 * time.Millisecond) + } + }() + + err := Run(ctx, Command{Path: "/bin/sh"}, processScenario(5_000, pidFile)) + + if err == nil || !strings.Contains(err.Error(), "command interrupted") { + t.Fatalf("Run() error = %v, want interruption", err) + } + assertRecordedProcessGone(t, pidFile) +} + +func processScenario(timeout int, pidFile string) Scenario { + exitCode, stdout, stderr := 0, "", "" + return Scenario{ + Args: []string{"-c", `sleep 30 & child=$!; echo "$child" > "$PIDFILE"; wait`}, + Timeout: &timeout, + Env: map[string]string{"PIDFILE": pidFile}, + Expected: &ProcessExpected{ + ExitCode: &exitCode, + Stdout: &stdout, + Stderr: &stderr, + }, + } +} + +func assertRecordedProcessGone(t *testing.T, pidFile string) { + t.Helper() + data, err := os.ReadFile(pidFile) + if err != nil { + t.Fatalf("read child pid: %v", err) + } + pid, err := strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil { + t.Fatalf("parse child pid: %v", err) + } + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + err = syscall.Kill(pid, 0) + if errors.Is(err, syscall.ESRCH) { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("descendant process %d remained alive after Run returned", pid) +} diff --git a/internal/conformance/runner.go b/internal/conformance/runner.go new file mode 100644 index 0000000..21854be --- /dev/null +++ b/internal/conformance/runner.go @@ -0,0 +1,160 @@ +package conformance + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "net/http/httptest" + "os" + "os/exec" + "strings" + "time" +) + +const ( + maxProcessOutput = 4 << 20 + processWaitDelay = 2 * time.Second +) + +type Command struct { + Path string + PrefixArgs []string + Dir string +} + +func Run(ctx context.Context, command Command, scenario Scenario) error { + if command.Path == "" { + return fmt.Errorf("command path is required") + } + handler := newSequentialServer(scenario.HTTP, protectedCredentials(scenario)...) + server := httptest.NewServer(handler) + defer func() { + server.CloseClientConnections() + server.Close() + }() + + tempHome, err := os.MkdirTemp("", "mm-conformance-home-") + if err != nil { + return fmt.Errorf("create isolated home: %w", err) + } + defer func() { _ = os.RemoveAll(tempHome) }() + tempDir := tempHome + "/tmp" + if err := os.MkdirAll(tempDir, 0o700); err != nil { + return fmt.Errorf("create isolated temp: %w", err) + } + + timeout := 15 * time.Second + if scenario.Timeout != nil { + timeout = time.Duration(*scenario.Timeout) * time.Millisecond + } + runCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + args := append(append([]string{}, command.PrefixArgs...), scenario.Args...) + cmd := exec.CommandContext(runCtx, command.Path, args...) // #nosec G204 -- explicit operator-selected binary under test. + configureProcessGroup(cmd) + defer cleanupProcessGroup(cmd) + cmd.WaitDelay = processWaitDelay + cmd.Dir = command.Dir + cmd.Stdin = strings.NewReader(scenario.Stdin) + stdout := newLimitedBuffer(maxProcessOutput) + stderr := newLimitedBuffer(maxProcessOutput) + cmd.Stdout = stdout + cmd.Stderr = stderr + cmd.Env, err = isolatedEnv(tempHome, tempDir, server.URL, scenario.Env) + if err != nil { + return err + } + + runErr := cmd.Run() + if ctx.Err() != nil { + return fmt.Errorf("command interrupted: %w", ctx.Err()) + } + if runCtx.Err() != nil { + return fmt.Errorf("command exceeded scenario timeout: %w", runCtx.Err()) + } + if stdout.Exceeded() || stderr.Exceeded() { + return fmt.Errorf("command output exceeded %d bytes per stream", maxProcessOutput) + } + exitCode := 0 + if runErr != nil { + var exitErr *exec.ExitError + if !errors.As(runErr, &exitErr) { + return fmt.Errorf("run command: %w", runErr) + } + exitCode = exitErr.ExitCode() + } + + if err := handler.verify(); err != nil { + return err + } + want := scenario.Expected + if exitCode != *want.ExitCode { + return fmt.Errorf("exit code = %d, want %d; stderr %s", exitCode, *want.ExitCode, byteSummary(stderr.String())) + } + if got, expected := stdout.String(), expand(*want.Stdout, server.URL); got != expected { + return fmt.Errorf("stdout mismatch: got %s, want %s", byteSummary(got), byteSummary(expected)) + } + if got, expected := stderr.String(), expand(*want.Stderr, server.URL); got != expected { + return fmt.Errorf("stderr mismatch: got %s, want %s", byteSummary(got), byteSummary(expected)) + } + return nil +} + +func protectedCredentials(scenario Scenario) []string { + protected := []string{scenario.Env["MM_TOKEN"]} + for _, exchange := range scenario.HTTP { + for name, value := range exchange.Request.Headers { + if !strings.EqualFold(name, "Authorization") { + continue + } + protected = append(protected, value) + if _, credential, found := strings.Cut(value, " "); found { + protected = append(protected, credential) + } + } + } + return protected +} + +func isolatedEnv(home, tempDir, serverURL string, scenarioEnv map[string]string) ([]string, error) { + reserved := map[string]bool{ + "HOME": true, "XDG_CONFIG_HOME": true, "XDG_STATE_HOME": true, "MM_URL": true, + "PATH": true, "TMPDIR": true, "TMP": true, "TEMP": true, + "LANG": true, "LC_ALL": true, "TZ": true, "NO_COLOR": true, "TERM": true, + } + for name := range scenarioEnv { + if reserved[name] { + return nil, fmt.Errorf("scenario environment cannot override reserved variable %q", name) + } + } + env := []string{ + "HOME=" + home, + "XDG_CONFIG_HOME=" + home + "/.config", + "XDG_STATE_HOME=" + home + "/.local/state", + "MM_URL=" + serverURL, + "PATH=" + os.Getenv("PATH"), + "TMPDIR=" + tempDir, + "TMP=" + tempDir, + "TEMP=" + tempDir, + "LANG=C.UTF-8", + "LC_ALL=C.UTF-8", + "TZ=UTC", + "NO_COLOR=1", + "TERM=dumb", + } + for name, value := range scenarioEnv { + env = append(env, name+"="+expand(value, serverURL)) + } + return env, nil +} + +func expand(value, serverURL string) string { + return strings.ReplaceAll(value, "${SERVER_URL}", serverURL) +} + +func byteSummary(value string) string { + sum := sha256.Sum256([]byte(value)) + return fmt.Sprintf("%d bytes sha256:%x", len(value), sum) +} diff --git a/internal/conformance/runner_test.go b/internal/conformance/runner_test.go new file mode 100644 index 0000000..1af668a --- /dev/null +++ b/internal/conformance/runner_test.go @@ -0,0 +1,152 @@ +package conformance + +import ( + "net/http" + "net/http/httptest" + "slices" + "strings" + "testing" +) + +func TestLimitedBufferBoundsStoredOutput(t *testing.T) { + buffer := newLimitedBuffer(4) + input := []byte("abcdefgh") + + n, err := buffer.Write(input) + + if err != nil || n != len(input) { + t.Fatalf("Write() = (%d, %v), want (%d, nil)", n, err, len(input)) + } + if got := buffer.String(); got != "abcd" { + t.Fatalf("String() = %q, want %q", got, "abcd") + } + if !buffer.Exceeded() { + t.Fatal("Exceeded() = false, want true") + } +} + +func TestIsolatedEnvRejectsReservedOverride(t *testing.T) { + _, err := isolatedEnv(t.TempDir(), t.TempDir(), "http://127.0.0.1", map[string]string{"MM_URL": "https://real.example"}) + if err == nil || !strings.Contains(err.Error(), "reserved variable") { + t.Fatalf("isolatedEnv() error = %v, want reserved-variable failure", err) + } +} + +func TestIsolatedEnvDoesNotInheritParentSecrets(t *testing.T) { + t.Setenv("AWS_SECRET_ACCESS_KEY", "must-not-cross-boundary") + env, err := isolatedEnv(t.TempDir(), t.TempDir(), "http://127.0.0.1", nil) + if err != nil { + t.Fatal(err) + } + for _, entry := range env { + if strings.Contains(entry, "must-not-cross-boundary") { + t.Fatalf("isolated env inherited parent secret in %q", entry) + } + } +} + +func TestSequentialServerRejectsDuplicateExpectedHeader(t *testing.T) { + server := newSequentialServer([]HTTPExchange{{ + Request: HTTPRequestExpected{ + Method: "GET", + URI: "/api/v4/users/me", + Headers: map[string]string{"Authorization": "Bearer fixture"}, + }, + Response: HTTPResponse{Status: 200}, + }}) + req := httptest.NewRequest(http.MethodGet, "http://example.test/api/v4/users/me", nil) + req.Header.Add("Authorization", "Bearer fixture") + req.Header.Add("Authorization", "Bearer second") + response := httptest.NewRecorder() + + server.ServeHTTP(response, req) + + if err := server.verify(); err == nil || !strings.Contains(err.Error(), "header \"Authorization\" mismatched") { + t.Fatalf("verify() error = %v, want duplicate-header failure", err) + } +} + +func TestSequentialServerRejectsUnexpectedHeader(t *testing.T) { + server := newSequentialServer([]HTTPExchange{{ + Request: HTTPRequestExpected{Method: "GET", URI: "/"}, + Response: HTTPResponse{Status: 200}, + }}) + req := httptest.NewRequest(http.MethodGet, "http://example.test/", nil) + req.Header.Set("X-Unexpected", "value") + response := httptest.NewRecorder() + + server.ServeHTTP(response, req) + + if err := server.verify(); err == nil || !strings.Contains(err.Error(), "unexpected header \"X-Unexpected\"") { + t.Fatalf("verify() error = %v, want unexpected-header failure", err) + } +} + +func TestSequentialServerRejectsCredentialInIgnoredHeader(t *testing.T) { + const token = "fixture-active-token" + server := newSequentialServer([]HTTPExchange{{ + Request: HTTPRequestExpected{ + Method: "GET", + URI: "/", + Headers: map[string]string{"Authorization": "Bearer " + token}, + IgnoreHeaders: []string{"User-Agent"}, + }, + Response: HTTPResponse{Status: 200}, + }}, token) + req := httptest.NewRequest(http.MethodGet, "http://example.test/", nil) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("User-Agent", "accidental/"+token) + response := httptest.NewRecorder() + + server.ServeHTTP(response, req) + err := server.verify() + + if err == nil || !strings.Contains(err.Error(), "protected credential") { + t.Fatalf("verify() error = %v, want protected-credential failure", err) + } + if strings.Contains(err.Error(), token) { + t.Fatal("verify() reflected protected credential") + } +} + +func TestSequentialServerRejectsCredentialInURIWithoutReflection(t *testing.T) { + const token = "fixture-active-token" + server := newSequentialServer([]HTTPExchange{{ + Request: HTTPRequestExpected{Method: "GET", URI: "/safe"}, + Response: HTTPResponse{Status: 200}, + }}, token) + req := httptest.NewRequest(http.MethodGet, "http://example.test/leak?token="+token, nil) + response := httptest.NewRecorder() + + server.ServeHTTP(response, req) + err := server.verify() + + if err == nil || !strings.Contains(err.Error(), "protected credential") { + t.Fatalf("verify() error = %v, want protected-credential failure", err) + } + if strings.Contains(err.Error(), token) { + t.Fatal("verify() reflected protected credential") + } +} + +func TestProtectedCredentialsIncludesEnvAndExpectedAuthorization(t *testing.T) { + scenario := Scenario{ + Env: map[string]string{"MM_TOKEN": "lower-precedence-env-token"}, + HTTP: []HTTPExchange{{ + Request: HTTPRequestExpected{ + Headers: map[string]string{"authorization": "Bearer active-cli-or-config-token"}, + }, + }}, + } + + got := protectedCredentials(scenario) + for _, want := range []string{ + "lower-precedence-env-token", + "Bearer active-cli-or-config-token", + "active-cli-or-config-token", + } { + if !slices.Contains(got, want) { + t.Fatalf("protectedCredentials() = %q, missing %q", got, want) + } + } +} diff --git a/internal/conformance/scenario.go b/internal/conformance/scenario.go new file mode 100644 index 0000000..a0c85c9 --- /dev/null +++ b/internal/conformance/scenario.go @@ -0,0 +1,111 @@ +package conformance + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "regexp" +) + +const SchemaV1 = "mm/conformance/v1" + +var envNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +type Scenario struct { + Schema string `json:"schema"` + Name string `json:"name"` + Args []string `json:"args"` + Stdin string `json:"stdin,omitempty"` + Timeout *int `json:"timeoutMs,omitempty"` + Env map[string]string `json:"env,omitempty"` + HTTP []HTTPExchange `json:"http,omitempty"` + Expected *ProcessExpected `json:"expected"` +} + +type HTTPExchange struct { + Request HTTPRequestExpected `json:"request"` + Response HTTPResponse `json:"response"` +} + +type HTTPRequestExpected struct { + Method string `json:"method"` + URI string `json:"uri"` + Headers map[string]string `json:"headers,omitempty"` + IgnoreHeaders []string `json:"ignoreHeaders,omitempty"` + Body string `json:"body,omitempty"` +} + +type HTTPResponse struct { + Status int `json:"status"` + Headers map[string]string `json:"headers,omitempty"` + Body string `json:"body,omitempty"` +} + +type ProcessExpected struct { + ExitCode *int `json:"exitCode"` + Stdout *string `json:"stdout"` + Stderr *string `json:"stderr"` +} + +func Load(path string) (Scenario, error) { + f, err := os.Open(path) // #nosec G304 -- operator-selected local scenario fixture. + if err != nil { + return Scenario{}, err + } + defer func() { _ = f.Close() }() + + decoder := json.NewDecoder(f) + decoder.DisallowUnknownFields() + var scenario Scenario + if err := decoder.Decode(&scenario); err != nil { + return Scenario{}, fmt.Errorf("decode scenario: %w", err) + } + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return Scenario{}, fmt.Errorf("decode scenario: trailing JSON value") + } + return Scenario{}, fmt.Errorf("decode scenario trailer: %w", err) + } + if scenario.Schema != SchemaV1 { + return Scenario{}, fmt.Errorf("unsupported scenario schema %q", scenario.Schema) + } + if scenario.Name == "" { + return Scenario{}, fmt.Errorf("scenario name is required") + } + if scenario.Args == nil { + return Scenario{}, fmt.Errorf("scenario args are required") + } + if scenario.Expected == nil || scenario.Expected.ExitCode == nil || scenario.Expected.Stdout == nil || scenario.Expected.Stderr == nil { + return Scenario{}, fmt.Errorf("scenario expected exitCode, stdout, and stderr are required") + } + if *scenario.Expected.ExitCode < 0 || *scenario.Expected.ExitCode > 255 { + return Scenario{}, fmt.Errorf("scenario expected exitCode must be between 0 and 255") + } + if scenario.Timeout != nil && (*scenario.Timeout < 1 || *scenario.Timeout > 300_000) { + return Scenario{}, fmt.Errorf("scenario timeoutMs must be between 1 and 300000 when set") + } + for name := range scenario.Env { + if !envNamePattern.MatchString(name) { + return Scenario{}, fmt.Errorf("invalid scenario environment name %q", name) + } + } + for i, exchange := range scenario.HTTP { + if exchange.Request.Method == "" || exchange.Request.URI == "" { + return Scenario{}, fmt.Errorf("http exchange %d requires request method and uri", i) + } + if exchange.Response.Status < 100 || exchange.Response.Status > 599 { + return Scenario{}, fmt.Errorf("http exchange %d has invalid response status", i) + } + for _, name := range exchange.Request.IgnoreHeaders { + switch http.CanonicalHeaderKey(name) { + case "Authorization", "Cookie", "Proxy-Authorization": + return Scenario{}, fmt.Errorf("http exchange %d cannot ignore security-sensitive header %q", i, name) + } + } + } + return scenario, nil +} diff --git a/internal/conformance/scenario_test.go b/internal/conformance/scenario_test.go new file mode 100644 index 0000000..3f8bdb8 --- /dev/null +++ b/internal/conformance/scenario_test.go @@ -0,0 +1,59 @@ +package conformance + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestLoadRejectsUnknownFields(t *testing.T) { + path := filepath.Join(t.TempDir(), "scenario.json") + content := `{"schema":"mm/conformance/v1","name":"bad","unexpected":true,"expected":{"exitCode":0,"stdout":"","stderr":""}}` + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + + _, err := Load(path) + if err == nil || !strings.Contains(err.Error(), "unknown field") { + t.Fatalf("Load() error = %v, want unknown-field failure", err) + } +} + +func TestLoadRequiresCompleteProcessExpectation(t *testing.T) { + path := filepath.Join(t.TempDir(), "scenario.json") + content := `{"schema":"mm/conformance/v1","name":"bad","args":[],"expected":{"exitCode":0,"stdout":""}}` + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + + _, err := Load(path) + if err == nil || !strings.Contains(err.Error(), "expected exitCode, stdout, and stderr are required") { + t.Fatalf("Load() error = %v, want incomplete-expectation failure", err) + } +} + +func TestLoadRejectsIgnoredAuthorizationHeader(t *testing.T) { + path := filepath.Join(t.TempDir(), "scenario.json") + content := `{"schema":"mm/conformance/v1","name":"bad","args":[],"http":[{"request":{"method":"GET","uri":"/","ignoreHeaders":["authorization"]},"response":{"status":200}}],"expected":{"exitCode":0,"stdout":"","stderr":""}}` + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + + _, err := Load(path) + if err == nil || !strings.Contains(err.Error(), "cannot ignore security-sensitive header") { + t.Fatalf("Load() error = %v, want ignored-authorization failure", err) + } +} + +func TestSequentialServerReportsMissingRequests(t *testing.T) { + server := newSequentialServer([]HTTPExchange{{ + Request: HTTPRequestExpected{Method: "GET", URI: "/api/v4/users/me"}, + Response: HTTPResponse{Status: 200}, + }}) + + err := server.verify() + if err == nil || !strings.Contains(err.Error(), "observed 0 requests, want 1") { + t.Fatalf("verify() error = %v, want missing-request failure", err) + } +} diff --git a/internal/conformance/server.go b/internal/conformance/server.go new file mode 100644 index 0000000..44c225a --- /dev/null +++ b/internal/conformance/server.go @@ -0,0 +1,135 @@ +package conformance + +import ( + "fmt" + "io" + "net/http" + "slices" + "strings" + "sync" +) + +const maxRequestBody = 16 << 20 + +type sequentialServer struct { + mu sync.Mutex + exchanges []HTTPExchange + protected []string + next int + errors []string +} + +func newSequentialServer(exchanges []HTTPExchange, protected ...string) *sequentialServer { + nonempty := make([]string, 0, len(protected)) + for _, value := range protected { + if value != "" { + nonempty = append(nonempty, value) + } + } + return &sequentialServer{exchanges: exchanges, protected: nonempty} +} + +func (s *sequentialServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + + if s.next >= len(s.exchanges) { + s.fail(w, "unexpected request method/path") + return + } + exchange := s.exchanges[s.next] + s.next++ + + body, err := io.ReadAll(io.LimitReader(r.Body, maxRequestBody+1)) + if err != nil { + s.fail(w, "request %d body read failed", s.next) + return + } + if len(body) > maxRequestBody { + s.fail(w, "request %d body exceeded harness limit", s.next) + return + } + + want := exchange.Request + if r.Method != want.Method { + s.errors = append(s.errors, fmt.Sprintf("request %d method = %q, want %q", s.next, r.Method, want.Method)) + } + if got := r.URL.RequestURI(); got != want.URI { + s.errors = append(s.errors, fmt.Sprintf("request %d uri mismatched", s.next)) + } + expectedHeaders := make(map[string]string, len(want.Headers)) + for name, expected := range want.Headers { + expectedHeaders[http.CanonicalHeaderKey(name)] = expected + } + ignoredHeaders := make(map[string]bool, len(want.IgnoreHeaders)) + for _, name := range want.IgnoreHeaders { + ignoredHeaders[http.CanonicalHeaderKey(name)] = true + } + s.scanProtectedCredential(r, body, expectedHeaders) + for name, expected := range expectedHeaders { + if got := r.Header.Values(name); !slices.Equal(got, []string{expected}) { + s.errors = append(s.errors, fmt.Sprintf("request %d header %q mismatched", s.next, name)) + } + } + for name := range r.Header { + canonical := http.CanonicalHeaderKey(name) + if _, expected := expectedHeaders[canonical]; !expected && !ignoredHeaders[canonical] { + s.errors = append(s.errors, fmt.Sprintf("request %d had unexpected header %q", s.next, canonical)) + } + } + if got := string(body); got != want.Body { + s.errors = append(s.errors, fmt.Sprintf("request %d body mismatched", s.next)) + } + if len(s.errors) > 0 { + http.Error(w, "scenario request mismatch", http.StatusInternalServerError) + return + } + + for name, value := range exchange.Response.Headers { + w.Header().Set(name, value) + } + w.WriteHeader(exchange.Response.Status) + _, _ = io.WriteString(w, exchange.Response.Body) +} + +func (s *sequentialServer) scanProtectedCredential(r *http.Request, body []byte, expectedHeaders map[string]string) { + for _, protected := range s.protected { + if strings.Contains(r.URL.RequestURI(), protected) || strings.Contains(string(body), protected) { + s.errors = append(s.errors, fmt.Sprintf("request %d exposed protected credential outside authorization", s.next)) + } + for name, values := range r.Header { + canonical := http.CanonicalHeaderKey(name) + contains := false + for _, value := range values { + if strings.Contains(value, protected) { + contains = true + break + } + } + if !contains { + continue + } + allowed := canonical == "Authorization" && slices.Equal(values, []string{expectedHeaders[canonical]}) + if !allowed { + s.errors = append(s.errors, fmt.Sprintf("request %d exposed protected credential outside expected authorization", s.next)) + } + } + } +} + +func (s *sequentialServer) fail(w http.ResponseWriter, format string, args ...any) { + s.errors = append(s.errors, fmt.Sprintf(format, args...)) + http.Error(w, "unexpected scenario request", http.StatusInternalServerError) +} + +func (s *sequentialServer) verify() error { + s.mu.Lock() + defer s.mu.Unlock() + if s.next != len(s.exchanges) { + s.errors = append(s.errors, fmt.Sprintf("observed %d requests, want %d", s.next, len(s.exchanges))) + } + if len(s.errors) == 0 { + return nil + } + return fmt.Errorf("http conformance failed: %s", strings.Join(s.errors, "; ")) +} diff --git a/justfile b/justfile index cdf7129..e6aa4d6 100644 --- a/justfile +++ b/justfile @@ -21,10 +21,14 @@ go-modules: go-build: go build -o "${TMPDIR:-/tmp}/mattermost-cli-mm" ./cmd/mm +oracle-smoke: + git diff --quiet v1.6.0 -- src package.json bun.lock tsconfig.json + go run ./cmd/conformance --scenario conformance/scenarios/v1/whoami.json --cwd . -- bun src/index.ts + go-gate: go-format-check go-test go-race go-vet go-modules go-build git diff --check legacy-gate: bun run verify -gate: go-gate legacy-gate +gate: go-gate legacy-gate oracle-smoke From d6cd131f9df3b76f191973c911cd67ceef25924a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 03:25:23 +0300 Subject: [PATCH 005/119] feat: add v2 machine schema registry --- .github/workflows/ci.yml | 15 ++++ cmd/mm/main.go | 1 + cmd/mm/signal_other.go | 5 ++ cmd/mm/signal_unix.go | 15 ++++ cmd/mm/signal_unix_test.go | 65 +++++++++++++++ go.mod | 3 + go.sum | 6 ++ internal/cli/root.go | 96 +++++++++++++++++++++- internal/cli/root_test.go | 70 ++++++++++++++++ internal/schema/registry.go | 132 +++++++++++++++++++++++++++++++ internal/schema/registry_test.go | 82 +++++++++++++++++++ schemas/embed.go | 8 ++ schemas/v2/error.schema.json | 85 ++++++++++++++++++++ schemas/v2/examples/error.json | 8 ++ 14 files changed, 590 insertions(+), 1 deletion(-) create mode 100644 cmd/mm/signal_other.go create mode 100644 cmd/mm/signal_unix.go create mode 100644 cmd/mm/signal_unix_test.go create mode 100644 internal/schema/registry.go create mode 100644 internal/schema/registry_test.go create mode 100644 schemas/embed.go create mode 100644 schemas/v2/error.schema.json create mode 100644 schemas/v2/examples/error.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5fd4e95..fd1dbe9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,21 @@ permissions: contents: read jobs: + go: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version-file: go.mod + cache: true + - run: go test ./... + - run: go test -race ./... + - run: go vet ./... + - run: go mod verify + - run: go build -o "$RUNNER_TEMP/mm" ./cmd/mm + - run: git diff --check + verify: runs-on: ubuntu-latest steps: diff --git a/cmd/mm/main.go b/cmd/mm/main.go index 32f6cad..b600837 100644 --- a/cmd/mm/main.go +++ b/cmd/mm/main.go @@ -8,5 +8,6 @@ import ( ) func main() { + handleBrokenPipe() os.Exit(cli.Execute(context.Background(), os.Args[1:], os.Stdin, os.Stdout, os.Stderr)) } diff --git a/cmd/mm/signal_other.go b/cmd/mm/signal_other.go new file mode 100644 index 0000000..1af0e8c --- /dev/null +++ b/cmd/mm/signal_other.go @@ -0,0 +1,5 @@ +//go:build !darwin && !linux + +package main + +func handleBrokenPipe() {} diff --git a/cmd/mm/signal_unix.go b/cmd/mm/signal_unix.go new file mode 100644 index 0000000..6f0f11a --- /dev/null +++ b/cmd/mm/signal_unix.go @@ -0,0 +1,15 @@ +//go:build darwin || linux + +package main + +import ( + "os" + "os/signal" + "syscall" +) + +var brokenPipeSignals = make(chan os.Signal, 1) + +func handleBrokenPipe() { + signal.Notify(brokenPipeSignals, syscall.SIGPIPE) +} diff --git a/cmd/mm/signal_unix_test.go b/cmd/mm/signal_unix_test.go new file mode 100644 index 0000000..3f1306c --- /dev/null +++ b/cmd/mm/signal_unix_test.go @@ -0,0 +1,65 @@ +//go:build darwin || linux + +package main + +import ( + "bytes" + "context" + "errors" + "os" + "os/exec" + "os/signal" + "strings" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/cli" +) + +func TestClosedStdoutUsesStableExitClass(t *testing.T) { + command := exec.Command(os.Args[0], "-test.run=TestMMBrokenPipeHelper") + command.Env = append(os.Environ(), "MM_BROKEN_PIPE_HELPER=1") + reader, writer, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + if err := reader.Close(); err != nil { + t.Fatal(err) + } + defer func() { _ = writer.Close() }() + command.Stdout = writer + var stderr bytes.Buffer + command.Stderr = &stderr + + err = command.Run() + + var exitError *exec.ExitError + if !errors.As(err, &exitError) || exitError.ExitCode() != 3 { + t.Fatalf("closed-stdout result = %v; stderr = %q, want exit 3", err, stderr.String()) + } + if strings.Contains(stderr.String(), "broken pipe") { + t.Fatalf("stderr reflected low-level output error: %q", stderr.String()) + } +} + +func TestMMBrokenPipeHelper(t *testing.T) { + if os.Getenv("MM_BROKEN_PIPE_HELPER") != "1" { + return + } + handleBrokenPipe() + os.Exit(cli.Execute(context.Background(), []string{"schema", "list"}, strings.NewReader(""), os.Stdout, os.Stderr)) +} + +func TestBrokenPipeHandlerIsNotInheritedByChild(t *testing.T) { + handleBrokenPipe() + defer signal.Stop(brokenPipeSignals) + + output, err := exec.Command("/bin/sh", "-c", `kill -PIPE $$; echo survived`).CombinedOutput() + + var exitError *exec.ExitError + if !errors.As(err, &exitError) { + t.Fatalf("child error = %v, output = %q; want SIGPIPE termination", err, output) + } + if strings.Contains(string(output), "survived") { + t.Fatalf("child inherited handled SIGPIPE: %q", output) + } +} diff --git a/go.mod b/go.mod index 2612d55..855eb66 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,10 @@ go 1.26.5 require github.com/spf13/cobra v1.10.2 +require github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 + require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/spf13/pflag v1.0.9 // indirect + golang.org/x/text v0.14.0 // indirect ) diff --git a/go.sum b/go.sum index a6ee3e0..6e609e2 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,16 @@ github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/cli/root.go b/internal/cli/root.go index b955728..1a1e1bc 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -2,12 +2,16 @@ package cli import ( "context" + "errors" "fmt" "io" + "os" + "strings" "github.com/spf13/cobra" "github.com/ardasevinc/mattermost-cli/internal/buildinfo" + mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" ) type streams struct { @@ -20,7 +24,15 @@ func Execute(ctx context.Context, args []string, in io.Reader, out, errOut io.Wr cmd := newRoot(streams{in: in, out: out, err: errOut}) cmd.SetArgs(args) if err := cmd.ExecuteContext(ctx); err != nil { - _, _ = fmt.Fprintf(errOut, "error: %s\n", err) + message := err.Error() + if token := os.Getenv("MM_TOKEN"); token != "" { + message = strings.ReplaceAll(message, token, "[REDACTED:mattermost_credential]") + } + _, _ = fmt.Fprintf(errOut, "error: %s\n", message) + var outputFailure outputError + if errors.As(err, &outputFailure) { + return 3 + } return 2 } return 0 @@ -44,5 +56,87 @@ func newRoot(s streams) *cobra.Command { cmd.SetIn(s.in) cmd.SetOut(s.out) cmd.SetErr(s.err) + cmd.AddCommand(newSchemaCommand(s)) return cmd } + +func newSchemaCommand(s streams) *cobra.Command { + command := &cobra.Command{ + Use: "schema", + Short: "Inspect and validate machine schemas", + Args: cobra.NoArgs, + } + command.AddCommand(&cobra.Command{ + Use: "list", + Short: "List embedded schema identifiers", + Args: cobra.NoArgs, + RunE: func(_ *cobra.Command, _ []string) error { + registry, err := mmSchema.Load() + if err != nil { + return err + } + return writeAll(s.out, []byte(strings.Join(registry.IDs(), "\n")+"\n")) + }, + }) + command.AddCommand(&cobra.Command{ + Use: "show ", + Short: "Print one embedded JSON Schema", + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + registry, err := mmSchema.Load() + if err != nil { + return err + } + data, err := registry.Show(args[0]) + if err != nil { + return err + } + if err := writeAll(s.out, data); err != nil { + return err + } + if len(data) == 0 || data[len(data)-1] != '\n' { + err = writeAll(s.out, []byte("\n")) + } + return err + }, + }) + command.AddCommand(&cobra.Command{ + Use: "validate ", + Short: "Validate one JSON document from stdin", + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + registry, err := mmSchema.Load() + if err != nil { + return err + } + if err := registry.Validate(args[0], s.in); err != nil { + return err + } + return writeAll(s.out, []byte("valid: "+args[0]+"\n")) + }, + }) + return command +} + +type outputError struct { + err error +} + +func (e outputError) Error() string { + return "write output failed" +} + +func (e outputError) Unwrap() error { + return e.err +} + +func writeAll(output io.Writer, data []byte) error { + written, err := io.Copy(output, strings.NewReader(string(data))) + if err != nil { + return outputError{err: err} + } + if written != int64(len(data)) { + return outputError{err: io.ErrShortWrite} + } + return nil +} diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index 1c754e2..41b964a 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -3,6 +3,7 @@ package cli import ( "bytes" "context" + "io" "strings" "testing" ) @@ -43,3 +44,72 @@ func TestExecuteRejectsUnknownCommandWithoutUsage(t *testing.T) { t.Fatalf("stderr = %q, must not contain usage", stderr.String()) } } + +func TestSchemaList(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + code := Execute(context.Background(), []string{"schema", "list"}, strings.NewReader(""), &stdout, &stderr) + + if code != 0 { + t.Fatalf("exit code = %d, want 0; stderr = %q", code, stderr.String()) + } + if got, want := stdout.String(), "mm/v2/error\n"; got != want { + t.Fatalf("stdout = %q, want %q", got, want) + } +} + +func TestSchemaValidate(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + document := `{"schema":"mm/v2/error","code":"invalid_input","message":"bad input","exitCode":2,"recovery":"none"}` + + code := Execute(context.Background(), []string{"schema", "validate", "mm/v2/error"}, strings.NewReader(document), &stdout, &stderr) + + if code != 0 { + t.Fatalf("exit code = %d, want 0; stderr = %q", code, stderr.String()) + } + if got, want := stdout.String(), "valid: mm/v2/error\n"; got != want { + t.Fatalf("stdout = %q, want %q", got, want) + } +} + +func TestSchemaLookupDoesNotReflectActiveCredential(t *testing.T) { + const token = "super-secret-mm-token" + t.Setenv("MM_TOKEN", token) + var stdout bytes.Buffer + var stderr bytes.Buffer + + code := Execute(context.Background(), []string{"schema", "show", token}, strings.NewReader(""), &stdout, &stderr) + + if code != 2 { + t.Fatalf("exit code = %d, want 2", code) + } + if strings.Contains(stderr.String(), token) { + t.Fatalf("stderr reflected active credential: %q", stderr.String()) + } +} + +func TestSchemaShowTreatsShortWriteAsReadFailure(t *testing.T) { + var stderr bytes.Buffer + + code := Execute(context.Background(), []string{"schema", "show", "mm/v2/error"}, strings.NewReader(""), shortWriter{}, &stderr) + + if code != 3 { + t.Fatalf("exit code = %d, want 3; stderr = %q", code, stderr.String()) + } + if !strings.Contains(stderr.String(), "write output failed") { + t.Fatalf("stderr = %q, want generic output failure", stderr.String()) + } +} + +type shortWriter struct{} + +func (shortWriter) Write(data []byte) (int, error) { + if len(data) == 0 { + return 0, nil + } + return len(data) - 1, nil +} + +var _ io.Writer = shortWriter{} diff --git a/internal/schema/registry.go b/internal/schema/registry.go new file mode 100644 index 0000000..3a1eb36 --- /dev/null +++ b/internal/schema/registry.go @@ -0,0 +1,132 @@ +package schema + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/fs" + "slices" + "strings" + + jsonschema "github.com/santhosh-tekuri/jsonschema/v6" + + publicschemas "github.com/ardasevinc/mattermost-cli/schemas" +) + +const maxDocumentBytes = 4 << 20 + +type Registry struct { + raw map[string][]byte + compiled map[string]*jsonschema.Schema +} + +func Load() (*Registry, error) { + paths, err := fs.Glob(publicschemas.FS, "v2/*.schema.json") + if err != nil { + return nil, fmt.Errorf("discover embedded schemas: %w", err) + } + compiler := jsonschema.NewCompiler() + compiler.AssertFormat() + raw := make(map[string][]byte, len(paths)) + for _, path := range paths { + data, err := fs.ReadFile(publicschemas.FS, path) + if err != nil { + return nil, fmt.Errorf("read embedded schema: %w", err) + } + var document map[string]any + if err := json.Unmarshal(data, &document); err != nil { + return nil, fmt.Errorf("decode embedded schema %s: %w", path, err) + } + resourceID, ok := document["$id"].(string) + if !ok || !strings.HasPrefix(resourceID, "urn:mm:schema:v2:") { + return nil, fmt.Errorf("embedded schema %s has invalid identifier", path) + } + logicalID, ok := logicalIdentifier(document) + if !ok { + return nil, fmt.Errorf("embedded schema %s has invalid logical identifier", path) + } + if _, duplicate := raw[logicalID]; duplicate { + return nil, fmt.Errorf("duplicate embedded schema identifier") + } + if err := compiler.AddResource(resourceID, document); err != nil { + return nil, fmt.Errorf("register embedded schema: %w", err) + } + raw[logicalID] = data + } + compiled := make(map[string]*jsonschema.Schema, len(raw)) + for logicalID, data := range raw { + var document map[string]any + if err := json.Unmarshal(data, &document); err != nil { + return nil, fmt.Errorf("decode embedded schema: %w", err) + } + resourceID, _ := document["$id"].(string) + compiled[logicalID], err = compiler.Compile(resourceID) + if err != nil { + return nil, fmt.Errorf("compile embedded schema: %w", err) + } + } + return &Registry{raw: raw, compiled: compiled}, nil +} + +func (r *Registry) IDs() []string { + ids := make([]string, 0, len(r.raw)) + for id := range r.raw { + ids = append(ids, id) + } + slices.Sort(ids) + return ids +} + +func (r *Registry) Show(id string) ([]byte, error) { + data, ok := r.raw[id] + if !ok { + return nil, fmt.Errorf("unknown schema") + } + return bytes.Clone(data), nil +} + +func (r *Registry) Validate(id string, input io.Reader) error { + compiled, ok := r.compiled[id] + if !ok { + return fmt.Errorf("unknown schema") + } + limited := io.LimitReader(input, maxDocumentBytes+1) + data, err := io.ReadAll(limited) + if err != nil { + return fmt.Errorf("read JSON document: %w", err) + } + if len(data) > maxDocumentBytes { + return fmt.Errorf("JSON document exceeds %d bytes", maxDocumentBytes) + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.UseNumber() + var document any + if err := decoder.Decode(&document); err != nil { + return fmt.Errorf("decode JSON document: %w", err) + } + var trailing any + if err := decoder.Decode(&trailing); err != io.EOF { + if err == nil { + return fmt.Errorf("decode JSON document: trailing JSON value") + } + return fmt.Errorf("decode JSON document trailer: %w", err) + } + if err := compiled.Validate(document); err != nil { + return fmt.Errorf("document does not match %s", id) + } + return nil +} + +func logicalIdentifier(document map[string]any) (string, bool) { + properties, ok := document["properties"].(map[string]any) + if !ok { + return "", false + } + schemaProperty, ok := properties["schema"].(map[string]any) + if !ok { + return "", false + } + logicalID, ok := schemaProperty["const"].(string) + return logicalID, ok && strings.HasPrefix(logicalID, "mm/v2/") +} diff --git a/internal/schema/registry_test.go b/internal/schema/registry_test.go new file mode 100644 index 0000000..d587e32 --- /dev/null +++ b/internal/schema/registry_test.go @@ -0,0 +1,82 @@ +package schema + +import ( + "bytes" + "encoding/json" + "io/fs" + "strings" + "testing" + + publicschemas "github.com/ardasevinc/mattermost-cli/schemas" +) + +func TestEmbeddedExamplesValidate(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + paths, err := fs.Glob(publicschemas.FS, "v2/examples/*.json") + if err != nil { + t.Fatal(err) + } + if len(paths) == 0 { + t.Fatal("no embedded schema examples") + } + for _, path := range paths { + data, err := fs.ReadFile(publicschemas.FS, path) + if err != nil { + t.Fatal(err) + } + var envelope struct { + Schema string `json:"schema"` + } + if err := json.Unmarshal(data, &envelope); err != nil { + t.Fatalf("decode %s: %v", path, err) + } + if err := registry.Validate(envelope.Schema, bytes.NewReader(data)); err != nil { + t.Fatalf("validate %s: %v", path, err) + } + } +} + +func TestValidateRejectsUnknownFields(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + document := `{"schema":"mm/v2/error","code":"internal","message":"failed","exitCode":3,"secret":"nope"}` + + err = registry.Validate("mm/v2/error", strings.NewReader(document)) + + if err == nil || !strings.Contains(err.Error(), "document does not match mm/v2/error") { + t.Fatalf("Validate() error = %v, want unknown-field rejection", err) + } +} + +func TestValidateRejectsTrailingJSON(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + + err = registry.Validate("mm/v2/error", strings.NewReader(`{} {}`)) + + if err == nil || !strings.Contains(err.Error(), "trailing JSON value") { + t.Fatalf("Validate() error = %v, want trailing-value rejection", err) + } +} + +func TestErrorSchemaBindsCodesToExitClassesAndRequiresRecovery(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + for _, document := range []string{ + `{"schema":"mm/v2/error","code":"authentication","message":"failed","exitCode":7,"recovery":"none"}`, + `{"schema":"mm/v2/error","code":"state_conflict","message":"failed","exitCode":6}`, + } { + if err := registry.Validate("mm/v2/error", strings.NewReader(document)); err == nil { + t.Fatalf("Validate() accepted contradictory error envelope: %s", document) + } + } +} diff --git a/schemas/embed.go b/schemas/embed.go new file mode 100644 index 0000000..6f0fb4b --- /dev/null +++ b/schemas/embed.go @@ -0,0 +1,8 @@ +package schemas + +import "embed" + +// FS contains the checked-in public machine contracts shipped with mm. +// +//go:embed v2/*.schema.json v2/examples/*.json +var FS embed.FS diff --git a/schemas/v2/error.schema.json b/schemas/v2/error.schema.json new file mode 100644 index 0000000..f1410f9 --- /dev/null +++ b/schemas/v2/error.schema.json @@ -0,0 +1,85 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:error", + "title": "mm v2 machine error", + "type": "object", + "additionalProperties": false, + "required": ["schema", "code", "message", "exitCode", "recovery"], + "properties": { + "schema": { "const": "mm/v2/error" }, + "code": { + "type": "string", + "enum": [ + "invalid_invocation", + "invalid_input", + "configuration", + "authentication", + "authorization", + "read_failed", + "mutation_rejected", + "mutation_partial", + "mutation_unknown", + "state_conflict", + "confirmed_effect_local_failure", + "internal" + ] + }, + "message": { "type": "string", "minLength": 1 }, + "exitCode": { "type": "integer", "enum": [2, 3, 4, 5, 6, 7] }, + "stageRef": { "type": "string", "minLength": 1 }, + "recovery": { + "type": "string", + "enum": ["none", "resume_partial", "force_unknown", "forbidden"] + } + }, + "allOf": [ + { + "if": { + "properties": { + "code": { "enum": ["invalid_invocation", "invalid_input"] } + } + }, + "then": { "properties": { "exitCode": { "const": 2 } } } + }, + { + "if": { + "properties": { + "code": { + "enum": [ + "configuration", + "authentication", + "authorization", + "read_failed", + "internal" + ] + } + } + }, + "then": { "properties": { "exitCode": { "const": 3 } } } + }, + { + "if": { "properties": { "code": { "const": "mutation_rejected" } } }, + "then": { "properties": { "exitCode": { "const": 4 } } } + }, + { + "if": { + "properties": { + "code": { "enum": ["mutation_partial", "mutation_unknown"] } + } + }, + "then": { "properties": { "exitCode": { "const": 5 } } } + }, + { + "if": { "properties": { "code": { "const": "state_conflict" } } }, + "then": { "properties": { "exitCode": { "const": 6 } } } + }, + { + "if": { + "properties": { + "code": { "const": "confirmed_effect_local_failure" } + } + }, + "then": { "properties": { "exitCode": { "const": 7 } } } + } + ] +} diff --git a/schemas/v2/examples/error.json b/schemas/v2/examples/error.json new file mode 100644 index 0000000..372203e --- /dev/null +++ b/schemas/v2/examples/error.json @@ -0,0 +1,8 @@ +{ + "schema": "mm/v2/error", + "code": "state_conflict", + "message": "stage revision no longer matches", + "exitCode": 6, + "stageRef": "stage_fixture@2", + "recovery": "none" +} From 8c1ab7bf9a1600afd27b8a30121ab2cbac101570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 03:58:22 +0300 Subject: [PATCH 006/119] feat: port config and URL policy --- docs/V1_PARITY_MATRIX.md | 3 +- docs/V2_CONTRACT.md | 1 + go.mod | 8 +- go.sum | 10 +- internal/config/config.go | 279 +++++++++++++++++++++++++++ internal/config/config_test.go | 289 ++++++++++++++++++++++++++++ internal/config/paths.go | 34 ++++ internal/config/secure_other.go | 16 ++ internal/config/secure_unix.go | 140 ++++++++++++++ internal/config/secure_unix_test.go | 35 ++++ internal/serverurl/url.go | 147 ++++++++++++++ internal/serverurl/url_test.go | 136 +++++++++++++ 12 files changed, 1094 insertions(+), 4 deletions(-) create mode 100644 internal/config/config.go create mode 100644 internal/config/config_test.go create mode 100644 internal/config/paths.go create mode 100644 internal/config/secure_other.go create mode 100644 internal/config/secure_unix.go create mode 100644 internal/config/secure_unix_test.go create mode 100644 internal/serverurl/url.go create mode 100644 internal/serverurl/url_test.go diff --git a/docs/V1_PARITY_MATRIX.md b/docs/V1_PARITY_MATRIX.md index 0944f69..67fc500 100644 --- a/docs/V1_PARITY_MATRIX.md +++ b/docs/V1_PARITY_MATRIX.md @@ -33,6 +33,7 @@ This is the removal gate for the TypeScript implementation. A row may become `ve | `--no-threads` | selected seeds only except `thread` | preserve | oracle | | numeric validation | canonical positive safe integer only | preserve bounded canonical integer validation | oracle | | duration validation | `^\d+[hdwm]$` | preserve | oracle | +| URL normalization | WHATWG normalization plus custom loopback test | preserve safe canonicalization; reject transport-ambiguous IPv4/backslash forms | intentionally_changed | ## Configuration @@ -137,7 +138,7 @@ Every row requires a named Go regression test and, where applicable, a language- | uncertain mutation requests are never automatically replayed | oracle | | confirmed effect plus local receipt/output failure says do not retry | oracle | | reads retry only bounded safe failures | oracle | -| HTTP is allowed only for loopback; unsafe URL components fail closed | oracle | +| HTTP is allowed only for transport-canonical loopback; unsafe or parser-ambiguous URL components fail closed | intentionally_changed | | complete normalized base path is preserved and stage-bound | oracle | | read commands never create DMs or perform POST fallback | oracle | | dry-run performs no local persistence or remote mutation | oracle | diff --git a/docs/V2_CONTRACT.md b/docs/V2_CONTRACT.md index f868db4..89b5203 100644 --- a/docs/V2_CONTRACT.md +++ b/docs/V2_CONTRACT.md @@ -407,6 +407,7 @@ The v1 security boundary survives the rewrite: - HTTPS is required except explicitly allowed loopback HTTP for local testing. - URLs with credentials, query strings, fragments, unsafe schemes, or ambiguous normalization fail closed. +- URL normalization is transport-aligned rather than blindly WHATWG-compatible. Safe canonicalizations such as IDNA hostnames, dot-segment resolution, lowercase hosts, and default-port removal are preserved. Host spellings that Go's dialer could interpret differently from validation, including shorthand or leading-zero IPv4, and backslash authority forms are rejected. Loopback HTTP requires `localhost` or a canonical `netip` loopback address. - User-controlled path components are encoded. - Mutation requests do not follow redirects. - Active credentials are ownership-scoped and fully masked everywhere. diff --git a/go.mod b/go.mod index 855eb66..9002b17 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,14 @@ require github.com/spf13/cobra v1.10.2 require github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 +require github.com/pelletier/go-toml/v2 v2.4.3 + +require golang.org/x/net v0.57.0 + +require golang.org/x/sys v0.47.0 + require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/spf13/pflag v1.0.9 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/text v0.40.0 // indirect ) diff --git a/go.sum b/go.sum index 6e609e2..8799a06 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxK github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= +github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= @@ -11,6 +13,10 @@ github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiT github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..a97b30a --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,279 @@ +package config + +import ( + "errors" + "fmt" + "io" + "os" + "strings" + + "github.com/pelletier/go-toml/v2" +) + +const maxConfigBytes = 1 << 20 + +const Template = `# Mattermost CLI Configuration +# https://github.com/ardasevinc/mattermost-cli + +url = "https://mattermost.example.com" +token = "your-personal-access-token" +# mention_names = ["Arda", "arda.sevinc"] +` + +type Source string + +const ( + SourceCLI Source = "cli" + SourceEnv Source = "env" + SourceFile Source = "file" + SourceDefault Source = "default" + SourceMissing Source = "missing" +) + +type Migration string + +const ( + MigrationNone Migration = "none" + MigrationLegacyFallback Migration = "legacy_fallback" + MigrationLegacyIgnored Migration = "legacy_ignored" +) + +type FileError string + +const ( + FileErrorRead FileError = "read" + FileErrorParse FileError = "parse" +) + +type UnsafeReason string + +const ( + UnsafeType UnsafeReason = "type" + UnsafeChanged UnsafeReason = "changed" + UnsafeOwnership UnsafeReason = "ownership" + UnsafeUnsupported UnsafeReason = "unsupported" +) + +type File struct { + URL string + Token string + Redact *bool + MentionNames []string +} + +type FileState struct { + Config File + SelectedPath string + ReadPath string + LegacyPath string + Exists bool + InsecurePermissions bool + Error FileError + Unsafe UnsafeReason + Migration Migration +} + +type Options struct { + URL string + Token string + Redact *bool +} + +type Resolved struct { + URL string + Token string + Redact bool + URLSource Source + TokenSource Source + RedactSource Source + MentionNames []string + File FileState +} + +func Load(paths Paths) FileState { + selected := inspect(paths.ConfigPath) + selected.SelectedPath = paths.ConfigPath + selected.LegacyPath = paths.LegacyPath + if paths.ConfigPath == paths.LegacyPath { + return selected + } + if selected.Exists || selected.Error != "" { + if configPathPresent(paths.LegacyPath) { + selected.Migration = MigrationLegacyIgnored + } + return selected + } + legacy := inspect(paths.LegacyPath) + if !legacy.Exists && legacy.Error == "" { + return selected + } + legacy.SelectedPath = paths.ConfigPath + legacy.LegacyPath = paths.LegacyPath + legacy.Migration = MigrationLegacyFallback + return legacy +} + +func configPathPresent(path string) bool { + file, _, _, err := openConfigFile(path) + if file != nil { + _ = file.Close() + } + return !errors.Is(err, os.ErrNotExist) +} + +func Init(path string) (bool, error) { + file, err := createConfigFile(path) + if errors.Is(err, os.ErrExist) { + existing := inspect(path) + if existing.Unsafe != "" || existing.Error == FileErrorRead || (existing.InsecurePermissions && existing.Config.Token != "") { + return false, fmt.Errorf("existing config path is unsafe") + } + return false, nil + } + if err != nil { + return false, fmt.Errorf("could not create config file") + } + complete := false + defer func() { + _ = file.Close() + if !complete { + _ = os.Remove(path) + } + }() + written, err := io.WriteString(file, Template) + if err != nil { + return false, fmt.Errorf("could not write config template") + } + if written != len(Template) { + return false, fmt.Errorf("could not write config template") + } + if err := file.Sync(); err != nil { + return false, fmt.Errorf("could not sync config template") + } + if err := file.Close(); err != nil { + return false, fmt.Errorf("could not close config template") + } + complete = true + return true, nil +} + +func Resolve(options Options, lookup LookupEnv, file FileState) Resolved { + url, urlSource := first(options.URL, envNonempty(lookup, "MM_URL"), file.Config.URL) + token, tokenSource := first(options.Token, envNonempty(lookup, "MM_TOKEN"), file.Config.Token) + redact, redactSource := resolveRedact(options.Redact, lookup, file.Config.Redact) + return Resolved{ + URL: url, + Token: token, + Redact: redact, + URLSource: urlSource, + TokenSource: tokenSource, + RedactSource: redactSource, + MentionNames: append([]string(nil), file.Config.MentionNames...), + File: file, + } +} + +func inspect(path string) FileState { + state := FileState{SelectedPath: path, ReadPath: path} + file, info, unsafe, err := openConfigFile(path) + if errors.Is(err, os.ErrNotExist) { + return state + } + if err != nil { + state.Exists = true + state.Error = FileErrorRead + state.Unsafe = unsafe + return state + } + state.Exists = true + defer func() { _ = file.Close() }() + state.InsecurePermissions = info.Mode().Perm()&0o077 != 0 + data, err := io.ReadAll(io.LimitReader(file, maxConfigBytes+1)) + if err != nil || len(data) > maxConfigBytes { + state.Error = FileErrorRead + return state + } + parsed, err := parse(data) + if err != nil { + state.Error = FileErrorParse + return state + } + state.Config = parsed + return state +} + +func parse(data []byte) (File, error) { + var raw map[string]any + if err := toml.Unmarshal(data, &raw); err != nil { + return File{}, err + } + result := File{ + URL: trimmedString(raw["url"]), + Token: trimmedString(raw["token"]), + } + if value, ok := raw["redact"].(bool); ok { + result.Redact = &value + } + if values, ok := raw["mention_names"].([]any); ok { + result.MentionNames = make([]string, 0, len(values)) + for _, rawValue := range values { + if value, ok := rawValue.(string); ok { + value = strings.TrimSpace(value) + if value != "" { + result.MentionNames = append(result.MentionNames, value) + } + } + } + } + return result, nil +} + +func first(cli string, envValue string, file string) (string, Source) { + if cli != "" { + return cli, SourceCLI + } + if envValue != "" { + return envValue, SourceEnv + } + if file != "" { + return file, SourceFile + } + return "", SourceMissing +} + +func envNonempty(lookup LookupEnv, name string) string { + value, _ := lookup(name) + return value +} + +func resolveRedact(cli *bool, lookup LookupEnv, file *bool) (bool, Source) { + if cli != nil { + return *cli, SourceCLI + } + if value, ok := lookup("MM_REDACT"); ok { + return value != "false", SourceEnv + } + if file != nil { + return *file, SourceFile + } + return true, SourceDefault +} + +func trimmedString(value any) string { + text, ok := value.(string) + if !ok { + return "" + } + return strings.TrimSpace(text) +} + +func (f FileState) Warning() string { + switch f.Migration { + case MigrationLegacyFallback: + return fmt.Sprintf("selected config %q is absent; reading legacy config %q without modifying it", f.SelectedPath, f.LegacyPath) + case MigrationLegacyIgnored: + return fmt.Sprintf("selected config %q is authoritative; ignoring legacy config %q", f.SelectedPath, f.LegacyPath) + default: + return "" + } +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..c79cbe5 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,289 @@ +package config + +import ( + "os" + "path/filepath" + "slices" + "testing" +) + +func TestResolvePathsUsesOnlyAbsoluteXDGValues(t *testing.T) { + home := t.TempDir() + env := map[string]string{ + "XDG_CONFIG_HOME": filepath.Join(home, "xdg-config"), + "XDG_STATE_HOME": "relative-state", + } + paths, err := ResolvePaths(home, mapLookup(env)) + if err != nil { + t.Fatal(err) + } + if got, want := paths.ConfigPath, filepath.Join(home, "xdg-config", "mattermost-cli", "config.toml"); got != want { + t.Fatalf("ConfigPath = %q, want %q", got, want) + } + if got, want := paths.StateDir, filepath.Join(home, ".local", "state", "mattermost-cli"); got != want { + t.Fatalf("StateDir = %q, want %q", got, want) + } +} + +func TestLoadFallsBackToLegacyWithoutMovingIt(t *testing.T) { + home := t.TempDir() + selected := filepath.Join(home, "selected", "mattermost-cli", "config.toml") + legacy := filepath.Join(home, ".config", "mattermost-cli", "config.toml") + writeConfig(t, legacy, `url = "https://legacy.example"`, 0o600) + + state := Load(Paths{ConfigPath: selected, LegacyPath: legacy}) + + if state.Migration != MigrationLegacyFallback || state.ReadPath != legacy || state.SelectedPath != selected { + t.Fatalf("Load() state = %+v, want legacy fallback", state) + } + if state.Config.URL != "https://legacy.example" { + t.Fatalf("URL = %q, want legacy URL", state.Config.URL) + } + if _, err := os.Stat(selected); !os.IsNotExist(err) { + t.Fatalf("selected config unexpectedly created: %v", err) + } +} + +func TestInitCreatesSecureTemplateWithoutOverwrite(t *testing.T) { + path := filepath.Join(t.TempDir(), "nested", "config.toml") + created, err := Init(path) + if err != nil { + t.Fatal(err) + } + if !created { + t.Fatal("Init() created = false, want true") + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Fatalf("config mode = %#o, want 0600", got) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(data) != Template { + t.Fatalf("config template = %q, want exact template", data) + } + if err := os.WriteFile(path, []byte("keep me"), 0o600); err != nil { + t.Fatal(err) + } + created, err = Init(path) + if err != nil || created { + t.Fatalf("second Init() = (%v, %v), want (false, nil)", created, err) + } + data, err = os.ReadFile(path) + if err != nil || string(data) != "keep me" { + t.Fatalf("second Init() changed file: data=%q err=%v", data, err) + } +} + +func TestInitRejectsUnsafeExistingPath(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "target.toml") + link := filepath.Join(root, "config.toml") + writeConfig(t, target, "keep me", 0o600) + if err := os.Symlink(target, link); err != nil { + t.Fatal(err) + } + + created, err := Init(link) + + if err == nil || created { + t.Fatalf("Init() = (%v, %v), want unsafe-path failure", created, err) + } + data, readErr := os.ReadFile(target) + if readErr != nil || string(data) != "keep me" { + t.Fatalf("Init() changed symlink target: data=%q err=%v", data, readErr) + } +} + +func TestInitRejectsExistingTokenWithInsecurePermissions(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + writeConfig(t, path, `token = "fixture-token"`, 0o644) + + created, err := Init(path) + + if err == nil || created { + t.Fatalf("Init() = (%v, %v), want insecure-token failure", created, err) + } +} + +func TestLoadRejectsSymlinkedAncestorInsideUserDirectory(t *testing.T) { + root := t.TempDir() + realDirectory := filepath.Join(root, "real") + linkDirectory := filepath.Join(root, "linked") + path := filepath.Join(realDirectory, "config.toml") + writeConfig(t, path, `token = "fixture-token"`, 0o600) + if err := os.Symlink(realDirectory, linkDirectory); err != nil { + t.Fatal(err) + } + + state := Load(Paths{ + ConfigPath: filepath.Join(linkDirectory, "config.toml"), + LegacyPath: filepath.Join(linkDirectory, "config.toml"), + }) + + if state.Error != FileErrorRead || state.Unsafe != UnsafeType || state.Config.Token != "" { + t.Fatalf("Load() ancestor-symlink state = %+v, want fail closed", state) + } + if created, err := Init(filepath.Join(linkDirectory, "new.toml")); err == nil || created { + t.Fatalf("Init() through ancestor symlink = (%v, %v), want failure", created, err) + } +} + +func TestLoadSelectedPathIsAuthoritative(t *testing.T) { + home := t.TempDir() + selected := filepath.Join(home, "selected", "config.toml") + legacy := filepath.Join(home, "legacy", "config.toml") + writeConfig(t, selected, `url = "https://selected.example"`, 0o600) + writeConfig(t, legacy, `url = "https://legacy.example"`, 0o600) + + state := Load(Paths{ConfigPath: selected, LegacyPath: legacy}) + + if state.Migration != MigrationLegacyIgnored || state.Config.URL != "https://selected.example" { + t.Fatalf("Load() state = %+v, want selected authoritative", state) + } +} + +func TestLoadParsesSupportedFieldsAndIgnoresWrongTypes(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + writeConfig(t, path, ` +url = " https://mattermost.example/chat/ " +token = " fixture-token " +redact = false +mention_names = [" Arda ", "", 42, "arda.sevinc"] +unknown = "ignored" +`, 0o600) + + state := Load(Paths{ConfigPath: path, LegacyPath: path}) + + if state.Error != "" { + t.Fatalf("Load() error = %q", state.Error) + } + if state.Config.URL != "https://mattermost.example/chat/" || state.Config.Token != "fixture-token" { + t.Fatalf("Load() config = %+v", state.Config) + } + if state.Config.Redact == nil || *state.Config.Redact { + t.Fatalf("Redact = %v, want false", state.Config.Redact) + } + if !slices.Equal(state.Config.MentionNames, []string{"Arda", "arda.sevinc"}) { + t.Fatalf("MentionNames = %q", state.Config.MentionNames) + } +} + +func TestLoadCharacterizesMissingReadParseAndPermissions(t *testing.T) { + root := t.TempDir() + missing := filepath.Join(root, "missing.toml") + if state := Load(Paths{ConfigPath: missing, LegacyPath: missing}); state.Exists || state.Error != "" { + t.Fatalf("missing state = %+v", state) + } + if state := Load(Paths{ConfigPath: root, LegacyPath: root}); !state.Exists || state.Error != FileErrorRead { + t.Fatalf("directory state = %+v", state) + } + malformed := filepath.Join(root, "malformed.toml") + writeConfig(t, malformed, `not valid = [toml`, 0o600) + if state := Load(Paths{ConfigPath: malformed, LegacyPath: malformed}); state.Error != FileErrorParse { + t.Fatalf("malformed state = %+v", state) + } + insecure := filepath.Join(root, "insecure.toml") + writeConfig(t, insecure, `token = "secret"`, 0o640) + if state := Load(Paths{ConfigPath: insecure, LegacyPath: insecure}); !state.InsecurePermissions { + t.Fatalf("insecure state = %+v", state) + } +} + +func TestLoadIgnoresUnsupportedTopLevelTypes(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + writeConfig(t, path, `url = 42 +token = false +redact = "false" +mention_names = "Arda" +unknown = "ignored" +`, 0o600) + + state := Load(Paths{ConfigPath: path, LegacyPath: path}) + + if state.Error != "" || state.Config.URL != "" || state.Config.Token != "" || state.Config.Redact != nil || state.Config.MentionNames != nil { + t.Fatalf("Load() state = %+v, want unsupported fields ignored", state) + } +} + +func TestLoadRejectsSymlinkedConfig(t *testing.T) { + root := t.TempDir() + target := filepath.Join(root, "target.toml") + link := filepath.Join(root, "config.toml") + writeConfig(t, target, `token = "fixture-token"`, 0o600) + if err := os.Symlink(target, link); err != nil { + t.Fatal(err) + } + + state := Load(Paths{ConfigPath: link, LegacyPath: link}) + + if !state.Exists || state.Error != FileErrorRead || state.Unsafe != UnsafeType || state.Config.Token != "" { + t.Fatalf("Load() symlink state = %+v, want fail-closed read error", state) + } +} + +func TestResolvePreservesPrecedenceAndRedactSemantics(t *testing.T) { + fileRedact := false + file := FileState{Config: File{ + URL: "https://file.example", Token: "file-token", Redact: &fileRedact, + MentionNames: []string{"Arda"}, + }} + env := map[string]string{ + "MM_URL": "https://env.example", "MM_TOKEN": "env-token", "MM_REDACT": "", + } + cliRedact := false + resolved := Resolve(Options{URL: "https://cli.example", Redact: &cliRedact}, mapLookup(env), file) + + if resolved.URLSource != SourceCLI || resolved.TokenSource != SourceEnv || resolved.RedactSource != SourceCLI { + t.Fatalf("Resolve() sources = %s/%s/%s", resolved.URLSource, resolved.TokenSource, resolved.RedactSource) + } + if resolved.URL != "https://cli.example" || resolved.Token != "env-token" || resolved.Redact { + t.Fatalf("Resolve() = %+v", resolved) + } + + resolved = Resolve(Options{}, mapLookup(env), file) + if !resolved.Redact || resolved.RedactSource != SourceEnv { + t.Fatalf("empty MM_REDACT = (%v, %s), want (true, env)", resolved.Redact, resolved.RedactSource) + } + env["MM_REDACT"] = "false" + resolved = Resolve(Options{}, mapLookup(env), file) + if resolved.Redact { + t.Fatal("MM_REDACT=false did not disable redaction") + } +} + +func TestResolveEmptyURLAndTokenOverridesFallThrough(t *testing.T) { + file := FileState{Config: File{URL: "https://file.example", Token: "file-token"}} + env := map[string]string{"MM_URL": "", "MM_TOKEN": ""} + + resolved := Resolve(Options{}, mapLookup(env), file) + + if resolved.URLSource != SourceFile || resolved.TokenSource != SourceFile || resolved.URL != file.Config.URL || resolved.Token != file.Config.Token { + t.Fatalf("Resolve() = %+v, want file fallbacks", resolved) + } +} + +func mapLookup(values map[string]string) LookupEnv { + return func(name string) (string, bool) { + value, ok := values[name] + return value, ok + } +} + +func writeConfig(t *testing.T, path, content string, mode os.FileMode) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), mode); err != nil { + t.Fatal(err) + } + if err := os.Chmod(path, mode); err != nil { + t.Fatal(err) + } +} diff --git a/internal/config/paths.go b/internal/config/paths.go new file mode 100644 index 0000000..7da7b87 --- /dev/null +++ b/internal/config/paths.go @@ -0,0 +1,34 @@ +package config + +import ( + "fmt" + "path/filepath" +) + +type LookupEnv func(string) (string, bool) + +type Paths struct { + ConfigPath string + LegacyPath string + StateDir string +} + +func ResolvePaths(home string, lookup LookupEnv) (Paths, error) { + if !filepath.IsAbs(home) { + return Paths{}, fmt.Errorf("home directory must be absolute") + } + legacy := filepath.Join(home, ".config", "mattermost-cli", "config.toml") + configPath := legacy + if value, ok := lookup("XDG_CONFIG_HOME"); ok && filepath.IsAbs(value) { + configPath = filepath.Join(value, "mattermost-cli", "config.toml") + } + stateRoot := filepath.Join(home, ".local", "state") + if value, ok := lookup("XDG_STATE_HOME"); ok && filepath.IsAbs(value) { + stateRoot = value + } + return Paths{ + ConfigPath: configPath, + LegacyPath: legacy, + StateDir: filepath.Join(stateRoot, "mattermost-cli"), + }, nil +} diff --git a/internal/config/secure_other.go b/internal/config/secure_other.go new file mode 100644 index 0000000..dbb7922 --- /dev/null +++ b/internal/config/secure_other.go @@ -0,0 +1,16 @@ +//go:build !darwin && !linux + +package config + +import ( + "errors" + "os" +) + +func openConfigFile(_ string) (*os.File, os.FileInfo, UnsafeReason, error) { + return nil, nil, UnsafeUnsupported, errors.New("secure config access is unsupported") +} + +func createConfigFile(_ string) (*os.File, error) { + return nil, errors.New("secure config access is unsupported") +} diff --git a/internal/config/secure_unix.go b/internal/config/secure_unix.go new file mode 100644 index 0000000..a2b71e7 --- /dev/null +++ b/internal/config/secure_unix.go @@ -0,0 +1,140 @@ +//go:build darwin || linux + +package config + +import ( + "errors" + "os" + "path/filepath" + "strings" + + "golang.org/x/sys/unix" +) + +var errUnsafePath = errors.New("unsafe config path") + +func openConfigFile(path string) (*os.File, os.FileInfo, UnsafeReason, error) { + directory, name, unsafe, err := walkConfigParent(path, false) + if err != nil { + return nil, nil, unsafe, err + } + defer func() { _ = unix.Close(directory) }() + fd, err := unix.Openat(directory, name, unix.O_RDONLY|unix.O_CLOEXEC|unix.O_NOFOLLOW|unix.O_NONBLOCK, 0) + if err != nil { + unsafe, classified := classifyOpenError(err) + return nil, nil, unsafe, classified + } + file := os.NewFile(uintptr(fd), "mattermost-cli-config") + var stat unix.Stat_t + if err := unix.Fstat(fd, &stat); err != nil { + _ = file.Close() + return nil, nil, "", err + } + info, err := file.Stat() + if err != nil { + _ = file.Close() + return nil, nil, "", err + } + if !info.Mode().IsRegular() { + _ = file.Close() + return nil, nil, UnsafeType, errUnsafePath + } + if stat.Uid != uint32(os.Geteuid()) { + _ = file.Close() + return nil, nil, UnsafeOwnership, errUnsafePath + } + return file, info, "", nil +} + +func createConfigFile(path string) (*os.File, error) { + directory, name, _, err := walkConfigParent(path, true) + if err != nil { + return nil, err + } + defer func() { _ = unix.Close(directory) }() + fd, err := unix.Openat(directory, name, unix.O_WRONLY|unix.O_CREAT|unix.O_EXCL|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0o600) + if err != nil { + return nil, err + } + return os.NewFile(uintptr(fd), "mattermost-cli-config"), nil +} + +func walkConfigParent(path string, create bool) (int, string, UnsafeReason, error) { + if !filepath.IsAbs(path) { + return -1, "", UnsafeType, errUnsafePath + } + clean := filepath.Clean(path) + parts := strings.Split(strings.TrimPrefix(clean, string(filepath.Separator)), string(filepath.Separator)) + if len(parts) == 0 || parts[len(parts)-1] == "" || parts[len(parts)-1] == "." { + return -1, "", UnsafeType, errUnsafePath + } + current, err := unix.Open(string(filepath.Separator), unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) + if err != nil { + return -1, "", "", err + } + boundary := false + for _, part := range parts[:len(parts)-1] { + next, nextBoundary, unsafe, err := openConfigDirectory(current, part, boundary, create) + _ = unix.Close(current) + if err != nil { + return -1, "", unsafe, err + } + current = next + boundary = nextBoundary + } + return current, parts[len(parts)-1], "", nil +} + +func openConfigDirectory(parent int, name string, boundary, create bool) (int, bool, UnsafeReason, error) { + var entry unix.Stat_t + err := unix.Fstatat(parent, name, &entry, unix.AT_SYMLINK_NOFOLLOW) + if errors.Is(err, unix.ENOENT) && create { + if err := unix.Mkdirat(parent, name, 0o700); err != nil && !errors.Is(err, unix.EEXIST) { + return -1, boundary, "", err + } + err = unix.Fstatat(parent, name, &entry, unix.AT_SYMLINK_NOFOLLOW) + } + if err != nil { + return -1, boundary, "", err + } + isSymlink := entry.Mode&unix.S_IFMT == unix.S_IFLNK + currentUser := uint32(os.Geteuid()) + if isSymlink && (boundary || (currentUser != 0 && entry.Uid == currentUser)) { + return -1, boundary, UnsafeType, errUnsafePath + } + flags := unix.O_RDONLY | unix.O_DIRECTORY | unix.O_CLOEXEC + if !isSymlink { + flags |= unix.O_NOFOLLOW + } + fd, err := unix.Openat(parent, name, flags, 0) + if err != nil { + unsafe, classified := classifyOpenError(err) + return -1, boundary, unsafe, classified + } + var opened unix.Stat_t + if err := unix.Fstat(fd, &opened); err != nil { + _ = unix.Close(fd) + return -1, boundary, "", err + } + if opened.Mode&unix.S_IFMT != unix.S_IFDIR { + _ = unix.Close(fd) + return -1, boundary, UnsafeType, errUnsafePath + } + if opened.Uid != 0 && opened.Uid != currentUser { + _ = unix.Close(fd) + return -1, boundary, UnsafeOwnership, errUnsafePath + } + nextBoundary := boundary || (opened.Uid == currentUser && (currentUser != 0 || opened.Mode&0o077 == 0)) + if nextBoundary && opened.Mode&0o022 != 0 { + _ = unix.Close(fd) + return -1, boundary, UnsafeOwnership, errUnsafePath + } + return fd, nextBoundary, "", nil +} + +func classifyOpenError(err error) (UnsafeReason, error) { + if errors.Is(err, unix.ELOOP) || errors.Is(err, unix.ENOTDIR) { + return UnsafeType, errUnsafePath + } + return "", err +} diff --git a/internal/config/secure_unix_test.go b/internal/config/secure_unix_test.go new file mode 100644 index 0000000..8a909bd --- /dev/null +++ b/internal/config/secure_unix_test.go @@ -0,0 +1,35 @@ +//go:build darwin || linux + +package config + +import ( + "path/filepath" + "testing" + "time" + + "golang.org/x/sys/unix" +) + +func TestLoadRejectsFIFOWithoutBlocking(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + if err := unix.Mkfifo(path, 0o600); err != nil { + t.Fatal(err) + } + done := make(chan FileState, 1) + go func() { + done <- Load(Paths{ConfigPath: path, LegacyPath: path}) + }() + + select { + case state := <-done: + if state.Error != FileErrorRead || state.Unsafe != UnsafeType { + t.Fatalf("Load() FIFO state = %+v, want unsafe type", state) + } + case <-time.After(time.Second): + t.Fatal("Load() blocked opening FIFO") + } + created, err := Init(path) + if err == nil || created { + t.Fatalf("Init() FIFO = (%v, %v), want unsafe-path failure", created, err) + } +} diff --git a/internal/serverurl/url.go b/internal/serverurl/url.go new file mode 100644 index 0000000..c007335 --- /dev/null +++ b/internal/serverurl/url.go @@ -0,0 +1,147 @@ +package serverurl + +import ( + "errors" + "net/netip" + "net/url" + "strconv" + "strings" + + "golang.org/x/net/idna" +) + +var ( + ErrInvalid = errors.New("invalid Mattermost URL") + ErrAmbiguous = errors.New("Mattermost URL cannot contain credentials, query strings, or fragments") + ErrScheme = errors.New("Mattermost URL must use HTTPS, or HTTP on a loopback host") + ErrPlaintext = errors.New("refusing to send a Mattermost token over plaintext HTTP; use HTTPS or a loopback URL") +) + +func Normalize(input string) (string, error) { + value := strings.TrimSpace(input) + if strings.Contains(value, "\\") { + return "", ErrInvalid + } + parsed, err := url.Parse(value) + if err != nil || parsed.Scheme == "" || parsed.Host == "" || parsed.Opaque != "" { + return "", ErrInvalid + } + if parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" { + return "", ErrAmbiguous + } + scheme := strings.ToLower(parsed.Scheme) + if scheme != "https" && scheme != "http" { + return "", ErrScheme + } + rawHostname := strings.ToLower(parsed.Hostname()) + hostname := "" + if address, parseErr := netip.ParseAddr(rawHostname); parseErr == nil { + hostname = address.String() + } else { + if looksLikeAmbiguousIPv4(rawHostname) { + return "", ErrInvalid + } + hostname, err = idna.Lookup.ToASCII(rawHostname) + } + if err != nil || hostname == "" || strings.ContainsAny(hostname, "\\\x00\r\n\t") { + return "", ErrInvalid + } + port := parsed.Port() + if port != "" { + portNumber, err := strconv.Atoi(port) + if err != nil || portNumber < 0 || portNumber > 65535 { + return "", ErrInvalid + } + port = strconv.Itoa(portNumber) + } + if scheme == "http" && !isLoopback(hostname) { + return "", ErrPlaintext + } + if (scheme == "https" && numericPort(port) == 443) || (scheme == "http" && numericPort(port) == 80) { + port = "" + } + return strings.TrimRight(scheme+"://"+canonicalHost(hostname, port)+normalizePath(parsed.EscapedPath()), "/"), nil +} + +func looksLikeAmbiguousIPv4(hostname string) bool { + hostname = strings.TrimSuffix(hostname, ".") + parts := strings.Split(hostname, ".") + if len(parts) < 1 || len(parts) > 4 { + return false + } + for _, part := range parts { + if part == "" { + return false + } + if strings.HasPrefix(strings.ToLower(part), "0x") { + if len(part) == 2 { + return false + } + for _, char := range part[2:] { + if !((char >= '0' && char <= '9') || (char >= 'a' && char <= 'f') || (char >= 'A' && char <= 'F')) { + return false + } + } + continue + } + for _, char := range part { + if char < '0' || char > '9' { + return false + } + } + } + return true +} + +func numericPort(port string) int { + value, err := strconv.Atoi(port) + if err != nil { + return -1 + } + return value +} + +func normalizePath(escapedPath string) string { + segments := strings.Split(escapedPath, "/") + normalized := make([]string, 0, len(segments)) + for _, segment := range segments { + switch strings.ToLower(segment) { + case ".", "%2e": + continue + case "..", ".%2e", "%2e.", "%2e%2e": + if len(normalized) > 1 { + normalized = normalized[:len(normalized)-1] + } + default: + normalized = append(normalized, segment) + } + } + return strings.Join(normalized, "/") +} + +func BuildPostPermalink(baseURL, postID string) (string, error) { + normalized, err := Normalize(baseURL) + if err != nil { + return "", err + } + return normalized + "/_redirect/pl/" + url.PathEscape(postID), nil +} + +func isLoopback(hostname string) bool { + if hostname == "localhost" { + return true + } + address, err := netip.ParseAddr(hostname) + return err == nil && address.IsLoopback() +} + +func canonicalHost(hostname, port string) string { + host := hostname + if strings.Contains(hostname, ":") { + host = "[" + hostname + "]" + } + if port != "" { + host += ":" + port + } + return host +} diff --git a/internal/serverurl/url_test.go b/internal/serverurl/url_test.go new file mode 100644 index 0000000..c592f98 --- /dev/null +++ b/internal/serverurl/url_test.go @@ -0,0 +1,136 @@ +package serverurl + +import ( + "errors" + "strings" + "testing" +) + +func TestNormalizeAllowsHTTPSAndLoopbackHTTP(t *testing.T) { + for _, input := range []string{ + "https://mattermost.example.com", + "https://mattermost.example.com:8443/chat", + "http://localhost:8065", + "http://127.0.0.1:8065", + "http://127.0.0.2:8065", + "http://[::1]:8065", + } { + t.Run(input, func(t *testing.T) { + if _, err := Normalize(input); err != nil { + t.Fatalf("Normalize() error = %v", err) + } + }) + } +} + +func TestNormalizeRejectsRemotePlaintext(t *testing.T) { + for _, input := range []string{ + "http://mattermost.example.com", + "http://localhost.evil:8065", + "http://127.evil:8065", + } { + t.Run(input, func(t *testing.T) { + if _, err := Normalize(input); !errors.Is(err, ErrPlaintext) { + t.Fatalf("Normalize() error = %v, want ErrPlaintext", err) + } + }) + } +} + +func TestNormalizeRejectsUnsafeOrAmbiguousURLsWithoutReflection(t *testing.T) { + secret := "secret-token-value" + tests := []struct { + input string + want error + }{ + {"not a url with " + secret, ErrInvalid}, + {"ftp://mattermost.example.com", ErrScheme}, + {"https://user:" + secret + "@mattermost.example.com", ErrAmbiguous}, + {"https://mattermost.example.com?token=" + secret, ErrAmbiguous}, + {"https://mattermost.example.com#" + secret, ErrAmbiguous}, + {"https://mattermost.example.com:invalid", ErrInvalid}, + {"https://mattermost.example.com:65536", ErrInvalid}, + {"http://127.0.0.256:8065", ErrInvalid}, + {"http://127.000.000.001:8065", ErrInvalid}, + {"http://127.1:8065", ErrInvalid}, + {"https://127.000.000.001", ErrInvalid}, + {"https://127.1", ErrInvalid}, + {"https://0177.0.0.1", ErrInvalid}, + {"https://0x7f.0.0.1", ErrInvalid}, + {"https://127.0.0.1.", ErrInvalid}, + {"https://127.000.000.001.", ErrInvalid}, + {"https://127.1.", ErrInvalid}, + {"https://0x7f.", ErrInvalid}, + {"https://2130706433.", ErrInvalid}, + {"https:\\mattermost.example.com", ErrInvalid}, + } + for _, test := range tests { + _, err := Normalize(test.input) + if !errors.Is(err, test.want) { + t.Errorf("Normalize(%q) error = %v, want %v", test.input, err, test.want) + } + if err != nil && strings.Contains(err.Error(), secret) { + t.Errorf("Normalize() reflected secret in error: %v", err) + } + } +} + +func TestNormalizeCanonicalizesWithoutLosingBasePath(t *testing.T) { + tests := map[string]string{ + " HTTPS://Mattermost.Example.Com/// ": "https://mattermost.example.com", + "https://Mattermost.Example.Com/chat/": "https://mattermost.example.com/chat", + "https://mattermost.example.com?": "https://mattermost.example.com", + "https://mattermost.example.com#": "https://mattermost.example.com", + "https://mattermost.example.com:0443/": "https://mattermost.example.com", + "http://localhost:080/": "http://localhost", + "https://mattermost.example.com:00081/": "https://mattermost.example.com:81", + "https://bücher.example/": "https://xn--bcher-kva.example", + "https://mattermost.example.com/a/../b/": "https://mattermost.example.com/b", + "https://mattermost.example.com/%2e%2e/b": "https://mattermost.example.com/b", + } + for input, want := range tests { + got, err := Normalize(input) + if err != nil { + t.Fatalf("Normalize(%q) error = %v", input, err) + } + if got != want { + t.Fatalf("Normalize(%q) = %q, want %q", input, got, want) + } + } +} + +func TestBuildPostPermalinkEncodesIDAndPreservesBasePath(t *testing.T) { + got, err := BuildPostPermalink("https://mattermost.example.com/chat///", "post(special)") + if err != nil { + t.Fatal(err) + } + if want := "https://mattermost.example.com/chat/_redirect/pl/post%28special%29"; got != want { + t.Fatalf("BuildPostPermalink() = %q, want %q", got, want) + } +} + +func FuzzNormalizeIsStable(f *testing.F) { + for _, seed := range []string{ + "https://mattermost.example.com", + "http://127.0.0.2:8065/chat/", + "https://bücher.example/a/../b", + "https://127.000.000.001", + "https://user:secret@example.com", + "not a url", + } { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, input string) { + first, err := Normalize(input) + if err != nil { + return + } + second, err := Normalize(first) + if err != nil { + t.Fatalf("Normalize(normalized) error = %v; normalized = %q", err, first) + } + if second != first { + t.Fatalf("Normalize() is not stable: first %q, second %q", first, second) + } + }) +} From cffe4b39041bd411a632dbe4b51136a6df62e83c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 04:10:04 +0300 Subject: [PATCH 007/119] feat: port credential masking and terminal sanitization --- internal/presentation/credential.go | 82 ++++++++++++++++ internal/presentation/sanitize.go | 127 +++++++++++++++++++++++++ internal/presentation/sanitize_test.go | 99 +++++++++++++++++++ 3 files changed, 308 insertions(+) create mode 100644 internal/presentation/credential.go create mode 100644 internal/presentation/sanitize.go create mode 100644 internal/presentation/sanitize_test.go diff --git a/internal/presentation/credential.go b/internal/presentation/credential.go new file mode 100644 index 0000000..0ae49d6 --- /dev/null +++ b/internal/presentation/credential.go @@ -0,0 +1,82 @@ +package presentation + +import "sync" + +type credentialOwner struct { + id uint64 + value string +} + +type CredentialRegistry struct { + mu sync.RWMutex + nextID uint64 + owners []credentialOwner +} + +func (r *CredentialRegistry) SetDefault(value string) { + if value == "" { + return + } + r.mu.Lock() + defer r.mu.Unlock() + for index := range r.owners { + if r.owners[index].id == 0 { + r.owners[index].value = value + return + } + } + r.owners = append(r.owners, credentialOwner{value: value}) +} + +func (r *CredentialRegistry) Clear() { + r.mu.Lock() + defer r.mu.Unlock() + r.owners = nil +} + +func (r *CredentialRegistry) Register(value string) func() { + if value == "" { + return func() {} + } + r.mu.Lock() + r.nextID++ + id := r.nextID + r.owners = append(r.owners, credentialOwner{id: id, value: value}) + r.mu.Unlock() + var once sync.Once + return func() { + once.Do(func() { + r.mu.Lock() + defer r.mu.Unlock() + for index := range r.owners { + if r.owners[index].id == id { + r.owners = append(r.owners[:index], r.owners[index+1:]...) + return + } + } + }) + } +} + +func (r *CredentialRegistry) Values() []string { + r.mu.RLock() + defer r.mu.RUnlock() + values := make([]string, 0, len(r.owners)) + seen := make(map[string]struct{}, len(r.owners)) + appendValue := func(value string) { + if value == "" { + return + } + if _, exists := seen[value]; exists { + return + } + seen[value] = struct{}{} + values = append(values, value) + } + for _, owner := range r.owners { + appendValue(owner.value) + } + return values +} + +var ActiveCredentials CredentialRegistry diff --git a/internal/presentation/sanitize.go b/internal/presentation/sanitize.go new file mode 100644 index 0000000..c88fa88 --- /dev/null +++ b/internal/presentation/sanitize.go @@ -0,0 +1,127 @@ +package presentation + +import ( + "fmt" + "sort" + "strings" + "unicode/utf16" +) + +const credentialMask = "[REDACTED:mattermost_credential]" + +type Redaction struct { + Type string `json:"type"` + Masked string `json:"masked"` + Position int `json:"position"` + Field string `json:"field,omitempty"` +} + +type Result struct { + Text string `json:"text"` + Redactions []Redaction `json:"redactions"` +} + +type span struct { + start int + end int +} + +func SanitizeControls(text string) string { + text = strings.ReplaceAll(text, "\r\n", "\n") + var output strings.Builder + output.Grow(len(text)) + for _, character := range text { + if unsafeControl(character) { + _, _ = fmt.Fprintf(&output, "\\u%04x", character) + continue + } + output.WriteRune(character) + } + return output.String() +} + +func SanitizeLabel(text string) string { + return strings.NewReplacer("\n", "\\n", "\t", "\\t").Replace(SanitizeControls(text)) +} + +func Preprocess(text string, credentials []string) Result { + spans := credentialSpans(text, credentials) + if len(spans) == 0 { + return Result{Text: SanitizeControls(text), Redactions: []Redaction{}} + } + var raw strings.Builder + raw.Grow(len(text)) + redactions := make([]Redaction, 0, len(spans)) + positions := make([]int, 0, len(spans)) + cursor := 0 + for _, current := range spans { + raw.WriteString(text[cursor:current.start]) + positions = append(positions, raw.Len()) + raw.WriteString(credentialMask) + redactions = append(redactions, Redaction{Type: "mattermost_credential", Masked: credentialMask}) + cursor = current.end + } + raw.WriteString(text[cursor:]) + rawText := raw.String() + for index, position := range positions { + redactions[index].Position = utf16Length(SanitizeControls(rawText[:position])) + } + return Result{Text: SanitizeControls(rawText), Redactions: redactions} +} + +func PreprocessActive(text string) Result { + return Preprocess(text, ActiveCredentials.Values()) +} + +func credentialSpans(text string, credentials []string) []span { + var spans []span + seenCredential := make(map[string]struct{}, len(credentials)) + for _, credential := range credentials { + if credential == "" { + continue + } + if _, seen := seenCredential[credential]; seen { + continue + } + seenCredential[credential] = struct{}{} + for offset := 0; offset <= len(text)-len(credential); { + index := strings.Index(text[offset:], credential) + if index < 0 { + break + } + start := offset + index + spans = append(spans, span{start: start, end: start + len(credential)}) + offset = start + len(credential) + } + } + sort.Slice(spans, func(i, j int) bool { + if spans[i].start != spans[j].start { + return spans[i].start < spans[j].start + } + return spans[i].end > spans[j].end + }) + merged := make([]span, 0, len(spans)) + for _, current := range spans { + if len(merged) == 0 || current.start >= merged[len(merged)-1].end { + merged = append(merged, current) + continue + } + if current.end > merged[len(merged)-1].end { + merged[len(merged)-1].end = current.end + } + } + return merged +} + +func unsafeControl(character rune) bool { + return (character >= 0x0000 && character <= 0x0008) || + (character >= 0x000b && character <= 0x001f) || + (character >= 0x007f && character <= 0x009f) || + character == 0x061c || character == 0x200e || character == 0x200f || + (character >= 0x202a && character <= 0x202e) || + (character >= 0x2066 && character <= 0x2069) +} + +func utf16Length(text string) int { + return len(utf16.Encode([]rune(text))) +} diff --git a/internal/presentation/sanitize_test.go b/internal/presentation/sanitize_test.go new file mode 100644 index 0000000..2fc653c --- /dev/null +++ b/internal/presentation/sanitize_test.go @@ -0,0 +1,99 @@ +package presentation + +import ( + "encoding/json" + "slices" + "strings" + "testing" +) + +func TestSanitizeControlsMakesTerminalHazardsVisible(t *testing.T) { + input := "before\x1b[2Jmiddle\x1b]52;c;YXR0YWNrZXI=\aafter" + want := `before\u001b[2Jmiddle\u001b]52;c;YXR0YWNrZXI=\u0007after` + if got := SanitizeControls(input); got != want { + t.Fatalf("SanitizeControls() = %q, want %q", got, want) + } +} + +func TestSanitizeControlsNormalizesOnlyCRLFAndPreservesUnicode(t *testing.T) { + input := "first\r\nsecond\rforged\n\tindented 👩‍💻 café العربية\u202e" + want := "first\nsecond\\u000dforged\n\tindented 👩‍💻 café العربية\\u202e" + if got := SanitizeControls(input); got != want { + t.Fatalf("SanitizeControls() = %q, want %q", got, want) + } + if got := SanitizeLabel("line\n\tvalue"); got != `line\n\tvalue` { + t.Fatalf("SanitizeLabel() = %q", got) + } +} + +func TestCredentialRegistryOwnershipAndReplacement(t *testing.T) { + var registry CredentialRegistry + registry.SetDefault("default-one") + registry.SetDefault("") + releaseA := registry.Register("shared") + releaseB := registry.Register("shared") + registry.SetDefault("default-two") + if got := registry.Values(); !slices.Equal(got, []string{"default-two", "shared"}) { + t.Fatalf("Values() = %q", got) + } + releaseA() + if got := registry.Values(); !slices.Equal(got, []string{"default-two", "shared"}) { + t.Fatalf("Values() after one release = %q", got) + } + releaseB() + releaseB() + if got := registry.Values(); !slices.Equal(got, []string{"default-two"}) { + t.Fatalf("Values() after release = %q", got) + } + registry.Clear() + if len(registry.Values()) != 0 { + t.Fatal("Clear() retained credentials") + } +} + +func TestCredentialRegistryPreservesOwnerInsertionOrder(t *testing.T) { + var registry CredentialRegistry + release := registry.Register("socket") + defer release() + registry.SetDefault("config") + registry.SetDefault("replacement") + if got := registry.Values(); !slices.Equal(got, []string{"socket", "replacement"}) { + t.Fatalf("Values() = %q", got) + } +} + +func TestPreprocessMasksExactCredentialAndOmitsOriginal(t *testing.T) { + const credential = "short" + result := Preprocess("before short after short", []string{credential}) + want := "before " + credentialMask + " after " + credentialMask + if result.Text != want || len(result.Redactions) != 2 { + t.Fatalf("Preprocess() = %+v, want %q", result, want) + } + encoded, err := json.Marshal(result.Redactions) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(encoded), credential) { + t.Fatalf("redaction provenance leaked credential: %s", encoded) + } +} + +func TestPreprocessMergesOverlappingCredentialsAndUsesUTF16Position(t *testing.T) { + result := Preprocess("👩‍💻 xxabcdefyy", []string{"abcdef", "cdefyy"}) + if got, want := result.Text, "👩‍💻 xx"+credentialMask; got != want { + t.Fatalf("Text = %q, want %q", got, want) + } + if got, want := result.Redactions[0].Position, 8; got != want { + t.Fatalf("Position = %d, want UTF-16 offset %d", got, want) + } +} + +func TestPreprocessSanitizesPrefixBeforeComputingPosition(t *testing.T) { + result := Preprocess("a\r\n\x1btoken", []string{"token"}) + if got, want := result.Text, "a\n\\u001b"+credentialMask; got != want { + t.Fatalf("Text = %q, want %q", got, want) + } + if got, want := result.Redactions[0].Position, len("a\n\\u001b"); got != want { + t.Fatalf("Position = %d, want %d", got, want) + } +} From 39f55ad64f1958559f6e16872ed588f208739796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 04:36:20 +0300 Subject: [PATCH 008/119] feat: port heuristic secret redaction --- internal/presentation/fuzz_test.go | 58 +++++++++ internal/presentation/patterns.go | 160 +++++++++++++++++++++++++ internal/presentation/patterns_test.go | 153 +++++++++++++++++++++++ internal/presentation/sanitize.go | 93 +++++++++++--- internal/presentation/sanitize_test.go | 41 +++++++ 5 files changed, 486 insertions(+), 19 deletions(-) create mode 100644 internal/presentation/fuzz_test.go create mode 100644 internal/presentation/patterns.go create mode 100644 internal/presentation/patterns_test.go diff --git a/internal/presentation/fuzz_test.go b/internal/presentation/fuzz_test.go new file mode 100644 index 0000000..4bc0bcc --- /dev/null +++ b/internal/presentation/fuzz_test.go @@ -0,0 +1,58 @@ +package presentation + +import ( + "encoding/hex" + "encoding/json" + "strings" + "testing" +) + +func FuzzSanitizeControlsIsIdempotentAndRemovesUnsafeCodePoints(f *testing.F) { + for _, seed := range []string{ + "ordinary text", + "first\r\nsecond\rforged\x1b[2J", + "مرحبا\u200e 👩‍💻 café", + } { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, input string) { + once := SanitizeControls(input) + if twice := SanitizeControls(once); twice != once { + t.Fatalf("sanitization is not idempotent: once=%q twice=%q", once, twice) + } + for _, character := range once { + if unsafeControl(character) { + t.Fatalf("sanitized output retained unsafe code point U+%04X", character) + } + } + }) +} + +func FuzzPreprocessNeverEmitsExactActiveCredential(f *testing.F) { + f.Add("before ", " after", []byte("token")) + f.Add("👩‍💻\x1b", "\u202eend", []byte{0, 1, 2, 255}) + f.Fuzz(func(t *testing.T, prefix, suffix string, credentialBytes []byte) { + credential := "ACTIVE-" + hex.EncodeToString(credentialBytes) + result := PreprocessWithOptions(prefix+credential+suffix, Options{ + Credentials: []string{credential}, + DisableHeuristics: true, + }) + if strings.Contains(result.Text, credential) { + t.Fatal("preprocessed output retained the exact active credential") + } + encoded, err := json.Marshal(result.Redactions) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(encoded), credential) { + t.Fatal("redaction provenance retained the exact active credential") + } + position := strings.Index(result.Text, credentialMask) + if position < 0 || len(result.Redactions) == 0 { + t.Fatalf("credential mask missing: %+v", result) + } + if got, want := result.Redactions[0].Position, utf16Length(result.Text[:position]); got != want { + t.Fatalf("position = %d, want %d", got, want) + } + }) +} diff --git a/internal/presentation/patterns.go b/internal/presentation/patterns.go new file mode 100644 index 0000000..976a51f --- /dev/null +++ b/internal/presentation/patterns.go @@ -0,0 +1,160 @@ +package presentation + +import ( + "regexp" + "sort" + "strings" + "unicode/utf8" +) + +// DetectedSecret uses byte offsets so callers can safely slice the original +// UTF-8 string. Presentation boundaries may convert these to UTF-16 offsets. +type DetectedSecret struct { + Type string + Value string + Start int + End int +} + +type secretPattern struct { + name string + re *regexp.Regexp + valid func(string, int, int) bool +} + +func pattern(name, expression string) secretPattern { + return secretPattern{name: name, re: regexp.MustCompile(expression)} +} + +const jsWhitespaceClass = `\x{0009}\x{000a}\x{000b}\x{000c}\x{000d}\x{0020}\x{00a0}\x{1680}\x{2000}-\x{200a}\x{2028}\x{2029}\x{202f}\x{205f}\x{3000}\x{feff}` + +var secretPatterns = []secretPattern{ + pattern("aws_access_key", `\b((?:AKIA|ASIA)[0-9A-Z]{16})\b`), + pattern("aws_secret_key", `(?i)(?:aws[_-]?secret[_-]?(?:access[_-]?)?key|secret[_-]?key)["`+jsWhitespaceClass+`:=]+["']?([A-Za-z0-9/+=]{40})["']?`), + {name: "github_stateless_token", re: regexp.MustCompile(`(ghs_[A-Za-z0-9._-]{36,})`), valid: boundedBy(`[A-Za-z0-9._-]`)}, + pattern("github_token", `\b(gh[pousr]_[A-Za-z0-9_]{36,255})\b`), + pattern("github_oauth", `\b(gho_[A-Za-z0-9]{36,255})\b`), + pattern("github_fine_grained_token", `\b(github_pat_[A-Za-z0-9_]{22,255})\b`), + pattern("gitlab_token", `\b(glpat-[A-Za-z0-9_-]{20,})\b`), + {name: "slack_app_token", re: regexp.MustCompile(`(xapp-[0-9]+-[A-Za-z0-9-]{16,})`), valid: boundedBy(`[A-Za-z0-9-]`)}, + pattern("slack_token", `\b(xox[baprs]-[0-9]{10,13}-[0-9]{10,13}-[a-zA-Z0-9]{24,})\b`), + pattern("slack_webhook", `(https://hooks\.slack\.com/services/T[A-Z0-9]+/B[A-Z0-9]+/[a-zA-Z0-9]+)`), + pattern("discord_token", `\b([MN][A-Za-z\d]{23,}\.[\w-]{6}\.[\w-]{27,})\b`), + pattern("discord_webhook", `(https://discord(?:app)?\.com/api/webhooks/\d+/[A-Za-z0-9_-]+)`), + pattern("jwt", `\b(eyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+)\b`), + pattern("bearer_token", `(?i)\bBearer[`+jsWhitespaceClass+`]+([A-Za-z0-9_.-]{20,})\b`), + pattern("basic_auth", `(?i)\bBasic[`+jsWhitespaceClass+`]+([A-Za-z0-9+/=]{20,})\b`), + pattern("connection_string", `(?i)\b((?:mongodb|postgres|postgresql|mysql|redis|amqp|amqps)://[^:]+:[^@`+jsWhitespaceClass+`]+@[^`+jsWhitespaceClass+`"']+)\b`), + pattern("api_key", `(?i)(?:api[_-]?key|apikey|api[_-]?secret)["`+jsWhitespaceClass+`:=]+["']?([A-Za-z0-9_-]{20,})["']?`), + pattern("password", `(?i)(?:password|passwd|pwd|secret)["`+jsWhitespaceClass+`:=]+["']?([^`+jsWhitespaceClass+`"']{8,})["']?`), + // private_key is handled separately because RE2 has no backreferences. + pattern("mattermost_token", `(?i)(?:\bmm(?:auth)?[_-]?token\b|\bmm[_-]?pat\b|\bmattermost[`+jsWhitespaceClass+`_-]?(?:pat|token)\b|\bpat\b|\btoken\b)["`+jsWhitespaceClass+`:=]+["']?([a-z0-9]{26})["']?`), + pattern("stripe_key", `\b(sk_(?:live|test)_[A-Za-z0-9]{24,})\b`), + pattern("stripe_restricted_key", `\b(rk_(?:live|test)_[A-Za-z0-9]{24,})\b`), + pattern("sendgrid_key", `\b(SG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43})\b`), + pattern("twilio_key", `\b(SK[a-f0-9]{32})\b`), + pattern("openai_key", `\b(sk-[A-Za-z0-9]{32,})\b`), + pattern("openai_project_key", `\b(sk-proj-[A-Za-z0-9_-]{32,})\b`), + pattern("anthropic_key", `\b(sk-ant-[A-Za-z0-9_-]{32,})\b`), + pattern("google_api_key", `\b(AIza[A-Za-z0-9_-]{35})\b`), + pattern("heroku_key", `(?i)(?:heroku[_-]?api[_-]?key|HEROKU_API_KEY)["`+jsWhitespaceClass+`:=]+["']?([A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12})["']?`), + pattern("npm_token", `\b(npm_[A-Za-z0-9]{36})\b`), + pattern("high_entropy_secret", `(?i)(?:token|secret|key|auth|credential)["`+jsWhitespaceClass+`:=]+["']?([A-Za-z0-9_-]{32,64})["']?`), +} + +var privateKeyBegin = regexp.MustCompile(`-----BEGIN ((?:[A-Z0-9]+ )*PRIVATE KEY)-----`) + +func DetectSecrets(text string) []DetectedSecret { + secrets := make([]DetectedSecret, 0) + seen := make(map[[2]int]struct{}) + add := func(kind string, start, end int) { + key := [2]int{start, end} + if _, exists := seen[key]; exists { + return + } + seen[key] = struct{}{} + secrets = append(secrets, DetectedSecret{Type: kind, Value: text[start:end], Start: start, End: end}) + } + + for _, current := range secretPatterns { + for _, match := range current.re.FindAllStringSubmatchIndex(text, -1) { + start, end := match[0], match[1] + if len(match) >= 4 && match[2] >= 0 { + start, end = match[2], match[3] + } + if current.name == "mattermost_token" && end < len(text) && text[end] != '\'' && text[end] != '"' && isASCIIAlnum(text[end]) { + continue + } + if current.valid == nil || current.valid(text, start, end) { + add(current.name, start, end) + } + } + if current.name == "password" { // Preserve the TS pattern order. + for _, match := range privateKeyMatches(text) { + add("private_key", match[0], match[1]) + } + } + } + sort.SliceStable(secrets, func(i, j int) bool { return secrets[i].Start < secrets[j].Start }) + return secrets +} + +func MaskSecret(value, kind string) string { + if kind == "mattermost_credential" { + return credentialMask + } + runes := []rune(value) + if len(runes) <= 8 { + return "[REDACTED:" + kind + "]" + } + visible := len(runes) / 10 + if visible < 2 { + visible = 2 + } + if visible > 4 { + visible = 4 + } + return string(runes[:visible]) + "..." + string(runes[len(runes)-visible:]) +} + +func boundedBy(class string) func(string, int, int) bool { + re := regexp.MustCompile(`^` + class + `$`) + return func(text string, start, end int) bool { + beforeOK := start == 0 + if !beforeOK { + r, _ := utf8.DecodeLastRuneInString(text[:start]) + beforeOK = !re.MatchString(string(r)) + } + afterOK := end == len(text) + if !afterOK { + r, _ := utf8.DecodeRuneInString(text[end:]) + afterOK = !re.MatchString(string(r)) + } + return beforeOK && afterOK + } +} + +func isASCIIAlnum(value byte) bool { + return value >= 'a' && value <= 'z' || value >= 'A' && value <= 'Z' || value >= '0' && value <= '9' +} + +func privateKeyMatches(text string) [][2]int { + var matches [][2]int + for offset := 0; offset < len(text); { + begin := privateKeyBegin.FindStringSubmatchIndex(text[offset:]) + if begin == nil { + break + } + start := offset + begin[0] + bodyStart := offset + begin[1] + label := text[offset+begin[2] : offset+begin[3]] + endMarker := "-----END " + label + "-----" + end := len(text) + if relative := strings.Index(text[bodyStart:], endMarker); relative >= 0 { + end = bodyStart + relative + len(endMarker) + } + matches = append(matches, [2]int{start, end}) + offset = end + } + return matches +} diff --git a/internal/presentation/patterns_test.go b/internal/presentation/patterns_test.go new file mode 100644 index 0000000..3ed7910 --- /dev/null +++ b/internal/presentation/patterns_test.go @@ -0,0 +1,153 @@ +package presentation + +import ( + "strings" + "testing" +) + +func TestDetectSecretsPatterns(t *testing.T) { + repeat := strings.Repeat + cases := []struct{ kind, text, value string }{ + {"aws_access_key", "x AKIAIOSFODNN7EXAMPLE y", "AKIAIOSFODNN7EXAMPLE"}, + {"aws_secret_key", "aws_secret_access_key = " + repeat("a", 40), repeat("a", 40)}, + {"github_stateless_token", "(" + "ghs_" + repeat("a.b_-", 8) + ")", "ghs_" + repeat("a.b_-", 8)}, + {"github_token", "ghp_" + repeat("a", 36), "ghp_" + repeat("a", 36)}, + // github_oauth is intentionally suppressed by the earlier, identical-span + // github_token match, matching the TypeScript detector's seen-span rule. + {"github_token", "gho_" + repeat("a", 36), "gho_" + repeat("a", 36)}, + {"github_fine_grained_token", "github_pat_" + repeat("a", 22), "github_pat_" + repeat("a", 22)}, + {"gitlab_token", "glpat-" + repeat("a", 20), "glpat-" + repeat("a", 20)}, + {"slack_app_token", "xapp-1-" + repeat("a-", 8), "xapp-1-" + repeat("a-", 8)}, + {"slack_token", "xoxb-1234567890-1234567890-" + repeat("a", 24), "xoxb-1234567890-1234567890-" + repeat("a", 24)}, + {"slack_webhook", "https://hooks.slack.com/services/TABC/BDEF/abc123", "https://hooks.slack.com/services/TABC/BDEF/abc123"}, + {"discord_token", "M" + repeat("a", 23) + ".abcdef." + repeat("z", 27), "M" + repeat("a", 23) + ".abcdef." + repeat("z", 27)}, + {"discord_webhook", "https://discordapp.com/api/webhooks/123/abc_DEF", "https://discordapp.com/api/webhooks/123/abc_DEF"}, + {"jwt", "eyJ" + repeat("a", 10) + ".eyJ" + repeat("b", 10) + ".ccc", "eyJ" + repeat("a", 10) + ".eyJ" + repeat("b", 10) + ".ccc"}, + {"bearer_token", "Authorization: Bearer " + repeat("a", 20), repeat("a", 20)}, + {"basic_auth", "Basic " + repeat("A", 20), repeat("A", 20)}, + {"connection_string", "postgres://user:password@db.example/prod", "postgres://user:password@db.example/prod"}, + {"api_key", "api-key='" + repeat("A", 20) + "'", repeat("A", 20)}, + {"password", "passwd: hunter22", "hunter22"}, + {"private_key", "-----BEGIN EC PRIVATE KEY-----\nabc\n-----END EC PRIVATE KEY-----", "-----BEGIN EC PRIVATE KEY-----\nabc\n-----END EC PRIVATE KEY-----"}, + {"mattermost_token", "Mattermost token: 9xuqwrwgstrb3mzrxb83nb357a", "9xuqwrwgstrb3mzrxb83nb357a"}, + {"stripe_key", "sk_live_" + repeat("a", 24), "sk_live_" + repeat("a", 24)}, + {"stripe_restricted_key", "rk_test_" + repeat("a", 24), "rk_test_" + repeat("a", 24)}, + {"sendgrid_key", "SG." + repeat("a", 22) + "." + repeat("b", 43), "SG." + repeat("a", 22) + "." + repeat("b", 43)}, + {"twilio_key", "SK" + repeat("a", 32), "SK" + repeat("a", 32)}, + {"openai_key", "sk-" + repeat("A", 32), "sk-" + repeat("A", 32)}, + {"openai_project_key", "sk-proj-" + repeat("A", 32), "sk-proj-" + repeat("A", 32)}, + {"anthropic_key", "sk-ant-" + repeat("a", 32), "sk-ant-" + repeat("a", 32)}, + {"google_api_key", "AIza" + repeat("a", 35), "AIza" + repeat("a", 35)}, + // api_key owns this exact span before heroku_key gets to it. + {"api_key", "HEROKU_API_KEY=12345678-1234-abcd-9876-123456789abc", "12345678-1234-abcd-9876-123456789abc"}, + {"npm_token", "npm_" + repeat("a", 36), "npm_" + repeat("a", 36)}, + {"high_entropy_secret", "credential=" + repeat("Z", 32), repeat("Z", 32)}, + } + for _, tc := range cases { + t.Run(tc.kind, func(t *testing.T) { + for _, got := range DetectSecrets(tc.text) { + if got.Type == tc.kind && got.Value == tc.value && tc.text[got.Start:got.End] == tc.value { + return + } + } + t.Fatalf("missing %s=%q in %#v", tc.kind, tc.value, DetectSecrets(tc.text)) + }) + } +} + +func TestSecretPatternOrderMatchesV1(t *testing.T) { + want := []string{ + "aws_access_key", "aws_secret_key", "github_stateless_token", "github_token", "github_oauth", + "github_fine_grained_token", "gitlab_token", "slack_app_token", "slack_token", "slack_webhook", + "discord_token", "discord_webhook", "jwt", "bearer_token", "basic_auth", "connection_string", + "api_key", "password", "mattermost_token", "stripe_key", "stripe_restricted_key", "sendgrid_key", + "twilio_key", "openai_key", "openai_project_key", "anthropic_key", "google_api_key", "heroku_key", + "npm_token", "high_entropy_secret", + } + if len(secretPatterns) != len(want) { + t.Fatalf("pattern count = %d, want %d", len(secretPatterns), len(want)) + } + for index := range want { + if secretPatterns[index].name != want[index] { + t.Fatalf("pattern[%d] = %q, want %q", index, secretPatterns[index].name, want[index]) + } + } +} + +func TestDetectSecretsBoundariesAndSpecialCases(t *testing.T) { + stateless := "ghs_" + strings.Repeat("a", 36) + slack := "xapp-1-" + strings.Repeat("a", 16) + mm := "9xuqwrwgstrb3mzrxb83nb357a" + for _, text := range []string{"prefix" + stateless, "a" + slack, "xapp-this-is-not-a-token", "post id: " + mm, "token=" + mm + "z"} { + if got := DetectSecrets(text); len(got) != 0 { + t.Errorf("DetectSecrets(%q) = %#v", text, got) + } + } + for _, text := range []string{"token=" + mm + `"z`, "token=" + mm + `'z`} { + got := DetectSecrets(text) + if len(got) != 1 || got[0].Type != "mattermost_token" || got[0].Value != mm { + t.Errorf("quoted Mattermost token boundary %q: %#v", text, got) + } + } + + truncated := "-----BEGIN OPENSSH PRIVATE KEY-----\n秘密🙂" + got := DetectSecrets(truncated) + if len(got) != 1 || got[0].Type != "private_key" || got[0].Value != truncated { + t.Fatalf("truncated key: %#v", got) + } + for _, label := range []string{"PUBLIC KEY", "CERTIFICATE"} { + if got := DetectSecrets("-----BEGIN " + label + "-----\nx\n-----END " + label + "-----"); len(got) != 0 { + t.Errorf("matched %s: %#v", label, got) + } + } + + token := "ghp_" + strings.Repeat("a", 36) + unicodeText := "🙂é " + token + got = DetectSecrets(unicodeText) + if len(got) != 1 || got[0].Start != len("🙂é ") || got[0].Value != token { + t.Fatalf("UTF-8 offsets: %#v", got) + } +} + +func TestDetectSecretsUsesJavaScriptUnicodeWhitespace(t *testing.T) { + token := strings.Repeat("A", 20) + for _, text := range []string{ + "Bearer\u00a0" + token, + "Basic\u3000" + token, + "api_key\ufeff" + token, + "password\u2028" + token, + } { + if got := DetectSecrets(text); len(got) == 0 || got[0].Value != token { + t.Errorf("DetectSecrets(%q) = %#v", text, got) + } + } +} + +func TestDetectSecretsOrderAndExactSpanDeduplication(t *testing.T) { + oauth := "gho_" + strings.Repeat("a", 36) + got := DetectSecrets(oauth) + if len(got) != 1 || got[0].Type != "github_token" { + t.Fatalf("precedence/dedupe: %#v", got) + } + + overlap := "postgres://api_key=ABCDEFGHIJKLMNOPQRSTUVWXYZ123456:supersecret@db.internal/prod" + got = DetectSecrets(overlap) + if len(got) < 2 || got[0].Type != "connection_string" || got[1].Type != "api_key" { + t.Fatalf("overlap order: %#v", got) + } +} + +func TestMaskSecret(t *testing.T) { + if got := MaskSecret("abc123", "test"); got != "[REDACTED:test]" { + t.Fatal(got) + } + if got := MaskSecret(strings.Repeat("a", 40), "test"); got != "aaaa...aaaa" { + t.Fatal(got) + } + if got := MaskSecret("秘密🙂abcdefgh", "test"); got != "秘密...gh" { + t.Fatal(got) + } + if got := MaskSecret("anything", "mattermost_credential"); got != credentialMask { + t.Fatal(got) + } +} diff --git a/internal/presentation/sanitize.go b/internal/presentation/sanitize.go index c88fa88..7500971 100644 --- a/internal/presentation/sanitize.go +++ b/internal/presentation/sanitize.go @@ -22,8 +22,9 @@ type Result struct { } type span struct { - start int - end int + start int + end int + secrets []DetectedSecret } func SanitizeControls(text string) string { @@ -45,7 +46,21 @@ func SanitizeLabel(text string) string { } func Preprocess(text string, credentials []string) Result { - spans := credentialSpans(text, credentials) + return PreprocessWithOptions(text, Options{Credentials: credentials}) +} + +type Options struct { + Credentials []string + DisableHeuristics bool +} + +func PreprocessWithOptions(text string, options Options) Result { + secrets := make([]DetectedSecret, 0) + if !options.DisableHeuristics { + secrets = append(secrets, DetectSecrets(text)...) + } + secrets = append(secrets, exactCredentialSecrets(text, options.Credentials)...) + spans := groupOverlappingSecrets(secrets) if len(spans) == 0 { return Result{Text: SanitizeControls(text), Redactions: []Redaction{}} } @@ -57,8 +72,12 @@ func Preprocess(text string, credentials []string) Result { for _, current := range spans { raw.WriteString(text[cursor:current.start]) positions = append(positions, raw.Len()) - raw.WriteString(credentialMask) - redactions = append(redactions, Redaction{Type: "mattermost_credential", Masked: credentialMask}) + kind, types := dominantSecretTypes(current.secrets) + masked := MaskSecret(text[current.start:current.end], kind) + raw.WriteString(masked) + redactions = append(redactions, Redaction{ + Type: strings.Join(types, "+"), Masked: SanitizeControls(masked), + }) cursor = current.end } raw.WriteString(text[cursor:]) @@ -73,8 +92,8 @@ func PreprocessActive(text string) Result { return Preprocess(text, ActiveCredentials.Values()) } -func credentialSpans(text string, credentials []string) []span { - var spans []span +func exactCredentialSecrets(text string, credentials []string) []DetectedSecret { + var secrets []DetectedSecret seenCredential := make(map[string]struct{}, len(credentials)) for _, credential := range credentials { if credential == "" { @@ -90,27 +109,63 @@ func credentialSpans(text string, credentials []string) []span { break } start := offset + index - spans = append(spans, span{start: start, end: start + len(credential)}) + secrets = append(secrets, DetectedSecret{ + Type: "mattermost_credential", Value: credential, Start: start, End: start + len(credential), + }) offset = start + len(credential) } } - sort.Slice(spans, func(i, j int) bool { - if spans[i].start != spans[j].start { - return spans[i].start < spans[j].start + return secrets +} + +func groupOverlappingSecrets(secrets []DetectedSecret) []span { + sort.SliceStable(secrets, func(i, j int) bool { + if secrets[i].Start != secrets[j].Start { + return secrets[i].Start < secrets[j].Start } - return spans[i].end > spans[j].end + return secrets[i].End > secrets[j].End }) - merged := make([]span, 0, len(spans)) - for _, current := range spans { - if len(merged) == 0 || current.start >= merged[len(merged)-1].end { - merged = append(merged, current) + groups := make([]span, 0, len(secrets)) + for _, secret := range secrets { + if len(groups) == 0 || secret.Start >= groups[len(groups)-1].end { + groups = append(groups, span{start: secret.Start, end: secret.End, secrets: []DetectedSecret{secret}}) continue } - if current.end > merged[len(merged)-1].end { - merged[len(merged)-1].end = current.end + current := &groups[len(groups)-1] + if secret.End > current.end { + current.end = secret.End + } + current.secrets = append(current.secrets, secret) + } + return groups +} + +func dominantSecretTypes(secrets []DetectedSecret) (string, []string) { + types := make([]string, 0, len(secrets)) + seen := make(map[string]struct{}, len(secrets)) + dominant := "secret" + for _, secret := range secrets { + if _, exists := seen[secret.Type]; !exists { + seen[secret.Type] = struct{}{} + types = append(types, secret.Type) + } + if secret.Type == "mattermost_credential" { + dominant = secret.Type + } + } + if dominant == "secret" && len(types) > 0 { + dominant = types[0] + } + if dominant == "mattermost_credential" && len(types) > 0 && types[0] != dominant { + for index, kind := range types { + if kind == dominant { + types = append(types[:index], types[index+1:]...) + break + } } + types = append([]string{dominant}, types...) } - return merged + return dominant, types } func unsafeControl(character rune) bool { diff --git a/internal/presentation/sanitize_test.go b/internal/presentation/sanitize_test.go index 2fc653c..2409679 100644 --- a/internal/presentation/sanitize_test.go +++ b/internal/presentation/sanitize_test.go @@ -97,3 +97,44 @@ func TestPreprocessSanitizesPrefixBeforeComputingPosition(t *testing.T) { t.Fatalf("Position = %d, want %d", got, want) } } + +func TestPreprocessHeuristicRedactionCanBeDisabledWithoutDisablingCredentialMasking(t *testing.T) { + token := "ghp_" + strings.Repeat("a", 36) + active := "active-token" + withPatterns := Preprocess(token, nil) + if withPatterns.Text == token || len(withPatterns.Redactions) != 1 { + t.Fatalf("heuristic redaction missing: %+v", withPatterns) + } + withoutPatterns := PreprocessWithOptions(token+" "+active, Options{ + Credentials: []string{active}, DisableHeuristics: true, + }) + if got, want := withoutPatterns.Text, token+" "+credentialMask; got != want { + t.Fatalf("disabled heuristic result = %q, want %q", got, want) + } +} + +func TestPreprocessNeverReappendsOverlappingPlaintext(t *testing.T) { + text := "postgres://api_key=ABCDEFGHIJKLMNOPQRSTUVWXYZ123456:supersecret@db.internal/prod" + result := Preprocess(text, nil) + if len(result.Redactions) != 1 || result.Redactions[0].Type != "connection_string+api_key" { + t.Fatalf("redactions = %+v", result.Redactions) + } + for _, secret := range []string{"ABCDEFGHIJKLMNOPQRSTUVWXYZ123456", "supersecret", "db.internal"} { + if strings.Contains(result.Text, secret) { + t.Fatalf("redacted text leaked %q: %q", secret, result.Text) + } + } +} + +func TestPreprocessSanitizesRedactionProvenanceMasks(t *testing.T) { + result := Preprocess("password=\x1bAAAAAAAAX", nil) + if len(result.Redactions) != 1 { + t.Fatalf("redactions = %+v", result.Redactions) + } + if strings.ContainsRune(result.Redactions[0].Masked, '\x1b') { + t.Fatalf("provenance retained live escape: %+v", result.Redactions[0]) + } + if !strings.Contains(result.Redactions[0].Masked, `\u001b`) { + t.Fatalf("provenance did not make escape visible: %+v", result.Redactions[0]) + } +} From d67067cb9f2f8bb08b874dbb14465be0185c02e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 04:36:26 +0300 Subject: [PATCH 009/119] feat: add bounded Mattermost transport --- internal/api/client.go | 442 +++++++++++++++++++++++++++++++++ internal/api/client_test.go | 481 ++++++++++++++++++++++++++++++++++++ 2 files changed, 923 insertions(+) create mode 100644 internal/api/client.go create mode 100644 internal/api/client_test.go diff --git a/internal/api/client.go b/internal/api/client.go new file mode 100644 index 0000000..c3b05b4 --- /dev/null +++ b/internal/api/client.go @@ -0,0 +1,442 @@ +// Package api provides the bounded, authenticated Mattermost HTTP transport. +package api + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "time" + + "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/internal/serverurl" +) + +const ( + AttemptTimeout = 15 * time.Second + MaxResponseBody = 4 << 20 + maxRetries = 2 + maxRetryDelay = 30 * time.Second +) + +var ( + ErrNetwork = errors.New("unable to connect to Mattermost due to a network error") + ErrTimeout = errors.New("Mattermost request timed out") + ErrCanceled = errors.New("Mattermost request canceled") + ErrInvalidJSON = errors.New("Mattermost returned an invalid JSON response") + ErrBodyTooLarge = errors.New("Mattermost response exceeded the size limit") + ErrClientClosed = errors.New("Mattermost client is closed") +) + +type APIError struct{ Status int } + +func (e *APIError) Error() string { + return fmt.Sprintf("Mattermost API request failed with status %d", e.Status) +} + +type OutcomeUnknownError struct{} + +func (*OutcomeUnknownError) Error() string { + return "Mattermost did not confirm the write; its outcome is unknown, check the destination before retrying" +} + +type SleepFunc func(context.Context, time.Duration) error +type Option func(*Client) + +func WithRoundTripper(transport http.RoundTripper) Option { + return func(c *Client) { + if transport != nil { + c.transport = transport + } + } +} +func WithClock(now func() time.Time) Option { + return func(c *Client) { + if now != nil { + c.now = now + } + } +} +func WithSleep(sleep SleepFunc) Option { + return func(c *Client) { + if sleep != nil { + c.sleep = sleep + } + } +} +func WithAttemptTimeout(timeout time.Duration) Option { + return func(c *Client) { + if timeout > 0 { + c.timeout = timeout + } + } +} +func WithResponseLimit(limit int64) Option { + return func(c *Client) { + if limit > 0 { + c.bodyLimit = limit + } + } +} + +type Client struct { + base *url.URL + token string + http *http.Client + transport http.RoundTripper + now func() time.Time + sleep SleepFunc + timeout time.Duration + bodyLimit int64 + release func() + lifecycle sync.RWMutex + closed bool +} + +type requestContextKey uint8 + +const mutationRequestKey requestContextKey = 1 + +func New(baseURL, token string, options ...Option) (*Client, error) { + normalized, err := serverurl.Normalize(baseURL) + if err != nil { + return nil, err + } + base, err := url.Parse(normalized) + if err != nil { + return nil, errors.New("invalid Mattermost URL") + } + c := &Client{ + base: base, token: token, transport: http.DefaultTransport, now: time.Now, + timeout: AttemptTimeout, bodyLimit: MaxResponseBody, + } + c.sleep = sleepContext + for _, option := range options { + option(c) + } + c.http = &http.Client{Transport: c.transport, CheckRedirect: c.checkRedirect} + c.release = presentation.ActiveCredentials.Register(token) + return c, nil +} + +func (c *Client) Close() { + c.lifecycle.Lock() + defer c.lifecycle.Unlock() + if c.closed { + return + } + c.closed = true + if c.release != nil { + c.release() + } +} + +func (c *Client) Get(ctx context.Context, path string, out any) error { + return c.request(ctx, http.MethodGet, path, nil, out, false) +} +func (c *Client) Post(ctx context.Context, path string, body, out any) error { + return c.request(ctx, http.MethodPost, path, body, out, true) +} +func (c *Client) PostRead(ctx context.Context, path string, body, out any) error { + return c.request(ctx, http.MethodPost, path, body, out, false) +} +func (c *Client) Put(ctx context.Context, path string, body, out any) error { + return c.request(ctx, http.MethodPut, path, body, out, true) +} +func (c *Client) Delete(ctx context.Context, path string, out any) error { + return c.request(ctx, http.MethodDelete, path, nil, out, true) +} + +func (c *Client) request(ctx context.Context, method, path string, body, out any, mutation bool) error { + c.lifecycle.RLock() + defer c.lifecycle.RUnlock() + if c.closed { + return ErrClientClosed + } + endpoint, err := c.endpoint(path) + if err != nil { + return err + } + payload, err := encodeBody(body) + if err != nil { + return errors.New("unable to encode Mattermost request") + } + for attempt := 0; ; attempt++ { + if ctx.Err() != nil { + return classifyContext(ctx) + } + status, headers, data, failure := c.attempt(ctx, method, endpoint, payload, mutation) + if failure != nil { + if mutation { + return &OutcomeUnknownError{} + } + if retryableFailure(failure) && attempt < maxRetries { + if err := c.sleep(ctx, retryDelay(attempt)); err != nil { + return classifyContext(ctx) + } + continue + } + return publicReadError(ctx, failure) + } + if mutation { + if status >= 400 && status < 500 { + return &APIError{Status: status} + } + if status < 200 || status >= 300 { + return &OutcomeUnknownError{} + } + } else { + if (status == 429 || status == 502 || status == 503 || status == 504) && attempt < maxRetries { + delay := retryDelay(attempt) + if status == 429 { + delay = c.rateLimitDelay(headers, attempt) + } + if err := c.sleep(ctx, delay); err != nil { + return classifyContext(ctx) + } + continue + } + if status < 200 || status >= 300 { + return &APIError{Status: status} + } + } + decodeTarget := out + if decodeTarget == nil { + decodeTarget = new(any) + } + if err := decodeJSON(data, decodeTarget); err != nil { + if mutation { + return &OutcomeUnknownError{} + } + return ErrInvalidJSON + } + return nil + } +} + +type attemptFailure struct{ kind string } + +func (e *attemptFailure) Error() string { return e.kind } + +func (c *Client) attempt(parent context.Context, method string, endpoint *url.URL, payload []byte, mutation bool) (int, http.Header, []byte, error) { + ctx, cancel := context.WithTimeout(parent, c.timeout) + defer cancel() + if mutation { + ctx = context.WithValue(ctx, mutationRequestKey, true) + } + req, err := http.NewRequestWithContext(ctx, method, endpoint.String(), bytes.NewReader(payload)) + if err != nil { + return 0, nil, nil, &attemptFailure{kind: "transport"} + } + req.Header.Set("Authorization", "Bearer "+c.token) + req.Header.Set("Content-Type", "application/json") + resp, err := c.http.Do(req) + if err != nil { + if parent.Err() != nil { + return 0, nil, nil, &attemptFailure{kind: "canceled"} + } + if ctx.Err() != nil { + return 0, nil, nil, &attemptFailure{kind: "timeout"} + } + return 0, nil, nil, &attemptFailure{kind: "transport"} + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return resp.StatusCode, resp.Header.Clone(), nil, nil + } + data, readErr := io.ReadAll(io.LimitReader(resp.Body, c.bodyLimit+1)) + if parent.Err() != nil { + return 0, nil, nil, &attemptFailure{kind: "canceled"} + } + if ctx.Err() != nil { + return 0, nil, nil, &attemptFailure{kind: "timeout"} + } + if readErr != nil { + if mutation && resp.StatusCode >= 400 && resp.StatusCode < 500 { + return resp.StatusCode, resp.Header.Clone(), nil, nil + } + return 0, nil, nil, &attemptFailure{kind: "transport"} + } + if int64(len(data)) > c.bodyLimit { + if mutation && resp.StatusCode >= 400 && resp.StatusCode < 500 { + return resp.StatusCode, resp.Header.Clone(), nil, nil + } + return 0, nil, nil, &attemptFailure{kind: "oversized"} + } + return resp.StatusCode, resp.Header.Clone(), data, nil +} + +func (c *Client) endpoint(path string) (*url.URL, error) { + if !strings.HasPrefix(path, "/") || strings.HasPrefix(path, "//") || strings.Contains(path, "#") { + return nil, errors.New("invalid Mattermost API path") + } + relative, err := url.ParseRequestURI(path) + if err != nil || relative.IsAbs() || relative.Host != "" || relative.User != nil || relative.Fragment != "" { + return nil, errors.New("invalid Mattermost API path") + } + for _, segment := range strings.Split(relative.Path, "/") { + if segment == "." || segment == ".." { + return nil, errors.New("invalid Mattermost API path") + } + } + u := *c.base + u.Path = c.base.Path + "/api/v4" + relative.Path + u.RawPath = c.base.EscapedPath() + "/api/v4" + relative.EscapedPath() + u.RawQuery = relative.RawQuery + u.Fragment = "" + return &u, nil +} + +func (c *Client) checkRedirect(req *http.Request, via []*http.Request) error { + if len(via) == 0 { + return http.ErrUseLastResponse + } + first := via[0] + if mutation, _ := first.Context().Value(mutationRequestKey).(bool); mutation { + return http.ErrUseLastResponse + } + if !sameOrigin(first.URL, req.URL) { + return http.ErrUseLastResponse + } + req.Header.Set("Authorization", first.Header.Get("Authorization")) + return nil +} + +func sameOrigin(a, b *url.URL) bool { + return strings.EqualFold(a.Scheme, b.Scheme) && strings.EqualFold(a.Hostname(), b.Hostname()) && effectivePort(a) == effectivePort(b) +} +func effectivePort(u *url.URL) string { + if u.Port() != "" { + return u.Port() + } + if strings.EqualFold(u.Scheme, "https") { + return "443" + } + if strings.EqualFold(u.Scheme, "http") { + return "80" + } + return "" +} + +func encodeBody(body any) ([]byte, error) { + if body == nil { + return nil, nil + } + var buffer bytes.Buffer + encoder := json.NewEncoder(&buffer) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(body); err != nil { + return nil, err + } + data := bytes.TrimSuffix(buffer.Bytes(), []byte{'\n'}) + return restoreJSONLineSeparators(data), nil +} + +func restoreJSONLineSeparators(data []byte) []byte { + var output bytes.Buffer + output.Grow(len(data)) + for index := 0; index < len(data); { + if index+6 <= len(data) && data[index] == '\\' && + (bytes.Equal(data[index:index+6], []byte(`\u2028`)) || bytes.Equal(data[index:index+6], []byte(`\u2029`))) { + precedingSlashes := 0 + for cursor := index - 1; cursor >= 0 && data[cursor] == '\\'; cursor-- { + precedingSlashes++ + } + if precedingSlashes%2 == 0 { + if data[index+5] == '8' { + output.WriteString("\u2028") + } else { + output.WriteString("\u2029") + } + index += 6 + continue + } + } + output.WriteByte(data[index]) + index++ + } + return output.Bytes() +} + +func decodeJSON(data []byte, out any) error { + decoder := json.NewDecoder(bytes.NewReader(data)) + if err := decoder.Decode(out); err != nil { + return err + } + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + return errors.New("trailing JSON data") + } + return nil +} + +func retryableFailure(err error) bool { + var failure *attemptFailure + return errors.As(err, &failure) && (failure.kind == "transport" || failure.kind == "timeout" || failure.kind == "oversized") +} +func publicReadError(ctx context.Context, err error) error { + if ctx.Err() != nil { + return classifyContext(ctx) + } + var failure *attemptFailure + if errors.As(err, &failure) { + switch failure.kind { + case "timeout": + return ErrTimeout + case "canceled": + return ErrCanceled + case "oversized": + return ErrBodyTooLarge + } + } + return ErrNetwork +} +func classifyContext(ctx context.Context) error { + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + return ErrTimeout + } + return ErrCanceled +} +func retryDelay(attempt int) time.Duration { return time.Second << attempt } +func (c *Client) rateLimitDelay(headers http.Header, attempt int) time.Duration { + if value := headers.Get("Retry-After"); value != "" { + if seconds, err := strconv.ParseFloat(value, 64); err == nil { + return clamp(time.Duration(seconds * float64(time.Second))) + } + if when, err := http.ParseTime(value); err == nil { + return clamp(when.Sub(c.now())) + } + } + if value := headers.Get("X-RateLimit-Reset"); value != "" { + if seconds, err := strconv.ParseFloat(value, 64); err == nil && seconds > 0 { + return clamp(time.Duration(seconds * float64(time.Second))) + } + } + return retryDelay(attempt) +} +func clamp(delay time.Duration) time.Duration { + if delay < 0 { + return 0 + } + if delay > maxRetryDelay { + return maxRetryDelay + } + return delay +} +func sleepContext(ctx context.Context, delay time.Duration) error { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} diff --git a/internal/api/client_test.go b/internal/api/client_test.go new file mode 100644 index 0000000..3730430 --- /dev/null +++ b/internal/api/client_test.go @@ -0,0 +1,481 @@ +package api + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/ardasevinc/mattermost-cli/internal/presentation" +) + +func TestReadAuthJSONAndRetryCounts(t *testing.T) { + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Authorization"); got != "Bearer token-value" { + t.Errorf("Authorization = %q", got) + } + if got := r.URL.Path; got != "/api/v4/users/me" { + t.Errorf("path = %q", got) + } + if attempts.Add(1) < 3 { + w.WriteHeader(http.StatusBadGateway) + return + } + _, _ = io.WriteString(w, `{"id":"me"}`) + })) + defer server.Close() + c := newTestClient(t, server.URL, WithSleep(noSleep)) + var result struct { + ID string `json:"id"` + } + if err := c.Get(context.Background(), "/users/me", &result); err != nil { + t.Fatal(err) + } + if result.ID != "me" || attempts.Load() != 3 { + t.Fatalf("result=%+v attempts=%d", result, attempts.Load()) + } +} + +func TestEndpointPreservesQueryAndEscapedPathComponents(t *testing.T) { + var requestURI string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestURI = r.RequestURI + _, _ = io.WriteString(w, `{}`) + })) + defer server.Close() + c := newTestClient(t, server.URL) + if err := c.Get(context.Background(), "/users/a%2Fb?page=2&active=true", &struct{}{}); err != nil { + t.Fatal(err) + } + if want := "/api/v4/users/a%2Fb?page=2&active=true"; requestURI != want { + t.Fatalf("RequestURI = %q, want %q", requestURI, want) + } +} + +func TestEndpointPreservesBasePathEndingInEncodedSlash(t *testing.T) { + var requestURI string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestURI = r.RequestURI + _, _ = io.WriteString(w, `{}`) + })) + defer server.Close() + c := newTestClient(t, server.URL+"/mattermost%2F") + if err := c.Get(context.Background(), "/users/me", &struct{}{}); err != nil { + t.Fatal(err) + } + if want := "/mattermost%2F/api/v4/users/me"; requestURI != want { + t.Fatalf("RequestURI = %q, want %q", requestURI, want) + } +} + +func TestEndpointRejectsAmbiguousRequestTargetsBeforeDispatch(t *testing.T) { + transport := &countingTransport{err: errors.New("must not be called")} + c := newTestClient(t, "https://mattermost.example", WithRoundTripper(transport)) + for _, path := range []string{"users", "//evil.example/x", "/../posts", "/users#fragment", "/bad%zz"} { + if err := c.Get(context.Background(), path, &struct{}{}); err == nil || err.Error() != "invalid Mattermost API path" { + t.Errorf("Get(%q) error = %v", path, err) + } + } + if transport.count.Load() != 0 { + t.Fatalf("attempts = %d", transport.count.Load()) + } +} + +func TestReadRetriesTransportExactlyTwice(t *testing.T) { + transport := &countingTransport{err: errors.New("contains token-value")} + c := newTestClient(t, "https://mattermost.example", WithRoundTripper(transport), WithSleep(noSleep)) + err := c.Get(context.Background(), "/users/me", &struct{}{}) + if !errors.Is(err, ErrNetwork) || strings.Contains(err.Error(), "token-value") { + t.Fatalf("error = %v", err) + } + if transport.count.Load() != 3 { + t.Fatalf("attempts = %d", transport.count.Load()) + } +} + +func TestReadOnlyPostUsesBoundedSafeRetries(t *testing.T) { + transport := &sequenceTransport{statuses: []int{503, 503, 200}, headers: make([]http.Header, 3)} + c := newTestClient(t, "https://mattermost.example", WithRoundTripper(transport), WithSleep(noSleep)) + if err := c.PostRead(context.Background(), "/users/search", map[string]string{"term": "arda"}, &struct{}{}); err != nil { + t.Fatal(err) + } + if transport.index != 3 { + t.Fatalf("attempts = %d", transport.index) + } +} + +func TestMutationNeverReplaysAndClassifiesOutcomes(t *testing.T) { + for _, status := range []int{300, 307, 500, 503} { + t.Run(http.StatusText(status), func(t *testing.T) { + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { attempts.Add(1); w.WriteHeader(status) })) + defer server.Close() + c := newTestClient(t, server.URL, WithSleep(noSleep)) + err := c.Post(context.Background(), "/posts", map[string]string{"message": "hi"}, &struct{}{}) + var unknown *OutcomeUnknownError + if !errors.As(err, &unknown) || attempts.Load() != 1 { + t.Fatalf("error=%v attempts=%d", err, attempts.Load()) + } + }) + } + for _, status := range []int{400, 401, 429, 499} { + t.Run(http.StatusText(status), func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(status) })) + defer server.Close() + c := newTestClient(t, server.URL) + var apiErr *APIError + if err := c.Post(context.Background(), "/posts", nil, nil); !errors.As(err, &apiErr) || apiErr.Status != status { + t.Fatalf("error = %v", err) + } + }) + } +} + +func TestMutationInformationalStatusIsUnknown(t *testing.T) { + c := newTestClient(t, "https://mattermost.example", WithRoundTripper(&statusTransport{status: 101})) + var unknown *OutcomeUnknownError + if err := c.Post(context.Background(), "/posts", nil, nil); !errors.As(err, &unknown) { + t.Fatalf("error = %v", err) + } +} + +func TestMutationRedirectDoesNotReplay(t *testing.T) { + var original, redirected atomic.Int32 + server := httptest.NewServer(nil) + defer server.Close() + mux := http.NewServeMux() + mux.HandleFunc("/api/v4/posts", func(w http.ResponseWriter, r *http.Request) { + original.Add(1) + http.Redirect(w, r, server.URL+"/redirected", http.StatusTemporaryRedirect) + }) + mux.HandleFunc("/redirected", func(w http.ResponseWriter, r *http.Request) { redirected.Add(1) }) + server.Config.Handler = mux + c := newTestClient(t, server.URL) + var unknown *OutcomeUnknownError + if err := c.Post(context.Background(), "/posts", map[string]string{"x": "y"}, nil); !errors.As(err, &unknown) { + t.Fatalf("error = %v", err) + } + if original.Load() != 1 || redirected.Load() != 0 { + t.Fatalf("original=%d redirected=%d", original.Load(), redirected.Load()) + } +} + +func TestReadRedirectSameOriginOnlyAndNeverLeaksAuth(t *testing.T) { + var leaked atomic.Bool + evil := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "" { + leaked.Store(true) + } + _, _ = io.WriteString(w, `{}`) + })) + defer evil.Close() + var sameHits atomic.Int32 + var origin *httptest.Server + origin = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v4/same": + http.Redirect(w, r, origin.URL+"/api/v4/final", http.StatusFound) + case "/api/v4/final": + sameHits.Add(1) + if r.Header.Get("Authorization") != "Bearer token-value" { + t.Error("same-origin auth missing") + } + _, _ = io.WriteString(w, `{}`) + default: + http.Redirect(w, r, evil.URL, http.StatusFound) + } + })) + defer origin.Close() + c := newTestClient(t, origin.URL) + if err := c.Get(context.Background(), "/same", &struct{}{}); err != nil { + t.Fatal(err) + } + var apiErr *APIError + if err := c.Get(context.Background(), "/cross", &struct{}{}); !errors.As(err, &apiErr) || apiErr.Status != 302 { + t.Fatalf("error = %v", err) + } + if leaked.Load() || sameHits.Load() != 1 { + t.Fatalf("leaked=%v sameHits=%d", leaked.Load(), sameHits.Load()) + } +} + +func TestAttemptTimeoutCoversBodyAndCancellationStopsSleep(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "100") + w.(http.Flusher).Flush() + <-r.Context().Done() + })) + defer server.Close() + c := newTestClient(t, server.URL, WithAttemptTimeout(20*time.Millisecond), WithSleep(noSleep)) + if err := c.Get(context.Background(), "/slow", &struct{}{}); !errors.Is(err, ErrTimeout) { + t.Fatalf("error = %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + c2 := newTestClient(t, "https://mattermost.example", WithRoundTripper(&statusTransport{status: 503}), WithSleep(func(ctx context.Context, _ time.Duration) error { cancel(); <-ctx.Done(); return ctx.Err() })) + if err := c2.Get(ctx, "/x", &struct{}{}); !errors.Is(err, ErrCanceled) { + t.Fatalf("error = %v", err) + } +} + +func TestBoundedAndTruncatedBodies(t *testing.T) { + for name, handler := range map[string]http.HandlerFunc{ + "oversized": func(w http.ResponseWriter, r *http.Request) { _, _ = io.WriteString(w, strings.Repeat("x", 9)) }, + "truncated": func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "20") + _, _ = io.WriteString(w, `{"id":`) + }, + } { + t.Run(name, func(t *testing.T) { + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { attempts.Add(1); handler(w, r) })) + defer server.Close() + c := newTestClient(t, server.URL, WithResponseLimit(8), WithSleep(noSleep)) + if err := c.Get(context.Background(), "/x", &struct{}{}); err == nil { + t.Fatal("expected error") + } + if attempts.Load() != 3 { + t.Fatalf("attempts = %d", attempts.Load()) + } + }) + } +} + +func TestMutationMalformedSuccessAndTruncationAreUnknown(t *testing.T) { + for name, handler := range map[string]http.HandlerFunc{ + "malformed": func(w http.ResponseWriter, r *http.Request) { _, _ = io.WriteString(w, `{bad`) }, + "truncated": func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "20") + _, _ = io.WriteString(w, `{`) + }, + } { + t.Run(name, func(t *testing.T) { + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { attempts.Add(1); handler(w, r) })) + defer server.Close() + c := newTestClient(t, server.URL) + var unknown *OutcomeUnknownError + if err := c.Post(context.Background(), "/posts", nil, &struct{}{}); !errors.As(err, &unknown) { + t.Fatalf("error = %v", err) + } + if attempts.Load() != 1 { + t.Fatalf("attempts = %d", attempts.Load()) + } + }) + } +} + +func TestMutationWithIgnoredOutputStillValidatesSuccessJSON(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, `{bad`) + })) + defer server.Close() + c := newTestClient(t, server.URL) + var unknown *OutcomeUnknownError + if err := c.Delete(context.Background(), "/posts/id", nil); !errors.As(err, &unknown) { + t.Fatalf("error = %v", err) + } +} + +func TestJSONDoesNotEscapeHTMLAndCredentialLifecycle(t *testing.T) { + presentation.ActiveCredentials.Clear() + var body string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + data, _ := io.ReadAll(r.Body) + body = string(data) + _, _ = io.WriteString(w, `{}`) + })) + defer server.Close() + c, err := New(server.URL, "owned-token") + if err != nil { + t.Fatal(err) + } + if got := presentation.ActiveCredentials.Values(); len(got) != 1 || got[0] != "owned-token" { + t.Fatalf("credentials = %v", got) + } + if err := c.Post(context.Background(), "/posts", map[string]string{"message": "&"}, &struct{}{}); err != nil { + t.Fatal(err) + } + if body != `{"message":"&"}` { + t.Fatalf("body = %q", body) + } + c.Close() + c.Close() + if got := presentation.ActiveCredentials.Values(); len(got) != 0 { + t.Fatalf("credentials after close = %v", got) + } +} + +func TestJSONMatchesJavaScriptLineSeparatorEncoding(t *testing.T) { + encoded, err := encodeBody(map[string]string{"message": "before\u2028middle\u2029after literal \\u2028"}) + if err != nil { + t.Fatal(err) + } + want := "{\"message\":\"before\u2028middle\u2029after literal \\\\u2028\"}" + if string(encoded) != want { + t.Fatalf("encoded = %q, want %q", encoded, want) + } +} + +func TestContentTypeIsPresentWithoutRequestBody(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Content-Type"); got != "application/json" { + t.Errorf("Content-Type = %q", got) + } + _, _ = io.WriteString(w, `{}`) + })) + defer server.Close() + c := newTestClient(t, server.URL) + if err := c.Get(context.Background(), "/users/me", &struct{}{}); err != nil { + t.Fatal(err) + } +} + +func TestCloseWaitsForInFlightRequestBeforeReleasingCredential(t *testing.T) { + presentation.ActiveCredentials.Clear() + started := make(chan struct{}) + proceed := make(chan struct{}) + transport := roundTripFunc(func(*http.Request) (*http.Response, error) { + close(started) + <-proceed + return &http.Response{ + StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{}`)), + }, nil + }) + c, err := New("https://mattermost.example", "in-flight-token", WithRoundTripper(transport)) + if err != nil { + t.Fatal(err) + } + requestDone := make(chan error, 1) + go func() { requestDone <- c.Get(context.Background(), "/users/me", &struct{}{}) }() + <-started + closeDone := make(chan struct{}) + go func() { c.Close(); close(closeDone) }() + select { + case <-closeDone: + t.Fatal("Close returned while request still used the credential") + case <-time.After(20 * time.Millisecond): + } + if got := presentation.ActiveCredentials.Values(); len(got) != 1 || got[0] != "in-flight-token" { + t.Fatalf("credentials during request = %v", got) + } + close(proceed) + if err := <-requestDone; err != nil { + t.Fatal(err) + } + <-closeDone + if got := presentation.ActiveCredentials.Values(); len(got) != 0 { + t.Fatalf("credentials after close = %v", got) + } + if err := c.Get(context.Background(), "/users/me", &struct{}{}); !errors.Is(err, ErrClientClosed) { + t.Fatalf("request after close error = %v", err) + } +} + +func TestCanceledMutationBeforeDispatchIsDefinitive(t *testing.T) { + transport := &countingTransport{err: errors.New("must not be called")} + c := newTestClient(t, "https://mattermost.example", WithRoundTripper(transport)) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := c.Post(ctx, "/posts", map[string]string{"message": "hi"}, &struct{}{}); !errors.Is(err, ErrCanceled) { + t.Fatalf("error = %v", err) + } + if transport.count.Load() != 0 { + t.Fatalf("attempts = %d", transport.count.Load()) + } +} + +func TestErrorResponsesAreNotConsumed(t *testing.T) { + body := &trackingBody{} + transport := roundTripFunc(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusBadRequest, Header: make(http.Header), Body: body}, nil + }) + c := newTestClient(t, "https://mattermost.example", WithRoundTripper(transport)) + var apiErr *APIError + if err := c.Post(context.Background(), "/posts", nil, nil); !errors.As(err, &apiErr) || apiErr.Status != http.StatusBadRequest { + t.Fatalf("error = %v", err) + } + if body.reads.Load() != 0 || !body.closed.Load() { + t.Fatalf("reads=%d closed=%v", body.reads.Load(), body.closed.Load()) + } +} + +func TestRateLimitDelayBoundsHeaders(t *testing.T) { + var delays []time.Duration + retryAfter := make(http.Header) + retryAfter.Set("Retry-After", "Thu, 16 Jul 2026 12:00:03 GMT") + rateReset := make(http.Header) + rateReset.Set("X-RateLimit-Reset", "3600") + transport := &sequenceTransport{statuses: []int{429, 429, 200}, headers: []http.Header{retryAfter, rateReset, nil}} + c := newTestClient(t, "https://mattermost.example", WithRoundTripper(transport), WithClock(func() time.Time { return time.Date(2026, 7, 16, 12, 0, 0, 0, time.UTC) }), WithSleep(func(_ context.Context, d time.Duration) error { delays = append(delays, d); return nil })) + if err := c.Get(context.Background(), "/x", &struct{}{}); err != nil { + t.Fatal(err) + } + if len(delays) != 2 || delays[0] != 3*time.Second || delays[1] != 30*time.Second { + t.Fatalf("delays = %v", delays) + } +} + +func newTestClient(t *testing.T, base string, options ...Option) *Client { + t.Helper() + c, err := New(base, "token-value", options...) + if err != nil { + t.Fatal(err) + } + t.Cleanup(c.Close) + return c +} +func noSleep(context.Context, time.Duration) error { return nil } + +type countingTransport struct { + count atomic.Int32 + err error +} + +func (t *countingTransport) RoundTrip(*http.Request) (*http.Response, error) { + t.count.Add(1) + return nil, t.err +} + +type statusTransport struct{ status int } + +func (t *statusTransport) RoundTrip(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: t.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{}`))}, nil +} + +type sequenceTransport struct { + index int + statuses []int + headers []http.Header +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { return f(request) } + +type trackingBody struct { + reads atomic.Int32 + closed atomic.Bool +} + +func (b *trackingBody) Read([]byte) (int, error) { + b.reads.Add(1) + return 0, io.EOF +} + +func (b *trackingBody) Close() error { + b.closed.Store(true) + return nil +} + +func (t *sequenceTransport) RoundTrip(*http.Request) (*http.Response, error) { + i := t.index + t.index++ + return &http.Response{StatusCode: t.statuses[i], Header: t.headers[i], Body: io.NopCloser(strings.NewReader(`{}`))}, nil +} From 891634b6ec7f0f19d054bb07fa86267285379438 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 04:41:54 +0300 Subject: [PATCH 010/119] docs: define v2 redaction and JSON offsets --- docs/V1_PARITY_MATRIX.md | 2 ++ docs/V2_CONTRACT.md | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/V1_PARITY_MATRIX.md b/docs/V1_PARITY_MATRIX.md index 67fc500..58ed81a 100644 --- a/docs/V1_PARITY_MATRIX.md +++ b/docs/V1_PARITY_MATRIX.md @@ -150,6 +150,8 @@ Every row requires a named Go regression test and, where applicable, a language- | terminal/control/bidi hazards are visible or removed safely | oracle | | overlapping redaction matches never re-append plaintext | oracle | | truncated private-key blocks fail closed without plaintext leakage | oracle | +| public redaction positions remain UTF-16 offsets; v2 heuristic mask previews use whole Unicode scalar values | intentionally_changed | +| request JSON has no trailing newline and preserves JSON.stringify HTML/U+2028/U+2029 wire behavior | oracle | | invalid UTF-8, whitespace-only, oversized, and unintended TTY input fail before work | oracle | | watch validates events/sequences and bounds reconnect | oracle | | watch auth failure stops reconnect and releases credentials/timers | oracle | diff --git a/docs/V2_CONTRACT.md b/docs/V2_CONTRACT.md index 89b5203..84e96eb 100644 --- a/docs/V2_CONTRACT.md +++ b/docs/V2_CONTRACT.md @@ -415,7 +415,8 @@ The v1 security boundary survives the rewrite: - Original secret values and unredacted originals never appear in redaction provenance. - Canonical unsanitized IDs drive identity, ordering, grouping, and deduplication; sanitized copies are presentation-only. - Go RE2 incompatibilities are implemented with candidate matching plus explicit boundary validation, not weaker regex substitutions. -- Byte, rune, and structured-redaction offsets are defined explicitly and tested across non-ASCII text. +- Regex and literal-match spans use UTF-8 byte offsets internally so Go slicing remains exact. Public structured-redaction `position` values are UTF-16 code-unit offsets into the final sanitized emitted text, preserving the v1 oracle contract across non-BMP text. Partial heuristic masks compute visible prefix/suffix lengths in Unicode scalar values so v2 never splits a code point; this is an intentional presentation-only change from JavaScript UTF-16 slicing. +- JSON request bodies contain no encoder-added trailing newline, disable HTML escaping, and preserve literal U+2028/U+2029 like `JSON.stringify`; string content that literally contains `\\u2028` or `\\u2029` remains escaped data. - Files, config, database, editor temp handling, and subprocess invocation resist symlink and permission attacks within the documented local-user threat model. ## 17. Test and conformance contract From 2f3c43582064428cc135d00918e4da6fdaa8ea19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 05:03:34 +0300 Subject: [PATCH 011/119] feat: add unauthenticated health reads --- internal/api/client.go | 23 ++++++++++++++--------- internal/api/client_test.go | 14 ++++++++++++++ 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/internal/api/client.go b/internal/api/client.go index c3b05b4..4f32166 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -139,22 +139,25 @@ func (c *Client) Close() { } func (c *Client) Get(ctx context.Context, path string, out any) error { - return c.request(ctx, http.MethodGet, path, nil, out, false) + return c.request(ctx, http.MethodGet, path, nil, out, false, true) +} +func (c *Client) GetPublic(ctx context.Context, path string, out any) error { + return c.request(ctx, http.MethodGet, path, nil, out, false, false) } func (c *Client) Post(ctx context.Context, path string, body, out any) error { - return c.request(ctx, http.MethodPost, path, body, out, true) + return c.request(ctx, http.MethodPost, path, body, out, true, true) } func (c *Client) PostRead(ctx context.Context, path string, body, out any) error { - return c.request(ctx, http.MethodPost, path, body, out, false) + return c.request(ctx, http.MethodPost, path, body, out, false, true) } func (c *Client) Put(ctx context.Context, path string, body, out any) error { - return c.request(ctx, http.MethodPut, path, body, out, true) + return c.request(ctx, http.MethodPut, path, body, out, true, true) } func (c *Client) Delete(ctx context.Context, path string, out any) error { - return c.request(ctx, http.MethodDelete, path, nil, out, true) + return c.request(ctx, http.MethodDelete, path, nil, out, true, true) } -func (c *Client) request(ctx context.Context, method, path string, body, out any, mutation bool) error { +func (c *Client) request(ctx context.Context, method, path string, body, out any, mutation, authenticated bool) error { c.lifecycle.RLock() defer c.lifecycle.RUnlock() if c.closed { @@ -172,7 +175,7 @@ func (c *Client) request(ctx context.Context, method, path string, body, out any if ctx.Err() != nil { return classifyContext(ctx) } - status, headers, data, failure := c.attempt(ctx, method, endpoint, payload, mutation) + status, headers, data, failure := c.attempt(ctx, method, endpoint, payload, mutation, authenticated) if failure != nil { if mutation { return &OutcomeUnknownError{} @@ -225,7 +228,7 @@ type attemptFailure struct{ kind string } func (e *attemptFailure) Error() string { return e.kind } -func (c *Client) attempt(parent context.Context, method string, endpoint *url.URL, payload []byte, mutation bool) (int, http.Header, []byte, error) { +func (c *Client) attempt(parent context.Context, method string, endpoint *url.URL, payload []byte, mutation, authenticated bool) (int, http.Header, []byte, error) { ctx, cancel := context.WithTimeout(parent, c.timeout) defer cancel() if mutation { @@ -235,7 +238,9 @@ func (c *Client) attempt(parent context.Context, method string, endpoint *url.UR if err != nil { return 0, nil, nil, &attemptFailure{kind: "transport"} } - req.Header.Set("Authorization", "Bearer "+c.token) + if authenticated { + req.Header.Set("Authorization", "Bearer "+c.token) + } req.Header.Set("Content-Type", "application/json") resp, err := c.http.Do(req) if err != nil { diff --git a/internal/api/client_test.go b/internal/api/client_test.go index 3730430..05978e4 100644 --- a/internal/api/client_test.go +++ b/internal/api/client_test.go @@ -337,6 +337,20 @@ func TestContentTypeIsPresentWithoutRequestBody(t *testing.T) { } } +func TestPublicReadNeverSendsAuthorization(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Authorization"); got != "" { + t.Errorf("Authorization = %q", got) + } + _, _ = io.WriteString(w, `{}`) + })) + defer server.Close() + c := newTestClient(t, server.URL) + if err := c.GetPublic(context.Background(), "/system/ping?get_server_status=true", &struct{}{}); err != nil { + t.Fatal(err) + } +} + func TestCloseWaitsForInFlightRequestBeforeReleasingCredential(t *testing.T) { presentation.ActiveCredentials.Clear() started := make(chan struct{}) From b5fee31d9a3dc67638d47f0534460481f294516f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 05:05:17 +0300 Subject: [PATCH 012/119] fix: sanitize all CLI errors --- internal/cli/root.go | 8 ++++---- internal/cli/root_test.go | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/internal/cli/root.go b/internal/cli/root.go index 1a1e1bc..c4299fe 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -11,6 +11,7 @@ import ( "github.com/spf13/cobra" "github.com/ardasevinc/mattermost-cli/internal/buildinfo" + "github.com/ardasevinc/mattermost-cli/internal/presentation" mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" ) @@ -21,13 +22,12 @@ type streams struct { } func Execute(ctx context.Context, args []string, in io.Reader, out, errOut io.Writer) int { + releaseCredential := presentation.ActiveCredentials.Register(os.Getenv("MM_TOKEN")) + defer releaseCredential() cmd := newRoot(streams{in: in, out: out, err: errOut}) cmd.SetArgs(args) if err := cmd.ExecuteContext(ctx); err != nil { - message := err.Error() - if token := os.Getenv("MM_TOKEN"); token != "" { - message = strings.ReplaceAll(message, token, "[REDACTED:mattermost_credential]") - } + message := presentation.SanitizeLabel(presentation.PreprocessActive(err.Error()).Text) _, _ = fmt.Fprintf(errOut, "error: %s\n", message) var outputFailure outputError if errors.As(err, &outputFailure) { diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index 41b964a..cbb2b00 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -90,6 +90,28 @@ func TestSchemaLookupDoesNotReflectActiveCredential(t *testing.T) { } } +func TestExecuteSanitizesAndRedactsUnknownCommand(t *testing.T) { + const token = "super-secret-mm-token" + t.Setenv("MM_TOKEN", token) + hostile := token + " \x1b[2J " + "AKIAIOSFODNN7EXAMPLE" + var stdout bytes.Buffer + var stderr bytes.Buffer + + code := Execute(context.Background(), []string{hostile}, strings.NewReader(""), &stdout, &stderr) + + if code != 2 { + t.Fatalf("exit code = %d, want 2", code) + } + if strings.Contains(stderr.String(), token) || strings.ContainsRune(stderr.String(), '\x1b') || + strings.Contains(stderr.String(), "AKIAIOSFODNN7EXAMPLE") { + t.Fatalf("stderr retained hostile input: %q", stderr.String()) + } + if !strings.Contains(stderr.String(), `[REDACTED:mattermost_credential]`) || + (!strings.Contains(stderr.String(), `\u001b`) && !strings.Contains(stderr.String(), `\x1b`)) { + t.Fatalf("stderr did not expose safe provenance: %q", stderr.String()) + } +} + func TestSchemaShowTreatsShortWriteAsReadFailure(t *testing.T) { var stderr bytes.Buffer From 64286f417c2da197c3eede7e9e3e89289da684c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 05:06:41 +0300 Subject: [PATCH 013/119] feat: port validated user identity reads --- internal/mattermost/users.go | 293 ++++++++++++++++++++++++++++++ internal/mattermost/users_test.go | 229 +++++++++++++++++++++++ 2 files changed, 522 insertions(+) create mode 100644 internal/mattermost/users.go create mode 100644 internal/mattermost/users_test.go diff --git a/internal/mattermost/users.go b/internal/mattermost/users.go new file mode 100644 index 0000000..6eaad6b --- /dev/null +++ b/internal/mattermost/users.go @@ -0,0 +1,293 @@ +// Package mattermost provides narrow, validated access to the Mattermost API. +package mattermost + +import ( + "context" + "encoding/json" + "errors" + "net/url" + "strconv" + "strings" + "sync" +) + +const ( + maxUserListPage = 200 + maxUserSearchPage = 1000 + maxUsersByID = 200 +) + +var ( + ErrInvalidUserResponse = errors.New("Mattermost returned an invalid user response") + ErrInvalidUsersResponse = errors.New("Mattermost returned an invalid users response") + ErrInvalidUserRequest = errors.New("invalid Mattermost user request") +) + +type userTransport interface { + Get(context.Context, string, any) error + PostRead(context.Context, string, any, any) error +} + +// User is the deliberately narrow user profile exposed to the rest of v2. +// Sensitive and server-internal profile fields are never retained here. +type User struct { + ID string + Username string + Nickname string + FirstName string + LastName string + Roles string +} + +func (u *User) UnmarshalJSON(data []byte) error { + var raw struct { + ID json.RawMessage `json:"id"` + Username json.RawMessage `json:"username"` + Nickname json.RawMessage `json:"nickname"` + FirstName json.RawMessage `json:"first_name"` + LastName json.RawMessage `json:"last_name"` + Roles json.RawMessage `json:"roles"` + } + if err := json.Unmarshal(data, &raw); err != nil { + return ErrInvalidUserResponse + } + id, ok := requiredString(raw.ID) + if !ok { + return ErrInvalidUserResponse + } + username, ok := requiredString(raw.Username) + if !ok { + return ErrInvalidUserResponse + } + *u = User{ + ID: id, Username: username, + Nickname: optionalString(raw.Nickname), FirstName: optionalString(raw.FirstName), + LastName: optionalString(raw.LastName), Roles: optionalString(raw.Roles), + } + return nil +} + +func requiredString(raw json.RawMessage) (string, bool) { + var value string + if len(raw) == 0 || json.Unmarshal(raw, &value) != nil || strings.TrimSpace(value) == "" { + return "", false + } + return value, true +} + +func optionalString(raw json.RawMessage) string { + var value string + if len(raw) == 0 || json.Unmarshal(raw, &value) != nil { + return "" + } + return value +} + +// DirectoryResult reports whether more matching users are known to exist. +// Truncated is nil when an endpoint ceiling prevented an honest determination. +type DirectoryResult struct { + Users []User + Truncated *bool +} + +// Users provides concurrency-safe identity lookup and session-local caching. +type Users struct { + client userTransport + mu sync.RWMutex + byID map[string]User + byName map[string]string +} + +func NewUsers(client userTransport) *Users { + return &Users{client: client, byID: make(map[string]User), byName: make(map[string]string)} +} + +func (s *Users) Current(ctx context.Context) (User, error) { + var user User + if err := s.client.Get(ctx, "/users/me", &user); err != nil { + return User{}, err + } + s.cache(user) + return user, nil +} + +func (s *Users) ByID(ctx context.Context, id string) (User, error) { + if strings.TrimSpace(id) == "" { + return User{}, ErrInvalidUserRequest + } + if user, ok := s.cachedByID(id); ok { + return user, nil + } + var user User + if err := s.client.Get(ctx, "/users/"+url.PathEscape(id), &user); err != nil { + return User{}, err + } + if user.ID != id { + return User{}, ErrInvalidUserResponse + } + s.cache(user) + return user, nil +} + +func (s *Users) ByUsername(ctx context.Context, username string) (User, error) { + if strings.TrimSpace(username) == "" { + return User{}, ErrInvalidUserRequest + } + key := strings.ToLower(username) + s.mu.RLock() + id, ok := s.byName[key] + user, present := s.byID[id] + s.mu.RUnlock() + if ok && present && strings.EqualFold(user.Username, username) { + return user, nil + } + var fetched User + if err := s.client.Get(ctx, "/users/username/"+url.PathEscape(username), &fetched); err != nil { + return User{}, err + } + if !strings.EqualFold(fetched.Username, username) { + return User{}, ErrInvalidUserResponse + } + s.cache(fetched) + return fetched, nil +} + +// ByIDs performs one bounded, retry-safe semantic read POST. The response must +// contain each requested user exactly once; incomplete results fail closed. +func (s *Users) ByIDs(ctx context.Context, ids []string) ([]User, error) { + if len(ids) == 0 { + return []User{}, nil + } + unique := make([]string, 0, len(ids)) + seen := make(map[string]struct{}, len(ids)) + for _, id := range ids { + if strings.TrimSpace(id) == "" { + return nil, ErrInvalidUserRequest + } + if _, ok := seen[id]; !ok { + seen[id] = struct{}{} + unique = append(unique, id) + } + } + if len(unique) > maxUsersByID { + return nil, ErrInvalidUserRequest + } + + missing := make([]string, 0, len(unique)) + for _, id := range unique { + if _, ok := s.cachedByID(id); !ok { + missing = append(missing, id) + } + } + if len(missing) > 0 { + var users []User + if err := s.client.PostRead(ctx, "/users/ids", missing, &users); err != nil { + return nil, err + } + if users == nil || len(users) != len(missing) { + return nil, ErrInvalidUsersResponse + } + wanted := make(map[string]struct{}, len(missing)) + for _, id := range missing { + wanted[id] = struct{}{} + } + got := make(map[string]struct{}, len(users)) + for _, user := range users { + if _, ok := wanted[user.ID]; !ok { + return nil, ErrInvalidUsersResponse + } + if _, duplicate := got[user.ID]; duplicate { + return nil, ErrInvalidUsersResponse + } + got[user.ID] = struct{}{} + } + for _, user := range users { + s.cache(user) + } + } + + result := make([]User, 0, len(ids)) + for _, id := range ids { + user, ok := s.cachedByID(id) + if !ok { + return nil, ErrInvalidUsersResponse + } + result = append(result, user) + } + return result, nil +} + +func (s *Users) Directory(ctx context.Context, query, teamID string, limit int) (DirectoryResult, error) { + if limit <= 0 { + return DirectoryResult{}, ErrInvalidUserRequest + } + query = strings.TrimSpace(query) + ceiling := maxUserListPage + if query != "" { + ceiling = maxUserSearchPage + } + probe := limit + if probe < ceiling { + probe++ + } else { + probe = ceiling + } + + var users []User + if query != "" { + body := map[string]any{"term": query, "limit": probe, "allow_inactive": false} + if teamID != "" { + body["team_id"] = teamID + } + if err := s.client.PostRead(ctx, "/users/search", body, &users); err != nil { + return DirectoryResult{}, err + } + } else { + path := "/users?page=0&per_page=" + strconv.Itoa(probe) + "&active=true" + if teamID != "" { + path += "&in_team=" + encodeQueryComponent(teamID) + } + if err := s.client.Get(ctx, path, &users); err != nil { + return DirectoryResult{}, err + } + } + if users == nil || len(users) > probe { + return DirectoryResult{}, ErrInvalidUsersResponse + } + for _, user := range users { + s.cache(user) + } + truncated := len(users) > limit + if truncated { + users = users[:limit] + } + var coverage *bool + if probe != ceiling || len(users) < ceiling || truncated { + coverage = &truncated + } + return DirectoryResult{Users: users, Truncated: coverage}, nil +} + +func (s *Users) cachedByID(id string) (User, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + user, ok := s.byID[id] + return user, ok +} + +func (s *Users) cache(user User) { + s.mu.Lock() + defer s.mu.Unlock() + if previous, ok := s.byID[user.ID]; ok && !strings.EqualFold(previous.Username, user.Username) { + previousKey := strings.ToLower(previous.Username) + if s.byName[previousKey] == user.ID { + delete(s.byName, previousKey) + } + } + s.byID[user.ID] = user + s.byName[strings.ToLower(user.Username)] = user.ID +} + +func encodeQueryComponent(value string) string { + return strings.ReplaceAll(url.QueryEscape(value), "+", "%20") +} diff --git a/internal/mattermost/users_test.go b/internal/mattermost/users_test.go new file mode 100644 index 0000000..a091616 --- /dev/null +++ b/internal/mattermost/users_test.go @@ -0,0 +1,229 @@ +package mattermost + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "sync" + "testing" +) + +type userCall struct { + method string + path string + body any +} + +type fakeUsersTransport struct { + mu sync.Mutex + responses []string + calls []userCall +} + +func (f *fakeUsersTransport) respond(method, path string, body, out any) error { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, userCall{method: method, path: path, body: body}) + if len(f.responses) == 0 { + return errors.New("unexpected request") + } + response := f.responses[0] + f.responses = f.responses[1:] + return json.Unmarshal([]byte(response), out) +} + +func (f *fakeUsersTransport) Get(_ context.Context, path string, out any) error { + return f.respond("GET", path, nil, out) +} + +func (f *fakeUsersTransport) PostRead(_ context.Context, path string, body, out any) error { + return f.respond("POST_READ", path, body, out) +} + +func TestCurrentRetainsOnlyNarrowValidatedFields(t *testing.T) { + transport := &fakeUsersTransport{responses: []string{`{ + "id":"me","username":"arda","nickname":"a","first_name":"Arda", + "last_name":42,"roles":"system_user","email":"private@example.test","props":{"secret":true} + }`}} + users := NewUsers(transport) + got, err := users.Current(context.Background()) + if err != nil { + t.Fatal(err) + } + want := User{ID: "me", Username: "arda", Nickname: "a", FirstName: "Arda", Roles: "system_user"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("user = %+v, want %+v", got, want) + } +} + +func TestUserDecodingFailsClosedForMalformedRequiredFields(t *testing.T) { + for _, payload := range []string{`null`, `{}`, `[]`, `{"id":"","username":"user"}`, `{"id":"id","username":7}`} { + t.Run(payload, func(t *testing.T) { + transport := &fakeUsersTransport{responses: []string{payload}} + _, err := NewUsers(transport).Current(context.Background()) + if !errors.Is(err, ErrInvalidUserResponse) { + t.Fatalf("error = %v", err) + } + }) + } +} + +func TestLookupsEncodePathsRequireExactIdentityAndCache(t *testing.T) { + transport := &fakeUsersTransport{responses: []string{ + `{"id":"id/with space","username":"name"}`, + `{"id":"other","username":"Name/With Space"}`, + }} + users := NewUsers(transport) + if _, err := users.ByID(context.Background(), "id/with space"); err != nil { + t.Fatal(err) + } + if _, err := users.ByID(context.Background(), "id/with space"); err != nil { + t.Fatal(err) + } + if _, err := users.ByUsername(context.Background(), "name/with space"); err != nil { + t.Fatal(err) + } + if len(transport.calls) != 2 { + t.Fatalf("calls = %d", len(transport.calls)) + } + if got := transport.calls[0].path; got != "/users/id%2Fwith%20space" { + t.Fatalf("ID path = %q", got) + } + if got := transport.calls[1].path; got != "/users/username/name%2Fwith%20space" { + t.Fatalf("username path = %q", got) + } + if _, err := users.ByUsername(context.Background(), "NAME/WITH SPACE"); err != nil { + t.Fatal(err) + } +} + +func TestLookupRejectsMismatchedResponseWithoutCaching(t *testing.T) { + transport := &fakeUsersTransport{responses: []string{ + `{"id":"different","username":"user"}`, + `{"id":"id","username":"different"}`, + }} + users := NewUsers(transport) + if _, err := users.ByID(context.Background(), "id"); !errors.Is(err, ErrInvalidUserResponse) { + t.Fatalf("ByID error = %v", err) + } + if _, err := users.ByUsername(context.Background(), "user"); !errors.Is(err, ErrInvalidUserResponse) { + t.Fatalf("ByUsername error = %v", err) + } +} + +func TestCacheReplacementCannotResolveAStaleUsername(t *testing.T) { + transport := &fakeUsersTransport{responses: []string{ + `{"id":"id","username":"old"}`, + `{"id":"id","username":"new"}`, + `{"id":"different","username":"old"}`, + }} + users := NewUsers(transport) + if _, err := users.ByUsername(context.Background(), "old"); err != nil { + t.Fatal(err) + } + if _, err := users.Current(context.Background()); err != nil { + t.Fatal(err) + } + if _, err := users.ByUsername(context.Background(), "old"); err != nil { + t.Fatal(err) + } + if len(transport.calls) != 3 { + t.Fatalf("calls = %d, stale username was served from cache", len(transport.calls)) + } +} + +func TestByIDsDeduplicatesRequestPreservesOrderAndRequiresCompleteResult(t *testing.T) { + transport := &fakeUsersTransport{responses: []string{`[ + {"id":"b","username":"bob"},{"id":"a","username":"alice"} + ]`}} + users := NewUsers(transport) + got, err := users.ByIDs(context.Background(), []string{"a", "b", "a"}) + if err != nil { + t.Fatal(err) + } + if ids := []string{got[0].ID, got[1].ID, got[2].ID}; !reflect.DeepEqual(ids, []string{"a", "b", "a"}) { + t.Fatalf("IDs = %v", ids) + } + if len(transport.calls) != 1 || transport.calls[0].method != "POST_READ" { + t.Fatalf("calls = %+v", transport.calls) + } + if !reflect.DeepEqual(transport.calls[0].body, []string{"a", "b"}) { + t.Fatalf("body = %#v", transport.calls[0].body) + } + + incomplete := NewUsers(&fakeUsersTransport{responses: []string{`[{"id":"a","username":"alice"}]`}}) + if _, err := incomplete.ByIDs(context.Background(), []string{"a", "b"}); !errors.Is(err, ErrInvalidUsersResponse) { + t.Fatalf("incomplete error = %v", err) + } +} + +func TestDirectoryUsesBoundedProbeAndHonestCoverage(t *testing.T) { + transport := &fakeUsersTransport{responses: []string{ + `[{"id":"a","username":"a"},{"id":"b","username":"b"},{"id":"c","username":"c"}]`, + `[]`, + }} + users := NewUsers(transport) + result, err := users.Directory(context.Background(), " dev ", "team/one", 2) + if err != nil { + t.Fatal(err) + } + if result.Truncated == nil || !*result.Truncated || len(result.Users) != 2 { + t.Fatalf("result = %+v", result) + } + wantBody := map[string]any{"term": "dev", "team_id": "team/one", "limit": 3, "allow_inactive": false} + if !reflect.DeepEqual(transport.calls[0].body, wantBody) { + t.Fatalf("body = %#v", transport.calls[0].body) + } + + result, err = users.Directory(context.Background(), "", "team/one space", 2) + if err != nil { + t.Fatal(err) + } + if result.Truncated == nil || *result.Truncated { + t.Fatalf("result = %+v", result) + } + if got := transport.calls[1].path; got != "/users?page=0&per_page=3&active=true&in_team=team%2Fone%20space" { + t.Fatalf("path = %q", got) + } +} + +func TestDirectoryReportsUnknownAtEndpointCeilings(t *testing.T) { + items := make([]map[string]string, maxUserListPage) + for i := range items { + items[i] = map[string]string{"id": "id" + string(rune(i+1)), "username": "u" + string(rune(i+1))} + } + payload, err := json.Marshal(items) + if err != nil { + t.Fatal(err) + } + transport := &fakeUsersTransport{responses: []string{string(payload)}} + result, err := NewUsers(transport).Directory(context.Background(), "", "", 300) + if err != nil { + t.Fatal(err) + } + if result.Truncated != nil { + t.Fatalf("truncated = %v, want unknown", *result.Truncated) + } + if got := transport.calls[0].path; got != "/users?page=0&per_page=200&active=true" { + t.Fatalf("path = %q", got) + } +} + +func TestUsersCacheIsRaceSafe(t *testing.T) { + transport := &fakeUsersTransport{responses: []string{`{"id":"id","username":"user"}`}} + users := NewUsers(transport) + if _, err := users.ByID(context.Background(), "id"); err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + for range 50 { + wg.Add(2) + go func() { defer wg.Done(); _, _ = users.ByID(context.Background(), "id") }() + go func() { defer wg.Done(); _, _ = users.ByUsername(context.Background(), "USER") }() + } + wg.Wait() + if len(transport.calls) != 1 { + t.Fatalf("calls = %d", len(transport.calls)) + } +} From 11cd36cb0fe2957ca61c41a453b341597fb5bea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 05:14:34 +0300 Subject: [PATCH 014/119] fix: classify CLI error write failures --- internal/cli/root.go | 4 +++- internal/cli/root_test.go | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/internal/cli/root.go b/internal/cli/root.go index c4299fe..89b4e47 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -28,7 +28,9 @@ func Execute(ctx context.Context, args []string, in io.Reader, out, errOut io.Wr cmd.SetArgs(args) if err := cmd.ExecuteContext(ctx); err != nil { message := presentation.SanitizeLabel(presentation.PreprocessActive(err.Error()).Text) - _, _ = fmt.Fprintf(errOut, "error: %s\n", message) + if writeErr := writeAll(errOut, []byte(fmt.Sprintf("error: %s\n", message))); writeErr != nil { + return 3 + } var outputFailure outputError if errors.As(err, &outputFailure) { return 3 diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index cbb2b00..0f9d469 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -125,6 +125,14 @@ func TestSchemaShowTreatsShortWriteAsReadFailure(t *testing.T) { } } +func TestErrorOutputShortWriteReturnsOutputFailure(t *testing.T) { + var stdout bytes.Buffer + code := Execute(context.Background(), []string{"unknown"}, strings.NewReader(""), &stdout, shortWriter{}) + if code != 3 { + t.Fatalf("exit code = %d, want 3", code) + } +} + type shortWriter struct{} func (shortWriter) Write(data []byte) (int, error) { From 154f4a78f74a0922b48613a3d066811c564f4dc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 05:15:44 +0300 Subject: [PATCH 015/119] feat: port bounded readiness diagnostics --- internal/doctor/doctor.go | 212 ++++++++++++++++++++++++ internal/doctor/doctor_test.go | 287 +++++++++++++++++++++++++++++++++ 2 files changed, 499 insertions(+) create mode 100644 internal/doctor/doctor.go create mode 100644 internal/doctor/doctor_test.go diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go new file mode 100644 index 0000000..859f078 --- /dev/null +++ b/internal/doctor/doctor.go @@ -0,0 +1,212 @@ +// Package doctor performs the fixed, read-only Mattermost readiness checks. +package doctor + +import ( + "context" + "encoding/json" + "errors" + "strings" + "time" + + "github.com/ardasevinc/mattermost-cli/internal/api" + "github.com/ardasevinc/mattermost-cli/internal/config" + "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/internal/serverurl" +) + +type Status string + +const ( + StatusPass Status = "pass" + StatusWarn Status = "warn" + StatusFail Status = "fail" + StatusSkipped Status = "skipped" +) + +type Check struct { + Name string `json:"name"` + Status Status `json:"status"` + Message string `json:"message"` + Details map[string]any `json:"details,omitempty"` +} + +type Report struct { + OK bool `json:"ok"` + Checks []Check `json:"checks"` +} + +// Transport is the complete network authority required by doctor. +type Transport interface { + GetPublic(context.Context, string, any) error + Get(context.Context, string, any) error +} + +// Factory binds doctor's transport authority to the normalized resolved URL and +// exact resolved credential. The returned close function releases that authority. +type Factory func(baseURL, token string) (Transport, func(), error) + +const checkTimeout = 10 * time.Second + +func Run(ctx context.Context, resolved config.Resolved, factory Factory) Report { + checks := make([]Check, 0, 3) + checks = append(checks, configurationCheck(resolved)) + + normalizedURL, urlErr := serverurl.Normalize(resolved.URL) + var transport Transport + var factoryErr error + closeTransport := func() {} + if resolved.URL != "" && urlErr == nil { + if factory == nil { + factoryErr = errors.New("transport factory unavailable") + } else { + transport, closeTransport, factoryErr = factory(normalizedURL, resolved.Token) + if closeTransport == nil { + closeTransport = func() {} + } + } + } + defer closeTransport() + if resolved.URL == "" { + checks = append(checks, Check{Name: "server", Status: StatusSkipped, Message: "Mattermost URL is missing"}) + } else if urlErr != nil { + checks = append(checks, Check{Name: "server", Status: StatusFail, Message: "Mattermost URL is invalid or unsafe"}) + } else { + var ping pingResponse + requestCtx, cancel := context.WithTimeout(ctx, checkTimeout) + err := factoryErr + if err == nil { + err = transportError(transport, func(t Transport) error { + return t.GetPublic(requestCtx, "/system/ping?get_server_status=true", &ping) + }) + } + cancel() + checks = append(checks, serverCheck(resolved, ping, err)) + } + + if resolved.Token == "" { + checks = append(checks, Check{Name: "authentication", Status: StatusSkipped, Message: "Mattermost token is missing"}) + } else if resolved.URL == "" || urlErr != nil { + checks = append(checks, Check{Name: "authentication", Status: StatusSkipped, Message: "valid Mattermost URL is missing"}) + } else { + var identity identityResponse + requestCtx, cancel := context.WithTimeout(ctx, checkTimeout) + err := factoryErr + if err == nil { + err = transportError(transport, func(t Transport) error { + return t.Get(requestCtx, "/users/me", &identity) + }) + } + cancel() + checks = append(checks, authenticationCheck(resolved, identity, err)) + } + + ok := true + for _, check := range checks { + if check.Status == StatusFail { + ok = false + } + } + return Report{OK: ok, Checks: checks} +} + +func configurationCheck(resolved config.Resolved) Check { + details := map[string]any{"urlSource": resolved.URLSource, "tokenSource": resolved.TokenSource} + if resolved.File.Error != "" || resolved.File.Unsafe != "" { + return Check{Name: "configuration", Status: StatusFail, Message: "config file could not be loaded", Details: details} + } + if resolved.File.InsecurePermissions && resolved.File.Config.Token != "" { + return Check{Name: "configuration", Status: StatusFail, Message: "config file permissions expose a stored token; run chmod 600", Details: details} + } + if resolved.URL == "" || resolved.Token == "" { + return Check{Name: "configuration", Status: StatusFail, Message: "configuration is incomplete", Details: details} + } + if resolved.File.InsecurePermissions { + return Check{Name: "configuration", Status: StatusWarn, Message: "config file permissions are broader than recommended; run chmod 600", Details: details} + } + return Check{Name: "configuration", Status: StatusPass, Message: "credentials resolved", Details: details} +} + +type remoteString struct { + value string + valid bool +} + +func (s *remoteString) UnmarshalJSON(data []byte) error { + var value string + if json.Unmarshal(data, &value) == nil && value != "" { + s.value, s.valid = value, true + } + return nil +} + +type pingResponse struct { + Status remoteString `json:"status"` + DatabaseStatus remoteString `json:"database_status"` + FilestoreStatus remoteString `json:"filestore_status"` +} + +func serverCheck(resolved config.Resolved, ping pingResponse, err error) Check { + if err != nil { + return failedRequest("server", "server health request failed", err) + } + status := safeRemote(ping.Status, resolved) + database := safeRemote(ping.DatabaseStatus, resolved) + filestore := safeRemote(ping.FilestoreStatus, resolved) + details := map[string]any{"status": status, "databaseStatus": database, "filestoreStatus": filestore} + values := []string{status, database, filestore} + for _, value := range values { + if value != "OK" && value != "unknown" { + return Check{Name: "server", Status: StatusFail, Message: "server reported an unhealthy component", Details: details} + } + } + for _, value := range values { + if value == "unknown" { + return Check{Name: "server", Status: StatusWarn, Message: "server responded with incomplete health data", Details: details} + } + } + return Check{Name: "server", Status: StatusPass, Message: "server is healthy", Details: details} +} + +type identityResponse struct { + ID remoteString `json:"id"` + Username remoteString `json:"username"` +} + +func authenticationCheck(resolved config.Resolved, identity identityResponse, err error) Check { + if err != nil { + return failedRequest("authentication", "authentication request failed", err) + } + if !identity.ID.valid || strings.TrimSpace(identity.ID.value) == "" || + !identity.Username.valid || strings.TrimSpace(identity.Username.value) == "" { + return Check{Name: "authentication", Status: StatusFail, Message: "authentication response was invalid"} + } + return Check{Name: "authentication", Status: StatusPass, Message: "authenticated", Details: map[string]any{ + "id": safeRemote(identity.ID, resolved), "username": safeRemote(identity.Username, resolved), + }} +} + +func safeRemote(value remoteString, resolved config.Resolved) string { + if !value.valid { + return "unknown" + } + processed := presentation.PreprocessWithOptions(value.value, presentation.Options{ + Credentials: []string{resolved.Token}, DisableHeuristics: !resolved.Redact, + }) + return presentation.SanitizeLabel(processed.Text) +} + +func failedRequest(name, message string, err error) Check { + check := Check{Name: name, Status: StatusFail, Message: message} + var apiError *api.APIError + if errors.As(err, &apiError) && apiError.Status > 0 { + check.Details = map[string]any{"httpStatus": apiError.Status} + } + return check +} + +func transportError(transport Transport, request func(Transport) error) error { + if transport == nil { + return errors.New("transport unavailable") + } + return request(transport) +} diff --git a/internal/doctor/doctor_test.go b/internal/doctor/doctor_test.go new file mode 100644 index 0000000..6f65c49 --- /dev/null +++ b/internal/doctor/doctor_test.go @@ -0,0 +1,287 @@ +package doctor + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/ardasevinc/mattermost-cli/internal/api" + "github.com/ardasevinc/mattermost-cli/internal/config" +) + +type fakeCall struct { + public bool + path string + ctx context.Context +} + +type fakeResult struct { + value any + err error + run func(context.Context, any) error +} + +type fakeTransport struct { + results []fakeResult + calls []fakeCall +} + +type fakeFactory struct { + transport *fakeTransport + baseURL string + token string + closed int + err error +} + +func (f *fakeFactory) make(baseURL, token string) (Transport, func(), error) { + f.baseURL, f.token = baseURL, token + if f.err != nil { + return nil, nil, f.err + } + return f.transport, func() { f.closed++ }, nil +} + +func factoryFor(transport *fakeTransport) Factory { + factory := &fakeFactory{transport: transport} + return factory.make +} + +func (f *fakeTransport) GetPublic(ctx context.Context, path string, out any) error { + return f.call(ctx, path, out, true) +} + +func (f *fakeTransport) Get(ctx context.Context, path string, out any) error { + return f.call(ctx, path, out, false) +} + +func (f *fakeTransport) call(ctx context.Context, path string, out any, public bool) error { + f.calls = append(f.calls, fakeCall{public: public, path: path, ctx: ctx}) + result := f.results[len(f.calls)-1] + if result.run != nil { + return result.run(ctx, out) + } + if result.err != nil { + return result.err + } + data, err := json.Marshal(result.value) + if err != nil { + return err + } + return json.Unmarshal(data, out) +} + +func completeConfig() config.Resolved { + return config.Resolved{ + URL: "https://mm.example", Token: "active-token", Redact: true, + URLSource: config.SourceCLI, TokenSource: config.SourceEnv, + } +} + +func healthy() map[string]any { + return map[string]any{"status": "OK", "database_status": "OK", "filestore_status": "OK"} +} + +func TestRunHealthyUsesOnlyNarrowReadEndpoints(t *testing.T) { + transport := &fakeTransport{results: []fakeResult{{value: healthy()}, {value: map[string]any{ + "id": "user-id", "username": "arda", "email": "private", "roles": "system_admin", + }}}} + factory := &fakeFactory{transport: transport} + report := Run(context.Background(), config.Resolved{URL: " HTTPS://MM.Example:443/base/../ ", Token: "active-token", Redact: true}, factory.make) + if !report.OK || len(report.Checks) != 3 { + t.Fatalf("Run() = %+v", report) + } + if len(transport.calls) != 2 || !transport.calls[0].public || transport.calls[0].path != "/system/ping?get_server_status=true" || + transport.calls[1].public || transport.calls[1].path != "/users/me" { + t.Fatalf("calls = %+v", transport.calls) + } + if got := report.Checks[2].Details; len(got) != 2 || got["id"] != "user-id" || got["username"] != "arda" { + t.Fatalf("authentication details = %#v", got) + } + if factory.baseURL != "https://mm.example" || factory.token != "active-token" || factory.closed != 1 { + t.Fatalf("factory binding/lifecycle = url %q token %q closes %d", factory.baseURL, factory.token, factory.closed) + } +} + +func TestRunContinuesAfterFailuresWithoutReflectingErrors(t *testing.T) { + token := "never-print-this" + resolved := completeConfig() + resolved.Token = token + transport := &fakeTransport{results: []fakeResult{ + {err: fmt.Errorf("remote body: %s: %w", token, &api.APIError{Status: 503})}, + {err: fmt.Errorf("hostile %s: %w", token, &api.APIError{Status: 401})}, + }} + report := Run(context.Background(), resolved, factoryFor(transport)) + encoded, _ := json.Marshal(report) + if len(report.Checks) != 3 || len(transport.calls) != 2 || report.OK { + t.Fatalf("Run() = %+v, calls = %d", report, len(transport.calls)) + } + if strings.Contains(string(encoded), token) || strings.Contains(string(encoded), "remote body") || strings.Contains(string(encoded), "hostile") { + t.Fatalf("report reflected an error: %s", encoded) + } + if report.Checks[1].Details["httpStatus"] != 503 || report.Checks[2].Details["httpStatus"] != 401 { + t.Fatalf("status details = %#v / %#v", report.Checks[1].Details, report.Checks[2].Details) + } +} + +func TestRunSanitizesAndRedactsEveryRemoteString(t *testing.T) { + for _, redact := range []bool{true, false} { + resolved := completeConfig() + resolved.Redact = redact + token := resolved.Token + control := "\x1b]8;;https://evil.example\x07click\x1b]8;;\x07" + probe := "ghp_abcdefghijklmnopqrstuvwxyz1234567890" + transport := &fakeTransport{results: []fakeResult{ + {value: map[string]any{"status": "OK", "database_status": token + control, "filestore_status": "OK", "secret": token}}, + {value: map[string]any{"id": token + control, "username": "arda" + control + probe, "email": token}}, + }} + report := Run(context.Background(), resolved, factoryFor(transport)) + encoded, _ := json.Marshal(report) + output := string(encoded) + if strings.Contains(output, token) || strings.Contains(output, "\x1b") || strings.Contains(output, "\x07") { + t.Fatalf("redact=%v leaked unsafe value: %q", redact, output) + } + if !strings.Contains(output, "REDACTED") || !strings.Contains(output, `\\u001b`) { + t.Fatalf("redact=%v did not visibly sanitize/redact: %q", redact, output) + } + if redact && strings.Contains(output, probe) { + t.Fatalf("heuristic secret leaked: %q", output) + } + } +} + +func TestRunHandlesMalformedAndIncompleteResponses(t *testing.T) { + transport := &fakeTransport{results: []fakeResult{ + {value: map[string]any{"status": "OK", "database_status": 42, "filestore_status": "OK"}}, + {value: map[string]any{"id": " ", "username": []string{"arda"}}}, + }} + report := Run(context.Background(), completeConfig(), factoryFor(transport)) + if report.Checks[1].Status != StatusWarn || report.Checks[1].Details["databaseStatus"] != "unknown" { + t.Fatalf("server check = %+v", report.Checks[1]) + } + if report.Checks[2].Status != StatusFail || report.Checks[2].Message != "authentication response was invalid" { + t.Fatalf("authentication check = %+v", report.Checks[2]) + } +} + +func TestConfigurationPermissionAndSourceSemantics(t *testing.T) { + tests := []struct { + name string + mutate func(*config.Resolved) + status Status + message string + }{ + {"stored token exposed", func(r *config.Resolved) { r.File.InsecurePermissions = true; r.File.Config.Token = "file-secret" }, StatusFail, "config file permissions expose a stored token; run chmod 600"}, + {"tokenless file warns", func(r *config.Resolved) { r.File.InsecurePermissions = true }, StatusWarn, "config file permissions are broader than recommended; run chmod 600"}, + {"unsafe file fails", func(r *config.Resolved) { r.File.Unsafe = config.UnsafeOwnership }, StatusFail, "config file could not be loaded"}, + {"malformed file fails", func(r *config.Resolved) { r.File.Error = config.FileErrorParse }, StatusFail, "config file could not be loaded"}, + {"missing token fails", func(r *config.Resolved) { r.Token = ""; r.TokenSource = config.SourceMissing }, StatusFail, "configuration is incomplete"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + resolved := completeConfig() + test.mutate(&resolved) + transport := &fakeTransport{results: []fakeResult{{value: healthy()}, {value: map[string]any{"id": "id", "username": "user"}}}} + report := Run(context.Background(), resolved, factoryFor(transport)) + check := report.Checks[0] + if check.Status != test.status || check.Message != test.message { + t.Fatalf("configuration check = %+v", check) + } + if check.Details["urlSource"] != config.SourceCLI || check.Details["tokenSource"] != resolved.TokenSource { + t.Fatalf("source details = %#v", check.Details) + } + encoded, _ := json.Marshal(report) + if strings.Contains(string(encoded), "file-secret") || (resolved.Token != "" && strings.Contains(string(encoded), resolved.Token)) { + t.Fatalf("report leaked credential: %s", encoded) + } + }) + } +} + +func TestMissingAndUnsafeConfigurationStillProducesThreeChecks(t *testing.T) { + missing := completeConfig() + missing.Token = "" + transport := &fakeTransport{results: []fakeResult{{value: healthy()}}} + report := Run(context.Background(), missing, factoryFor(transport)) + if len(report.Checks) != 3 || report.Checks[1].Status != StatusPass || report.Checks[2].Status != StatusSkipped || len(transport.calls) != 1 { + t.Fatalf("missing token report = %+v, calls = %+v", report, transport.calls) + } + + unsafe := completeConfig() + unsafe.URL = "http://mm.example" + transport = &fakeTransport{} + report = Run(context.Background(), unsafe, factoryFor(transport)) + if len(report.Checks) != 3 || report.Checks[1].Status != StatusFail || report.Checks[2].Status != StatusSkipped || len(transport.calls) != 0 { + t.Fatalf("unsafe URL report = %+v, calls = %+v", report, transport.calls) + } +} + +func TestRequestsHaveIndependentChildContexts(t *testing.T) { + type contextKey string + parent := context.WithValue(context.Background(), contextKey("proof"), "inherited") + transport := &fakeTransport{results: []fakeResult{{value: healthy()}, {value: map[string]any{"id": "id", "username": "user"}}}} + Run(parent, completeConfig(), factoryFor(transport)) + if len(transport.calls) != 2 || transport.calls[0].ctx == parent || transport.calls[1].ctx == parent || transport.calls[0].ctx == transport.calls[1].ctx { + t.Fatalf("request contexts are not independent children") + } + for _, call := range transport.calls { + deadline, ok := call.ctx.Deadline() + if call.ctx.Value(contextKey("proof")) != "inherited" || !errors.Is(call.ctx.Err(), context.Canceled) || !ok || time.Until(deadline) > checkTimeout { + t.Fatalf("request context did not inherit parent or was not released") + } + } +} + +func TestPingTimeoutDoesNotPreventAuthentication(t *testing.T) { + transport := &fakeTransport{results: []fakeResult{ + {run: func(ctx context.Context, _ any) error { + deadline, ok := ctx.Deadline() + if !ok || time.Until(deadline) < 9*time.Second || time.Until(deadline) > checkTimeout { + return errors.New("missing doctor deadline") + } + return context.DeadlineExceeded + }}, + {value: map[string]any{"id": "id", "username": "user"}}, + }} + report := Run(context.Background(), completeConfig(), factoryFor(transport)) + if len(transport.calls) != 2 || report.Checks[1].Status != StatusFail || report.Checks[2].Status != StatusPass { + t.Fatalf("timeout continuation = %+v, calls = %+v", report, transport.calls) + } +} + +func TestParentDeadlineWins(t *testing.T) { + parent, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + transport := &fakeTransport{results: []fakeResult{ + {run: func(ctx context.Context, out any) error { + if deadline, ok := ctx.Deadline(); !ok || time.Until(deadline) > time.Second { + return errors.New("parent deadline did not win") + } + return json.Unmarshal([]byte(`{"status":"OK","database_status":"OK","filestore_status":"OK"}`), out) + }}, + {value: map[string]any{"id": "id", "username": "user"}}, + }} + report := Run(parent, completeConfig(), factoryFor(transport)) + if !report.OK { + t.Fatalf("Run() = %+v", report) + } +} + +func TestFactoryFailureIsGenericAndCloseIsNotRequired(t *testing.T) { + token := "factory-secret-token" + resolved := completeConfig() + resolved.Token = token + factory := &fakeFactory{err: fmt.Errorf("could not build for %s", token)} + report := Run(context.Background(), resolved, factory.make) + encoded, _ := json.Marshal(report) + if len(report.Checks) != 3 || report.Checks[1].Status != StatusFail || report.Checks[2].Status != StatusFail || factory.closed != 0 { + t.Fatalf("factory failure report = %+v, closes = %d", report, factory.closed) + } + if strings.Contains(string(encoded), token) || strings.Contains(string(encoded), "could not build") { + t.Fatalf("factory error reflected: %s", encoded) + } +} From b4813bf3aebee407e3717c1aa8865c1c6fbf154d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 05:43:34 +0300 Subject: [PATCH 016/119] feat: port validated team and channel reads --- internal/mattermost/channels.go | 215 +++++++++++++++++++++++++++ internal/mattermost/channels_test.go | 152 +++++++++++++++++++ internal/mattermost/teams.go | 141 ++++++++++++++++++ internal/mattermost/teams_test.go | 95 ++++++++++++ 4 files changed, 603 insertions(+) create mode 100644 internal/mattermost/channels.go create mode 100644 internal/mattermost/channels_test.go create mode 100644 internal/mattermost/teams.go create mode 100644 internal/mattermost/teams_test.go diff --git a/internal/mattermost/channels.go b/internal/mattermost/channels.go new file mode 100644 index 0000000..67a5b2e --- /dev/null +++ b/internal/mattermost/channels.go @@ -0,0 +1,215 @@ +package mattermost + +import ( + "context" + "encoding/json" + "errors" + "net/url" + "sort" + "strings" +) + +var ( + ErrInvalidChannelResponse = errors.New("Mattermost returned an invalid channel response") + ErrInvalidChannelsResponse = errors.New("Mattermost returned an invalid channels response") + ErrInvalidChannelRequest = errors.New("invalid Mattermost channel request") +) + +type channelTransport interface { + Get(context.Context, string, any) error +} + +type Channel struct { + ID string + TeamID string + Type string + Name string + DisplayName string +} + +func (c *Channel) UnmarshalJSON(data []byte) error { + var raw struct { + ID json.RawMessage `json:"id"` + TeamID json.RawMessage `json:"team_id"` + Type json.RawMessage `json:"type"` + Name json.RawMessage `json:"name"` + DisplayName json.RawMessage `json:"display_name"` + } + if err := json.Unmarshal(data, &raw); err != nil { + return ErrInvalidChannelResponse + } + id, idOK := requiredString(raw.ID) + typeCode, typeOK := requiredString(raw.Type) + name, nameOK := requiredString(raw.Name) + teamID, teamOK := strictString(raw.TeamID) + displayName, displayOK := strictString(raw.DisplayName) + if !idOK || !typeOK || !nameOK || !teamOK || !displayOK { + return ErrInvalidChannelResponse + } + if typeCode != "O" && typeCode != "P" && typeCode != "D" && typeCode != "G" { + return ErrInvalidChannelResponse + } + if typeCode == "O" || typeCode == "P" { + if strings.TrimSpace(teamID) == "" { + return ErrInvalidChannelResponse + } + } else if teamID != "" { + return ErrInvalidChannelResponse + } + if typeCode == "D" { + parts := strings.Split(name, "__") + if len(parts) != 2 || strings.TrimSpace(parts[0]) == "" || strings.TrimSpace(parts[1]) == "" { + return ErrInvalidChannelResponse + } + } + *c = Channel{ID: id, TeamID: teamID, Type: typeCode, Name: name, DisplayName: displayName} + return nil +} + +func strictString(raw json.RawMessage) (string, bool) { + var value string + if len(raw) == 0 || json.Unmarshal(raw, &value) != nil { + return "", false + } + return value, true +} + +type ChannelMember struct { + ChannelID string + UserID string +} + +func (m *ChannelMember) UnmarshalJSON(data []byte) error { + var raw struct { + ChannelID json.RawMessage `json:"channel_id"` + UserID json.RawMessage `json:"user_id"` + } + if err := json.Unmarshal(data, &raw); err != nil { + return ErrInvalidChannelResponse + } + channelID, channelOK := requiredString(raw.ChannelID) + userID, userOK := requiredString(raw.UserID) + if !channelOK || !userOK { + return ErrInvalidChannelResponse + } + *m = ChannelMember{ChannelID: channelID, UserID: userID} + return nil +} + +type Channels struct{ client channelTransport } + +func NewChannels(client channelTransport) *Channels { return &Channels{client: client} } + +type channelList []Channel + +func (l *channelList) UnmarshalJSON(data []byte) error { + var channels []Channel + if err := json.Unmarshal(data, &channels); err != nil || channels == nil { + return ErrInvalidChannelsResponse + } + *l = channels + return nil +} + +func (s *Channels) ByID(ctx context.Context, channelID string) (Channel, error) { + if strings.TrimSpace(channelID) == "" { + return Channel{}, ErrInvalidChannelRequest + } + var channel Channel + if err := s.client.Get(ctx, "/channels/"+url.PathEscape(channelID), &channel); err != nil { + return Channel{}, err + } + if channel.ID != channelID { + return Channel{}, ErrInvalidChannelResponse + } + return channel, nil +} + +func (s *Channels) ByName(ctx context.Context, teamID, name string) (Channel, error) { + name = strings.TrimPrefix(name, "#") + if strings.TrimSpace(teamID) == "" || strings.TrimSpace(name) == "" { + return Channel{}, ErrInvalidChannelRequest + } + var channel Channel + path := "/teams/" + url.PathEscape(teamID) + "/channels/name/" + url.PathEscape(name) + if err := s.client.Get(ctx, path, &channel); err != nil { + return Channel{}, err + } + if channel.TeamID != teamID || channel.Name != name || (channel.Type != "O" && channel.Type != "P") { + return Channel{}, ErrInvalidChannelResponse + } + return channel, nil +} + +func (s *Channels) Member(ctx context.Context, channelID, userID string) (ChannelMember, error) { + if strings.TrimSpace(channelID) == "" || strings.TrimSpace(userID) == "" || userID == "me" { + return ChannelMember{}, ErrInvalidChannelRequest + } + var member ChannelMember + path := "/channels/" + url.PathEscape(channelID) + "/members/" + url.PathEscape(userID) + if err := s.client.Get(ctx, path, &member); err != nil { + return ChannelMember{}, err + } + if member.ChannelID != channelID || member.UserID != userID { + return ChannelMember{}, ErrInvalidChannelResponse + } + return member, nil +} + +// List returns the authenticated user's channels with every identity binding +// checked before any result is released. Team membership is fetched through +// the same transport so proof cannot be mixed across sessions or servers. +func (s *Channels) List(ctx context.Context, userID string) ([]Channel, error) { + if strings.TrimSpace(userID) == "" || userID == "me" { + return nil, ErrInvalidChannelRequest + } + var decoded channelList + if err := s.client.Get(ctx, "/users/"+url.PathEscape(userID)+"/channels", &decoded); err != nil { + return nil, err + } + channels := []Channel(decoded) + var membership TeamMembership + for _, channel := range channels { + if channel.Type == "O" || channel.Type == "P" { + var err error + membership, err = NewTeams(s.client).List(ctx, userID) + if err != nil { + return nil, err + } + break + } + } + seen := make(map[string]Channel, len(channels)) + result := make([]Channel, 0, len(channels)) + for _, channel := range channels { + if previous, duplicate := seen[channel.ID]; duplicate { + if previous != channel { + return nil, ErrInvalidChannelsResponse + } + continue + } + seen[channel.ID] = channel + switch channel.Type { + case "O", "P": + if !membership.contains(channel.TeamID) { + return nil, ErrInvalidChannelsResponse + } + case "D": + if !directChannelContains(channel.Name, userID) { + return nil, ErrInvalidChannelResponse + } + case "G": + if _, err := s.Member(ctx, channel.ID, userID); err != nil { + return nil, err + } + } + result = append(result, channel) + } + sort.Slice(result, func(i, j int) bool { return result[i].ID < result[j].ID }) + return result, nil +} + +func directChannelContains(name, userID string) bool { + parts := strings.Split(name, "__") + return len(parts) == 2 && (parts[0] == userID || parts[1] == userID) +} diff --git a/internal/mattermost/channels_test.go b/internal/mattermost/channels_test.go new file mode 100644 index 0000000..f0fa1bc --- /dev/null +++ b/internal/mattermost/channels_test.go @@ -0,0 +1,152 @@ +package mattermost + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "sync" + "testing" +) + +type fakeChannelTransport struct { + mu sync.Mutex + responses map[string]string + paths []string +} + +func (f *fakeChannelTransport) Get(_ context.Context, path string, out any) error { + f.mu.Lock() + defer f.mu.Unlock() + f.paths = append(f.paths, path) + payload, ok := f.responses[path] + if !ok { + return errors.New("unexpected request") + } + return json.Unmarshal([]byte(payload), out) +} + +func TestChannelLookupsEncodeAndRequireExactIdentity(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{ + "/channels/channel%2Fone": `{"id":"channel/one","team_id":"team/one","type":"P","name":"release/name","display_name":"Release"}`, + "/teams/team%2Fone/channels/name/release%2Fname": `{"id":"channel/one","team_id":"team/one","type":"P","name":"release/name","display_name":"Release"}`, + }} + channels := NewChannels(f) + if _, err := channels.ByID(context.Background(), "channel/one"); err != nil { + t.Fatal(err) + } + if _, err := channels.ByName(context.Background(), "team/one", "#release/name"); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(f.paths, []string{"/channels/channel%2Fone", "/teams/team%2Fone/channels/name/release%2Fname"}) { + t.Fatalf("paths = %v", f.paths) + } +} + +func TestChannelDecodingFailsClosedForRequiredShape(t *testing.T) { + bad := []string{ + `null`, `{}`, `{"id":"remote-secret","team_id":"","type":"X","name":"x","display_name":""}`, + `{"id":"x","team_id":"","type":"O","name":"x","display_name":""}`, + `{"id":"x","team_id":"team","type":"G","name":"x","display_name":""}`, + `{"id":"x","team_id":" ","type":"G","name":"x","display_name":""}`, + `{"id":"x","team_id":" ","type":"D","name":"a__b","display_name":""}`, + `{"id":"x","team_id":"","type":"D","name":"alice","display_name":""}`, + } + for _, payload := range bad { + f := &fakeChannelTransport{responses: map[string]string{"/channels/x": payload}} + _, err := NewChannels(f).ByID(context.Background(), "x") + if !errors.Is(err, ErrInvalidChannelResponse) { + t.Fatalf("payload %s: error = %v", payload, err) + } + if contains(err.Error(), "remote-secret") { + t.Fatalf("error reflected remote data: %v", err) + } + } +} + +func TestChannelListBindsTeamsAndDirectAndGroupParticipants(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{ + "/users/user/teams": `[{"id":"team","name":"core","display_name":"Core","type":"O"}]`, + "/users/user/channels": `[ + {"id":"public","team_id":"team","type":"O","name":"general","display_name":"General"}, + {"id":"dm","team_id":"","type":"D","name":"user__alice","display_name":""}, + {"id":"group","team_id":"","type":"G","name":"opaque","display_name":"Crew"}, + {"id":"public","team_id":"team","type":"O","name":"general","display_name":"General"} + ]`, + "/channels/group/members/user": `{"channel_id":"group","user_id":"user","roles":"channel_user"}`, + }} + got, err := NewChannels(f).List(context.Background(), "user") + if err != nil { + t.Fatal(err) + } + if ids := []string{got[0].ID, got[1].ID, got[2].ID}; !reflect.DeepEqual(ids, []string{"dm", "group", "public"}) { + t.Fatalf("IDs = %v", ids) + } + for _, path := range f.paths { + if path == "/channels/direct" { + t.Fatal("read path attempted channel creation") + } + } +} + +func TestChannelListRejectsIncompleteBindings(t *testing.T) { + for name, payload := range map[string]string{ + "foreign team": `[{"id":"x","team_id":"other","type":"P","name":"private","display_name":""}]`, + "foreign direct participant": `[{"id":"x","team_id":"","type":"D","name":"alice__bob","display_name":""}]`, + "conflicting duplicate": `[{"id":"x","team_id":"team","type":"O","name":"one","display_name":""},{"id":"x","team_id":"team","type":"O","name":"two","display_name":""}]`, + } { + t.Run(name, func(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{ + "/users/user/teams": `[{"id":"team","name":"core","display_name":"Core","type":"O"}]`, + "/users/user/channels": payload, + }} + _, err := NewChannels(f).List(context.Background(), "user") + if err == nil { + t.Fatal("expected binding error") + } + }) + } + f := &fakeChannelTransport{responses: map[string]string{ + "/users/user/channels": `[{"id":"group","team_id":"","type":"G","name":"opaque","display_name":""}]`, + "/channels/group/members/user": `{"channel_id":"other","user_id":"user"}`, + }} + if _, err := NewChannels(f).List(context.Background(), "user"); !errors.Is(err, ErrInvalidChannelResponse) { + t.Fatalf("group binding error = %v", err) + } +} + +func TestChannelListAndMemberRejectAlias(t *testing.T) { + channels := NewChannels(&fakeChannelTransport{}) + if _, err := channels.List(context.Background(), "me"); !errors.Is(err, ErrInvalidChannelRequest) { + t.Fatalf("me alias error = %v", err) + } + if _, err := channels.Member(context.Background(), "channel", "me"); !errors.Is(err, ErrInvalidChannelRequest) { + t.Fatalf("member alias error = %v", err) + } +} + +func TestChannelListDoesNotRequireTeamsForDirectOnlyDiscovery(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{ + "/users/user/channels": `[{"id":"dm","team_id":"","type":"D","name":"user__other","display_name":""}]`, + }} + got, err := NewChannels(f).List(context.Background(), "user") + if err != nil || len(got) != 1 || got[0].ID != "dm" { + t.Fatalf("channels = %#v, error = %v", got, err) + } + if !reflect.DeepEqual(f.paths, []string{"/users/user/channels"}) { + t.Fatalf("paths = %v", f.paths) + } +} + +func TestChannelReadsAreRaceSafe(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{ + "/channels/x": `{"id":"x","team_id":"team","type":"O","name":"general","display_name":"General"}`, + }} + channels := NewChannels(f) + var wg sync.WaitGroup + for range 40 { + wg.Add(1) + go func() { defer wg.Done(); _, _ = channels.ByID(context.Background(), "x") }() + } + wg.Wait() +} diff --git a/internal/mattermost/teams.go b/internal/mattermost/teams.go new file mode 100644 index 0000000..1a0c5d1 --- /dev/null +++ b/internal/mattermost/teams.go @@ -0,0 +1,141 @@ +package mattermost + +import ( + "context" + "encoding/json" + "errors" + "net/url" + "sort" + "strings" +) + +var ( + ErrInvalidTeamResponse = errors.New("Mattermost returned an invalid team response") + ErrInvalidTeamsResponse = errors.New("Mattermost returned an invalid teams response") + ErrInvalidTeamRequest = errors.New("invalid Mattermost team request") + ErrTeamNotFound = errors.New("Mattermost team was not found") + ErrAmbiguousTeam = errors.New("Mattermost team selection is ambiguous") +) + +type teamTransport interface { + Get(context.Context, string, any) error +} + +type Team struct { + ID string + Name string + DisplayName string + Type string +} + +// TeamMembership is an exact, validated snapshot returned by Mattermost for +// one canonical user ID. Callers can inspect but cannot forge its contents. +type TeamMembership struct { + userID string + teams []Team +} + +func (m TeamMembership) Items() []Team { + return append([]Team(nil), m.teams...) +} + +func (m TeamMembership) contains(teamID string) bool { + for _, team := range m.teams { + if team.ID == teamID { + return true + } + } + return false +} + +func (t *Team) UnmarshalJSON(data []byte) error { + var raw struct { + ID json.RawMessage `json:"id"` + Name json.RawMessage `json:"name"` + DisplayName json.RawMessage `json:"display_name"` + Type json.RawMessage `json:"type"` + } + if err := json.Unmarshal(data, &raw); err != nil { + return ErrInvalidTeamResponse + } + id, idOK := requiredString(raw.ID) + name, nameOK := requiredString(raw.Name) + displayName, displayOK := strictString(raw.DisplayName) + typeCode, typeOK := requiredString(raw.Type) + if !idOK || !nameOK || !displayOK || !typeOK || (typeCode != "O" && typeCode != "I") { + return ErrInvalidTeamResponse + } + *t = Team{ID: id, Name: name, DisplayName: displayName, Type: typeCode} + return nil +} + +type Teams struct{ client teamTransport } + +func NewTeams(client teamTransport) *Teams { return &Teams{client: client} } + +type teamList []Team + +func (l *teamList) UnmarshalJSON(data []byte) error { + var teams []Team + if err := json.Unmarshal(data, &teams); err != nil || teams == nil { + return ErrInvalidTeamsResponse + } + *l = teams + return nil +} + +func (s *Teams) List(ctx context.Context, userID string) (TeamMembership, error) { + if strings.TrimSpace(userID) == "" || userID == "me" { + return TeamMembership{}, ErrInvalidTeamRequest + } + var decoded teamList + if err := s.client.Get(ctx, "/users/"+url.PathEscape(userID)+"/teams", &decoded); err != nil { + return TeamMembership{}, err + } + teams := []Team(decoded) + seen := make(map[string]struct{}, len(teams)) + for _, team := range teams { + if _, duplicate := seen[team.ID]; duplicate { + return TeamMembership{}, ErrInvalidTeamsResponse + } + seen[team.ID] = struct{}{} + } + sort.Slice(teams, func(i, j int) bool { + if teams[i].Name != teams[j].Name { + return teams[i].Name < teams[j].Name + } + return teams[i].ID < teams[j].ID + }) + return TeamMembership{userID: userID, teams: teams}, nil +} + +// Resolve selects a team only when the complete membership list identifies it +// uniquely. An empty selector is accepted only for a single-team account. +func (s *Teams) Resolve(ctx context.Context, userID, selector string) (Team, error) { + membership, err := s.List(ctx, userID) + if err != nil { + return Team{}, err + } + if selector == "" { + if len(membership.teams) == 0 { + return Team{}, ErrTeamNotFound + } + if len(membership.teams) != 1 { + return Team{}, ErrAmbiguousTeam + } + return membership.teams[0], nil + } + var matches []Team + for _, team := range membership.teams { + if team.Name == selector || team.DisplayName == selector { + matches = append(matches, team) + } + } + if len(matches) == 0 { + return Team{}, ErrTeamNotFound + } + if len(matches) != 1 { + return Team{}, ErrAmbiguousTeam + } + return matches[0], nil +} diff --git a/internal/mattermost/teams_test.go b/internal/mattermost/teams_test.go new file mode 100644 index 0000000..ae5f256 --- /dev/null +++ b/internal/mattermost/teams_test.go @@ -0,0 +1,95 @@ +package mattermost + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "sync" + "testing" +) + +type fakeTeamTransport struct { + mu sync.Mutex + payload string + paths []string +} + +func (f *fakeTeamTransport) Get(_ context.Context, path string, out any) error { + f.mu.Lock() + defer f.mu.Unlock() + f.paths = append(f.paths, path) + return json.Unmarshal([]byte(f.payload), out) +} + +func TestTeamsListValidatesEncodesSortsAndNarrows(t *testing.T) { + f := &fakeTeamTransport{payload: `[ + {"id":"z","name":"beta","display_name":"","type":"I","email":"secret"}, + {"id":"b","name":"alpha","display_name":"B","type":"O"}, + {"id":"a","name":"alpha","display_name":"A","type":"O"} + ]`} + got, err := NewTeams(f).List(context.Background(), "user/one space") + if err != nil { + t.Fatal(err) + } + want := []Team{{ID: "a", Name: "alpha", DisplayName: "A", Type: "O"}, {ID: "b", Name: "alpha", DisplayName: "B", Type: "O"}, {ID: "z", Name: "beta", Type: "I"}} + if !reflect.DeepEqual(got.Items(), want) { + t.Fatalf("teams = %#v, want %#v", got.Items(), want) + } + if f.paths[0] != "/users/user%2Fone%20space/teams" { + t.Fatalf("path = %q", f.paths[0]) + } +} + +func TestTeamsFailClosedWithoutReflectingRemoteValues(t *testing.T) { + for _, payload := range []string{`null`, `{}`, `[{"id":"remote-secret","name":"x","display_name":"X","type":"X"}]`, `[{"id":"a","name":"x","display_name":7,"type":"O"}]`, `[{"id":"a","name":"x","display_name":"X","type":"O"},{"id":"a","name":"y","display_name":"Y","type":"I"}]`} { + _, err := NewTeams(&fakeTeamTransport{payload: payload}).List(context.Background(), "user") + if !errors.Is(err, ErrInvalidTeamResponse) && !errors.Is(err, ErrInvalidTeamsResponse) { + t.Fatalf("payload %s: error = %v", payload, err) + } + if err != nil && contains(err.Error(), "remote-secret") { + t.Fatalf("error reflected remote data: %v", err) + } + } +} + +func TestResolveRequiresUniqueCompleteSelection(t *testing.T) { + teams := NewTeams(&fakeTeamTransport{payload: `[{"id":"a","name":"core","display_name":"Shared","type":"O"},{"id":"b","name":"eng","display_name":"Shared","type":"I"}]`}) + if _, err := teams.Resolve(context.Background(), "user", ""); !errors.Is(err, ErrAmbiguousTeam) { + t.Fatalf("empty selector error = %v", err) + } + if _, err := teams.Resolve(context.Background(), "user", "Shared"); !errors.Is(err, ErrAmbiguousTeam) { + t.Fatalf("duplicate display name error = %v", err) + } + got, err := teams.Resolve(context.Background(), "user", "eng") + if err != nil || got.ID != "b" { + t.Fatalf("resolved = %+v, error = %v", got, err) + } +} + +func TestTeamsReadsAreRaceSafe(t *testing.T) { + f := &fakeTeamTransport{payload: `[{"id":"a","name":"core","display_name":"Core","type":"O"}]`} + teams := NewTeams(f) + var wg sync.WaitGroup + for range 40 { + wg.Add(1) + go func() { defer wg.Done(); _, _ = teams.List(context.Background(), "user") }() + } + wg.Wait() +} + +func TestTeamsRejectMeAlias(t *testing.T) { + _, err := NewTeams(&fakeTeamTransport{}).List(context.Background(), "me") + if !errors.Is(err, ErrInvalidTeamRequest) { + t.Fatalf("error = %v", err) + } +} + +func contains(value, fragment string) bool { + for i := 0; i+len(fragment) <= len(value); i++ { + if value[i:i+len(fragment)] == fragment { + return true + } + } + return false +} From 924fcea167bf57caa8fb1ee109ec0bbdb0a0af36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 06:10:48 +0300 Subject: [PATCH 017/119] feat: port bounded channel history reads --- internal/mattermost/posts.go | 192 +++++++++++++++++++++++++++ internal/mattermost/posts_test.go | 129 ++++++++++++++++++ internal/retrieval/channel.go | 205 +++++++++++++++++++++++++++++ internal/retrieval/channel_test.go | 202 ++++++++++++++++++++++++++++ 4 files changed, 728 insertions(+) create mode 100644 internal/mattermost/posts.go create mode 100644 internal/mattermost/posts_test.go create mode 100644 internal/retrieval/channel.go create mode 100644 internal/retrieval/channel_test.go diff --git a/internal/mattermost/posts.go b/internal/mattermost/posts.go new file mode 100644 index 0000000..c4cfed9 --- /dev/null +++ b/internal/mattermost/posts.go @@ -0,0 +1,192 @@ +package mattermost + +import ( + "context" + "encoding/json" + "errors" + "net/url" + "strconv" + "strings" +) + +const MaxPostsPage = 200 + +const ( + maxPostIDLength = 128 + maxDateMilliseconds = int64(8_640_000_000_000_000) +) + +var ( + ErrInvalidPostResponse = errors.New("Mattermost returned an invalid post response") + ErrInvalidPostsResponse = errors.New("Mattermost returned an invalid posts response") + ErrInvalidPostsRequest = errors.New("invalid Mattermost posts request") +) + +type postTransport interface { + Get(context.Context, string, any) error +} + +// Post retains only fields needed for history selection. Presentation-specific +// fields belong in a later, wider model rather than leaking unchecked payloads. +type Post struct { + ID string + ChannelID string + Message string + CreateAt int64 + DeleteAt int64 +} + +func (p *Post) UnmarshalJSON(data []byte) error { + var raw struct { + ID json.RawMessage `json:"id"` + ChannelID json.RawMessage `json:"channel_id"` + Message json.RawMessage `json:"message"` + CreateAt json.RawMessage `json:"create_at"` + DeleteAt json.RawMessage `json:"delete_at"` + } + if err := json.Unmarshal(data, &raw); err != nil { + return ErrInvalidPostResponse + } + id, idOK := safePostID(raw.ID) + channelID, channelOK := requiredString(raw.ChannelID) + message, messageOK := strictString(raw.Message) + createAt, createOK := nonnegativeInteger(raw.CreateAt) + deleteAt, deleteOK := nonnegativeInteger(raw.DeleteAt) + if !idOK || !channelOK || !messageOK || !createOK || createAt == 0 || createAt > maxDateMilliseconds || !deleteOK || deleteAt > maxDateMilliseconds { + return ErrInvalidPostResponse + } + *p = Post{ID: id, ChannelID: channelID, Message: message, CreateAt: createAt, DeleteAt: deleteAt} + return nil +} + +func safePostID(raw json.RawMessage) (string, bool) { + value, ok := requiredString(raw) + if !ok || len(value) > maxPostIDLength { + return "", false + } + for i := range len(value) { + c := value[i] + if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '-') { + return "", false + } + } + return value, true +} + +func nonnegativeInteger(raw json.RawMessage) (int64, bool) { + var value int64 + if len(raw) == 0 || json.Unmarshal(raw, &value) != nil || value < 0 { + return 0, false + } + return value, true +} + +// OrderedPostsPage preserves the server's order and raw fullness separately +// from its validated, visible posts. +type OrderedPostsPage struct { + Posts []Post + RawCount int + HasNext *bool + FirstInaccessiblePostTime *int64 + Incomplete bool + bindings []postBinding +} + +type postBinding struct { + ID string + ChannelID string +} + +func (p *OrderedPostsPage) UnmarshalJSON(data []byte) error { + var raw struct { + Order json.RawMessage `json:"order"` + Posts map[string]json.RawMessage `json:"posts"` + HasNext json.RawMessage `json:"has_next"` + FirstInaccessiblePostTime json.RawMessage `json:"first_inaccessible_post_time"` + } + if err := json.Unmarshal(data, &raw); err != nil { + return ErrInvalidPostsResponse + } + var order []json.RawMessage + if len(raw.Order) == 0 || json.Unmarshal(raw.Order, &order) != nil || order == nil { + return ErrInvalidPostsResponse + } + + result := OrderedPostsPage{RawCount: len(order)} + if raw.Posts == nil && len(order) > 0 { + result.Incomplete = true + } + for _, rawID := range order { + var id string + if json.Unmarshal(rawID, &id) != nil || strings.TrimSpace(id) == "" { + result.Incomplete = true + continue + } + candidate, ok := raw.Posts[id] + if !ok { + result.Incomplete = true + continue + } + var post Post + if json.Unmarshal(candidate, &post) != nil || post.ID != id { + result.Incomplete = true + continue + } + result.bindings = append(result.bindings, postBinding{ID: post.ID, ChannelID: post.ChannelID}) + if post.DeleteAt == 0 { + result.Posts = append(result.Posts, post) + } + } + if len(raw.HasNext) > 0 { + var hasNext bool + if string(raw.HasNext) == "null" || json.Unmarshal(raw.HasNext, &hasNext) != nil { + result.Incomplete = true + } else { + result.HasNext = &hasNext + } + } + if len(raw.FirstInaccessiblePostTime) > 0 && string(raw.FirstInaccessiblePostTime) != "null" { + value, ok := nonnegativeInteger(raw.FirstInaccessiblePostTime) + if !ok { + result.Incomplete = true + } else if value > 0 { + result.FirstInaccessiblePostTime = &value + } + } + *p = result + return nil +} + +type ChannelPostsOptions struct { + PerPage int + Page int + Before string +} + +type Posts struct{ client postTransport } + +func NewPosts(client postTransport) *Posts { return &Posts{client: client} } + +func (s *Posts) ChannelPage(ctx context.Context, channelID string, options ChannelPostsOptions) (OrderedPostsPage, error) { + if strings.TrimSpace(channelID) == "" || options.PerPage <= 0 || options.PerPage > MaxPostsPage || options.Page < 0 { + return OrderedPostsPage{}, ErrInvalidPostsRequest + } + params := url.Values{} + params.Set("per_page", strconv.Itoa(options.PerPage)) + params.Set("page", strconv.Itoa(options.Page)) + params.Set("skipFetchThreads", "true") + if options.Before != "" { + params.Set("before", options.Before) + } + var page OrderedPostsPage + path := "/channels/" + url.PathEscape(channelID) + "/posts?" + params.Encode() + if err := s.client.Get(ctx, path, &page); err != nil { + return OrderedPostsPage{}, err + } + for _, binding := range page.bindings { + if binding.ChannelID != channelID { + return OrderedPostsPage{}, ErrInvalidPostsResponse + } + } + return page, nil +} diff --git a/internal/mattermost/posts_test.go b/internal/mattermost/posts_test.go new file mode 100644 index 0000000..3a603c6 --- /dev/null +++ b/internal/mattermost/posts_test.go @@ -0,0 +1,129 @@ +package mattermost + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" +) + +type postTransportFunc func(context.Context, string, any) error + +func (f postTransportFunc) Get(ctx context.Context, path string, out any) error { + return f(ctx, path, out) +} + +func TestOrderedPostsPageNormalizesAndSuppressesDeleted(t *testing.T) { + var page OrderedPostsPage + err := json.Unmarshal([]byte(`{ + "order":["live",7,"missing","deleted","mismatch"], + "posts":{ + "live":{"id":"live","channel_id":"chan","message":"ok","create_at":3,"delete_at":0}, + "deleted":{"id":"deleted","channel_id":"chan","message":"secret stale text","create_at":2,"delete_at":9}, + "mismatch":{"id":"other","channel_id":"chan","message":"bad","create_at":1,"delete_at":0} + }, + "has_next":false,"first_inaccessible_post_time":4 + }`), &page) + if err != nil { + t.Fatal(err) + } + if page.RawCount != 5 || len(page.Posts) != 1 || page.Posts[0].ID != "live" || !page.Incomplete { + t.Fatalf("page = %#v", page) + } + if page.HasNext == nil || *page.HasNext || page.FirstInaccessiblePostTime == nil || *page.FirstInaccessiblePostTime != 4 { + t.Fatalf("metadata = %#v", page) + } +} + +func TestOrderedPostsPageRejectsInvalidEnvelope(t *testing.T) { + for _, payload := range []string{`null`, `{}`, `{"order":null,"posts":{}}`, `{"order":{},"posts":{}}`} { + var page OrderedPostsPage + if !errors.Is(json.Unmarshal([]byte(payload), &page), ErrInvalidPostsResponse) { + t.Fatalf("payload %s accepted", payload) + } + } +} + +func TestOrderedPostsPageMarksMalformedMetadataIncomplete(t *testing.T) { + var page OrderedPostsPage + if err := json.Unmarshal([]byte(`{"order":[],"posts":{},"has_next":"yes","first_inaccessible_post_time":-1}`), &page); err != nil { + t.Fatal(err) + } + if !page.Incomplete || page.HasNext != nil || page.FirstInaccessiblePostTime != nil { + t.Fatalf("page = %#v", page) + } +} + +func TestOrderedPostsPageTreatsExplicitNullHasNextAsIncomplete(t *testing.T) { + var page OrderedPostsPage + if err := json.Unmarshal([]byte(`{"order":[],"posts":{},"has_next":null}`), &page); err != nil { + t.Fatal(err) + } + if !page.Incomplete || page.HasNext != nil { + t.Fatalf("page = %#v", page) + } +} + +func TestChannelPageBuildsBoundedGETAndChecksBinding(t *testing.T) { + var gotPath string + api := NewPosts(postTransportFunc(func(_ context.Context, path string, out any) error { + gotPath = path + return json.Unmarshal([]byte(`{"order":["p"],"posts":{"p":{"id":"p","channel_id":"channel/id","message":"","create_at":1,"delete_at":0}}}`), out) + })) + page, err := api.ChannelPage(context.Background(), "channel/id", ChannelPostsOptions{PerPage: 17, Page: 2, Before: "anchor/id"}) + if err != nil || len(page.Posts) != 1 { + t.Fatalf("page = %#v, err = %v", page, err) + } + for _, part := range []string{"/channels/channel%2Fid/posts?", "before=anchor%2Fid", "page=2", "per_page=17", "skipFetchThreads=true"} { + if !strings.Contains(gotPath, part) { + t.Fatalf("path %q missing %q", gotPath, part) + } + } +} + +func TestChannelPageRejectsInvalidRequestAndCrossChannelPost(t *testing.T) { + api := NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { + return json.Unmarshal([]byte(`{"order":["p"],"posts":{"p":{"id":"p","channel_id":"other","message":"x","create_at":1,"delete_at":0}}}`), out) + })) + for _, options := range []ChannelPostsOptions{{PerPage: 0}, {PerPage: 201}, {PerPage: 1, Page: -1}} { + if _, err := api.ChannelPage(context.Background(), "channel", options); !errors.Is(err, ErrInvalidPostsRequest) { + t.Fatalf("options %#v: %v", options, err) + } + } + if _, err := api.ChannelPage(context.Background(), "channel", ChannelPostsOptions{PerPage: 1}); !errors.Is(err, ErrInvalidPostsResponse) { + t.Fatalf("cross-channel error = %v", err) + } +} + +func TestChannelPageRejectsDeletedCrossChannelPost(t *testing.T) { + api := NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { + return json.Unmarshal([]byte(`{"order":["p"],"posts":{"p":{"id":"p","channel_id":"other","message":"stale","create_at":1,"delete_at":2}}}`), out) + })) + if _, err := api.ChannelPage(context.Background(), "channel", ChannelPostsOptions{PerPage: 1}); !errors.Is(err, ErrInvalidPostsResponse) { + t.Fatalf("error = %v", err) + } +} + +func TestPostRejectsValuesOutsideDeterministicDomain(t *testing.T) { + for _, payload := range []string{ + `{"id":"nonascii-é","channel_id":"channel","message":"x","create_at":1,"delete_at":0}`, + `{"id":"slash/id","channel_id":"channel","message":"x","create_at":1,"delete_at":0}`, + `{"id":"ok","channel_id":"channel","message":"x","create_at":8640000000000001,"delete_at":0}`, + `{"id":"ok","channel_id":"channel","message":"x","create_at":1,"delete_at":8640000000000001}`, + } { + var post Post + if !errors.Is(json.Unmarshal([]byte(payload), &post), ErrInvalidPostResponse) { + t.Fatalf("payload %s accepted", payload) + } + } +} + +func FuzzOrderedPostsPageNeverPanics(f *testing.F) { + f.Add([]byte(`{"order":[],"posts":{}}`)) + f.Add([]byte(`{"order":["x"],"posts":null}`)) + f.Fuzz(func(t *testing.T, payload []byte) { + var page OrderedPostsPage + _ = json.Unmarshal(payload, &page) + }) +} diff --git a/internal/retrieval/channel.go b/internal/retrieval/channel.go new file mode 100644 index 0000000..ce93a38 --- /dev/null +++ b/internal/retrieval/channel.go @@ -0,0 +1,205 @@ +// Package retrieval implements bounded, deterministic read selection. +package retrieval + +import ( + "context" + "errors" + "sort" + "strings" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" +) + +var ErrInvalidChannelHistoryRequest = errors.New("invalid channel history request") + +const ( + MaxChannelHistoryPages = 100 + maxSafeInteger = int64(9_007_199_254_740_991) + maxDateMilliseconds = int64(8_640_000_000_000_000) +) + +type Completeness uint8 + +const ( + CompletenessUnknown Completeness = iota + CompletenessComplete + CompletenessTruncated +) + +type Boundary struct { + CreateAt int64 + ID string +} + +type ChannelHistoryOptions struct { + Limit int + Since *int64 + Boundary *Boundary + SafeBeforePostID string +} + +type ChannelHistoryResult struct { + Posts []mattermost.Post + Completeness Completeness + SafeBeforeValid bool +} + +type channelPageSource interface { + ChannelPage(context.Context, string, mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) +} + +func ChannelHistory(ctx context.Context, source channelPageSource, channelID string, options ChannelHistoryOptions) (ChannelHistoryResult, error) { + if source == nil || strings.TrimSpace(channelID) == "" || options.Limit <= 0 || int64(options.Limit) > maxSafeInteger || + (options.Since != nil && (*options.Since < 0 || *options.Since > maxDateMilliseconds)) || + (options.Boundary != nil && (options.Boundary.CreateAt <= 0 || options.Boundary.CreateAt > maxDateMilliseconds || !safeBoundaryID(options.Boundary.ID))) { + return ChannelHistoryResult{}, ErrInvalidChannelHistoryRequest + } + target := options.Limit + 1 + pageSize := target + if pageSize > mattermost.MaxPostsPage { + pageSize = mattermost.MaxPostsPage + } + byID := make(map[string]mattermost.Post) + seen := make(map[string]struct{}) + pageNumber, stagnantPages := 0, 0 + uncertain, exhausted := false, false + activeBefore := options.SafeBeforePostID + retriedWithoutAnchor := false + + for { + if pageNumber >= MaxChannelHistoryPages { + uncertain = true + break + } + page, err := source.ChannelPage(ctx, channelID, mattermost.ChannelPostsOptions{ + PerPage: pageSize, Page: pageNumber, Before: activeBefore, + }) + if err != nil { + return ChannelHistoryResult{}, err + } + if page.Incomplete || page.FirstInaccessiblePostTime != nil { + uncertain = true + } + if page.RawCount == 0 { + if pageNumber == 0 && activeBefore != "" && !retriedWithoutAnchor { + activeBefore = "" + retriedWithoutAnchor = true + stagnantPages = 0 + continue + } + if page.HasNext != nil && *page.HasNext { + uncertain = true + } + exhausted = !uncertain + break + } + + madeProgress := false + for _, post := range page.Posts { + if _, duplicate := seen[post.ID]; duplicate { + continue + } + seen[post.ID] = struct{}{} + madeProgress = true + if withinSelection(post, options) { + byID[post.ID] = post + } + } + pageNumber++ + if madeProgress { + stagnantPages = 0 + } else { + stagnantPages++ + } + + if options.Since != nil && len(page.Posts) > 0 && allOlderThan(page.Posts, *options.Since) { + exhausted = !uncertain + break + } + if len(byID) >= target { + selected := mostRecent(byID, options.Limit) + cutoff := selected[len(selected)-1].CreateAt + if containsOlderThan(page.Posts, cutoff) { + break + } + } + if page.RawCount < pageSize && (page.HasNext == nil || !*page.HasNext) { + exhausted = !uncertain + break + } + if stagnantPages >= 2 { + uncertain = true + break + } + } + + result := ChannelHistoryResult{Posts: mostRecent(byID, options.Limit), SafeBeforeValid: options.SafeBeforePostID == "" || !retriedWithoutAnchor} + switch { + case len(byID) > options.Limit: + result.Completeness = CompletenessTruncated + case uncertain || !exhausted: + result.Completeness = CompletenessUnknown + default: + result.Completeness = CompletenessComplete + } + return result, nil +} + +func safeBoundaryID(value string) bool { + if len(value) == 0 || len(value) > 128 { + return false + } + for i := range len(value) { + c := value[i] + if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '-') { + return false + } + } + return true +} + +func withinSelection(post mattermost.Post, options ChannelHistoryOptions) bool { + if options.Since != nil && post.CreateAt < *options.Since { + return false + } + if options.Boundary == nil { + return true + } + return post.CreateAt < options.Boundary.CreateAt || + (post.CreateAt == options.Boundary.CreateAt && post.ID > options.Boundary.ID) +} + +func mostRecent(posts map[string]mattermost.Post, limit int) []mattermost.Post { + result := make([]mattermost.Post, 0, len(posts)) + for _, post := range posts { + result = append(result, post) + } + sort.Slice(result, func(i, j int) bool { + if result[i].CreateAt != result[j].CreateAt { + return result[i].CreateAt > result[j].CreateAt + } + return result[i].ID < result[j].ID + }) + if len(result) > limit { + result = result[:limit] + } + return result +} + +func allOlderThan(posts []mattermost.Post, since int64) bool { + for _, post := range posts { + if post.CreateAt >= since { + return false + } + } + return true +} + +func containsOlderThan(posts []mattermost.Post, cutoff int64) bool { + for _, post := range posts { + if post.CreateAt < cutoff { + return true + } + } + return false +} diff --git a/internal/retrieval/channel_test.go b/internal/retrieval/channel_test.go new file mode 100644 index 0000000..5950aad --- /dev/null +++ b/internal/retrieval/channel_test.go @@ -0,0 +1,202 @@ +package retrieval + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" +) + +type pageSourceFunc func(context.Context, string, mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) + +func (f pageSourceFunc) ChannelPage(ctx context.Context, channelID string, options mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) { + return f(ctx, channelID, options) +} + +func testPost(id string, at int64) mattermost.Post { + return mattermost.Post{ID: id, ChannelID: "channel", Message: id, CreateAt: at} +} + +func testPage(posts ...mattermost.Post) mattermost.OrderedPostsPage { + return mattermost.OrderedPostsPage{Posts: posts, RawCount: len(posts)} +} + +func TestChannelHistoryUsesLimitPlusOneAndDeterministicOrder(t *testing.T) { + var requested mattermost.ChannelPostsOptions + result, err := ChannelHistory(context.Background(), pageSourceFunc(func(_ context.Context, _ string, options mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) { + requested = options + return testPage(testPost("b", 3), testPost("a", 3), testPost("older", 2)), nil + }), "channel", ChannelHistoryOptions{Limit: 2}) + if err != nil { + t.Fatal(err) + } + if requested.PerPage != 3 || result.Completeness != CompletenessTruncated || fmt.Sprint(ids(result.Posts)) != "[a b]" { + t.Fatalf("requested=%#v result=%#v", requested, result) + } +} + +func TestChannelHistoryCompletesEqualMillisecondTiesAndBoundary(t *testing.T) { + pages := []mattermost.OrderedPostsPage{ + testPage(testPost("newer", 300), testPost("anchor", 200), testPost("c", 200)), + testPage(testPost("b", 200), testPost("a", 200), testPost("older", 100)), + } + result, err := ChannelHistory(context.Background(), indexedPages(t, pages), "channel", ChannelHistoryOptions{ + Limit: 2, Boundary: &Boundary{CreateAt: 200, ID: "anchor"}, + }) + if err != nil { + t.Fatal(err) + } + if fmt.Sprint(ids(result.Posts)) != "[b c]" || result.Completeness != CompletenessTruncated { + t.Fatalf("result = %#v", result) + } +} + +func TestChannelHistoryAppliesExactLocalSinceWithoutSendingIt(t *testing.T) { + since := int64(100) + pages := []mattermost.OrderedPostsPage{ + testPage(testPost("new", 101), testPost("edge", 100), testPost("old", 99)), + } + result, err := ChannelHistory(context.Background(), indexedPages(t, pages), "channel", ChannelHistoryOptions{Limit: 4, Since: &since}) + if err != nil { + t.Fatal(err) + } + if fmt.Sprint(ids(result.Posts)) != "[new edge]" || result.Completeness != CompletenessComplete { + t.Fatalf("result = %#v", result) + } +} + +func TestChannelHistorySafeBeforeFallbackPreservesBoundary(t *testing.T) { + var requests []mattermost.ChannelPostsOptions + result, err := ChannelHistory(context.Background(), pageSourceFunc(func(_ context.Context, _ string, options mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) { + requests = append(requests, options) + if options.Before != "" { + return mattermost.OrderedPostsPage{RawCount: 0}, nil + } + return testPage(testPost("peer", 200), testPost("older", 100)), nil + }), "channel", ChannelHistoryOptions{Limit: 2, Boundary: &Boundary{CreateAt: 200, ID: "anchor"}, SafeBeforePostID: "gone"}) + if err != nil { + t.Fatal(err) + } + if len(requests) != 2 || requests[0].Page != 0 || requests[1].Page != 0 || requests[1].Before != "" { + t.Fatalf("requests = %#v", requests) + } + if result.SafeBeforeValid || result.Completeness != CompletenessComplete || fmt.Sprint(ids(result.Posts)) != "[peer older]" { + t.Fatalf("result = %#v", result) + } +} + +func TestChannelHistorySafeBeforeRemainsOnDeepPages(t *testing.T) { + pages := []mattermost.OrderedPostsPage{ + testPage(testPost("newer", 300), testPost("anchor", 200), testPost("c", 200)), + testPage(testPost("b", 200), testPost("older", 100)), + } + index := 0 + result, err := ChannelHistory(context.Background(), pageSourceFunc(func(_ context.Context, _ string, options mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) { + if options.Before != "safe" || options.Page != index { + t.Fatalf("options = %#v, index = %d", options, index) + } + page := pages[index] + index++ + return page, nil + }), "channel", ChannelHistoryOptions{Limit: 2, Boundary: &Boundary{CreateAt: 200, ID: "anchor"}, SafeBeforePostID: "safe"}) + if err != nil || !result.SafeBeforeValid || fmt.Sprint(ids(result.Posts)) != "[b c]" { + t.Fatalf("result=%#v err=%v", result, err) + } +} + +func TestChannelHistoryUnknownOnTwoStagnantFullPages(t *testing.T) { + calls := 0 + result, err := ChannelHistory(context.Background(), pageSourceFunc(func(_ context.Context, _ string, options mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) { + calls++ + return mattermost.OrderedPostsPage{Posts: []mattermost.Post{testPost("a", 2), testPost("b", 1), testPost("a", 2), testPost("b", 1)}, RawCount: options.PerPage}, nil + }), "channel", ChannelHistoryOptions{Limit: 3}) + if err != nil { + t.Fatal(err) + } + if calls != 3 || result.Completeness != CompletenessUnknown { + t.Fatalf("calls=%d result=%#v", calls, result) + } +} + +func TestChannelHistoryIncompleteDeletedPageContinuesButCannotClaimComplete(t *testing.T) { + pages := []mattermost.OrderedPostsPage{ + {RawCount: 3, Incomplete: true}, + {Posts: []mattermost.Post{testPost("live", 2)}, RawCount: 1}, + } + result, err := ChannelHistory(context.Background(), indexedPages(t, pages), "channel", ChannelHistoryOptions{Limit: 2}) + if err != nil || fmt.Sprint(ids(result.Posts)) != "[live]" || result.Completeness != CompletenessUnknown { + t.Fatalf("result=%#v err=%v", result, err) + } +} + +func TestChannelHistoryEmptyHasNextAndInaccessibleAreUnknown(t *testing.T) { + truth := true + inaccessible := int64(1) + for name, page := range map[string]mattermost.OrderedPostsPage{ + "has next": {HasNext: &truth}, + "inaccessible": {Posts: []mattermost.Post{testPost("x", 1)}, RawCount: 1, FirstInaccessiblePostTime: &inaccessible}, + } { + t.Run(name, func(t *testing.T) { + result, err := ChannelHistory(context.Background(), pageSourceFunc(func(context.Context, string, mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) { + return page, nil + }), "channel", ChannelHistoryOptions{Limit: 2}) + if err != nil || result.Completeness != CompletenessUnknown { + t.Fatalf("result=%#v err=%v", result, err) + } + }) + } +} + +func TestChannelHistoryStopsAtHardPageBoundAsUnknown(t *testing.T) { + calls := 0 + result, err := ChannelHistory(context.Background(), pageSourceFunc(func(_ context.Context, _ string, options mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) { + calls++ + posts := make([]mattermost.Post, options.PerPage) + for i := range posts { + posts[i] = testPost(fmt.Sprintf("p%03d_%03d", options.Page, i), 300) + } + return mattermost.OrderedPostsPage{Posts: posts, RawCount: options.PerPage}, nil + }), "channel", ChannelHistoryOptions{Limit: 200, Boundary: &Boundary{CreateAt: 200, ID: "anchor"}}) + if err != nil || calls != MaxChannelHistoryPages || result.Completeness != CompletenessUnknown || len(result.Posts) != 0 { + t.Fatalf("calls=%d result=%#v error=%v", calls, result, err) + } +} + +func TestChannelHistoryRejectsValuesOutsideCursorDomain(t *testing.T) { + source := pageSourceFunc(func(context.Context, string, mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) { + t.Fatal("source called for invalid request") + return mattermost.OrderedPostsPage{}, nil + }) + tooLate := int64(8_640_000_000_000_001) + for name, options := range map[string]ChannelHistoryOptions{ + "since": {Limit: 1, Since: &tooLate}, + "boundary id": {Limit: 1, Boundary: &Boundary{CreateAt: 1, ID: "bad/é"}}, + "boundary at": {Limit: 1, Boundary: &Boundary{CreateAt: tooLate, ID: "ok"}}, + } { + t.Run(name, func(t *testing.T) { + if _, err := ChannelHistory(context.Background(), source, "channel", options); !errors.Is(err, ErrInvalidChannelHistoryRequest) { + t.Fatalf("error = %v", err) + } + }) + } +} + +func indexedPages(t *testing.T, pages []mattermost.OrderedPostsPage) pageSourceFunc { + t.Helper() + return func(_ context.Context, _ string, options mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) { + if options.Page >= len(pages) { + t.Fatalf("unexpected page %d", options.Page) + } + return pages[options.Page], nil + } +} + +func ids(posts []mattermost.Post) []string { + result := make([]string, len(posts)) + for i := range posts { + result[i] = posts[i].ID + } + return result +} From eb2d7c063ef077c60ea107560364334b376e8f41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 06:21:20 +0300 Subject: [PATCH 018/119] feat: port strict history cursors --- internal/cursor/cursor.go | 178 +++++++++++++++++++++++++++++++++ internal/cursor/cursor_test.go | 124 +++++++++++++++++++++++ 2 files changed, 302 insertions(+) create mode 100644 internal/cursor/cursor.go create mode 100644 internal/cursor/cursor_test.go diff --git a/internal/cursor/cursor.go b/internal/cursor/cursor.go new file mode 100644 index 0000000..1a6c77e --- /dev/null +++ b/internal/cursor/cursor.go @@ -0,0 +1,178 @@ +// Package cursor encodes and decodes opaque channel-history cursors. +package cursor + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "io" + "math" + "regexp" + "unicode/utf8" +) + +const ( + maxEncodedLength = 2048 + maxDecodedLength = 1536 + maxDateMillis = int64(8_640_000_000_000_000) + maxIDLength = 128 +) + +var ( + // ErrInvalidCursor is returned for every invalid cursor without exposing its input. + ErrInvalidCursor = errors.New("invalid cursor") + safeIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) +) + +// ChannelHistory identifies a deterministic boundary in one channel's history. +type ChannelHistory struct { + Version int `json:"v"` + Scope string `json:"scope"` + ChannelID string `json:"channelId"` + Boundary Boundary `json:"boundary"` + Since *int64 `json:"since"` + SafeBeforePostID string `json:"safeBeforePostId,omitempty"` +} + +// Boundary is the newest post included in a channel-history page. +type Boundary struct { + CreateAt int64 `json:"createAt"` + ID string `json:"id"` +} + +// EncodeChannelHistory returns canonical, unpadded base64url JSON. +func EncodeChannelHistory(value ChannelHistory) (string, error) { + if !valid(value) { + return "", ErrInvalidCursor + } + + data, err := json.Marshal(value) + if err != nil { + return "", ErrInvalidCursor + } + encoded := base64.RawURLEncoding.EncodeToString(data) + if len(encoded) > maxEncodedLength { + return "", ErrInvalidCursor + } + return encoded, nil +} + +// DecodeChannelHistory validates and decodes an opaque channel-history cursor. +func DecodeChannelHistory(encoded string) (ChannelHistory, error) { + if len(encoded) == 0 || len(encoded) > maxEncodedLength || !safeIDPattern.MatchString(encoded) { + return ChannelHistory{}, ErrInvalidCursor + } + + data, err := base64.RawURLEncoding.DecodeString(encoded) + if err != nil || len(data) == 0 || len(data) > maxDecodedLength || !utf8.Valid(data) || + base64.RawURLEncoding.EncodeToString(data) != encoded { + return ChannelHistory{}, ErrInvalidCursor + } + + value, ok := decodeJSON(data) + if !ok || !valid(value) { + return ChannelHistory{}, ErrInvalidCursor + } + return value, nil +} + +// ComparePostIDs applies the bytewise ASCII ordering used for equal timestamps. +func ComparePostIDs(a, b string) int { + if a < b { + return -1 + } + if a > b { + return 1 + } + return 0 +} + +func valid(value ChannelHistory) bool { + return value.Version == 1 && value.Scope == "channel" && + isSafeID(value.ChannelID) && isSafeID(value.Boundary.ID) && + value.Boundary.CreateAt >= 0 && value.Boundary.CreateAt <= maxDateMillis && + (value.Since == nil || (*value.Since >= 0 && *value.Since <= value.Boundary.CreateAt)) && + (value.SafeBeforePostID == "" || isSafeID(value.SafeBeforePostID)) +} + +func isSafeID(value string) bool { + return len(value) >= 1 && len(value) <= maxIDLength && safeIDPattern.MatchString(value) +} + +func decodeJSON(data []byte) (ChannelHistory, bool) { + var outer map[string]json.RawMessage + decoder := json.NewDecoder(bytes.NewReader(data)) + if err := decoder.Decode(&outer); err != nil || !onlyKeys(outer, + []string{"v", "scope", "channelId", "boundary", "since"}, []string{"safeBeforePostId"}) { + return ChannelHistory{}, false + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return ChannelHistory{}, false + } + + var boundary map[string]json.RawMessage + if err := json.Unmarshal(outer["boundary"], &boundary); err != nil || + !onlyKeys(boundary, []string{"createAt", "id"}, nil) { + return ChannelHistory{}, false + } + + version, ok := integer(outer["v"]) + if !ok || version != 1 { + return ChannelHistory{}, false + } + createAt, ok := integer(boundary["createAt"]) + if !ok { + return ChannelHistory{}, false + } + + value := ChannelHistory{Version: int(version), Boundary: Boundary{CreateAt: createAt}} + if json.Unmarshal(outer["scope"], &value.Scope) != nil || + json.Unmarshal(outer["channelId"], &value.ChannelID) != nil || + json.Unmarshal(boundary["id"], &value.Boundary.ID) != nil { + return ChannelHistory{}, false + } + if !bytes.Equal(outer["since"], []byte("null")) { + since, valid := integer(outer["since"]) + if !valid { + return ChannelHistory{}, false + } + value.Since = &since + } + if raw, exists := outer["safeBeforePostId"]; exists { + if json.Unmarshal(raw, &value.SafeBeforePostID) != nil || value.SafeBeforePostID == "" { + return ChannelHistory{}, false + } + } + return value, true +} + +func integer(raw json.RawMessage) (int64, bool) { + var number float64 + if json.Unmarshal(raw, &number) != nil || math.IsNaN(number) || math.IsInf(number, 0) || + math.Trunc(number) != number || number < -9_007_199_254_740_991 || number > 9_007_199_254_740_991 { + return 0, false + } + return int64(number), true +} + +func onlyKeys(value map[string]json.RawMessage, required, optional []string) bool { + if value == nil || len(value) < len(required) || len(value) > len(required)+len(optional) { + return false + } + allowed := make(map[string]struct{}, len(required)+len(optional)) + for _, key := range append(required, optional...) { + allowed[key] = struct{}{} + } + for _, key := range required { + if _, exists := value[key]; !exists { + return false + } + } + for key := range value { + if _, exists := allowed[key]; !exists { + return false + } + } + return true +} diff --git a/internal/cursor/cursor_test.go b/internal/cursor/cursor_test.go new file mode 100644 index 0000000..9ec36d9 --- /dev/null +++ b/internal/cursor/cursor_test.go @@ -0,0 +1,124 @@ +package cursor + +import ( + "encoding/base64" + "errors" + "reflect" + "strings" + "testing" +) + +func testCursor() ChannelHistory { + since := int64(10) + return ChannelHistory{ + Version: 1, Scope: "channel", ChannelID: "channel", + Boundary: Boundary{CreateAt: 123, ID: "post"}, Since: &since, + } +} + +func TestChannelHistoryRoundTrip(t *testing.T) { + want := testCursor() + encoded, err := EncodeChannelHistory(want) + if err != nil { + t.Fatal(err) + } + const canonical = `eyJ2IjoxLCJzY29wZSI6ImNoYW5uZWwiLCJjaGFubmVsSWQiOiJjaGFubmVsIiwiYm91bmRhcnkiOnsiY3JlYXRlQXQiOjEyMywiaWQiOiJwb3N0In0sInNpbmNlIjoxMH0` + if encoded != canonical { + t.Fatalf("encoded = %q, want %q", encoded, canonical) + } + got, err := DecodeChannelHistory(encoded) + if err != nil || !reflect.DeepEqual(got, want) { + t.Fatalf("DecodeChannelHistory() = %#v, %v; want %#v", got, err, want) + } +} + +func TestDecodeChannelHistoryRejectsInvalid(t *testing.T) { + encode := func(json string) string { return base64.RawURLEncoding.EncodeToString([]byte(json)) } + tests := map[string]string{ + "empty": "", "not JSON": "not_json", "padding": "e30=", "too long": strings.Repeat("a", 2049), + "version": encode(`{"v":2,"scope":"channel","channelId":"channel","boundary":{"createAt":123,"id":"post"},"since":10}`), + "bad boundary": encode(`{"v":1,"scope":"channel","channelId":"channel","boundary":{"createAt":-1,"id":""},"since":10}`), + "extra": encode(`{"v":1,"scope":"channel","channelId":"channel","boundary":{"createAt":123,"id":"post"},"since":10,"extra":true}`), + "boundary extra": encode(`{"v":1,"scope":"channel","channelId":"channel","boundary":{"createAt":123,"id":"post","extra":true},"since":10}`), + "since beyond boundary": encode(`{"v":1,"scope":"channel","channelId":"channel","boundary":{"createAt":123,"id":"post"},"since":124}`), + "optional null": encode(`{"v":1,"scope":"channel","channelId":"channel","boundary":{"createAt":123,"id":"post"},"since":10,"safeBeforePostId":null}`), + "invalid UTF-8": base64.RawURLEncoding.EncodeToString([]byte{0xff}), + "oversized decoded": base64.RawURLEncoding.EncodeToString([]byte(strings.Repeat(" ", 1537))), + "trailing garbage": encode(`{"v":1,"scope":"channel","channelId":"channel","boundary":{"createAt":123,"id":"post"},"since":10}x`), + } + for name, encoded := range tests { + t.Run(name, func(t *testing.T) { + _, err := DecodeChannelHistory(encoded) + if !errors.Is(err, ErrInvalidCursor) || err.Error() != "invalid cursor" { + t.Fatalf("error = %v, want generic ErrInvalidCursor", err) + } + }) + } +} + +func TestChannelHistoryValidationBoundaries(t *testing.T) { + valid := testCursor() + longID := strings.Repeat("a", 128) + valid.ChannelID, valid.Boundary.ID, valid.SafeBeforePostID = longID, "-_AZaz09", "safe_post" + valid.Boundary.CreateAt = maxDateMillis + *valid.Since = maxDateMillis + if _, err := EncodeChannelHistory(valid); err != nil { + t.Fatalf("boundary cursor rejected: %v", err) + } + + invalid := []ChannelHistory{testCursor(), testCursor(), testCursor(), testCursor()} + invalid[0].ChannelID = strings.Repeat("a", 129) + invalid[1].Boundary.CreateAt = maxDateMillis + 1 + invalid[2].SafeBeforePostID = "not safe!" + minusOne := int64(-1) + invalid[3].Since = &minusOne + for i, value := range invalid { + if _, err := EncodeChannelHistory(value); !errors.Is(err, ErrInvalidCursor) { + t.Errorf("case %d: error = %v, want ErrInvalidCursor", i, err) + } + } +} + +func TestDecodeAcceptsJSONSafeIntegerSpellings(t *testing.T) { + encoded := base64.RawURLEncoding.EncodeToString([]byte(`{"v":1.0,"scope":"channel","channelId":"channel","boundary":{"createAt":1.23e2,"id":"post"},"since":1e1}`)) + got, err := DecodeChannelHistory(encoded) + if err != nil || got.Version != 1 || got.Boundary.CreateAt != 123 || got.Since == nil || *got.Since != 10 { + t.Fatalf("DecodeChannelHistory() = %#v, %v", got, err) + } +} + +func TestComparePostIDs(t *testing.T) { + values := []string{"-", "A", "_", "z"} + for i := 1; i < len(values); i++ { + if ComparePostIDs(values[i-1], values[i]) != -1 || ComparePostIDs(values[i], values[i-1]) != 1 { + t.Fatalf("unexpected ordering for %q and %q", values[i-1], values[i]) + } + } + if ComparePostIDs("same", "same") != 0 { + t.Fatal("equal IDs did not compare equal") + } +} + +func FuzzDecodeChannelHistory(f *testing.F) { + encoded, _ := EncodeChannelHistory(testCursor()) + for _, seed := range []string{encoded, "", "e30=", "not_json"} { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, input string) { + value, err := DecodeChannelHistory(input) + if err != nil { + if !errors.Is(err, ErrInvalidCursor) { + t.Fatalf("unexpected error: %v", err) + } + return + } + roundTrip, err := EncodeChannelHistory(value) + if err != nil { + t.Fatalf("decoded cursor failed to encode: %v", err) + } + decoded, err := DecodeChannelHistory(roundTrip) + if err != nil || !reflect.DeepEqual(decoded, value) { + t.Fatalf("round trip = %#v, %v; want %#v", decoded, err, value) + } + }) +} From a2827a2519f88eccc7379ef12e437e82daca4c65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 06:39:04 +0300 Subject: [PATCH 019/119] feat: port bounded thread hydration --- internal/mattermost/posts.go | 141 ++++++++++++++++++--- internal/mattermost/posts_test.go | 57 +++++++++ internal/retrieval/hydration.go | 159 +++++++++++++++++++++++ internal/retrieval/thread.go | 148 ++++++++++++++++++++++ internal/retrieval/thread_test.go | 202 ++++++++++++++++++++++++++++++ 5 files changed, 689 insertions(+), 18 deletions(-) create mode 100644 internal/retrieval/hydration.go create mode 100644 internal/retrieval/thread.go create mode 100644 internal/retrieval/thread_test.go diff --git a/internal/mattermost/posts.go b/internal/mattermost/posts.go index c4cfed9..4482415 100644 --- a/internal/mattermost/posts.go +++ b/internal/mattermost/posts.go @@ -29,20 +29,25 @@ type postTransport interface { // Post retains only fields needed for history selection. Presentation-specific // fields belong in a later, wider model rather than leaking unchecked payloads. type Post struct { - ID string - ChannelID string - Message string - CreateAt int64 - DeleteAt int64 + ID string + ChannelID string + Message string + CreateAt int64 + DeleteAt int64 + RootID string + ReplyCount int + ThreadShapeKnown bool } func (p *Post) UnmarshalJSON(data []byte) error { var raw struct { - ID json.RawMessage `json:"id"` - ChannelID json.RawMessage `json:"channel_id"` - Message json.RawMessage `json:"message"` - CreateAt json.RawMessage `json:"create_at"` - DeleteAt json.RawMessage `json:"delete_at"` + ID json.RawMessage `json:"id"` + ChannelID json.RawMessage `json:"channel_id"` + Message json.RawMessage `json:"message"` + CreateAt json.RawMessage `json:"create_at"` + DeleteAt json.RawMessage `json:"delete_at"` + RootID json.RawMessage `json:"root_id"` + ReplyCount json.RawMessage `json:"reply_count"` } if err := json.Unmarshal(data, &raw); err != nil { return ErrInvalidPostResponse @@ -52,25 +57,60 @@ func (p *Post) UnmarshalJSON(data []byte) error { message, messageOK := strictString(raw.Message) createAt, createOK := nonnegativeInteger(raw.CreateAt) deleteAt, deleteOK := nonnegativeInteger(raw.DeleteAt) - if !idOK || !channelOK || !messageOK || !createOK || createAt == 0 || createAt > maxDateMilliseconds || !deleteOK || deleteAt > maxDateMilliseconds { + rootID, rootOK, rootKnown := optionalPostID(raw.RootID) + replyCount, replyOK, replyKnown := optionalNonnegativeInteger(raw.ReplyCount) + if !idOK || !channelOK || !messageOK || !createOK || createAt == 0 || createAt > maxDateMilliseconds || !deleteOK || deleteAt > maxDateMilliseconds || !rootOK || !replyOK { return ErrInvalidPostResponse } - *p = Post{ID: id, ChannelID: channelID, Message: message, CreateAt: createAt, DeleteAt: deleteAt} + *p = Post{ID: id, ChannelID: channelID, Message: message, CreateAt: createAt, DeleteAt: deleteAt, RootID: rootID, ReplyCount: replyCount, ThreadShapeKnown: rootKnown && replyKnown} return nil } +func optionalPostID(raw json.RawMessage) (string, bool, bool) { + if len(raw) == 0 { + return "", true, false + } + var value string + if json.Unmarshal(raw, &value) != nil { + return "", false, true + } + if value == "" { + return "", true, true + } + value, ok := safePostID(raw) + return value, ok, true +} + +func optionalNonnegativeInteger(raw json.RawMessage) (int, bool, bool) { + if len(raw) == 0 { + return 0, true, false + } + value, ok := nonnegativeInteger(raw) + if !ok || value > int64(^uint(0)>>1) { + return 0, false, true + } + return int(value), true, true +} + func safePostID(raw json.RawMessage) (string, bool) { value, ok := requiredString(raw) - if !ok || len(value) > maxPostIDLength { + if !ok || !isSafePostID(value) { return "", false } + return value, true +} + +func isSafePostID(value string) bool { + if len(value) == 0 || len(value) > maxPostIDLength { + return false + } for i := range len(value) { c := value[i] if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '-') { - return "", false + return false } } - return value, true + return true } func nonnegativeInteger(raw json.RawMessage) (int64, bool) { @@ -89,12 +129,23 @@ type OrderedPostsPage struct { HasNext *bool FirstInaccessiblePostTime *int64 Incomplete bool + Continuation *ThreadCursor + ThreadRootID string + ThreadChannelID string + ContainsRequestedPost bool bindings []postBinding } +type ThreadCursor struct { + PostID string + CreateAt int64 +} + type postBinding struct { - ID string - ChannelID string + ID string + ChannelID string + RootID string + ThreadShapeKnown bool } func (p *OrderedPostsPage) UnmarshalJSON(data []byte) error { @@ -132,7 +183,8 @@ func (p *OrderedPostsPage) UnmarshalJSON(data []byte) error { result.Incomplete = true continue } - result.bindings = append(result.bindings, postBinding{ID: post.ID, ChannelID: post.ChannelID}) + result.bindings = append(result.bindings, postBinding{ID: post.ID, ChannelID: post.ChannelID, RootID: post.RootID, ThreadShapeKnown: post.ThreadShapeKnown}) + result.Continuation = &ThreadCursor{PostID: post.ID, CreateAt: post.CreateAt} if post.DeleteAt == 0 { result.Posts = append(result.Posts, post) } @@ -163,6 +215,12 @@ type ChannelPostsOptions struct { Before string } +type ThreadPageOptions struct { + PerPage int + FromPost string + FromCreateAt *int64 +} + type Posts struct{ client postTransport } func NewPosts(client postTransport) *Posts { return &Posts{client: client} } @@ -190,3 +248,50 @@ func (s *Posts) ChannelPage(ctx context.Context, channelID string, options Chann } return page, nil } + +func (s *Posts) ThreadPage(ctx context.Context, postID string, options ThreadPageOptions) (OrderedPostsPage, error) { + if !isSafePostID(postID) || options.PerPage <= 0 || options.PerPage > MaxPostsPage || + (options.FromPost != "" && !isSafePostID(options.FromPost)) || + (options.FromCreateAt != nil && (*options.FromCreateAt <= 0 || *options.FromCreateAt > maxDateMilliseconds)) { + return OrderedPostsPage{}, ErrInvalidPostsRequest + } + params := []string{"perPage=" + strconv.Itoa(options.PerPage), "direction=down"} + if options.FromPost != "" { + params = append(params, "fromPost="+url.QueryEscape(options.FromPost)) + } + if options.FromCreateAt != nil { + params = append(params, "fromCreateAt="+strconv.FormatInt(*options.FromCreateAt, 10)) + } + var page OrderedPostsPage + path := "/posts/" + url.PathEscape(postID) + "/thread?" + strings.Join(params, "&") + if err := s.client.Get(ctx, path, &page); err != nil { + return OrderedPostsPage{}, err + } + for _, binding := range page.bindings { + if page.ThreadChannelID == "" { + page.ThreadChannelID = binding.ChannelID + } else if page.ThreadChannelID != binding.ChannelID { + return OrderedPostsPage{}, ErrInvalidPostsResponse + } + if binding.ID == postID { + page.ContainsRequestedPost = true + } + if !binding.ThreadShapeKnown { + page.Incomplete = true + continue + } + candidateRoot := binding.RootID + if candidateRoot == "" { + candidateRoot = binding.ID + } + if page.ThreadRootID == "" { + page.ThreadRootID = candidateRoot + } else if page.ThreadRootID != candidateRoot { + return OrderedPostsPage{}, ErrInvalidPostsResponse + } + } + if page.ThreadRootID == postID { + page.ContainsRequestedPost = true + } + return page, nil +} diff --git a/internal/mattermost/posts_test.go b/internal/mattermost/posts_test.go index 3a603c6..483e624 100644 --- a/internal/mattermost/posts_test.go +++ b/internal/mattermost/posts_test.go @@ -105,6 +105,63 @@ func TestChannelPageRejectsDeletedCrossChannelPost(t *testing.T) { } } +func TestThreadPageBuildsMattermostCursorAndChecksDeletedBindings(t *testing.T) { + var gotPath string + api := NewPosts(postTransportFunc(func(_ context.Context, path string, out any) error { + gotPath = path + return json.Unmarshal([]byte(`{"order":["root","gone"],"posts":{"root":{"id":"root","channel_id":"chan","message":"root","create_at":1,"delete_at":0,"root_id":"","reply_count":1},"gone":{"id":"gone","channel_id":"chan","message":"stale","create_at":2,"delete_at":3,"root_id":"other","reply_count":0}},"has_next":true}`), out) + })) + fromAt := int64(7) + _, err := api.ThreadPage(context.Background(), "root", ThreadPageOptions{PerPage: 200, FromPost: "cursor", FromCreateAt: &fromAt}) + if !errors.Is(err, ErrInvalidPostsResponse) { + t.Fatalf("error = %v", err) + } + for _, part := range []string{"/posts/root/thread?", "direction=down", "fromCreateAt=7", "fromPost=cursor", "perPage=200"} { + if !strings.Contains(gotPath, part) { + t.Fatalf("path %q missing %q", gotPath, part) + } + } +} + +func TestThreadPageContinuationUsesLastValidOrderedDeletedPost(t *testing.T) { + api := NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { + return json.Unmarshal([]byte(`{"order":["root","gone"],"posts":{"root":{"id":"root","channel_id":"chan","message":"root","create_at":1,"delete_at":0,"root_id":"","reply_count":1},"gone":{"id":"gone","channel_id":"chan","message":"stale","create_at":2,"delete_at":3,"root_id":"root","reply_count":0}},"has_next":true}`), out) + })) + page, err := api.ThreadPage(context.Background(), "root", ThreadPageOptions{PerPage: 200}) + if err != nil || len(page.Posts) != 1 || page.Continuation == nil || page.Continuation.PostID != "gone" || page.Continuation.CreateAt != 2 { + t.Fatalf("page=%#v error=%v", page, err) + } +} + +func TestThreadPageAcceptsReplyIDAndInfersCanonicalIdentity(t *testing.T) { + api := NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { + return json.Unmarshal([]byte(`{"order":["root","reply"],"posts":{"root":{"id":"root","channel_id":"chan","message":"root","create_at":1,"delete_at":0,"root_id":"","reply_count":1},"reply":{"id":"reply","channel_id":"chan","message":"reply","create_at":2,"delete_at":0,"root_id":"root","reply_count":0}},"has_next":false}`), out) + })) + page, err := api.ThreadPage(context.Background(), "reply", ThreadPageOptions{PerPage: 200}) + if err != nil || page.ThreadRootID != "root" || page.ThreadChannelID != "chan" || !page.ContainsRequestedPost { + t.Fatalf("page=%#v error=%v", page, err) + } +} + +func TestThreadPageRejectsCrossChannelCandidates(t *testing.T) { + api := NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { + return json.Unmarshal([]byte(`{"order":["root","reply"],"posts":{"root":{"id":"root","channel_id":"one","message":"root","create_at":1,"delete_at":0,"root_id":"","reply_count":1},"reply":{"id":"reply","channel_id":"two","message":"reply","create_at":2,"delete_at":3,"root_id":"root","reply_count":0}}}`), out) + })) + if _, err := api.ThreadPage(context.Background(), "root", ThreadPageOptions{PerPage: 200}); !errors.Is(err, ErrInvalidPostsResponse) { + t.Fatalf("error = %v", err) + } +} + +func TestThreadPageMarksMissingThreadShapeIncomplete(t *testing.T) { + api := NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { + return json.Unmarshal([]byte(`{"order":["root"],"posts":{"root":{"id":"root","channel_id":"chan","message":"root","create_at":1,"delete_at":0}},"has_next":false}`), out) + })) + page, err := api.ThreadPage(context.Background(), "root", ThreadPageOptions{PerPage: 200}) + if err != nil || !page.Incomplete || page.Posts[0].ThreadShapeKnown { + t.Fatalf("page=%#v error=%v", page, err) + } +} + func TestPostRejectsValuesOutsideDeterministicDomain(t *testing.T) { for _, payload := range []string{ `{"id":"nonascii-é","channel_id":"channel","message":"x","create_at":1,"delete_at":0}`, diff --git a/internal/retrieval/hydration.go b/internal/retrieval/hydration.go new file mode 100644 index 0000000..1542728 --- /dev/null +++ b/internal/retrieval/hydration.go @@ -0,0 +1,159 @@ +package retrieval + +import ( + "context" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" +) + +type VisibleThreadsStatus uint8 + +const ( + VisibleThreadsNotRequested VisibleThreadsStatus = iota + VisibleThreadsComplete + VisibleThreadsPartial +) + +type VisibleThreadsMetadata struct { + Status VisibleThreadsStatus + HydratedRootCount int + FailedRootIDs []string +} + +type HydrationResult struct { + Posts []mattermost.Post + VisibleThreads VisibleThreadsMetadata +} + +func HydrateVisibleThreads(ctx context.Context, source threadPageSource, seeds []mattermost.Post, requested bool) (HydrationResult, error) { + if !requested { + return HydrationResult{Posts: seeds, VisibleThreads: VisibleThreadsMetadata{Status: VisibleThreadsNotRequested, FailedRootIDs: []string{}}}, nil + } + rootIDs, unknownShape := visibleRootIDs(seeds) + if len(rootIDs) == 0 { + status := VisibleThreadsComplete + if unknownShape { + status = VisibleThreadsPartial + } + return HydrationResult{Posts: seeds, VisibleThreads: VisibleThreadsMetadata{Status: status, FailedRootIDs: []string{}}}, nil + } + type outcome struct { + result ThreadResult + err error + reused bool + } + outcomes := make([]outcome, len(rootIDs)) + jobs := make(chan int) + worker := func() { + for index := range jobs { + if ctx.Err() != nil { + return + } + rootID := rootIDs[index] + if seedThreadComplete(seeds, rootID) { + outcomes[index].reused = true + continue + } + outcomes[index].result, outcomes[index].err = Thread(ctx, source, rootID) + } + } + workers := len(rootIDs) + if workers > 4 { + workers = 4 + } + done := make(chan struct{}, workers) + for range workers { + go func() { worker(); done <- struct{}{} }() + } + dispatchCanceled := false +dispatch: + for index := range rootIDs { + select { + case jobs <- index: + case <-ctx.Done(): + dispatchCanceled = true + break dispatch + } + } + close(jobs) + for range workers { + <-done + } + if dispatchCanceled || ctx.Err() != nil { + return HydrationResult{}, ctx.Err() + } + + posts := append([]mattermost.Post(nil), seeds...) + seen := make(map[string]struct{}, len(seeds)) + for _, post := range seeds { + seen[post.ID] = struct{}{} + } + metadata := VisibleThreadsMetadata{Status: VisibleThreadsComplete, FailedRootIDs: []string{}} + if unknownShape { + metadata.Status = VisibleThreadsPartial + } + for index, rootID := range rootIDs { + outcome := outcomes[index] + for _, post := range outcome.result.Posts { + if _, exists := seen[post.ID]; exists { + continue + } + seen[post.ID] = struct{}{} + posts = append(posts, post) + } + rootPresent := false + for _, post := range outcome.result.Posts { + if post.ID == rootID && post.RootID == "" { + rootPresent = true + break + } + } + if outcome.reused || (outcome.err == nil && outcome.result.Completeness == CompletenessComplete && rootPresent) { + metadata.HydratedRootCount++ + } else { + metadata.Status = VisibleThreadsPartial + metadata.FailedRootIDs = append(metadata.FailedRootIDs, rootID) + } + } + return HydrationResult{Posts: posts, VisibleThreads: metadata}, nil +} + +func visibleRootIDs(posts []mattermost.Post) ([]string, bool) { + seen := make(map[string]struct{}) + result := make([]string, 0) + unknownShape := false + for _, post := range posts { + if !post.ThreadShapeKnown { + unknownShape = true + continue + } + rootID := post.RootID + if rootID == "" && post.ReplyCount > 0 { + rootID = post.ID + } + if rootID == "" { + continue + } + if _, exists := seen[rootID]; exists { + continue + } + seen[rootID] = struct{}{} + result = append(result, rootID) + } + return result, unknownShape +} + +func seedThreadComplete(posts []mattermost.Post, rootID string) bool { + var root *mattermost.Post + replies := 0 + for index := range posts { + post := &posts[index] + if post.ThreadShapeKnown && post.ID == rootID && post.RootID == "" { + root = post + } + if post.ThreadShapeKnown && post.RootID == rootID { + replies++ + } + } + return root != nil && replies >= root.ReplyCount +} diff --git a/internal/retrieval/thread.go b/internal/retrieval/thread.go new file mode 100644 index 0000000..adcd616 --- /dev/null +++ b/internal/retrieval/thread.go @@ -0,0 +1,148 @@ +package retrieval + +import ( + "context" + "errors" + "strings" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" +) + +const MaxThreadPages = 100 + +type ThreadResult struct { + Posts []mattermost.Post + Completeness Completeness +} + +type threadPageSource interface { + ThreadPage(context.Context, string, mattermost.ThreadPageOptions) (mattermost.OrderedPostsPage, error) +} + +func Thread(ctx context.Context, source threadPageSource, rootID string) (ThreadResult, error) { + if source == nil || strings.TrimSpace(rootID) == "" { + return ThreadResult{}, mattermost.ErrInvalidPostsRequest + } + byID := make(map[string]mattermost.Post) + order := make([]string, 0) + var fromPost string + var fromCreateAt *int64 + uncertain, stagnantPages := false, 0 + var canonicalRootID, canonicalChannelID string + + for pageNumber := 0; pageNumber < MaxThreadPages; pageNumber++ { + if err := ctx.Err(); err != nil { + return ThreadResult{}, err + } + page, err := source.ThreadPage(ctx, rootID, mattermost.ThreadPageOptions{ + PerPage: mattermost.MaxPostsPage, FromPost: fromPost, FromCreateAt: fromCreateAt, + }) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || ctx.Err() != nil { + return ThreadResult{}, contextError(ctx, err) + } + if pageNumber == 0 { + return ThreadResult{}, err + } + return ThreadResult{Posts: orderedPosts(byID, order), Completeness: CompletenessUnknown}, nil + } + if page.Incomplete || page.FirstInaccessiblePostTime != nil { + uncertain = true + } + if pageNumber == 0 && page.ThreadChannelID != "" && !page.ContainsRequestedPost { + return ThreadResult{}, mattermost.ErrInvalidPostsResponse + } + if page.ThreadRootID != "" { + if canonicalRootID == "" { + canonicalRootID = page.ThreadRootID + } else if canonicalRootID != page.ThreadRootID { + return ThreadResult{Posts: orderedPosts(byID, order), Completeness: CompletenessUnknown}, nil + } + } + if page.ThreadChannelID != "" { + if canonicalChannelID == "" { + canonicalChannelID = page.ThreadChannelID + } else if canonicalChannelID != page.ThreadChannelID { + return ThreadResult{Posts: orderedPosts(byID, order), Completeness: CompletenessUnknown}, nil + } + } + added := 0 + for _, post := range page.Posts { + if _, exists := byID[post.ID]; exists { + continue + } + byID[post.ID] = post + order = append(order, post.ID) + added++ + } + + if page.HasNext == nil || !*page.HasNext { + complete := page.HasNext != nil && !*page.HasNext + if page.HasNext == nil && !page.Incomplete { + completionRootID := canonicalRootID + if completionRootID == "" { + completionRootID = rootID + } + complete = legacyThreadComplete(byID, completionRootID) + } + completeness := CompletenessUnknown + if !uncertain && complete { + completeness = CompletenessComplete + } + return ThreadResult{Posts: orderedPosts(byID, order), Completeness: completeness}, nil + } + + if page.Continuation == nil { + return ThreadResult{Posts: orderedPosts(byID, order), Completeness: threadPartialCompleteness(uncertain)}, nil + } + advanced := page.Continuation.PostID != fromPost || fromCreateAt == nil || page.Continuation.CreateAt != *fromCreateAt + if added > 0 && advanced { + stagnantPages = 0 + } else { + stagnantPages++ + } + if stagnantPages >= 2 { + return ThreadResult{Posts: orderedPosts(byID, order), Completeness: threadPartialCompleteness(uncertain)}, nil + } + fromPost = page.Continuation.PostID + value := page.Continuation.CreateAt + fromCreateAt = &value + } + return ThreadResult{Posts: orderedPosts(byID, order), Completeness: threadPartialCompleteness(uncertain)}, nil +} + +func contextError(ctx context.Context, fallback error) error { + if err := ctx.Err(); err != nil { + return err + } + return fallback +} + +func threadPartialCompleteness(uncertain bool) Completeness { + if uncertain { + return CompletenessUnknown + } + return CompletenessTruncated +} + +func legacyThreadComplete(posts map[string]mattermost.Post, rootID string) bool { + root, ok := posts[rootID] + if !ok || !root.ThreadShapeKnown || root.RootID != "" { + return false + } + replies := 0 + for _, post := range posts { + if post.RootID == rootID { + replies++ + } + } + return replies >= root.ReplyCount +} + +func orderedPosts(byID map[string]mattermost.Post, order []string) []mattermost.Post { + posts := make([]mattermost.Post, 0, len(order)) + for _, id := range order { + posts = append(posts, byID[id]) + } + return posts +} diff --git a/internal/retrieval/thread_test.go b/internal/retrieval/thread_test.go new file mode 100644 index 0000000..fc43b8f --- /dev/null +++ b/internal/retrieval/thread_test.go @@ -0,0 +1,202 @@ +package retrieval + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" +) + +type threadSourceFunc func(context.Context, string, mattermost.ThreadPageOptions) (mattermost.OrderedPostsPage, error) + +func (f threadSourceFunc) ThreadPage(ctx context.Context, rootID string, options mattermost.ThreadPageOptions) (mattermost.OrderedPostsPage, error) { + return f(ctx, rootID, options) +} + +func threadPost(id, rootID string, at int64, replies int) mattermost.Post { + return mattermost.Post{ID: id, RootID: rootID, ChannelID: "channel", Message: id, CreateAt: at, ReplyCount: replies, ThreadShapeKnown: true} +} + +func boolPointer(value bool) *bool { return &value } + +func TestThreadPaginatesInResponseOrderAndDeduplicates(t *testing.T) { + root := threadPost("root", "", 1, 2) + reply1 := threadPost("reply-1", "root", 2, 0) + reply2 := threadPost("reply-2", "root", 3, 0) + var requests []mattermost.ThreadPageOptions + result, err := Thread(context.Background(), threadSourceFunc(func(_ context.Context, _ string, options mattermost.ThreadPageOptions) (mattermost.OrderedPostsPage, error) { + requests = append(requests, options) + if len(requests) == 1 { + return mattermost.OrderedPostsPage{Posts: []mattermost.Post{root, reply1}, RawCount: 2, HasNext: boolPointer(true), Continuation: &mattermost.ThreadCursor{PostID: reply1.ID, CreateAt: reply1.CreateAt}}, nil + } + return mattermost.OrderedPostsPage{Posts: []mattermost.Post{root, reply1, reply2}, RawCount: 3, HasNext: boolPointer(false)}, nil + }), "root") + if err != nil || result.Completeness != CompletenessComplete || fmt.Sprint(ids(result.Posts)) != "[root reply-1 reply-2]" { + t.Fatalf("result=%#v error=%v", result, err) + } + if len(requests) != 2 || requests[1].FromPost != "reply-1" || requests[1].FromCreateAt == nil || *requests[1].FromCreateAt != 2 { + t.Fatalf("requests = %#v", requests) + } +} + +func TestThreadRetainsLaterPageContentAsUnknown(t *testing.T) { + root := threadPost("root", "", 1, 2) + reply := threadPost("reply", "root", 2, 0) + calls := 0 + result, err := Thread(context.Background(), threadSourceFunc(func(context.Context, string, mattermost.ThreadPageOptions) (mattermost.OrderedPostsPage, error) { + calls++ + if calls == 2 { + return mattermost.OrderedPostsPage{}, errors.New("later page failed") + } + return mattermost.OrderedPostsPage{Posts: []mattermost.Post{root, reply}, HasNext: boolPointer(true), Continuation: &mattermost.ThreadCursor{PostID: reply.ID, CreateAt: reply.CreateAt}}, nil + }), "root") + if err != nil || result.Completeness != CompletenessUnknown || fmt.Sprint(ids(result.Posts)) != "[root reply]" { + t.Fatalf("result=%#v error=%v", result, err) + } +} + +func TestThreadLegacyCompletionAndStagnation(t *testing.T) { + root := threadPost("root", "", 1, 1) + reply := threadPost("reply", "root", 2, 0) + complete, err := Thread(context.Background(), threadSourceFunc(func(context.Context, string, mattermost.ThreadPageOptions) (mattermost.OrderedPostsPage, error) { + return mattermost.OrderedPostsPage{Posts: []mattermost.Post{root, reply}}, nil + }), "root") + if err != nil || complete.Completeness != CompletenessComplete { + t.Fatalf("complete=%#v error=%v", complete, err) + } + calls := 0 + partial, err := Thread(context.Background(), threadSourceFunc(func(context.Context, string, mattermost.ThreadPageOptions) (mattermost.OrderedPostsPage, error) { + calls++ + return mattermost.OrderedPostsPage{Posts: []mattermost.Post{root}, HasNext: boolPointer(true), Continuation: &mattermost.ThreadCursor{PostID: root.ID, CreateAt: root.CreateAt}}, nil + }), "root") + if err != nil || calls != 3 || partial.Completeness != CompletenessTruncated { + t.Fatalf("calls=%d partial=%#v error=%v", calls, partial, err) + } +} + +func TestHydrateVisibleThreadsReusesCompleteRootsAndBoundsConcurrency(t *testing.T) { + seeds := []mattermost.Post{threadPost("complete", "", 1, 1), threadPost("complete-reply", "complete", 2, 0)} + for index := range 9 { + seeds = append(seeds, threadPost(fmt.Sprintf("seed-%d", index), fmt.Sprintf("root-%d", index), int64(index+10), 0)) + } + var mu sync.Mutex + active, maximum, calls := 0, 0, 0 + source := threadSourceFunc(func(_ context.Context, rootID string, _ mattermost.ThreadPageOptions) (mattermost.OrderedPostsPage, error) { + mu.Lock() + active++ + calls++ + if active > maximum { + maximum = active + } + mu.Unlock() + time.Sleep(time.Millisecond) + mu.Lock() + active-- + mu.Unlock() + root := threadPost(rootID, "", 1, 1) + return mattermost.OrderedPostsPage{Posts: []mattermost.Post{root}, HasNext: boolPointer(false)}, nil + }) + result, err := HydrateVisibleThreads(context.Background(), source, seeds, true) + if err != nil { + t.Fatal(err) + } + if calls != 9 || maximum != 4 || result.VisibleThreads.Status != VisibleThreadsComplete || result.VisibleThreads.HydratedRootCount != 10 { + t.Fatalf("calls=%d maximum=%d metadata=%#v", calls, maximum, result.VisibleThreads) + } +} + +func TestHydrationFailureMetadataFollowsSeedOrderAndRetainsPartialPosts(t *testing.T) { + seeds := []mattermost.Post{threadPost("seed-a", "root-a", 3, 0), threadPost("seed-b", "root-b", 4, 0)} + source := threadSourceFunc(func(_ context.Context, rootID string, options mattermost.ThreadPageOptions) (mattermost.OrderedPostsPage, error) { + if rootID == "root-b" { + return mattermost.OrderedPostsPage{}, errors.New("failed") + } + if options.FromPost != "" { + return mattermost.OrderedPostsPage{}, errors.New("later failed") + } + root := threadPost(rootID, "", 1, 2) + contextPost := threadPost("context-a", rootID, 2, 0) + return mattermost.OrderedPostsPage{Posts: []mattermost.Post{root, contextPost}, HasNext: boolPointer(true), Continuation: &mattermost.ThreadCursor{PostID: contextPost.ID, CreateAt: 2}}, nil + }) + result, err := HydrateVisibleThreads(context.Background(), source, seeds, true) + if err != nil { + t.Fatal(err) + } + if result.VisibleThreads.Status != VisibleThreadsPartial || result.VisibleThreads.HydratedRootCount != 0 || fmt.Sprint(result.VisibleThreads.FailedRootIDs) != "[root-a root-b]" || fmt.Sprint(ids(result.Posts)) != "[seed-a seed-b root-a context-a]" { + t.Fatalf("result = %#v", result) + } +} + +func TestThreadMissingLegacyShapeCannotProveCompleteness(t *testing.T) { + root := mattermost.Post{ID: "root", ChannelID: "channel", Message: "root", CreateAt: 1} + result, err := Thread(context.Background(), threadSourceFunc(func(context.Context, string, mattermost.ThreadPageOptions) (mattermost.OrderedPostsPage, error) { + return mattermost.OrderedPostsPage{Posts: []mattermost.Post{root}}, nil + }), "root") + if err != nil || result.Completeness != CompletenessUnknown { + t.Fatalf("result=%#v error=%v", result, err) + } + hydrated, err := HydrateVisibleThreads(context.Background(), nil, []mattermost.Post{root}, true) + if err != nil || hydrated.VisibleThreads.Status != VisibleThreadsPartial { + t.Fatalf("hydrated=%#v error=%v", hydrated, err) + } +} + +func TestThreadReplyIDUsesInferredRootForLegacyCompletion(t *testing.T) { + root := threadPost("root", "", 1, 1) + reply := threadPost("reply", "root", 2, 0) + result, err := Thread(context.Background(), threadSourceFunc(func(context.Context, string, mattermost.ThreadPageOptions) (mattermost.OrderedPostsPage, error) { + return mattermost.OrderedPostsPage{ + Posts: []mattermost.Post{root, reply}, ThreadRootID: "root", ThreadChannelID: "channel", ContainsRequestedPost: true, + }, nil + }), "reply") + if err != nil || result.Completeness != CompletenessComplete { + t.Fatalf("result=%#v error=%v", result, err) + } +} + +func TestThreadRejectsCrossPageIdentityWithoutMergingForeignPosts(t *testing.T) { + root := threadPost("root", "", 1, 1) + foreign := threadPost("foreign", "root", 2, 0) + foreign.ChannelID = "other" + calls := 0 + result, err := Thread(context.Background(), threadSourceFunc(func(context.Context, string, mattermost.ThreadPageOptions) (mattermost.OrderedPostsPage, error) { + calls++ + if calls == 1 { + return mattermost.OrderedPostsPage{Posts: []mattermost.Post{root}, HasNext: boolPointer(true), Continuation: &mattermost.ThreadCursor{PostID: root.ID, CreateAt: root.CreateAt}, ThreadRootID: "root", ThreadChannelID: "channel", ContainsRequestedPost: true}, nil + } + return mattermost.OrderedPostsPage{Posts: []mattermost.Post{foreign}, HasNext: boolPointer(false), ThreadRootID: "root", ThreadChannelID: "other"}, nil + }), "root") + if err != nil || result.Completeness != CompletenessUnknown || fmt.Sprint(ids(result.Posts)) != "[root]" { + t.Fatalf("result=%#v error=%v", result, err) + } +} + +func TestThreadAndHydrationPropagateCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + root := threadPost("root", "", 1, 1) + calls := 0 + _, err := Thread(ctx, threadSourceFunc(func(ctx context.Context, _ string, _ mattermost.ThreadPageOptions) (mattermost.OrderedPostsPage, error) { + calls++ + if calls == 1 { + cancel() + return mattermost.OrderedPostsPage{Posts: []mattermost.Post{root}, HasNext: boolPointer(true), Continuation: &mattermost.ThreadCursor{PostID: root.ID, CreateAt: root.CreateAt}}, nil + } + return mattermost.OrderedPostsPage{}, ctx.Err() + }), "root") + if !errors.Is(err, context.Canceled) { + t.Fatalf("thread error = %v", err) + } + + canceled, stop := context.WithCancel(context.Background()) + stop() + if _, err := HydrateVisibleThreads(canceled, threadSourceFunc(func(context.Context, string, mattermost.ThreadPageOptions) (mattermost.OrderedPostsPage, error) { + t.Fatal("source called after cancellation") + return mattermost.OrderedPostsPage{}, nil + }), []mattermost.Post{threadPost("seed", "root", 2, 0)}, true); !errors.Is(err, context.Canceled) { + t.Fatalf("hydration error = %v", err) + } +} From 73f0ac462ed601ca1093b21bb64abb6a528b2f89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 06:58:26 +0300 Subject: [PATCH 020/119] feat: port bounded message search --- internal/mattermost/posts.go | 123 +++++++++++++++++++++++- internal/mattermost/posts_test.go | 48 +++++++++ internal/retrieval/search.go | 133 +++++++++++++++++++++++++ internal/retrieval/search_test.go | 155 ++++++++++++++++++++++++++++++ 4 files changed, 456 insertions(+), 3 deletions(-) create mode 100644 internal/retrieval/search.go create mode 100644 internal/retrieval/search_test.go diff --git a/internal/mattermost/posts.go b/internal/mattermost/posts.go index 4482415..16b9099 100644 --- a/internal/mattermost/posts.go +++ b/internal/mattermost/posts.go @@ -17,13 +17,15 @@ const ( ) var ( - ErrInvalidPostResponse = errors.New("Mattermost returned an invalid post response") - ErrInvalidPostsResponse = errors.New("Mattermost returned an invalid posts response") - ErrInvalidPostsRequest = errors.New("invalid Mattermost posts request") + ErrInvalidPostResponse = errors.New("Mattermost returned an invalid post response") + ErrInvalidPostsResponse = errors.New("Mattermost returned an invalid posts response") + ErrInvalidSearchResponse = errors.New("Mattermost returned an invalid search response") + ErrInvalidPostsRequest = errors.New("invalid Mattermost posts request") ) type postTransport interface { Get(context.Context, string, any) error + PostRead(context.Context, string, any, any) error } // Post retains only fields needed for history selection. Presentation-specific @@ -221,10 +223,125 @@ type ThreadPageOptions struct { FromCreateAt *int64 } +const MaxSearchPage = 100 + +type SearchPageOptions struct { + Terms string + Page int + PerPage int +} + +type SearchPage struct { + Posts []Post + OrderedIDs []string + Matches map[string][]string + RawCount int + HasNext *bool + FirstInaccessiblePostTime *int64 + Incomplete bool +} + +func (p *SearchPage) UnmarshalJSON(data []byte) error { + var raw struct { + Order json.RawMessage `json:"order"` + Posts map[string]json.RawMessage `json:"posts"` + Matches map[string]json.RawMessage `json:"matches"` + HasNext json.RawMessage `json:"has_next"` + FirstInaccessiblePostTime json.RawMessage `json:"first_inaccessible_post_time"` + } + if err := json.Unmarshal(data, &raw); err != nil { + return ErrInvalidSearchResponse + } + var order []json.RawMessage + if len(raw.Order) == 0 || json.Unmarshal(raw.Order, &order) != nil || order == nil { + return ErrInvalidSearchResponse + } + result := SearchPage{RawCount: len(order), Matches: make(map[string][]string)} + seen := make(map[string]struct{}) + if raw.Posts == nil && len(order) > 0 { + result.Incomplete = true + } + for _, rawID := range order { + var id string + if json.Unmarshal(rawID, &id) != nil || !isSafePostID(id) { + result.Incomplete = true + continue + } + if _, duplicate := seen[id]; duplicate { + continue + } + seen[id] = struct{}{} + result.OrderedIDs = append(result.OrderedIDs, id) + candidate, ok := raw.Posts[id] + if !ok { + result.Incomplete = true + continue + } + var post Post + if json.Unmarshal(candidate, &post) != nil || post.ID != id { + result.Incomplete = true + continue + } + if post.DeleteAt != 0 { + continue + } + result.Posts = append(result.Posts, post) + if value, exists := raw.Matches[id]; exists { + var rawMatches []json.RawMessage + if json.Unmarshal(value, &rawMatches) == nil && rawMatches != nil { + matches := make([]string, 0, len(rawMatches)) + for _, rawMatch := range rawMatches { + var match string + if json.Unmarshal(rawMatch, &match) == nil { + matches = append(matches, match) + } + } + result.Matches[id] = matches + } + } + } + if len(raw.HasNext) > 0 { + var hasNext bool + if string(raw.HasNext) == "null" || json.Unmarshal(raw.HasNext, &hasNext) != nil { + result.Incomplete = true + } else { + result.HasNext = &hasNext + } + } + if len(raw.FirstInaccessiblePostTime) > 0 && string(raw.FirstInaccessiblePostTime) != "null" { + value, ok := nonnegativeInteger(raw.FirstInaccessiblePostTime) + if !ok { + result.Incomplete = true + } else if value > 0 { + result.FirstInaccessiblePostTime = &value + } + } + *p = result + return nil +} + type Posts struct{ client postTransport } func NewPosts(client postTransport) *Posts { return &Posts{client: client} } +func (s *Posts) SearchPage(ctx context.Context, teamID string, options SearchPageOptions) (SearchPage, error) { + if strings.TrimSpace(teamID) == "" || strings.TrimSpace(options.Terms) == "" || options.Page < 0 || options.PerPage <= 0 || options.PerPage > MaxSearchPage { + return SearchPage{}, ErrInvalidPostsRequest + } + body := struct { + Terms string `json:"terms"` + IsOrSearch bool `json:"is_or_search"` + Page int `json:"page"` + PerPage int `json:"per_page"` + }{options.Terms, false, options.Page, options.PerPage} + var page SearchPage + path := "/teams/" + url.PathEscape(teamID) + "/posts/search" + if err := s.client.PostRead(ctx, path, body, &page); err != nil { + return SearchPage{}, err + } + return page, nil +} + func (s *Posts) ChannelPage(ctx context.Context, channelID string, options ChannelPostsOptions) (OrderedPostsPage, error) { if strings.TrimSpace(channelID) == "" || options.PerPage <= 0 || options.PerPage > MaxPostsPage || options.Page < 0 { return OrderedPostsPage{}, ErrInvalidPostsRequest diff --git a/internal/mattermost/posts_test.go b/internal/mattermost/posts_test.go index 483e624..62b6179 100644 --- a/internal/mattermost/posts_test.go +++ b/internal/mattermost/posts_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "reflect" "strings" "testing" ) @@ -14,6 +15,19 @@ func (f postTransportFunc) Get(ctx context.Context, path string, out any) error return f(ctx, path, out) } +func (f postTransportFunc) PostRead(ctx context.Context, path string, _ any, out any) error { + return f(ctx, path, out) +} + +type searchTransportFunc func(context.Context, string, any, any) error + +func (f searchTransportFunc) Get(context.Context, string, any) error { + return errors.New("unexpected GET") +} +func (f searchTransportFunc) PostRead(ctx context.Context, path string, body, out any) error { + return f(ctx, path, body, out) +} + func TestOrderedPostsPageNormalizesAndSuppressesDeleted(t *testing.T) { var page OrderedPostsPage err := json.Unmarshal([]byte(`{ @@ -65,6 +79,40 @@ func TestOrderedPostsPageTreatsExplicitNullHasNextAsIncomplete(t *testing.T) { } } +func TestSearchPageBuildsExactReadPOSTAndNormalizesEnvelope(t *testing.T) { + var gotPath string + var gotBody []byte + api := NewPosts(searchTransportFunc(func(_ context.Context, path string, body, out any) error { + gotPath = path + gotBody, _ = json.Marshal(body) + return json.Unmarshal([]byte(`{"order":["live","live","missing","gone"],"posts":{"live":{"id":"live","channel_id":"chan","message":"ok","create_at":2,"delete_at":0},"gone":{"id":"gone","channel_id":"chan","message":"stale","create_at":1,"delete_at":3}},"matches":{"live":["needle",7]},"has_next":true}`), out) + })) + page, err := api.SearchPage(context.Background(), "team/id", SearchPageOptions{Terms: "needle", Page: 2, PerPage: 17}) + if err != nil { + t.Fatal(err) + } + if gotPath != "/teams/team%2Fid/posts/search" || string(gotBody) != `{"terms":"needle","is_or_search":false,"page":2,"per_page":17}` { + t.Fatalf("path=%q body=%s", gotPath, gotBody) + } + if page.RawCount != 4 || len(page.OrderedIDs) != 3 || len(page.Posts) != 1 || page.Posts[0].ID != "live" || !page.Incomplete || !reflect.DeepEqual(page.Matches["live"], []string{"needle"}) { + t.Fatalf("page=%#v", page) + } +} + +func TestSearchPageRejectsMalformedEnvelopeAndInvalidRequest(t *testing.T) { + api := NewPosts(searchTransportFunc(func(_ context.Context, _ string, _ any, out any) error { + return json.Unmarshal([]byte(`null`), out) + })) + if _, err := api.SearchPage(context.Background(), "team", SearchPageOptions{Terms: "x", PerPage: 1}); !errors.Is(err, ErrInvalidSearchResponse) { + t.Fatalf("malformed error=%v", err) + } + for _, options := range []SearchPageOptions{{Terms: "", PerPage: 1}, {Terms: "x", PerPage: 0}, {Terms: "x", PerPage: 101}, {Terms: "x", Page: -1, PerPage: 1}} { + if _, err := api.SearchPage(context.Background(), "team", options); !errors.Is(err, ErrInvalidPostsRequest) { + t.Fatalf("options=%#v error=%v", options, err) + } + } +} + func TestChannelPageBuildsBoundedGETAndChecksBinding(t *testing.T) { var gotPath string api := NewPosts(postTransportFunc(func(_ context.Context, path string, out any) error { diff --git a/internal/retrieval/search.go b/internal/retrieval/search.go new file mode 100644 index 0000000..7cffbbc --- /dev/null +++ b/internal/retrieval/search.go @@ -0,0 +1,133 @@ +package retrieval + +import ( + "context" + "errors" + "strings" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" +) + +const MaxSearchPages = 100 + +var ErrInvalidSearchRequest = errors.New("invalid search request") + +type SearchOptions struct { + Limit int + Accept func(mattermost.Post) bool +} + +type SearchResult struct { + Posts []mattermost.Post + Matches map[string][]string + Completeness Completeness +} + +type searchPageSource interface { + SearchPage(context.Context, string, mattermost.SearchPageOptions) (mattermost.SearchPage, error) +} + +func Search(ctx context.Context, source searchPageSource, teamID, terms string, options SearchOptions) (SearchResult, error) { + if source == nil || strings.TrimSpace(teamID) == "" || strings.TrimSpace(terms) == "" || options.Limit <= 0 || int64(options.Limit) > maxSafeInteger { + return SearchResult{}, ErrInvalidSearchRequest + } + accept := options.Accept + if accept == nil { + accept = func(mattermost.Post) bool { return true } + } + target := options.Limit + 1 + perPage := target + if perPage > mattermost.MaxSearchPage { + perPage = mattermost.MaxSearchPage + } + byID := make(map[string]mattermost.Post) + matches := make(map[string][]string) + seen := make(map[string]struct{}) + uncertain, exhausted, stagnantPages := false, false, 0 + + for pageNumber := 0; pageNumber < MaxSearchPages; pageNumber++ { + if err := ctx.Err(); err != nil { + return SearchResult{}, err + } + page, err := source.SearchPage(ctx, teamID, mattermost.SearchPageOptions{Terms: terms, Page: pageNumber, PerPage: perPage}) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || ctx.Err() != nil { + return SearchResult{}, contextError(ctx, err) + } + return SearchResult{}, err + } + if page.Incomplete || page.FirstInaccessiblePostTime != nil { + uncertain = true + } + madeProgress := false + firstSeen := make(map[string]struct{}, len(page.OrderedIDs)) + for _, id := range page.OrderedIDs { + if _, duplicate := seen[id]; duplicate { + continue + } + seen[id] = struct{}{} + firstSeen[id] = struct{}{} + madeProgress = true + } + acceptedThisPage := make([]mattermost.Post, 0, len(page.Posts)) + for _, post := range page.Posts { + if _, first := firstSeen[post.ID]; !first || !accept(post) { + continue + } + byID[post.ID] = post + acceptedThisPage = append(acceptedThisPage, post) + if value, ok := page.Matches[post.ID]; ok { + matches[post.ID] = append([]string(nil), value...) + } + } + if madeProgress { + stagnantPages = 0 + } else { + stagnantPages++ + } + + if page.RawCount == 0 { + if page.HasNext != nil && *page.HasNext { + uncertain = true + } + exhausted = !uncertain + break + } + if page.HasNext != nil && !*page.HasNext { + exhausted = !uncertain + break + } + if stagnantPages >= 2 { + uncertain = true + break + } + if len(byID) >= target { + selected := mostRecent(byID, options.Limit) + cutoff := selected[len(selected)-1].CreateAt + if containsOlderThan(acceptedThisPage, cutoff) { + break + } + } + if pageNumber == MaxSearchPages-1 { + uncertain = true + } + } + + selected := mostRecent(byID, options.Limit) + selectedMatches := make(map[string][]string) + for _, post := range selected { + if value, ok := matches[post.ID]; ok { + selectedMatches[post.ID] = value + } + } + result := SearchResult{Posts: selected, Matches: selectedMatches} + switch { + case len(byID) > options.Limit: + result.Completeness = CompletenessTruncated + case uncertain || !exhausted: + result.Completeness = CompletenessUnknown + default: + result.Completeness = CompletenessComplete + } + return result, nil +} diff --git a/internal/retrieval/search_test.go b/internal/retrieval/search_test.go new file mode 100644 index 0000000..52bd112 --- /dev/null +++ b/internal/retrieval/search_test.go @@ -0,0 +1,155 @@ +package retrieval + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" +) + +type searchSourceFunc func(context.Context, string, mattermost.SearchPageOptions) (mattermost.SearchPage, error) + +func (f searchSourceFunc) SearchPage(ctx context.Context, teamID string, options mattermost.SearchPageOptions) (mattermost.SearchPage, error) { + return f(ctx, teamID, options) +} + +func searchPage(posts ...mattermost.Post) mattermost.SearchPage { + ids := make([]string, len(posts)) + for i := range posts { + ids[i] = posts[i].ID + } + return mattermost.SearchPage{Posts: posts, OrderedIDs: ids, RawCount: len(ids), Matches: map[string][]string{}} +} + +func TestSearchUsesLimitPlusOneDedupeAndDeterministicOrder(t *testing.T) { + var requests []mattermost.SearchPageOptions + result, err := Search(context.Background(), searchSourceFunc(func(_ context.Context, team string, options mattermost.SearchPageOptions) (mattermost.SearchPage, error) { + requests = append(requests, options) + if team != "team" { + t.Fatalf("team=%q", team) + } + if options.Page == 0 { + page := searchPage(testPost("b", 3), testPost("a", 3), testPost("old", 2)) + page.Matches = map[string][]string{"a": {"hit"}, "old": {"drop"}} + return page, nil + } + return mattermost.SearchPage{}, nil + }), "team", "needle", SearchOptions{Limit: 2}) + if err != nil || len(requests) != 1 || requests[0].PerPage != 3 || fmt.Sprint(ids(result.Posts)) != "[a b]" || result.Completeness != CompletenessTruncated || fmt.Sprint(result.Matches["a"]) != "[hit]" { + t.Fatalf("requests=%#v result=%#v err=%v", requests, result, err) + } +} + +func TestSearchContinuesRejectedMissingShortPagesAndEqualTimeCutoff(t *testing.T) { + var calls int + result, err := Search(context.Background(), searchSourceFunc(func(_ context.Context, _ string, options mattermost.SearchPageOptions) (mattermost.SearchPage, error) { + calls++ + switch options.Page { + case 0: + return mattermost.SearchPage{OrderedIDs: []string{"missing", "rejected"}, Posts: []mattermost.Post{testPost("rejected", 101)}, RawCount: 2, Incomplete: true}, nil + case 1: + return searchPage(testPost("d", 100), testPost("c", 100)), nil + case 2: + return searchPage(testPost("b", 100), testPost("a", 100)), nil + default: + return searchPage(testPost("older", 99)), nil + } + }), "team", "needle", SearchOptions{Limit: 2, Accept: func(post mattermost.Post) bool { return post.ID != "rejected" }}) + if err != nil || calls != 4 || fmt.Sprint(ids(result.Posts)) != "[a b]" || result.Completeness != CompletenessTruncated { + t.Fatalf("calls=%d result=%#v err=%v", calls, result, err) + } +} + +func TestSearchExhaustionPoisonAndStagnation(t *testing.T) { + falseValue, trueValue := false, true + tests := []struct { + name string + page mattermost.SearchPage + want Completeness + }{ + {"empty", mattermost.SearchPage{}, CompletenessComplete}, + {"explicit false", mattermost.SearchPage{RawCount: 1, OrderedIDs: []string{"missing"}, Incomplete: true, HasNext: &falseValue}, CompletenessUnknown}, + {"empty has next", mattermost.SearchPage{HasNext: &trueValue}, CompletenessUnknown}, + {"inaccessible", mattermost.SearchPage{FirstInaccessiblePostTime: ptr64(1)}, CompletenessUnknown}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := Search(context.Background(), searchSourceFunc(func(context.Context, string, mattermost.SearchPageOptions) (mattermost.SearchPage, error) { + return tt.page, nil + }), "team", "x", SearchOptions{Limit: 1}) + if err != nil || result.Completeness != tt.want { + t.Fatalf("result=%#v err=%v", result, err) + } + }) + } + calls := 0 + result, err := Search(context.Background(), searchSourceFunc(func(context.Context, string, mattermost.SearchPageOptions) (mattermost.SearchPage, error) { + calls++ + return mattermost.SearchPage{RawCount: 1, OrderedIDs: []string{"same"}}, nil + }), "team", "x", SearchOptions{Limit: 1}) + if err != nil || calls != 3 || result.Completeness != CompletenessUnknown { + t.Fatalf("calls=%d result=%#v err=%v", calls, result, err) + } +} + +func TestSearchBoundsPagesValidatesAndPropagatesCancellation(t *testing.T) { + calls := 0 + result, err := Search(context.Background(), searchSourceFunc(func(_ context.Context, _ string, options mattermost.SearchPageOptions) (mattermost.SearchPage, error) { + calls++ + return mattermost.SearchPage{RawCount: 1, OrderedIDs: []string{fmt.Sprintf("stale-%d", options.Page)}, Incomplete: true}, nil + }), "team", "x", SearchOptions{Limit: 1}) + if err != nil || calls != MaxSearchPages || result.Completeness != CompletenessUnknown { + t.Fatalf("calls=%d result=%#v err=%v", calls, result, err) + } + if _, err := Search(context.Background(), nil, "team", "x", SearchOptions{Limit: 1}); !errors.Is(err, ErrInvalidSearchRequest) { + t.Fatalf("validation error=%v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := Search(ctx, searchSourceFunc(func(context.Context, string, mattermost.SearchPageOptions) (mattermost.SearchPage, error) { + panic("called") + }), "team", "x", SearchOptions{Limit: 1}); !errors.Is(err, context.Canceled) { + t.Fatalf("cancel error=%v", err) + } +} + +func TestSearchPropagatesLaterPageFailure(t *testing.T) { + want := errors.New("page two failed") + _, err := Search(context.Background(), searchSourceFunc(func(_ context.Context, _ string, options mattermost.SearchPageOptions) (mattermost.SearchPage, error) { + if options.Page == 0 { + return searchPage(testPost("first", 2)), nil + } + return mattermost.SearchPage{}, want + }), "team", "x", SearchOptions{Limit: 2}) + if !errors.Is(err, want) { + t.Fatalf("error = %v", err) + } +} + +func TestSearchNeverReconsidersFirstSeenIDs(t *testing.T) { + calls := 0 + result, err := Search(context.Background(), searchSourceFunc(func(_ context.Context, _ string, options mattermost.SearchPageOptions) (mattermost.SearchPage, error) { + calls++ + switch options.Page { + case 0: + rejected := testPost("rejected", 3) + rejected.Message = "deny" + return mattermost.SearchPage{OrderedIDs: []string{"missing", "deleted", "rejected"}, Posts: []mattermost.Post{rejected}, RawCount: 3, Incomplete: true}, nil + case 1: + missing := testPost("missing", 5) + deleted := testPost("deleted", 4) + rejected := testPost("rejected", 3) + fresh := testPost("fresh", 2) + return searchPage(missing, deleted, rejected, fresh), nil + default: + return mattermost.SearchPage{}, nil + } + }), "team", "x", SearchOptions{Limit: 4, Accept: func(post mattermost.Post) bool { return post.Message != "deny" }}) + if err != nil || calls != 3 || fmt.Sprint(ids(result.Posts)) != "[fresh]" || result.Completeness != CompletenessUnknown { + t.Fatalf("calls=%d result=%#v error=%v", calls, result, err) + } +} + +func ptr64(value int64) *int64 { return &value } From 012403e5ef074b4041204789aa94a78dec31271e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 10:13:47 +0300 Subject: [PATCH 021/119] feat: add deterministic output model --- internal/output/date.go | 99 ++++++++++++++++++++++++++ internal/output/date_test.go | 113 ++++++++++++++++++++++++++++++ internal/output/model.go | 113 ++++++++++++++++++++++++++++++ internal/output/model_test.go | 35 +++++++++ internal/output/threading.go | 62 ++++++++++++++++ internal/output/threading_test.go | 82 ++++++++++++++++++++++ 6 files changed, 504 insertions(+) create mode 100644 internal/output/date.go create mode 100644 internal/output/date_test.go create mode 100644 internal/output/model.go create mode 100644 internal/output/model_test.go create mode 100644 internal/output/threading.go create mode 100644 internal/output/threading_test.go diff --git a/internal/output/date.go b/internal/output/date.go new file mode 100644 index 0000000..b0308cd --- /dev/null +++ b/internal/output/date.go @@ -0,0 +1,99 @@ +package output + +import ( + "fmt" + "math" + "time" +) + +type DateFormatter struct { + now func() time.Time + location *time.Location +} + +func NewDateFormatter(now func() time.Time, location *time.Location) DateFormatter { + if now == nil { + panic("output: nil clock") + } + if location == nil { + panic("output: nil location") + } + return DateFormatter{now: now, location: location} +} + +func (f DateFormatter) FormatDate(date time.Time, includeYear bool) string { + date = date.In(f.location) + if includeYear { + return fmt.Sprintf("%d %s %d", date.Day(), date.Format("Jan"), date.Year()) + } + return fmt.Sprintf("%d %s", date.Day(), date.Format("Jan")) +} + +func (f DateFormatter) FormatDateLong(date time.Time) string { + return date.In(f.location).Format("Monday, 2 January 2006") +} + +func (f DateFormatter) FormatTime(date time.Time) string { + return date.In(f.location).Format("15:04") +} + +func (f DateFormatter) FormatRelativeTime(date time.Time) string { + diffMillis := date.UnixMilli() - f.now().UnixMilli() + absMillis := diffMillis + if absMillis < 0 { + absMillis = -absMillis + } + unitMillis := int64(time.Second / time.Millisecond) + name := "" + switch { + case absMillis < int64(time.Minute/time.Millisecond): + return "just now" + case absMillis < int64(time.Hour/time.Millisecond): + unitMillis, name = int64(time.Minute/time.Millisecond), "minute" + case absMillis < int64((24*time.Hour)/time.Millisecond): + unitMillis, name = int64(time.Hour/time.Millisecond), "hour" + case absMillis < int64((7*24*time.Hour)/time.Millisecond): + unitMillis, name = int64((24*time.Hour)/time.Millisecond), "day" + case absMillis < int64((30*24*time.Hour)/time.Millisecond): + unitMillis, name = int64((7*24*time.Hour)/time.Millisecond), "week" + case absMillis < int64((365*24*time.Hour)/time.Millisecond): + unitMillis, name = int64((30*24*time.Hour)/time.Millisecond), "month" + default: + unitMillis, name = int64((365*24*time.Hour)/time.Millisecond), "year" + } + value := int64(math.Floor(float64(diffMillis)/float64(unitMillis) + 0.5)) // JavaScript Math.round + if name == "day" && value == -1 { + return "yesterday" + } + if name == "day" && value == 1 { + return "tomorrow" + } + if value == -1 && (name == "week" || name == "month" || name == "year") { + return "last " + name + } + if value == 1 && (name == "week" || name == "month" || name == "year") { + return "next " + name + } + label := name + if value != -1 && value != 1 { + label += "s" + } + if value < 0 { + return fmt.Sprintf("%d %s ago", -value, label) + } + return fmt.Sprintf("in %d %s", value, label) +} + +func (f DateFormatter) DateGroupLabel(date time.Time) string { + now := f.now().In(f.location) + date = date.In(f.location) + today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, f.location) + day := time.Date(date.Year(), date.Month(), date.Day(), 0, 0, 0, 0, f.location) + if day.Equal(today) { + return "Today" + } + if day.Equal(today.AddDate(0, 0, -1)) { + return "Yesterday" + } + return f.FormatDate(date, date.Year() != now.Year()) +} diff --git a/internal/output/date_test.go b/internal/output/date_test.go new file mode 100644 index 0000000..e3a296c --- /dev/null +++ b/internal/output/date_test.go @@ -0,0 +1,113 @@ +package output + +import ( + "testing" + "time" +) + +func TestDateFormatterUsesInjectedLocation(t *testing.T) { + istanbul := mustLocation(t, "Europe/Istanbul") + now := time.Date(2026, time.January, 15, 12, 0, 0, 0, time.UTC) + formatter := NewDateFormatter(func() time.Time { return now }, istanbul) + date := time.Date(2025, time.December, 31, 22, 5, 0, 0, time.UTC) + + if got := formatter.FormatDate(date, false); got != "1 Jan" { + t.Fatalf("FormatDate = %q", got) + } + if got := formatter.FormatDate(date, true); got != "1 Jan 2026" { + t.Fatalf("FormatDate with year = %q", got) + } + if got := formatter.FormatDateLong(date); got != "Thursday, 1 January 2026" { + t.Fatalf("FormatDateLong = %q", got) + } + if got := formatter.FormatTime(date); got != "01:05" { + t.Fatalf("FormatTime = %q", got) + } +} + +func TestRelativeTimeThresholdsAndRounding(t *testing.T) { + now := time.Date(2026, time.July, 16, 12, 0, 0, 0, time.UTC) + formatter := NewDateFormatter(func() time.Time { return now }, time.UTC) + tests := []struct { + name string + diff time.Duration + want string + }{ + {"past under minute", -59 * time.Second, "just now"}, + {"future under minute", 59 * time.Second, "just now"}, + {"minute boundary", -time.Minute, "1 minute ago"}, + {"javascript negative half rounding", -90 * time.Second, "1 minute ago"}, + {"positive half rounding", 90 * time.Second, "in 2 minutes"}, + {"hour boundary", -time.Hour, "1 hour ago"}, + {"day boundary", -24 * time.Hour, "yesterday"}, + {"future day", 24 * time.Hour, "tomorrow"}, + {"week boundary", -7 * 24 * time.Hour, "last week"}, + {"future week", 7 * 24 * time.Hour, "next week"}, + {"month boundary", -30 * 24 * time.Hour, "last month"}, + {"future month", 30 * 24 * time.Hour, "next month"}, + {"year boundary", -365 * 24 * time.Hour, "last year"}, + {"future year", 365 * 24 * time.Hour, "next year"}, + {"plural years", -730 * 24 * time.Hour, "2 years ago"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := formatter.FormatRelativeTime(now.Add(test.diff)); got != test.want { + t.Fatalf("FormatRelativeTime(%s) = %q, want %q", test.diff, got, test.want) + } + }) + } +} + +func TestDateGroupLabelUsesCalendarDaysAcrossDST(t *testing.T) { + newYork := mustLocation(t, "America/New_York") + // The previous local calendar day is only 23 hours away across spring-forward. + now := time.Date(2026, time.March, 9, 0, 30, 0, 0, newYork) + formatter := NewDateFormatter(func() time.Time { return now }, newYork) + tests := []struct { + date time.Time + want string + }{ + {time.Date(2026, time.March, 9, 23, 0, 0, 0, time.UTC), "Today"}, + {time.Date(2026, time.March, 8, 0, 30, 0, 0, newYork), "Yesterday"}, + {time.Date(2026, time.March, 7, 23, 0, 0, 0, newYork), "7 Mar"}, + {time.Date(2025, time.December, 31, 23, 0, 0, 0, newYork), "31 Dec 2025"}, + } + for _, test := range tests { + if got := formatter.DateGroupLabel(test.date); got != test.want { + t.Errorf("DateGroupLabel(%s) = %q, want %q", test.date, got, test.want) + } + } +} + +func TestRelativeTimeDoesNotSaturateBeyondDurationRange(t *testing.T) { + now := time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC) + formatter := NewDateFormatter(func() time.Time { return now }, time.UTC) + if got := formatter.FormatRelativeTime(time.Date(2400, time.January, 1, 0, 0, 0, 0, time.UTC)); got != "in 374 years" { + t.Fatalf("FormatRelativeTime = %q", got) + } +} + +func TestNewDateFormatterRejectsMissingDependencies(t *testing.T) { + assertPanic := func(name string, call func()) { + t.Helper() + t.Run(name, func(t *testing.T) { + defer func() { + if recover() == nil { + t.Fatal("expected panic") + } + }() + call() + }) + } + assertPanic("clock", func() { NewDateFormatter(nil, time.UTC) }) + assertPanic("location", func() { NewDateFormatter(time.Now, nil) }) +} + +func mustLocation(t *testing.T, name string) *time.Location { + t.Helper() + location, err := time.LoadLocation(name) + if err != nil { + t.Fatal(err) + } + return location +} diff --git a/internal/output/model.go b/internal/output/model.go new file mode 100644 index 0000000..e66b4ad --- /dev/null +++ b/internal/output/model.go @@ -0,0 +1,113 @@ +package output + +import ( + "time" + + "github.com/ardasevinc/mattermost-cli/internal/presentation" +) + +// Redaction is presentation-owned because its Position is a UTF-16 code-unit +// offset into the final sanitized text. Output models must not recompute it. +type Redaction = presentation.Redaction + +type Message struct { + ID string `json:"id"` + Permalink string `json:"permalink"` + User string `json:"user"` + UserID string `json:"userId"` + Text string `json:"text"` + Timestamp time.Time `json:"timestamp"` + UpdatedAt time.Time `json:"updatedAt"` + EditedAt *time.Time `json:"editedAt,omitempty"` + DeletedAt *time.Time `json:"deletedAt,omitempty"` + IsDeleted bool `json:"isDeleted"` + PostType string `json:"postType"` + IsSystem bool `json:"isSystem"` + IsPinned bool `json:"isPinned"` + Files []string `json:"files"` + FileDetails []File `json:"fileDetails"` + Attachments []Attachment `json:"attachments"` + Reactions []Reaction `json:"reactions"` + RootID string `json:"rootId,omitempty"` + ReplyCount *int `json:"replyCount,omitempty"` + Replies []Message `json:"replies,omitempty"` + + // Canonical identities are unsanitized internal values. They drive + // ordering and grouping but are never serialized. + CanonicalID string `json:"-"` + CanonicalRootID string `json:"-"` +} + +type File struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + MIME string `json:"mime,omitempty"` + Size *int64 `json:"size,omitempty"` + Extension string `json:"extension,omitempty"` +} + +type AttachmentField struct { + Title string `json:"title,omitempty"` + Value string `json:"value,omitempty"` + Short *bool `json:"short,omitempty"` +} + +type Attachment struct { + Fallback string `json:"fallback,omitempty"` + Pretext string `json:"pretext,omitempty"` + Title string `json:"title,omitempty"` + TitleLink string `json:"titleLink,omitempty"` + Text string `json:"text,omitempty"` + Fields []AttachmentField `json:"fields,omitempty"` + Footer string `json:"footer,omitempty"` + FooterIcon string `json:"footerIcon,omitempty"` + AuthorName string `json:"authorName,omitempty"` + AuthorLink string `json:"authorLink,omitempty"` + AuthorIcon string `json:"authorIcon,omitempty"` + Color string `json:"color,omitempty"` + ImageURL string `json:"imageUrl,omitempty"` + ThumbURL string `json:"thumbUrl,omitempty"` + Timestamp string `json:"timestamp,omitempty"` +} + +type ReactionActor struct { + ID string `json:"id"` + Username string `json:"username,omitempty"` +} + +type Reaction struct { + Emoji string `json:"emoji"` + Count int `json:"count"` + Actors []ReactionActor `json:"actors"` +} + +type Channel struct { + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + DisplayName string `json:"displayName,omitempty"` + MetadataStatus string `json:"metadataStatus"` +} + +type Retrieval struct { + Selection Selection `json:"selection"` + VisibleThreads VisibleThreads `json:"visibleThreads"` + VisiblePostCount int `json:"visiblePostCount"` + DeletedPostsIncluded bool `json:"deletedPostsIncluded"` +} + +type Selection struct { + Source string `json:"source"` + SelectedCount int `json:"selectedCount"` + RequestedLimit *int `json:"requestedLimit"` + Since *string `json:"since"` + QueryTruncated *bool `json:"queryTruncated"` + InputCursor *string `json:"inputCursor"` + NextCursor *string `json:"nextCursor"` +} + +type VisibleThreads struct { + Status string `json:"status"` + HydratedRootCount int `json:"hydratedRootCount"` + FailedRootIDs []string `json:"failedRootIds"` +} diff --git a/internal/output/model_test.go b/internal/output/model_test.go new file mode 100644 index 0000000..e528ddc --- /dev/null +++ b/internal/output/model_test.go @@ -0,0 +1,35 @@ +package output + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +func TestMessageJSONUsesSemanticFieldsAndHidesCanonicalIdentity(t *testing.T) { + encoded, err := json.Marshal(Message{ + ID: "visible", CanonicalID: "raw", CanonicalRootID: "raw-root", + Timestamp: time.Date(2026, time.July, 16, 12, 0, 0, 0, time.UTC), + UpdatedAt: time.Date(2026, time.July, 16, 12, 1, 0, 0, time.UTC), + Files: []string{}, FileDetails: []File{}, Attachments: []Attachment{}, Reactions: []Reaction{}, + }) + if err != nil { + t.Fatal(err) + } + got := string(encoded) + if strings.Contains(got, "raw") || !strings.Contains(got, `"updatedAt"`) || !strings.Contains(got, `"fileDetails":[]`) { + t.Fatalf("JSON = %s", got) + } +} + +func TestOutputRedactionReusesPresentationUTF16Contract(t *testing.T) { + redaction := Redaction{Type: "api_key", Masked: "[REDACTED]", Position: 2, Field: "messages[0].text"} + encoded, err := json.Marshal(redaction) + if err != nil { + t.Fatal(err) + } + if got, want := string(encoded), `{"type":"api_key","masked":"[REDACTED]","position":2,"field":"messages[0].text"}`; got != want { + t.Fatalf("JSON = %s, want %s", got, want) + } +} diff --git a/internal/output/threading.go b/internal/output/threading.go new file mode 100644 index 0000000..0bb31fd --- /dev/null +++ b/internal/output/threading.go @@ -0,0 +1,62 @@ +package output + +import "slices" + +func GroupIntoThreads(messages []Message) []Message { + sorted := slices.Clone(messages) + slices.SortStableFunc(sorted, compareMessages) + + roots := make([]Message, 0, len(sorted)) + rootIndexes := make(map[string]int, len(sorted)) + orphans := make([]Message, 0) + for _, message := range sorted { + if canonicalRootID(message) != "" { + continue + } + root := message + root.Replies = []Message{} + rootIndexes[canonicalID(root)] = len(roots) + roots = append(roots, root) + } + for _, message := range sorted { + rootID := canonicalRootID(message) + if rootID == "" { + continue + } + if index, ok := rootIndexes[rootID]; ok { + roots[index].Replies = append(roots[index].Replies, message) + } else { + orphans = append(orphans, message) + } + } + result := append(roots, orphans...) + slices.SortStableFunc(result, compareMessages) + return result +} + +func compareMessages(a, b Message) int { + if order := a.Timestamp.Compare(b.Timestamp); order != 0 { + return order + } + if canonicalID(a) < canonicalID(b) { + return -1 + } + if canonicalID(a) > canonicalID(b) { + return 1 + } + return 0 +} + +func canonicalID(message Message) string { + if message.CanonicalID != "" { + return message.CanonicalID + } + return message.ID +} + +func canonicalRootID(message Message) string { + if message.CanonicalRootID != "" { + return message.CanonicalRootID + } + return message.RootID +} diff --git a/internal/output/threading_test.go b/internal/output/threading_test.go new file mode 100644 index 0000000..bb45fc7 --- /dev/null +++ b/internal/output/threading_test.go @@ -0,0 +1,82 @@ +package output + +import ( + "reflect" + "testing" + "time" +) + +func TestGroupIntoThreadsGroupsSortsAndDoesNotMutateInput(t *testing.T) { + base := time.Date(2026, time.February, 21, 10, 0, 0, 0, time.UTC) + messages := []Message{ + {ID: "reply-2", RootID: "root", Timestamp: base.Add(3 * time.Minute)}, + {ID: "root", Timestamp: base, Replies: []Message{{ID: "stale"}}}, + {ID: "reply-1", RootID: "root", Timestamp: base.Add(time.Minute)}, + } + result := GroupIntoThreads(messages) + if got := messageIDs(result); !reflect.DeepEqual(got, []string{"root"}) { + t.Fatalf("roots = %v", got) + } + if got := messageIDs(result[0].Replies); !reflect.DeepEqual(got, []string{"reply-1", "reply-2"}) { + t.Fatalf("replies = %v", got) + } + if got := messageIDs(messages); !reflect.DeepEqual(got, []string{"reply-2", "root", "reply-1"}) || messages[1].Replies[0].ID != "stale" { + t.Fatalf("input mutated: %#v", messages) + } +} + +func TestGroupIntoThreadsKeepsOrphansInGlobalTimestampThenIDOrder(t *testing.T) { + stamp := time.Date(2026, time.February, 21, 10, 0, 0, 0, time.UTC) + messages := []Message{ + {ID: "root-z", Timestamp: stamp}, + {ID: "orphan-b", RootID: "missing", Timestamp: stamp}, + {ID: "orphan-a", RootID: "missing", Timestamp: stamp}, + {ID: "root-early", Timestamp: stamp.Add(-time.Minute)}, + } + if got := messageIDs(GroupIntoThreads(messages)); !reflect.DeepEqual(got, []string{"root-early", "orphan-a", "orphan-b", "root-z"}) { + t.Fatalf("order = %v", got) + } +} + +func TestGroupIntoThreadsUsesCanonicalIdentityForGroupingAndTies(t *testing.T) { + stamp := time.Date(2026, time.February, 21, 10, 0, 0, 0, time.UTC) + messages := []Message{ + {ID: "visible-first", CanonicalID: "z", Timestamp: stamp}, + {ID: "visible-last", CanonicalID: "a", Timestamp: stamp}, + {ID: "masked-reply", RootID: "masked-root", CanonicalID: "r", CanonicalRootID: "z", Timestamp: stamp.Add(time.Second)}, + } + result := GroupIntoThreads(messages) + if got := messageIDs(result); !reflect.DeepEqual(got, []string{"visible-last", "visible-first"}) { + t.Fatalf("canonical root order = %v", got) + } + if got := messageIDs(result[1].Replies); !reflect.DeepEqual(got, []string{"masked-reply"}) { + t.Fatalf("canonical grouping = %v", got) + } +} + +func TestGroupIntoThreadsSortsEqualTimestampRepliesByCanonicalID(t *testing.T) { + stamp := time.Date(2026, time.February, 21, 10, 0, 0, 0, time.UTC) + result := GroupIntoThreads([]Message{ + {ID: "root", Timestamp: stamp}, + {ID: "visible-a", CanonicalID: "b", RootID: "root", Timestamp: stamp.Add(time.Second)}, + {ID: "visible-z", CanonicalID: "a", RootID: "root", Timestamp: stamp.Add(time.Second)}, + }) + if got := messageIDs(result[0].Replies); !reflect.DeepEqual(got, []string{"visible-z", "visible-a"}) { + t.Fatalf("reply order = %v", got) + } +} + +func TestGroupIntoThreadsEmptyIsNonNil(t *testing.T) { + result := GroupIntoThreads(nil) + if result == nil || len(result) != 0 { + t.Fatalf("result = %#v", result) + } +} + +func messageIDs(messages []Message) []string { + ids := make([]string, len(messages)) + for index := range messages { + ids[index] = messages[index].ID + } + return ids +} From 170435a7bead582b5e32f7dbd1fac5c2baa7e918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 10:35:37 +0300 Subject: [PATCH 022/119] feat: port bounded mention retrieval --- go.mod | 6 +- internal/retrieval/mentions.go | 215 ++++++++++++++++++++++++++++ internal/retrieval/mentions_test.go | 145 +++++++++++++++++++ 3 files changed, 364 insertions(+), 2 deletions(-) create mode 100644 internal/retrieval/mentions.go create mode 100644 internal/retrieval/mentions_test.go diff --git a/go.mod b/go.mod index 9002b17..8a522b5 100644 --- a/go.mod +++ b/go.mod @@ -10,10 +10,12 @@ require github.com/pelletier/go-toml/v2 v2.4.3 require golang.org/x/net v0.57.0 -require golang.org/x/sys v0.47.0 +require ( + golang.org/x/sys v0.47.0 + golang.org/x/text v0.40.0 +) require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/spf13/pflag v1.0.9 // indirect - golang.org/x/text v0.40.0 // indirect ) diff --git a/internal/retrieval/mentions.go b/internal/retrieval/mentions.go new file mode 100644 index 0000000..3b0db6e --- /dev/null +++ b/internal/retrieval/mentions.go @@ -0,0 +1,215 @@ +package retrieval + +import ( + "context" + "errors" + "strings" + "time" + "unicode" + "unicode/utf16" + "unicode/utf8" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "golang.org/x/text/cases" + "golang.org/x/text/language" +) + +const ( + MaxMentionAliases = 64 + MaxMentionTermBytes = 256 +) + +var ErrInvalidMentionsRequest = errors.New("invalid mentions request") + +type MentionsOptions struct { + Username string + Aliases []string + Channel *string + Since *int64 + Limit int +} + +type MentionsResult struct { + Posts []mattermost.Post + Completeness Completeness +} + +// Mentions performs one bounded Search per distinct mention term, then applies +// the exact local predicate that the remote search endpoint cannot express. +func Mentions(ctx context.Context, source searchPageSource, teamID string, options MentionsOptions) (MentionsResult, error) { + if source == nil { + return MentionsResult{}, ErrInvalidMentionsRequest + } + return retrieveMentions(ctx, func(ctx context.Context, teamID, terms string, options SearchOptions) (SearchResult, error) { + return Search(ctx, source, teamID, terms, options) + }, teamID, options) +} + +type mentionSearchFunc func(context.Context, string, string, SearchOptions) (SearchResult, error) + +func retrieveMentions(ctx context.Context, search mentionSearchFunc, teamID string, options MentionsOptions) (MentionsResult, error) { + terms, channel, err := mentionTerms(options) + if err != nil || search == nil || strings.TrimSpace(teamID) == "" { + return MentionsResult{}, ErrInvalidMentionsRequest + } + byID := make(map[string]mattermost.Post) + states := make([]Completeness, 0, len(terms)) + for _, term := range terms { + if err := ctx.Err(); err != nil { + return MentionsResult{}, err + } + query := searchTermWithScope(term, channel, options.Since) + result, err := search(ctx, teamID, query, SearchOptions{ + Limit: options.Limit, + Accept: func(post mattermost.Post) bool { return isExactMention(post, term, options.Since) }, + }) + if err != nil { + return MentionsResult{}, err + } + states = append(states, result.Completeness) + for _, post := range result.Posts { + byID[post.ID] = post + } + } + return MentionsResult{ + Posts: mostRecent(byID, options.Limit), + Completeness: mergeMentionCompleteness(states, len(byID), options.Limit), + }, nil +} + +func mentionTerms(options MentionsOptions) ([]string, string, error) { + username := strings.TrimSpace(options.Username) + channel := "" + if options.Channel != nil { + channel = strings.TrimPrefix(strings.TrimSpace(*options.Channel), "#") + if channel == "" { + return nil, "", ErrInvalidMentionsRequest + } + } + if options.Limit <= 0 || int64(options.Limit) > maxSafeInteger || !safeSearchAtom(username) || + len(options.Aliases) > MaxMentionAliases || (options.Since != nil && (*options.Since < 0 || *options.Since > maxDateMilliseconds)) || + (channel != "" && !safeSearchAtom(channel)) { + return nil, "", ErrInvalidMentionsRequest + } + terms := make([]string, 0, len(options.Aliases)+1) + seen := make(map[string]struct{}, len(options.Aliases)+1) + add := func(term string) { + if _, ok := seen[term]; ok { + return + } + seen[term] = struct{}{} + terms = append(terms, term) + } + add("@" + username) + for _, raw := range options.Aliases { + alias := strings.TrimSpace(raw) + if alias == "" { + continue + } + if len(alias) > MaxMentionTermBytes || !utf8.ValidString(alias) || strings.ContainsAny(alias, "\"\\\r\n\x00") { + return nil, "", ErrInvalidMentionsRequest + } + for _, r := range alias { + if unicode.IsControl(r) { + return nil, "", ErrInvalidMentionsRequest + } + } + add("\"" + alias + "\"") + } + return terms, channel, nil +} + +func safeSearchAtom(value string) bool { + if value == "" || len(value) > MaxMentionTermBytes { + return false + } + for i := range len(value) { + c := value[i] + if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '.' || c == '_' || c == '-') { + return false + } + } + return true +} + +func searchTermWithScope(term, channel string, since *int64) string { + parts := []string{term} + if since != nil { + date := time.UnixMilli(*since).UTC().Add(-24 * time.Hour).Format("2006-01-02") + parts = append(parts, "after:"+date) + } + if channel != "" { + parts = append(parts, "in:"+channel) + } + return strings.Join(parts, " ") +} + +func isExactMention(post mattermost.Post, term string, since *int64) bool { + if post.DeleteAt != 0 || (since != nil && post.CreateAt < *since) { + return false + } + literal := term + if !strings.HasPrefix(term, "@") { + literal = strings.TrimSuffix(strings.TrimPrefix(term, "\""), "\"") + } + alias := !strings.HasPrefix(literal, "@") + return hasLiteralMention(post.Message, literal, alias) +} + +func hasLiteralMention(message, literal string, alias bool) bool { + lower := cases.Lower(language.Und) + messageUnits := utf16.Encode([]rune(lower.String(message))) + literalUnits := utf16.Encode([]rune(lower.String(literal))) + if len(literalUnits) == 0 { + return false + } + for start := 0; start+len(literalUnits) <= len(messageUnits); start++ { + matched := true + for index := range literalUnits { + if messageUnits[start+index] != literalUnits[index] { + matched = false + break + } + } + if !matched { + continue + } + end := start + len(literalUnits) + if !mentionBoundaryUnit(messageUnits, start-1, alias) && !mentionBoundaryUnit(messageUnits, end, alias) { + return true + } + } + return false +} + +func mentionBoundaryUnit(value []uint16, index int, alias bool) bool { + if index < 0 || index >= len(value) { + return false + } + unit := value[index] + if alias { + if unit >= 0xD800 && unit <= 0xDFFF { + return false + } + r := rune(unit) + return unicode.IsLetter(r) || unicode.IsMark(r) || unicode.IsNumber(r) + } + return unit <= unicode.MaxASCII && ((unit >= 'a' && unit <= 'z') || (unit >= '0' && unit <= '9') || strings.ContainsRune("._-", rune(unit))) +} + +func mergeMentionCompleteness(states []Completeness, candidates, limit int) Completeness { + if candidates > limit { + return CompletenessTruncated + } + for _, state := range states { + if state == CompletenessTruncated { + return CompletenessTruncated + } + } + for _, state := range states { + if state != CompletenessComplete { + return CompletenessUnknown + } + } + return CompletenessComplete +} diff --git a/internal/retrieval/mentions_test.go b/internal/retrieval/mentions_test.go new file mode 100644 index 0000000..3fa2635 --- /dev/null +++ b/internal/retrieval/mentions_test.go @@ -0,0 +1,145 @@ +package retrieval + +import ( + "context" + "errors" + "fmt" + "reflect" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" +) + +func TestMentionTermsTrimQuoteDedupeAndScope(t *testing.T) { + since := int64(1_768_404_600_000) // 2026-01-14 15:30:00 UTC + terms, channel, err := mentionTerms(MentionsOptions{ + Username: " arda ", Aliases: []string{" Arda Sevinc ", "", "Arda Sevinc", "arda"}, + Channel: stringPointer(" #general "), Since: &since, Limit: 20, + }) + if err != nil || channel != "general" || !reflect.DeepEqual(terms, []string{"@arda", `"Arda Sevinc"`, `"arda"`}) { + t.Fatalf("terms=%q channel=%q err=%v", terms, channel, err) + } + got := searchTermWithScope(terms[1], channel, &since) + if got != `"Arda Sevinc" after:2026-01-13 in:general` { + t.Fatalf("query=%q", got) + } +} + +func TestExactMentionLiteralUnicodeAndPunctuationBoundaries(t *testing.T) { + tests := []struct { + name, message, term string + want bool + }{ + {"alias case", "hello ARDA SEVINC", `"Arda Sevinc"`, true}, + {"alias punctuation", "(Arda).com", `"Arda"`, true}, + {"alias unicode letter before", "İArda", `"Arda"`, false}, + {"alias unicode letter after", "Arda東京", `"Arda"`, false}, + {"alias number", "Arda2", `"Arda"`, false}, + {"alias regex metacharacters literal", "ping (a+b)[x] now", `"(a+b)[x]"`, true}, + {"alias regex metacharacters absent", "ping aaabx now", `"a+b[x]"`, false}, + {"username punctuation", "(@arda)!", "@arda", true}, + {"username dot", "@arda.com", "@arda", false}, + {"username underscore", "@arda_more", "@arda", false}, + {"username hyphen", "@arda-name", "@arda", false}, + {"username prefix", "x@arda", "@arda", false}, + {"username unicode adjacent allowed", "ğ@ardağ", "@arda", true}, + {"quoted username alias uses username boundary", "@ops.com", `"@ops"`, false}, + {"contextual sigma", "ος", `"ΟΣ"`, true}, + {"non-final sigma differs", "οσ", `"ΟΣ"`, false}, + {"full lowercase expansion", "İ", `"i"`, false}, + {"utf16 supplementary boundary parity", "𐐀Arda", `"Arda"`, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + post := mattermost.Post{ID: "p", Message: tt.message, CreateAt: 10} + if got := isExactMention(post, tt.term, nil); got != tt.want { + t.Fatalf("isExactMention(%q, %q)=%v want %v", tt.message, tt.term, got, tt.want) + } + }) + } +} + +func TestExactMentionFiltersDeletedAndExactSince(t *testing.T) { + since := int64(1000) + for _, post := range []mattermost.Post{ + {Message: "@arda", CreateAt: 999}, + {Message: "@arda", CreateAt: 1000, DeleteAt: 1}, + } { + if isExactMention(post, "@arda", &since) { + t.Fatalf("accepted %#v", post) + } + } + if !isExactMention(mattermost.Post{Message: "@arda", CreateAt: 1000}, "@arda", &since) { + t.Fatal("rejected exact since boundary") + } +} + +func TestRetrieveMentionsMergesGloballyAndCompleteness(t *testing.T) { + var queries []string + result, err := retrieveMentions(context.Background(), func(_ context.Context, _, query string, options SearchOptions) (SearchResult, error) { + queries = append(queries, query) + candidates := []mattermost.Post{ + {ID: "shared", Message: "@arda Arda", CreateAt: 2}, + {ID: query, Message: query, CreateAt: 1}, + } + accepted := make([]mattermost.Post, 0, len(candidates)) + for _, post := range candidates { + if options.Accept(post) { + accepted = append(accepted, post) + } + } + state := CompletenessComplete + if query == `"Arda"` { + state = CompletenessUnknown + } + return SearchResult{Posts: accepted, Completeness: state}, nil + }, "team", MentionsOptions{Username: "arda", Aliases: []string{"Arda"}, Limit: 2}) + if err != nil || fmt.Sprint(queries) != `[@arda "Arda"]` || fmt.Sprint(ids(result.Posts)) != `[shared "Arda"]` || result.Completeness != CompletenessTruncated { + t.Fatalf("queries=%q result=%#v err=%v", queries, result, err) + } + + if got := mergeMentionCompleteness([]Completeness{CompletenessComplete, CompletenessUnknown}, 1, 2); got != CompletenessUnknown { + t.Fatalf("unknown merge=%v", got) + } + if got := mergeMentionCompleteness([]Completeness{CompletenessUnknown, CompletenessTruncated}, 1, 2); got != CompletenessTruncated { + t.Fatalf("truncated merge=%v", got) + } +} + +func TestRetrieveMentionsPropagatesErrorsAndCancellation(t *testing.T) { + want := errors.New("search failed") + _, err := retrieveMentions(context.Background(), func(context.Context, string, string, SearchOptions) (SearchResult, error) { + return SearchResult{}, want + }, "team", MentionsOptions{Username: "arda", Limit: 1}) + if !errors.Is(err, want) { + t.Fatalf("error=%v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = retrieveMentions(ctx, func(context.Context, string, string, SearchOptions) (SearchResult, error) { + t.Fatal("search called after cancellation") + return SearchResult{}, nil + }, "team", MentionsOptions{Username: "arda", Limit: 1}) + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancel error=%v", err) + } +} + +func TestMentionValidationBoundsWithoutReflectingInput(t *testing.T) { + tests := []MentionsOptions{ + {Username: "bad user", Limit: 1}, + {Username: "arda", Channel: stringPointer("general after:2000-01-01"), Limit: 1}, + {Username: "arda", Channel: stringPointer("#"), Limit: 1}, + {Username: "arda", Aliases: []string{`hostile" after:2000-01-01`}, Limit: 1}, + {Username: "arda", Aliases: []string{string(make([]byte, MaxMentionTermBytes+1))}, Limit: 1}, + } + for _, options := range tests { + _, _, err := mentionTerms(options) + if !errors.Is(err, ErrInvalidMentionsRequest) || err.Error() != "invalid mentions request" { + t.Fatalf("error=%v", err) + } + } +} + +func stringPointer(value string) *string { return &value } From 8237b2332558459be5a48b15c5e08f877f5b6a9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 10:47:23 +0300 Subject: [PATCH 023/119] feat: port terminal message rendering --- internal/output/pretty.go | 354 +++++++++++++++++++++++++++++++++ internal/output/pretty_test.go | 119 +++++++++++ 2 files changed, 473 insertions(+) create mode 100644 internal/output/pretty.go create mode 100644 internal/output/pretty_test.go diff --git a/internal/output/pretty.go b/internal/output/pretty.go new file mode 100644 index 0000000..db9365a --- /dev/null +++ b/internal/output/pretty.go @@ -0,0 +1,354 @@ +package output + +import ( + "fmt" + "io" + "slices" + "strings" + "time" + "unicode/utf16" +) + +const ( + ansiReset = "\x1b[0m" + ansiBold = "\x1b[1m" + ansiDim = "\x1b[2m" + ansiCyan = "\x1b[36m" +) + +type PrettyOptions struct { + Color bool + Relative bool +} + +// MessageOutput is one independently headed channel section. +type MessageOutput struct { + Channel Channel + Messages []Message + Redactions []Redaction + Retrieval Retrieval +} + +// FormatPretty renders already-presented values. It deliberately performs no +// sanitization or redaction of its own. +func FormatPretty(outputs []MessageOutput, dates DateFormatter, options PrettyOptions) string { + sections := make([]string, 0, len(outputs)) + for _, output := range outputs { + if options.Color { + sections = append(sections, formatColorSection(output, dates, options.Relative)) + } else { + sections = append(sections, formatPlainSection(output, dates, options.Relative)) + } + } + separator := "\n" + strings.Repeat("=", 60) + "\n\n" + if options.Color { + separator = "\n" + dim(strings.Repeat("─", 60)) + "\n\n" + } + return strings.Join(sections, separator) +} + +// WritePretty renders once and writes once. A short write is returned as +// io.ErrShortWrite and is never retried, since the writer's state is unknown. +func WritePretty(w io.Writer, outputs []MessageOutput, dates DateFormatter, options PrettyOptions) (int, error) { + formatted := FormatPretty(outputs, dates, options) + n, err := io.WriteString(w, formatted) + if err == nil && n != len(formatted) { + err = io.ErrShortWrite + } + return n, err +} + +func formatColorSection(output MessageOutput, dates DateFormatter, relative bool) string { + lines := []string{bold(colorHeader(output.Channel)), ""} + appendMessagesByDate(&lines, output.Messages, dates, func(message Message) { + lines = append(lines, formatColorMessage(message, dates, relative, " ")) + }, true) + if len(output.Redactions) > 0 { + lines = append(lines, "", dim(fmt.Sprintf(" ⚠ %d secret(s) redacted", len(output.Redactions)))) + } + appendCoverage(&lines, output) + return strings.Join(lines, "\n") +} + +func formatPlainSection(output MessageOutput, dates DateFormatter, relative bool) string { + lines := []string{plainHeader(output.Channel), strings.Repeat("─", 40)} + appendMessagesByDate(&lines, output.Messages, dates, func(message Message) { + appendPlainMessage(&lines, message, dates, relative, " ") + }, false) + if len(output.Redactions) > 0 { + lines = append(lines, fmt.Sprintf(" [%d secret(s) redacted]", len(output.Redactions))) + } + appendCoverage(&lines, output) + return strings.Join(lines, "\n") +} + +func appendMessagesByDate(lines *[]string, messages []Message, dates DateFormatter, appendMessage func(Message), color bool) { + sorted := slices.Clone(messages) + slices.SortStableFunc(sorted, func(a, b Message) int { return a.Timestamp.Compare(b.Timestamp) }) + last := "" + for _, message := range sorted { + label := dates.DateGroupLabel(message.Timestamp) + if label != last { + if color { + *lines = append(*lines, dim(" ── "+label+" ──"), "") + } else { + *lines = append(*lines, " -- "+label+" --", "") + } + last = label + } + appendMessage(message) + } +} + +func formatColorMessage(message Message, dates DateFormatter, relative bool, indent string) string { + timeText := dates.FormatTime(message.Timestamp) + if relative { + timeText = dates.FormatRelativeTime(message.Timestamp) + } + markers := messageMarkers(message) + header := indent + dim(timeText) + " " + bold(userColor(message.User)) + if markers != "" { + header += " " + dim(markers) + } + header += " " + dim(compactPostRef(message)) + textIndent := indent + " " + lines := []string{header, indentLines(message.Text, textIndent), dim(textIndent + formatStateTimes(message))} + if len(message.FileDetails) > 0 { + lines = append(lines, dim(textIndent+"📎 "+formatFiles(message))) + } + appendRichContent(&lines, message, textIndent, dim) + lines = append(lines, "") + for _, reply := range message.Replies { + lines = append(lines, formatColorMessage(reply, dates, relative, indent+" ↳ ")) + } + return strings.Join(lines, "\n") +} + +func appendPlainMessage(lines *[]string, message Message, dates DateFormatter, relative bool, indent string) { + timeText := dates.FormatTime(message.Timestamp) + if relative { + timeText = dates.FormatRelativeTime(message.Timestamp) + } + markers := messageMarkers(message) + header := fmt.Sprintf("%s[%s] %s", indent, timeText, message.User) + if markers != "" { + header += " " + markers + } + header += " " + compactPostRef(message) + textIndent := indent + " " + *lines = append(*lines, header, indentLines(message.Text, textIndent), textIndent+formatStateTimes(message)) + if len(message.FileDetails) > 0 { + *lines = append(*lines, textIndent+"Files: "+formatFiles(message)) + } + appendRichContent(lines, message, textIndent, func(value string) string { return value }) + *lines = append(*lines, "") + for _, reply := range message.Replies { + appendPlainMessage(lines, reply, dates, relative, indent+" > ") + } +} + +func colorHeader(channel Channel) string { + switch channel.Type { + case "unknown": + return "⚠ Unknown channel (" + cyan(channel.ID) + ")" + case "dm": + return "💬 DMs with " + cyan(channel.Name) + case "group": + return "💬 Group DM: " + cyan(channel.Name) + default: + display := "" + if channel.DisplayName != "" { + display = " (" + channel.DisplayName + ")" + } + return "📢 " + cyan("#"+channel.Name) + display + } +} + +func plainHeader(channel Channel) string { + switch channel.Type { + case "unknown": + return "Unknown channel (" + channel.ID + ")" + case "dm": + return "DMs with " + channel.Name + case "group": + return "Group DM: " + channel.Name + default: + display := "" + if channel.DisplayName != "" { + display = " (" + channel.DisplayName + ")" + } + return "#" + channel.Name + display + } +} + +func compactPostRef(message Message) string { + id := message.ID + if len(id) > 8 { + id = id[:8] + } + return id + " " + message.Permalink +} + +func messageMarkers(message Message) string { + markers := make([]string, 0, 3) + if message.IsDeleted { + markers = append(markers, "[deleted]") + } else if message.EditedAt != nil { + markers = append(markers, "[edited]") + } + if message.IsSystem { + if message.PostType != "" { + markers = append(markers, "[system:"+message.PostType+"]") + } else { + markers = append(markers, "[system]") + } + } + if message.IsPinned { + markers = append(markers, "[pinned]") + } + return strings.Join(markers, " ") +} + +func formatFiles(message Message) string { + files := make([]string, 0, len(message.FileDetails)) + for _, file := range message.FileDetails { + label := file.ID + if file.Name != "" { + label = file.Name + " (" + file.ID + ")" + } + details := make([]string, 0, 3) + if file.MIME != "" { + details = append(details, file.MIME) + } + if file.Extension != "" { + details = append(details, file.Extension) + } + if file.Size != nil { + details = append(details, fmt.Sprintf("%d B", *file.Size)) + } + if len(details) > 0 { + label += ", " + strings.Join(details, ", ") + } + files = append(files, label) + } + return strings.Join(files, ", ") +} + +func formatStateTimes(message Message) string { + values := []string{"Updated " + isoTime(message.UpdatedAt)} + if message.EditedAt != nil { + values = append(values, "edited "+isoTime(*message.EditedAt)) + } + if message.DeletedAt != nil { + values = append(values, "deleted "+isoTime(*message.DeletedAt)) + } + return strings.Join(values, "; ") +} + +func isoTime(value time.Time) string { + return value.UTC().Format("2006-01-02T15:04:05.000Z") +} + +func appendRichContent(lines *[]string, message Message, indent string, decorate func(string) string) { + for _, attachment := range message.Attachments { + if attachment.Pretext != "" { + *lines = append(*lines, decorate(indent+attachment.Pretext)) + } + if attachment.Title != "" { + *lines = append(*lines, decorate(indent+"Attachment: "+attachment.Title)) + } + if attachment.TitleLink != "" { + *lines = append(*lines, decorate(indent+" Link: "+attachment.TitleLink)) + } + if attachment.AuthorName != "" { + *lines = append(*lines, decorate(indent+" By: "+attachment.AuthorName)) + } + if attachment.Text != "" { + *lines = append(*lines, decorate(indent+" "+attachment.Text)) + } + for _, field := range attachment.Fields { + prefix := "" + if field.Title != "" { + prefix = field.Title + ": " + } + *lines = append(*lines, decorate(indent+" "+prefix+field.Value)) + } + if attachment.Footer != "" { + *lines = append(*lines, decorate(indent+" "+attachment.Footer)) + } + if attachment.Fallback != "" { + *lines = append(*lines, decorate(indent+" Fallback: "+attachment.Fallback)) + } + if attachment.Color != "" { + *lines = append(*lines, decorate(indent+" Color: "+attachment.Color)) + } + if attachment.Timestamp != "" { + *lines = append(*lines, decorate(indent+" Timestamp: "+attachment.Timestamp)) + } + for _, value := range []string{attachment.AuthorLink, attachment.AuthorIcon, attachment.FooterIcon, attachment.ImageURL, attachment.ThumbURL} { + if value != "" { + *lines = append(*lines, decorate(indent+" "+value)) + } + } + } + if len(message.Reactions) > 0 { + reactions := make([]string, 0, len(message.Reactions)) + for _, reaction := range message.Reactions { + names := make([]string, 0, len(reaction.Actors)) + for _, actor := range reaction.Actors { + if actor.Username != "" { + names = append(names, actor.Username) + } else { + names = append(names, actor.ID) + } + } + value := fmt.Sprintf(":%s: %d", reaction.Emoji, reaction.Count) + if len(names) > 0 { + value += " (" + strings.Join(names, ", ") + ")" + } + reactions = append(reactions, value) + } + *lines = append(*lines, decorate(indent+"Reactions: "+strings.Join(reactions, " "))) + } +} + +func appendCoverage(lines *[]string, output MessageOutput) { + state := "completeness unknown" + if output.Retrieval.Selection.QueryTruncated != nil { + if *output.Retrieval.Selection.QueryTruncated { + state = "truncated" + } else { + state = "complete" + } + } + *lines = append(*lines, fmt.Sprintf(" Coverage: %d selected, %d visible; query %s", output.Retrieval.Selection.SelectedCount, output.Retrieval.VisiblePostCount, state)) + if output.Retrieval.Selection.NextCursor != nil && *output.Retrieval.Selection.NextCursor != "" { + *lines = append(*lines, " Next cursor: "+*output.Retrieval.Selection.NextCursor) + } +} + +func indentLines(value, indent string) string { + return indent + strings.ReplaceAll(value, "\n", "\n"+indent) +} + +func bold(value string) string { return ansiBold + value + ansiReset } +func dim(value string) string { return ansiDim + value + ansiReset } +func cyan(value string) string { return ansiCyan + value + ansiReset } + +func userColor(value string) string { + colors := [...]string{ansiCyan, "\x1b[33m", "\x1b[32m", "\x1b[35m", "\x1b[34m"} + var hash int32 + for _, r := range value { + unit := uint16(r) + if r > 0xFFFF { + high, _ := utf16.EncodeRune(r) + unit = uint16(high) + } + hash = hash*31 + int32(unit) + } + index := int64(hash) + if index < 0 { + index = -index + } + return colors[index%int64(len(colors))] + value + ansiReset +} diff --git a/internal/output/pretty_test.go b/internal/output/pretty_test.go new file mode 100644 index 0000000..5667b4f --- /dev/null +++ b/internal/output/pretty_test.go @@ -0,0 +1,119 @@ +package output + +import ( + "errors" + "io" + "testing" + "time" +) + +var prettyNow = time.Date(2026, 2, 21, 12, 0, 0, 0, time.UTC) + +func prettyDates() DateFormatter { + return NewDateFormatter(func() time.Time { return prettyNow }, time.UTC) +} + +func TestFormatPrettyPlainGolden(t *testing.T) { + complete := false + next := "opaque_123" + size := int64(42) + edited := time.Date(2026, 2, 20, 10, 1, 0, 0, time.UTC) + deleted := time.Date(2026, 2, 20, 10, 2, 0, 0, time.UTC) + output := MessageOutput{ + Channel: Channel{ID: "ch", Type: "public", Name: "general\\n", DisplayName: "Gen\\u001b"}, + Messages: []Message{{ + ID: "123456789-hostile", Permalink: "https://mm.test/_redirect/pl/x", User: "a\\tb", Text: "one\ntwo\\u202e", Timestamp: time.Date(2026, 2, 20, 10, 0, 0, 0, time.UTC), UpdatedAt: time.Date(2026, 2, 20, 10, 0, 0, 0, time.UTC), + EditedAt: &edited, DeletedAt: &deleted, IsDeleted: true, IsSystem: true, PostType: "system_x", IsPinned: true, + FileDetails: []File{{ID: "f1", Name: "a.txt", MIME: "text/plain", Extension: "txt", Size: &size}}, + Attachments: []Attachment{{Pretext: "pre", Title: "title", TitleLink: "link", AuthorName: "author", Text: "body", Fields: []AttachmentField{{Title: "key", Value: "value"}, {Value: "bare"}}, Footer: "foot", Fallback: "fallback", Color: "red", Timestamp: "stamp", AuthorLink: "author-link", AuthorIcon: "author-icon", FooterIcon: "footer-icon", ImageURL: "image", ThumbURL: "thumb"}}, + Reactions: []Reaction{{Emoji: "eyes", Count: 2, Actors: []ReactionActor{{ID: "u1", Username: "bob"}, {ID: "u2"}}}}, + Replies: []Message{{ID: "r", Permalink: "reply-link", User: "eve", Text: "reply", Timestamp: time.Date(2026, 2, 20, 10, 3, 0, 0, time.UTC), UpdatedAt: time.Date(2026, 2, 20, 10, 3, 0, 0, time.UTC), Replies: []Message{{ID: "rr", Permalink: "deep-link", User: "zed", Text: "deep", Timestamp: time.Date(2026, 2, 20, 10, 4, 0, 0, time.UTC), UpdatedAt: time.Date(2026, 2, 20, 10, 4, 0, 0, time.UTC)}}}}, + }}, + Redactions: []Redaction{{Type: "token"}}, + Retrieval: Retrieval{Selection: Selection{SelectedCount: 1, QueryTruncated: &complete, NextCursor: &next}, VisiblePostCount: 3}, + } + want := "#general\\n (Gen\\u001b)\n" + + "────────────────────────────────────────\n" + + " -- Yesterday --\n\n" + + " [10:00] a\\tb [deleted] [system:system_x] [pinned] 12345678 https://mm.test/_redirect/pl/x\n" + + " one\n two\\u202e\n" + + " Updated 2026-02-20T10:00:00.000Z; edited 2026-02-20T10:01:00.000Z; deleted 2026-02-20T10:02:00.000Z\n" + + " Files: a.txt (f1), text/plain, txt, 42 B\n" + + " pre\n Attachment: title\n Link: link\n By: author\n body\n key: value\n bare\n foot\n Fallback: fallback\n Color: red\n Timestamp: stamp\n author-link\n author-icon\n footer-icon\n image\n thumb\n" + + " Reactions: :eyes: 2 (bob, u2)\n\n" + + " > [10:03] eve r reply-link\n > reply\n > Updated 2026-02-20T10:03:00.000Z\n\n" + + " > > [10:04] zed rr deep-link\n > > deep\n > > Updated 2026-02-20T10:04:00.000Z\n\n" + + " [1 secret(s) redacted]\n" + + " Coverage: 1 selected, 3 visible; query complete\n" + + " Next cursor: opaque_123" + if got := FormatPretty([]MessageOutput{output}, prettyDates(), PrettyOptions{}); got != want { + t.Fatalf("plain bytes mismatch\n--- got ---\n%q\n--- want ---\n%q", got, want) + } +} + +func TestFormatPrettyColorMultipleAndRelativeGolden(t *testing.T) { + truncated := true + message := Message{ID: "m", Permalink: "p", User: "alice", Text: "hi", Timestamp: prettyNow.Add(-2 * time.Hour), UpdatedAt: prettyNow} + outputs := []MessageOutput{ + {Channel: Channel{Type: "dm", Name: "@bob"}, Messages: []Message{message}, Retrieval: Retrieval{Selection: Selection{SelectedCount: 1, QueryTruncated: &truncated}, VisiblePostCount: 1}}, + {Channel: Channel{ID: "cid", Type: "unknown"}, Retrieval: Retrieval{}}, + } + want := "\x1b[1m💬 DMs with \x1b[36m@bob\x1b[0m\x1b[0m\n\n" + + "\x1b[2m ── Today ──\x1b[0m\n\n" + + " \x1b[2m2 hours ago\x1b[0m \x1b[1m\x1b[36malice\x1b[0m\x1b[0m \x1b[2mm p\x1b[0m\n" + + " hi\n\x1b[2m Updated 2026-02-21T12:00:00.000Z\x1b[0m\n\n" + + " Coverage: 1 selected, 1 visible; query truncated\n" + + "\x1b[2m────────────────────────────────────────────────────────────\x1b[0m\n\n" + + "\x1b[1m⚠ Unknown channel (\x1b[36mcid\x1b[0m)\x1b[0m\n\n" + + " Coverage: 0 selected, 0 visible; query completeness unknown" + if got := FormatPretty(outputs, prettyDates(), PrettyOptions{Color: true, Relative: true}); got != want { + t.Fatalf("color bytes mismatch\n--- got ---\n%q\n--- want ---\n%q", got, want) + } + if got := FormatPretty(nil, prettyDates(), PrettyOptions{Color: true}); got != "" { + t.Fatalf("empty output = %q", got) + } +} + +func TestUserColorMatchesJavaScriptSupplementaryHash(t *testing.T) { + if got, want := userColor("😀"), "\x1b[32m😀\x1b[0m"; got != want { + t.Fatalf("userColor = %q, want %q", got, want) + } +} + +type controlledWriter struct { + n int + err error + calls int +} + +func (w *controlledWriter) Write(p []byte) (int, error) { + w.calls++ + if w.n > len(p) { + return len(p), w.err + } + return w.n, w.err +} + +func TestWritePrettyPropagatesWriterResultsWithoutRetry(t *testing.T) { + output := []MessageOutput{{Channel: Channel{Type: "dm", Name: "@x"}, Retrieval: Retrieval{}}} + for _, test := range []struct { + name string + n int + err error + want error + }{ + {name: "short", n: 3, want: io.ErrShortWrite}, + {name: "failure", n: 2, err: errors.New("disk gone"), want: errors.New("disk gone")}, + } { + t.Run(test.name, func(t *testing.T) { + writer := &controlledWriter{n: test.n, err: test.err} + n, err := WritePretty(writer, output, prettyDates(), PrettyOptions{}) + if n != test.n || writer.calls != 1 { + t.Fatalf("n=%d calls=%d", n, writer.calls) + } + if err == nil || err.Error() != test.want.Error() { + t.Fatalf("error = %v, want %v", err, test.want) + } + }) + } +} From ab4eb5dd8cbf57893ad7f35155b318e4faa44a0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 10:48:46 +0300 Subject: [PATCH 024/119] feat: port safe Markdown rendering --- internal/output/markdown.go | 221 +++++++++++++++++++++++++++++++ internal/output/markdown_test.go | 133 +++++++++++++++++++ 2 files changed, 354 insertions(+) create mode 100644 internal/output/markdown.go create mode 100644 internal/output/markdown_test.go diff --git a/internal/output/markdown.go b/internal/output/markdown.go new file mode 100644 index 0000000..418d2b4 --- /dev/null +++ b/internal/output/markdown.go @@ -0,0 +1,221 @@ +package output + +import ( + "fmt" + "io" + "net/url" + "slices" + "strings" +) + +type MarkdownOptions struct { + Relative bool +} + +// FormatMarkdown renders already-presented values without further sanitizing them. +func FormatMarkdown(outputs []MessageOutput, dates DateFormatter, options MarkdownOptions) string { + sections := make([]string, 0, len(outputs)) + for _, output := range outputs { + sections = append(sections, formatMarkdownSection(output, dates, options.Relative)) + } + return strings.Join(sections, "\n\n---\n\n") +} + +// WriteMarkdown renders and writes exactly once. It never retries an uncertain write. +func WriteMarkdown(w io.Writer, outputs []MessageOutput, dates DateFormatter, options MarkdownOptions) (int, error) { + formatted := FormatMarkdown(outputs, dates, options) + n, err := io.WriteString(w, formatted) + if err == nil && n != len(formatted) { + err = io.ErrShortWrite + } + return n, err +} + +func formatMarkdownSection(output MessageOutput, dates DateFormatter, relative bool) string { + lines := []string{markdownChannelHeader(output.Channel), ""} + messages := slices.Clone(output.Messages) + slices.SortStableFunc(messages, func(a, b Message) int { return a.Timestamp.Compare(b.Timestamp) }) + lastDate := "" + for _, message := range messages { + date := dates.FormatDateLong(message.Timestamp) + if date != lastDate { + lines = append(lines, "### "+date, "") + lastDate = date + } + lines = append(lines, formatMarkdownMessage(message, dates, relative, 0), "") + } + if len(output.Redactions) > 0 { + lines = append(lines, "", fmt.Sprintf("_%d secret(s) redacted_", len(output.Redactions))) + } + state := "completeness unknown" + if output.Retrieval.Selection.QueryTruncated != nil { + if *output.Retrieval.Selection.QueryTruncated { + state = "truncated" + } else { + state = "complete" + } + } + lines = append(lines, "", fmt.Sprintf("_Coverage: %d selected, %d visible; query %s_", output.Retrieval.Selection.SelectedCount, output.Retrieval.VisiblePostCount, state)) + if cursor := output.Retrieval.Selection.NextCursor; cursor != nil && *cursor != "" { + lines = append(lines, "Next cursor: `"+*cursor+"`") + } + return strings.Join(lines, "\n") +} + +func markdownChannelHeader(channel Channel) string { + switch channel.Type { + case "unknown": + return "## Unknown channel (" + escapeMarkdown(channel.ID) + ")" + case "dm": + return "## DMs with " + escapeMarkdown(channel.Name) + case "group": + return "## Group DM: " + escapeMarkdown(channel.Name) + default: + display := "" + if channel.DisplayName != "" { + display = " (" + escapeMarkdown(channel.DisplayName) + ")" + } + return "## #" + escapeMarkdown(channel.Name) + display + } +} + +func formatMarkdownMessage(message Message, dates DateFormatter, relative bool, depth int) string { + timeText := dates.FormatTime(message.Timestamp) + if relative { + timeText = dates.FormatRelativeTime(message.Timestamp) + } + prefix := strings.Repeat("> ", depth) + postID := escapeMarkdown(message.ID) + postRef := postID + if permalink, ok := safeHTTPURL(message.Permalink); ok { + postRef = "[" + postID + "](<" + permalink + ">)" + } + markers := markdownMessageMarkers(message) + header := prefix + "**" + escapeMarkdown(message.User) + "**" + if markers != "" { + header += " " + markers + } + header += " (" + timeText + ", " + postRef + "):" + quotePrefix := prefix + "> " + lines := []string{header, markdownQuoteLines(escapeMarkdown(message.Text), quotePrefix)} + lines = append(lines, quotePrefix+"_"+formatStateTimes(message)+"_") + if len(message.FileDetails) > 0 { + lines = append(lines, quotePrefix+"_Files: "+escapeMarkdown(formatFiles(message))+"_") + } + for _, attachment := range message.Attachments { + title := attachment.Title + if title == "" { + title = "Attachment" + } + if attachment.Pretext != "" { + lines = append(lines, markdownQuoteLines(escapeMarkdown(attachment.Pretext), quotePrefix)) + } + var titleLine string + if titleLink, ok := safeHTTPURL(attachment.TitleLink); ok { + titleLine = "**[" + escapeMarkdown(title) + "](<" + titleLink + ">)**" + } else { + titleLine = "**" + escapeMarkdown(title) + "**" + } + lines = append(lines, markdownQuoteLines(titleLine, quotePrefix)) + if attachment.AuthorName != "" { + lines = append(lines, markdownQuoteLines("By: "+escapeMarkdown(attachment.AuthorName), quotePrefix)) + } + if attachment.Text != "" { + lines = append(lines, markdownQuoteLines(escapeMarkdown(attachment.Text), quotePrefix)) + } + for _, field := range attachment.Fields { + value := escapeMarkdown(field.Value) + if field.Title != "" { + value = "**" + escapeMarkdown(field.Title) + ":** " + value + } + lines = append(lines, markdownQuoteLines(value, quotePrefix)) + } + if attachment.Fallback != "" { + lines = append(lines, markdownQuoteLines("Fallback: "+escapeMarkdown(attachment.Fallback), quotePrefix)) + } + if attachment.Footer != "" { + lines = append(lines, markdownQuoteLines("_"+escapeMarkdown(attachment.Footer)+"_", quotePrefix)) + } + if attachment.Color != "" { + lines = append(lines, quotePrefix+"Color: "+escapeMarkdown(attachment.Color)) + } + if attachment.Timestamp != "" { + lines = append(lines, quotePrefix+"Timestamp: "+escapeMarkdown(attachment.Timestamp)) + } + for _, candidate := range []string{attachment.AuthorLink, attachment.AuthorIcon, attachment.FooterIcon, attachment.ImageURL, attachment.ThumbURL} { + if safe, ok := safeHTTPURL(candidate); ok { + lines = append(lines, quotePrefix+"<"+safe+">") + } + } + } + if len(message.Reactions) > 0 { + reactions := make([]string, 0, len(message.Reactions)) + for _, reaction := range message.Reactions { + actors := make([]string, 0, len(reaction.Actors)) + for _, actor := range reaction.Actors { + name := actor.Username + if name == "" { + name = actor.ID + } + actors = append(actors, escapeMarkdown(name)) + } + value := fmt.Sprintf(":%s: %d", escapeMarkdown(reaction.Emoji), reaction.Count) + if len(actors) > 0 { + value += " (" + strings.Join(actors, ", ") + ")" + } + reactions = append(reactions, value) + } + lines = append(lines, quotePrefix+"_Reactions: "+strings.Join(reactions, " · ")+"_") + } + if len(message.Replies) > 0 { + lines = append(lines, "") + for _, reply := range message.Replies { + lines = append(lines, formatMarkdownMessage(reply, dates, relative, depth+1), "") + } + } + return strings.Join(lines, "\n") +} + +func markdownMessageMarkers(message Message) string { + markers := make([]string, 0, 3) + if message.IsDeleted { + markers = append(markers, "[deleted]") + } else if message.EditedAt != nil { + markers = append(markers, "[edited]") + } + if message.IsSystem { + if message.PostType != "" { + markers = append(markers, "[system:"+escapeMarkdown(message.PostType)+"]") + } else { + markers = append(markers, "[system]") + } + } + if message.IsPinned { + markers = append(markers, "[pinned]") + } + return strings.Join(markers, " ") +} + +var markdownEscaper = strings.NewReplacer( + "\\", "\\\\", "`", "\\`", "*", "\\*", "_", "\\_", "[", "\\[", "]", "\\]", + "{", "\\{", "}", "\\}", "(", "\\(", ")", "\\)", "<", "\\<", ">", "\\>", + "#", "\\#", "+", "\\+", "-", "\\-", ".", "\\.", "!", "\\!", "|", "\\|", "~", "\\~", +) + +func escapeMarkdown(value string) string { return markdownEscaper.Replace(value) } + +func markdownQuoteLines(value, prefix string) string { + return prefix + strings.ReplaceAll(value, "\n", "\n"+prefix) +} + +func safeHTTPURL(value string) (string, bool) { + if value == "" { + return "", false + } + parsed, err := url.Parse(value) + if err != nil || parsed.Host == "" || (parsed.Scheme != "http" && parsed.Scheme != "https") { + return "", false + } + replacer := strings.NewReplacer("<", "%3C", ">", "%3E", "\\", "%5C") + return replacer.Replace(value), true +} diff --git a/internal/output/markdown_test.go b/internal/output/markdown_test.go new file mode 100644 index 0000000..d822164 --- /dev/null +++ b/internal/output/markdown_test.go @@ -0,0 +1,133 @@ +package output + +import ( + "errors" + "io" + "strings" + "testing" + "time" +) + +func markdownDates() DateFormatter { + return NewDateFormatter(func() time.Time { return time.Date(2026, 2, 21, 12, 0, 0, 0, time.UTC) }, time.UTC) +} + +func TestFormatMarkdownHostileGolden(t *testing.T) { + complete := false + next := "opaque_123" + size := int64(42) + edited := time.Date(2026, 2, 20, 10, 1, 0, 0, time.UTC) + deleted := time.Date(2026, 2, 20, 10, 2, 0, 0, time.UTC) + output := MessageOutput{ + Channel: Channel{Type: "private", Name: "secret-stuff", DisplayName: "[General]"}, + Messages: []Message{ + {ID: "later", Permalink: "javascript:alert(1)", User: "mallory", Text: "last", Timestamp: time.Date(2026, 2, 21, 8, 0, 0, 0, time.UTC), UpdatedAt: time.Date(2026, 2, 21, 8, 0, 0, 0, time.UTC)}, + { + ID: "m[1]", Permalink: "https://mm.test/a>b\\c", User: "[admin](https://evil.test)", Text: "# heading\n---\na | b\n*bold*\ncontrol \\u001b", Timestamp: time.Date(2026, 2, 20, 10, 0, 0, 0, time.UTC), UpdatedAt: time.Date(2026, 2, 20, 10, 0, 0, 123000000, time.FixedZone("x", 3*60*60)), + EditedAt: &edited, DeletedAt: &deleted, IsDeleted: true, IsSystem: true, PostType: "system_webhook", IsPinned: true, + FileDetails: []File{{ID: "f(1)", Name: "a*.txt", MIME: "text/plain", Extension: "txt", Size: &size}}, + Attachments: []Attachment{{ + Pretext: "pre\ntext", Title: "[click](evil)", TitleLink: "https://example.test/a>b\\c", AuthorName: "a_b", Text: "first\nsecond", Fields: []AttachmentField{{Title: "key*", Value: "v|"}, {Value: "bare"}}, Fallback: "fall#", Footer: "foot_", Color: "#fff", Timestamp: "now!", AuthorLink: "https://example.test/", AuthorIcon: "javascript:bad", FooterIcon: "https://example.test/footer", ImageURL: "ftp://bad", ThumbURL: "https://example.test/thumb", + }}, + Reactions: []Reaction{{Emoji: "white_check_mark", Count: 2, Actors: []ReactionActor{{Username: "b*b"}, {ID: "u[2]"}}}}, + Replies: []Message{{ID: "r", Permalink: "not a url", User: "eve", Text: "reply\nline", Timestamp: time.Date(2026, 2, 20, 10, 3, 0, 0, time.UTC), UpdatedAt: time.Date(2026, 2, 20, 10, 3, 0, 0, time.UTC), Replies: []Message{{ID: "rr", Permalink: "https://mm.test/rr", User: "zed", Text: "deep", Timestamp: time.Date(2026, 2, 20, 10, 4, 0, 0, time.UTC), UpdatedAt: time.Date(2026, 2, 20, 10, 4, 0, 0, time.UTC)}}}}, + }, + }, + Redactions: []Redaction{{Type: "token"}}, + Retrieval: Retrieval{Selection: Selection{SelectedCount: 2, QueryTruncated: &complete, NextCursor: &next}, VisiblePostCount: 4}, + } + want := "## #secret\\-stuff (\\[General\\])\n\n" + + "### Friday, 20 February 2026\n\n" + + "**\\[admin\\]\\(https://evil\\.test\\)** [deleted] [system:system\\_webhook] [pinned] (10:00, [m\\[1\\]]()):\n" + + "> \\# heading\n> \\-\\-\\-\n> a \\| b\n> \\*bold\\*\n> control \\\\u001b\n" + + "> _Updated 2026-02-20T07:00:00.123Z; edited 2026-02-20T10:01:00.000Z; deleted 2026-02-20T10:02:00.000Z_\n" + + "> _Files: a\\*\\.txt \\(f\\(1\\)\\), text/plain, txt, 42 B_\n" + + "> pre\n> text\n" + + "> **[\\[click\\]\\(evil\\)]()**\n" + + "> By: a\\_b\n> first\n> second\n> **key\\*:** v\\|\n> bare\n> Fallback: fall\\#\n> _foot\\__\n> Color: \\#fff\n> Timestamp: now\\!\n" + + "> \n> \n> \n" + + "> _Reactions: :white\\_check\\_mark: 2 (b\\*b, u\\[2\\])_\n\n" + + "> **eve** (10:03, r):\n> > reply\n> > line\n> > _Updated 2026-02-20T10:03:00.000Z_\n\n" + + "> > **zed** (10:04, [rr]()):\n> > > deep\n> > > _Updated 2026-02-20T10:04:00.000Z_\n\n\n\n" + + "### Saturday, 21 February 2026\n\n**mallory** (08:00, later):\n> last\n> _Updated 2026-02-21T08:00:00.000Z_\n\n\n" + + "_1 secret(s) redacted_\n\n_Coverage: 2 selected, 4 visible; query complete_\nNext cursor: `opaque_123`" + if got := FormatMarkdown([]MessageOutput{output}, markdownDates(), MarkdownOptions{}); got != want { + t.Fatalf("markdown bytes mismatch\n--- got ---\n%q\n--- want ---\n%q", got, want) + } +} + +func TestFormatMarkdownHeadersCoverageRelativeAndSections(t *testing.T) { + truncated := true + unknown := MessageOutput{Channel: Channel{ID: "id(1)", Type: "unknown"}, Retrieval: Retrieval{}} + dm := MessageOutput{Channel: Channel{Type: "dm", Name: "@bob"}, Messages: []Message{{ID: "m", User: "a", Text: "hi", Timestamp: time.Date(2026, 2, 21, 10, 0, 0, 0, time.UTC), UpdatedAt: time.Date(2026, 2, 21, 10, 0, 0, 0, time.UTC)}}, Retrieval: Retrieval{Selection: Selection{SelectedCount: 1, QueryTruncated: &truncated}, VisiblePostCount: 1}} + got := FormatMarkdown([]MessageOutput{unknown, dm}, markdownDates(), MarkdownOptions{Relative: true}) + for _, want := range []string{"## Unknown channel (id\\(1\\))", "\n\n---\n\n", "## DMs with @bob", "(2 hours ago, m)", "query completeness unknown", "query truncated"} { + if !strings.Contains(got, want) { + t.Errorf("missing %q in %q", want, got) + } + } + if got := FormatMarkdown(nil, markdownDates(), MarkdownOptions{}); got != "" { + t.Fatalf("empty output = %q", got) + } +} + +func TestFormatMarkdownUnsafeAttachmentTitleIsUnlinked(t *testing.T) { + output := MessageOutput{Channel: Channel{Type: "group", Name: "crew"}, Messages: []Message{{ID: "m", User: "u", Text: "", Timestamp: time.Unix(0, 0), UpdatedAt: time.Unix(0, 0), Attachments: []Attachment{{Title: "[click](https://evil.test)", TitleLink: "javascript:alert(1)"}, {}}}}, Retrieval: Retrieval{}} + got := FormatMarkdown([]MessageOutput{output}, markdownDates(), MarkdownOptions{}) + if strings.Contains(got, "javascript:") || strings.Contains(got, "[click](https://evil.test)") { + t.Fatalf("unsafe markdown survived: %q", got) + } + if !strings.Contains(got, "> **\\[click\\]\\(https://evil\\.test\\)**\n> **Attachment**") { + t.Fatalf("safe fallback missing: %q", got) + } +} + +func TestFormatMarkdownMultilineAttachmentTitlesStayQuoted(t *testing.T) { + message := Message{ID: "m", User: "u", Text: "body", Timestamp: time.Unix(0, 0), UpdatedAt: time.Unix(0, 0), Attachments: []Attachment{ + {Title: "linked\n\n forged", TitleLink: "https://example.test"}, + {Title: "plain\n\n forged"}, + }} + got := FormatMarkdown([]MessageOutput{{Channel: Channel{Type: "dm", Name: "@x"}, Messages: []Message{message}}}, markdownDates(), MarkdownOptions{}) + for _, want := range []string{ + "> **[linked\n> \n> forged]()**", + "> **plain\n> \n> forged**", + } { + if !strings.Contains(got, want) { + t.Fatalf("missing safely quoted title %q in %q", want, got) + } + } + if strings.Contains(got, "\n\n forged") { + t.Fatalf("title escaped blockquote: %q", got) + } +} + +type markdownControlledWriter struct { + n, calls int + err error +} + +func (w *markdownControlledWriter) Write(p []byte) (int, error) { + w.calls++ + if w.n > len(p) { + return len(p), w.err + } + return w.n, w.err +} + +func TestWriteMarkdownPropagatesWriterResultsWithoutRetry(t *testing.T) { + output := []MessageOutput{{Channel: Channel{Type: "dm", Name: "@x"}}} + for _, test := range []struct { + name string + n int + err error + want error + }{{"short", 3, nil, io.ErrShortWrite}, {"failure", 2, errors.New("disk gone"), errors.New("disk gone")}} { + t.Run(test.name, func(t *testing.T) { + writer := &markdownControlledWriter{n: test.n, err: test.err} + n, err := WriteMarkdown(writer, output, markdownDates(), MarkdownOptions{}) + if n != test.n || writer.calls != 1 || err == nil || err.Error() != test.want.Error() { + t.Fatalf("n=%d calls=%d error=%v, want %v", n, writer.calls, err, test.want) + } + }) + } +} From b593f1edf3b4736437b0d2fee5805c30001816f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 11:48:38 +0300 Subject: [PATCH 025/119] feat: retain validated post presentation data --- internal/mattermost/post_presentation_test.go | 144 ++++++++++ internal/mattermost/posts.go | 267 +++++++++++++++++- internal/mattermost/posts_test.go | 10 + 3 files changed, 415 insertions(+), 6 deletions(-) create mode 100644 internal/mattermost/post_presentation_test.go diff --git a/internal/mattermost/post_presentation_test.go b/internal/mattermost/post_presentation_test.go new file mode 100644 index 0000000..051a9f6 --- /dev/null +++ b/internal/mattermost/post_presentation_test.go @@ -0,0 +1,144 @@ +package mattermost + +import ( + "bytes" + "encoding/json" + "errors" + "reflect" + "testing" +) + +func TestPostRetainsValidatedEmbeddedPresentationData(t *testing.T) { + var post Post + payload := `{ + "id":"post_1","channel_id":"channel_1","user_id":"user raw","message":"latest","create_at":1000, + "update_at":3000,"edit_at":2000,"delete_at":0,"root_id":"","reply_count":0,"type":"system_webhook", + "is_pinned":true,"override_username":"direct hook","file_ids":["file raw",7,"file-2",""], + "props":{"override_username":"props hook","ignored":{"secret":"no"},"attachments":[null,7,{}, + {"pretext":"before","title":"title","title_link":"https://example.test","text":"body","fields":[null,7,{"title":7,"value":9,"short":false},{"short":true}],"footer":"foot","footer_icon":"fi","author_name":"author","author_link":"al","author_icon":"ai","color":"#fff","image_url":"image","thumb_url":"thumb","ts":123}]}, + "metadata":{"files":[null,7,{"id":"file raw","name":"name","mime_type":"text/plain","size":12,"extension":"txt"},{"id":7},{"id":"file-2","size":-1}], + "reactions":[null,7,{"user_id":"u2","post_id":"post_1","emoji_name":"eyes","create_at":2},{"user_id":"u1","post_id":"post_1","emoji_name":"eyes","create_at":1},{"user_id":7,"emoji_name":"bad"}]}} + ` + if err := json.Unmarshal([]byte(payload), &post); err != nil { + t.Fatal(err) + } + if post.ID != "post_1" || post.ChannelID != "channel_1" || post.UserID != "user raw" || post.Message != "latest" || post.CreateAt != 1000 || post.UpdateAt != 3000 || post.EditAt != 2000 || post.Type != "system_webhook" || !post.IsPinned || post.OverrideUsername != "direct hook" { + t.Fatalf("post scalars = %#v", post) + } + if !reflect.DeepEqual(post.FileIDs, []string{"file raw", "file-2"}) { + t.Fatalf("file ids = %#v", post.FileIDs) + } + if len(post.Files) != 2 || post.Files[0].ID != "file raw" || post.Files[0].Size == nil || *post.Files[0].Size != 12 || post.Files[1].ID != "file-2" || post.Files[1].Size != nil { + t.Fatalf("files = %#v", post.Files) + } + if len(post.Attachments) != 1 || post.Attachments[0].Timestamp != "123" || !reflect.DeepEqual(post.Attachments[0].Fields, []PostAttachmentField{{Title: "7", Value: "9", Short: boolPostPointer(false)}, {Short: boolPostPointer(true)}}) { + t.Fatalf("attachments = %#v", post.Attachments) + } + wantReactions := []PostReaction{{UserID: "u2", PostID: "post_1", EmojiName: "eyes", CreateAt: 2}, {UserID: "u1", PostID: "post_1", EmojiName: "eyes", CreateAt: 1}} + if !reflect.DeepEqual(post.Reactions, wantReactions) { + t.Fatalf("reactions = %#v", post.Reactions) + } +} + +func TestPostOptionalPresentationMetadataFailsLocally(t *testing.T) { + var post Post + payload := `{"id":"post","channel_id":"channel","user_id":7,"message":"ok","create_at":10,"update_at":"bad","edit_at":8640000000000001,"delete_at":0,"type":7,"is_pinned":"true","override_username":7,"file_ids":"bad","props":{"override_username":"props hook","attachments":"bad"},"metadata":{"files":"bad","reactions":[{"user_id":"u","emoji_name":"eyes"},{"user_id":"bad","emoji_name":7}]}}` + if err := json.Unmarshal([]byte(payload), &post); err != nil { + t.Fatal(err) + } + if post.UserID != "" || post.UpdateAt != post.CreateAt || post.EditAt != 0 || post.Type != "" || post.IsPinned || post.OverrideUsername != "props hook" || len(post.FileIDs) != 0 || len(post.Files) != 0 || len(post.Attachments) != 0 || !reflect.DeepEqual(post.Reactions, []PostReaction{{UserID: "u", EmojiName: "eyes"}}) { + t.Fatalf("post = %#v", post) + } +} + +func TestDeletedPostSuppressesEveryStalePresentationField(t *testing.T) { + var post Post + secret := "ghp_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + payload := `{"id":"gone","channel_id":"channel","user_id":"author","message":{"stale":"` + secret + `"},"create_at":10,"update_at":11,"edit_at":12,"delete_at":13,"type":"` + secret + `","is_pinned":true,"override_username":"` + secret + `","file_ids":["` + secret + `"],"props":{"attachments":[{"text":"` + secret + `"}]},"metadata":{"files":[{"id":"secret-file","name":"` + secret + `"}],"reactions":[{"user_id":"reactor","emoji_name":"` + secret + `"}]}}` + if err := json.Unmarshal([]byte(payload), &post); err != nil { + t.Fatal(err) + } + if post.Message != "" || post.DeleteAt != 13 || post.Type != "" || post.IsPinned || post.OverrideUsername != "" || len(post.FileIDs) != 0 || len(post.Files) != 0 || len(post.Attachments) != 0 || len(post.Reactions) != 0 { + t.Fatalf("deleted post retained stale data: %#v", post) + } + if encoded, err := json.Marshal(post); err != nil || bytes.Contains(encoded, []byte(secret)) { + t.Fatalf("deleted post encoded stale secret: %s (error %v)", encoded, err) + } + + var page OrderedPostsPage + if err := json.Unmarshal([]byte(`{"order":["gone"],"posts":{"gone":`+payload+`}}`), &page); err != nil { + t.Fatal(err) + } + if len(page.Posts) != 0 || page.RawCount != 1 || page.Continuation == nil || page.Continuation.PostID != "gone" { + t.Fatalf("page policy changed: %#v", page) + } +} + +func TestPostNullPresentationScalarsAreOmitted(t *testing.T) { + var post Post + payload := `{"id":"post","channel_id":"channel","message":"ok","create_at":10,"update_at":null,"edit_at":null,"delete_at":0,"props":{"attachments":[{"fields":[{"title":null,"value":null,"short":null}],"ts":null}]},"metadata":{"files":[{"id":"file","size":null}]}}` + if err := json.Unmarshal([]byte(payload), &post); err != nil { + t.Fatal(err) + } + if post.UpdateAt != 10 || post.EditAt != 0 || len(post.Files) != 1 || post.Files[0].Size != nil || len(post.Attachments) != 0 { + t.Fatalf("post = %#v", post) + } +} + +func TestPostOverrideUsernameUsesNullishNotEmptyFallback(t *testing.T) { + for _, tt := range []struct { + name string + direct string + expected string + }{ + {"missing", "", "props"}, + {"null", `,"override_username":null`, "props"}, + {"empty", `,"override_username":""`, ""}, + {"direct", `,"override_username":"direct"`, "direct"}, + } { + t.Run(tt.name, func(t *testing.T) { + var post Post + payload := `{"id":"post","channel_id":"channel","message":"ok","create_at":10,"delete_at":0` + tt.direct + `,"props":{"override_username":"props"}}` + if err := json.Unmarshal([]byte(payload), &post); err != nil { + t.Fatal(err) + } + if post.OverrideUsername != tt.expected { + t.Fatalf("override = %q, want %q", post.OverrideUsername, tt.expected) + } + }) + } +} + +func TestECMAScriptNumberString(t *testing.T) { + for _, tt := range []struct { + input float64 + want string + }{ + {1e-6, "0.000001"}, + {1e20, "100000000000000000000"}, + {1e-7, "1e-7"}, + {-0.0, "0"}, + {1e21, "1e+21"}, + {-1e-7, "-1e-7"}, + } { + if got := ecmaScriptNumberString(tt.input); got != tt.want { + t.Fatalf("ecmaScriptNumberString(%v) = %q, want %q", tt.input, got, tt.want) + } + } +} + +func TestPostStillRejectsMalformedRequiredIdentityAndTimestamps(t *testing.T) { + for _, payload := range []string{ + `{"id":7,"channel_id":"channel","message":"x","create_at":1,"delete_at":0}`, + `{"id":"post","channel_id":7,"message":"x","create_at":1,"delete_at":0}`, + `{"id":"post","channel_id":"channel","message":"x","create_at":"1","delete_at":0}`, + `{"id":"post","channel_id":"channel","message":"x","create_at":1,"delete_at":"0"}`, + } { + var post Post + if !errors.Is(json.Unmarshal([]byte(payload), &post), ErrInvalidPostResponse) { + t.Fatalf("payload accepted: %s", payload) + } + } +} + +func boolPostPointer(value bool) *bool { return &value } diff --git a/internal/mattermost/posts.go b/internal/mattermost/posts.go index 16b9099..494679a 100644 --- a/internal/mattermost/posts.go +++ b/internal/mattermost/posts.go @@ -1,6 +1,7 @@ package mattermost import ( + "bytes" "context" "encoding/json" "errors" @@ -28,46 +29,296 @@ type postTransport interface { PostRead(context.Context, string, any, any) error } -// Post retains only fields needed for history selection. Presentation-specific -// fields belong in a later, wider model rather than leaking unchecked payloads. type Post struct { ID string ChannelID string + UserID string Message string CreateAt int64 + UpdateAt int64 + EditAt int64 DeleteAt int64 RootID string ReplyCount int ThreadShapeKnown bool + Type string + IsPinned bool + OverrideUsername string + FileIDs []string + Files []PostFile + Attachments []PostAttachment + Reactions []PostReaction +} + +type PostFile struct { + ID string + Name string + MIMEType string + Size *int64 + Extension string +} + +type PostAttachmentField struct { + Title string + Value string + Short *bool +} + +type PostAttachment struct { + Fallback string + Pretext string + Title string + TitleLink string + Text string + Fields []PostAttachmentField + Footer string + FooterIcon string + AuthorName string + AuthorLink string + AuthorIcon string + Color string + ImageURL string + ThumbURL string + Timestamp string +} + +type PostReaction struct { + UserID string + PostID string + EmojiName string + CreateAt int64 } func (p *Post) UnmarshalJSON(data []byte) error { var raw struct { ID json.RawMessage `json:"id"` ChannelID json.RawMessage `json:"channel_id"` + UserID json.RawMessage `json:"user_id"` Message json.RawMessage `json:"message"` CreateAt json.RawMessage `json:"create_at"` + UpdateAt json.RawMessage `json:"update_at"` + EditAt json.RawMessage `json:"edit_at"` DeleteAt json.RawMessage `json:"delete_at"` RootID json.RawMessage `json:"root_id"` ReplyCount json.RawMessage `json:"reply_count"` + Type json.RawMessage `json:"type"` + IsPinned json.RawMessage `json:"is_pinned"` + Override json.RawMessage `json:"override_username"` + FileIDs json.RawMessage `json:"file_ids"` + Props json.RawMessage `json:"props"` + Metadata json.RawMessage `json:"metadata"` } if err := json.Unmarshal(data, &raw); err != nil { return ErrInvalidPostResponse } id, idOK := safePostID(raw.ID) channelID, channelOK := requiredString(raw.ChannelID) - message, messageOK := strictString(raw.Message) createAt, createOK := nonnegativeInteger(raw.CreateAt) deleteAt, deleteOK := nonnegativeInteger(raw.DeleteAt) rootID, rootOK, rootKnown := optionalPostID(raw.RootID) replyCount, replyOK, replyKnown := optionalNonnegativeInteger(raw.ReplyCount) - if !idOK || !channelOK || !messageOK || !createOK || createAt == 0 || createAt > maxDateMilliseconds || !deleteOK || deleteAt > maxDateMilliseconds || !rootOK || !replyOK { + message, messageOK := strictString(raw.Message) + if !idOK || !channelOK || (!messageOK && deleteAt == 0) || !createOK || createAt == 0 || createAt > maxDateMilliseconds || !deleteOK || deleteAt > maxDateMilliseconds || !rootOK || !replyOK { return ErrInvalidPostResponse } - *p = Post{ID: id, ChannelID: channelID, Message: message, CreateAt: createAt, DeleteAt: deleteAt, RootID: rootID, ReplyCount: replyCount, ThreadShapeKnown: rootKnown && replyKnown} + post := Post{ + ID: id, ChannelID: channelID, UserID: postOptionalString(raw.UserID), + CreateAt: createAt, UpdateAt: optionalTimestamp(raw.UpdateAt, createAt), EditAt: optionalTimestamp(raw.EditAt, 0), DeleteAt: deleteAt, + RootID: rootID, ReplyCount: replyCount, ThreadShapeKnown: rootKnown && replyKnown, + } + if deleteAt == 0 { + post.Message = message + post.Type = postOptionalString(raw.Type) + post.IsPinned = optionalBool(raw.IsPinned) + directOverride, directOverrideKnown := optionalStringValue(raw.Override) + if directOverrideKnown { + post.OverrideUsername = directOverride + } + post.FileIDs = stringArray(raw.FileIDs) + post.Files, post.Reactions = parsePostMetadata(raw.Metadata) + post.Attachments, post.OverrideUsername = parsePostProps(raw.Props, post.OverrideUsername, directOverrideKnown) + } + *p = post return nil } +func postOptionalString(raw json.RawMessage) string { + value, ok := optionalStringValue(raw) + if !ok { + return "" + } + return value +} + +func optionalStringValue(raw json.RawMessage) (string, bool) { + if len(raw) == 0 || isJSONNull(raw) { + return "", false + } + return strictString(raw) +} + +func optionalBool(raw json.RawMessage) bool { + var value bool + if len(raw) == 0 || json.Unmarshal(raw, &value) != nil { + return false + } + return value +} + +func optionalTimestamp(raw json.RawMessage, fallback int64) int64 { + value, ok := nonnegativeInteger(raw) + if !ok || value > maxDateMilliseconds { + return fallback + } + return value +} + +func stringArray(raw json.RawMessage) []string { + var values []json.RawMessage + if json.Unmarshal(raw, &values) != nil { + return nil + } + result := make([]string, 0, len(values)) + for _, candidate := range values { + if value := postOptionalString(candidate); value != "" { + result = append(result, value) + } + } + return result +} + +func parsePostMetadata(raw json.RawMessage) ([]PostFile, []PostReaction) { + var metadata map[string]json.RawMessage + if json.Unmarshal(raw, &metadata) != nil { + return nil, nil + } + var rawFiles []json.RawMessage + _ = json.Unmarshal(metadata["files"], &rawFiles) + files := make([]PostFile, 0, len(rawFiles)) + for _, candidate := range rawFiles { + var file struct { + ID json.RawMessage `json:"id"` + Name json.RawMessage `json:"name"` + MIMEType json.RawMessage `json:"mime_type"` + Size json.RawMessage `json:"size"` + Extension json.RawMessage `json:"extension"` + } + if json.Unmarshal(candidate, &file) != nil || postOptionalString(file.ID) == "" { + continue + } + value := PostFile{ID: postOptionalString(file.ID), Name: postOptionalString(file.Name), MIMEType: postOptionalString(file.MIMEType), Extension: postOptionalString(file.Extension)} + if size, ok := nonnegativeInteger(file.Size); ok { + value.Size = &size + } + files = append(files, value) + } + var rawReactions []json.RawMessage + _ = json.Unmarshal(metadata["reactions"], &rawReactions) + reactions := make([]PostReaction, 0, len(rawReactions)) + for _, candidate := range rawReactions { + var reaction struct { + UserID json.RawMessage `json:"user_id"` + PostID json.RawMessage `json:"post_id"` + EmojiName json.RawMessage `json:"emoji_name"` + CreateAt json.RawMessage `json:"create_at"` + } + if json.Unmarshal(candidate, &reaction) != nil { + continue + } + value := PostReaction{UserID: postOptionalString(reaction.UserID), PostID: postOptionalString(reaction.PostID), EmojiName: postOptionalString(reaction.EmojiName), CreateAt: optionalTimestamp(reaction.CreateAt, 0)} + if value.UserID != "" && value.EmojiName != "" { + reactions = append(reactions, value) + } + } + return files, reactions +} + +func parsePostProps(raw json.RawMessage, override string, overrideKnown bool) ([]PostAttachment, string) { + var props map[string]json.RawMessage + if json.Unmarshal(raw, &props) != nil { + return nil, override + } + if !overrideKnown { + if propsOverride, ok := optionalStringValue(props["override_username"]); ok { + override = propsOverride + } + } + var rawAttachments []json.RawMessage + _ = json.Unmarshal(props["attachments"], &rawAttachments) + attachments := make([]PostAttachment, 0, len(rawAttachments)) + for _, candidate := range rawAttachments { + var source map[string]json.RawMessage + if json.Unmarshal(candidate, &source) != nil || source == nil { + continue + } + attachment := PostAttachment{ + Fallback: postOptionalString(source["fallback"]), Pretext: postOptionalString(source["pretext"]), Title: postOptionalString(source["title"]), + TitleLink: postOptionalString(source["title_link"]), Text: postOptionalString(source["text"]), Footer: postOptionalString(source["footer"]), + FooterIcon: postOptionalString(source["footer_icon"]), AuthorName: postOptionalString(source["author_name"]), AuthorLink: postOptionalString(source["author_link"]), + AuthorIcon: postOptionalString(source["author_icon"]), Color: postOptionalString(source["color"]), ImageURL: postOptionalString(source["image_url"]), + ThumbURL: postOptionalString(source["thumb_url"]), Timestamp: stringOrNumber(source["ts"]), + } + var fields []json.RawMessage + if json.Unmarshal(source["fields"], &fields) == nil { + for _, rawField := range fields { + var fieldSource map[string]json.RawMessage + if json.Unmarshal(rawField, &fieldSource) != nil || fieldSource == nil { + continue + } + field := PostAttachmentField{Title: stringOrNumber(fieldSource["title"]), Value: stringOrNumber(fieldSource["value"])} + var short bool + if rawShort := fieldSource["short"]; len(rawShort) > 0 && !isJSONNull(rawShort) && json.Unmarshal(rawShort, &short) == nil { + field.Short = &short + } + if field.Title != "" || field.Value != "" || field.Short != nil { + attachment.Fields = append(attachment.Fields, field) + } + } + } + if postAttachmentHasContent(attachment) { + attachments = append(attachments, attachment) + } + } + return attachments, override +} + +func postAttachmentHasContent(value PostAttachment) bool { + return value.Fallback != "" || value.Pretext != "" || value.Title != "" || value.TitleLink != "" || value.Text != "" || + len(value.Fields) > 0 || value.Footer != "" || value.FooterIcon != "" || value.AuthorName != "" || value.AuthorLink != "" || + value.AuthorIcon != "" || value.Color != "" || value.ImageURL != "" || value.ThumbURL != "" || value.Timestamp != "" +} + +func stringOrNumber(raw json.RawMessage) string { + if value := postOptionalString(raw); value != "" { + return value + } + var number float64 + if len(raw) > 0 && !isJSONNull(raw) && json.Unmarshal(raw, &number) == nil { + return ecmaScriptNumberString(number) + } + return "" +} + +func ecmaScriptNumberString(value float64) string { + if value == 0 { + return "0" + } + abs := value + if abs < 0 { + abs = -abs + } + if abs >= 1e-6 && abs < 1e21 { + return strconv.FormatFloat(value, 'f', -1, 64) + } + formatted := strconv.FormatFloat(value, 'e', -1, 64) + parts := strings.SplitN(formatted, "e", 2) + exponent, _ := strconv.Atoi(parts[1]) + if exponent >= 0 { + return parts[0] + "e+" + strconv.Itoa(exponent) + } + return parts[0] + "e" + strconv.Itoa(exponent) +} + func optionalPostID(raw json.RawMessage) (string, bool, bool) { if len(raw) == 0 { return "", true, false @@ -117,12 +368,16 @@ func isSafePostID(value string) bool { func nonnegativeInteger(raw json.RawMessage) (int64, bool) { var value int64 - if len(raw) == 0 || json.Unmarshal(raw, &value) != nil || value < 0 { + if len(raw) == 0 || isJSONNull(raw) || json.Unmarshal(raw, &value) != nil || value < 0 { return 0, false } return value, true } +func isJSONNull(raw json.RawMessage) bool { + return string(bytes.TrimSpace(raw)) == "null" +} + // OrderedPostsPage preserves the server's order and raw fullness separately // from its validated, visible posts. type OrderedPostsPage struct { diff --git a/internal/mattermost/posts_test.go b/internal/mattermost/posts_test.go index 62b6179..5579bbc 100644 --- a/internal/mattermost/posts_test.go +++ b/internal/mattermost/posts_test.go @@ -79,6 +79,16 @@ func TestOrderedPostsPageTreatsExplicitNullHasNextAsIncomplete(t *testing.T) { } } +func TestOrderedPostsPageRejectsNullDeleteTimestampWithoutEmittingStalePost(t *testing.T) { + var page OrderedPostsPage + if err := json.Unmarshal([]byte(`{"order":["stale"],"posts":{"stale":{"id":"stale","channel_id":"channel","message":"secret stale text","create_at":1,"delete_at":null}}}`), &page); err != nil { + t.Fatal(err) + } + if !page.Incomplete || page.RawCount != 1 || len(page.Posts) != 0 || page.Continuation != nil { + t.Fatalf("page = %#v", page) + } +} + func TestSearchPageBuildsExactReadPOSTAndNormalizesEnvelope(t *testing.T) { var gotPath string var gotBody []byte From 56c7d2170566e1ef359f6922ce27bf87801d2c96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 11:55:10 +0300 Subject: [PATCH 026/119] feat: add strict read machine contracts --- internal/cli/root_test.go | 2 +- internal/output/machine.go | 501 +++++++++++++++++++++++ internal/output/machine_test.go | 210 ++++++++++ internal/schema/read_test.go | 96 +++++ schemas/v2/channel.schema.json | 1 + schemas/v2/dms.schema.json | 1 + schemas/v2/examples/channel-empty.json | 1 + schemas/v2/examples/channel.json | 1 + schemas/v2/examples/dms-empty.json | 1 + schemas/v2/examples/dms.json | 1 + schemas/v2/examples/group-dms-empty.json | 1 + schemas/v2/examples/group-dms.json | 1 + schemas/v2/examples/mentions-empty.json | 1 + schemas/v2/examples/mentions.json | 1 + schemas/v2/examples/search-empty.json | 1 + schemas/v2/examples/search.json | 1 + schemas/v2/examples/thread.json | 1 + schemas/v2/group-dms.schema.json | 1 + schemas/v2/mentions.schema.json | 1 + schemas/v2/search.schema.json | 1 + schemas/v2/thread.schema.json | 1 + 21 files changed, 825 insertions(+), 1 deletion(-) create mode 100644 internal/output/machine.go create mode 100644 internal/output/machine_test.go create mode 100644 internal/schema/read_test.go create mode 100644 schemas/v2/channel.schema.json create mode 100644 schemas/v2/dms.schema.json create mode 100644 schemas/v2/examples/channel-empty.json create mode 100644 schemas/v2/examples/channel.json create mode 100644 schemas/v2/examples/dms-empty.json create mode 100644 schemas/v2/examples/dms.json create mode 100644 schemas/v2/examples/group-dms-empty.json create mode 100644 schemas/v2/examples/group-dms.json create mode 100644 schemas/v2/examples/mentions-empty.json create mode 100644 schemas/v2/examples/mentions.json create mode 100644 schemas/v2/examples/search-empty.json create mode 100644 schemas/v2/examples/search.json create mode 100644 schemas/v2/examples/thread.json create mode 100644 schemas/v2/group-dms.schema.json create mode 100644 schemas/v2/mentions.schema.json create mode 100644 schemas/v2/search.schema.json create mode 100644 schemas/v2/thread.schema.json diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index 0f9d469..175298f 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -54,7 +54,7 @@ func TestSchemaList(t *testing.T) { if code != 0 { t.Fatalf("exit code = %d, want 0; stderr = %q", code, stderr.String()) } - if got, want := stdout.String(), "mm/v2/error\n"; got != want { + if got, want := stdout.String(), "mm/v2/channel\nmm/v2/dms\nmm/v2/error\nmm/v2/group-dms\nmm/v2/mentions\nmm/v2/search\nmm/v2/thread\n"; got != want { t.Fatalf("stdout = %q, want %q", got, want) } } diff --git a/internal/output/machine.go b/internal/output/machine.go new file mode 100644 index 0000000..cca00d0 --- /dev/null +++ b/internal/output/machine.go @@ -0,0 +1,501 @@ +package output + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "reflect" + "strings" + "time" + "unicode/utf8" +) + +const MaxMachineDocumentBytes = 4 << 20 + +type MachineCompleteness string + +const ( + MachineComplete MachineCompleteness = "complete" + MachineTruncated MachineCompleteness = "truncated" + MachineUnknown MachineCompleteness = "unknown" +) + +// MachineMetadata preserves the distinction between false/empty and unknown. +// Pointer fields are always encoded: nil is the machine contract's explicit null. +type MachineMetadata struct { + Completeness MachineCompleteness `json:"completeness"` + Selection Selection `json:"selection"` + VisibleThreads VisibleThreads `json:"visibleThreads"` + VisiblePostCount int `json:"visiblePostCount"` + DeletedPostsIncluded bool `json:"deletedPostsIncluded"` +} + +type MachineChannel struct { + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + DisplayName string `json:"displayName"` + MetadataStatus string `json:"metadataStatus"` +} + +type MachineMessage struct { + ID string `json:"id"` + Permalink string `json:"permalink"` + User string `json:"user"` + UserID string `json:"userId"` + Text string `json:"text"` + Timestamp MillisTime `json:"timestamp"` + UpdatedAt MillisTime `json:"updatedAt"` + EditedAt *MillisTime `json:"editedAt"` + DeletedAt *MillisTime `json:"deletedAt"` + IsDeleted bool `json:"isDeleted"` + PostType string `json:"postType"` + IsSystem bool `json:"isSystem"` + IsPinned bool `json:"isPinned"` + RootID *string `json:"rootId"` + ReplyCount *int `json:"replyCount"` + Files []string `json:"files"` + FileDetails []File `json:"fileDetails"` + Attachments []Attachment `json:"attachments"` + Reactions []Reaction `json:"reactions"` + Replies []MachineMessage `json:"replies"` +} + +type MillisTime struct{ time.Time } + +func (value MillisTime) MarshalJSON() ([]byte, error) { + _, offset := value.Time.Zone() + if offset != 0 { + return nil, fmt.Errorf("machine timestamp must have zero UTC offset") + } + value.Time = value.Time.UTC() + if value.Year() < 1 || value.Year() > 9999 { + return nil, fmt.Errorf("machine timestamp year must be between 0001 and 9999") + } + return []byte(`"` + value.Truncate(time.Millisecond).Format("2006-01-02T15:04:05.000Z") + `"`), nil +} + +type MachineHistory struct { + Channel MachineChannel `json:"channel"` + Messages []MachineMessage `json:"messages"` + Redactions []Redaction `json:"redactions"` + Metadata MachineMetadata `json:"metadata"` +} + +type DMSEnvelope struct { + Schema string `json:"schema"` + Channels []MachineHistory `json:"channels"` +} +type GroupDMSEnvelope struct { + Schema string `json:"schema"` + Channels []MachineHistory `json:"channels"` +} +type ChannelEnvelope struct { + Schema string `json:"schema"` + Data MachineHistory `json:"data"` +} +type ThreadData struct { + Channel MachineChannel `json:"channel"` + Root MachineMessage `json:"root"` + Redactions []Redaction `json:"redactions"` + Metadata MachineMetadata `json:"metadata"` +} +type ThreadEnvelope struct { + Schema string `json:"schema"` + Data ThreadData `json:"data"` +} +type SearchEnvelope struct { + Schema string `json:"schema"` + Results []MachineHistory `json:"results"` +} +type MentionsEnvelope struct { + Schema string `json:"schema"` + Results []MachineHistory `json:"results"` +} + +type MachineDocument interface{ machineDocument() } + +func (DMSEnvelope) machineDocument() {} +func (GroupDMSEnvelope) machineDocument() {} +func (ChannelEnvelope) machineDocument() {} +func (ThreadEnvelope) machineDocument() {} +func (SearchEnvelope) machineDocument() {} +func (MentionsEnvelope) machineDocument() {} + +type wireMessage struct { + ID string `json:"id"` + Permalink string `json:"permalink"` + User string `json:"user"` + UserID string `json:"userId"` + Text string `json:"text"` + Timestamp MillisTime `json:"timestamp"` + UpdatedAt MillisTime `json:"updatedAt"` + EditedAt *MillisTime `json:"editedAt"` + DeletedAt *MillisTime `json:"deletedAt"` + IsDeleted bool `json:"isDeleted"` + PostType string `json:"postType"` + IsSystem bool `json:"isSystem"` + IsPinned bool `json:"isPinned"` + RootID *string `json:"rootId"` + ReplyCount *int `json:"replyCount"` + Files []string `json:"files"` + FileDetails []File `json:"fileDetails"` + Attachments []wireAttachment `json:"attachments"` + Reactions []wireReaction `json:"reactions"` + Replies []wireMessage `json:"replies"` +} + +type wireAttachment struct { + Fallback string `json:"fallback,omitempty"` + Pretext string `json:"pretext,omitempty"` + Title string `json:"title,omitempty"` + TitleLink string `json:"titleLink,omitempty"` + Text string `json:"text,omitempty"` + Fields []AttachmentField `json:"fields"` + Footer string `json:"footer,omitempty"` + FooterIcon string `json:"footerIcon,omitempty"` + AuthorName string `json:"authorName,omitempty"` + AuthorLink string `json:"authorLink,omitempty"` + AuthorIcon string `json:"authorIcon,omitempty"` + Color string `json:"color,omitempty"` + ImageURL string `json:"imageUrl,omitempty"` + ThumbURL string `json:"thumbUrl,omitempty"` + Timestamp string `json:"timestamp,omitempty"` +} + +type wireReaction struct { + Emoji string `json:"emoji"` + Count int `json:"count"` + Actors []ReactionActor `json:"actors"` +} + +type wireMetadata struct { + Completeness MachineCompleteness `json:"completeness"` + Selection Selection `json:"selection"` + VisibleThreads wireVisibleThreads `json:"visibleThreads"` + VisiblePostCount int `json:"visiblePostCount"` + DeletedPostsIncluded bool `json:"deletedPostsIncluded"` +} + +type wireVisibleThreads struct { + Status string `json:"status"` + HydratedRootCount int `json:"hydratedRootCount"` + FailedRootIDs []string `json:"failedRootIds"` +} + +type wireHistory struct { + Channel MachineChannel `json:"channel"` + Messages []wireMessage `json:"messages"` + Redactions []Redaction `json:"redactions"` + Metadata wireMetadata `json:"metadata"` +} + +func canonicalMachineDocument(document MachineDocument) (any, error) { + switch value := document.(type) { + case DMSEnvelope: + return struct { + Schema string `json:"schema"` + Channels []wireHistory `json:"channels"` + }{value.Schema, canonicalHistories(value.Channels)}, nil + case GroupDMSEnvelope: + return struct { + Schema string `json:"schema"` + Channels []wireHistory `json:"channels"` + }{value.Schema, canonicalHistories(value.Channels)}, nil + case ChannelEnvelope: + return struct { + Schema string `json:"schema"` + Data wireHistory `json:"data"` + }{value.Schema, canonicalHistory(value.Data)}, nil + case ThreadEnvelope: + return struct { + Schema string `json:"schema"` + Data struct { + Channel MachineChannel `json:"channel"` + Root wireMessage `json:"root"` + Redactions []Redaction `json:"redactions"` + Metadata wireMetadata `json:"metadata"` + } `json:"data"` + }{value.Schema, struct { + Channel MachineChannel `json:"channel"` + Root wireMessage `json:"root"` + Redactions []Redaction `json:"redactions"` + Metadata wireMetadata `json:"metadata"` + }{value.Data.Channel, canonicalMessage(value.Data.Root), cloneSlice(value.Data.Redactions), canonicalMetadata(value.Data.Metadata)}}, nil + case SearchEnvelope: + return struct { + Schema string `json:"schema"` + Results []wireHistory `json:"results"` + }{value.Schema, canonicalHistories(value.Results)}, nil + case MentionsEnvelope: + return struct { + Schema string `json:"schema"` + Results []wireHistory `json:"results"` + }{value.Schema, canonicalHistories(value.Results)}, nil + default: + return nil, fmt.Errorf("unsupported machine document type %T", document) + } +} + +func canonicalHistories(values []MachineHistory) []wireHistory { + result := make([]wireHistory, len(values)) + for index := range values { + result[index] = canonicalHistory(values[index]) + } + return result +} + +func canonicalHistory(value MachineHistory) wireHistory { + messages := make([]wireMessage, len(value.Messages)) + for index := range value.Messages { + messages[index] = canonicalMessage(value.Messages[index]) + } + return wireHistory{value.Channel, messages, cloneSlice(value.Redactions), canonicalMetadata(value.Metadata)} +} + +func canonicalMessage(value MachineMessage) wireMessage { + attachments := make([]wireAttachment, len(value.Attachments)) + for index, attachment := range value.Attachments { + attachments[index] = wireAttachment{attachment.Fallback, attachment.Pretext, attachment.Title, attachment.TitleLink, attachment.Text, cloneSlice(attachment.Fields), attachment.Footer, attachment.FooterIcon, attachment.AuthorName, attachment.AuthorLink, attachment.AuthorIcon, attachment.Color, attachment.ImageURL, attachment.ThumbURL, attachment.Timestamp} + } + reactions := make([]wireReaction, len(value.Reactions)) + for index, reaction := range value.Reactions { + reactions[index] = wireReaction{reaction.Emoji, reaction.Count, cloneSlice(reaction.Actors)} + } + replies := make([]wireMessage, len(value.Replies)) + for index := range value.Replies { + replies[index] = canonicalMessage(value.Replies[index]) + } + return wireMessage{value.ID, value.Permalink, value.User, value.UserID, value.Text, value.Timestamp, value.UpdatedAt, value.EditedAt, value.DeletedAt, value.IsDeleted, value.PostType, value.IsSystem, value.IsPinned, value.RootID, value.ReplyCount, cloneSlice(value.Files), cloneSlice(value.FileDetails), attachments, reactions, replies} +} + +func canonicalMetadata(value MachineMetadata) wireMetadata { + return wireMetadata{value.Completeness, value.Selection, wireVisibleThreads{value.VisibleThreads.Status, value.VisibleThreads.HydratedRootCount, cloneSlice(value.VisibleThreads.FailedRootIDs)}, value.VisiblePostCount, value.DeletedPostsIncluded} +} + +func cloneSlice[T any](values []T) []T { + result := make([]T, len(values)) + copy(result, values) + return result +} + +// WriteMachineJSON encodes one bounded value, appends exactly one newline, and +// performs one write. A short or failed write is returned and never retried. +func WriteMachineJSON(w io.Writer, value MachineDocument) (int, error) { + if err := preflightMachineDocument(value); err != nil { + return 0, err + } + wireValue, err := canonicalMachineDocument(value) + if err != nil { + return 0, err + } + buffer := boundedBuffer{limit: MaxMachineDocumentBytes} + wire := separatorWriter{destination: &buffer} + encoder := json.NewEncoder(&wire) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(wireValue); err != nil { + return 0, err + } + if err := wire.flush(); err != nil { + return 0, err + } + data := buffer.Bytes() + n, err := w.Write(data) + if err == nil && n != len(data) { + err = io.ErrShortWrite + } + return n, err +} + +var errMachineDocumentTooLarge = errors.New("machine JSON document exceeds size limit") + +type boundedBuffer struct { + bytes.Buffer + limit int +} + +type separatorWriter struct { + destination io.Writer + pending [5]byte + pendingLen int + slashRun int +} + +func (writer *separatorWriter) Write(data []byte) (int, error) { + for index, current := range data { + if err := writer.feed(current); err != nil { + return index, err + } + } + return len(data), nil +} + +func (writer *separatorWriter) feed(current byte) error { + if writer.pendingLen < len(writer.pending) { + writer.pending[writer.pendingLen] = current + writer.pendingLen++ + return nil + } + if writer.pending[0] == '\\' && writer.slashRun%2 == 0 && + bytes.Equal(writer.pending[1:], []byte("u202")) && (current == '8' || current == '9') { + separator := []byte("\u2028") + if current == '9' { + separator = []byte("\u2029") + } + if _, err := writer.destination.Write(separator); err != nil { + return err + } + writer.pendingLen = 0 + writer.slashRun = 0 + return nil + } + if err := writer.emit(writer.pending[0]); err != nil { + return err + } + copy(writer.pending[:], writer.pending[1:]) + writer.pending[len(writer.pending)-1] = current + return nil +} + +func (writer *separatorWriter) flush() error { + for index := 0; index < writer.pendingLen; index++ { + if err := writer.emit(writer.pending[index]); err != nil { + return err + } + } + writer.pendingLen = 0 + return nil +} + +func (writer *separatorWriter) emit(current byte) error { + if _, err := writer.destination.Write([]byte{current}); err != nil { + return err + } + if current == '\\' { + writer.slashRun++ + } else { + writer.slashRun = 0 + } + return nil +} + +const ( + machinePreflightMaxValues = 32768 + machinePreflightMaxDepth = 256 +) + +func preflightMachineDocument(document MachineDocument) error { + switch document.(type) { + case DMSEnvelope, GroupDMSEnvelope, ChannelEnvelope, ThreadEnvelope, SearchEnvelope, MentionsEnvelope: + default: + return fmt.Errorf("unsupported machine document type %T", document) + } + contentBudget := int64(MaxMachineDocumentBytes) + valueBudget := machinePreflightMaxValues + stack := make(map[preflightVisit]bool) + if err := consumeMachineBudget(reflect.ValueOf(document), &contentBudget, &valueBudget, stack, 0); err != nil { + return err + } + return nil +} + +type preflightVisit struct { + typeName reflect.Type + pointer uintptr +} + +func consumeMachineBudget(value reflect.Value, contentBudget *int64, valueBudget *int, stack map[preflightVisit]bool, depth int) error { + if depth > machinePreflightMaxDepth { + return fmt.Errorf("machine JSON document exceeds nesting limit") + } + if !value.IsValid() { + return nil + } + *valueBudget-- + if *valueBudget < 0 { + return fmt.Errorf("machine JSON document exceeds complexity limit of %d values", machinePreflightMaxValues) + } + if value.Type() == reflect.TypeFor[time.Time]() || value.Type() == reflect.TypeFor[MillisTime]() { + return nil + } + switch value.Kind() { + case reflect.String: + *contentBudget -= int64(machineJSONStringBytes(value.String())) + case reflect.Pointer, reflect.Interface: + if !value.IsNil() { + return consumeMachineBudget(value.Elem(), contentBudget, valueBudget, stack, depth+1) + } + case reflect.Struct: + for index := 0; index < value.NumField(); index++ { + if err := consumeMachineBudget(value.Field(index), contentBudget, valueBudget, stack, depth+1); err != nil { + return err + } + } + case reflect.Slice: + if value.IsNil() { + break + } + visit := preflightVisit{typeName: value.Type(), pointer: value.Pointer()} + if stack[visit] { + return fmt.Errorf("machine JSON document contains a collection cycle") + } + stack[visit] = true + defer delete(stack, visit) + for index := 0; index < value.Len(); index++ { + if err := consumeMachineBudget(value.Index(index), contentBudget, valueBudget, stack, depth+1); err != nil { + return err + } + } + } + if *contentBudget < 0 { + return fmt.Errorf("%w: %d bytes", errMachineDocumentTooLarge, MaxMachineDocumentBytes) + } + return nil +} + +func machineJSONStringBytes(value string) int { + length := 2 + for index := 0; index < len(value); { + current := value[index] + if current < utf8.RuneSelf { + switch current { + case '\\', '"', '\b', '\f', '\n', '\r', '\t': + length += 2 + default: + if current < 0x20 { + length += 6 + } else { + length++ + } + } + index++ + continue + } + decoded, size := utf8.DecodeRuneInString(value[index:]) + if decoded == utf8.RuneError && size == 1 { + length += 6 + index++ + continue + } + length += size + index += size + } + return length +} + +func (buffer *boundedBuffer) Write(data []byte) (int, error) { + remaining := buffer.limit - buffer.Len() + if len(data) > remaining { + if remaining > 0 { + _, _ = buffer.Buffer.Write(data[:remaining]) + } + return remaining, fmt.Errorf("%w: %d bytes", errMachineDocumentTooLarge, buffer.limit) + } + return buffer.Buffer.Write(data) +} + +func ValidMachineSchema(command, schema string) bool { + return schema == "mm/v2/"+strings.TrimSpace(command) +} diff --git a/internal/output/machine_test.go b/internal/output/machine_test.go new file mode 100644 index 0000000..d0a5b8a --- /dev/null +++ b/internal/output/machine_test.go @@ -0,0 +1,210 @@ +package output + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "strings" + "sync" + "testing" + "time" +) + +func TestWriteMachineJSONWireContract(t *testing.T) { + var output bytes.Buffer + value := ChannelEnvelope{Schema: "mm/v2/channel", Data: MachineHistory{Messages: []MachineMessage{{ + Text: "<>&\u2028\u2029\\u2028", Timestamp: MillisTime{time.Date(2026, 7, 16, 1, 2, 3, 456789000, time.UTC)}, UpdatedAt: MillisTime{time.Date(2026, 7, 16, 1, 2, 3, 456789000, time.UTC)}, + }}}} + if _, err := WriteMachineJSON(&output, value); err != nil { + t.Fatal(err) + } + if !strings.Contains(output.String(), "\"text\":\"<>&\u2028\u2029\\\\u2028\"") || !strings.HasSuffix(output.String(), "\n") { + t.Fatalf("unexpected wire bytes: %q", output.String()) + } +} + +type shortMachineWriter struct{ calls int } + +func (w *shortMachineWriter) Write(p []byte) (int, error) { w.calls++; return len(p) - 1, nil } + +func TestWriteMachineJSONNeverRetriesShortWrite(t *testing.T) { + w := &shortMachineWriter{} + _, err := WriteMachineJSON(w, DMSEnvelope{Schema: "mm/v2/dms"}) + if !errors.Is(err, io.ErrShortWrite) || w.calls != 1 { + t.Fatalf("err=%v calls=%d", err, w.calls) + } +} + +func TestWriteMachineJSONBoundsBeforeWrite(t *testing.T) { + w := &shortMachineWriter{} + _, err := WriteMachineJSON(w, ChannelEnvelope{Schema: "mm/v2/channel", Data: MachineHistory{Messages: []MachineMessage{{Text: strings.Repeat("x", MaxMachineDocumentBytes)}}}}) + if err == nil || w.calls != 0 { + t.Fatalf("err=%v calls=%d", err, w.calls) + } +} + +func TestMachineEmptyCollectionsAreArrays(t *testing.T) { + values := []MachineDocument{ + DMSEnvelope{Schema: "mm/v2/dms"}, + GroupDMSEnvelope{Schema: "mm/v2/group-dms"}, + SearchEnvelope{Schema: "mm/v2/search"}, + MentionsEnvelope{Schema: "mm/v2/mentions"}, + ChannelEnvelope{Schema: "mm/v2/channel", Data: MachineHistory{}}, + ThreadEnvelope{Schema: "mm/v2/thread", Data: ThreadData{}}, + } + for _, value := range values { + var encoded bytes.Buffer + _, err := WriteMachineJSON(&encoded, value) + if err != nil { + t.Fatalf("WriteMachineJSON(%T): %v", value, err) + } + if strings.Contains(encoded.String(), `"channels":null`) || strings.Contains(encoded.String(), `"results":null`) || strings.Contains(encoded.String(), `"messages":null`) || strings.Contains(encoded.String(), `"replies":null`) || strings.Contains(encoded.String(), `"files":null`) || strings.Contains(encoded.String(), `"fields":null`) || strings.Contains(encoded.String(), `"actors":null`) { + t.Fatalf("WriteMachineJSON(%T) emitted a null collection: %s", value, encoded.String()) + } + } +} + +func TestMachineMarshalDoesNotMutateCallerAndIsConcurrentSafe(t *testing.T) { + message := MachineMessage{ + Attachments: []Attachment{{Fields: nil}}, + Reactions: []Reaction{{Emoji: "eyes", Actors: nil}}, + } + var wait sync.WaitGroup + for range 32 { + wait.Add(1) + go func() { + defer wait.Done() + if _, err := WriteMachineJSON(io.Discard, ChannelEnvelope{Data: MachineHistory{Messages: []MachineMessage{message}}}); err != nil { + t.Errorf("WriteMachineJSON: %v", err) + } + }() + } + wait.Wait() + if message.Attachments[0].Fields != nil || message.Reactions[0].Actors != nil { + t.Fatalf("marshal mutated caller: %+v", message) + } +} + +func TestWriteMachineJSONAcceptsLargeASCIIWithinWireLimit(t *testing.T) { + var output bytes.Buffer + text := strings.Repeat("a", 700<<10) + _, err := WriteMachineJSON(&output, ChannelEnvelope{Schema: "mm/v2/channel", Data: MachineHistory{Messages: []MachineMessage{{Text: text}}}}) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(output.Bytes(), []byte(text[:1024])) { + t.Fatal("large ASCII text missing from output") + } +} + +func TestWriteMachineJSONRejectsTypedNilBeforeWrite(t *testing.T) { + var envelope *DMSEnvelope + var document MachineDocument = envelope + writer := &shortMachineWriter{} + if _, err := WriteMachineJSON(writer, document); err == nil || writer.calls != 0 { + t.Fatalf("err=%v calls=%d", err, writer.calls) + } +} + +func TestMachinePreflightSeparatesTinyValueComplexityFromByteSize(t *testing.T) { + withinLimit := make([]string, 1000) + for index := range withinLimit { + withinLimit[index] = "x" + } + if _, err := WriteMachineJSON(io.Discard, ChannelEnvelope{Data: MachineHistory{Messages: []MachineMessage{{Files: withinLimit}}}}); err != nil { + t.Fatalf("small value graph: %v", err) + } + + tooMany := make([]string, machinePreflightMaxValues) + for index := range tooMany { + tooMany[index] = "x" + } + writer := &shortMachineWriter{} + _, err := WriteMachineJSON(writer, ChannelEnvelope{Data: MachineHistory{Messages: []MachineMessage{{Files: tooMany}}}}) + if err == nil || !strings.Contains(err.Error(), "exceeds complexity limit") || errors.Is(err, errMachineDocumentTooLarge) || writer.calls != 0 { + t.Fatalf("err=%v calls=%d", err, writer.calls) + } +} + +func TestWriteMachineJSONHandlesDeepRepliesWithOneCanonicalPass(t *testing.T) { + message := MachineMessage{Text: "leaf"} + for range 100 { + message = MachineMessage{Replies: []MachineMessage{message}} + } + var output bytes.Buffer + if _, err := WriteMachineJSON(&output, ChannelEnvelope{Data: MachineHistory{Messages: []MachineMessage{message}}}); err != nil { + t.Fatal(err) + } + if got := bytes.Count(output.Bytes(), []byte(`"replies"`)); got != 101 { + t.Fatalf("reply arrays=%d, want 101", got) + } +} + +func TestSeparatorWriterRetainsAtMostFiveBytesAcrossLargeChunks(t *testing.T) { + var output bytes.Buffer + writer := separatorWriter{destination: &output} + chunk := []byte(strings.Repeat("x", 1<<20) + `\u2028`) + if _, err := writer.Write(chunk); err != nil { + t.Fatal(err) + } + if writer.pendingLen > 5 { + t.Fatalf("pendingLen=%d", writer.pendingLen) + } + if err := writer.flush(); err != nil { + t.Fatal(err) + } + if !bytes.HasSuffix(output.Bytes(), []byte("\u2028")) { + t.Fatal("line separator was not preserved literally") + } +} + +func TestWriteMachineJSONRejectsRichOversizeAndCyclesBeforeWrite(t *testing.T) { + for name, message := range map[string]MachineMessage{ + "rich oversized": {Attachments: []Attachment{{Text: strings.Repeat("x", MaxMachineDocumentBytes)}}}, + "nested oversized": {Replies: []MachineMessage{{Replies: []MachineMessage{{Text: strings.Repeat("y", MaxMachineDocumentBytes)}}}}}, + } { + writer := &shortMachineWriter{} + _, err := WriteMachineJSON(writer, ChannelEnvelope{Schema: "mm/v2/channel", Data: MachineHistory{Messages: []MachineMessage{message}}}) + if !errors.Is(err, errMachineDocumentTooLarge) || writer.calls != 0 { + t.Errorf("%s: err=%v calls=%d", name, err, writer.calls) + } + } + cycle := make([]MachineMessage, 1) + cycle[0].Replies = cycle + writer := &shortMachineWriter{} + _, err := WriteMachineJSON(writer, ChannelEnvelope{Data: MachineHistory{Messages: cycle}}) + if err == nil || writer.calls != 0 { + t.Fatalf("cycle: err=%v calls=%d", err, writer.calls) + } +} + +func TestMachineMessagePreservesRichAgentFields(t *testing.T) { + message := MachineMessage{ + Files: []string{}, FileDetails: []File{{ID: "f"}}, Attachments: []Attachment{{Fields: nil}}, + Reactions: []Reaction{{Emoji: "eyes", Actors: nil}}, Replies: []MachineMessage{{Files: nil}}, + } + var encoded bytes.Buffer + _, err := WriteMachineJSON(&encoded, ChannelEnvelope{Data: MachineHistory{Messages: []MachineMessage{message}}}) + if err != nil { + t.Fatal(err) + } + for _, field := range []string{`"fileDetails":[`, `"attachments":[`, `"reactions":[`, `"actors":[]`, `"replies":[`, `"files":[]`} { + if !strings.Contains(encoded.String(), field) { + t.Fatalf("missing %s in %s", field, encoded.String()) + } + } +} + +func TestMillisTimeNormalizesZeroOffsetAndRejectsInvalidYears(t *testing.T) { + zeroOffset := time.FixedZone("UTC spelled differently", 0) + encoded, err := json.Marshal(MillisTime{time.Date(2026, 7, 16, 1, 2, 3, 999999999, zeroOffset)}) + if err != nil || string(encoded) != `"2026-07-16T01:02:03.999Z"` { + t.Fatalf("encoded=%s err=%v", encoded, err) + } + for _, value := range []time.Time{time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(2026, 1, 1, 0, 0, 0, 0, time.FixedZone("plus one", 3600))} { + if _, err := json.Marshal(MillisTime{value}); err == nil { + t.Fatalf("accepted %v", value) + } + } +} diff --git a/internal/schema/read_test.go b/internal/schema/read_test.go new file mode 100644 index 0000000..a602d24 --- /dev/null +++ b/internal/schema/read_test.go @@ -0,0 +1,96 @@ +package schema + +import ( + "encoding/json" + "io/fs" + "strings" + "testing" + + jsonschema "github.com/santhosh-tekuri/jsonschema/v6" + + publicschemas "github.com/ardasevinc/mattermost-cli/schemas" +) + +func TestReadSchemasAreRegisteredAndStrict(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + for _, command := range []string{"dms", "group-dms", "channel", "thread", "search", "mentions"} { + id := "mm/v2/" + command + if _, err := registry.Show(id); err != nil { + t.Fatalf("Show(%q): %v", id, err) + } + } + if _, err := registry.Show("mm/v2/read-defs"); err == nil { + t.Fatal("shared definitions resource must not be public") + } + for _, id := range registry.IDs() { + if !strings.HasPrefix(id, "mm/v2/") || id == "mm/v2/error" { + continue + } + raw, err := registry.Show(id) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), `"$ref":"urn:`) { + t.Fatalf("Show(%q) returned a schema with an external reference", id) + } + var document any + if err := json.Unmarshal(raw, &document); err != nil { + t.Fatal(err) + } + compiler := jsonschema.NewCompiler() + compiler.AssertFormat() + if err := compiler.AddResource("standalone.json", document); err != nil { + t.Fatalf("standalone AddResource(%q): %v", id, err) + } + if _, err := compiler.Compile("standalone.json"); err != nil { + t.Fatalf("standalone Compile(%q): %v", id, err) + } + } + + invalid := map[string]string{ + "mm/v2/dms": `{"schema":"mm/v2/dms","channels":[],"unknown":true}`, + "mm/v2/group-dms": `{"schema":"mm/v2/dms","channels":[]}`, + "mm/v2/channel": `{"schema":"mm/v2/search","data":{}}`, + "mm/v2/thread": `{"schema":"mm/v2/thread","data":{"root":null,"replies":[],"metadata":{"completeness":"maybe","nextCursor":null,"queryTruncated":null}}}`, + "mm/v2/search": `{"schema":"mm/v2/search","results":[],"metadata":{"completeness":"complete","nextCursor":null,"queryTruncated":null,"unknown":true}}`, + "mm/v2/mentions": `{"schema":"mm/v2/search","results":[],"metadata":{"completeness":"complete","nextCursor":null,"queryTruncated":null}}`, + } + for id, document := range invalid { + if err := registry.Validate(id, strings.NewReader(document)); err == nil { + t.Errorf("Validate(%q) accepted %s", id, document) + } + } +} + +func TestReadSchemaRejectsNullThreadRootAndInvalidTimestamps(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + valid, err := fs.ReadFile(publicschemas.FS, "v2/examples/thread.json") + if err != nil { + t.Fatal(err) + } + for name, replacement := range map[string]string{ + "null root": `"root":null`, + "impossible date": `"timestamp":"2026-02-30T01:02:03.456Z"`, + "short year": `"timestamp":"026-07-16T01:02:03.456Z"`, + "year zero": `"timestamp":"0000-07-16T01:02:03.456Z"`, + } { + document := string(valid) + switch name { + case "null root": + start := strings.Index(document, `"root":{`) + end := strings.Index(document[start:], `,"redactions":[]`) + document = document[:start] + replacement + document[start+end:] + default: + document = strings.Replace(document, `"timestamp":"2026-07-16T01:02:03.456Z"`, replacement, 1) + } + if err := registry.Validate("mm/v2/thread", strings.NewReader(document)); err == nil { + t.Errorf("accepted %s", name) + } + } +} diff --git a/schemas/v2/channel.schema.json b/schemas/v2/channel.schema.json new file mode 100644 index 0000000..35da4cf --- /dev/null +++ b/schemas/v2/channel.schema.json @@ -0,0 +1 @@ +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"urn:mm:schema:v2:channel","type":"object","additionalProperties":false,"required":["schema","data"],"properties":{"schema":{"const":"mm/v2/channel"},"data":{"$ref":"#/$defs/history"}},"$defs":{"timestamp":{"type":"string","format":"date-time","pattern":"^(?:000[1-9]|00[1-9][0-9]|0[1-9][0-9]{2}|[1-9][0-9]{3})-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\\.[0-9]{3}Z$"},"nullableTimestamp":{"anyOf":[{"$ref":"#/$defs/timestamp"},{"type":"null"}]},"file":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"name":{"type":"string"},"mime":{"type":"string"},"size":{"type":"integer","minimum":0},"extension":{"type":"string"}}},"attachmentField":{"type":"object","additionalProperties":false,"properties":{"title":{"type":"string"},"value":{"type":"string"},"short":{"type":"boolean"}}},"attachment":{"type":"object","additionalProperties":false,"properties":{"fallback":{"type":"string"},"pretext":{"type":"string"},"title":{"type":"string"},"titleLink":{"type":"string"},"text":{"type":"string"},"fields":{"type":"array","items":{"$ref":"#/$defs/attachmentField"}},"footer":{"type":"string"},"footerIcon":{"type":"string"},"authorName":{"type":"string"},"authorLink":{"type":"string"},"authorIcon":{"type":"string"},"color":{"type":"string"},"imageUrl":{"type":"string"},"thumbUrl":{"type":"string"},"timestamp":{"type":"string"}}},"reactionActor":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"username":{"type":"string"}}},"reaction":{"type":"object","additionalProperties":false,"required":["emoji","count","actors"],"properties":{"emoji":{"type":"string"},"count":{"type":"integer","minimum":0},"actors":{"type":"array","items":{"$ref":"#/$defs/reactionActor"}}}},"redaction":{"type":"object","additionalProperties":false,"required":["type","masked","position"],"properties":{"type":{"type":"string"},"masked":{"type":"string"},"position":{"type":"integer","minimum":0},"field":{"type":"string"}}},"channel":{"type":"object","additionalProperties":false,"required":["id","type","name","displayName","metadataStatus"],"properties":{"id":{"type":"string"},"type":{"enum":["dm","public","private","group","unknown"]},"name":{"type":"string"},"displayName":{"type":"string"},"metadataStatus":{"enum":["resolved","unavailable"]}}},"selection":{"type":"object","additionalProperties":false,"required":["source","selectedCount","requestedLimit","since","queryTruncated","inputCursor","nextCursor"],"properties":{"source":{"enum":["recent","search","mentions","unread","thread"]},"selectedCount":{"type":"integer","minimum":0},"requestedLimit":{"type":["integer","null"],"minimum":1},"since":{"type":["string","null"]},"queryTruncated":{"type":["boolean","null"]},"inputCursor":{"type":["string","null"]},"nextCursor":{"type":["string","null"]}}},"visibleThreads":{"type":"object","additionalProperties":false,"required":["status","hydratedRootCount","failedRootIds"],"properties":{"status":{"enum":["not_requested","complete","partial"]},"hydratedRootCount":{"type":"integer","minimum":0},"failedRootIds":{"type":"array","items":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":false,"required":["completeness","selection","visibleThreads","visiblePostCount","deletedPostsIncluded"],"properties":{"completeness":{"enum":["complete","truncated","unknown"]},"selection":{"$ref":"#/$defs/selection"},"visibleThreads":{"$ref":"#/$defs/visibleThreads"},"visiblePostCount":{"type":"integer","minimum":0},"deletedPostsIncluded":{"const":false}}},"message":{"type":"object","additionalProperties":false,"required":["id","permalink","user","userId","text","timestamp","updatedAt","editedAt","deletedAt","isDeleted","postType","isSystem","isPinned","rootId","replyCount","files","fileDetails","attachments","reactions","replies"],"properties":{"id":{"type":"string"},"permalink":{"type":"string"},"user":{"type":"string"},"userId":{"type":"string"},"text":{"type":"string"},"timestamp":{"$ref":"#/$defs/timestamp"},"updatedAt":{"$ref":"#/$defs/timestamp"},"editedAt":{"$ref":"#/$defs/nullableTimestamp"},"deletedAt":{"$ref":"#/$defs/nullableTimestamp"},"isDeleted":{"type":"boolean"},"postType":{"type":"string"},"isSystem":{"type":"boolean"},"isPinned":{"type":"boolean"},"rootId":{"type":["string","null"]},"replyCount":{"type":["integer","null"],"minimum":0},"files":{"type":"array","items":{"type":"string"}},"fileDetails":{"type":"array","items":{"$ref":"#/$defs/file"}},"attachments":{"type":"array","items":{"$ref":"#/$defs/attachment"}},"reactions":{"type":"array","items":{"$ref":"#/$defs/reaction"}},"replies":{"type":"array","items":{"$ref":"#/$defs/message"}}}},"history":{"type":"object","additionalProperties":false,"required":["channel","messages","redactions","metadata"],"properties":{"channel":{"$ref":"#/$defs/channel"},"messages":{"type":"array","items":{"$ref":"#/$defs/message"}},"redactions":{"type":"array","items":{"$ref":"#/$defs/redaction"}},"metadata":{"$ref":"#/$defs/metadata"}}}}} diff --git a/schemas/v2/dms.schema.json b/schemas/v2/dms.schema.json new file mode 100644 index 0000000..763f1b8 --- /dev/null +++ b/schemas/v2/dms.schema.json @@ -0,0 +1 @@ +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"urn:mm:schema:v2:dms","type":"object","additionalProperties":false,"required":["schema","channels"],"properties":{"schema":{"const":"mm/v2/dms"},"channels":{"type":"array","items":{"$ref":"#/$defs/history"}}},"$defs":{"timestamp":{"type":"string","format":"date-time","pattern":"^(?:000[1-9]|00[1-9][0-9]|0[1-9][0-9]{2}|[1-9][0-9]{3})-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\\.[0-9]{3}Z$"},"nullableTimestamp":{"anyOf":[{"$ref":"#/$defs/timestamp"},{"type":"null"}]},"file":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"name":{"type":"string"},"mime":{"type":"string"},"size":{"type":"integer","minimum":0},"extension":{"type":"string"}}},"attachmentField":{"type":"object","additionalProperties":false,"properties":{"title":{"type":"string"},"value":{"type":"string"},"short":{"type":"boolean"}}},"attachment":{"type":"object","additionalProperties":false,"properties":{"fallback":{"type":"string"},"pretext":{"type":"string"},"title":{"type":"string"},"titleLink":{"type":"string"},"text":{"type":"string"},"fields":{"type":"array","items":{"$ref":"#/$defs/attachmentField"}},"footer":{"type":"string"},"footerIcon":{"type":"string"},"authorName":{"type":"string"},"authorLink":{"type":"string"},"authorIcon":{"type":"string"},"color":{"type":"string"},"imageUrl":{"type":"string"},"thumbUrl":{"type":"string"},"timestamp":{"type":"string"}}},"reactionActor":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"username":{"type":"string"}}},"reaction":{"type":"object","additionalProperties":false,"required":["emoji","count","actors"],"properties":{"emoji":{"type":"string"},"count":{"type":"integer","minimum":0},"actors":{"type":"array","items":{"$ref":"#/$defs/reactionActor"}}}},"redaction":{"type":"object","additionalProperties":false,"required":["type","masked","position"],"properties":{"type":{"type":"string"},"masked":{"type":"string"},"position":{"type":"integer","minimum":0},"field":{"type":"string"}}},"channel":{"type":"object","additionalProperties":false,"required":["id","type","name","displayName","metadataStatus"],"properties":{"id":{"type":"string"},"type":{"enum":["dm","public","private","group","unknown"]},"name":{"type":"string"},"displayName":{"type":"string"},"metadataStatus":{"enum":["resolved","unavailable"]}}},"selection":{"type":"object","additionalProperties":false,"required":["source","selectedCount","requestedLimit","since","queryTruncated","inputCursor","nextCursor"],"properties":{"source":{"enum":["recent","search","mentions","unread","thread"]},"selectedCount":{"type":"integer","minimum":0},"requestedLimit":{"type":["integer","null"],"minimum":1},"since":{"type":["string","null"]},"queryTruncated":{"type":["boolean","null"]},"inputCursor":{"type":["string","null"]},"nextCursor":{"type":["string","null"]}}},"visibleThreads":{"type":"object","additionalProperties":false,"required":["status","hydratedRootCount","failedRootIds"],"properties":{"status":{"enum":["not_requested","complete","partial"]},"hydratedRootCount":{"type":"integer","minimum":0},"failedRootIds":{"type":"array","items":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":false,"required":["completeness","selection","visibleThreads","visiblePostCount","deletedPostsIncluded"],"properties":{"completeness":{"enum":["complete","truncated","unknown"]},"selection":{"$ref":"#/$defs/selection"},"visibleThreads":{"$ref":"#/$defs/visibleThreads"},"visiblePostCount":{"type":"integer","minimum":0},"deletedPostsIncluded":{"const":false}}},"message":{"type":"object","additionalProperties":false,"required":["id","permalink","user","userId","text","timestamp","updatedAt","editedAt","deletedAt","isDeleted","postType","isSystem","isPinned","rootId","replyCount","files","fileDetails","attachments","reactions","replies"],"properties":{"id":{"type":"string"},"permalink":{"type":"string"},"user":{"type":"string"},"userId":{"type":"string"},"text":{"type":"string"},"timestamp":{"$ref":"#/$defs/timestamp"},"updatedAt":{"$ref":"#/$defs/timestamp"},"editedAt":{"$ref":"#/$defs/nullableTimestamp"},"deletedAt":{"$ref":"#/$defs/nullableTimestamp"},"isDeleted":{"type":"boolean"},"postType":{"type":"string"},"isSystem":{"type":"boolean"},"isPinned":{"type":"boolean"},"rootId":{"type":["string","null"]},"replyCount":{"type":["integer","null"],"minimum":0},"files":{"type":"array","items":{"type":"string"}},"fileDetails":{"type":"array","items":{"$ref":"#/$defs/file"}},"attachments":{"type":"array","items":{"$ref":"#/$defs/attachment"}},"reactions":{"type":"array","items":{"$ref":"#/$defs/reaction"}},"replies":{"type":"array","items":{"$ref":"#/$defs/message"}}}},"history":{"type":"object","additionalProperties":false,"required":["channel","messages","redactions","metadata"],"properties":{"channel":{"$ref":"#/$defs/channel"},"messages":{"type":"array","items":{"$ref":"#/$defs/message"}},"redactions":{"type":"array","items":{"$ref":"#/$defs/redaction"}},"metadata":{"$ref":"#/$defs/metadata"}}}}} diff --git a/schemas/v2/examples/channel-empty.json b/schemas/v2/examples/channel-empty.json new file mode 100644 index 0000000..def2e8c --- /dev/null +++ b/schemas/v2/examples/channel-empty.json @@ -0,0 +1 @@ +{"schema":"mm/v2/channel","data":{"channel":{"id":"c1","type":"public","name":"town-square","displayName":"Town Square","metadataStatus":"resolved"},"messages":[],"redactions":[],"metadata":{"completeness":"complete","selection":{"source":"recent","selectedCount":0,"requestedLimit":50,"since":null,"queryTruncated":false,"inputCursor":null,"nextCursor":null},"visibleThreads":{"status":"not_requested","hydratedRootCount":0,"failedRootIds":[]},"visiblePostCount":0,"deletedPostsIncluded":false}}} diff --git a/schemas/v2/examples/channel.json b/schemas/v2/examples/channel.json new file mode 100644 index 0000000..639a99a --- /dev/null +++ b/schemas/v2/examples/channel.json @@ -0,0 +1 @@ +{"schema":"mm/v2/channel","data":{"channel":{"id":"c1","type":"public","name":"town-square","displayName":"Town Square","metadataStatus":"resolved"},"messages":[{"id":"p1","permalink":"https://mm.example/team/pl/p1","user":"arda","userId":"u1","text":"hello","timestamp":"2026-07-16T01:02:03.456Z","updatedAt":"2026-07-16T01:02:03.456Z","editedAt":null,"deletedAt":null,"isDeleted":false,"postType":"","isSystem":false,"isPinned":false,"rootId":null,"replyCount":0,"files":[],"fileDetails":[],"attachments":[],"reactions":[],"replies":[]}],"redactions":[],"metadata":{"completeness":"complete","selection":{"source":"recent","selectedCount":1,"requestedLimit":50,"since":null,"queryTruncated":false,"inputCursor":null,"nextCursor":null},"visibleThreads":{"status":"not_requested","hydratedRootCount":0,"failedRootIds":[]},"visiblePostCount":1,"deletedPostsIncluded":false}}} diff --git a/schemas/v2/examples/dms-empty.json b/schemas/v2/examples/dms-empty.json new file mode 100644 index 0000000..21d339c --- /dev/null +++ b/schemas/v2/examples/dms-empty.json @@ -0,0 +1 @@ +{"schema":"mm/v2/dms","channels":[]} diff --git a/schemas/v2/examples/dms.json b/schemas/v2/examples/dms.json new file mode 100644 index 0000000..8787c9f --- /dev/null +++ b/schemas/v2/examples/dms.json @@ -0,0 +1 @@ +{"schema":"mm/v2/dms","channels":[{"channel":{"id":"c1","type":"public","name":"town-square","displayName":"Town Square","metadataStatus":"resolved"},"messages":[],"redactions":[],"metadata":{"completeness":"complete","selection":{"source":"recent","selectedCount":0,"requestedLimit":50,"since":null,"queryTruncated":false,"inputCursor":null,"nextCursor":null},"visibleThreads":{"status":"not_requested","hydratedRootCount":0,"failedRootIds":[]},"visiblePostCount":0,"deletedPostsIncluded":false}}]} diff --git a/schemas/v2/examples/group-dms-empty.json b/schemas/v2/examples/group-dms-empty.json new file mode 100644 index 0000000..d4cbf29 --- /dev/null +++ b/schemas/v2/examples/group-dms-empty.json @@ -0,0 +1 @@ +{"schema":"mm/v2/group-dms","channels":[]} diff --git a/schemas/v2/examples/group-dms.json b/schemas/v2/examples/group-dms.json new file mode 100644 index 0000000..96eecc0 --- /dev/null +++ b/schemas/v2/examples/group-dms.json @@ -0,0 +1 @@ +{"schema":"mm/v2/group-dms","channels":[{"channel":{"id":"g1","type":"group","name":"arda, deniz","displayName":"","metadataStatus":"resolved"},"messages":[],"redactions":[],"metadata":{"completeness":"complete","selection":{"source":"recent","selectedCount":0,"requestedLimit":50,"since":null,"queryTruncated":false,"inputCursor":null,"nextCursor":null},"visibleThreads":{"status":"not_requested","hydratedRootCount":0,"failedRootIds":[]},"visiblePostCount":0,"deletedPostsIncluded":false}}]} diff --git a/schemas/v2/examples/mentions-empty.json b/schemas/v2/examples/mentions-empty.json new file mode 100644 index 0000000..eb2299b --- /dev/null +++ b/schemas/v2/examples/mentions-empty.json @@ -0,0 +1 @@ +{"schema":"mm/v2/mentions","results":[]} diff --git a/schemas/v2/examples/mentions.json b/schemas/v2/examples/mentions.json new file mode 100644 index 0000000..97d4e0e --- /dev/null +++ b/schemas/v2/examples/mentions.json @@ -0,0 +1 @@ +{"schema":"mm/v2/mentions","results":[{"channel":{"id":"c1","type":"public","name":"town-square","displayName":"Town Square","metadataStatus":"resolved"},"messages":[{"id":"p1","permalink":"https://mm.example/team/pl/p1","user":"arda","userId":"u1","text":"hello","timestamp":"2026-07-16T01:02:03.456Z","updatedAt":"2026-07-16T01:02:03.456Z","editedAt":null,"deletedAt":null,"isDeleted":false,"postType":"","isSystem":false,"isPinned":false,"rootId":null,"replyCount":0,"files":[],"fileDetails":[],"attachments":[],"reactions":[],"replies":[]}],"redactions":[],"metadata":{"completeness":"complete","selection":{"source":"mentions","selectedCount":1,"requestedLimit":50,"since":null,"queryTruncated":false,"inputCursor":null,"nextCursor":null},"visibleThreads":{"status":"not_requested","hydratedRootCount":0,"failedRootIds":[]},"visiblePostCount":1,"deletedPostsIncluded":false}}]} diff --git a/schemas/v2/examples/search-empty.json b/schemas/v2/examples/search-empty.json new file mode 100644 index 0000000..5a78cbf --- /dev/null +++ b/schemas/v2/examples/search-empty.json @@ -0,0 +1 @@ +{"schema":"mm/v2/search","results":[]} diff --git a/schemas/v2/examples/search.json b/schemas/v2/examples/search.json new file mode 100644 index 0000000..1bf7c2d --- /dev/null +++ b/schemas/v2/examples/search.json @@ -0,0 +1 @@ +{"schema":"mm/v2/search","results":[{"channel":{"id":"c1","type":"public","name":"town-square","displayName":"Town Square","metadataStatus":"resolved"},"messages":[{"id":"p1","permalink":"https://mm.example/team/pl/p1","user":"arda","userId":"u1","text":"hello","timestamp":"2026-07-16T01:02:03.456Z","updatedAt":"2026-07-16T01:02:03.456Z","editedAt":null,"deletedAt":null,"isDeleted":false,"postType":"","isSystem":false,"isPinned":false,"rootId":null,"replyCount":0,"files":[],"fileDetails":[],"attachments":[],"reactions":[],"replies":[]}],"redactions":[],"metadata":{"completeness":"complete","selection":{"source":"search","selectedCount":1,"requestedLimit":50,"since":null,"queryTruncated":false,"inputCursor":null,"nextCursor":null},"visibleThreads":{"status":"not_requested","hydratedRootCount":0,"failedRootIds":[]},"visiblePostCount":1,"deletedPostsIncluded":false}}]} diff --git a/schemas/v2/examples/thread.json b/schemas/v2/examples/thread.json new file mode 100644 index 0000000..c120286 --- /dev/null +++ b/schemas/v2/examples/thread.json @@ -0,0 +1 @@ +{"schema":"mm/v2/thread","data":{"channel":{"id":"c1","type":"public","name":"town-square","displayName":"Town Square","metadataStatus":"resolved"},"root":{"id":"p1","permalink":"https://mm.example/team/pl/p1","user":"arda","userId":"u1","text":"hello","timestamp":"2026-07-16T01:02:03.456Z","updatedAt":"2026-07-16T01:02:03.456Z","editedAt":null,"deletedAt":null,"isDeleted":false,"postType":"","isSystem":false,"isPinned":false,"rootId":null,"replyCount":0,"files":[],"fileDetails":[],"attachments":[],"reactions":[],"replies":[]},"redactions":[],"metadata":{"completeness":"complete","selection":{"source":"thread","selectedCount":1,"requestedLimit":null,"since":null,"queryTruncated":false,"inputCursor":null,"nextCursor":null},"visibleThreads":{"status":"not_requested","hydratedRootCount":0,"failedRootIds":[]},"visiblePostCount":1,"deletedPostsIncluded":false}}} diff --git a/schemas/v2/group-dms.schema.json b/schemas/v2/group-dms.schema.json new file mode 100644 index 0000000..372a8b0 --- /dev/null +++ b/schemas/v2/group-dms.schema.json @@ -0,0 +1 @@ +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"urn:mm:schema:v2:group-dms","type":"object","additionalProperties":false,"required":["schema","channels"],"properties":{"schema":{"const":"mm/v2/group-dms"},"channels":{"type":"array","items":{"$ref":"#/$defs/history"}}},"$defs":{"timestamp":{"type":"string","format":"date-time","pattern":"^(?:000[1-9]|00[1-9][0-9]|0[1-9][0-9]{2}|[1-9][0-9]{3})-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\\.[0-9]{3}Z$"},"nullableTimestamp":{"anyOf":[{"$ref":"#/$defs/timestamp"},{"type":"null"}]},"file":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"name":{"type":"string"},"mime":{"type":"string"},"size":{"type":"integer","minimum":0},"extension":{"type":"string"}}},"attachmentField":{"type":"object","additionalProperties":false,"properties":{"title":{"type":"string"},"value":{"type":"string"},"short":{"type":"boolean"}}},"attachment":{"type":"object","additionalProperties":false,"properties":{"fallback":{"type":"string"},"pretext":{"type":"string"},"title":{"type":"string"},"titleLink":{"type":"string"},"text":{"type":"string"},"fields":{"type":"array","items":{"$ref":"#/$defs/attachmentField"}},"footer":{"type":"string"},"footerIcon":{"type":"string"},"authorName":{"type":"string"},"authorLink":{"type":"string"},"authorIcon":{"type":"string"},"color":{"type":"string"},"imageUrl":{"type":"string"},"thumbUrl":{"type":"string"},"timestamp":{"type":"string"}}},"reactionActor":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"username":{"type":"string"}}},"reaction":{"type":"object","additionalProperties":false,"required":["emoji","count","actors"],"properties":{"emoji":{"type":"string"},"count":{"type":"integer","minimum":0},"actors":{"type":"array","items":{"$ref":"#/$defs/reactionActor"}}}},"redaction":{"type":"object","additionalProperties":false,"required":["type","masked","position"],"properties":{"type":{"type":"string"},"masked":{"type":"string"},"position":{"type":"integer","minimum":0},"field":{"type":"string"}}},"channel":{"type":"object","additionalProperties":false,"required":["id","type","name","displayName","metadataStatus"],"properties":{"id":{"type":"string"},"type":{"enum":["dm","public","private","group","unknown"]},"name":{"type":"string"},"displayName":{"type":"string"},"metadataStatus":{"enum":["resolved","unavailable"]}}},"selection":{"type":"object","additionalProperties":false,"required":["source","selectedCount","requestedLimit","since","queryTruncated","inputCursor","nextCursor"],"properties":{"source":{"enum":["recent","search","mentions","unread","thread"]},"selectedCount":{"type":"integer","minimum":0},"requestedLimit":{"type":["integer","null"],"minimum":1},"since":{"type":["string","null"]},"queryTruncated":{"type":["boolean","null"]},"inputCursor":{"type":["string","null"]},"nextCursor":{"type":["string","null"]}}},"visibleThreads":{"type":"object","additionalProperties":false,"required":["status","hydratedRootCount","failedRootIds"],"properties":{"status":{"enum":["not_requested","complete","partial"]},"hydratedRootCount":{"type":"integer","minimum":0},"failedRootIds":{"type":"array","items":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":false,"required":["completeness","selection","visibleThreads","visiblePostCount","deletedPostsIncluded"],"properties":{"completeness":{"enum":["complete","truncated","unknown"]},"selection":{"$ref":"#/$defs/selection"},"visibleThreads":{"$ref":"#/$defs/visibleThreads"},"visiblePostCount":{"type":"integer","minimum":0},"deletedPostsIncluded":{"const":false}}},"message":{"type":"object","additionalProperties":false,"required":["id","permalink","user","userId","text","timestamp","updatedAt","editedAt","deletedAt","isDeleted","postType","isSystem","isPinned","rootId","replyCount","files","fileDetails","attachments","reactions","replies"],"properties":{"id":{"type":"string"},"permalink":{"type":"string"},"user":{"type":"string"},"userId":{"type":"string"},"text":{"type":"string"},"timestamp":{"$ref":"#/$defs/timestamp"},"updatedAt":{"$ref":"#/$defs/timestamp"},"editedAt":{"$ref":"#/$defs/nullableTimestamp"},"deletedAt":{"$ref":"#/$defs/nullableTimestamp"},"isDeleted":{"type":"boolean"},"postType":{"type":"string"},"isSystem":{"type":"boolean"},"isPinned":{"type":"boolean"},"rootId":{"type":["string","null"]},"replyCount":{"type":["integer","null"],"minimum":0},"files":{"type":"array","items":{"type":"string"}},"fileDetails":{"type":"array","items":{"$ref":"#/$defs/file"}},"attachments":{"type":"array","items":{"$ref":"#/$defs/attachment"}},"reactions":{"type":"array","items":{"$ref":"#/$defs/reaction"}},"replies":{"type":"array","items":{"$ref":"#/$defs/message"}}}},"history":{"type":"object","additionalProperties":false,"required":["channel","messages","redactions","metadata"],"properties":{"channel":{"$ref":"#/$defs/channel"},"messages":{"type":"array","items":{"$ref":"#/$defs/message"}},"redactions":{"type":"array","items":{"$ref":"#/$defs/redaction"}},"metadata":{"$ref":"#/$defs/metadata"}}}}} diff --git a/schemas/v2/mentions.schema.json b/schemas/v2/mentions.schema.json new file mode 100644 index 0000000..b116532 --- /dev/null +++ b/schemas/v2/mentions.schema.json @@ -0,0 +1 @@ +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"urn:mm:schema:v2:mentions","type":"object","additionalProperties":false,"required":["schema","results"],"properties":{"schema":{"const":"mm/v2/mentions"},"results":{"type":"array","items":{"$ref":"#/$defs/history"}}},"$defs":{"timestamp":{"type":"string","format":"date-time","pattern":"^(?:000[1-9]|00[1-9][0-9]|0[1-9][0-9]{2}|[1-9][0-9]{3})-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\\.[0-9]{3}Z$"},"nullableTimestamp":{"anyOf":[{"$ref":"#/$defs/timestamp"},{"type":"null"}]},"file":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"name":{"type":"string"},"mime":{"type":"string"},"size":{"type":"integer","minimum":0},"extension":{"type":"string"}}},"attachmentField":{"type":"object","additionalProperties":false,"properties":{"title":{"type":"string"},"value":{"type":"string"},"short":{"type":"boolean"}}},"attachment":{"type":"object","additionalProperties":false,"properties":{"fallback":{"type":"string"},"pretext":{"type":"string"},"title":{"type":"string"},"titleLink":{"type":"string"},"text":{"type":"string"},"fields":{"type":"array","items":{"$ref":"#/$defs/attachmentField"}},"footer":{"type":"string"},"footerIcon":{"type":"string"},"authorName":{"type":"string"},"authorLink":{"type":"string"},"authorIcon":{"type":"string"},"color":{"type":"string"},"imageUrl":{"type":"string"},"thumbUrl":{"type":"string"},"timestamp":{"type":"string"}}},"reactionActor":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"username":{"type":"string"}}},"reaction":{"type":"object","additionalProperties":false,"required":["emoji","count","actors"],"properties":{"emoji":{"type":"string"},"count":{"type":"integer","minimum":0},"actors":{"type":"array","items":{"$ref":"#/$defs/reactionActor"}}}},"redaction":{"type":"object","additionalProperties":false,"required":["type","masked","position"],"properties":{"type":{"type":"string"},"masked":{"type":"string"},"position":{"type":"integer","minimum":0},"field":{"type":"string"}}},"channel":{"type":"object","additionalProperties":false,"required":["id","type","name","displayName","metadataStatus"],"properties":{"id":{"type":"string"},"type":{"enum":["dm","public","private","group","unknown"]},"name":{"type":"string"},"displayName":{"type":"string"},"metadataStatus":{"enum":["resolved","unavailable"]}}},"selection":{"type":"object","additionalProperties":false,"required":["source","selectedCount","requestedLimit","since","queryTruncated","inputCursor","nextCursor"],"properties":{"source":{"enum":["recent","search","mentions","unread","thread"]},"selectedCount":{"type":"integer","minimum":0},"requestedLimit":{"type":["integer","null"],"minimum":1},"since":{"type":["string","null"]},"queryTruncated":{"type":["boolean","null"]},"inputCursor":{"type":["string","null"]},"nextCursor":{"type":["string","null"]}}},"visibleThreads":{"type":"object","additionalProperties":false,"required":["status","hydratedRootCount","failedRootIds"],"properties":{"status":{"enum":["not_requested","complete","partial"]},"hydratedRootCount":{"type":"integer","minimum":0},"failedRootIds":{"type":"array","items":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":false,"required":["completeness","selection","visibleThreads","visiblePostCount","deletedPostsIncluded"],"properties":{"completeness":{"enum":["complete","truncated","unknown"]},"selection":{"$ref":"#/$defs/selection"},"visibleThreads":{"$ref":"#/$defs/visibleThreads"},"visiblePostCount":{"type":"integer","minimum":0},"deletedPostsIncluded":{"const":false}}},"message":{"type":"object","additionalProperties":false,"required":["id","permalink","user","userId","text","timestamp","updatedAt","editedAt","deletedAt","isDeleted","postType","isSystem","isPinned","rootId","replyCount","files","fileDetails","attachments","reactions","replies"],"properties":{"id":{"type":"string"},"permalink":{"type":"string"},"user":{"type":"string"},"userId":{"type":"string"},"text":{"type":"string"},"timestamp":{"$ref":"#/$defs/timestamp"},"updatedAt":{"$ref":"#/$defs/timestamp"},"editedAt":{"$ref":"#/$defs/nullableTimestamp"},"deletedAt":{"$ref":"#/$defs/nullableTimestamp"},"isDeleted":{"type":"boolean"},"postType":{"type":"string"},"isSystem":{"type":"boolean"},"isPinned":{"type":"boolean"},"rootId":{"type":["string","null"]},"replyCount":{"type":["integer","null"],"minimum":0},"files":{"type":"array","items":{"type":"string"}},"fileDetails":{"type":"array","items":{"$ref":"#/$defs/file"}},"attachments":{"type":"array","items":{"$ref":"#/$defs/attachment"}},"reactions":{"type":"array","items":{"$ref":"#/$defs/reaction"}},"replies":{"type":"array","items":{"$ref":"#/$defs/message"}}}},"history":{"type":"object","additionalProperties":false,"required":["channel","messages","redactions","metadata"],"properties":{"channel":{"$ref":"#/$defs/channel"},"messages":{"type":"array","items":{"$ref":"#/$defs/message"}},"redactions":{"type":"array","items":{"$ref":"#/$defs/redaction"}},"metadata":{"$ref":"#/$defs/metadata"}}}}} diff --git a/schemas/v2/search.schema.json b/schemas/v2/search.schema.json new file mode 100644 index 0000000..89c54f4 --- /dev/null +++ b/schemas/v2/search.schema.json @@ -0,0 +1 @@ +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"urn:mm:schema:v2:search","type":"object","additionalProperties":false,"required":["schema","results"],"properties":{"schema":{"const":"mm/v2/search"},"results":{"type":"array","items":{"$ref":"#/$defs/history"}}},"$defs":{"timestamp":{"type":"string","format":"date-time","pattern":"^(?:000[1-9]|00[1-9][0-9]|0[1-9][0-9]{2}|[1-9][0-9]{3})-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\\.[0-9]{3}Z$"},"nullableTimestamp":{"anyOf":[{"$ref":"#/$defs/timestamp"},{"type":"null"}]},"file":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"name":{"type":"string"},"mime":{"type":"string"},"size":{"type":"integer","minimum":0},"extension":{"type":"string"}}},"attachmentField":{"type":"object","additionalProperties":false,"properties":{"title":{"type":"string"},"value":{"type":"string"},"short":{"type":"boolean"}}},"attachment":{"type":"object","additionalProperties":false,"properties":{"fallback":{"type":"string"},"pretext":{"type":"string"},"title":{"type":"string"},"titleLink":{"type":"string"},"text":{"type":"string"},"fields":{"type":"array","items":{"$ref":"#/$defs/attachmentField"}},"footer":{"type":"string"},"footerIcon":{"type":"string"},"authorName":{"type":"string"},"authorLink":{"type":"string"},"authorIcon":{"type":"string"},"color":{"type":"string"},"imageUrl":{"type":"string"},"thumbUrl":{"type":"string"},"timestamp":{"type":"string"}}},"reactionActor":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"username":{"type":"string"}}},"reaction":{"type":"object","additionalProperties":false,"required":["emoji","count","actors"],"properties":{"emoji":{"type":"string"},"count":{"type":"integer","minimum":0},"actors":{"type":"array","items":{"$ref":"#/$defs/reactionActor"}}}},"redaction":{"type":"object","additionalProperties":false,"required":["type","masked","position"],"properties":{"type":{"type":"string"},"masked":{"type":"string"},"position":{"type":"integer","minimum":0},"field":{"type":"string"}}},"channel":{"type":"object","additionalProperties":false,"required":["id","type","name","displayName","metadataStatus"],"properties":{"id":{"type":"string"},"type":{"enum":["dm","public","private","group","unknown"]},"name":{"type":"string"},"displayName":{"type":"string"},"metadataStatus":{"enum":["resolved","unavailable"]}}},"selection":{"type":"object","additionalProperties":false,"required":["source","selectedCount","requestedLimit","since","queryTruncated","inputCursor","nextCursor"],"properties":{"source":{"enum":["recent","search","mentions","unread","thread"]},"selectedCount":{"type":"integer","minimum":0},"requestedLimit":{"type":["integer","null"],"minimum":1},"since":{"type":["string","null"]},"queryTruncated":{"type":["boolean","null"]},"inputCursor":{"type":["string","null"]},"nextCursor":{"type":["string","null"]}}},"visibleThreads":{"type":"object","additionalProperties":false,"required":["status","hydratedRootCount","failedRootIds"],"properties":{"status":{"enum":["not_requested","complete","partial"]},"hydratedRootCount":{"type":"integer","minimum":0},"failedRootIds":{"type":"array","items":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":false,"required":["completeness","selection","visibleThreads","visiblePostCount","deletedPostsIncluded"],"properties":{"completeness":{"enum":["complete","truncated","unknown"]},"selection":{"$ref":"#/$defs/selection"},"visibleThreads":{"$ref":"#/$defs/visibleThreads"},"visiblePostCount":{"type":"integer","minimum":0},"deletedPostsIncluded":{"const":false}}},"message":{"type":"object","additionalProperties":false,"required":["id","permalink","user","userId","text","timestamp","updatedAt","editedAt","deletedAt","isDeleted","postType","isSystem","isPinned","rootId","replyCount","files","fileDetails","attachments","reactions","replies"],"properties":{"id":{"type":"string"},"permalink":{"type":"string"},"user":{"type":"string"},"userId":{"type":"string"},"text":{"type":"string"},"timestamp":{"$ref":"#/$defs/timestamp"},"updatedAt":{"$ref":"#/$defs/timestamp"},"editedAt":{"$ref":"#/$defs/nullableTimestamp"},"deletedAt":{"$ref":"#/$defs/nullableTimestamp"},"isDeleted":{"type":"boolean"},"postType":{"type":"string"},"isSystem":{"type":"boolean"},"isPinned":{"type":"boolean"},"rootId":{"type":["string","null"]},"replyCount":{"type":["integer","null"],"minimum":0},"files":{"type":"array","items":{"type":"string"}},"fileDetails":{"type":"array","items":{"$ref":"#/$defs/file"}},"attachments":{"type":"array","items":{"$ref":"#/$defs/attachment"}},"reactions":{"type":"array","items":{"$ref":"#/$defs/reaction"}},"replies":{"type":"array","items":{"$ref":"#/$defs/message"}}}},"history":{"type":"object","additionalProperties":false,"required":["channel","messages","redactions","metadata"],"properties":{"channel":{"$ref":"#/$defs/channel"},"messages":{"type":"array","items":{"$ref":"#/$defs/message"}},"redactions":{"type":"array","items":{"$ref":"#/$defs/redaction"}},"metadata":{"$ref":"#/$defs/metadata"}}}}} diff --git a/schemas/v2/thread.schema.json b/schemas/v2/thread.schema.json new file mode 100644 index 0000000..8f7060c --- /dev/null +++ b/schemas/v2/thread.schema.json @@ -0,0 +1 @@ +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"urn:mm:schema:v2:thread","type":"object","additionalProperties":false,"required":["schema","data"],"properties":{"schema":{"const":"mm/v2/thread"},"data":{"type":"object","additionalProperties":false,"required":["channel","root","redactions","metadata"],"properties":{"channel":{"$ref":"#/$defs/channel"},"root":{"$ref":"#/$defs/message"},"redactions":{"type":"array","items":{"$ref":"#/$defs/redaction"}},"metadata":{"$ref":"#/$defs/metadata"}}}},"$defs":{"timestamp":{"type":"string","format":"date-time","pattern":"^(?:000[1-9]|00[1-9][0-9]|0[1-9][0-9]{2}|[1-9][0-9]{3})-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\\.[0-9]{3}Z$"},"nullableTimestamp":{"anyOf":[{"$ref":"#/$defs/timestamp"},{"type":"null"}]},"file":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"name":{"type":"string"},"mime":{"type":"string"},"size":{"type":"integer","minimum":0},"extension":{"type":"string"}}},"attachmentField":{"type":"object","additionalProperties":false,"properties":{"title":{"type":"string"},"value":{"type":"string"},"short":{"type":"boolean"}}},"attachment":{"type":"object","additionalProperties":false,"properties":{"fallback":{"type":"string"},"pretext":{"type":"string"},"title":{"type":"string"},"titleLink":{"type":"string"},"text":{"type":"string"},"fields":{"type":"array","items":{"$ref":"#/$defs/attachmentField"}},"footer":{"type":"string"},"footerIcon":{"type":"string"},"authorName":{"type":"string"},"authorLink":{"type":"string"},"authorIcon":{"type":"string"},"color":{"type":"string"},"imageUrl":{"type":"string"},"thumbUrl":{"type":"string"},"timestamp":{"type":"string"}}},"reactionActor":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"username":{"type":"string"}}},"reaction":{"type":"object","additionalProperties":false,"required":["emoji","count","actors"],"properties":{"emoji":{"type":"string"},"count":{"type":"integer","minimum":0},"actors":{"type":"array","items":{"$ref":"#/$defs/reactionActor"}}}},"redaction":{"type":"object","additionalProperties":false,"required":["type","masked","position"],"properties":{"type":{"type":"string"},"masked":{"type":"string"},"position":{"type":"integer","minimum":0},"field":{"type":"string"}}},"channel":{"type":"object","additionalProperties":false,"required":["id","type","name","displayName","metadataStatus"],"properties":{"id":{"type":"string"},"type":{"enum":["dm","public","private","group","unknown"]},"name":{"type":"string"},"displayName":{"type":"string"},"metadataStatus":{"enum":["resolved","unavailable"]}}},"selection":{"type":"object","additionalProperties":false,"required":["source","selectedCount","requestedLimit","since","queryTruncated","inputCursor","nextCursor"],"properties":{"source":{"enum":["recent","search","mentions","unread","thread"]},"selectedCount":{"type":"integer","minimum":0},"requestedLimit":{"type":["integer","null"],"minimum":1},"since":{"type":["string","null"]},"queryTruncated":{"type":["boolean","null"]},"inputCursor":{"type":["string","null"]},"nextCursor":{"type":["string","null"]}}},"visibleThreads":{"type":"object","additionalProperties":false,"required":["status","hydratedRootCount","failedRootIds"],"properties":{"status":{"enum":["not_requested","complete","partial"]},"hydratedRootCount":{"type":"integer","minimum":0},"failedRootIds":{"type":"array","items":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":false,"required":["completeness","selection","visibleThreads","visiblePostCount","deletedPostsIncluded"],"properties":{"completeness":{"enum":["complete","truncated","unknown"]},"selection":{"$ref":"#/$defs/selection"},"visibleThreads":{"$ref":"#/$defs/visibleThreads"},"visiblePostCount":{"type":"integer","minimum":0},"deletedPostsIncluded":{"const":false}}},"message":{"type":"object","additionalProperties":false,"required":["id","permalink","user","userId","text","timestamp","updatedAt","editedAt","deletedAt","isDeleted","postType","isSystem","isPinned","rootId","replyCount","files","fileDetails","attachments","reactions","replies"],"properties":{"id":{"type":"string"},"permalink":{"type":"string"},"user":{"type":"string"},"userId":{"type":"string"},"text":{"type":"string"},"timestamp":{"$ref":"#/$defs/timestamp"},"updatedAt":{"$ref":"#/$defs/timestamp"},"editedAt":{"$ref":"#/$defs/nullableTimestamp"},"deletedAt":{"$ref":"#/$defs/nullableTimestamp"},"isDeleted":{"type":"boolean"},"postType":{"type":"string"},"isSystem":{"type":"boolean"},"isPinned":{"type":"boolean"},"rootId":{"type":["string","null"]},"replyCount":{"type":["integer","null"],"minimum":0},"files":{"type":"array","items":{"type":"string"}},"fileDetails":{"type":"array","items":{"$ref":"#/$defs/file"}},"attachments":{"type":"array","items":{"$ref":"#/$defs/attachment"}},"reactions":{"type":"array","items":{"$ref":"#/$defs/reaction"}},"replies":{"type":"array","items":{"$ref":"#/$defs/message"}}}},"history":{"type":"object","additionalProperties":false,"required":["channel","messages","redactions","metadata"],"properties":{"channel":{"$ref":"#/$defs/channel"},"messages":{"type":"array","items":{"$ref":"#/$defs/message"}},"redactions":{"type":"array","items":{"$ref":"#/$defs/redaction"}},"metadata":{"$ref":"#/$defs/metadata"}}}}} From 15f65aae63911f8356b3620a9345ca8a86a2afed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 12:21:41 +0300 Subject: [PATCH 027/119] feat: normalize rich post presentation --- internal/normalization/post.go | 280 +++++++++++++++++++++++ internal/normalization/post_test.go | 340 ++++++++++++++++++++++++++++ 2 files changed, 620 insertions(+) create mode 100644 internal/normalization/post.go create mode 100644 internal/normalization/post_test.go diff --git a/internal/normalization/post.go b/internal/normalization/post.go new file mode 100644 index 0000000..43e9f24 --- /dev/null +++ b/internal/normalization/post.go @@ -0,0 +1,280 @@ +// Package normalization converts validated Mattermost data into safe output models. +package normalization + +import ( + "sort" + "strings" + "time" + "unicode/utf16" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/output" + "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/internal/serverurl" +) + +const deletedPostText = "[deleted post]" + +// PostUserIDs returns unique author and reaction actor IDs in encounter order. +func PostUserIDs(posts []mattermost.Post) []string { + seen := make(map[string]struct{}) + ids := make([]string, 0) + add := func(id string) { + if id == "" { + return + } + if _, ok := seen[id]; ok { + return + } + seen[id] = struct{}{} + ids = append(ids, id) + } + for _, post := range posts { + add(post.UserID) + for _, reaction := range post.Reactions { + add(reaction.UserID) + } + } + return ids +} + +// NormalizePosts safely prepares validated posts for presentation. +func NormalizePosts(posts []mattermost.Post, users map[string]mattermost.User, myUserID, serverURL string, options presentation.Options) ([]output.Message, []output.Redaction, error) { + redactions := make([]output.Redaction, 0) + clean := func(value, field string, label bool) string { + result := presentation.PreprocessWithOptions(value, options) + if label { + remapLabelRedactionPositions(result.Text, result.Redactions) + result.Text = presentation.SanitizeLabel(result.Text) + } + for _, item := range result.Redactions { + item.Field = field + redactions = append(redactions, item) + } + return result.Text + } + messages := make([]output.Message, 0, len(posts)) + for _, post := range posts { + permalink, err := serverurl.BuildPostPermalink(serverURL, post.ID) + if err != nil { + return nil, nil, err + } + deleted := post.DeleteAt > 0 + username := post.UserID + if user, ok := users[post.UserID]; ok { + username = user.Username + } + displayUser := "" + switch { + case post.OverrideUsername != "": + displayUser = clean(post.OverrideUsername, "user", true) + case post.UserID == "" || strings.HasPrefix(post.Type, "system_"): + displayUser = "system" + case post.UserID == myUserID: + displayUser = "you" + default: + displayUser = clean(username, "user", true) + } + + files, details := normalizeFiles(post, deleted, clean) + attachments := normalizeAttachments(post, deleted, clean) + reactions := normalizeReactions(post, users, deleted, clean) + messageText := deletedPostText + if !deleted { + messageText = clean(post.Message, "post.message", false) + } + message := output.Message{ + ID: clean(post.ID, "post.id", true), Permalink: clean(permalink, "post.permalink", true), + User: displayUser, UserID: clean(post.UserID, "post.userId", true), Text: messageText, + Timestamp: time.UnixMilli(post.CreateAt).UTC(), UpdatedAt: time.UnixMilli(post.UpdateAt).UTC(), + IsDeleted: deleted, PostType: clean(post.Type, "post.type", true), + IsSystem: post.UserID == "" || strings.HasPrefix(post.Type, "system_"), IsPinned: post.IsPinned, + Files: files, FileDetails: details, Attachments: attachments, Reactions: reactions, + CanonicalID: post.ID, CanonicalRootID: post.RootID, + } + if post.EditAt > 0 { + value := time.UnixMilli(post.EditAt).UTC() + message.EditedAt = &value + } + if post.DeleteAt > 0 { + value := time.UnixMilli(post.DeleteAt).UTC() + message.DeletedAt = &value + } + if post.RootID != "" { + message.RootID = clean(post.RootID, "post.rootId", true) + } + if post.ReplyCount > 0 { + value := post.ReplyCount + message.ReplyCount = &value + } + messages = append(messages, message) + } + return messages, redactions, nil +} + +type cleaner func(string, string, bool) string + +func normalizeFiles(post mattermost.Post, deleted bool, clean cleaner) ([]string, []output.File) { + if deleted { + return []string{}, []output.File{} + } + metadata := make(map[string]mattermost.PostFile, len(post.Files)) + for _, file := range post.Files { + if file.ID != "" { + metadata[file.ID] = file + } + } + ids := make([]string, 0, len(post.FileIDs)+len(post.Files)) + seen := make(map[string]struct{}) + add := func(id string) { + if id != "" { + if _, ok := seen[id]; !ok { + seen[id] = struct{}{} + ids = append(ids, id) + } + } + } + for _, id := range post.FileIDs { + add(id) + } + for _, file := range post.Files { + add(file.ID) + } + visible := make([]string, 0, len(ids)) + details := make([]output.File, 0, len(ids)) + for _, id := range ids { + visible = append(visible, clean(id, "file.id", true)) + source := metadata[id] + item := output.File{ID: clean(id, "file.id", true)} + if source.Name != "" { + item.Name = clean(source.Name, "file.name", true) + } + if source.MIMEType != "" { + item.MIME = clean(source.MIMEType, "file.mime", true) + } + if source.Size != nil { + value := *source.Size + item.Size = &value + } + if source.Extension != "" { + item.Extension = clean(source.Extension, "file.extension", true) + } + details = append(details, item) + } + return visible, details +} + +func normalizeAttachments(post mattermost.Post, deleted bool, clean cleaner) []output.Attachment { + result := make([]output.Attachment, 0) + if deleted { + return result + } + for i, source := range post.Attachments { + field := func(name string) string { return "attachment." + itoa(i) + "." + name } + item := output.Attachment{ + Fallback: cleanOptional(source.Fallback, field("fallback"), false, clean), Pretext: cleanOptional(source.Pretext, field("pretext"), false, clean), + Title: cleanOptional(source.Title, field("title"), false, clean), TitleLink: cleanOptional(source.TitleLink, field("title_link"), true, clean), + Text: cleanOptional(source.Text, field("text"), false, clean), Footer: cleanOptional(source.Footer, field("footer"), false, clean), + FooterIcon: cleanOptional(source.FooterIcon, field("footer_icon"), true, clean), AuthorName: cleanOptional(source.AuthorName, field("author_name"), false, clean), + AuthorLink: cleanOptional(source.AuthorLink, field("author_link"), true, clean), AuthorIcon: cleanOptional(source.AuthorIcon, field("author_icon"), true, clean), + Color: cleanOptional(source.Color, field("color"), true, clean), ImageURL: cleanOptional(source.ImageURL, field("image_url"), true, clean), + ThumbURL: cleanOptional(source.ThumbURL, field("thumb_url"), true, clean), Timestamp: cleanOptional(source.Timestamp, field("ts"), true, clean), + } + for j, sourceField := range source.Fields { + prefix := field("fields." + itoa(j)) + f := output.AttachmentField{Title: cleanOptional(sourceField.Title, prefix+".title", false, clean), Value: cleanOptional(sourceField.Value, prefix+".value", false, clean)} + if sourceField.Short != nil { + value := *sourceField.Short + f.Short = &value + } + if f.Title != "" || f.Value != "" || f.Short != nil { + item.Fields = append(item.Fields, f) + } + } + result = append(result, item) + } + return result +} + +func normalizeReactions(post mattermost.Post, users map[string]mattermost.User, deleted bool, clean cleaner) []output.Reaction { + result := make([]output.Reaction, 0) + if deleted { + return result + } + groups := make(map[string][]mattermost.PostReaction) + for _, reaction := range post.Reactions { + if reaction.EmojiName != "" && reaction.UserID != "" { + groups[reaction.EmojiName] = append(groups[reaction.EmojiName], reaction) + } + } + emojis := make([]string, 0, len(groups)) + for emoji := range groups { + emojis = append(emojis, emoji) + } + sort.Strings(emojis) + for _, emoji := range emojis { + entries := groups[emoji] + sort.SliceStable(entries, func(i, j int) bool { return entries[i].UserID < entries[j].UserID }) + actors := make([]output.ReactionActor, 0, len(entries)) + for _, entry := range entries { + actor := output.ReactionActor{ID: clean(entry.UserID, "reaction.actor.id", true)} + if user, ok := users[entry.UserID]; ok && user.Username != "" { + actor.Username = clean(user.Username, "reaction.actor.username", true) + } + actors = append(actors, actor) + } + result = append(result, output.Reaction{Emoji: clean(emoji, "reaction.emoji", true), Count: len(entries), Actors: actors}) + } + return result +} + +func cleanOptional(value, field string, label bool, clean cleaner) string { + if value == "" { + return "" + } + return clean(value, field, label) +} + +func remapLabelRedactionPositions(text string, redactions []presentation.Redaction) { + next := 0 + originalPosition := 0 + expansion := 0 + applyThrough := func(position int) { + for next < len(redactions) && redactions[next].Position <= position { + redactions[next].Position += expansion + next++ + } + } + applyThrough(0) + for _, character := range text { + width := 1 + if character > 0xffff { + width = 2 + } + originalPosition += width + if character == '\n' || character == '\t' { + expansion++ + } + applyThrough(originalPosition) + } + for next < len(redactions) { + redactions[next].Position += expansion + next++ + } +} + +func utf16Length(value string) int { return len(utf16.Encode([]rune(value))) } + +func itoa(value int) string { + if value == 0 { + return "0" + } + var digits [20]byte + cursor := len(digits) + for value > 0 { + cursor-- + digits[cursor] = byte('0' + value%10) + value /= 10 + } + return string(digits[cursor:]) +} diff --git a/internal/normalization/post_test.go b/internal/normalization/post_test.go new file mode 100644 index 0000000..c63b7ec --- /dev/null +++ b/internal/normalization/post_test.go @@ -0,0 +1,340 @@ +package normalization + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + "time" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/presentation" +) + +const token = "ghp_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +func basePost() mattermost.Post { + return mattermost.Post{ID: "post-1", ChannelID: "channel", UserID: "author", Message: "latest visible text", CreateAt: 1000, UpdateAt: 3000, EditAt: 2000} +} + +func normalizeOne(t *testing.T, post mattermost.Post, options presentation.Options) ([]byte, int) { + t.Helper() + users := map[string]mattermost.User{"author": {ID: "author", Username: "alice"}, "reactor": {ID: "reactor", Username: "bob"}} + messages, redactions, err := NormalizePosts([]mattermost.Post{post}, users, "me", "https://mattermost.test", options) + if err != nil { + t.Fatal(err) + } + encoded, err := json.Marshal(messages[0]) + if err != nil { + t.Fatal(err) + } + return encoded, len(redactions) +} + +func TestNormalizePosts(t *testing.T) { + t.Run("latest visible edited state and long markdown exactness", func(t *testing.T) { + post := basePost() + post.Message = strings.Repeat("# heading\n\n- exact `markdown`\n", 600) + messages, _, err := NormalizePosts([]mattermost.Post{post}, nil, "me", "https://mattermost.test", presentation.Options{}) + if err != nil { + t.Fatal(err) + } + got := messages[0] + if got.Text != post.Message || !got.UpdatedAt.Equal(time.UnixMilli(3000)) || got.EditedAt == nil || !got.EditedAt.Equal(time.UnixMilli(2000)) || got.IsDeleted { + t.Fatalf("unexpected message: %#v", got) + } + }) + + t.Run("deleted post never leaks stale nested data", func(t *testing.T) { + post := basePost() + post.DeleteAt = 4000 + post.Message = "stale " + token + post.FileIDs = []string{"secret-file"} + post.Files = []mattermost.PostFile{{ID: "secret-file", Name: "stale-" + token}} + post.Attachments = []mattermost.PostAttachment{{Text: "stale " + token}} + post.Reactions = []mattermost.PostReaction{{UserID: "reactor", EmojiName: "yes"}} + encoded, _ := normalizeOne(t, post, presentation.Options{}) + got := string(encoded) + if strings.Contains(got, token) || !strings.Contains(got, `"text":"[deleted post]"`) || !strings.Contains(got, `"files":[]`) || !strings.Contains(got, `"attachments":[]`) || !strings.Contains(got, `"reactions":[]`) { + t.Fatalf("deleted output = %s", got) + } + }) + + t.Run("author precedence", func(t *testing.T) { + cases := []struct { + name string + post mattermost.Post + myID, want string + }{ + {"override", func() mattermost.Post { p := basePost(); p.OverrideUsername = "hook"; p.Type = "system_x"; return p }(), "author", "hook"}, + {"system", func() mattermost.Post { p := basePost(); p.Type = "system_x"; return p }(), "author", "system"}, + {"you", basePost(), "author", "you"}, {"username", basePost(), "me", "alice"}, + {"missing user id", func() mattermost.Post { p := basePost(); p.UserID = "missing"; return p }(), "me", "missing"}, + } + users := map[string]mattermost.User{"author": {ID: "author", Username: "alice"}} + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + messages, _, err := NormalizePosts([]mattermost.Post{tc.post}, users, tc.myID, "https://mattermost.test", presentation.Options{}) + if err != nil { + t.Fatal(err) + } + if messages[0].User != tc.want { + t.Fatalf("user = %q, want %q", messages[0].User, tc.want) + } + }) + } + }) + + t.Run("file id and metadata dedupe with nested metadata", func(t *testing.T) { + size := int64(12) + post := basePost() + post.FileIDs = []string{"a", "a"} + post.Files = []mattermost.PostFile{{ID: "a", Name: "old"}, {ID: "a", Name: "new", MIMEType: "text/plain", Size: &size}, {ID: "b", Extension: "txt"}} + messages, _, err := NormalizePosts([]mattermost.Post{post}, nil, "", "https://mattermost.test", presentation.Options{}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(messages[0].Files, []string{"a", "b"}) || len(messages[0].FileDetails) != 2 || messages[0].FileDetails[0].Name != "new" || messages[0].FileDetails[0].Size == nil || *messages[0].FileDetails[0].Size != 12 { + t.Fatalf("files = %#v / %#v", messages[0].Files, messages[0].FileDetails) + } + }) + + t.Run("rich attachments preserve numeric-derived strings", func(t *testing.T) { + short := true + post := basePost() + post.Attachments = []mattermost.PostAttachment{{Pretext: "before", TitleLink: "https://x.test/a\n", Fields: []mattermost.PostAttachmentField{{Title: "7", Value: "9", Short: &short}}, Timestamp: "123"}} + messages, _, err := NormalizePosts([]mattermost.Post{post}, nil, "", "https://mattermost.test", presentation.Options{}) + if err != nil { + t.Fatal(err) + } + got := messages[0].Attachments[0] + if got.Pretext != "before" || got.TitleLink != "https://x.test/a\\n" || got.Timestamp != "123" || len(got.Fields) != 1 || got.Fields[0].Title != "7" || got.Fields[0].Short == nil || !*got.Fields[0].Short { + t.Fatalf("attachment = %#v", got) + } + }) + + t.Run("reactions group by raw emoji and sort actors by raw id", func(t *testing.T) { + first := "ghp_a" + strings.Repeat("x", 35) + second := "ghp_b" + strings.Repeat("x", 35) + post := basePost() + post.Reactions = []mattermost.PostReaction{{UserID: "reactor", EmojiName: second}, {UserID: "missing", EmojiName: first}, {UserID: "reactor", EmojiName: first}} + messages, _, err := NormalizePosts([]mattermost.Post{post}, map[string]mattermost.User{"reactor": {Username: "bob"}}, "", "https://mattermost.test", presentation.Options{}) + if err != nil { + t.Fatal(err) + } + got := messages[0].Reactions + if len(got) != 2 || got[0].Count != 2 || got[1].Count != 1 || got[0].Actors[0].ID != "missing" || got[0].Actors[1].Username != "bob" || got[0].Emoji != got[1].Emoji { + t.Fatalf("reactions = %#v", got) + } + }) + + t.Run("sanitizes all emitted strings and records provenance without originals", func(t *testing.T) { + post := basePost() + post.ID = token + post.RootID = "root\x1b" + token + post.UserID = "user\x1b" + token + post.OverrideUsername = "hook\x1b" + token + post.FileIDs = []string{"file\x1b" + token} + post.Attachments = []mattermost.PostAttachment{{Text: "body " + token}} + post.Reactions = []mattermost.PostReaction{{UserID: "reactor", EmojiName: "yes\x1b" + token}} + messages, redactions, err := NormalizePosts([]mattermost.Post{post}, nil, "", "https://mattermost.test", presentation.Options{}) + if err != nil { + t.Fatal(err) + } + encoded, _ := json.Marshal(struct { + Messages any + Redactions any + }{messages, redactions}) + text := string(encoded) + if strings.Contains(text, token) || strings.ContainsRune(text, '\x1b') || strings.Contains(strings.ToLower(text), "original") { + t.Fatalf("unsafe output = %s", text) + } + fields := map[string]bool{} + for _, r := range redactions { + fields[r.Field] = true + } + for _, field := range []string{"post.id", "post.userId", "post.rootId", "file.id", "post.permalink", "attachment.0.text", "reaction.emoji"} { + if !fields[field] { + t.Errorf("missing field %q in %#v", field, fields) + } + } + if messages[0].CanonicalID != post.ID || messages[0].CanonicalRootID != post.RootID { + t.Fatal("canonical raw identities were not retained") + } + }) + + t.Run("label redaction positions use final UTF-16 coordinates", func(t *testing.T) { + credential := "mm-active-secret" + cases := []struct { + name, value, mask string + options presentation.Options + }{ + {"heuristic secret", token, "ghp_...aaaa", presentation.Options{}}, + {"exact credential", credential, "[REDACTED:mattermost_credential]", presentation.Options{DisableHeuristics: true, Credentials: []string{credential}}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + post := basePost() + post.OverrideUsername = "😀\n\t" + tc.value + messages, redactions, err := NormalizePosts([]mattermost.Post{post}, nil, "", "https://mattermost.test", tc.options) + if err != nil { + t.Fatal(err) + } + if got, want := messages[0].User, "😀\\n\\t"+tc.mask; got != want { + t.Fatalf("user = %q, want %q", got, want) + } + for _, redaction := range redactions { + if redaction.Field != "user" { + continue + } + if redaction.Position != 6 { + t.Fatalf("position = %d, want 6 in %q", redaction.Position, messages[0].User) + } + return + } + t.Fatal("missing user redaction") + }) + } + }) + + t.Run("multiple label redactions track expansions and astral characters", func(t *testing.T) { + credential := "mm-active-secret" + post := basePost() + post.OverrideUsername = "😀\n" + credential + "\t😀\n" + credential + messages, redactions, err := NormalizePosts([]mattermost.Post{post}, nil, "", "https://mattermost.test", presentation.Options{DisableHeuristics: true, Credentials: []string{credential}}) + if err != nil { + t.Fatal(err) + } + positions := make([]int, 0, 2) + for _, redaction := range redactions { + if redaction.Field == "user" { + positions = append(positions, redaction.Position) + } + } + mask := "[REDACTED:mattermost_credential]" + firstByte := strings.Index(messages[0].User, mask) + secondRelative := strings.Index(messages[0].User[firstByte+len(mask):], mask) + secondByte := firstByte + len(mask) + secondRelative + want := []int{utf16Length(messages[0].User[:firstByte]), utf16Length(messages[0].User[:secondByte])} + if !reflect.DeepEqual(positions, want) { + t.Fatalf("positions = %v, want %v in %q", positions, want, messages[0].User) + } + }) + + t.Run("many label redactions remain exact at practical size", func(t *testing.T) { + credential := "mm-active-secret" + const count = 1000 + post := basePost() + post.OverrideUsername = strings.Repeat("\n"+credential, count) + messages, redactions, err := NormalizePosts([]mattermost.Post{post}, nil, "", "https://mattermost.test", presentation.Options{DisableHeuristics: true, Credentials: []string{credential}}) + if err != nil { + t.Fatal(err) + } + userRedactions := 0 + lastPosition := -1 + for _, redaction := range redactions { + if redaction.Field != "user" { + continue + } + userRedactions++ + if redaction.Position <= lastPosition { + t.Fatalf("positions not strictly increasing at %d: %d <= %d", userRedactions, redaction.Position, lastPosition) + } + lastPosition = redaction.Position + } + if userRedactions != count || strings.Count(messages[0].User, "\\n") != count || strings.Contains(messages[0].User, credential) { + t.Fatalf("redactions=%d newlines=%d", userRedactions, strings.Count(messages[0].User, "\\n")) + } + }) + + t.Run("permalink redaction position addresses mask in final canonical URL", func(t *testing.T) { + post := basePost() + post.ID = token + messages, redactions, err := NormalizePosts([]mattermost.Post{post}, nil, "", "https://mattermost.test", presentation.Options{}) + if err != nil { + t.Fatal(err) + } + permalink := messages[0].Permalink + maskAt := strings.Index(permalink, "ghp_...aaaa") + if maskAt < 0 { + t.Fatalf("permalink has no mask: %q", permalink) + } + want := utf16Length(permalink[:maskAt]) + for _, redaction := range redactions { + if redaction.Field == "post.permalink" { + if redaction.Position != want { + t.Fatalf("position = %d, want %d in %q", redaction.Position, want, permalink) + } + return + } + } + t.Fatal("missing permalink redaction") + }) + + t.Run("disabled heuristics still masks active credential", func(t *testing.T) { + credential := "mm-active-secret" + post := basePost() + post.Message = token + " " + credential + "\x1b" + encoded, count := normalizeOne(t, post, presentation.Options{DisableHeuristics: true, Credentials: []string{credential}}) + got := string(encoded) + if !strings.Contains(got, token) || strings.Contains(got, credential) || strings.ContainsRune(got, '\x1b') || count != 1 { + t.Fatalf("output=%s redactions=%d", got, count) + } + }) + + t.Run("reply count is nil at zero", func(t *testing.T) { + post := basePost() + messages, _, err := NormalizePosts([]mattermost.Post{post}, nil, "", "https://mattermost.test", presentation.Options{}) + if err != nil { + t.Fatal(err) + } + if messages[0].ReplyCount != nil { + t.Fatalf("reply count = %#v", messages[0].ReplyCount) + } + post.ReplyCount = 3 + messages, _, err = NormalizePosts([]mattermost.Post{post}, nil, "", "https://mattermost.test", presentation.Options{}) + if err != nil { + t.Fatal(err) + } + if messages[0].ReplyCount == nil || *messages[0].ReplyCount != 3 { + t.Fatalf("reply count = %#v", messages[0].ReplyCount) + } + }) + + t.Run("does not mutate posts or users", func(t *testing.T) { + size := int64(4) + post := basePost() + post.Files = []mattermost.PostFile{{ID: "a", Size: &size}} + post.Reactions = []mattermost.PostReaction{{UserID: "z", EmojiName: "b"}, {UserID: "a", EmojiName: "b"}} + users := map[string]mattermost.User{"z": {Username: "zed"}} + beforePost := post + beforePost.Files = append([]mattermost.PostFile(nil), post.Files...) + beforePost.Reactions = append([]mattermost.PostReaction(nil), post.Reactions...) + beforeUsers := map[string]mattermost.User{"z": users["z"]} + _, _, err := NormalizePosts([]mattermost.Post{post}, users, "", "https://mattermost.test", presentation.Options{}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(post, beforePost) || !reflect.DeepEqual(users, beforeUsers) { + t.Fatalf("inputs mutated: %#v %#v", post, users) + } + }) +} + +func TestPostUserIDs(t *testing.T) { + posts := []mattermost.Post{{UserID: "author", Reactions: []mattermost.PostReaction{{UserID: "reactor"}, {UserID: "reactor"}}}, {UserID: "", Reactions: []mattermost.PostReaction{{UserID: "other"}}}} + if got, want := PostUserIDs(posts), []string{"author", "reactor", "other"}; !reflect.DeepEqual(got, want) { + t.Fatalf("PostUserIDs()=%v, want %v", got, want) + } +} + +func TestRemapLabelRedactionPositionsHandlesSharedBoundaries(t *testing.T) { + redactions := []presentation.Redaction{{Position: 0}, {Position: 0}, {Position: 3}, {Position: 4}} + remapLabelRedactionPositions("😀\nx", redactions) + want := []int{0, 0, 4, 5} + for index, redaction := range redactions { + if redaction.Position != want[index] { + t.Fatalf("position[%d] = %d, want %d", index, redaction.Position, want[index]) + } + } +} From c3461d13ca8ce905f298127479e3a488552d717a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 12:54:05 +0300 Subject: [PATCH 028/119] feat: convert read output to machine envelopes --- internal/output/machine_convert.go | 363 ++++++++++++++++++++++++ internal/output/machine_convert_test.go | 253 +++++++++++++++++ 2 files changed, 616 insertions(+) create mode 100644 internal/output/machine_convert.go create mode 100644 internal/output/machine_convert_test.go diff --git a/internal/output/machine_convert.go b/internal/output/machine_convert.go new file mode 100644 index 0000000..166a3b6 --- /dev/null +++ b/internal/output/machine_convert.go @@ -0,0 +1,363 @@ +package output + +import ( + "fmt" + "time" +) + +// MachineMessageFromMessage losslessly copies a presentation message into its +// machine representation. Timestamps retain their instant and are normalized +// to UTC; collection and pointer fields do not alias the input. +func MachineMessageFromMessage(message Message) MachineMessage { + replies := make([]MachineMessage, len(message.Replies)) + for index := range message.Replies { + replies[index] = MachineMessageFromMessage(message.Replies[index]) + } + + return MachineMessage{ + ID: message.ID, + Permalink: message.Permalink, + User: message.User, + UserID: message.UserID, + Text: message.Text, + Timestamp: machineMillis(message.Timestamp), + UpdatedAt: machineMillis(message.UpdatedAt), + EditedAt: machineMillisPointer(message.EditedAt), + DeletedAt: machineMillisPointer(message.DeletedAt), + IsDeleted: message.IsDeleted, + PostType: message.PostType, + IsSystem: message.IsSystem, + IsPinned: message.IsPinned, + RootID: nullableString(message.RootID), + ReplyCount: clonePointer(message.ReplyCount), + Files: cloneSlice(message.Files), + FileDetails: cloneFiles(message.FileDetails), + Attachments: cloneAttachments(message.Attachments), + Reactions: cloneReactions(message.Reactions), + Replies: replies, + } +} + +// MachineHistoryFromOutput converts one independently headed output section. +func MachineHistoryFromOutput(value MessageOutput, completeness MachineCompleteness) (MachineHistory, error) { + if !validMachineCompleteness(completeness) { + return MachineHistory{}, fmt.Errorf("invalid machine completeness %q", completeness) + } + channel, err := machineChannel(value.Channel) + if err != nil { + return MachineHistory{}, err + } + if err := validateRetrieval(value.Retrieval); err != nil { + return MachineHistory{}, err + } + for index := range value.Messages { + if err := validateMessage(value.Messages[index], 0); err != nil { + return MachineHistory{}, fmt.Errorf("message %d: %w", index, err) + } + } + for index, redaction := range value.Redactions { + if redaction.Position < 0 { + return MachineHistory{}, fmt.Errorf("redaction %d has negative position", index) + } + } + messages := make([]MachineMessage, len(value.Messages)) + for index := range value.Messages { + messages[index] = MachineMessageFromMessage(value.Messages[index]) + } + return MachineHistory{ + Channel: channel, + Messages: messages, + Redactions: cloneSlice(value.Redactions), + Metadata: MachineMetadata{ + Completeness: completeness, + Selection: cloneSelection(value.Retrieval.Selection), + VisibleThreads: cloneVisibleThreads(value.Retrieval.VisibleThreads), + VisiblePostCount: value.Retrieval.VisiblePostCount, + DeletedPostsIncluded: value.Retrieval.DeletedPostsIncluded, + }, + }, nil +} + +func NewChannelEnvelope(value MessageOutput, completeness MachineCompleteness) (ChannelEnvelope, error) { + if err := validateSource(value.Retrieval.Selection.Source, "recent"); err != nil { + return ChannelEnvelope{}, err + } + history, err := MachineHistoryFromOutput(value, completeness) + return ChannelEnvelope{Schema: "mm/v2/channel", Data: history}, err +} + +func NewDMSEnvelope(values []MessageOutput, completeness MachineCompleteness) (DMSEnvelope, error) { + histories, err := machineHistories(values, completeness, "recent", "dm", "unknown") + return DMSEnvelope{Schema: "mm/v2/dms", Channels: histories}, err +} + +func NewGroupDMSEnvelope(values []MessageOutput, completeness MachineCompleteness) (GroupDMSEnvelope, error) { + histories, err := machineHistories(values, completeness, "recent", "group", "unknown") + return GroupDMSEnvelope{Schema: "mm/v2/group-dms", Channels: histories}, err +} + +func NewSearchEnvelope(values []MessageOutput, completeness MachineCompleteness) (SearchEnvelope, error) { + histories, err := machineHistories(values, completeness, "search") + return SearchEnvelope{Schema: "mm/v2/search", Results: histories}, err +} + +func NewMentionsEnvelope(values []MessageOutput, completeness MachineCompleteness) (MentionsEnvelope, error) { + histories, err := machineHistories(values, completeness, "mentions") + return MentionsEnvelope{Schema: "mm/v2/mentions", Results: histories}, err +} + +// NewThreadEnvelope requires the grouped thread representation: exactly one +// top-level root, with every reply represented only through root.Replies. +func NewThreadEnvelope(value MessageOutput, completeness MachineCompleteness) (ThreadEnvelope, error) { + if err := validateSource(value.Retrieval.Selection.Source, "thread"); err != nil { + return ThreadEnvelope{}, err + } + history, err := MachineHistoryFromOutput(value, completeness) + if err != nil { + return ThreadEnvelope{}, err + } + if len(value.Messages) != 1 { + return ThreadEnvelope{}, fmt.Errorf("thread output must contain exactly one top-level root, got %d", len(value.Messages)) + } + root := value.Messages[0] + if root.RootID != "" || root.CanonicalRootID != "" { + return ThreadEnvelope{}, fmt.Errorf("thread output top-level message is a reply, not a root") + } + rootIdentity := root.CanonicalID + if rootIdentity == "" { + rootIdentity = root.ID + } + if rootIdentity == "" || root.ID == "" { + return ThreadEnvelope{}, fmt.Errorf("thread root must have presented and canonical identity") + } + seen := map[string]bool{rootIdentity: true, root.ID: true} + for index, reply := range root.Replies { + if len(reply.Replies) != 0 { + return ThreadEnvelope{}, fmt.Errorf("thread reply %d contains nested replies", index) + } + replyRootIdentity := reply.CanonicalRootID + if replyRootIdentity == "" { + replyRootIdentity = reply.RootID + } + if reply.RootID != root.ID || replyRootIdentity != rootIdentity { + return ThreadEnvelope{}, fmt.Errorf("thread reply %d does not reference root", index) + } + replyIdentity := reply.CanonicalID + if replyIdentity == "" { + replyIdentity = reply.ID + } + if replyIdentity == "" || reply.ID == "" || seen[replyIdentity] || seen[reply.ID] { + return ThreadEnvelope{}, fmt.Errorf("thread reply %d has missing or duplicate identity", index) + } + seen[replyIdentity], seen[reply.ID] = true, true + } + return ThreadEnvelope{Schema: "mm/v2/thread", Data: ThreadData{ + Channel: history.Channel, Root: history.Messages[0], + Redactions: history.Redactions, Metadata: history.Metadata, + }}, nil +} + +func machineHistories(values []MessageOutput, completeness MachineCompleteness, source string, channelTypes ...string) ([]MachineHistory, error) { + if !validMachineCompleteness(completeness) { + return nil, fmt.Errorf("invalid machine completeness %q", completeness) + } + result := make([]MachineHistory, len(values)) + for index := range values { + if err := validateSource(values[index].Retrieval.Selection.Source, source); err != nil { + return nil, fmt.Errorf("convert output %d: %w", index, err) + } + if len(channelTypes) != 0 && !containsString(channelTypes, values[index].Channel.Type) { + return nil, fmt.Errorf("convert output %d: channel type %q is not valid for this envelope", index, values[index].Channel.Type) + } + converted, err := MachineHistoryFromOutput(values[index], completeness) + if err != nil { + return nil, fmt.Errorf("convert output %d: %w", index, err) + } + result[index] = converted + } + return result, nil +} + +func machineChannel(channel Channel) (MachineChannel, error) { + switch channel.Type { + case "dm", "public", "private", "group", "unknown": + default: + return MachineChannel{}, fmt.Errorf("invalid machine channel type %q", channel.Type) + } + switch channel.MetadataStatus { + case "resolved", "unavailable": + default: + return MachineChannel{}, fmt.Errorf("invalid machine channel metadata status %q", channel.MetadataStatus) + } + if (channel.Type == "unknown") != (channel.MetadataStatus == "unavailable") { + return MachineChannel{}, fmt.Errorf("machine channel type %q and metadata status %q are inconsistent", channel.Type, channel.MetadataStatus) + } + return MachineChannel{ID: channel.ID, Type: channel.Type, Name: channel.Name, DisplayName: channel.DisplayName, MetadataStatus: channel.MetadataStatus}, nil +} + +func validMachineCompleteness(value MachineCompleteness) bool { + return value == MachineComplete || value == MachineTruncated || value == MachineUnknown +} + +func validateSource(source string, allowed ...string) error { + for _, value := range allowed { + if source == value { + return nil + } + } + return fmt.Errorf("selection source %q is not valid for this envelope", source) +} + +func validateRetrieval(value Retrieval) error { + switch value.Selection.Source { + case "recent", "search", "mentions", "unread", "thread": + default: + return fmt.Errorf("invalid selection source %q", value.Selection.Source) + } + if value.Selection.SelectedCount < 0 { + return fmt.Errorf("selected count must be nonnegative") + } + if value.Selection.RequestedLimit != nil && *value.Selection.RequestedLimit < 1 { + return fmt.Errorf("requested limit must be positive") + } + if value.VisiblePostCount < 0 || value.VisibleThreads.HydratedRootCount < 0 { + return fmt.Errorf("visible post and hydrated root counts must be nonnegative") + } + switch value.VisibleThreads.Status { + case "not_requested": + if value.VisibleThreads.HydratedRootCount != 0 || len(value.VisibleThreads.FailedRootIDs) != 0 { + return fmt.Errorf("threads not requested cannot have hydration results") + } + case "complete": + if len(value.VisibleThreads.FailedRootIDs) != 0 { + return fmt.Errorf("complete thread hydration cannot have failed roots") + } + case "partial": + default: + return fmt.Errorf("invalid visible thread status %q", value.VisibleThreads.Status) + } + if value.DeletedPostsIncluded { + return fmt.Errorf("machine schema does not permit deleted posts to be included") + } + return nil +} + +func containsString(values []string, target string) bool { + for _, value := range values { + if value == target { + return true + } + } + return false +} + +func validateMessage(message Message, depth int) error { + if depth > machinePreflightMaxDepth { + return fmt.Errorf("reply nesting exceeds machine limit") + } + for name, value := range map[string]time.Time{"timestamp": message.Timestamp, "updatedAt": message.UpdatedAt} { + if err := validateMachineTime(value); err != nil { + return fmt.Errorf("%s: %w", name, err) + } + } + for name, value := range map[string]*time.Time{"editedAt": message.EditedAt, "deletedAt": message.DeletedAt} { + if value != nil { + if err := validateMachineTime(*value); err != nil { + return fmt.Errorf("%s: %w", name, err) + } + } + } + if message.ReplyCount != nil && *message.ReplyCount < 0 { + return fmt.Errorf("reply count must be nonnegative") + } + for index, file := range message.FileDetails { + if file.Size != nil && *file.Size < 0 { + return fmt.Errorf("file %d size must be nonnegative", index) + } + } + for index, reaction := range message.Reactions { + if reaction.Count < 0 { + return fmt.Errorf("reaction %d count must be nonnegative", index) + } + } + for index := range message.Replies { + if err := validateMessage(message.Replies[index], depth+1); err != nil { + return fmt.Errorf("reply %d: %w", index, err) + } + } + return nil +} + +func validateMachineTime(value time.Time) error { + utc := value.UTC() + if utc.Year() < 1 || utc.Year() > 9999 { + return fmt.Errorf("timestamp year must be between 0001 and 9999 after UTC normalization") + } + return nil +} + +func machineMillis(value time.Time) MillisTime { return MillisTime{Time: value.UTC()} } + +func machineMillisPointer(value *time.Time) *MillisTime { + if value == nil { + return nil + } + converted := machineMillis(*value) + return &converted +} + +func nullableString(value string) *string { + if value == "" { + return nil + } + return &value +} + +func clonePointer[T any](value *T) *T { + if value == nil { + return nil + } + copy := *value + return © +} + +func cloneFiles(values []File) []File { + result := cloneSlice(values) + for index := range result { + result[index].Size = clonePointer(result[index].Size) + } + return result +} + +func cloneAttachments(values []Attachment) []Attachment { + result := cloneSlice(values) + for index := range result { + result[index].Fields = cloneSlice(result[index].Fields) + for fieldIndex := range result[index].Fields { + result[index].Fields[fieldIndex].Short = clonePointer(result[index].Fields[fieldIndex].Short) + } + } + return result +} + +func cloneReactions(values []Reaction) []Reaction { + result := cloneSlice(values) + for index := range result { + result[index].Actors = cloneSlice(result[index].Actors) + } + return result +} + +func cloneSelection(value Selection) Selection { + value.RequestedLimit = clonePointer(value.RequestedLimit) + value.Since = clonePointer(value.Since) + value.QueryTruncated = clonePointer(value.QueryTruncated) + value.InputCursor = clonePointer(value.InputCursor) + value.NextCursor = clonePointer(value.NextCursor) + return value +} + +func cloneVisibleThreads(value VisibleThreads) VisibleThreads { + value.FailedRootIDs = cloneSlice(value.FailedRootIDs) + return value +} diff --git a/internal/output/machine_convert_test.go b/internal/output/machine_convert_test.go new file mode 100644 index 0000000..cd513cc --- /dev/null +++ b/internal/output/machine_convert_test.go @@ -0,0 +1,253 @@ +package output_test + +import ( + "bytes" + "strings" + "testing" + "time" + + "github.com/ardasevinc/mattermost-cli/internal/output" + "github.com/ardasevinc/mattermost-cli/internal/schema" +) + +func TestMachineMessageFromMessagePreservesRichRecursiveDataWithoutAliasing(t *testing.T) { + location := time.FixedZone("offset", 3*60*60) + edited := time.Date(2026, 7, 16, 13, 14, 15, 987654321, location) + count, size, short := 0, int64(42), false + reply := output.Message{ID: "reply", RootID: "root", Text: "nested", Timestamp: edited, UpdatedAt: edited} + message := output.Message{ + ID: "root", Permalink: "https://mm.test/_redirect/pl/root", User: "arda", UserID: "u1", Text: "hello", + Timestamp: edited, UpdatedAt: edited, EditedAt: &edited, IsPinned: true, PostType: "custom", ReplyCount: &count, + Files: []string{"report.txt"}, FileDetails: []output.File{{ID: "f1", Name: "report.txt", Size: &size}}, + Attachments: []output.Attachment{{Title: "rich", Fields: []output.AttachmentField{{Title: "field", Value: "value", Short: &short}}}}, + Reactions: []output.Reaction{{Emoji: "wave", Count: 1, Actors: []output.ReactionActor{{ID: "u2", Username: "bob"}}}}, + Replies: []output.Message{reply}, + } + + converted := output.MachineMessageFromMessage(message) + if converted.Timestamp.Location() != time.UTC || converted.EditedAt == nil || converted.EditedAt.Location() != time.UTC { + t.Fatalf("timestamps were not normalized to UTC: %#v", converted) + } + if !converted.Timestamp.Equal(edited) || !converted.EditedAt.Equal(edited) { + t.Fatalf("timestamp instant changed: got %v / %v, want %v", converted.Timestamp, converted.EditedAt, edited) + } + if converted.RootID != nil || converted.ReplyCount == nil || *converted.ReplyCount != 0 || len(converted.Replies) != 1 || converted.Replies[0].RootID == nil || *converted.Replies[0].RootID != "root" { + t.Fatalf("nullable or recursive fields changed: %#v", converted) + } + + converted.Files[0] = "changed" + *converted.FileDetails[0].Size = 99 + *converted.Attachments[0].Fields[0].Short = true + converted.Reactions[0].Actors[0].Username = "changed" + converted.Replies[0].Text = "changed" + *converted.ReplyCount = 2 + if message.Files[0] != "report.txt" || *message.FileDetails[0].Size != 42 || *message.Attachments[0].Fields[0].Short || message.Reactions[0].Actors[0].Username != "bob" || message.Replies[0].Text != "nested" || *message.ReplyCount != 0 { + t.Fatal("converted message aliases input storage") + } +} + +func TestMachineEnvelopeFactoriesWriteSchemaValidDocuments(t *testing.T) { + registry, err := schema.Load() + if err != nil { + t.Fatal(err) + } + value := validOutput() + + channel, err := output.NewChannelEnvelope(value, output.MachineComplete) + if err != nil { + t.Fatal(err) + } + dmValue := value + dmValue.Channel.Type = "dm" + dms, err := output.NewDMSEnvelope([]output.MessageOutput{dmValue}, output.MachineComplete) + if err != nil { + t.Fatal(err) + } + groupValue := value + groupValue.Channel.Type = "group" + groups, err := output.NewGroupDMSEnvelope([]output.MessageOutput{groupValue}, output.MachineComplete) + if err != nil { + t.Fatal(err) + } + searchValue := value + searchValue.Retrieval.Selection.Source = "search" + search, err := output.NewSearchEnvelope([]output.MessageOutput{searchValue}, output.MachineComplete) + if err != nil { + t.Fatal(err) + } + mentionsValue := value + mentionsValue.Retrieval.Selection.Source = "mentions" + mentions, err := output.NewMentionsEnvelope([]output.MessageOutput{mentionsValue}, output.MachineComplete) + if err != nil { + t.Fatal(err) + } + threadValue := value + threadValue.Retrieval.Selection.Source = "thread" + thread, err := output.NewThreadEnvelope(threadValue, output.MachineComplete) + if err != nil { + t.Fatal(err) + } + + cases := []struct { + id string + document output.MachineDocument + }{ + {"mm/v2/channel", channel}, {"mm/v2/dms", dms}, {"mm/v2/group-dms", groups}, + {"mm/v2/search", search}, {"mm/v2/mentions", mentions}, {"mm/v2/thread", thread}, + } + for _, test := range cases { + t.Run(test.id, func(t *testing.T) { + var wire bytes.Buffer + if _, err := output.WriteMachineJSON(&wire, test.document); err != nil { + t.Fatal(err) + } + if strings.Contains(wire.String(), ":null") && (strings.Contains(wire.String(), `"files":null`) || strings.Contains(wire.String(), `"redactions":null`) || strings.Contains(wire.String(), `"replies":null`)) { + t.Fatalf("writer did not canonicalize nil collections: %s", wire.String()) + } + if err := registry.Validate(test.id, bytes.NewReader(wire.Bytes())); err != nil { + t.Fatalf("schema validation: %v\n%s", err, wire.String()) + } + if !strings.Contains(wire.String(), `"timestamp":"2026-07-16T10:14:15.987Z"`) { + t.Fatalf("timestamp is not exact UTC milliseconds: %s", wire.String()) + } + }) + } +} + +func TestMachineConversionRejectsInvalidVocabularyAndAmbiguousThreads(t *testing.T) { + value := validOutput() + value.Channel.Type = "town-square" + if _, err := output.MachineHistoryFromOutput(value, output.MachineComplete); err == nil { + t.Fatal("invalid channel type accepted") + } + value = validOutput() + value.Channel.MetadataStatus = "maybe" + if _, err := output.MachineHistoryFromOutput(value, output.MachineComplete); err == nil { + t.Fatal("invalid metadata status accepted") + } + value = validOutput() + if _, err := output.MachineHistoryFromOutput(value, output.MachineCompleteness("maybe")); err == nil { + t.Fatal("invalid completeness accepted") + } + value = validOutput() + value.Channel.Type = "unknown" + if _, err := output.MachineHistoryFromOutput(value, output.MachineComplete); err == nil { + t.Fatal("inconsistent channel type/status accepted") + } + + badRetrievals := map[string]func(*output.MessageOutput){ + "source": func(v *output.MessageOutput) { v.Retrieval.Selection.Source = "other" }, + "selected count": func(v *output.MessageOutput) { v.Retrieval.Selection.SelectedCount = -1 }, + "requested limit": func(v *output.MessageOutput) { n := 0; v.Retrieval.Selection.RequestedLimit = &n }, + "visible count": func(v *output.MessageOutput) { v.Retrieval.VisiblePostCount = -1 }, + "hydrated count": func(v *output.MessageOutput) { v.Retrieval.VisibleThreads.HydratedRootCount = -1 }, + "visible status": func(v *output.MessageOutput) { v.Retrieval.VisibleThreads.Status = "other" }, + "complete failures": func(v *output.MessageOutput) { v.Retrieval.VisibleThreads.FailedRootIDs = []string{"root"} }, + "deleted posts": func(v *output.MessageOutput) { v.Retrieval.DeletedPostsIncluded = true }, + } + for name, corrupt := range badRetrievals { + t.Run(name, func(t *testing.T) { + value := validOutput() + corrupt(&value) + if _, err := output.MachineHistoryFromOutput(value, output.MachineComplete); err == nil { + t.Fatal("invalid retrieval accepted") + } + }) + } + + badMessages := map[string]func(*output.Message){ + "timestamp": func(m *output.Message) { m.Timestamp = time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC) }, + "reply count": func(m *output.Message) { n := -1; m.ReplyCount = &n }, + "file size": func(m *output.Message) { n := int64(-1); m.FileDetails = []output.File{{ID: "f", Size: &n}} }, + "reaction count": func(m *output.Message) { m.Reactions = []output.Reaction{{Count: -1}} }, + "recursive reply": func(m *output.Message) { m.Replies[0].Reactions = []output.Reaction{{Count: -1}} }, + } + value = validOutput() + value.Redactions[0].Position = -1 + if _, err := output.MachineHistoryFromOutput(value, output.MachineComplete); err == nil { + t.Fatal("negative redaction position accepted") + } + for name, corrupt := range badMessages { + t.Run(name, func(t *testing.T) { + value := validOutput() + corrupt(&value.Messages[0]) + if _, err := output.MachineHistoryFromOutput(value, output.MachineComplete); err == nil { + t.Fatal("schema-invalid message accepted") + } + }) + } + if _, err := output.NewDMSEnvelope(nil, output.MachineCompleteness("bad")); err == nil { + t.Fatal("empty factory skipped completeness validation") + } + if _, err := output.NewSearchEnvelope([]output.MessageOutput{validOutput()}, output.MachineComplete); err == nil { + t.Fatal("search factory accepted recent source") + } + value = validOutput() + value.Retrieval.VisibleThreads.Status = "partial" + if _, err := output.MachineHistoryFromOutput(value, output.MachineComplete); err != nil { + t.Fatalf("partial retrieval with unknown failed root identities rejected: %v", err) + } + if _, err := output.NewDMSEnvelope([]output.MessageOutput{validOutput()}, output.MachineComplete); err == nil { + t.Fatal("DMS factory accepted public channel") + } + value = validOutput() + value.Channel.Type = "dm" + if _, err := output.NewGroupDMSEnvelope([]output.MessageOutput{value}, output.MachineComplete); err == nil { + t.Fatal("group-DMS factory accepted DM channel") + } + + for name, messages := range map[string][]output.Message{ + "zero roots": {}, + "multiple roots": {{ID: "one"}, {ID: "two"}}, + "top-level reply": {{ID: "reply", RootID: "root"}}, + } { + t.Run(name, func(t *testing.T) { + value := validOutput() + value.Retrieval.Selection.Source = "thread" + value.Messages = messages + if _, err := output.NewThreadEnvelope(value, output.MachineComplete); err == nil { + t.Fatal("ambiguous thread accepted") + } + }) + } + + threadCases := map[string]func(*output.Message){ + "orphan": func(root *output.Message) { root.Replies[0].RootID = "other" }, + "canonical orphan": func(root *output.Message) { root.Replies[0].CanonicalRootID = "other" }, + "nested": func(root *output.Message) { root.Replies[0].Replies = []output.Message{{ID: "nested"}} }, + "duplicate": func(root *output.Message) { root.Replies = append(root.Replies, root.Replies[0]) }, + } + for name, corrupt := range threadCases { + t.Run(name, func(t *testing.T) { + value := validOutput() + value.Retrieval.Selection.Source = "thread" + corrupt(&value.Messages[0]) + if _, err := output.NewThreadEnvelope(value, output.MachineComplete); err == nil { + t.Fatal("invalid thread accepted") + } + }) + } + value = validOutput() + value.Retrieval.Selection.Source = "thread" + value.Messages[0].CanonicalID = "" + value.Messages[0].Replies[0].CanonicalID = "" + value.Messages[0].Replies[0].CanonicalRootID = "" + if _, err := output.NewThreadEnvelope(value, output.MachineComplete); err != nil { + t.Fatalf("thread identity fallback rejected valid grouped model: %v", err) + } +} + +func validOutput() output.MessageOutput { + zone := time.FixedZone("source", 3*60*60) + stamp := time.Date(2026, 7, 16, 13, 14, 15, 987654321, zone) + limit, truncated := 10, false + return output.MessageOutput{ + Channel: output.Channel{ID: "c1", Type: "public", Name: "town-square", DisplayName: "Town Square", MetadataStatus: "resolved"}, + Messages: []output.Message{{ID: "root", CanonicalID: "root", User: "arda", UserID: "u1", Text: "root", Timestamp: stamp, UpdatedAt: stamp, Replies: []output.Message{{ID: "reply", CanonicalID: "reply", RootID: "root", CanonicalRootID: "root", User: "bob", UserID: "u2", Text: "reply", Timestamp: stamp, UpdatedAt: stamp}}}}, + Redactions: []output.Redaction{{Type: "token", Masked: "abc***xyz", Position: 0}}, + Retrieval: output.Retrieval{ + Selection: output.Selection{Source: "recent", SelectedCount: 2, RequestedLimit: &limit, QueryTruncated: &truncated}, + VisibleThreads: output.VisibleThreads{Status: "complete", HydratedRootCount: 1}, VisiblePostCount: 2, + }, + } +} From ae0287e4d471b6db7b7bc696777fe2c52c5f0f05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 13:09:57 +0300 Subject: [PATCH 029/119] feat: add secure CLI runtime foundation --- internal/cli/root.go | 59 ++++-- internal/cli/root_test.go | 35 ++++ internal/cli/runtime.go | 251 +++++++++++++++++++++++++ internal/cli/runtime_test.go | 304 +++++++++++++++++++++++++++++++ internal/schema/registry.go | 15 +- internal/schema/registry_test.go | 22 +++ 6 files changed, 674 insertions(+), 12 deletions(-) create mode 100644 internal/cli/runtime.go create mode 100644 internal/cli/runtime_test.go diff --git a/internal/cli/root.go b/internal/cli/root.go index 89b4e47..9639359 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -2,7 +2,6 @@ package cli import ( "context" - "errors" "fmt" "io" "os" @@ -22,25 +21,41 @@ type streams struct { } func Execute(ctx context.Context, args []string, in io.Reader, out, errOut io.Writer) int { - releaseCredential := presentation.ActiveCredentials.Register(os.Getenv("MM_TOKEN")) - defer releaseCredential() - cmd := newRoot(streams{in: in, out: out, err: errOut}) + trackedOut := &writeTracker{writer: out} + s := streams{in: in, out: trackedOut, err: errOut} + deps := defaultDependencies(out) + credentials := append([]string{os.Getenv("MM_TOKEN")}, earlyTokens(args)...) + credentials = append(credentials, bestEffortFileToken(deps)) + state := &rootState{streams: s, deps: deps, credentials: credentials} + for _, credential := range credentials { + state.releases = append(state.releases, presentation.ActiveCredentials.Register(credential)) + } + defer state.releaseCredentials() + defer state.close() + cmd := newRootWithState(state) cmd.SetArgs(args) if err := cmd.ExecuteContext(ctx); err != nil { - message := presentation.SanitizeLabel(presentation.PreprocessActive(err.Error()).Text) + message := presentation.SanitizeLabel(presentation.Preprocess(err.Error(), state.credentials).Text) if writeErr := writeAll(errOut, []byte(fmt.Sprintf("error: %s\n", message))); writeErr != nil { return 3 } - var outputFailure outputError - if errors.As(err, &outputFailure) { + if trackedOut.Failed() { return 3 } - return 2 + return exitCode(err) + } + if trackedOut.Failed() { + return 3 } return 0 } func newRoot(s streams) *cobra.Command { + return newRootWithState(&rootState{streams: s, deps: defaultDependencies(s.out)}) +} + +func newRootWithState(state *rootState) *cobra.Command { + s := state.streams cmd := &cobra.Command{ Use: "mm", Short: "Mattermost CLI for agents and humans", @@ -58,6 +73,10 @@ func newRoot(s streams) *cobra.Command { cmd.SetIn(s.in) cmd.SetOut(s.out) cmd.SetErr(s.err) + cmd.PersistentFlags().StringVar(&state.flags.url, "url", "", "Mattermost server URL") + cmd.PersistentFlags().StringVar(&state.flags.token, "token", "", "Mattermost personal access token") + cmd.PersistentFlags().BoolVar(&state.flags.redact, "redact", true, "redact detected secrets") + cmd.PersistentFlags().BoolVar(&state.flags.noRedact, "no-redact", false, "disable heuristic secret redaction") cmd.AddCommand(newSchemaCommand(s)) return cmd } @@ -75,7 +94,7 @@ func newSchemaCommand(s streams) *cobra.Command { RunE: func(_ *cobra.Command, _ []string) error { registry, err := mmSchema.Load() if err != nil { - return err + return readFailure(err) } return writeAll(s.out, []byte(strings.Join(registry.IDs(), "\n")+"\n")) }, @@ -87,7 +106,7 @@ func newSchemaCommand(s streams) *cobra.Command { RunE: func(_ *cobra.Command, args []string) error { registry, err := mmSchema.Load() if err != nil { - return err + return readFailure(err) } data, err := registry.Show(args[0]) if err != nil { @@ -109,9 +128,12 @@ func newSchemaCommand(s streams) *cobra.Command { RunE: func(_ *cobra.Command, args []string) error { registry, err := mmSchema.Load() if err != nil { - return err + return readFailure(err) } if err := registry.Validate(args[0], s.in); err != nil { + if mmSchema.IsInputReadError(err) { + return readFailure(err) + } return err } return writeAll(s.out, []byte("valid: "+args[0]+"\n")) @@ -124,6 +146,21 @@ type outputError struct { err error } +type writeTracker struct { + writer io.Writer + failed bool +} + +func (w *writeTracker) Write(data []byte) (int, error) { + written, err := w.writer.Write(data) + if err != nil || written != len(data) { + w.failed = true + } + return written, err +} + +func (w *writeTracker) Failed() bool { return w.failed } + func (e outputError) Error() string { return "write output failed" } diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index 175298f..49e5610 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -3,6 +3,7 @@ package cli import ( "bytes" "context" + "errors" "io" "strings" "testing" @@ -74,6 +75,30 @@ func TestSchemaValidate(t *testing.T) { } } +func TestSchemaValidatePhysicalReadFailureIsExitThree(t *testing.T) { + var stdout, stderr bytes.Buffer + physical := errors.New("hostile reader detail \x1b[2J") + code := Execute(context.Background(), []string{"schema", "validate", "mm/v2/error"}, errorInput{err: physical}, &stdout, &stderr) + if code != 3 { + t.Fatalf("exit code = %d, want 3; stderr=%q", code, stderr.String()) + } + if stdout.Len() != 0 || strings.Contains(stderr.String(), physical.Error()) || !strings.Contains(stderr.String(), "could not read JSON document") { + t.Fatalf("stdout=%q stderr=%q", stdout.String(), stderr.String()) + } +} + +func TestSchemaValidateInvalidJSONRemainsExitTwo(t *testing.T) { + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"schema", "validate", "mm/v2/error"}, strings.NewReader("{"), &stdout, &stderr) + if code != 2 { + t.Fatalf("exit code = %d, want 2; stderr=%q", code, stderr.String()) + } +} + +type errorInput struct{ err error } + +func (r errorInput) Read([]byte) (int, error) { return 0, r.err } + func TestSchemaLookupDoesNotReflectActiveCredential(t *testing.T) { const token = "super-secret-mm-token" t.Setenv("MM_TOKEN", token) @@ -133,6 +158,16 @@ func TestErrorOutputShortWriteReturnsOutputFailure(t *testing.T) { } } +func TestRootHelpAndVersionShortWritesReturnOutputFailure(t *testing.T) { + tests := [][]string{nil, {"--help"}, {"--version"}} + for _, args := range tests { + var stderr bytes.Buffer + if code := Execute(context.Background(), args, strings.NewReader(""), shortWriter{}, &stderr); code != 3 { + t.Fatalf("Execute(%q) exit = %d, want 3", args, code) + } + } +} + type shortWriter struct{} func (shortWriter) Write(data []byte) (int, error) { diff --git a/internal/cli/runtime.go b/internal/cli/runtime.go new file mode 100644 index 0000000..e0354b6 --- /dev/null +++ b/internal/cli/runtime.go @@ -0,0 +1,251 @@ +package cli + +import ( + "errors" + "os" + "strings" + "sync" + + "github.com/spf13/cobra" + + "github.com/ardasevinc/mattermost-cli/internal/api" + "github.com/ardasevinc/mattermost-cli/internal/config" + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/internal/serverurl" +) + +type clientFactory func(baseURL, token string) (*api.Client, error) + +type dependencies struct { + lookupEnv config.LookupEnv + homeDir func() (string, error) + newClient clientFactory + stdoutTTY func() bool +} + +func defaultDependencies(out any) dependencies { + return dependencies{ + lookupEnv: os.LookupEnv, + homeDir: os.UserHomeDir, + newClient: func(baseURL, token string) (*api.Client, error) { return api.New(baseURL, token) }, + stdoutTTY: func() bool { + file, ok := out.(*os.File) + if !ok { + return false + } + info, err := file.Stat() + return err == nil && info.Mode()&os.ModeCharDevice != 0 + }, + } +} + +type Runtime struct { + Config config.Resolved + Client *api.Client + Users *mattermost.Users + Teams *mattermost.Teams + Channels *mattermost.Channels + Posts *mattermost.Posts + StdoutTTY bool +} + +func (r *Runtime) Close() { + if r != nil && r.Client != nil { + r.Client.Close() + } +} + +type rootState struct { + streams streams + deps dependencies + flags runtimeFlags + + mu sync.Mutex + runtime *Runtime + runtimeErr error + resolved bool + warned bool + releases []func() + credentials []string +} + +type runtimeFlags struct { + url string + token string + redact bool + noRedact bool +} + +func (s *rootState) close() { + s.mu.Lock() + defer s.mu.Unlock() + if s.runtime != nil { + s.runtime.Close() + s.runtime = nil + } +} + +func (s *rootState) releaseCredentials() { + s.mu.Lock() + releases := s.releases + s.releases = nil + s.mu.Unlock() + for i := len(releases) - 1; i >= 0; i-- { + releases[i]() + } +} + +func (s *rootState) runtimeFor(cmd *cobra.Command) (*Runtime, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.resolved { + return s.runtime, s.runtimeErr + } + s.resolved = true + + home, err := s.deps.homeDir() + if err != nil { + s.runtimeErr = configFailure("could not resolve the home directory") + return nil, s.runtimeErr + } + paths, err := config.ResolvePaths(home, s.deps.lookupEnv) + if err != nil { + s.runtimeErr = configFailure(err.Error()) + return nil, s.runtimeErr + } + file := config.Load(paths) + if file.Config.Token != "" { + s.releases = append(s.releases, presentation.ActiveCredentials.Register(file.Config.Token)) + s.credentials = append(s.credentials, file.Config.Token) + } + if file.Error == config.FileErrorRead || file.Unsafe != "" { + s.runtimeErr = configFailure("could not safely read the Mattermost configuration") + return nil, s.runtimeErr + } + if file.Error == config.FileErrorParse { + s.runtimeErr = configFailure("could not parse the Mattermost configuration") + return nil, s.runtimeErr + } + if file.InsecurePermissions && file.Config.Token != "" { + s.runtimeErr = configFailure("Mattermost configuration containing a token must not be accessible by other users") + return nil, s.runtimeErr + } + if warning := file.Warning(); warning != "" && !s.warned { + warning = presentation.SanitizeLabel(presentation.Preprocess(warning, s.credentials).Text) + if err := writeAll(s.streams.err, []byte("warning: "+warning+"\n")); err != nil { + s.runtimeErr = err + return nil, err + } + s.warned = true + } + + var redact *bool + redactFlag := cmd.Flags().Lookup("redact") + noRedactFlag := cmd.Flags().Lookup("no-redact") + redactChanged := redactFlag != nil && redactFlag.Changed + noRedactChanged := noRedactFlag != nil && noRedactFlag.Changed + if redactChanged && noRedactChanged { + s.runtimeErr = invalidFailure("--redact and --no-redact cannot be used together") + return nil, s.runtimeErr + } + if redactChanged { + value := s.flags.redact + redact = &value + } else if noRedactChanged { + value := !s.flags.noRedact + redact = &value + } + resolved := config.Resolve(config.Options{URL: s.flags.url, Token: s.flags.token, Redact: redact}, s.deps.lookupEnv, file) + if resolved.URL == "" || resolved.Token == "" { + s.runtimeErr = configFailure("Mattermost URL and token are required") + return nil, s.runtimeErr + } + normalized, err := serverurl.Normalize(resolved.URL) + if err != nil { + s.runtimeErr = configFailure(err.Error()) + return nil, s.runtimeErr + } + resolved.URL = normalized + client, err := s.deps.newClient(resolved.URL, resolved.Token) + if err != nil { + s.runtimeErr = configFailure("could not initialize the Mattermost client") + return nil, s.runtimeErr + } + s.runtime = &Runtime{ + Config: resolved, Client: client, StdoutTTY: s.deps.stdoutTTY(), + Users: mattermost.NewUsers(client), Teams: mattermost.NewTeams(client), + Channels: mattermost.NewChannels(client), Posts: mattermost.NewPosts(client), + } + return s.runtime, nil +} + +type errorClass uint8 + +const ( + classInvalid errorClass = iota + classRead +) + +type classifiedError struct { + class errorClass + msg string +} + +func (e classifiedError) Error() string { return e.msg } + +func invalidFailure(message string) error { return classifiedError{class: classInvalid, msg: message} } +func configFailure(message string) error { return classifiedError{class: classRead, msg: message} } + +type operationFailure struct { + class errorClass + err error +} + +func (e operationFailure) Error() string { return e.err.Error() } +func (e operationFailure) Unwrap() error { return e.err } + +// readFailure and authFailure preserve the v2 exit contract at command boundaries. +func readFailure(err error) error { return operationFailure{class: classRead, err: err} } +func authFailure(err error) error { return operationFailure{class: classRead, err: err} } + +func exitCode(err error) int { + var outputFailure outputError + if errors.As(err, &outputFailure) { + return 3 + } + var classified classifiedError + if errors.As(err, &classified) && classified.class == classRead { + return 3 + } + var operation operationFailure + if errors.As(err, &operation) && operation.class == classRead { + return 3 + } + return 2 +} + +func earlyTokens(args []string) []string { + var tokens []string + for index, arg := range args { + if arg == "--token" && index+1 < len(args) { + tokens = append(tokens, args[index+1]) + } + if strings.HasPrefix(arg, "--token=") { + tokens = append(tokens, strings.TrimPrefix(arg, "--token=")) + } + } + return tokens +} + +func bestEffortFileToken(deps dependencies) string { + home, err := deps.homeDir() + if err != nil { + return "" + } + paths, err := config.ResolvePaths(home, deps.lookupEnv) + if err != nil { + return "" + } + return config.Load(paths).Config.Token +} diff --git a/internal/cli/runtime_test.go b/internal/cli/runtime_test.go new file mode 100644 index 0000000..0edc5e9 --- /dev/null +++ b/internal/cli/runtime_test.go @@ -0,0 +1,304 @@ +package cli + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/cobra" + + "github.com/ardasevinc/mattermost-cli/internal/api" + "github.com/ardasevinc/mattermost-cli/internal/config" + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/presentation" +) + +func TestRuntimeResolvesCLIEnvFilePrecedenceAndNormalizesURL(t *testing.T) { + home := t.TempDir() + writeRuntimeConfig(t, home, `url = "https://file.example/base/" +token = "file-token" +redact = false +`) + env := map[string]string{"MM_URL": "https://env.example/path/", "MM_TOKEN": "env-token"} + state, command, captured := runtimeProbe(t, home, env, false) + command.SetArgs([]string{"--url", "https://CLI.example:443/chat/", "--token", "cli-token", "--redact", "probe"}) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + defer state.close() + defer state.releaseCredentials() + if captured.runtime.Config.URL != "https://cli.example/chat" || captured.runtime.Config.Token != "cli-token" { + t.Fatalf("resolved config = %+v", captured.runtime.Config) + } + if !captured.runtime.Config.Redact || captured.runtime.Config.RedactSource != config.SourceCLI { + t.Fatalf("redact = %v/%s, want true/cli", captured.runtime.Config.Redact, captured.runtime.Config.RedactSource) + } + if captured.runtime.Users == nil || captured.runtime.Teams == nil || captured.runtime.Channels == nil || captured.runtime.Posts == nil { + t.Fatal("runtime did not construct all Mattermost services") + } +} + +func TestRuntimeUsesMacIndependentXDGPath(t *testing.T) { + home := t.TempDir() + xdg := filepath.Join(t.TempDir(), "config") + path := filepath.Join(xdg, "mattermost-cli", "config.toml") + writeFile(t, path, "url = \"https://xdg.example\"\ntoken = \"xdg-token\"\n", 0o600) + state, command, captured := runtimeProbe(t, home, map[string]string{"XDG_CONFIG_HOME": xdg}, false) + defer state.close() + defer state.releaseCredentials() + command.SetArgs([]string{"probe"}) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + if captured.runtime.Config.File.ReadPath != path { + t.Fatalf("ReadPath = %q, want %q", captured.runtime.Config.File.ReadPath, path) + } +} + +func TestRuntimeRejectsUnsafeParseAndInsecureTokenConfig(t *testing.T) { + tests := []struct { + name, body, want string + mode os.FileMode + }{ + {name: "parse", body: "broken = [", mode: 0o600, want: "could not parse"}, + {name: "insecure token", body: "token = \"private-token\"", mode: 0o644, want: "must not be accessible"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + home := t.TempDir() + writeFile(t, filepath.Join(home, ".config", "mattermost-cli", "config.toml"), test.body, test.mode) + state, command, _ := runtimeProbe(t, home, nil, false) + defer state.close() + defer state.releaseCredentials() + command.SetArgs([]string{"probe"}) + err := command.Execute() + if err == nil || !strings.Contains(err.Error(), test.want) || exitCode(err) != 3 { + t.Fatalf("Execute() error = %v, exit=%d", err, exitCode(err)) + } + if strings.Contains(err.Error(), "private-token") { + t.Fatalf("error reflected token: %q", err) + } + }) + } +} + +func TestRuntimeRejectsUnsafeConfigPathBeforeClientCreation(t *testing.T) { + home := t.TempDir() + directory := filepath.Join(home, ".config", "mattermost-cli") + writeFile(t, filepath.Join(directory, "target.toml"), "url = \"https://example.com\"\ntoken = \"unsafe-token\"\n", 0o600) + if err := os.Symlink(filepath.Join(directory, "target.toml"), filepath.Join(directory, "config.toml")); err != nil { + t.Fatal(err) + } + state, command, _ := runtimeProbe(t, home, nil, false) + created := false + state.deps.newClient = func(string, string) (*api.Client, error) { + created = true + return nil, errors.New("must not run") + } + defer state.close() + defer state.releaseCredentials() + command.SetArgs([]string{"probe"}) + err := command.Execute() + if err == nil || exitCode(err) != 3 || created { + t.Fatalf("Execute() error=%v exit=%d clientCreated=%v", err, exitCode(err), created) + } +} + +func TestRuntimeMissingConfigIsNormalWhenCLIHasCredentials(t *testing.T) { + state, command, captured := runtimeProbe(t, t.TempDir(), nil, false) + defer state.close() + defer state.releaseCredentials() + command.SetArgs([]string{"--url", "https://example.com", "--token", "token", "probe"}) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + if captured.runtime.Config.File.Exists { + t.Fatal("missing config reported as existing") + } +} + +func TestRuntimeMigrationWarningOnceAndTTYInjection(t *testing.T) { + home := t.TempDir() + xdg := filepath.Join(t.TempDir(), "xdg") + writeRuntimeConfig(t, home, "url = \"https://legacy.example\"\ntoken = \"legacy-token\"\n") + state, command, captured := runtimeProbe(t, home, map[string]string{"XDG_CONFIG_HOME": xdg}, true) + defer state.close() + defer state.releaseCredentials() + command.SetArgs([]string{"probe"}) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + if _, err := state.runtimeFor(command); err != nil { + t.Fatal(err) + } + if got := strings.Count(state.streams.err.(*bytes.Buffer).String(), "warning:"); got != 1 { + t.Fatalf("warning count = %d, want 1", got) + } + if !captured.runtime.StdoutTTY { + t.Fatal("injected stdout TTY capability was not retained") + } +} + +func TestRuntimeCloseReleasesClientAndFileCredential(t *testing.T) { + home := t.TempDir() + const token = "file-owned-token" + writeRuntimeConfig(t, home, "url = \"https://example.com\"\ntoken = \""+token+"\"\n") + state, command, captured := runtimeProbe(t, home, nil, false) + command.SetArgs([]string{"probe"}) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + if !contains(presentation.ActiveCredentials.Values(), token) { + t.Fatal("resolved file credential was not active during command lifetime") + } + client := captured.runtime.Client + state.close() + state.releaseCredentials() + if contains(presentation.ActiveCredentials.Values(), token) { + t.Fatal("resolved file credential remained active after execution cleanup") + } + if err := client.Get(context.Background(), "/api/v4/users/me", new(any)); !errors.Is(err, api.ErrClientClosed) { + t.Fatalf("closed client Get() error = %v", err) + } +} + +func TestExitClassesAndSchemaIsolation(t *testing.T) { + if exitCode(invalidFailure("bad input")) != 2 || exitCode(configFailure("bad config")) != 3 || exitCode(outputError{err: errors.New("write")}) != 3 { + t.Fatal("stable exit class mapping changed") + } + t.Setenv("HOME", "relative-home-must-not-be-read") + var stdout, stderr bytes.Buffer + if code := Execute(context.Background(), []string{"schema", "list"}, strings.NewReader(""), &stdout, &stderr); code != 0 { + t.Fatalf("schema required runtime config: exit=%d stderr=%q", code, stderr.String()) + } +} + +func TestCLIFlagTokenIsNeverReflected(t *testing.T) { + const token = "cli-only-super-secret" + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--token", token, "not-a-command"}, strings.NewReader(""), &stdout, &stderr) + if code != 2 || strings.Contains(stderr.String(), token) { + t.Fatalf("exit=%d stderr reflected CLI token: %q", code, stderr.String()) + } +} + +func TestFileOnlyTokenMasksPreParseErrorsWithoutMakingSchemaDependOnConfig(t *testing.T) { + home := t.TempDir() + const token = "file-only-opaque-token" + writeRuntimeConfig(t, home, "token = \""+token+"\"\n") + t.Setenv("HOME", home) + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{token}, strings.NewReader(""), &stdout, &stderr) + if code != 2 || strings.Contains(stderr.String(), token) || !strings.Contains(stderr.String(), "mattermost_credential") { + t.Fatalf("exit=%d stderr=%q", code, stderr.String()) + } + writeFile(t, filepath.Join(home, ".config", "mattermost-cli", "config.toml"), "broken = [", 0o600) + stdout.Reset() + stderr.Reset() + if code := Execute(context.Background(), []string{"schema", "list"}, strings.NewReader(""), &stdout, &stderr); code != 0 { + t.Fatalf("schema exit=%d stderr=%q", code, stderr.String()) + } +} + +func TestAllRepeatedCLITokenFormsAreDiscoveredAndMasked(t *testing.T) { + const first = "first-opaque-decoy" + const second = "second-opaque-final" + if got := earlyTokens([]string{"--token", first, "--token=" + second}); len(got) != 2 || got[0] != first || got[1] != second { + t.Fatalf("earlyTokens() = %q", got) + } + for _, surface := range []string{first, second} { + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--token", first, "--token=" + second, "schema", "show", surface}, strings.NewReader(""), &stdout, &stderr) + if code != 2 || strings.Contains(stderr.String(), surface) { + t.Fatalf("surface %q: exit=%d stderr=%q", surface, code, stderr.String()) + } + } +} + +func TestReadAndAuthFailuresMapToThreeWhileRawErrorsRemainInvocationFailures(t *testing.T) { + apiErr := &api.APIError{Status: 503} + if code := exitCode(readFailure(apiErr)); code != 3 || !errors.Is(readFailure(apiErr), apiErr) { + t.Fatalf("read failure exit=%d", code) + } + authErr := &api.APIError{Status: 401} + if code := exitCode(authFailure(authErr)); code != 3 || !errors.Is(authFailure(authErr), authErr) { + t.Fatalf("auth failure exit=%d", code) + } + if code := exitCode(apiErr); code != 2 { + t.Fatalf("unwrapped operation error exit=%d, want 2", code) + } + if code := exitCode(readFailure(mattermost.ErrInvalidUsersResponse)); code != 3 { + t.Fatalf("wrapped Mattermost read error exit=%d, want 3", code) + } +} + +func TestConcurrentExecuteUsesOnlyInvocationLocalCredentialsForErrors(t *testing.T) { + const tokenA = "invocation-a-opaque" + const tokenB = "invocation-b-opaque" + for iteration := 0; iteration < 50; iteration++ { + start := make(chan struct{}) + results := make(chan string, 2) + run := func(own, other string) { + <-start + var stdout, stderr bytes.Buffer + _ = Execute(context.Background(), []string{"--token", own, other}, strings.NewReader(""), &stdout, &stderr) + results <- stderr.String() + } + go run(tokenA, tokenB) + go run(tokenB, tokenA) + close(start) + first, second := <-results, <-results + if !(strings.Contains(first, tokenA) || strings.Contains(first, tokenB)) || !(strings.Contains(second, tokenA) || strings.Contains(second, tokenB)) { + t.Fatalf("cross-invocation token was over-redacted: %q / %q", first, second) + } + } +} + +type capturedRuntime struct{ runtime *Runtime } + +func runtimeProbe(t *testing.T, home string, env map[string]string, tty bool) (*rootState, *cobra.Command, *capturedRuntime) { + t.Helper() + var stdout, stderr bytes.Buffer + lookup := func(key string) (string, bool) { value, ok := env[key]; return value, ok } + state := &rootState{streams: streams{in: strings.NewReader(""), out: &stdout, err: &stderr}, deps: dependencies{ + lookupEnv: lookup, homeDir: func() (string, error) { return home, nil }, + newClient: func(url, token string) (*api.Client, error) { return api.New(url, token) }, + stdoutTTY: func() bool { return tty }, + }} + command := newRootWithState(state) + captured := new(capturedRuntime) + command.AddCommand(&cobra.Command{Use: "probe", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { + runtime, err := state.runtimeFor(cmd) + captured.runtime = runtime + return err + }}) + return state, command, captured +} + +func writeRuntimeConfig(t *testing.T, home, body string) { + t.Helper() + writeFile(t, filepath.Join(home, ".config", "mattermost-cli", "config.toml"), body, 0o600) +} + +func writeFile(t *testing.T, path, body string, mode os.FileMode) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), mode); err != nil { + t.Fatal(err) + } +} + +func contains(values []string, target string) bool { + for _, value := range values { + if value == target { + return true + } + } + return false +} diff --git a/internal/schema/registry.go b/internal/schema/registry.go index 3a1eb36..ab23a2d 100644 --- a/internal/schema/registry.go +++ b/internal/schema/registry.go @@ -3,6 +3,7 @@ package schema import ( "bytes" "encoding/json" + "errors" "fmt" "io" "io/fs" @@ -16,6 +17,18 @@ import ( const maxDocumentBytes = 4 << 20 +// InputReadError identifies a physical failure while reading a validation +// document. JSON and schema validation errors deliberately do not use it. +type InputReadError struct{ err error } + +func (e *InputReadError) Error() string { return "could not read JSON document" } +func (e *InputReadError) Unwrap() error { return e.err } + +func IsInputReadError(err error) bool { + var target *InputReadError + return errors.As(err, &target) +} + type Registry struct { raw map[string][]byte compiled map[string]*jsonschema.Schema @@ -94,7 +107,7 @@ func (r *Registry) Validate(id string, input io.Reader) error { limited := io.LimitReader(input, maxDocumentBytes+1) data, err := io.ReadAll(limited) if err != nil { - return fmt.Errorf("read JSON document: %w", err) + return &InputReadError{err: err} } if len(data) > maxDocumentBytes { return fmt.Errorf("JSON document exceeds %d bytes", maxDocumentBytes) diff --git a/internal/schema/registry_test.go b/internal/schema/registry_test.go index d587e32..dfdac26 100644 --- a/internal/schema/registry_test.go +++ b/internal/schema/registry_test.go @@ -3,6 +3,7 @@ package schema import ( "bytes" "encoding/json" + "errors" "io/fs" "strings" "testing" @@ -39,6 +40,27 @@ func TestEmbeddedExamplesValidate(t *testing.T) { } } +func TestValidateTypesOnlyPhysicalInputReadFailures(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + physical := errors.New("fixture read failure") + err = registry.Validate("mm/v2/error", errorReader{err: physical}) + if !IsInputReadError(err) || !errors.Is(err, physical) || strings.Contains(err.Error(), physical.Error()) { + t.Fatalf("physical read error = %v", err) + } + for _, input := range []string{"{", `{}`} { + if err := registry.Validate("mm/v2/error", strings.NewReader(input)); err == nil || IsInputReadError(err) { + t.Fatalf("content error incorrectly typed as input read failure: %v", err) + } + } +} + +type errorReader struct{ err error } + +func (r errorReader) Read([]byte) (int, error) { return 0, r.err } + func TestValidateRejectsUnknownFields(t *testing.T) { registry, err := Load() if err != nil { From c124b8216913b940b35a732d82507af0cbbe8069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 13:51:15 +0300 Subject: [PATCH 030/119] feat: port channel read command --- internal/cli/channel.go | 210 +++++++++++++++++++++++++ internal/cli/channel_test.go | 286 +++++++++++++++++++++++++++++++++++ internal/cli/read.go | 180 ++++++++++++++++++++++ internal/cli/root.go | 76 ++++++++-- internal/cli/root_test.go | 32 ++++ internal/cli/runtime.go | 81 +++++++--- internal/cli/runtime_test.go | 52 +++++++ internal/cursor/cursor.go | 2 +- internal/output/machine.go | 12 +- 9 files changed, 899 insertions(+), 32 deletions(-) create mode 100644 internal/cli/channel.go create mode 100644 internal/cli/channel_test.go create mode 100644 internal/cli/read.go diff --git a/internal/cli/channel.go b/internal/cli/channel.go new file mode 100644 index 0000000..36cfc56 --- /dev/null +++ b/internal/cli/channel.go @@ -0,0 +1,210 @@ +package cli + +import ( + "regexp" + "strconv" + "time" + + "github.com/spf13/cobra" + + "github.com/ardasevinc/mattermost-cli/internal/cursor" + "github.com/ardasevinc/mattermost-cli/internal/output" + "github.com/ardasevinc/mattermost-cli/internal/retrieval" +) + +type channelFlags struct { + team, limit, since, cursor string +} + +func newChannelCommand(state *rootState) *cobra.Command { + flags := new(channelFlags) + command := &cobra.Command{ + Use: "channel ", Short: "Fetch messages from a channel by name", Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { return runChannel(cmd, state, args[0], *flags) }, + } + command.Flags().StringVar(&flags.team, "team", "", "team name (auto-detected for one team)") + command.Flags().StringVarP(&flags.limit, "limit", "l", "50", "maximum seed messages") + command.Flags().StringVarP(&flags.since, "since", "s", "7d", "time range such as 24h, 7d, 1w, or 2m") + command.Flags().StringVar(&flags.cursor, "cursor", "", "resume deterministic channel history") + return command +} + +func runChannel(cmd *cobra.Command, state *rootState, name string, flags channelFlags) error { + limit, err := positiveInteger(flags.limit) + if err != nil { + return err + } + display, err := state.readDisplay(cmd) + if err != nil { + return err + } + var resume *cursor.ChannelHistory + if flags.cursor != "" { + decoded, decodeErr := cursor.DecodeChannelHistory(flags.cursor) + if decodeErr != nil { + return invalidFailure("invalid channel cursor") + } + resume = &decoded + if cmd.Flags().Changed("since") { + return invalidFailure("a cursor cannot be combined with --since") + } + } + var since *int64 + if resume != nil { + since = resume.Since + } else { + value, durationErr := durationBoundary(flags.since, time.Now()) + if durationErr != nil { + return durationErr + } + since = &value + } + runtime, err := state.runtimeFor(cmd) + if err != nil { + return err + } + if !display.json { + if err := warnRedactionDisabled(state, runtime); err != nil { + return err + } + } + me, err := runtime.Users.Current(cmd.Context()) + if err != nil { + return readFailure(err) + } + team, err := runtime.Teams.Resolve(cmd.Context(), me.ID, flags.team) + if err != nil { + return readFailure(err) + } + channel, err := runtime.Channels.ByName(cmd.Context(), team.ID, name) + if err != nil { + return readFailure(err) + } + if resume != nil && resume.ChannelID != channel.ID { + return invalidFailure("cursor does not match the selected channel") + } + options := retrieval.ChannelHistoryOptions{Limit: limit, Since: since} + if resume != nil { + options.Boundary = &retrieval.Boundary{CreateAt: resume.Boundary.CreateAt, ID: resume.Boundary.ID} + options.SafeBeforePostID = resume.SafeBeforePostID + } + page, err := retrieval.ChannelHistory(cmd.Context(), runtime.Posts, channel.ID, options) + if err != nil { + return readFailure(err) + } + if len(page.Posts) == 0 && page.Completeness == retrieval.CompletenessUnknown && resume == nil { + return readError("Mattermost could not confirm an empty channel history") + } + + nextCursor := "" + if len(page.Posts) > 0 && page.Completeness != retrieval.CompletenessComplete { + boundary := page.Posts[len(page.Posts)-1] + safeBefore := "" + for index := len(page.Posts) - 1; index >= 0; index-- { + if page.Posts[index].CreateAt > boundary.CreateAt { + safeBefore = page.Posts[index].ID + break + } + } + if safeBefore == "" && page.SafeBeforeValid && resume != nil { + safeBefore = resume.SafeBeforePostID + } + nextCursor, err = cursor.EncodeChannelHistory(cursor.ChannelHistory{ + Version: 1, Scope: "channel", ChannelID: channel.ID, + Boundary: cursor.Boundary{CreateAt: boundary.CreateAt, ID: boundary.ID}, Since: since, SafeBeforePostID: safeBefore, + }) + if err != nil { + return readFailure(err) + } + } else if len(page.Posts) == 0 && page.Completeness == retrieval.CompletenessUnknown && resume != nil { + nextCursor = flags.cursor + } + + hydrated, err := retrieval.HydrateVisibleThreads(cmd.Context(), runtime.Posts, page.Posts, display.threads) + if err != nil { + return readFailure(err) + } + messages, redactions, err := normalizeReadPosts(cmd.Context(), runtime, hydrated.Posts, me.ID) + if err != nil { + return readFailure(err) + } + if display.threads { + messages = output.GroupIntoThreads(messages) + } + presentedChannel, channelRedactions := processedChannel(channel, runtime) + redactions = append(channelRedactions, redactions...) + threadsMetadata, threadRedactions := processedVisibleThreads(hydrated.VisibleThreads, runtime) + redactions = append(redactions, threadRedactions...) + selectedCount := len(page.Posts) + requestedLimit := limit + var sinceText *string + if since != nil { + value := time.UnixMilli(*since).UTC().Format("2006-01-02T15:04:05.000Z") + sinceText = &value + } + inputCursor, next := stringPointer(flags.cursor), stringPointer(nextCursor) + section := output.MessageOutput{Channel: presentedChannel, Messages: messages, Redactions: redactions, Retrieval: output.Retrieval{ + Selection: output.Selection{Source: "recent", SelectedCount: selectedCount, RequestedLimit: &requestedLimit, Since: sinceText, + QueryTruncated: truncatedPointer(page.Completeness), InputCursor: inputCursor, NextCursor: next}, + VisibleThreads: threadsMetadata, VisiblePostCount: len(hydrated.Posts), DeletedPostsIncluded: false, + }} + envelope, err := output.NewChannelEnvelope(section, machineCompleteness(page.Completeness)) + if err != nil { + return internalFailure(err) + } + humanOutputs := []output.MessageOutput{section} + if len(page.Posts) == 0 && page.Completeness == retrieval.CompletenessComplete { + humanOutputs = nil + } + return state.renderRead(humanOutputs, envelope, display) +} + +var durationPattern = regexp.MustCompile(`^([0-9]+)([hHdDwWmM])$`) + +func positiveInteger(value string) (int, error) { + if value == "" || value[0] == '0' { + return 0, invalidFailure("--limit must be a positive number") + } + for _, character := range value { + if character < '0' || character > '9' { + return 0, invalidFailure("--limit must be a positive number") + } + } + parsed, err := strconv.ParseInt(value, 10, 64) + if err != nil || parsed <= 0 || parsed > 9_007_199_254_740_991 { + return 0, invalidFailure("--limit must be a positive number") + } + return int(parsed), nil +} + +func durationBoundary(value string, now time.Time) (int64, error) { + match := durationPattern.FindStringSubmatch(value) + if match == nil { + return 0, invalidFailure(`--since must use a duration such as "24h", "7d", "1w", or "2m"`) + } + amount, err := strconv.ParseUint(match[1], 10, 63) + if err != nil { + return 0, invalidFailure("--since duration is too large") + } + hours := uint64(1) + switch match[2][0] | 0x20 { + case 'd': + hours = 24 + case 'w': + hours = 24 * 7 + case 'm': + hours = 24 * 30 + } + if amount > uint64(now.UnixMilli())/(hours*uint64(time.Hour/time.Millisecond)) { + return 0, invalidFailure("--since duration is too large") + } + return now.UnixMilli() - int64(amount*hours*uint64(time.Hour/time.Millisecond)), nil +} + +func stringPointer(value string) *string { + if value == "" { + return nil + } + copy := value + return © +} diff --git a/internal/cli/channel_test.go b/internal/cli/channel_test.go new file mode 100644 index 0000000..2ef0291 --- /dev/null +++ b/internal/cli/channel_test.go @@ -0,0 +1,286 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/ardasevinc/mattermost-cli/internal/api" + "github.com/ardasevinc/mattermost-cli/internal/config" + "github.com/ardasevinc/mattermost-cli/internal/cursor" + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/internal/retrieval" + mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" +) + +func TestChannelJSONRunsValidatedReadPipeline(t *testing.T) { + postTime := time.Now().Add(-time.Hour).UnixMilli() + server := channelServer(t, func(w http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/users/me": + writeJSON(t, w, `{"id":"user1","username":"arda"}`) + case "/api/v4/users/user1/teams": + writeJSON(t, w, `[{"id":"team1","name":"main","display_name":"Main","type":"O"}]`) + case "/api/v4/teams/team1/channels/name/town-square": + writeJSON(t, w, `{"id":"channel1","team_id":"team1","type":"O","name":"town-square","display_name":"Town Square"}`) + case "/api/v4/channels/channel1/posts": + post := fmt.Sprintf(`{"id":"post1","channel_id":"channel1","user_id":"user1","message":"**hello**","create_at":%d,"delete_at":0,"root_id":"","reply_count":0}`, postTime) + writeJSON(t, w, `{"order":["post1"],"posts":{"post1":`+post+`},"has_next":false}`) + case "/api/v4/users/ids": + if request.Method != http.MethodPost { + t.Fatalf("users/ids method = %s", request.Method) + } + writeJSON(t, w, `[{"id":"user1","username":"arda"}]`) + default: + t.Fatalf("unexpected request: %s %s", request.Method, request.URL.String()) + } + }) + defer server.Close() + + stdout, stderr, code := executeChannel(t, server.URL, "--json", "--no-threads", "channel", "town-square", "--team", "main", "--limit", "1") + if code != 0 || stderr != "" { + t.Fatalf("exit=%d stderr=%q", code, stderr) + } + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/channel", strings.NewReader(stdout)); err != nil { + t.Fatalf("machine output did not validate: %v\n%s", err, stdout) + } + var document struct { + Schema string `json:"schema"` + Data struct { + Messages []struct{ Text, User string } `json:"messages"` + Metadata struct { + Completeness string `json:"completeness"` + Selection struct { + SelectedCount int `json:"selectedCount"` + } `json:"selection"` + } `json:"metadata"` + } `json:"data"` + } + if err := json.Unmarshal([]byte(stdout), &document); err != nil { + t.Fatal(err) + } + if document.Schema != "mm/v2/channel" || len(document.Data.Messages) != 1 || document.Data.Messages[0].Text != "**hello**" || document.Data.Messages[0].User != "you" || document.Data.Metadata.Completeness != "complete" || document.Data.Metadata.Selection.SelectedCount != 1 { + t.Fatalf("unexpected document: %+v", document) + } +} + +func TestChannelRejectsInvalidCursorBeforeNetwork(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { requests.Add(1) })) + defer server.Close() + _, stderr, code := executeChannel(t, server.URL, "channel", "town-square", "--cursor", "not-a-cursor") + if code != 2 || !strings.Contains(stderr, "invalid channel cursor") || requests.Load() != 0 { + t.Fatalf("exit=%d stderr=%q requests=%d", code, stderr, requests.Load()) + } +} + +func TestChannelJSONFailureUsesMachineErrorSchema(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Fatal("network must not be used") })) + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "--json", "channel", "town-square", "--cursor", "not-a-cursor") + if code != 2 || stdout != "" || strings.HasPrefix(stderr, "error:") { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/error", strings.NewReader(stderr)); err != nil { + t.Fatalf("machine error did not validate: %v\n%s", err, stderr) + } +} + +func TestChannelMachineOutputFailureDistinguishesZeroFromPartialWrite(t *testing.T) { + server := basicEmptyChannelServer(t, false) + defer server.Close() + for _, test := range []struct { + name string + writer io.Writer + wantStderr bool + }{ + {name: "zero", writer: zeroErrorWriter{}, wantStderr: true}, + {name: "partial", writer: shortWriter{}, wantStderr: false}, + } { + t.Run(test.name, func(t *testing.T) { + setChannelEnvironment(t, server.URL) + var stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "--no-threads", "channel", "town-square"}, strings.NewReader(""), test.writer, &stderr) + if code != 3 || (stderr.Len() > 0) != test.wantStderr { + t.Fatalf("exit=%d stderr=%q", code, stderr.String()) + } + if test.wantStderr && !strings.Contains(stderr.String(), `"code":"internal"`) { + t.Fatalf("stderr=%q, want internal machine error", stderr.String()) + } + }) + } +} + +func TestNormalizeReadPostsBatchesMoreThanTwoHundredUsers(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/api/v4/users/ids" { + t.Errorf("unexpected request path %q", request.URL.Path) + http.Error(writer, "unexpected", http.StatusNotFound) + return + } + calls.Add(1) + var ids []string + if err := json.NewDecoder(request.Body).Decode(&ids); err != nil { + t.Error(err) + return + } + users := make([]map[string]string, len(ids)) + for index, id := range ids { + users[index] = map[string]string{"id": id, "username": "user_" + id} + } + _ = json.NewEncoder(writer).Encode(users) + })) + defer server.Close() + client, err := api.New(server.URL, "token") + if err != nil { + t.Fatal(err) + } + defer client.Close() + runtime := &Runtime{Config: config.Resolved{URL: server.URL, Token: "token", Redact: true}, Client: client, Users: mattermost.NewUsers(client)} + posts := make([]mattermost.Post, 201) + for index := range posts { + posts[index] = mattermost.Post{ID: fmt.Sprintf("post%d", index), ChannelID: "channel1", UserID: fmt.Sprintf("user%d", index), Message: "hello", CreateAt: 1, UpdateAt: 1} + } + messages, _, err := normalizeReadPosts(context.Background(), runtime, posts, "me") + if err != nil || len(messages) != len(posts) || calls.Load() != 2 { + t.Fatalf("messages=%d calls=%d err=%v", len(messages), calls.Load(), err) + } +} + +func TestFailedThreadRootAlwaysMasksActiveCredential(t *testing.T) { + runtime := &Runtime{Config: config.Resolved{Token: "secret-token", Redact: false}} + metadata, redactions := processedVisibleThreads(retrieval.VisibleThreadsMetadata{ + Status: retrieval.VisibleThreadsPartial, FailedRootIDs: []string{"secret-token"}, + }, runtime) + if len(metadata.FailedRootIDs) != 1 || metadata.FailedRootIDs[0] == "secret-token" || !strings.Contains(metadata.FailedRootIDs[0], "REDACTED") || len(redactions) != 1 || redactions[0].Field != "retrieval.failedRootId" { + t.Fatalf("metadata=%+v redactions=%+v", metadata, redactions) + } + if got := presentation.Preprocess(metadata.FailedRootIDs[0], []string{"secret-token"}).Text; got == "secret-token" { + t.Fatal("failed root was not masked") + } +} + +func TestChannelRejectsCursorWithExplicitSinceBeforeNetwork(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { requests.Add(1) })) + defer server.Close() + since := time.Now().Add(-24 * time.Hour).UnixMilli() + encoded, err := cursor.EncodeChannelHistory(cursor.ChannelHistory{Version: 1, Scope: "channel", ChannelID: "channel1", Boundary: cursor.Boundary{CreateAt: time.Now().UnixMilli(), ID: "post1"}, Since: &since}) + if err != nil { + t.Fatal(err) + } + _, stderr, code := executeChannel(t, server.URL, "channel", "town-square", "--cursor", encoded, "--since", "1d") + if code != 2 || !strings.Contains(stderr, "cannot be combined") || requests.Load() != 0 { + t.Fatalf("exit=%d stderr=%q requests=%d", code, stderr, requests.Load()) + } +} + +func TestDurationAndLimitValidation(t *testing.T) { + now := time.UnixMilli(10 * int64(24*time.Hour/time.Millisecond)) + boundary, err := durationBoundary("2d", now) + if err != nil || boundary != now.Add(-48*time.Hour).UnixMilli() { + t.Fatalf("durationBoundary = %d, %v", boundary, err) + } + for _, invalid := range []string{"", "01", "0", "-1", "1.5", "9007199254740992"} { + if _, err := positiveInteger(invalid); err == nil { + t.Errorf("positiveInteger(%q) succeeded", invalid) + } + } +} + +func TestChannelCompleteEmptyHumanOutput(t *testing.T) { + server := basicEmptyChannelServer(t, false) + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "--no-threads", "channel", "town-square") + if code != 0 || stderr != "" || stdout != "No messages found.\n" { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } +} + +func TestChannelUnknownEmptyContinuationPreservesCursor(t *testing.T) { + server := basicEmptyChannelServer(t, true) + defer server.Close() + since := time.Now().Add(-24 * time.Hour).UnixMilli() + encoded, err := cursor.EncodeChannelHistory(cursor.ChannelHistory{Version: 1, Scope: "channel", ChannelID: "channel1", Boundary: cursor.Boundary{CreateAt: time.Now().UnixMilli(), ID: "post1"}, Since: &since}) + if err != nil { + t.Fatal(err) + } + stdout, stderr, code := executeChannel(t, server.URL, "--no-threads", "channel", "town-square", "--cursor", encoded) + if code != 0 || stderr != "" || !strings.Contains(stdout, "completeness unknown") || !strings.Contains(stdout, "Next cursor: `"+encoded+"`") { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } +} + +func basicEmptyChannelServer(t *testing.T, uncertain bool) *httptest.Server { + t.Helper() + return channelServer(t, func(w http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/users/me": + writeJSON(t, w, `{"id":"user1","username":"arda"}`) + case "/api/v4/users/user1/teams": + writeJSON(t, w, `[{"id":"team1","name":"main","display_name":"Main","type":"O"}]`) + case "/api/v4/teams/team1/channels/name/town-square": + writeJSON(t, w, `{"id":"channel1","team_id":"team1","type":"O","name":"town-square","display_name":"Town Square"}`) + case "/api/v4/channels/channel1/posts": + if uncertain { + writeJSON(t, w, `{"order":[],"posts":{},"has_next":true}`) + } else { + writeJSON(t, w, `{"order":[],"posts":{},"has_next":false}`) + } + default: + t.Fatalf("unexpected request: %s %s", request.Method, request.URL.String()) + } + }) +} + +func channelServer(t *testing.T, handler http.HandlerFunc) *httptest.Server { + t.Helper() + return httptest.NewServer(handler) +} + +func executeChannel(t *testing.T, serverURL string, args ...string) (string, string, int) { + t.Helper() + setChannelEnvironment(t, serverURL) + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), args, strings.NewReader(""), &stdout, &stderr) + return stdout.String(), stderr.String(), code +} + +func setChannelEnvironment(t *testing.T, serverURL string) { + t.Helper() + t.Setenv("HOME", t.TempDir()) + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + t.Setenv("MM_URL", serverURL) + t.Setenv("MM_TOKEN", "test-token") +} + +type zeroErrorWriter struct{} + +func (zeroErrorWriter) Write([]byte) (int, error) { return 0, errors.New("closed") } + +func writeJSON(t *testing.T, writer http.ResponseWriter, body string) { + t.Helper() + writer.Header().Set("Content-Type", "application/json") + if _, err := writer.Write([]byte(body)); err != nil { + t.Fatal(err) + } +} diff --git a/internal/cli/read.go b/internal/cli/read.go new file mode 100644 index 0000000..9270599 --- /dev/null +++ b/internal/cli/read.go @@ -0,0 +1,180 @@ +package cli + +import ( + "context" + "fmt" + "time" + "unicode/utf16" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/normalization" + "github.com/ardasevinc/mattermost-cli/internal/output" + "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/internal/retrieval" + "github.com/spf13/cobra" +) + +type readDisplayOptions struct { + json, color, relative, threads bool +} + +func (s *rootState) readDisplay(cmd *cobra.Command) (readDisplayOptions, error) { + if flagChanged(cmd, "relative") && flagChanged(cmd, "no-relative") { + return readDisplayOptions{}, invalidFailure("--relative and --no-relative cannot be used together") + } + if flagChanged(cmd, "threads") && flagChanged(cmd, "no-threads") { + return readDisplayOptions{}, invalidFailure("--threads and --no-threads cannot be used together") + } + return readDisplayOptions{ + json: s.flags.json, color: !s.flags.noColor, + relative: (s.flags.relative || detectsAgent(s.deps.lookupEnv)) && !s.flags.noRelative, + threads: s.flags.threads && !s.flags.noThreads, + }, nil +} + +func detectsAgent(lookup func(string) (string, bool)) bool { + for _, name := range []string{"CLAUDECODE", "GEMINI_CLI", "CODEX_CI", "OPENCODE"} { + if value, ok := lookup(name); ok && value == "1" { + return true + } + } + return false +} + +func flagChanged(cmd *cobra.Command, name string) bool { + flag := cmd.Flags().Lookup(name) + return flag != nil && flag.Changed +} + +func normalizeReadPosts(ctx context.Context, runtime *Runtime, posts []mattermost.Post, myUserID string) ([]output.Message, []output.Redaction, error) { + users := make(map[string]mattermost.User) + ids := normalization.PostUserIDs(posts) + if len(ids) != 0 { + for start := 0; start < len(ids); start += 200 { + end := start + 200 + if end > len(ids) { + end = len(ids) + } + items, err := runtime.Users.ByIDs(ctx, ids[start:end]) + if err != nil { + return nil, nil, err + } + for _, user := range items { + users[user.ID] = user + } + } + } + return normalization.NormalizePosts(posts, users, myUserID, runtime.Config.URL, presentation.Options{ + Credentials: []string{runtime.Config.Token}, DisableHeuristics: !runtime.Config.Redact, + }) +} + +func processedChannel(raw mattermost.Channel, runtime *Runtime) (output.Channel, []output.Redaction) { + options := presentation.Options{Credentials: []string{runtime.Config.Token}, DisableHeuristics: !runtime.Config.Redact} + redactions := make([]output.Redaction, 0) + clean := func(value, field string) string { + result := presentation.PreprocessWithOptions(value, options) + remapLabelPositions(result.Text, result.Redactions) + result.Text = presentation.SanitizeLabel(result.Text) + for _, redaction := range result.Redactions { + redaction.Field = field + redactions = append(redactions, redaction) + } + return result.Text + } + typeName := map[string]string{"O": "public", "P": "private", "D": "dm", "G": "group"}[raw.Type] + return output.Channel{ + ID: clean(raw.ID, "channel.id"), Type: typeName, Name: clean(raw.Name, "channel.name"), + DisplayName: clean(raw.DisplayName, "channel.displayName"), MetadataStatus: "resolved", + }, redactions +} + +func remapLabelPositions(text string, redactions []presentation.Redaction) { + for index := range redactions { + target, original, added := redactions[index].Position, 0, 0 + for _, character := range text { + width := len(utf16.Encode([]rune{character})) + if original+width > target { + break + } + original += width + if character == '\n' || character == '\t' { + added++ + } + } + redactions[index].Position += added + } +} + +func processedVisibleThreads(value retrieval.VisibleThreadsMetadata, runtime *Runtime) (output.VisibleThreads, []output.Redaction) { + status := "not_requested" + switch value.Status { + case retrieval.VisibleThreadsComplete: + status = "complete" + case retrieval.VisibleThreadsPartial: + status = "partial" + } + failed := make([]string, len(value.FailedRootIDs)) + redactions := make([]output.Redaction, 0) + options := presentation.Options{Credentials: []string{runtime.Config.Token}, DisableHeuristics: !runtime.Config.Redact} + for index, id := range value.FailedRootIDs { + result := presentation.PreprocessWithOptions(id, options) + remapLabelPositions(result.Text, result.Redactions) + failed[index] = presentation.SanitizeLabel(result.Text) + for _, redaction := range result.Redactions { + redaction.Field = "retrieval.failedRootId" + redactions = append(redactions, redaction) + } + } + return output.VisibleThreads{Status: status, HydratedRootCount: value.HydratedRootCount, FailedRootIDs: failed}, redactions +} + +func machineCompleteness(value retrieval.Completeness) output.MachineCompleteness { + switch value { + case retrieval.CompletenessComplete: + return output.MachineComplete + case retrieval.CompletenessTruncated: + return output.MachineTruncated + default: + return output.MachineUnknown + } +} + +func truncatedPointer(value retrieval.Completeness) *bool { + if value == retrieval.CompletenessUnknown { + return nil + } + result := value == retrieval.CompletenessTruncated + return &result +} + +func (s *rootState) renderRead(outputs []output.MessageOutput, document output.MachineDocument, display readDisplayOptions) error { + if display.json { + if _, err := output.WriteMachineJSON(s.streams.out, document); err != nil { + return outputError{err: err} + } + return nil + } + if len(outputs) == 0 { + return writeAll(s.streams.out, []byte("No messages found.\n")) + } + dates := output.NewDateFormatter(time.Now, time.Local) + var formatted string + if display.color && s.deps.stdoutTTY() { + formatted = output.FormatPretty(outputs, dates, output.PrettyOptions{Color: true, Relative: display.relative}) + } else if s.deps.stdoutTTY() { + formatted = output.FormatPretty(outputs, dates, output.PrettyOptions{Relative: display.relative}) + } else { + formatted = output.FormatMarkdown(outputs, dates, output.MarkdownOptions{Relative: display.relative}) + } + return writeAll(s.streams.out, []byte(formatted+"\n")) +} + +func warnRedactionDisabled(s *rootState, runtime *Runtime) error { + if runtime.Config.Redact { + return nil + } + return writeAll(s.streams.err, []byte("warning: secret redaction is disabled; output may contain secrets\n")) +} + +func readError(message string) error { return readFailure(fmt.Errorf("%s", message)) } diff --git a/internal/cli/root.go b/internal/cli/root.go index 9639359..e11d4b4 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -2,6 +2,7 @@ package cli import ( "context" + "errors" "fmt" "io" "os" @@ -10,6 +11,7 @@ import ( "github.com/spf13/cobra" "github.com/ardasevinc/mattermost-cli/internal/buildinfo" + "github.com/ardasevinc/mattermost-cli/internal/output" "github.com/ardasevinc/mattermost-cli/internal/presentation" mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" ) @@ -36,20 +38,50 @@ func Execute(ctx context.Context, args []string, in io.Reader, out, errOut io.Wr cmd.SetArgs(args) if err := cmd.ExecuteContext(ctx); err != nil { message := presentation.SanitizeLabel(presentation.Preprocess(err.Error(), state.credentials).Text) - if writeErr := writeAll(errOut, []byte(fmt.Sprintf("error: %s\n", message))); writeErr != nil { + code := exitCode(err) + if state.flags.json && trackedOut.BytesWritten() > 0 { + return 3 + } + if state.flags.json { + document := output.ErrorEnvelope{Schema: "mm/v2/error", Code: machineErrorCode(err), Message: message, ExitCode: code, Recovery: "none"} + if _, writeErr := output.WriteMachineJSON(errOut, document); writeErr != nil { + return 3 + } + } else if writeErr := writeAll(errOut, []byte(fmt.Sprintf("error: %s\n", message))); writeErr != nil { return 3 } if trackedOut.Failed() { return 3 } - return exitCode(err) + return code } if trackedOut.Failed() { return 3 } + if state.flags.json { + if err := state.flushMachineWarnings(); err != nil { + return 3 + } + } return 0 } +func machineErrorCode(err error) string { + var outputFailure outputError + if errors.As(err, &outputFailure) { + return "internal" + } + var classified classifiedError + if errors.As(err, &classified) { + return classified.code + } + var operation operationFailure + if errors.As(err, &operation) { + return operation.code + } + return "invalid_invocation" +} + func newRoot(s streams) *cobra.Command { return newRootWithState(&rootState{streams: s, deps: defaultDependencies(s.out)}) } @@ -77,15 +109,32 @@ func newRootWithState(state *rootState) *cobra.Command { cmd.PersistentFlags().StringVar(&state.flags.token, "token", "", "Mattermost personal access token") cmd.PersistentFlags().BoolVar(&state.flags.redact, "redact", true, "redact detected secrets") cmd.PersistentFlags().BoolVar(&state.flags.noRedact, "no-redact", false, "disable heuristic secret redaction") - cmd.AddCommand(newSchemaCommand(s)) + cmd.PersistentFlags().BoolVar(&state.flags.json, "json", false, "output a versioned JSON document") + cmd.PersistentFlags().BoolVar(&state.flags.noColor, "no-color", false, "disable colored output") + cmd.PersistentFlags().BoolVarP(&state.flags.relative, "relative", "r", false, "show relative times") + cmd.PersistentFlags().BoolVar(&state.flags.noRelative, "no-relative", false, "show absolute times") + cmd.PersistentFlags().BoolVar(&state.flags.threads, "threads", true, "show visible thread structure") + cmd.PersistentFlags().BoolVar(&state.flags.noThreads, "no-threads", false, "return selected seed posts only") + cmd.AddCommand(newSchemaCommand(state)) + cmd.AddCommand(newChannelCommand(state)) return cmd } -func newSchemaCommand(s streams) *cobra.Command { +func newSchemaCommand(state *rootState) *cobra.Command { + s := state.streams command := &cobra.Command{ Use: "schema", Short: "Inspect and validate machine schemas", Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return cmd.Help() + }, + PersistentPreRunE: func(_ *cobra.Command, _ []string) error { + if state.flags.json { + return invalidFailure("--json is not supported by schema inspection commands") + } + return nil + }, } command.AddCommand(&cobra.Command{ Use: "list", @@ -94,7 +143,7 @@ func newSchemaCommand(s streams) *cobra.Command { RunE: func(_ *cobra.Command, _ []string) error { registry, err := mmSchema.Load() if err != nil { - return readFailure(err) + return internalFailure(err) } return writeAll(s.out, []byte(strings.Join(registry.IDs(), "\n")+"\n")) }, @@ -106,11 +155,11 @@ func newSchemaCommand(s streams) *cobra.Command { RunE: func(_ *cobra.Command, args []string) error { registry, err := mmSchema.Load() if err != nil { - return readFailure(err) + return internalFailure(err) } data, err := registry.Show(args[0]) if err != nil { - return err + return invalidFailure(err.Error()) } if err := writeAll(s.out, data); err != nil { return err @@ -128,13 +177,13 @@ func newSchemaCommand(s streams) *cobra.Command { RunE: func(_ *cobra.Command, args []string) error { registry, err := mmSchema.Load() if err != nil { - return readFailure(err) + return internalFailure(err) } if err := registry.Validate(args[0], s.in); err != nil { if mmSchema.IsInputReadError(err) { return readFailure(err) } - return err + return invalidFailure(err.Error()) } return writeAll(s.out, []byte("valid: "+args[0]+"\n")) }, @@ -147,19 +196,22 @@ type outputError struct { } type writeTracker struct { - writer io.Writer - failed bool + writer io.Writer + failed bool + written int64 } func (w *writeTracker) Write(data []byte) (int, error) { written, err := w.writer.Write(data) + w.written += int64(written) if err != nil || written != len(data) { w.failed = true } return written, err } -func (w *writeTracker) Failed() bool { return w.failed } +func (w *writeTracker) Failed() bool { return w.failed } +func (w *writeTracker) BytesWritten() int64 { return w.written } func (e outputError) Error() string { return "write output failed" diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index 49e5610..93ff9ae 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -7,6 +7,8 @@ import ( "io" "strings" "testing" + + "github.com/ardasevinc/mattermost-cli/internal/api" ) func TestExecuteVersion(t *testing.T) { @@ -150,6 +152,36 @@ func TestSchemaShowTreatsShortWriteAsReadFailure(t *testing.T) { } } +func TestSchemaCommandsRejectJSONWithMachineError(t *testing.T) { + for _, args := range [][]string{{"--json", "schema"}, {"--json", "schema", "list"}} { + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), args, strings.NewReader(""), &stdout, &stderr) + if code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), `"code":"invalid_input"`) { + t.Fatalf("args=%q exit=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String()) + } + } +} + +func TestMachineErrorCodePreservesSemantics(t *testing.T) { + tests := []struct { + err error + want string + }{ + {errors.New("cobra"), "invalid_invocation"}, + {invalidFailure("bad"), "invalid_input"}, + {configFailure("bad"), "configuration"}, + {readFailure(errors.New("bad")), "read_failed"}, + {readFailure(&api.APIError{Status: 401}), "authentication"}, + {readFailure(&api.APIError{Status: 403}), "authorization"}, + {outputError{err: errors.New("bad")}, "internal"}, + } + for _, test := range tests { + if got := machineErrorCode(test.err); got != test.want { + t.Errorf("machineErrorCode(%T) = %q, want %q", test.err, got, test.want) + } + } +} + func TestErrorOutputShortWriteReturnsOutputFailure(t *testing.T) { var stdout bytes.Buffer code := Execute(context.Background(), []string{"unknown"}, strings.NewReader(""), &stdout, shortWriter{}) diff --git a/internal/cli/runtime.go b/internal/cli/runtime.go index e0354b6..3610a48 100644 --- a/internal/cli/runtime.go +++ b/internal/cli/runtime.go @@ -61,20 +61,27 @@ type rootState struct { deps dependencies flags runtimeFlags - mu sync.Mutex - runtime *Runtime - runtimeErr error - resolved bool - warned bool - releases []func() - credentials []string + mu sync.Mutex + runtime *Runtime + runtimeErr error + resolved bool + warned bool + releases []func() + credentials []string + pendingWarnings []string } type runtimeFlags struct { - url string - token string - redact bool - noRedact bool + url string + token string + redact bool + noRedact bool + json bool + noColor bool + relative bool + noRelative bool + threads bool + noThreads bool } func (s *rootState) close() { @@ -133,9 +140,13 @@ func (s *rootState) runtimeFor(cmd *cobra.Command) (*Runtime, error) { } if warning := file.Warning(); warning != "" && !s.warned { warning = presentation.SanitizeLabel(presentation.Preprocess(warning, s.credentials).Text) - if err := writeAll(s.streams.err, []byte("warning: "+warning+"\n")); err != nil { - s.runtimeErr = err - return nil, err + if s.flags.json { + s.pendingWarnings = append(s.pendingWarnings, "warning: "+warning+"\n") + } else { + if err := writeAll(s.streams.err, []byte("warning: "+warning+"\n")); err != nil { + s.runtimeErr = err + return nil, err + } } s.warned = true } @@ -180,6 +191,17 @@ func (s *rootState) runtimeFor(cmd *cobra.Command) (*Runtime, error) { return s.runtime, nil } +func (s *rootState) flushMachineWarnings() error { + s.mu.Lock() + warnings := strings.Join(s.pendingWarnings, "") + s.pendingWarnings = nil + s.mu.Unlock() + if warnings == "" { + return nil + } + return writeAll(s.streams.err, []byte(warnings)) +} + type errorClass uint8 const ( @@ -189,16 +211,22 @@ const ( type classifiedError struct { class errorClass + code string msg string } func (e classifiedError) Error() string { return e.msg } -func invalidFailure(message string) error { return classifiedError{class: classInvalid, msg: message} } -func configFailure(message string) error { return classifiedError{class: classRead, msg: message} } +func invalidFailure(message string) error { + return classifiedError{class: classInvalid, code: "invalid_input", msg: message} +} +func configFailure(message string) error { + return classifiedError{class: classRead, code: "configuration", msg: message} +} type operationFailure struct { class errorClass + code string err error } @@ -206,8 +234,25 @@ func (e operationFailure) Error() string { return e.err.Error() } func (e operationFailure) Unwrap() error { return e.err } // readFailure and authFailure preserve the v2 exit contract at command boundaries. -func readFailure(err error) error { return operationFailure{class: classRead, err: err} } -func authFailure(err error) error { return operationFailure{class: classRead, err: err} } +func readFailure(err error) error { + code := "read_failed" + var remote *api.APIError + if errors.As(err, &remote) { + switch remote.Status { + case 401: + code = "authentication" + case 403: + code = "authorization" + } + } + return operationFailure{class: classRead, code: code, err: err} +} +func authFailure(err error) error { + return operationFailure{class: classRead, code: "authentication", err: err} +} +func internalFailure(err error) error { + return operationFailure{class: classRead, code: "internal", err: err} +} func exitCode(err error) int { var outputFailure outputError diff --git a/internal/cli/runtime_test.go b/internal/cli/runtime_test.go index 0edc5e9..96a5d44 100644 --- a/internal/cli/runtime_test.go +++ b/internal/cli/runtime_test.go @@ -59,6 +59,36 @@ func TestRuntimeUsesMacIndependentXDGPath(t *testing.T) { } } +func TestReadDisplayAcceptsNegativeBooleanFlags(t *testing.T) { + state, command, _ := runtimeProbe(t, t.TempDir(), map[string]string{"MM_URL": "https://example.com", "MM_TOKEN": "token"}, false) + defer state.close() + defer state.releaseCredentials() + + command.SetArgs([]string{"--no-threads", "probe"}) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + display, err := state.readDisplay(command) + if err != nil || display.threads { + t.Fatalf("readDisplay = %+v, %v", display, err) + } +} + +func TestReadDisplayPreservesAgentRelativeDefault(t *testing.T) { + env := map[string]string{"MM_URL": "https://example.com", "MM_TOKEN": "token", "CODEX_CI": "1"} + state, command, _ := runtimeProbe(t, t.TempDir(), env, false) + defer state.close() + defer state.releaseCredentials() + command.SetArgs([]string{"probe"}) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + display, err := state.readDisplay(command) + if err != nil || !display.relative { + t.Fatalf("readDisplay = %+v, %v", display, err) + } +} + func TestRuntimeRejectsUnsafeParseAndInsecureTokenConfig(t *testing.T) { tests := []struct { name, body, want string @@ -143,6 +173,28 @@ func TestRuntimeMigrationWarningOnceAndTTYInjection(t *testing.T) { } } +func TestMachineRuntimeDefersMigrationWarningUntilSuccessfulCompletion(t *testing.T) { + home := t.TempDir() + xdg := filepath.Join(t.TempDir(), "xdg") + writeRuntimeConfig(t, home, "url = \"https://legacy.example\"\ntoken = \"legacy-token\"\n") + state, command, _ := runtimeProbe(t, home, map[string]string{"XDG_CONFIG_HOME": xdg}, false) + defer state.close() + defer state.releaseCredentials() + command.SetArgs([]string{"--json", "probe"}) + if err := command.Execute(); err != nil { + t.Fatal(err) + } + if got := state.streams.err.(*bytes.Buffer).String(); got != "" { + t.Fatalf("machine stderr before completion = %q", got) + } + if err := state.flushMachineWarnings(); err != nil { + t.Fatal(err) + } + if got := state.streams.err.(*bytes.Buffer).String(); !strings.Contains(got, "warning:") || !strings.Contains(got, ".config/mattermost-cli") { + t.Fatalf("machine completion warning = %q", got) + } +} + func TestRuntimeCloseReleasesClientAndFileCredential(t *testing.T) { home := t.TempDir() const token = "file-owned-token" diff --git a/internal/cursor/cursor.go b/internal/cursor/cursor.go index 1a6c77e..0635477 100644 --- a/internal/cursor/cursor.go +++ b/internal/cursor/cursor.go @@ -91,7 +91,7 @@ func ComparePostIDs(a, b string) int { func valid(value ChannelHistory) bool { return value.Version == 1 && value.Scope == "channel" && isSafeID(value.ChannelID) && isSafeID(value.Boundary.ID) && - value.Boundary.CreateAt >= 0 && value.Boundary.CreateAt <= maxDateMillis && + value.Boundary.CreateAt > 0 && value.Boundary.CreateAt <= maxDateMillis && (value.Since == nil || (*value.Since >= 0 && *value.Since <= value.Boundary.CreateAt)) && (value.SafeBeforePostID == "" || isSafeID(value.SafeBeforePostID)) } diff --git a/internal/output/machine.go b/internal/output/machine.go index cca00d0..4fbe955 100644 --- a/internal/output/machine.go +++ b/internal/output/machine.go @@ -114,6 +114,13 @@ type MentionsEnvelope struct { Schema string `json:"schema"` Results []MachineHistory `json:"results"` } +type ErrorEnvelope struct { + Schema string `json:"schema"` + Code string `json:"code"` + Message string `json:"message"` + ExitCode int `json:"exitCode"` + Recovery string `json:"recovery"` +} type MachineDocument interface{ machineDocument() } @@ -123,6 +130,7 @@ func (ChannelEnvelope) machineDocument() {} func (ThreadEnvelope) machineDocument() {} func (SearchEnvelope) machineDocument() {} func (MentionsEnvelope) machineDocument() {} +func (ErrorEnvelope) machineDocument() {} type wireMessage struct { ID string `json:"id"` @@ -234,6 +242,8 @@ func canonicalMachineDocument(document MachineDocument) (any, error) { Schema string `json:"schema"` Results []wireHistory `json:"results"` }{value.Schema, canonicalHistories(value.Results)}, nil + case ErrorEnvelope: + return value, nil default: return nil, fmt.Errorf("unsupported machine document type %T", document) } @@ -388,7 +398,7 @@ const ( func preflightMachineDocument(document MachineDocument) error { switch document.(type) { - case DMSEnvelope, GroupDMSEnvelope, ChannelEnvelope, ThreadEnvelope, SearchEnvelope, MentionsEnvelope: + case DMSEnvelope, GroupDMSEnvelope, ChannelEnvelope, ThreadEnvelope, SearchEnvelope, MentionsEnvelope, ErrorEnvelope: default: return fmt.Errorf("unsupported machine document type %T", document) } From b3fb7999cfceb10cffd76d7a8698224e49fa41b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 14:50:18 +0300 Subject: [PATCH 031/119] feat: port thread read command --- docs/V2_CONTRACT.md | 2 + internal/cli/channel.go | 6 +- internal/cli/read.go | 78 +++++++++++++- internal/cli/root.go | 1 + internal/cli/runtime.go | 6 ++ internal/cli/thread.go | 117 ++++++++++++++++++++ internal/cli/thread_test.go | 137 ++++++++++++++++++++++++ internal/normalization/post.go | 2 + internal/output/machine.go | 45 +++++--- internal/output/machine_convert.go | 127 +++++++++++++++++----- internal/output/machine_convert_test.go | 56 +++++++++- internal/output/model.go | 5 +- internal/output/threading.go | 8 +- internal/output/threading_test.go | 30 +++--- internal/schema/read_test.go | 121 +++++++++++++++++++-- schemas/v2/examples/thread.json | 2 +- schemas/v2/thread.schema.json | 2 +- 17 files changed, 667 insertions(+), 78 deletions(-) create mode 100644 internal/cli/thread.go create mode 100644 internal/cli/thread_test.go diff --git a/docs/V2_CONTRACT.md b/docs/V2_CONTRACT.md index 84e96eb..239b40e 100644 --- a/docs/V2_CONTRACT.md +++ b/docs/V2_CONTRACT.md @@ -123,6 +123,8 @@ Examples: Checked-in JSON Schemas and golden examples are part of the release contract. Unknown fields may be added only where the schema explicitly permits them. Field removal, meaning changes, or type changes require a new schema identifier. +Thread envelopes never invent a root from zero-value or incomplete remote shape. `data.root` is the proven canonical root or `null`; rootless and otherwise unbound partial posts are retained in `data.unboundPosts`, with non-complete retrieval and partial visible-thread metadata. + `--json`, `--from-json`, and JSONL watch mode imply machine mode. Machine mode never prompts, launches an editor, emits ANSI, or mixes diagnostics into stdout. For a one-shot command that fails before any successful envelope emission, stdout is empty and one schema-valid `mm/v2/error` object is written to stderr. Watch diagnostics are JSONL error/diagnostic envelopes on stderr after any prior events. A low-level partial stream write can make byte-perfect recovery impossible; it is classified honestly and never followed by another object pretending the stream remained valid. Human and machine surfaces use the same underlying operations and validation. Exit classes are stable: diff --git a/internal/cli/channel.go b/internal/cli/channel.go index 36cfc56..0329374 100644 --- a/internal/cli/channel.go +++ b/internal/cli/channel.go @@ -63,10 +63,8 @@ func runChannel(cmd *cobra.Command, state *rootState, name string, flags channel if err != nil { return err } - if !display.json { - if err := warnRedactionDisabled(state, runtime); err != nil { - return err - } + if err := emitRedactionWarning(state, runtime, display.json); err != nil { + return err } me, err := runtime.Users.Current(cmd.Context()) if err != nil { diff --git a/internal/cli/read.go b/internal/cli/read.go index 9270599..17484ee 100644 --- a/internal/cli/read.go +++ b/internal/cli/read.go @@ -2,10 +2,13 @@ package cli import ( "context" + "errors" "fmt" + "strings" "time" "unicode/utf16" + "github.com/ardasevinc/mattermost-cli/internal/api" "github.com/ardasevinc/mattermost-cli/internal/mattermost" "github.com/ardasevinc/mattermost-cli/internal/normalization" "github.com/ardasevinc/mattermost-cli/internal/output" @@ -83,12 +86,78 @@ func processedChannel(raw mattermost.Channel, runtime *Runtime) (output.Channel, return result.Text } typeName := map[string]string{"O": "public", "P": "private", "D": "dm", "G": "group"}[raw.Type] + if raw.Type == "G" { + name := raw.DisplayName + if name == "" { + name = raw.Name + } + return output.Channel{ID: clean(raw.ID, "channel.id"), Type: "group", Name: clean(name, "channel.displayName"), MetadataStatus: "resolved"}, redactions + } return output.Channel{ ID: clean(raw.ID, "channel.id"), Type: typeName, Name: clean(raw.Name, "channel.name"), DisplayName: clean(raw.DisplayName, "channel.displayName"), MetadataStatus: "resolved", }, redactions } +func resolvedReadChannel(ctx context.Context, runtime *Runtime, channelID, myUserID string) (output.Channel, []output.Redaction, bool, error) { + if channelID == "" { + channel, redactions := unavailableChannel(channelID, runtime) + return channel, redactions, true, nil + } + raw, err := runtime.Channels.ByID(ctx, channelID) + if err != nil { + var remote *api.APIError + if errors.As(err, &remote) && remote.Status == 401 { + return output.Channel{}, nil, false, err + } + channel, redactions := unavailableChannel(channelID, runtime) + return channel, redactions, true, nil + } + if raw.Type != "D" { + channel, redactions := processedChannel(raw, runtime) + return channel, redactions, false, nil + } + parts := strings.Split(raw.Name, "__") + if len(parts) != 2 || (parts[0] != myUserID && parts[1] != myUserID) { + channel, redactions := unavailableChannel(channelID, runtime) + return channel, redactions, true, nil + } + otherID := parts[0] + if otherID == myUserID { + otherID = parts[1] + } + other, err := runtime.Users.ByID(ctx, otherID) + if err != nil { + var remote *api.APIError + if errors.As(err, &remote) && remote.Status == 401 { + return output.Channel{}, nil, false, err + } + channel, redactions := unavailableChannel(channelID, runtime) + return channel, redactions, true, nil + } + id, redactions := processLabel(raw.ID, "channel.id", runtime) + name, nameRedactions := processLabel(other.Username, "channel.dmUsername", runtime) + channel := output.Channel{ID: id, Type: "dm", Name: "@" + name, MetadataStatus: "resolved"} + redactions = append(redactions, nameRedactions...) + return channel, redactions, false, nil +} + +func unavailableChannel(channelID string, runtime *Runtime) (output.Channel, []output.Redaction) { + id, redactions := processLabel(channelID, "channel.id", runtime) + return output.Channel{ID: id, Type: "unknown", Name: "unknown", MetadataStatus: "unavailable"}, redactions +} + +func processLabel(value, field string, runtime *Runtime) (string, []output.Redaction) { + result := presentation.PreprocessWithOptions(value, presentation.Options{ + Credentials: []string{runtime.Config.Token}, DisableHeuristics: !runtime.Config.Redact, + }) + remapLabelPositions(result.Text, result.Redactions) + for index := range result.Redactions { + result.Redactions[index].Field = field + } + return presentation.SanitizeLabel(result.Text), result.Redactions +} + func remapLabelPositions(text string, redactions []presentation.Redaction) { for index := range redactions { target, original, added := redactions[index].Position, 0, 0 @@ -170,11 +239,16 @@ func (s *rootState) renderRead(outputs []output.MessageOutput, document output.M return writeAll(s.streams.out, []byte(formatted+"\n")) } -func warnRedactionDisabled(s *rootState, runtime *Runtime) error { +func emitRedactionWarning(s *rootState, runtime *Runtime, machine bool) error { if runtime.Config.Redact { return nil } - return writeAll(s.streams.err, []byte("warning: secret redaction is disabled; output may contain secrets\n")) + warning := "warning: secret redaction is disabled; output may contain secrets\n" + if machine { + s.queueMachineWarning(warning) + return nil + } + return writeAll(s.streams.err, []byte(warning)) } func readError(message string) error { return readFailure(fmt.Errorf("%s", message)) } diff --git a/internal/cli/root.go b/internal/cli/root.go index e11d4b4..a8bb23b 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -117,6 +117,7 @@ func newRootWithState(state *rootState) *cobra.Command { cmd.PersistentFlags().BoolVar(&state.flags.noThreads, "no-threads", false, "return selected seed posts only") cmd.AddCommand(newSchemaCommand(state)) cmd.AddCommand(newChannelCommand(state)) + cmd.AddCommand(newThreadCommand(state)) return cmd } diff --git a/internal/cli/runtime.go b/internal/cli/runtime.go index 3610a48..1d78eb8 100644 --- a/internal/cli/runtime.go +++ b/internal/cli/runtime.go @@ -202,6 +202,12 @@ func (s *rootState) flushMachineWarnings() error { return writeAll(s.streams.err, []byte(warnings)) } +func (s *rootState) queueMachineWarning(message string) { + s.mu.Lock() + s.pendingWarnings = append(s.pendingWarnings, message) + s.mu.Unlock() +} + type errorClass uint8 const ( diff --git a/internal/cli/thread.go b/internal/cli/thread.go new file mode 100644 index 0000000..d576a9a --- /dev/null +++ b/internal/cli/thread.go @@ -0,0 +1,117 @@ +package cli + +import ( + "fmt" + "regexp" + + "github.com/spf13/cobra" + + "github.com/ardasevinc/mattermost-cli/internal/output" + "github.com/ardasevinc/mattermost-cli/internal/retrieval" +) + +var safePostIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,128}$`) + +func newThreadCommand(state *rootState) *cobra.Command { + return &cobra.Command{ + Use: "thread ", Short: "Fetch and display a specific thread", Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { return runThread(cmd, state, args[0]) }, + } +} + +func runThread(cmd *cobra.Command, state *rootState, postID string) error { + if !safePostIDPattern.MatchString(postID) { + return invalidFailure("post ID must be a nonempty Mattermost identifier") + } + display, err := state.readDisplay(cmd) + if err != nil { + return err + } + runtime, err := state.runtimeFor(cmd) + if err != nil { + return err + } + if err := emitRedactionWarning(state, runtime, display.json); err != nil { + return err + } + me, err := runtime.Users.Current(cmd.Context()) + if err != nil { + return readFailure(err) + } + result, err := retrieval.Thread(cmd.Context(), runtime.Posts, postID) + if err != nil { + return readFailure(err) + } + if len(result.Posts) == 0 { + return readError("thread was not found or is empty") + } + rootIndex := -1 + channelID := result.Posts[0].ChannelID + requestedRootID := postID + for index := range result.Posts { + if result.Posts[index].ThreadShapeKnown && result.Posts[index].RootID == "" { + rootIndex = index + requestedRootID = result.Posts[index].ID + channelID = result.Posts[index].ChannelID + break + } + if result.Posts[index].ThreadShapeKnown && result.Posts[index].RootID != "" { + requestedRootID = result.Posts[index].RootID + } + } + complete := result.Completeness == retrieval.CompletenessComplete && rootIndex >= 0 + effectiveCompleteness := result.Completeness + if rootIndex < 0 && effectiveCompleteness == retrieval.CompletenessComplete { + effectiveCompleteness = retrieval.CompletenessUnknown + } + threadMetadata := retrieval.VisibleThreadsMetadata{Status: retrieval.VisibleThreadsPartial, FailedRootIDs: []string{requestedRootID}} + if complete { + threadMetadata = retrieval.VisibleThreadsMetadata{Status: retrieval.VisibleThreadsComplete, HydratedRootCount: 1, FailedRootIDs: []string{}} + } + messages, redactions, err := normalizeReadPosts(cmd.Context(), runtime, result.Posts, me.ID) + if err != nil { + return readFailure(err) + } + messages = output.GroupIntoThreads(messages) + channel, channelRedactions, unavailable, err := resolvedReadChannel(cmd.Context(), runtime, channelID, me.ID) + if err != nil { + return readFailure(err) + } + redactions = append(channelRedactions, redactions...) + presentedThreads, threadRedactions := processedVisibleThreads(threadMetadata, runtime) + redactions = append(redactions, threadRedactions...) + section := output.MessageOutput{Channel: channel, Messages: messages, Redactions: redactions, Retrieval: output.Retrieval{ + Selection: output.Selection{Source: "thread", SelectedCount: len(result.Posts), QueryTruncated: truncatedPointer(effectiveCompleteness)}, + VisibleThreads: presentedThreads, VisiblePostCount: len(result.Posts), DeletedPostsIncluded: false, + }} + if unavailable { + label := channel.ID + if label == "" { + label = "an unknown channel" + } + warning := fmt.Sprintf("warning: channel metadata is unavailable for %s\n", label) + if display.json { + state.queueMachineWarning(warning) + } else { + if err := writeAll(state.streams.err, []byte(warning)); err != nil { + return err + } + } + } + if !complete { + root := presentedThreads.FailedRootIDs[0] + warning := fmt.Sprintf("warning: thread %s could only be partially hydrated\n", root) + if display.json { + state.queueMachineWarning(warning) + } else { + if err := writeAll(state.streams.err, []byte(warning)); err != nil { + return err + } + } + } + envelope, err := output.NewThreadEnvelope(section, machineCompleteness(effectiveCompleteness)) + if err != nil { + return readFailure(err) + } + return state.renderRead([]output.MessageOutput{section}, envelope, display) +} diff --git a/internal/cli/thread_test.go b/internal/cli/thread_test.go new file mode 100644 index 0000000..6b91278 --- /dev/null +++ b/internal/cli/thread_test.go @@ -0,0 +1,137 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/ardasevinc/mattermost-cli/internal/api" + "github.com/ardasevinc/mattermost-cli/internal/config" + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" +) + +func TestThreadJSONRunsRootBoundReadPipeline(t *testing.T) { + created := time.Now().Add(-time.Hour).UnixMilli() + server := threadServer(t, created, true) + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "--json", "thread", "root") + if code != 0 || stderr != "" { + t.Fatalf("exit=%d stderr=%q", code, stderr) + } + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/thread", strings.NewReader(stdout)); err != nil { + t.Fatalf("schema validation: %v\n%s", err, stdout) + } + var document struct { + Data struct { + Root struct { + ID string `json:"id"` + Replies []struct { + ID string `json:"id"` + } `json:"replies"` + } `json:"root"` + Metadata struct { + Completeness string `json:"completeness"` + VisibleThreads struct { + HydratedRootCount int `json:"hydratedRootCount"` + } `json:"visibleThreads"` + } `json:"metadata"` + } `json:"data"` + } + if err := json.Unmarshal([]byte(stdout), &document); err != nil { + t.Fatal(err) + } + if document.Data.Root.ID != "root" || len(document.Data.Root.Replies) != 1 || document.Data.Root.Replies[0].ID != "reply" || document.Data.Metadata.Completeness != "complete" || document.Data.Metadata.VisibleThreads.HydratedRootCount != 1 { + t.Fatalf("unexpected thread document: %+v", document) + } +} + +func TestThreadRejectsInvalidPostIDBeforeNetwork(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { requests.Add(1) })) + defer server.Close() + _, stderr, code := executeChannel(t, server.URL, "thread", "not/a/post") + if code != 2 || !strings.Contains(stderr, "post ID") || requests.Load() != 0 { + t.Fatalf("exit=%d stderr=%q requests=%d", code, stderr, requests.Load()) + } +} + +func TestResolvedReadChannelAcceptsSelfDM(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/channels/dm1": + writeJSON(t, writer, `{"id":"dm1","team_id":"","type":"D","name":"user1__user1","display_name":""}`) + case "/api/v4/users/user1": + writeJSON(t, writer, `{"id":"user1","username":"arda"}`) + default: + t.Errorf("unexpected request %s", request.URL.Path) + http.Error(writer, "unexpected", http.StatusNotFound) + } + })) + defer server.Close() + client, err := api.New(server.URL, "token") + if err != nil { + t.Fatal(err) + } + defer client.Close() + runtime := &Runtime{Config: config.Resolved{URL: server.URL, Token: "token", Redact: true}, Client: client, Users: mattermost.NewUsers(client), Channels: mattermost.NewChannels(client)} + channel, _, unavailable, err := resolvedReadChannel(context.Background(), runtime, "dm1", "user1") + if err != nil || unavailable || channel.Type != "dm" || channel.Name != "@arda" || channel.MetadataStatus != "resolved" { + t.Fatalf("channel=%+v unavailable=%v err=%v", channel, unavailable, err) + } +} + +func TestThreadMissingRootIsExplicitlyPartialInHumanAndMachineOutput(t *testing.T) { + server := threadServer(t, time.Now().UnixMilli(), false) + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "thread", "reply") + if code != 0 || !strings.Contains(stderr, "partially hydrated") || !strings.Contains(stdout, "reply") { + t.Fatalf("human exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } + stdout, stderr, code = executeChannel(t, server.URL, "--json", "thread", "reply") + if code != 0 || !strings.Contains(stdout, `"root":null`) || !strings.Contains(stdout, `"unboundPosts":[{"id":"reply"`) || !strings.Contains(stdout, `"completeness":"unknown"`) || !strings.Contains(stderr, "partially hydrated") { + t.Fatalf("machine exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/thread", strings.NewReader(stdout)); err != nil { + t.Fatalf("rootless machine output did not validate: %v", err) + } +} + +func threadServer(t *testing.T, created int64, includeRoot bool) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/users/me": + writeJSON(t, writer, `{"id":"user1","username":"arda"}`) + case "/api/v4/posts/root/thread", "/api/v4/posts/reply/thread": + root := fmt.Sprintf(`{"id":"root","channel_id":"channel1","user_id":"user1","message":"root text","create_at":%d,"delete_at":0,"root_id":"","reply_count":1}`, created) + reply := fmt.Sprintf(`{"id":"reply","channel_id":"channel1","user_id":"user2","message":"reply text","create_at":%d,"delete_at":0,"root_id":"root","reply_count":0}`, created+1) + if includeRoot { + writeJSON(t, writer, `{"order":["root","reply"],"posts":{"root":`+root+`,"reply":`+reply+`},"has_next":false}`) + } else { + writeJSON(t, writer, `{"order":["reply"],"posts":{"reply":`+reply+`},"has_next":false}`) + } + case "/api/v4/channels/channel1": + writeJSON(t, writer, `{"id":"channel1","team_id":"team1","type":"O","name":"town-square","display_name":"Town Square"}`) + case "/api/v4/users/ids": + writeJSON(t, writer, `[{"id":"user2","username":"bob"}]`) + default: + t.Errorf("unexpected request: %s %s", request.Method, request.URL.String()) + http.Error(writer, "unexpected", http.StatusNotFound) + } + })) +} diff --git a/internal/normalization/post.go b/internal/normalization/post.go index 43e9f24..b3c5772 100644 --- a/internal/normalization/post.go +++ b/internal/normalization/post.go @@ -92,6 +92,8 @@ func NormalizePosts(posts []mattermost.Post, users map[string]mattermost.User, m Files: files, FileDetails: details, Attachments: attachments, Reactions: reactions, CanonicalID: post.ID, CanonicalRootID: post.RootID, } + shapeKnown := post.ThreadShapeKnown + message.CanonicalThreadShapeKnown = &shapeKnown if post.EditAt > 0 { value := time.UnixMilli(post.EditAt).UTC() message.EditedAt = &value diff --git a/internal/output/machine.go b/internal/output/machine.go index 4fbe955..f40049c 100644 --- a/internal/output/machine.go +++ b/internal/output/machine.go @@ -97,10 +97,11 @@ type ChannelEnvelope struct { Data MachineHistory `json:"data"` } type ThreadData struct { - Channel MachineChannel `json:"channel"` - Root MachineMessage `json:"root"` - Redactions []Redaction `json:"redactions"` - Metadata MachineMetadata `json:"metadata"` + Channel MachineChannel `json:"channel"` + Root *MachineMessage `json:"root"` + UnboundPosts []MachineMessage `json:"unboundPosts"` + Redactions []Redaction `json:"redactions"` + Metadata MachineMetadata `json:"metadata"` } type ThreadEnvelope struct { Schema string `json:"schema"` @@ -221,17 +222,19 @@ func canonicalMachineDocument(document MachineDocument) (any, error) { return struct { Schema string `json:"schema"` Data struct { - Channel MachineChannel `json:"channel"` - Root wireMessage `json:"root"` - Redactions []Redaction `json:"redactions"` - Metadata wireMetadata `json:"metadata"` + Channel MachineChannel `json:"channel"` + Root *wireMessage `json:"root"` + UnboundPosts []wireMessage `json:"unboundPosts"` + Redactions []Redaction `json:"redactions"` + Metadata wireMetadata `json:"metadata"` } `json:"data"` }{value.Schema, struct { - Channel MachineChannel `json:"channel"` - Root wireMessage `json:"root"` - Redactions []Redaction `json:"redactions"` - Metadata wireMetadata `json:"metadata"` - }{value.Data.Channel, canonicalMessage(value.Data.Root), cloneSlice(value.Data.Redactions), canonicalMetadata(value.Data.Metadata)}}, nil + Channel MachineChannel `json:"channel"` + Root *wireMessage `json:"root"` + UnboundPosts []wireMessage `json:"unboundPosts"` + Redactions []Redaction `json:"redactions"` + Metadata wireMetadata `json:"metadata"` + }{value.Data.Channel, canonicalMessagePointer(value.Data.Root), canonicalMessages(value.Data.UnboundPosts), cloneSlice(value.Data.Redactions), canonicalMetadata(value.Data.Metadata)}}, nil case SearchEnvelope: return struct { Schema string `json:"schema"` @@ -257,6 +260,22 @@ func canonicalHistories(values []MachineHistory) []wireHistory { return result } +func canonicalMessagePointer(value *MachineMessage) *wireMessage { + if value == nil { + return nil + } + result := canonicalMessage(*value) + return &result +} + +func canonicalMessages(values []MachineMessage) []wireMessage { + result := make([]wireMessage, len(values)) + for index := range values { + result[index] = canonicalMessage(values[index]) + } + return result +} + func canonicalHistory(value MachineHistory) wireHistory { messages := make([]wireMessage, len(value.Messages)) for index := range value.Messages { diff --git a/internal/output/machine_convert.go b/internal/output/machine_convert.go index 166a3b6..c38acfc 100644 --- a/internal/output/machine_convert.go +++ b/internal/output/machine_convert.go @@ -106,57 +106,126 @@ func NewMentionsEnvelope(values []MessageOutput, completeness MachineCompletenes return MentionsEnvelope{Schema: "mm/v2/mentions", Results: histories}, err } -// NewThreadEnvelope requires the grouped thread representation: exactly one -// top-level root, with every reply represented only through root.Replies. +// NewThreadEnvelope preserves a proven root when available and carries +// rootless partial posts separately. It never promotes unknown shape to root. func NewThreadEnvelope(value MessageOutput, completeness MachineCompleteness) (ThreadEnvelope, error) { if err := validateSource(value.Retrieval.Selection.Source, "thread"); err != nil { return ThreadEnvelope{}, err } + if err := validateThreadRetrieval(value.Retrieval, completeness); err != nil { + return ThreadEnvelope{}, err + } history, err := MachineHistoryFromOutput(value, completeness) if err != nil { return ThreadEnvelope{}, err } - if len(value.Messages) != 1 { - return ThreadEnvelope{}, fmt.Errorf("thread output must contain exactly one top-level root, got %d", len(value.Messages)) - } - root := value.Messages[0] - if root.RootID != "" || root.CanonicalRootID != "" { - return ThreadEnvelope{}, fmt.Errorf("thread output top-level message is a reply, not a root") + rootIndex := -1 + unboundIndexes := make([]int, 0) + for index, message := range value.Messages { + if threadShapeKnown(message) && message.RootID == "" && message.CanonicalRootID == "" { + if rootIndex >= 0 { + return ThreadEnvelope{}, fmt.Errorf("thread output contains multiple proven roots") + } + rootIndex = index + } else { + unboundIndexes = append(unboundIndexes, index) + } } - rootIdentity := root.CanonicalID - if rootIdentity == "" { - rootIdentity = root.ID + if completeness == MachineComplete && (rootIndex < 0 || len(unboundIndexes) != 0) { + return ThreadEnvelope{}, fmt.Errorf("complete thread output requires one root and no unbound posts") } - if rootIdentity == "" || root.ID == "" { - return ThreadEnvelope{}, fmt.Errorf("thread root must have presented and canonical identity") + if rootIndex < 0 && value.Retrieval.VisibleThreads.Status != "partial" { + return ThreadEnvelope{}, fmt.Errorf("rootless thread output must report partial hydration") } - seen := map[string]bool{rootIdentity: true, root.ID: true} - for index, reply := range root.Replies { - if len(reply.Replies) != 0 { - return ThreadEnvelope{}, fmt.Errorf("thread reply %d contains nested replies", index) + var machineRoot *MachineMessage + rootIdentity := "" + seenTopLevel := make(map[string]bool) + if rootIndex >= 0 { + root := value.Messages[rootIndex] + rootIdentity = messageIdentity(root) + if rootIdentity == "" || root.ID == "" { + return ThreadEnvelope{}, fmt.Errorf("thread root must have presented and canonical identity") } - replyRootIdentity := reply.CanonicalRootID - if replyRootIdentity == "" { - replyRootIdentity = reply.RootID + seen := map[string]bool{rootIdentity: true, root.ID: true} + seenTopLevel[rootIdentity], seenTopLevel[root.ID] = true, true + for index, reply := range root.Replies { + if len(reply.Replies) != 0 { + return ThreadEnvelope{}, fmt.Errorf("thread reply %d contains nested replies", index) + } + replyRootIdentity := reply.CanonicalRootID + if replyRootIdentity == "" { + replyRootIdentity = reply.RootID + } + if reply.RootID != root.ID || replyRootIdentity != rootIdentity { + return ThreadEnvelope{}, fmt.Errorf("thread reply %d does not reference root", index) + } + replyIdentity := messageIdentity(reply) + if replyIdentity == "" || reply.ID == "" || seen[replyIdentity] || seen[reply.ID] { + return ThreadEnvelope{}, fmt.Errorf("thread reply %d has missing or duplicate identity", index) + } + seen[replyIdentity], seen[reply.ID] = true, true + seenTopLevel[replyIdentity], seenTopLevel[reply.ID] = true, true } - if reply.RootID != root.ID || replyRootIdentity != rootIdentity { - return ThreadEnvelope{}, fmt.Errorf("thread reply %d does not reference root", index) + converted := history.Messages[rootIndex] + machineRoot = &converted + } + unbound := make([]MachineMessage, len(unboundIndexes)) + for outputIndex, messageIndex := range unboundIndexes { + post := value.Messages[messageIndex] + if len(post.Replies) != 0 { + return ThreadEnvelope{}, fmt.Errorf("unbound post %d contains nested replies", outputIndex) } - replyIdentity := reply.CanonicalID - if replyIdentity == "" { - replyIdentity = reply.ID + identity := messageIdentity(post) + if identity == "" || post.ID == "" || seenTopLevel[identity] || seenTopLevel[post.ID] { + return ThreadEnvelope{}, fmt.Errorf("unbound post %d has missing or duplicate identity", outputIndex) } - if replyIdentity == "" || reply.ID == "" || seen[replyIdentity] || seen[reply.ID] { - return ThreadEnvelope{}, fmt.Errorf("thread reply %d has missing or duplicate identity", index) + if rootIdentity != "" { + postRoot := post.CanonicalRootID + if postRoot == "" { + postRoot = post.RootID + } + if postRoot == rootIdentity { + return ThreadEnvelope{}, fmt.Errorf("unbound post %d should be grouped under the proven root", outputIndex) + } } - seen[replyIdentity], seen[reply.ID] = true, true + seenTopLevel[identity], seenTopLevel[post.ID] = true, true + unbound[outputIndex] = history.Messages[messageIndex] } return ThreadEnvelope{Schema: "mm/v2/thread", Data: ThreadData{ - Channel: history.Channel, Root: history.Messages[0], + Channel: history.Channel, Root: machineRoot, UnboundPosts: unbound, Redactions: history.Redactions, Metadata: history.Metadata, }}, nil } +func validateThreadRetrieval(value Retrieval, completeness MachineCompleteness) error { + queryTruncated := value.Selection.QueryTruncated + visible := value.VisibleThreads + switch completeness { + case MachineComplete: + if queryTruncated == nil || *queryTruncated || visible.Status != "complete" || visible.HydratedRootCount != 1 || len(visible.FailedRootIDs) != 0 { + return fmt.Errorf("complete thread retrieval requires confirmed complete query and hydration metadata") + } + case MachineTruncated: + if queryTruncated == nil || !*queryTruncated || visible.Status != "partial" || visible.HydratedRootCount != 0 || len(visible.FailedRootIDs) == 0 { + return fmt.Errorf("truncated thread retrieval requires confirmed truncation and partial hydration metadata") + } + case MachineUnknown: + if queryTruncated != nil || visible.Status != "partial" || visible.HydratedRootCount != 0 || len(visible.FailedRootIDs) == 0 { + return fmt.Errorf("unknown thread retrieval requires unknown query and partial hydration metadata") + } + default: + return fmt.Errorf("invalid machine completeness %q", completeness) + } + return nil +} + +func messageIdentity(message Message) string { + if message.CanonicalID != "" { + return message.CanonicalID + } + return message.ID +} + func machineHistories(values []MessageOutput, completeness MachineCompleteness, source string, channelTypes ...string) ([]MachineHistory, error) { if !validMachineCompleteness(completeness) { return nil, fmt.Errorf("invalid machine completeness %q", completeness) diff --git a/internal/output/machine_convert_test.go b/internal/output/machine_convert_test.go index cd513cc..6443304 100644 --- a/internal/output/machine_convert_test.go +++ b/internal/output/machine_convert_test.go @@ -237,13 +237,67 @@ func TestMachineConversionRejectsInvalidVocabularyAndAmbiguousThreads(t *testing } } +func TestThreadEnvelopeKeepsUnknownShapeUnboundInsteadOfInventingRoot(t *testing.T) { + value := validOutput() + value.Retrieval.Selection.Source = "thread" + value.Retrieval.Selection.QueryTruncated = nil + value.Retrieval.VisibleThreads = output.VisibleThreads{Status: "partial", FailedRootIDs: []string{"requested"}} + known := false + value.Messages[0].CanonicalThreadShapeKnown = &known + value.Messages[0].Replies = nil + envelope, err := output.NewThreadEnvelope(value, output.MachineUnknown) + if err != nil { + t.Fatal(err) + } + if envelope.Data.Root != nil || len(envelope.Data.UnboundPosts) != 1 || envelope.Data.UnboundPosts[0].ID != value.Messages[0].ID { + t.Fatalf("unexpected rootless envelope: %#v", envelope.Data) + } +} + +func TestThreadEnvelopeRejectsContradictoryRetrievalMetadata(t *testing.T) { + for name, corrupt := range map[string]func(*output.MessageOutput, *output.MachineCompleteness){ + "complete with partial hydration": func(value *output.MessageOutput, _ *output.MachineCompleteness) { + value.Retrieval.VisibleThreads = output.VisibleThreads{Status: "partial", FailedRootIDs: []string{"root"}} + }, + "truncated with complete hydration": func(value *output.MessageOutput, completeness *output.MachineCompleteness) { + *completeness = output.MachineTruncated + truncated := true + value.Retrieval.Selection.QueryTruncated = &truncated + }, + "unknown with known query status": func(value *output.MessageOutput, completeness *output.MachineCompleteness) { + *completeness = output.MachineUnknown + value.Retrieval.VisibleThreads = output.VisibleThreads{Status: "partial", FailedRootIDs: []string{"root"}} + }, + } { + t.Run(name, func(t *testing.T) { + value := validOutput() + value.Retrieval.Selection.Source = "thread" + completeness := output.MachineComplete + corrupt(&value, &completeness) + if _, err := output.NewThreadEnvelope(value, completeness); err == nil { + t.Fatal("contradictory thread retrieval metadata accepted") + } + }) + } + + value := validOutput() + value.Retrieval.Selection.Source = "thread" + truncated := true + value.Retrieval.Selection.QueryTruncated = &truncated + value.Retrieval.VisibleThreads = output.VisibleThreads{Status: "partial", FailedRootIDs: []string{"root"}} + if _, err := output.NewThreadEnvelope(value, output.MachineTruncated); err != nil { + t.Fatalf("consistent truncated thread rejected: %v", err) + } +} + func validOutput() output.MessageOutput { zone := time.FixedZone("source", 3*60*60) stamp := time.Date(2026, 7, 16, 13, 14, 15, 987654321, zone) limit, truncated := 10, false + known := true return output.MessageOutput{ Channel: output.Channel{ID: "c1", Type: "public", Name: "town-square", DisplayName: "Town Square", MetadataStatus: "resolved"}, - Messages: []output.Message{{ID: "root", CanonicalID: "root", User: "arda", UserID: "u1", Text: "root", Timestamp: stamp, UpdatedAt: stamp, Replies: []output.Message{{ID: "reply", CanonicalID: "reply", RootID: "root", CanonicalRootID: "root", User: "bob", UserID: "u2", Text: "reply", Timestamp: stamp, UpdatedAt: stamp}}}}, + Messages: []output.Message{{ID: "root", CanonicalID: "root", CanonicalThreadShapeKnown: &known, User: "arda", UserID: "u1", Text: "root", Timestamp: stamp, UpdatedAt: stamp, Replies: []output.Message{{ID: "reply", CanonicalID: "reply", RootID: "root", CanonicalRootID: "root", CanonicalThreadShapeKnown: &known, User: "bob", UserID: "u2", Text: "reply", Timestamp: stamp, UpdatedAt: stamp}}}}, Redactions: []output.Redaction{{Type: "token", Masked: "abc***xyz", Position: 0}}, Retrieval: output.Retrieval{ Selection: output.Selection{Source: "recent", SelectedCount: 2, RequestedLimit: &limit, QueryTruncated: &truncated}, diff --git a/internal/output/model.go b/internal/output/model.go index e66b4ad..cb95328 100644 --- a/internal/output/model.go +++ b/internal/output/model.go @@ -34,8 +34,9 @@ type Message struct { // Canonical identities are unsanitized internal values. They drive // ordering and grouping but are never serialized. - CanonicalID string `json:"-"` - CanonicalRootID string `json:"-"` + CanonicalID string `json:"-"` + CanonicalRootID string `json:"-"` + CanonicalThreadShapeKnown *bool `json:"-"` } type File struct { diff --git a/internal/output/threading.go b/internal/output/threading.go index 0bb31fd..937b045 100644 --- a/internal/output/threading.go +++ b/internal/output/threading.go @@ -10,7 +10,7 @@ func GroupIntoThreads(messages []Message) []Message { rootIndexes := make(map[string]int, len(sorted)) orphans := make([]Message, 0) for _, message := range sorted { - if canonicalRootID(message) != "" { + if canonicalRootID(message) != "" || !threadShapeKnown(message) { continue } root := message @@ -20,7 +20,7 @@ func GroupIntoThreads(messages []Message) []Message { } for _, message := range sorted { rootID := canonicalRootID(message) - if rootID == "" { + if rootID == "" && threadShapeKnown(message) { continue } if index, ok := rootIndexes[rootID]; ok { @@ -34,6 +34,10 @@ func GroupIntoThreads(messages []Message) []Message { return result } +func threadShapeKnown(message Message) bool { + return message.CanonicalThreadShapeKnown != nil && *message.CanonicalThreadShapeKnown +} + func compareMessages(a, b Message) int { if order := a.Timestamp.Compare(b.Timestamp); order != 0 { return order diff --git a/internal/output/threading_test.go b/internal/output/threading_test.go index bb45fc7..8647e68 100644 --- a/internal/output/threading_test.go +++ b/internal/output/threading_test.go @@ -8,10 +8,11 @@ import ( func TestGroupIntoThreadsGroupsSortsAndDoesNotMutateInput(t *testing.T) { base := time.Date(2026, time.February, 21, 10, 0, 0, 0, time.UTC) + known := true messages := []Message{ - {ID: "reply-2", RootID: "root", Timestamp: base.Add(3 * time.Minute)}, - {ID: "root", Timestamp: base, Replies: []Message{{ID: "stale"}}}, - {ID: "reply-1", RootID: "root", Timestamp: base.Add(time.Minute)}, + {ID: "reply-2", RootID: "root", Timestamp: base.Add(3 * time.Minute), CanonicalThreadShapeKnown: &known}, + {ID: "root", Timestamp: base, Replies: []Message{{ID: "stale"}}, CanonicalThreadShapeKnown: &known}, + {ID: "reply-1", RootID: "root", Timestamp: base.Add(time.Minute), CanonicalThreadShapeKnown: &known}, } result := GroupIntoThreads(messages) if got := messageIDs(result); !reflect.DeepEqual(got, []string{"root"}) { @@ -27,11 +28,12 @@ func TestGroupIntoThreadsGroupsSortsAndDoesNotMutateInput(t *testing.T) { func TestGroupIntoThreadsKeepsOrphansInGlobalTimestampThenIDOrder(t *testing.T) { stamp := time.Date(2026, time.February, 21, 10, 0, 0, 0, time.UTC) + known := true messages := []Message{ - {ID: "root-z", Timestamp: stamp}, - {ID: "orphan-b", RootID: "missing", Timestamp: stamp}, - {ID: "orphan-a", RootID: "missing", Timestamp: stamp}, - {ID: "root-early", Timestamp: stamp.Add(-time.Minute)}, + {ID: "root-z", Timestamp: stamp, CanonicalThreadShapeKnown: &known}, + {ID: "orphan-b", RootID: "missing", Timestamp: stamp, CanonicalThreadShapeKnown: &known}, + {ID: "orphan-a", RootID: "missing", Timestamp: stamp, CanonicalThreadShapeKnown: &known}, + {ID: "root-early", Timestamp: stamp.Add(-time.Minute), CanonicalThreadShapeKnown: &known}, } if got := messageIDs(GroupIntoThreads(messages)); !reflect.DeepEqual(got, []string{"root-early", "orphan-a", "orphan-b", "root-z"}) { t.Fatalf("order = %v", got) @@ -40,10 +42,11 @@ func TestGroupIntoThreadsKeepsOrphansInGlobalTimestampThenIDOrder(t *testing.T) func TestGroupIntoThreadsUsesCanonicalIdentityForGroupingAndTies(t *testing.T) { stamp := time.Date(2026, time.February, 21, 10, 0, 0, 0, time.UTC) + known := true messages := []Message{ - {ID: "visible-first", CanonicalID: "z", Timestamp: stamp}, - {ID: "visible-last", CanonicalID: "a", Timestamp: stamp}, - {ID: "masked-reply", RootID: "masked-root", CanonicalID: "r", CanonicalRootID: "z", Timestamp: stamp.Add(time.Second)}, + {ID: "visible-first", CanonicalID: "z", Timestamp: stamp, CanonicalThreadShapeKnown: &known}, + {ID: "visible-last", CanonicalID: "a", Timestamp: stamp, CanonicalThreadShapeKnown: &known}, + {ID: "masked-reply", RootID: "masked-root", CanonicalID: "r", CanonicalRootID: "z", Timestamp: stamp.Add(time.Second), CanonicalThreadShapeKnown: &known}, } result := GroupIntoThreads(messages) if got := messageIDs(result); !reflect.DeepEqual(got, []string{"visible-last", "visible-first"}) { @@ -56,10 +59,11 @@ func TestGroupIntoThreadsUsesCanonicalIdentityForGroupingAndTies(t *testing.T) { func TestGroupIntoThreadsSortsEqualTimestampRepliesByCanonicalID(t *testing.T) { stamp := time.Date(2026, time.February, 21, 10, 0, 0, 0, time.UTC) + known := true result := GroupIntoThreads([]Message{ - {ID: "root", Timestamp: stamp}, - {ID: "visible-a", CanonicalID: "b", RootID: "root", Timestamp: stamp.Add(time.Second)}, - {ID: "visible-z", CanonicalID: "a", RootID: "root", Timestamp: stamp.Add(time.Second)}, + {ID: "root", Timestamp: stamp, CanonicalThreadShapeKnown: &known}, + {ID: "visible-a", CanonicalID: "b", RootID: "root", Timestamp: stamp.Add(time.Second), CanonicalThreadShapeKnown: &known}, + {ID: "visible-z", CanonicalID: "a", RootID: "root", Timestamp: stamp.Add(time.Second), CanonicalThreadShapeKnown: &known}, }) if got := messageIDs(result[0].Replies); !reflect.DeepEqual(got, []string{"visible-z", "visible-a"}) { t.Fatalf("reply order = %v", got) diff --git a/internal/schema/read_test.go b/internal/schema/read_test.go index a602d24..5dd9e88 100644 --- a/internal/schema/read_test.go +++ b/internal/schema/read_test.go @@ -65,7 +65,7 @@ func TestReadSchemasAreRegisteredAndStrict(t *testing.T) { } } -func TestReadSchemaRejectsNullThreadRootAndInvalidTimestamps(t *testing.T) { +func TestReadSchemaRejectsInvalidThreadTimestamps(t *testing.T) { registry, err := Load() if err != nil { t.Fatal(err) @@ -75,22 +75,123 @@ func TestReadSchemaRejectsNullThreadRootAndInvalidTimestamps(t *testing.T) { t.Fatal(err) } for name, replacement := range map[string]string{ - "null root": `"root":null`, "impossible date": `"timestamp":"2026-02-30T01:02:03.456Z"`, "short year": `"timestamp":"026-07-16T01:02:03.456Z"`, "year zero": `"timestamp":"0000-07-16T01:02:03.456Z"`, } { document := string(valid) - switch name { - case "null root": - start := strings.Index(document, `"root":{`) - end := strings.Index(document[start:], `,"redactions":[]`) - document = document[:start] + replacement + document[start+end:] - default: - document = strings.Replace(document, `"timestamp":"2026-07-16T01:02:03.456Z"`, replacement, 1) - } + document = strings.Replace(document, `"timestamp":"2026-07-16T01:02:03.456Z"`, replacement, 1) if err := registry.Validate("mm/v2/thread", strings.NewReader(document)); err == nil { t.Errorf("accepted %s", name) } } } + +func TestThreadSchemaEnforcesRootAndUnboundShape(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + valid, err := fs.ReadFile(publicschemas.FS, "v2/examples/thread.json") + if err != nil { + t.Fatal(err) + } + + decode := func(t *testing.T) map[string]any { + t.Helper() + var document map[string]any + if err := json.Unmarshal(valid, &document); err != nil { + t.Fatal(err) + } + return document + } + validate := func(t *testing.T, document map[string]any, wantValid bool) { + t.Helper() + raw, err := json.Marshal(document) + if err != nil { + t.Fatal(err) + } + err = registry.Validate("mm/v2/thread", strings.NewReader(string(raw))) + if wantValid && err != nil { + t.Fatalf("valid thread document rejected: %v", err) + } + if !wantValid && err == nil { + t.Fatalf("invalid thread document accepted: %s", raw) + } + } + data := func(document map[string]any) map[string]any { + return document["data"].(map[string]any) + } + + t.Run("root must be canonical root", func(t *testing.T) { + document := decode(t) + data(document)["root"].(map[string]any)["rootId"] = "other" + validate(t, document, false) + }) + + t.Run("unbound post cannot contain nested replies", func(t *testing.T) { + document := decode(t) + thread := data(document) + unbound := thread["root"].(map[string]any) + nested := data(decode(t))["root"].(map[string]any) + unbound["rootId"] = "missing-root" + unbound["replies"] = []any{nested} + thread["root"] = nil + thread["unboundPosts"] = []any{unbound} + metadata := thread["metadata"].(map[string]any) + metadata["completeness"] = "unknown" + metadata["visibleThreads"].(map[string]any)["status"] = "partial" + validate(t, document, false) + }) + + t.Run("unbound post requires partial hydration", func(t *testing.T) { + document := decode(t) + thread := data(document) + unbound := thread["root"].(map[string]any) + unbound["rootId"] = "missing-root" + thread["root"] = nil + thread["unboundPosts"] = []any{unbound} + thread["metadata"].(map[string]any)["completeness"] = "unknown" + validate(t, document, false) + }) + + t.Run("rootless partial output is valid", func(t *testing.T) { + document := decode(t) + thread := data(document) + unbound := thread["root"].(map[string]any) + unbound["rootId"] = "missing-root" + thread["root"] = nil + thread["unboundPosts"] = []any{unbound} + metadata := thread["metadata"].(map[string]any) + metadata["completeness"] = "unknown" + metadata["selection"].(map[string]any)["queryTruncated"] = nil + visible := metadata["visibleThreads"].(map[string]any) + visible["status"] = "partial" + visible["hydratedRootCount"] = float64(0) + visible["failedRootIds"] = []any{"missing-root"} + validate(t, document, true) + }) + + t.Run("complete retrieval cannot report partial hydration", func(t *testing.T) { + document := decode(t) + visible := data(document)["metadata"].(map[string]any)["visibleThreads"].(map[string]any) + visible["status"] = "partial" + visible["hydratedRootCount"] = float64(0) + visible["failedRootIds"] = []any{"p1"} + validate(t, document, false) + }) + + t.Run("truncated retrieval cannot report complete hydration", func(t *testing.T) { + document := decode(t) + metadata := data(document)["metadata"].(map[string]any) + metadata["completeness"] = "truncated" + metadata["selection"].(map[string]any)["queryTruncated"] = true + validate(t, document, false) + }) + + t.Run("query truncation must match completeness", func(t *testing.T) { + document := decode(t) + data(document)["metadata"].(map[string]any)["selection"].(map[string]any)["queryTruncated"] = true + validate(t, document, false) + }) +} diff --git a/schemas/v2/examples/thread.json b/schemas/v2/examples/thread.json index c120286..32086de 100644 --- a/schemas/v2/examples/thread.json +++ b/schemas/v2/examples/thread.json @@ -1 +1 @@ -{"schema":"mm/v2/thread","data":{"channel":{"id":"c1","type":"public","name":"town-square","displayName":"Town Square","metadataStatus":"resolved"},"root":{"id":"p1","permalink":"https://mm.example/team/pl/p1","user":"arda","userId":"u1","text":"hello","timestamp":"2026-07-16T01:02:03.456Z","updatedAt":"2026-07-16T01:02:03.456Z","editedAt":null,"deletedAt":null,"isDeleted":false,"postType":"","isSystem":false,"isPinned":false,"rootId":null,"replyCount":0,"files":[],"fileDetails":[],"attachments":[],"reactions":[],"replies":[]},"redactions":[],"metadata":{"completeness":"complete","selection":{"source":"thread","selectedCount":1,"requestedLimit":null,"since":null,"queryTruncated":false,"inputCursor":null,"nextCursor":null},"visibleThreads":{"status":"not_requested","hydratedRootCount":0,"failedRootIds":[]},"visiblePostCount":1,"deletedPostsIncluded":false}}} +{"schema":"mm/v2/thread","data":{"channel":{"id":"c1","type":"public","name":"town-square","displayName":"Town Square","metadataStatus":"resolved"},"root":{"id":"p1","permalink":"https://mm.example/team/pl/p1","user":"arda","userId":"u1","text":"hello","timestamp":"2026-07-16T01:02:03.456Z","updatedAt":"2026-07-16T01:02:03.456Z","editedAt":null,"deletedAt":null,"isDeleted":false,"isPinned":false,"postType":"","isSystem":false,"rootId":null,"replyCount":0,"files":[],"fileDetails":[],"attachments":[],"reactions":[],"replies":[]},"unboundPosts":[],"redactions":[],"metadata":{"completeness":"complete","selection":{"source":"thread","selectedCount":1,"requestedLimit":null,"since":null,"queryTruncated":false,"inputCursor":null,"nextCursor":null},"visibleThreads":{"status":"complete","hydratedRootCount":1,"failedRootIds":[]},"visiblePostCount":1,"deletedPostsIncluded":false}}} diff --git a/schemas/v2/thread.schema.json b/schemas/v2/thread.schema.json index 8f7060c..b095cd4 100644 --- a/schemas/v2/thread.schema.json +++ b/schemas/v2/thread.schema.json @@ -1 +1 @@ -{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"urn:mm:schema:v2:thread","type":"object","additionalProperties":false,"required":["schema","data"],"properties":{"schema":{"const":"mm/v2/thread"},"data":{"type":"object","additionalProperties":false,"required":["channel","root","redactions","metadata"],"properties":{"channel":{"$ref":"#/$defs/channel"},"root":{"$ref":"#/$defs/message"},"redactions":{"type":"array","items":{"$ref":"#/$defs/redaction"}},"metadata":{"$ref":"#/$defs/metadata"}}}},"$defs":{"timestamp":{"type":"string","format":"date-time","pattern":"^(?:000[1-9]|00[1-9][0-9]|0[1-9][0-9]{2}|[1-9][0-9]{3})-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\\.[0-9]{3}Z$"},"nullableTimestamp":{"anyOf":[{"$ref":"#/$defs/timestamp"},{"type":"null"}]},"file":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"name":{"type":"string"},"mime":{"type":"string"},"size":{"type":"integer","minimum":0},"extension":{"type":"string"}}},"attachmentField":{"type":"object","additionalProperties":false,"properties":{"title":{"type":"string"},"value":{"type":"string"},"short":{"type":"boolean"}}},"attachment":{"type":"object","additionalProperties":false,"properties":{"fallback":{"type":"string"},"pretext":{"type":"string"},"title":{"type":"string"},"titleLink":{"type":"string"},"text":{"type":"string"},"fields":{"type":"array","items":{"$ref":"#/$defs/attachmentField"}},"footer":{"type":"string"},"footerIcon":{"type":"string"},"authorName":{"type":"string"},"authorLink":{"type":"string"},"authorIcon":{"type":"string"},"color":{"type":"string"},"imageUrl":{"type":"string"},"thumbUrl":{"type":"string"},"timestamp":{"type":"string"}}},"reactionActor":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"username":{"type":"string"}}},"reaction":{"type":"object","additionalProperties":false,"required":["emoji","count","actors"],"properties":{"emoji":{"type":"string"},"count":{"type":"integer","minimum":0},"actors":{"type":"array","items":{"$ref":"#/$defs/reactionActor"}}}},"redaction":{"type":"object","additionalProperties":false,"required":["type","masked","position"],"properties":{"type":{"type":"string"},"masked":{"type":"string"},"position":{"type":"integer","minimum":0},"field":{"type":"string"}}},"channel":{"type":"object","additionalProperties":false,"required":["id","type","name","displayName","metadataStatus"],"properties":{"id":{"type":"string"},"type":{"enum":["dm","public","private","group","unknown"]},"name":{"type":"string"},"displayName":{"type":"string"},"metadataStatus":{"enum":["resolved","unavailable"]}}},"selection":{"type":"object","additionalProperties":false,"required":["source","selectedCount","requestedLimit","since","queryTruncated","inputCursor","nextCursor"],"properties":{"source":{"enum":["recent","search","mentions","unread","thread"]},"selectedCount":{"type":"integer","minimum":0},"requestedLimit":{"type":["integer","null"],"minimum":1},"since":{"type":["string","null"]},"queryTruncated":{"type":["boolean","null"]},"inputCursor":{"type":["string","null"]},"nextCursor":{"type":["string","null"]}}},"visibleThreads":{"type":"object","additionalProperties":false,"required":["status","hydratedRootCount","failedRootIds"],"properties":{"status":{"enum":["not_requested","complete","partial"]},"hydratedRootCount":{"type":"integer","minimum":0},"failedRootIds":{"type":"array","items":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":false,"required":["completeness","selection","visibleThreads","visiblePostCount","deletedPostsIncluded"],"properties":{"completeness":{"enum":["complete","truncated","unknown"]},"selection":{"$ref":"#/$defs/selection"},"visibleThreads":{"$ref":"#/$defs/visibleThreads"},"visiblePostCount":{"type":"integer","minimum":0},"deletedPostsIncluded":{"const":false}}},"message":{"type":"object","additionalProperties":false,"required":["id","permalink","user","userId","text","timestamp","updatedAt","editedAt","deletedAt","isDeleted","postType","isSystem","isPinned","rootId","replyCount","files","fileDetails","attachments","reactions","replies"],"properties":{"id":{"type":"string"},"permalink":{"type":"string"},"user":{"type":"string"},"userId":{"type":"string"},"text":{"type":"string"},"timestamp":{"$ref":"#/$defs/timestamp"},"updatedAt":{"$ref":"#/$defs/timestamp"},"editedAt":{"$ref":"#/$defs/nullableTimestamp"},"deletedAt":{"$ref":"#/$defs/nullableTimestamp"},"isDeleted":{"type":"boolean"},"postType":{"type":"string"},"isSystem":{"type":"boolean"},"isPinned":{"type":"boolean"},"rootId":{"type":["string","null"]},"replyCount":{"type":["integer","null"],"minimum":0},"files":{"type":"array","items":{"type":"string"}},"fileDetails":{"type":"array","items":{"$ref":"#/$defs/file"}},"attachments":{"type":"array","items":{"$ref":"#/$defs/attachment"}},"reactions":{"type":"array","items":{"$ref":"#/$defs/reaction"}},"replies":{"type":"array","items":{"$ref":"#/$defs/message"}}}},"history":{"type":"object","additionalProperties":false,"required":["channel","messages","redactions","metadata"],"properties":{"channel":{"$ref":"#/$defs/channel"},"messages":{"type":"array","items":{"$ref":"#/$defs/message"}},"redactions":{"type":"array","items":{"$ref":"#/$defs/redaction"}},"metadata":{"$ref":"#/$defs/metadata"}}}}} +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"urn:mm:schema:v2:thread","type":"object","additionalProperties":false,"required":["schema","data"],"properties":{"schema":{"const":"mm/v2/thread"},"data":{"type":"object","additionalProperties":false,"required":["channel","root","unboundPosts","redactions","metadata"],"properties":{"channel":{"$ref":"#/$defs/channel"},"root":{"anyOf":[{"$ref":"#/$defs/threadRoot"},{"type":"null"}]},"unboundPosts":{"type":"array","items":{"$ref":"#/$defs/unboundPost"}},"redactions":{"type":"array","items":{"$ref":"#/$defs/redaction"}},"metadata":{"$ref":"#/$defs/metadata"}},"allOf":[{"if":{"properties":{"root":{"type":"null"}}},"then":{"properties":{"unboundPosts":{"minItems":1},"metadata":{"allOf":[{"properties":{"completeness":{"enum":["truncated","unknown"]},"visibleThreads":{"properties":{"status":{"const":"partial"}}}}}]}}}},{"if":{"properties":{"metadata":{"properties":{"completeness":{"const":"complete"}}}}},"then":{"properties":{"root":{"$ref":"#/$defs/threadRoot"},"unboundPosts":{"maxItems":0},"metadata":{"properties":{"selection":{"properties":{"queryTruncated":{"const":false}}},"visibleThreads":{"properties":{"status":{"const":"complete"},"hydratedRootCount":{"const":1},"failedRootIds":{"maxItems":0}}}}}}}},{"if":{"properties":{"metadata":{"properties":{"completeness":{"const":"truncated"}}}}},"then":{"properties":{"metadata":{"properties":{"selection":{"properties":{"queryTruncated":{"const":true}}},"visibleThreads":{"properties":{"status":{"const":"partial"},"hydratedRootCount":{"const":0},"failedRootIds":{"minItems":1}}}}}}}},{"if":{"properties":{"metadata":{"properties":{"completeness":{"const":"unknown"}}}}},"then":{"properties":{"metadata":{"properties":{"selection":{"properties":{"queryTruncated":{"type":"null"}}},"visibleThreads":{"properties":{"status":{"const":"partial"},"hydratedRootCount":{"const":0},"failedRootIds":{"minItems":1}}}}}}}},{"if":{"properties":{"unboundPosts":{"minItems":1}}},"then":{"properties":{"metadata":{"properties":{"completeness":{"enum":["truncated","unknown"]},"visibleThreads":{"properties":{"status":{"const":"partial"}}}}}}}}]}},"$defs":{"timestamp":{"type":"string","format":"date-time","pattern":"^(?:000[1-9]|00[1-9][0-9]|0[1-9][0-9]{2}|[1-9][0-9]{3})-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\\.[0-9]{3}Z$"},"nullableTimestamp":{"anyOf":[{"$ref":"#/$defs/timestamp"},{"type":"null"}]},"file":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"name":{"type":"string"},"mime":{"type":"string"},"size":{"type":"integer","minimum":0},"extension":{"type":"string"}}},"attachmentField":{"type":"object","additionalProperties":false,"properties":{"title":{"type":"string"},"value":{"type":"string"},"short":{"type":"boolean"}}},"attachment":{"type":"object","additionalProperties":false,"properties":{"fallback":{"type":"string"},"pretext":{"type":"string"},"title":{"type":"string"},"titleLink":{"type":"string"},"text":{"type":"string"},"fields":{"type":"array","items":{"$ref":"#/$defs/attachmentField"}},"footer":{"type":"string"},"footerIcon":{"type":"string"},"authorName":{"type":"string"},"authorLink":{"type":"string"},"authorIcon":{"type":"string"},"color":{"type":"string"},"imageUrl":{"type":"string"},"thumbUrl":{"type":"string"},"timestamp":{"type":"string"}}},"reactionActor":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"username":{"type":"string"}}},"reaction":{"type":"object","additionalProperties":false,"required":["emoji","count","actors"],"properties":{"emoji":{"type":"string"},"count":{"type":"integer","minimum":0},"actors":{"type":"array","items":{"$ref":"#/$defs/reactionActor"}}}},"redaction":{"type":"object","additionalProperties":false,"required":["type","masked","position"],"properties":{"type":{"type":"string"},"masked":{"type":"string"},"position":{"type":"integer","minimum":0},"field":{"type":"string"}}},"channel":{"type":"object","additionalProperties":false,"required":["id","type","name","displayName","metadataStatus"],"properties":{"id":{"type":"string"},"type":{"enum":["dm","public","private","group","unknown"]},"name":{"type":"string"},"displayName":{"type":"string"},"metadataStatus":{"enum":["resolved","unavailable"]}}},"selection":{"type":"object","additionalProperties":false,"required":["source","selectedCount","requestedLimit","since","queryTruncated","inputCursor","nextCursor"],"properties":{"source":{"enum":["recent","search","mentions","unread","thread"]},"selectedCount":{"type":"integer","minimum":0},"requestedLimit":{"type":["integer","null"],"minimum":1},"since":{"type":["string","null"]},"queryTruncated":{"type":["boolean","null"]},"inputCursor":{"type":["string","null"]},"nextCursor":{"type":["string","null"]}}},"visibleThreads":{"type":"object","additionalProperties":false,"required":["status","hydratedRootCount","failedRootIds"],"properties":{"status":{"enum":["not_requested","complete","partial"]},"hydratedRootCount":{"type":"integer","minimum":0},"failedRootIds":{"type":"array","items":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":false,"required":["completeness","selection","visibleThreads","visiblePostCount","deletedPostsIncluded"],"properties":{"completeness":{"enum":["complete","truncated","unknown"]},"selection":{"$ref":"#/$defs/selection"},"visibleThreads":{"$ref":"#/$defs/visibleThreads"},"visiblePostCount":{"type":"integer","minimum":0},"deletedPostsIncluded":{"const":false}}},"message":{"type":"object","additionalProperties":false,"required":["id","permalink","user","userId","text","timestamp","updatedAt","editedAt","deletedAt","isDeleted","postType","isSystem","isPinned","rootId","replyCount","files","fileDetails","attachments","reactions","replies"],"properties":{"id":{"type":"string"},"permalink":{"type":"string"},"user":{"type":"string"},"userId":{"type":"string"},"text":{"type":"string"},"timestamp":{"$ref":"#/$defs/timestamp"},"updatedAt":{"$ref":"#/$defs/timestamp"},"editedAt":{"$ref":"#/$defs/nullableTimestamp"},"deletedAt":{"$ref":"#/$defs/nullableTimestamp"},"isDeleted":{"type":"boolean"},"postType":{"type":"string"},"isSystem":{"type":"boolean"},"isPinned":{"type":"boolean"},"rootId":{"type":["string","null"]},"replyCount":{"type":["integer","null"],"minimum":0},"files":{"type":"array","items":{"type":"string"}},"fileDetails":{"type":"array","items":{"$ref":"#/$defs/file"}},"attachments":{"type":"array","items":{"$ref":"#/$defs/attachment"}},"reactions":{"type":"array","items":{"$ref":"#/$defs/reaction"}},"replies":{"type":"array","items":{"$ref":"#/$defs/message"}}}},"threadReply":{"allOf":[{"$ref":"#/$defs/message"},{"properties":{"rootId":{"type":"string","minLength":1},"replies":{"maxItems":0}}}]},"threadRoot":{"allOf":[{"$ref":"#/$defs/message"},{"properties":{"rootId":{"type":"null"},"replies":{"type":"array","items":{"$ref":"#/$defs/threadReply"}}}}]},"unboundPost":{"allOf":[{"$ref":"#/$defs/message"},{"properties":{"replies":{"maxItems":0}}}]},"history":{"type":"object","additionalProperties":false,"required":["channel","messages","redactions","metadata"],"properties":{"channel":{"$ref":"#/$defs/channel"},"messages":{"type":"array","items":{"$ref":"#/$defs/message"}},"redactions":{"type":"array","items":{"$ref":"#/$defs/redaction"}},"metadata":{"$ref":"#/$defs/metadata"}}}}} From 73af149ad5db455f37a0db44fbcdb08662baf564 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 15:19:13 +0300 Subject: [PATCH 032/119] feat: port search read command --- internal/cli/read.go | 48 ++++++- internal/cli/root.go | 1 + internal/cli/search.go | 152 ++++++++++++++++++++ internal/cli/search_test.go | 181 ++++++++++++++++++++++++ internal/cli/thread.go | 2 +- internal/cli/thread_test.go | 46 +++++- internal/output/machine_convert.go | 26 ++++ internal/output/machine_convert_test.go | 17 +++ internal/retrieval/hydration.go | 34 +++++ internal/retrieval/thread_test.go | 17 +++ internal/schema/read_test.go | 20 +++ schemas/v2/search.schema.json | 2 +- 12 files changed, 537 insertions(+), 9 deletions(-) create mode 100644 internal/cli/search.go create mode 100644 internal/cli/search_test.go diff --git a/internal/cli/read.go b/internal/cli/read.go index 17484ee..e6ddff6 100644 --- a/internal/cli/read.go +++ b/internal/cli/read.go @@ -50,6 +50,14 @@ func flagChanged(cmd *cobra.Command, name string) bool { } func normalizeReadPosts(ctx context.Context, runtime *Runtime, posts []mattermost.Post, myUserID string) ([]output.Message, []output.Redaction, error) { + users, err := loadReadUsers(ctx, runtime, posts) + if err != nil { + return nil, nil, err + } + return normalizeReadPostsWithUsers(runtime, posts, users, myUserID) +} + +func loadReadUsers(ctx context.Context, runtime *Runtime, posts []mattermost.Post) (map[string]mattermost.User, error) { users := make(map[string]mattermost.User) ids := normalization.PostUserIDs(posts) if len(ids) != 0 { @@ -60,18 +68,27 @@ func normalizeReadPosts(ctx context.Context, runtime *Runtime, posts []mattermos } items, err := runtime.Users.ByIDs(ctx, ids[start:end]) if err != nil { - return nil, nil, err + return nil, err } for _, user := range items { users[user.ID] = user } } } + return users, nil +} + +func normalizeReadPostsWithUsers(runtime *Runtime, posts []mattermost.Post, users map[string]mattermost.User, myUserID string) ([]output.Message, []output.Redaction, error) { return normalization.NormalizePosts(posts, users, myUserID, runtime.Config.URL, presentation.Options{ Credentials: []string{runtime.Config.Token}, DisableHeuristics: !runtime.Config.Redact, }) } +type readChannelBinding struct { + selectedTeamID string + requireGroupMembership bool +} + func processedChannel(raw mattermost.Channel, runtime *Runtime) (output.Channel, []output.Redaction) { options := presentation.Options{Credentials: []string{runtime.Config.Token}, DisableHeuristics: !runtime.Config.Redact} redactions := make([]output.Redaction, 0) @@ -99,20 +116,32 @@ func processedChannel(raw mattermost.Channel, runtime *Runtime) (output.Channel, }, redactions } -func resolvedReadChannel(ctx context.Context, runtime *Runtime, channelID, myUserID string) (output.Channel, []output.Redaction, bool, error) { +func resolvedReadChannel(ctx context.Context, runtime *Runtime, channelID, myUserID string, binding readChannelBinding) (output.Channel, []output.Redaction, bool, error) { if channelID == "" { channel, redactions := unavailableChannel(channelID, runtime) return channel, redactions, true, nil } raw, err := runtime.Channels.ByID(ctx, channelID) if err != nil { - var remote *api.APIError - if errors.As(err, &remote) && remote.Status == 401 { + if fatalMetadataResolutionError(ctx, err) { return output.Channel{}, nil, false, err } channel, redactions := unavailableChannel(channelID, runtime) return channel, redactions, true, nil } + if (raw.Type == "O" || raw.Type == "P") && binding.selectedTeamID != "" && raw.TeamID != binding.selectedTeamID { + channel, redactions := unavailableChannel(channelID, runtime) + return channel, redactions, true, nil + } + if raw.Type == "G" && binding.requireGroupMembership { + if _, err := runtime.Channels.Member(ctx, raw.ID, myUserID); err != nil { + if fatalMetadataResolutionError(ctx, err) { + return output.Channel{}, nil, false, err + } + channel, redactions := unavailableChannel(channelID, runtime) + return channel, redactions, true, nil + } + } if raw.Type != "D" { channel, redactions := processedChannel(raw, runtime) return channel, redactions, false, nil @@ -128,8 +157,7 @@ func resolvedReadChannel(ctx context.Context, runtime *Runtime, channelID, myUse } other, err := runtime.Users.ByID(ctx, otherID) if err != nil { - var remote *api.APIError - if errors.As(err, &remote) && remote.Status == 401 { + if fatalMetadataResolutionError(ctx, err) { return output.Channel{}, nil, false, err } channel, redactions := unavailableChannel(channelID, runtime) @@ -142,6 +170,14 @@ func resolvedReadChannel(ctx context.Context, runtime *Runtime, channelID, myUse return channel, redactions, false, nil } +func fatalMetadataResolutionError(ctx context.Context, err error) bool { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || ctx.Err() != nil { + return true + } + var remote *api.APIError + return errors.As(err, &remote) && remote.Status == 401 +} + func unavailableChannel(channelID string, runtime *Runtime) (output.Channel, []output.Redaction) { id, redactions := processLabel(channelID, "channel.id", runtime) return output.Channel{ID: id, Type: "unknown", Name: "unknown", MetadataStatus: "unavailable"}, redactions diff --git a/internal/cli/root.go b/internal/cli/root.go index a8bb23b..9e7feee 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -118,6 +118,7 @@ func newRootWithState(state *rootState) *cobra.Command { cmd.AddCommand(newSchemaCommand(state)) cmd.AddCommand(newChannelCommand(state)) cmd.AddCommand(newThreadCommand(state)) + cmd.AddCommand(newSearchCommand(state)) return cmd } diff --git a/internal/cli/search.go b/internal/cli/search.go new file mode 100644 index 0000000..2f43119 --- /dev/null +++ b/internal/cli/search.go @@ -0,0 +1,152 @@ +package cli + +import ( + "fmt" + "strings" + + "github.com/spf13/cobra" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/output" + "github.com/ardasevinc/mattermost-cli/internal/retrieval" +) + +type searchFlags struct { + team, limit string +} + +func newSearchCommand(state *rootState) *cobra.Command { + flags := new(searchFlags) + command := &cobra.Command{ + Use: "search ", Short: "Search messages within one selected team", Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { return runSearch(cmd, state, args[0], *flags) }, + } + command.Flags().StringVar(&flags.team, "team", "", "team name (auto-detected for one team)") + command.Flags().StringVarP(&flags.limit, "limit", "l", "50", "maximum seed results") + return command +} + +func runSearch(cmd *cobra.Command, state *rootState, rawQuery string, flags searchFlags) error { + query := strings.TrimSpace(rawQuery) + if query == "" { + return invalidFailure("search query cannot be empty") + } + limit, err := positiveInteger(flags.limit) + if err != nil { + return err + } + display, err := state.readDisplay(cmd) + if err != nil { + return err + } + runtime, err := state.runtimeFor(cmd) + if err != nil { + return err + } + if err := emitRedactionWarning(state, runtime, display.json); err != nil { + return err + } + me, err := runtime.Users.Current(cmd.Context()) + if err != nil { + return readFailure(err) + } + team, err := runtime.Teams.Resolve(cmd.Context(), me.ID, flags.team) + if err != nil { + return readFailure(err) + } + result, err := retrieval.Search(cmd.Context(), runtime.Posts, team.ID, query, retrieval.SearchOptions{ + Limit: limit, + Accept: func(post mattermost.Post) bool { return post.DeleteAt == 0 }, + }) + if err != nil { + return readFailure(err) + } + if len(result.Posts) == 0 && result.Completeness == retrieval.CompletenessUnknown { + return readError("Mattermost could not confirm an empty search result") + } + + groups, order := groupSearchPosts(result.Posts) + type hydratedSearchGroup struct { + channelID string + seeds []mattermost.Post + hydrated retrieval.HydrationResult + } + hydratedGroups := make([]hydratedSearchGroup, 0, len(order)) + allPosts := make([]mattermost.Post, 0, len(result.Posts)) + for _, channelID := range order { + seeds := groups[channelID] + hydrated, hydrateErr := retrieval.HydrateVisibleThreads(cmd.Context(), runtime.Posts, seeds, display.threads) + if hydrateErr != nil { + return readFailure(hydrateErr) + } + hydratedGroups = append(hydratedGroups, hydratedSearchGroup{channelID: channelID, seeds: seeds, hydrated: hydrated}) + allPosts = append(allPosts, hydrated.Posts...) + } + users, err := loadReadUsers(cmd.Context(), runtime, allPosts) + if err != nil { + return readFailure(err) + } + sections := make([]output.MessageOutput, 0, len(order)) + for _, group := range hydratedGroups { + messages, redactions, normalizeErr := normalizeReadPostsWithUsers(runtime, group.hydrated.Posts, users, me.ID) + if normalizeErr != nil { + return readFailure(normalizeErr) + } + if display.threads { + messages = output.GroupIntoThreads(messages) + } + channel, channelRedactions, unavailable, channelErr := resolvedReadChannel(cmd.Context(), runtime, group.channelID, me.ID, readChannelBinding{ + selectedTeamID: team.ID, requireGroupMembership: true, + }) + if channelErr != nil { + return readFailure(channelErr) + } + redactions = append(channelRedactions, redactions...) + threads, threadRedactions := processedVisibleThreads(group.hydrated.VisibleThreads, runtime) + redactions = append(redactions, threadRedactions...) + requestedLimit := limit + sections = append(sections, output.MessageOutput{Channel: channel, Messages: messages, Redactions: redactions, Retrieval: output.Retrieval{ + Selection: output.Selection{Source: "search", SelectedCount: len(group.seeds), RequestedLimit: &requestedLimit, QueryTruncated: truncatedPointer(result.Completeness)}, + VisibleThreads: threads, VisiblePostCount: len(group.hydrated.Posts), DeletedPostsIncluded: false, + }}) + if unavailable { + label := channel.ID + if label == "" { + label = "an unknown channel" + } + if err := searchWarning(state, display.json, fmt.Sprintf("warning: channel metadata is unavailable for %s\n", label)); err != nil { + return err + } + } + for _, rootID := range threads.FailedRootIDs { + if err := searchWarning(state, display.json, fmt.Sprintf("warning: thread %s could only be partially hydrated\n", rootID)); err != nil { + return err + } + } + } + envelope, err := output.NewSearchEnvelope(sections, machineCompleteness(result.Completeness)) + if err != nil { + return internalFailure(err) + } + return state.renderRead(sections, envelope, display) +} + +func groupSearchPosts(posts []mattermost.Post) (map[string][]mattermost.Post, []string) { + groups := make(map[string][]mattermost.Post) + order := make([]string, 0) + for _, post := range posts { + if _, exists := groups[post.ChannelID]; !exists { + order = append(order, post.ChannelID) + } + groups[post.ChannelID] = append(groups[post.ChannelID], post) + } + return groups, order +} + +func searchWarning(state *rootState, machine bool, warning string) error { + if machine { + state.queueMachineWarning(warning) + return nil + } + return writeAll(state.streams.err, []byte(warning)) +} diff --git a/internal/cli/search_test.go b/internal/cli/search_test.go new file mode 100644 index 0000000..71389b3 --- /dev/null +++ b/internal/cli/search_test.go @@ -0,0 +1,181 @@ +package cli + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" +) + +func TestSearchJSONRunsSelectedTeamReadPipeline(t *testing.T) { + postTime := time.Now().Add(-time.Hour).UnixMilli() + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/users/me": + writeJSON(t, writer, `{"id":"user1","username":"arda"}`) + case "/api/v4/users/user1/teams": + writeJSON(t, writer, `[{"id":"team1","name":"main","display_name":"Main","type":"O"}]`) + case "/api/v4/teams/team1/posts/search": + if request.Method != http.MethodPost { + t.Fatalf("search method = %s", request.Method) + } + var body struct { + Terms string `json:"terms"` + Page int `json:"page"` + PerPage int `json:"per_page"` + } + if err := json.NewDecoder(request.Body).Decode(&body); err != nil { + t.Fatal(err) + } + if body.Terms != "needle" || body.Page != 0 || body.PerPage != 2 { + t.Fatalf("search body = %+v", body) + } + post := fmt.Sprintf(`{"id":"post1","channel_id":"channel1","user_id":"user1","message":"needle and test-token","create_at":%d,"update_at":%d,"delete_at":0,"root_id":"","reply_count":0}`, postTime, postTime) + writeJSON(t, writer, `{"order":["post1"],"posts":{"post1":`+post+`},"matches":{"post1":["needle"]},"has_next":false}`) + case "/api/v4/users/ids": + writeJSON(t, writer, `[{"id":"user1","username":"arda"}]`) + case "/api/v4/channels/channel1": + writeJSON(t, writer, `{"id":"channel1","team_id":"team1","type":"O","name":"town-square","display_name":"Town Square"}`) + default: + t.Fatalf("unexpected request: %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + + stdout, stderr, code := executeChannel(t, server.URL, "--json", "--no-threads", "search", " needle ", "--team", "main", "--limit", "1") + if code != 0 || stderr != "" { + t.Fatalf("exit=%d stderr=%q", code, stderr) + } + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/search", strings.NewReader(stdout)); err != nil { + t.Fatalf("machine output did not validate: %v\n%s", err, stdout) + } + var document struct { + Schema string `json:"schema"` + Results []struct { + Messages []struct{ Text, User string } `json:"messages"` + Metadata struct { + Completeness string `json:"completeness"` + Selection struct { + Source, Query string + SelectedCount int `json:"selectedCount"` + } `json:"selection"` + } `json:"metadata"` + } `json:"results"` + } + if err := json.Unmarshal([]byte(stdout), &document); err != nil { + t.Fatal(err) + } + if document.Schema != "mm/v2/search" || len(document.Results) != 1 || len(document.Results[0].Messages) != 1 || document.Results[0].Messages[0].User != "you" || strings.Contains(document.Results[0].Messages[0].Text, "test-token") || document.Results[0].Metadata.Completeness != "complete" || document.Results[0].Metadata.Selection.Source != "search" || document.Results[0].Metadata.Selection.SelectedCount != 1 { + t.Fatalf("unexpected document: %+v", document) + } +} + +func TestSearchRejectsBlankQueryBeforeNetwork(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { requests.Add(1) })) + defer server.Close() + _, stderr, code := executeChannel(t, server.URL, "search", " \t ") + if code != 2 || !strings.Contains(stderr, "search query cannot be empty") || requests.Load() != 0 { + t.Fatalf("exit=%d stderr=%q requests=%d", code, stderr, requests.Load()) + } +} + +func TestSearchCompleteEmptyEmitsEmptyEnvelopeAndUnknownEmptyFailsClosed(t *testing.T) { + for _, test := range []struct { + name, response string + wantCode int + }{ + {name: "complete", response: `{"order":[],"posts":{},"has_next":false}`, wantCode: 0}, + {name: "unknown", response: `{"order":[],"posts":{},"has_next":true}`, wantCode: 3}, + } { + t.Run(test.name, func(t *testing.T) { + server := searchEmptyServer(t, test.response) + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "--json", "--no-threads", "search", "needle") + if code != test.wantCode { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } + if test.wantCode == 0 && !strings.Contains(stdout, `"schema":"mm/v2/search"`) || test.wantCode != 0 && !strings.Contains(stderr, `"code":"read_failed"`) { + t.Fatalf("stdout=%q stderr=%q", stdout, stderr) + } + }) + } +} + +func TestSearchDoesNotBindForeignTeamChannelMetadata(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/users/me": + writeJSON(t, writer, `{"id":"user1","username":"arda"}`) + case "/api/v4/users/user1/teams": + writeJSON(t, writer, `[{"id":"team1","name":"main","display_name":"Main","type":"O"}]`) + case "/api/v4/teams/team1/posts/search": + writeJSON(t, writer, `{"order":["post1"],"posts":{"post1":{"id":"post1","channel_id":"channel1","user_id":"user1","message":"needle","create_at":1,"update_at":1,"delete_at":0,"root_id":"","reply_count":0}},"has_next":false}`) + case "/api/v4/users/ids": + writeJSON(t, writer, `[{"id":"user1","username":"arda"}]`) + case "/api/v4/channels/channel1": + writeJSON(t, writer, `{"id":"channel1","team_id":"team2","type":"O","name":"foreign","display_name":"Foreign"}`) + default: + t.Fatalf("unexpected request: %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "--json", "--no-threads", "search", "needle", "--team", "main") + if code != 0 || !strings.Contains(stderr, "channel metadata is unavailable") || !strings.Contains(stdout, `"type":"unknown"`) || strings.Contains(stdout, "Foreign") { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } +} + +func TestSearchBatchesUserHydrationAcrossChannels(t *testing.T) { + var userBatches atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/users/me": + writeJSON(t, writer, `{"id":"user1","username":"arda"}`) + case "/api/v4/users/user1/teams": + writeJSON(t, writer, `[{"id":"team1","name":"main","display_name":"Main","type":"O"}]`) + case "/api/v4/teams/team1/posts/search": + writeJSON(t, writer, `{"order":["post2","post1"],"posts":{"post1":{"id":"post1","channel_id":"channel1","user_id":"user2","message":"one","create_at":1,"update_at":1,"delete_at":0,"root_id":"","reply_count":0},"post2":{"id":"post2","channel_id":"channel2","user_id":"user3","message":"two","create_at":2,"update_at":2,"delete_at":0,"root_id":"","reply_count":0}},"has_next":false}`) + case "/api/v4/users/ids": + userBatches.Add(1) + writeJSON(t, writer, `[{"id":"user2","username":"two"},{"id":"user3","username":"three"}]`) + case "/api/v4/channels/channel1": + writeJSON(t, writer, `{"id":"channel1","team_id":"team1","type":"O","name":"one","display_name":"One"}`) + case "/api/v4/channels/channel2": + writeJSON(t, writer, `{"id":"channel2","team_id":"team1","type":"O","name":"two","display_name":"Two"}`) + default: + t.Fatalf("unexpected request: %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + _, stderr, code := executeChannel(t, server.URL, "--json", "--no-threads", "search", "needle", "--team", "main") + if code != 0 || stderr != "" || userBatches.Load() != 1 { + t.Fatalf("exit=%d stderr=%q user batches=%d", code, stderr, userBatches.Load()) + } +} + +func searchEmptyServer(t *testing.T, response string) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/users/me": + writeJSON(t, writer, `{"id":"user1","username":"arda"}`) + case "/api/v4/users/user1/teams": + writeJSON(t, writer, `[{"id":"team1","name":"main","display_name":"Main","type":"O"}]`) + case "/api/v4/teams/team1/posts/search": + writeJSON(t, writer, response) + default: + t.Fatalf("unexpected request: %s %s", request.Method, request.URL.String()) + } + })) +} diff --git a/internal/cli/thread.go b/internal/cli/thread.go index d576a9a..5652a78 100644 --- a/internal/cli/thread.go +++ b/internal/cli/thread.go @@ -73,7 +73,7 @@ func runThread(cmd *cobra.Command, state *rootState, postID string) error { return readFailure(err) } messages = output.GroupIntoThreads(messages) - channel, channelRedactions, unavailable, err := resolvedReadChannel(cmd.Context(), runtime, channelID, me.ID) + channel, channelRedactions, unavailable, err := resolvedReadChannel(cmd.Context(), runtime, channelID, me.ID, readChannelBinding{}) if err != nil { return readFailure(err) } diff --git a/internal/cli/thread_test.go b/internal/cli/thread_test.go index 6b91278..fe0328f 100644 --- a/internal/cli/thread_test.go +++ b/internal/cli/thread_test.go @@ -85,12 +85,56 @@ func TestResolvedReadChannelAcceptsSelfDM(t *testing.T) { } defer client.Close() runtime := &Runtime{Config: config.Resolved{URL: server.URL, Token: "token", Redact: true}, Client: client, Users: mattermost.NewUsers(client), Channels: mattermost.NewChannels(client)} - channel, _, unavailable, err := resolvedReadChannel(context.Background(), runtime, "dm1", "user1") + channel, _, unavailable, err := resolvedReadChannel(context.Background(), runtime, "dm1", "user1", readChannelBinding{}) if err != nil || unavailable || channel.Type != "dm" || channel.Name != "@arda" || channel.MetadataStatus != "resolved" { t.Fatalf("channel=%+v unavailable=%v err=%v", channel, unavailable, err) } } +func TestResolvedReadChannelPropagatesCancellation(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { requests.Add(1) })) + defer server.Close() + client, err := api.New(server.URL, "token") + if err != nil { + t.Fatal(err) + } + defer client.Close() + runtime := &Runtime{Config: config.Resolved{URL: server.URL, Token: "token", Redact: true}, Client: client, Channels: mattermost.NewChannels(client)} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, _, unavailable, err := resolvedReadChannel(ctx, runtime, "channel1", "user1", readChannelBinding{}); err == nil || unavailable { + t.Fatalf("unavailable=%v err=%v", unavailable, err) + } + if requests.Load() != 0 { + t.Fatalf("canceled resolution issued %d requests", requests.Load()) + } +} + +func TestResolvedReadChannelRequiresSearchGroupMembership(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/channels/group1": + writeJSON(t, writer, `{"id":"group1","team_id":"","type":"G","name":"group-name","display_name":"Group"}`) + case "/api/v4/channels/group1/members/user1": + http.Error(writer, "not found", http.StatusNotFound) + default: + t.Fatalf("unexpected request: %s", request.URL.Path) + } + })) + defer server.Close() + client, err := api.New(server.URL, "token") + if err != nil { + t.Fatal(err) + } + defer client.Close() + runtime := &Runtime{Config: config.Resolved{URL: server.URL, Token: "token", Redact: true}, Client: client, Channels: mattermost.NewChannels(client)} + channel, _, unavailable, err := resolvedReadChannel(context.Background(), runtime, "group1", "user1", readChannelBinding{requireGroupMembership: true}) + if err != nil || !unavailable || channel.Type != "unknown" || channel.MetadataStatus != "unavailable" { + t.Fatalf("channel=%+v unavailable=%v err=%v", channel, unavailable, err) + } +} + func TestThreadMissingRootIsExplicitlyPartialInHumanAndMachineOutput(t *testing.T) { server := threadServer(t, time.Now().UnixMilli(), false) defer server.Close() diff --git a/internal/output/machine_convert.go b/internal/output/machine_convert.go index c38acfc..f6a0e81 100644 --- a/internal/output/machine_convert.go +++ b/internal/output/machine_convert.go @@ -50,6 +50,9 @@ func MachineHistoryFromOutput(value MessageOutput, completeness MachineCompleten if err := validateRetrieval(value.Retrieval); err != nil { return MachineHistory{}, err } + if err := validateQueryCompleteness(value.Retrieval.Selection.QueryTruncated, completeness); err != nil { + return MachineHistory{}, err + } for index := range value.Messages { if err := validateMessage(value.Messages[index], 0); err != nil { return MachineHistory{}, fmt.Errorf("message %d: %w", index, err) @@ -78,6 +81,26 @@ func MachineHistoryFromOutput(value MessageOutput, completeness MachineCompleten }, nil } +func validateQueryCompleteness(queryTruncated *bool, completeness MachineCompleteness) error { + switch completeness { + case MachineComplete: + if queryTruncated == nil || *queryTruncated { + return fmt.Errorf("complete retrieval requires queryTruncated false") + } + case MachineTruncated: + if queryTruncated == nil || !*queryTruncated { + return fmt.Errorf("truncated retrieval requires queryTruncated true") + } + case MachineUnknown: + if queryTruncated != nil { + return fmt.Errorf("unknown retrieval requires queryTruncated null") + } + default: + return fmt.Errorf("invalid machine completeness %q", completeness) + } + return nil +} + func NewChannelEnvelope(value MessageOutput, completeness MachineCompleteness) (ChannelEnvelope, error) { if err := validateSource(value.Retrieval.Selection.Source, "recent"); err != nil { return ChannelEnvelope{}, err @@ -97,6 +120,9 @@ func NewGroupDMSEnvelope(values []MessageOutput, completeness MachineCompletenes } func NewSearchEnvelope(values []MessageOutput, completeness MachineCompleteness) (SearchEnvelope, error) { + if len(values) == 0 && completeness != MachineComplete { + return SearchEnvelope{}, fmt.Errorf("empty search output requires confirmed completeness") + } histories, err := machineHistories(values, completeness, "search") return SearchEnvelope{Schema: "mm/v2/search", Results: histories}, err } diff --git a/internal/output/machine_convert_test.go b/internal/output/machine_convert_test.go index 6443304..fc069f4 100644 --- a/internal/output/machine_convert_test.go +++ b/internal/output/machine_convert_test.go @@ -290,6 +290,23 @@ func TestThreadEnvelopeRejectsContradictoryRetrievalMetadata(t *testing.T) { } } +func TestHistoryEnvelopeRejectsCompletenessThatContradictsQueryTruncation(t *testing.T) { + if _, err := output.NewSearchEnvelope(nil, output.MachineUnknown); err == nil { + t.Fatal("unknown empty search envelope was accepted without metadata") + } + value := validOutput() + value.Retrieval.Selection.Source = "search" + truncated := true + value.Retrieval.Selection.QueryTruncated = &truncated + if _, err := output.NewSearchEnvelope([]output.MessageOutput{value}, output.MachineComplete); err == nil { + t.Fatal("complete search accepted queryTruncated true") + } + truncated = false + if _, err := output.NewSearchEnvelope([]output.MessageOutput{value}, output.MachineUnknown); err == nil { + t.Fatal("unknown search accepted known queryTruncated status") + } +} + func validOutput() output.MessageOutput { zone := time.FixedZone("source", 3*60*60) stamp := time.Date(2026, 7, 16, 13, 14, 15, 987654321, zone) diff --git a/internal/retrieval/hydration.go b/internal/retrieval/hydration.go index 1542728..80c1e5b 100644 --- a/internal/retrieval/hydration.go +++ b/internal/retrieval/hydration.go @@ -94,6 +94,10 @@ dispatch: } for index, rootID := range rootIDs { outcome := outcomes[index] + if !hydratedThreadMatchesSeedChannel(seeds, rootID, outcome.result.Posts) { + outcome.result.Posts = nil + outcome.err = mattermost.ErrInvalidPostsResponse + } for _, post := range outcome.result.Posts { if _, exists := seen[post.ID]; exists { continue @@ -118,6 +122,36 @@ dispatch: return HydrationResult{Posts: posts, VisibleThreads: metadata}, nil } +func hydratedThreadMatchesSeedChannel(seeds []mattermost.Post, rootID string, hydrated []mattermost.Post) bool { + expected := "" + for _, post := range seeds { + if !post.ThreadShapeKnown { + continue + } + candidate := post.RootID + if candidate == "" && post.ID == rootID { + candidate = post.ID + } + if candidate != rootID || post.ChannelID == "" { + continue + } + if expected == "" { + expected = post.ChannelID + } else if expected != post.ChannelID { + return false + } + } + if expected == "" { + return false + } + for _, post := range hydrated { + if post.ChannelID != expected { + return false + } + } + return true +} + func visibleRootIDs(posts []mattermost.Post) ([]string, bool) { seen := make(map[string]struct{}) result := make([]string, 0) diff --git a/internal/retrieval/thread_test.go b/internal/retrieval/thread_test.go index fc43b8f..27a5f34 100644 --- a/internal/retrieval/thread_test.go +++ b/internal/retrieval/thread_test.go @@ -131,6 +131,23 @@ func TestHydrationFailureMetadataFollowsSeedOrderAndRetainsPartialPosts(t *testi } } +func TestHydrationRejectsThreadContextFromAnotherSeedChannel(t *testing.T) { + seed := threadPost("seed", "root", 3, 0) + foreignRoot := threadPost("root", "", 1, 1) + foreignReply := threadPost("foreign-reply", "root", 2, 0) + foreignRoot.ChannelID = "other-channel" + foreignReply.ChannelID = "other-channel" + result, err := HydrateVisibleThreads(context.Background(), threadSourceFunc(func(context.Context, string, mattermost.ThreadPageOptions) (mattermost.OrderedPostsPage, error) { + return mattermost.OrderedPostsPage{Posts: []mattermost.Post{foreignRoot, foreignReply}, HasNext: boolPointer(false)}, nil + }), []mattermost.Post{seed}, true) + if err != nil { + t.Fatal(err) + } + if result.VisibleThreads.Status != VisibleThreadsPartial || fmt.Sprint(result.VisibleThreads.FailedRootIDs) != "[root]" || fmt.Sprint(ids(result.Posts)) != "[seed]" { + t.Fatalf("foreign thread context was not rejected: %#v", result) + } +} + func TestThreadMissingLegacyShapeCannotProveCompleteness(t *testing.T) { root := mattermost.Post{ID: "root", ChannelID: "channel", Message: "root", CreateAt: 1} result, err := Thread(context.Background(), threadSourceFunc(func(context.Context, string, mattermost.ThreadPageOptions) (mattermost.OrderedPostsPage, error) { diff --git a/internal/schema/read_test.go b/internal/schema/read_test.go index 5dd9e88..b9e9e7e 100644 --- a/internal/schema/read_test.go +++ b/internal/schema/read_test.go @@ -195,3 +195,23 @@ func TestThreadSchemaEnforcesRootAndUnboundShape(t *testing.T) { validate(t, document, false) }) } + +func TestSearchSchemaBindsSourceAndCompleteness(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + valid, err := fs.ReadFile(publicschemas.FS, "v2/examples/search.json") + if err != nil { + t.Fatal(err) + } + for name, document := range map[string]string{ + "wrong source": strings.Replace(string(valid), `"source":"search"`, `"source":"recent"`, 1), + "complete is truncated": strings.Replace(string(valid), `"queryTruncated":false`, `"queryTruncated":true`, 1), + "complete is unknown": strings.Replace(string(valid), `"queryTruncated":false`, `"queryTruncated":null`, 1), + } { + if err := registry.Validate("mm/v2/search", strings.NewReader(document)); err == nil { + t.Errorf("accepted %s", name) + } + } +} diff --git a/schemas/v2/search.schema.json b/schemas/v2/search.schema.json index 89c54f4..5468cbf 100644 --- a/schemas/v2/search.schema.json +++ b/schemas/v2/search.schema.json @@ -1 +1 @@ -{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"urn:mm:schema:v2:search","type":"object","additionalProperties":false,"required":["schema","results"],"properties":{"schema":{"const":"mm/v2/search"},"results":{"type":"array","items":{"$ref":"#/$defs/history"}}},"$defs":{"timestamp":{"type":"string","format":"date-time","pattern":"^(?:000[1-9]|00[1-9][0-9]|0[1-9][0-9]{2}|[1-9][0-9]{3})-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\\.[0-9]{3}Z$"},"nullableTimestamp":{"anyOf":[{"$ref":"#/$defs/timestamp"},{"type":"null"}]},"file":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"name":{"type":"string"},"mime":{"type":"string"},"size":{"type":"integer","minimum":0},"extension":{"type":"string"}}},"attachmentField":{"type":"object","additionalProperties":false,"properties":{"title":{"type":"string"},"value":{"type":"string"},"short":{"type":"boolean"}}},"attachment":{"type":"object","additionalProperties":false,"properties":{"fallback":{"type":"string"},"pretext":{"type":"string"},"title":{"type":"string"},"titleLink":{"type":"string"},"text":{"type":"string"},"fields":{"type":"array","items":{"$ref":"#/$defs/attachmentField"}},"footer":{"type":"string"},"footerIcon":{"type":"string"},"authorName":{"type":"string"},"authorLink":{"type":"string"},"authorIcon":{"type":"string"},"color":{"type":"string"},"imageUrl":{"type":"string"},"thumbUrl":{"type":"string"},"timestamp":{"type":"string"}}},"reactionActor":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"username":{"type":"string"}}},"reaction":{"type":"object","additionalProperties":false,"required":["emoji","count","actors"],"properties":{"emoji":{"type":"string"},"count":{"type":"integer","minimum":0},"actors":{"type":"array","items":{"$ref":"#/$defs/reactionActor"}}}},"redaction":{"type":"object","additionalProperties":false,"required":["type","masked","position"],"properties":{"type":{"type":"string"},"masked":{"type":"string"},"position":{"type":"integer","minimum":0},"field":{"type":"string"}}},"channel":{"type":"object","additionalProperties":false,"required":["id","type","name","displayName","metadataStatus"],"properties":{"id":{"type":"string"},"type":{"enum":["dm","public","private","group","unknown"]},"name":{"type":"string"},"displayName":{"type":"string"},"metadataStatus":{"enum":["resolved","unavailable"]}}},"selection":{"type":"object","additionalProperties":false,"required":["source","selectedCount","requestedLimit","since","queryTruncated","inputCursor","nextCursor"],"properties":{"source":{"enum":["recent","search","mentions","unread","thread"]},"selectedCount":{"type":"integer","minimum":0},"requestedLimit":{"type":["integer","null"],"minimum":1},"since":{"type":["string","null"]},"queryTruncated":{"type":["boolean","null"]},"inputCursor":{"type":["string","null"]},"nextCursor":{"type":["string","null"]}}},"visibleThreads":{"type":"object","additionalProperties":false,"required":["status","hydratedRootCount","failedRootIds"],"properties":{"status":{"enum":["not_requested","complete","partial"]},"hydratedRootCount":{"type":"integer","minimum":0},"failedRootIds":{"type":"array","items":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":false,"required":["completeness","selection","visibleThreads","visiblePostCount","deletedPostsIncluded"],"properties":{"completeness":{"enum":["complete","truncated","unknown"]},"selection":{"$ref":"#/$defs/selection"},"visibleThreads":{"$ref":"#/$defs/visibleThreads"},"visiblePostCount":{"type":"integer","minimum":0},"deletedPostsIncluded":{"const":false}}},"message":{"type":"object","additionalProperties":false,"required":["id","permalink","user","userId","text","timestamp","updatedAt","editedAt","deletedAt","isDeleted","postType","isSystem","isPinned","rootId","replyCount","files","fileDetails","attachments","reactions","replies"],"properties":{"id":{"type":"string"},"permalink":{"type":"string"},"user":{"type":"string"},"userId":{"type":"string"},"text":{"type":"string"},"timestamp":{"$ref":"#/$defs/timestamp"},"updatedAt":{"$ref":"#/$defs/timestamp"},"editedAt":{"$ref":"#/$defs/nullableTimestamp"},"deletedAt":{"$ref":"#/$defs/nullableTimestamp"},"isDeleted":{"type":"boolean"},"postType":{"type":"string"},"isSystem":{"type":"boolean"},"isPinned":{"type":"boolean"},"rootId":{"type":["string","null"]},"replyCount":{"type":["integer","null"],"minimum":0},"files":{"type":"array","items":{"type":"string"}},"fileDetails":{"type":"array","items":{"$ref":"#/$defs/file"}},"attachments":{"type":"array","items":{"$ref":"#/$defs/attachment"}},"reactions":{"type":"array","items":{"$ref":"#/$defs/reaction"}},"replies":{"type":"array","items":{"$ref":"#/$defs/message"}}}},"history":{"type":"object","additionalProperties":false,"required":["channel","messages","redactions","metadata"],"properties":{"channel":{"$ref":"#/$defs/channel"},"messages":{"type":"array","items":{"$ref":"#/$defs/message"}},"redactions":{"type":"array","items":{"$ref":"#/$defs/redaction"}},"metadata":{"$ref":"#/$defs/metadata"}}}}} +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"urn:mm:schema:v2:search","type":"object","additionalProperties":false,"required":["schema","results"],"properties":{"schema":{"const":"mm/v2/search"},"results":{"type":"array","items":{"$ref":"#/$defs/searchHistory"}}},"$defs":{"timestamp":{"type":"string","format":"date-time","pattern":"^(?:000[1-9]|00[1-9][0-9]|0[1-9][0-9]{2}|[1-9][0-9]{3})-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\\.[0-9]{3}Z$"},"nullableTimestamp":{"anyOf":[{"$ref":"#/$defs/timestamp"},{"type":"null"}]},"file":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"name":{"type":"string"},"mime":{"type":"string"},"size":{"type":"integer","minimum":0},"extension":{"type":"string"}}},"attachmentField":{"type":"object","additionalProperties":false,"properties":{"title":{"type":"string"},"value":{"type":"string"},"short":{"type":"boolean"}}},"attachment":{"type":"object","additionalProperties":false,"properties":{"fallback":{"type":"string"},"pretext":{"type":"string"},"title":{"type":"string"},"titleLink":{"type":"string"},"text":{"type":"string"},"fields":{"type":"array","items":{"$ref":"#/$defs/attachmentField"}},"footer":{"type":"string"},"footerIcon":{"type":"string"},"authorName":{"type":"string"},"authorLink":{"type":"string"},"authorIcon":{"type":"string"},"color":{"type":"string"},"imageUrl":{"type":"string"},"thumbUrl":{"type":"string"},"timestamp":{"type":"string"}}},"reactionActor":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"username":{"type":"string"}}},"reaction":{"type":"object","additionalProperties":false,"required":["emoji","count","actors"],"properties":{"emoji":{"type":"string"},"count":{"type":"integer","minimum":0},"actors":{"type":"array","items":{"$ref":"#/$defs/reactionActor"}}}},"redaction":{"type":"object","additionalProperties":false,"required":["type","masked","position"],"properties":{"type":{"type":"string"},"masked":{"type":"string"},"position":{"type":"integer","minimum":0},"field":{"type":"string"}}},"channel":{"type":"object","additionalProperties":false,"required":["id","type","name","displayName","metadataStatus"],"properties":{"id":{"type":"string"},"type":{"enum":["dm","public","private","group","unknown"]},"name":{"type":"string"},"displayName":{"type":"string"},"metadataStatus":{"enum":["resolved","unavailable"]}}},"selection":{"type":"object","additionalProperties":false,"required":["source","selectedCount","requestedLimit","since","queryTruncated","inputCursor","nextCursor"],"properties":{"source":{"enum":["recent","search","mentions","unread","thread"]},"selectedCount":{"type":"integer","minimum":0},"requestedLimit":{"type":["integer","null"],"minimum":1},"since":{"type":["string","null"]},"queryTruncated":{"type":["boolean","null"]},"inputCursor":{"type":["string","null"]},"nextCursor":{"type":["string","null"]}}},"visibleThreads":{"type":"object","additionalProperties":false,"required":["status","hydratedRootCount","failedRootIds"],"properties":{"status":{"enum":["not_requested","complete","partial"]},"hydratedRootCount":{"type":"integer","minimum":0},"failedRootIds":{"type":"array","items":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":false,"required":["completeness","selection","visibleThreads","visiblePostCount","deletedPostsIncluded"],"properties":{"completeness":{"enum":["complete","truncated","unknown"]},"selection":{"$ref":"#/$defs/selection"},"visibleThreads":{"$ref":"#/$defs/visibleThreads"},"visiblePostCount":{"type":"integer","minimum":0},"deletedPostsIncluded":{"const":false}}},"message":{"type":"object","additionalProperties":false,"required":["id","permalink","user","userId","text","timestamp","updatedAt","editedAt","deletedAt","isDeleted","postType","isSystem","isPinned","rootId","replyCount","files","fileDetails","attachments","reactions","replies"],"properties":{"id":{"type":"string"},"permalink":{"type":"string"},"user":{"type":"string"},"userId":{"type":"string"},"text":{"type":"string"},"timestamp":{"$ref":"#/$defs/timestamp"},"updatedAt":{"$ref":"#/$defs/timestamp"},"editedAt":{"$ref":"#/$defs/nullableTimestamp"},"deletedAt":{"$ref":"#/$defs/nullableTimestamp"},"isDeleted":{"type":"boolean"},"postType":{"type":"string"},"isSystem":{"type":"boolean"},"isPinned":{"type":"boolean"},"rootId":{"type":["string","null"]},"replyCount":{"type":["integer","null"],"minimum":0},"files":{"type":"array","items":{"type":"string"}},"fileDetails":{"type":"array","items":{"$ref":"#/$defs/file"}},"attachments":{"type":"array","items":{"$ref":"#/$defs/attachment"}},"reactions":{"type":"array","items":{"$ref":"#/$defs/reaction"}},"replies":{"type":"array","items":{"$ref":"#/$defs/message"}}}},"searchMetadata":{"allOf":[{"$ref":"#/$defs/metadata"},{"properties":{"selection":{"properties":{"source":{"const":"search"}}}}},{"if":{"properties":{"completeness":{"const":"complete"}}},"then":{"properties":{"selection":{"properties":{"queryTruncated":{"const":false}}}}}},{"if":{"properties":{"completeness":{"const":"truncated"}}},"then":{"properties":{"selection":{"properties":{"queryTruncated":{"const":true}}}}}},{"if":{"properties":{"completeness":{"const":"unknown"}}},"then":{"properties":{"selection":{"properties":{"queryTruncated":{"type":"null"}}}}}}]},"searchHistory":{"type":"object","additionalProperties":false,"required":["channel","messages","redactions","metadata"],"properties":{"channel":{"$ref":"#/$defs/channel"},"messages":{"type":"array","items":{"$ref":"#/$defs/message"}},"redactions":{"type":"array","items":{"$ref":"#/$defs/redaction"}},"metadata":{"$ref":"#/$defs/searchMetadata"}}}}} From 2838d1fc826b0e74f3d169344060ba911fb5e7d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 15:41:27 +0300 Subject: [PATCH 033/119] feat: port mentions read command --- internal/cli/mentions.go | 150 ++++++++++++++++++++++++ internal/cli/mentions_test.go | 104 ++++++++++++++++ internal/cli/root.go | 1 + internal/output/machine_convert.go | 3 + internal/output/machine_convert_test.go | 3 + internal/retrieval/mentions.go | 41 +++++-- internal/retrieval/mentions_test.go | 36 +++++- internal/schema/read_test.go | 20 ++++ schemas/v2/mentions.schema.json | 2 +- 9 files changed, 347 insertions(+), 13 deletions(-) create mode 100644 internal/cli/mentions.go create mode 100644 internal/cli/mentions_test.go diff --git a/internal/cli/mentions.go b/internal/cli/mentions.go new file mode 100644 index 0000000..4ef3db2 --- /dev/null +++ b/internal/cli/mentions.go @@ -0,0 +1,150 @@ +package cli + +import ( + "fmt" + "time" + + "github.com/spf13/cobra" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/output" + "github.com/ardasevinc/mattermost-cli/internal/retrieval" +) + +type mentionsFlags struct { + team, limit, since, channel string +} + +func newMentionsCommand(state *rootState) *cobra.Command { + flags := new(mentionsFlags) + command := &cobra.Command{ + Use: "mentions", Short: "Find mentions within one selected team", Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { return runMentions(cmd, state, *flags) }, + } + command.Flags().StringVar(&flags.team, "team", "", "team name (auto-detected for one team)") + command.Flags().StringVarP(&flags.limit, "limit", "l", "50", "maximum seed results") + command.Flags().StringVarP(&flags.since, "since", "s", "", "time range such as 24h, 7d, 1w, or 2m") + command.Flags().StringVar(&flags.channel, "channel", "", "scope mentions to a channel name") + return command +} + +func runMentions(cmd *cobra.Command, state *rootState, flags mentionsFlags) error { + limit, err := positiveInteger(flags.limit) + if err != nil { + return err + } + var since *int64 + if flags.since != "" { + value, durationErr := durationBoundary(flags.since, time.Now()) + if durationErr != nil { + return durationErr + } + since = &value + } + display, err := state.readDisplay(cmd) + if err != nil { + return err + } + runtime, err := state.runtimeFor(cmd) + if err != nil { + return err + } + if err := emitRedactionWarning(state, runtime, display.json); err != nil { + return err + } + me, err := runtime.Users.Current(cmd.Context()) + if err != nil { + return readFailure(err) + } + team, err := runtime.Teams.Resolve(cmd.Context(), me.ID, flags.team) + if err != nil { + return readFailure(err) + } + var channelName, channelID *string + if flags.channel != "" { + channel, channelErr := runtime.Channels.ByName(cmd.Context(), team.ID, flags.channel) + if channelErr != nil { + return readFailure(channelErr) + } + channelName, channelID = &channel.Name, &channel.ID + } + result, err := retrieval.Mentions(cmd.Context(), runtime.Posts, team.ID, retrieval.MentionsOptions{ + Username: me.Username, Aliases: runtime.Config.MentionNames, Channel: channelName, ChannelID: channelID, Since: since, Limit: limit, + }) + if err != nil { + return readFailure(err) + } + if len(result.Posts) == 0 && result.Completeness != retrieval.CompletenessComplete { + return readError("Mattermost could not confirm an empty mentions result") + } + + groups, order := groupSearchPosts(result.Posts) + type hydratedMentionGroup struct { + channelID string + seeds []mattermost.Post + hydrated retrieval.HydrationResult + } + hydratedGroups := make([]hydratedMentionGroup, 0, len(order)) + allPosts := make([]mattermost.Post, 0, len(result.Posts)) + for _, channelID := range order { + seeds := groups[channelID] + hydrated, hydrateErr := retrieval.HydrateVisibleThreads(cmd.Context(), runtime.Posts, seeds, display.threads) + if hydrateErr != nil { + return readFailure(hydrateErr) + } + hydratedGroups = append(hydratedGroups, hydratedMentionGroup{channelID: channelID, seeds: seeds, hydrated: hydrated}) + allPosts = append(allPosts, hydrated.Posts...) + } + users, err := loadReadUsers(cmd.Context(), runtime, allPosts) + if err != nil { + return readFailure(err) + } + sections := make([]output.MessageOutput, 0, len(order)) + for _, group := range hydratedGroups { + messages, redactions, normalizeErr := normalizeReadPostsWithUsers(runtime, group.hydrated.Posts, users, me.ID) + if normalizeErr != nil { + return readFailure(normalizeErr) + } + if display.threads { + messages = output.GroupIntoThreads(messages) + } + channel, channelRedactions, unavailable, channelErr := resolvedReadChannel(cmd.Context(), runtime, group.channelID, me.ID, readChannelBinding{ + selectedTeamID: team.ID, requireGroupMembership: true, + }) + if channelErr != nil { + return readFailure(channelErr) + } + redactions = append(channelRedactions, redactions...) + threads, threadRedactions := processedVisibleThreads(group.hydrated.VisibleThreads, runtime) + redactions = append(redactions, threadRedactions...) + requestedLimit := limit + var sinceText *string + if since != nil { + value := time.UnixMilli(*since).UTC().Format("2006-01-02T15:04:05.000Z") + sinceText = &value + } + sections = append(sections, output.MessageOutput{Channel: channel, Messages: messages, Redactions: redactions, Retrieval: output.Retrieval{ + Selection: output.Selection{Source: "mentions", SelectedCount: len(group.seeds), RequestedLimit: &requestedLimit, Since: sinceText, QueryTruncated: truncatedPointer(result.Completeness)}, + VisibleThreads: threads, VisiblePostCount: len(group.hydrated.Posts), DeletedPostsIncluded: false, + }}) + if unavailable { + label := channel.ID + if label == "" { + label = "an unknown channel" + } + if err := searchWarning(state, display.json, fmt.Sprintf("warning: channel metadata is unavailable for %s\n", label)); err != nil { + return err + } + } + for _, rootID := range threads.FailedRootIDs { + if err := searchWarning(state, display.json, fmt.Sprintf("warning: thread %s could only be partially hydrated\n", rootID)); err != nil { + return err + } + } + } + envelope, err := output.NewMentionsEnvelope(sections, machineCompleteness(result.Completeness)) + if err != nil { + return internalFailure(err) + } + return state.renderRead(sections, envelope, display) +} diff --git a/internal/cli/mentions_test.go b/internal/cli/mentions_test.go new file mode 100644 index 0000000..12433b8 --- /dev/null +++ b/internal/cli/mentions_test.go @@ -0,0 +1,104 @@ +package cli + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" +) + +func TestMentionsJSONUsesAliasesScopesGlobalSelectionAndHydration(t *testing.T) { + now := time.Now() + var searches, userBatches atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/users/me": + writeJSON(t, writer, `{"id":"user1","username":"arda"}`) + case "/api/v4/users/user1/teams": + writeJSON(t, writer, `[{"id":"team1","name":"main","display_name":"Main","type":"O"}]`) + case "/api/v4/teams/team1/channels/name/town-square": + writeJSON(t, writer, `{"id":"channel1","team_id":"team1","type":"O","name":"town-square","display_name":"Town Square"}`) + case "/api/v4/teams/team1/posts/search": + var body struct { + Terms string `json:"terms"` + } + if err := json.NewDecoder(request.Body).Decode(&body); err != nil { + t.Fatal(err) + } + if !strings.Contains(body.Terms, "after:") || !strings.HasSuffix(body.Terms, " in:town-square") { + t.Fatalf("scoped terms = %q", body.Terms) + } + searches.Add(1) + post := func(id, channel, user, message string, created int64) string { + return fmt.Sprintf(`{"id":%q,"channel_id":%q,"user_id":%q,"message":%q,"create_at":%d,"update_at":%d,"delete_at":0,"root_id":"","reply_count":0}`, id, channel, user, message, created, created) + } + switch { + case strings.HasPrefix(body.Terms, "@arda "): + writeJSON(t, writer, `{"order":["foreign","shared"],"posts":{"foreign":`+post("foreign", "channel2", "user2", "@arda exact but wrong channel", now.UnixMilli()+3)+`,"shared":`+post("shared", "channel1", "user2", "@arda exact", now.UnixMilli())+`},"has_next":false}`) + case strings.HasPrefix(body.Terms, `"Arda Sevinc" `): + writeJSON(t, writer, `{"order":["new","shared"],"posts":{"new":`+post("new", "channel1", "user3", "hello Arda Sevinc!", now.UnixMilli()+2)+`,"shared":`+post("shared", "channel1", "user2", "Arda Sevinc and @arda", now.UnixMilli())+`},"has_next":false}`) + default: + t.Fatalf("unexpected search terms %q", body.Terms) + } + case "/api/v4/users/ids": + userBatches.Add(1) + writeJSON(t, writer, `[{"id":"user2","username":"two"},{"id":"user3","username":"three"}]`) + case "/api/v4/channels/channel1": + writeJSON(t, writer, `{"id":"channel1","team_id":"team1","type":"O","name":"town-square","display_name":"Town Square"}`) + default: + t.Fatalf("unexpected request: %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + + configRoot := t.TempDir() + configDir := filepath.Join(configRoot, "mattermost-cli") + if err := os.Mkdir(configDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(configDir, "config.toml"), []byte(`mention_names = ["Arda Sevinc", "Arda Sevinc", ""]`), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("HOME", t.TempDir()) + t.Setenv("XDG_CONFIG_HOME", configRoot) + t.Setenv("MM_URL", server.URL) + t.Setenv("MM_TOKEN", "test-token") + stdout, stderr, code := executeWithCurrentEnvironment(t, "--json", "--no-threads", "mentions", "--team", "main", "--limit", "2", "--since", "24h", "--channel", "#town-square") + if code != 0 || stderr != "" || searches.Load() != 2 || userBatches.Load() != 1 { + t.Fatalf("exit=%d stderr=%q searches=%d user batches=%d", code, stderr, searches.Load(), userBatches.Load()) + } + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/mentions", strings.NewReader(stdout)); err != nil { + t.Fatalf("machine output did not validate: %v\n%s", err, stdout) + } + if !strings.Contains(stdout, `"source":"mentions"`) || !strings.Contains(stdout, `"selectedCount":2`) || !strings.Contains(stdout, `"requestedLimit":2`) || !strings.Contains(stdout, `"completeness":"complete"`) || strings.Contains(stdout, "wrong channel") { + t.Fatalf("unexpected output: %s", stdout) + } +} + +func TestMentionsIncompleteEmptyFailsClosed(t *testing.T) { + server := searchEmptyServer(t, `{"order":[],"posts":{},"has_next":true}`) + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "--json", "mentions") + if code != 3 || stdout != "" || !strings.Contains(stderr, `"code":"read_failed"`) { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } +} + +func executeWithCurrentEnvironment(t *testing.T, args ...string) (string, string, int) { + t.Helper() + var stdout, stderr strings.Builder + code := Execute(t.Context(), args, strings.NewReader(""), &stdout, &stderr) + return stdout.String(), stderr.String(), code +} diff --git a/internal/cli/root.go b/internal/cli/root.go index 9e7feee..1a356e5 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -119,6 +119,7 @@ func newRootWithState(state *rootState) *cobra.Command { cmd.AddCommand(newChannelCommand(state)) cmd.AddCommand(newThreadCommand(state)) cmd.AddCommand(newSearchCommand(state)) + cmd.AddCommand(newMentionsCommand(state)) return cmd } diff --git a/internal/output/machine_convert.go b/internal/output/machine_convert.go index f6a0e81..9d987a7 100644 --- a/internal/output/machine_convert.go +++ b/internal/output/machine_convert.go @@ -128,6 +128,9 @@ func NewSearchEnvelope(values []MessageOutput, completeness MachineCompleteness) } func NewMentionsEnvelope(values []MessageOutput, completeness MachineCompleteness) (MentionsEnvelope, error) { + if len(values) == 0 && completeness != MachineComplete { + return MentionsEnvelope{}, fmt.Errorf("empty mentions output requires confirmed completeness") + } histories, err := machineHistories(values, completeness, "mentions") return MentionsEnvelope{Schema: "mm/v2/mentions", Results: histories}, err } diff --git a/internal/output/machine_convert_test.go b/internal/output/machine_convert_test.go index fc069f4..72f7cf1 100644 --- a/internal/output/machine_convert_test.go +++ b/internal/output/machine_convert_test.go @@ -294,6 +294,9 @@ func TestHistoryEnvelopeRejectsCompletenessThatContradictsQueryTruncation(t *tes if _, err := output.NewSearchEnvelope(nil, output.MachineUnknown); err == nil { t.Fatal("unknown empty search envelope was accepted without metadata") } + if _, err := output.NewMentionsEnvelope(nil, output.MachineTruncated); err == nil { + t.Fatal("truncated empty mentions envelope was accepted without metadata") + } value := validOutput() value.Retrieval.Selection.Source = "search" truncated := true diff --git a/internal/retrieval/mentions.go b/internal/retrieval/mentions.go index 3b0db6e..c48a6bc 100644 --- a/internal/retrieval/mentions.go +++ b/internal/retrieval/mentions.go @@ -22,11 +22,12 @@ const ( var ErrInvalidMentionsRequest = errors.New("invalid mentions request") type MentionsOptions struct { - Username string - Aliases []string - Channel *string - Since *int64 - Limit int + Username string + Aliases []string + Channel *string + ChannelID *string + Since *int64 + Limit int } type MentionsResult struct { @@ -60,8 +61,10 @@ func retrieveMentions(ctx context.Context, search mentionSearchFunc, teamID stri } query := searchTermWithScope(term, channel, options.Since) result, err := search(ctx, teamID, query, SearchOptions{ - Limit: options.Limit, - Accept: func(post mattermost.Post) bool { return isExactMention(post, term, options.Since) }, + Limit: options.Limit, + Accept: func(post mattermost.Post) bool { + return (options.ChannelID == nil || post.ChannelID == *options.ChannelID) && isExactMention(post, term, options.Since) + }, }) if err != nil { return MentionsResult{}, err @@ -78,6 +81,9 @@ func retrieveMentions(ctx context.Context, search mentionSearchFunc, teamID stri } func mentionTerms(options MentionsOptions) ([]string, string, error) { + if (options.Channel == nil) != (options.ChannelID == nil) { + return nil, "", ErrInvalidMentionsRequest + } username := strings.TrimSpace(options.Username) channel := "" if options.Channel != nil { @@ -86,21 +92,29 @@ func mentionTerms(options MentionsOptions) ([]string, string, error) { return nil, "", ErrInvalidMentionsRequest } } + if options.ChannelID != nil { + channelID := strings.TrimSpace(*options.ChannelID) + if channelID != *options.ChannelID || !safeSearchAtom(channelID) { + return nil, "", ErrInvalidMentionsRequest + } + } if options.Limit <= 0 || int64(options.Limit) > maxSafeInteger || !safeSearchAtom(username) || - len(options.Aliases) > MaxMentionAliases || (options.Since != nil && (*options.Since < 0 || *options.Since > maxDateMilliseconds)) || + (options.Since != nil && (*options.Since < 0 || *options.Since > maxDateMilliseconds)) || (channel != "" && !safeSearchAtom(channel)) { return nil, "", ErrInvalidMentionsRequest } terms := make([]string, 0, len(options.Aliases)+1) seen := make(map[string]struct{}, len(options.Aliases)+1) - add := func(term string) { + add := func(term string) bool { if _, ok := seen[term]; ok { - return + return false } seen[term] = struct{}{} terms = append(terms, term) + return true } add("@" + username) + aliasCount := 0 for _, raw := range options.Aliases { alias := strings.TrimSpace(raw) if alias == "" { @@ -114,7 +128,12 @@ func mentionTerms(options MentionsOptions) ([]string, string, error) { return nil, "", ErrInvalidMentionsRequest } } - add("\"" + alias + "\"") + if add("\"" + alias + "\"") { + aliasCount++ + if aliasCount > MaxMentionAliases { + return nil, "", ErrInvalidMentionsRequest + } + } } return terms, channel, nil } diff --git a/internal/retrieval/mentions_test.go b/internal/retrieval/mentions_test.go index 3fa2635..df077e3 100644 --- a/internal/retrieval/mentions_test.go +++ b/internal/retrieval/mentions_test.go @@ -14,7 +14,7 @@ func TestMentionTermsTrimQuoteDedupeAndScope(t *testing.T) { since := int64(1_768_404_600_000) // 2026-01-14 15:30:00 UTC terms, channel, err := mentionTerms(MentionsOptions{ Username: " arda ", Aliases: []string{" Arda Sevinc ", "", "Arda Sevinc", "arda"}, - Channel: stringPointer(" #general "), Since: &since, Limit: 20, + Channel: stringPointer(" #general "), ChannelID: stringPointer("channel-id"), Since: &since, Limit: 20, }) if err != nil || channel != "general" || !reflect.DeepEqual(terms, []string{"@arda", `"Arda Sevinc"`, `"arda"`}) { t.Fatalf("terms=%q channel=%q err=%v", terms, channel, err) @@ -106,6 +106,27 @@ func TestRetrieveMentionsMergesGloballyAndCompleteness(t *testing.T) { } } +func TestRetrieveMentionsEnforcesExactResolvedChannelID(t *testing.T) { + result, err := retrieveMentions(context.Background(), func(_ context.Context, _, _ string, options SearchOptions) (SearchResult, error) { + posts := []mattermost.Post{ + {ID: "expected", ChannelID: "channel-1", Message: "@arda", CreateAt: 1}, + {ID: "foreign", ChannelID: "channel-2", Message: "@arda", CreateAt: 2}, + } + accepted := make([]mattermost.Post, 0, len(posts)) + for _, post := range posts { + if options.Accept(post) { + accepted = append(accepted, post) + } + } + return SearchResult{Posts: accepted, Completeness: CompletenessComplete}, nil + }, "team", MentionsOptions{ + Username: "arda", Channel: stringPointer("town-square"), ChannelID: stringPointer("channel-1"), Limit: 10, + }) + if err != nil || fmt.Sprint(ids(result.Posts)) != "[expected]" { + t.Fatalf("result=%#v err=%v", result, err) + } +} + func TestRetrieveMentionsPropagatesErrorsAndCancellation(t *testing.T) { want := errors.New("search failed") _, err := retrieveMentions(context.Background(), func(context.Context, string, string, SearchOptions) (SearchResult, error) { @@ -127,12 +148,25 @@ func TestRetrieveMentionsPropagatesErrorsAndCancellation(t *testing.T) { } func TestMentionValidationBoundsWithoutReflectingInput(t *testing.T) { + duplicates := make([]string, MaxMentionAliases+1) + for index := range duplicates { + duplicates[index] = "Arda Sevinc" + } + terms, _, err := mentionTerms(MentionsOptions{Username: "arda", Aliases: duplicates, Limit: 1}) + if err != nil || !reflect.DeepEqual(terms, []string{"@arda", `"Arda Sevinc"`}) { + t.Fatalf("duplicate aliases were not bounded after dedupe: terms=%q err=%v", terms, err) + } + distinct := make([]string, MaxMentionAliases+1) + for index := range distinct { + distinct[index] = fmt.Sprintf("alias-%d", index) + } tests := []MentionsOptions{ {Username: "bad user", Limit: 1}, {Username: "arda", Channel: stringPointer("general after:2000-01-01"), Limit: 1}, {Username: "arda", Channel: stringPointer("#"), Limit: 1}, {Username: "arda", Aliases: []string{`hostile" after:2000-01-01`}, Limit: 1}, {Username: "arda", Aliases: []string{string(make([]byte, MaxMentionTermBytes+1))}, Limit: 1}, + {Username: "arda", Aliases: distinct, Limit: 1}, } for _, options := range tests { _, _, err := mentionTerms(options) diff --git a/internal/schema/read_test.go b/internal/schema/read_test.go index b9e9e7e..3b821f5 100644 --- a/internal/schema/read_test.go +++ b/internal/schema/read_test.go @@ -215,3 +215,23 @@ func TestSearchSchemaBindsSourceAndCompleteness(t *testing.T) { } } } + +func TestMentionsSchemaBindsSourceAndCompleteness(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + valid, err := fs.ReadFile(publicschemas.FS, "v2/examples/mentions.json") + if err != nil { + t.Fatal(err) + } + for name, document := range map[string]string{ + "wrong source": strings.Replace(string(valid), `"source":"mentions"`, `"source":"search"`, 1), + "complete is truncated": strings.Replace(string(valid), `"queryTruncated":false`, `"queryTruncated":true`, 1), + "complete is unknown": strings.Replace(string(valid), `"queryTruncated":false`, `"queryTruncated":null`, 1), + } { + if err := registry.Validate("mm/v2/mentions", strings.NewReader(document)); err == nil { + t.Errorf("accepted %s", name) + } + } +} diff --git a/schemas/v2/mentions.schema.json b/schemas/v2/mentions.schema.json index b116532..4f8a751 100644 --- a/schemas/v2/mentions.schema.json +++ b/schemas/v2/mentions.schema.json @@ -1 +1 @@ -{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"urn:mm:schema:v2:mentions","type":"object","additionalProperties":false,"required":["schema","results"],"properties":{"schema":{"const":"mm/v2/mentions"},"results":{"type":"array","items":{"$ref":"#/$defs/history"}}},"$defs":{"timestamp":{"type":"string","format":"date-time","pattern":"^(?:000[1-9]|00[1-9][0-9]|0[1-9][0-9]{2}|[1-9][0-9]{3})-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\\.[0-9]{3}Z$"},"nullableTimestamp":{"anyOf":[{"$ref":"#/$defs/timestamp"},{"type":"null"}]},"file":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"name":{"type":"string"},"mime":{"type":"string"},"size":{"type":"integer","minimum":0},"extension":{"type":"string"}}},"attachmentField":{"type":"object","additionalProperties":false,"properties":{"title":{"type":"string"},"value":{"type":"string"},"short":{"type":"boolean"}}},"attachment":{"type":"object","additionalProperties":false,"properties":{"fallback":{"type":"string"},"pretext":{"type":"string"},"title":{"type":"string"},"titleLink":{"type":"string"},"text":{"type":"string"},"fields":{"type":"array","items":{"$ref":"#/$defs/attachmentField"}},"footer":{"type":"string"},"footerIcon":{"type":"string"},"authorName":{"type":"string"},"authorLink":{"type":"string"},"authorIcon":{"type":"string"},"color":{"type":"string"},"imageUrl":{"type":"string"},"thumbUrl":{"type":"string"},"timestamp":{"type":"string"}}},"reactionActor":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"username":{"type":"string"}}},"reaction":{"type":"object","additionalProperties":false,"required":["emoji","count","actors"],"properties":{"emoji":{"type":"string"},"count":{"type":"integer","minimum":0},"actors":{"type":"array","items":{"$ref":"#/$defs/reactionActor"}}}},"redaction":{"type":"object","additionalProperties":false,"required":["type","masked","position"],"properties":{"type":{"type":"string"},"masked":{"type":"string"},"position":{"type":"integer","minimum":0},"field":{"type":"string"}}},"channel":{"type":"object","additionalProperties":false,"required":["id","type","name","displayName","metadataStatus"],"properties":{"id":{"type":"string"},"type":{"enum":["dm","public","private","group","unknown"]},"name":{"type":"string"},"displayName":{"type":"string"},"metadataStatus":{"enum":["resolved","unavailable"]}}},"selection":{"type":"object","additionalProperties":false,"required":["source","selectedCount","requestedLimit","since","queryTruncated","inputCursor","nextCursor"],"properties":{"source":{"enum":["recent","search","mentions","unread","thread"]},"selectedCount":{"type":"integer","minimum":0},"requestedLimit":{"type":["integer","null"],"minimum":1},"since":{"type":["string","null"]},"queryTruncated":{"type":["boolean","null"]},"inputCursor":{"type":["string","null"]},"nextCursor":{"type":["string","null"]}}},"visibleThreads":{"type":"object","additionalProperties":false,"required":["status","hydratedRootCount","failedRootIds"],"properties":{"status":{"enum":["not_requested","complete","partial"]},"hydratedRootCount":{"type":"integer","minimum":0},"failedRootIds":{"type":"array","items":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":false,"required":["completeness","selection","visibleThreads","visiblePostCount","deletedPostsIncluded"],"properties":{"completeness":{"enum":["complete","truncated","unknown"]},"selection":{"$ref":"#/$defs/selection"},"visibleThreads":{"$ref":"#/$defs/visibleThreads"},"visiblePostCount":{"type":"integer","minimum":0},"deletedPostsIncluded":{"const":false}}},"message":{"type":"object","additionalProperties":false,"required":["id","permalink","user","userId","text","timestamp","updatedAt","editedAt","deletedAt","isDeleted","postType","isSystem","isPinned","rootId","replyCount","files","fileDetails","attachments","reactions","replies"],"properties":{"id":{"type":"string"},"permalink":{"type":"string"},"user":{"type":"string"},"userId":{"type":"string"},"text":{"type":"string"},"timestamp":{"$ref":"#/$defs/timestamp"},"updatedAt":{"$ref":"#/$defs/timestamp"},"editedAt":{"$ref":"#/$defs/nullableTimestamp"},"deletedAt":{"$ref":"#/$defs/nullableTimestamp"},"isDeleted":{"type":"boolean"},"postType":{"type":"string"},"isSystem":{"type":"boolean"},"isPinned":{"type":"boolean"},"rootId":{"type":["string","null"]},"replyCount":{"type":["integer","null"],"minimum":0},"files":{"type":"array","items":{"type":"string"}},"fileDetails":{"type":"array","items":{"$ref":"#/$defs/file"}},"attachments":{"type":"array","items":{"$ref":"#/$defs/attachment"}},"reactions":{"type":"array","items":{"$ref":"#/$defs/reaction"}},"replies":{"type":"array","items":{"$ref":"#/$defs/message"}}}},"history":{"type":"object","additionalProperties":false,"required":["channel","messages","redactions","metadata"],"properties":{"channel":{"$ref":"#/$defs/channel"},"messages":{"type":"array","items":{"$ref":"#/$defs/message"}},"redactions":{"type":"array","items":{"$ref":"#/$defs/redaction"}},"metadata":{"$ref":"#/$defs/metadata"}}}}} +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"urn:mm:schema:v2:mentions","type":"object","additionalProperties":false,"required":["schema","results"],"properties":{"schema":{"const":"mm/v2/mentions"},"results":{"type":"array","items":{"$ref":"#/$defs/mentionsHistory"}}},"$defs":{"timestamp":{"type":"string","format":"date-time","pattern":"^(?:000[1-9]|00[1-9][0-9]|0[1-9][0-9]{2}|[1-9][0-9]{3})-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\\.[0-9]{3}Z$"},"nullableTimestamp":{"anyOf":[{"$ref":"#/$defs/timestamp"},{"type":"null"}]},"file":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"name":{"type":"string"},"mime":{"type":"string"},"size":{"type":"integer","minimum":0},"extension":{"type":"string"}}},"attachmentField":{"type":"object","additionalProperties":false,"properties":{"title":{"type":"string"},"value":{"type":"string"},"short":{"type":"boolean"}}},"attachment":{"type":"object","additionalProperties":false,"properties":{"fallback":{"type":"string"},"pretext":{"type":"string"},"title":{"type":"string"},"titleLink":{"type":"string"},"text":{"type":"string"},"fields":{"type":"array","items":{"$ref":"#/$defs/attachmentField"}},"footer":{"type":"string"},"footerIcon":{"type":"string"},"authorName":{"type":"string"},"authorLink":{"type":"string"},"authorIcon":{"type":"string"},"color":{"type":"string"},"imageUrl":{"type":"string"},"thumbUrl":{"type":"string"},"timestamp":{"type":"string"}}},"reactionActor":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"username":{"type":"string"}}},"reaction":{"type":"object","additionalProperties":false,"required":["emoji","count","actors"],"properties":{"emoji":{"type":"string"},"count":{"type":"integer","minimum":0},"actors":{"type":"array","items":{"$ref":"#/$defs/reactionActor"}}}},"redaction":{"type":"object","additionalProperties":false,"required":["type","masked","position"],"properties":{"type":{"type":"string"},"masked":{"type":"string"},"position":{"type":"integer","minimum":0},"field":{"type":"string"}}},"channel":{"type":"object","additionalProperties":false,"required":["id","type","name","displayName","metadataStatus"],"properties":{"id":{"type":"string"},"type":{"enum":["dm","public","private","group","unknown"]},"name":{"type":"string"},"displayName":{"type":"string"},"metadataStatus":{"enum":["resolved","unavailable"]}}},"selection":{"type":"object","additionalProperties":false,"required":["source","selectedCount","requestedLimit","since","queryTruncated","inputCursor","nextCursor"],"properties":{"source":{"enum":["recent","search","mentions","unread","thread"]},"selectedCount":{"type":"integer","minimum":0},"requestedLimit":{"type":["integer","null"],"minimum":1},"since":{"type":["string","null"]},"queryTruncated":{"type":["boolean","null"]},"inputCursor":{"type":["string","null"]},"nextCursor":{"type":["string","null"]}}},"visibleThreads":{"type":"object","additionalProperties":false,"required":["status","hydratedRootCount","failedRootIds"],"properties":{"status":{"enum":["not_requested","complete","partial"]},"hydratedRootCount":{"type":"integer","minimum":0},"failedRootIds":{"type":"array","items":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":false,"required":["completeness","selection","visibleThreads","visiblePostCount","deletedPostsIncluded"],"properties":{"completeness":{"enum":["complete","truncated","unknown"]},"selection":{"$ref":"#/$defs/selection"},"visibleThreads":{"$ref":"#/$defs/visibleThreads"},"visiblePostCount":{"type":"integer","minimum":0},"deletedPostsIncluded":{"const":false}}},"message":{"type":"object","additionalProperties":false,"required":["id","permalink","user","userId","text","timestamp","updatedAt","editedAt","deletedAt","isDeleted","postType","isSystem","isPinned","rootId","replyCount","files","fileDetails","attachments","reactions","replies"],"properties":{"id":{"type":"string"},"permalink":{"type":"string"},"user":{"type":"string"},"userId":{"type":"string"},"text":{"type":"string"},"timestamp":{"$ref":"#/$defs/timestamp"},"updatedAt":{"$ref":"#/$defs/timestamp"},"editedAt":{"$ref":"#/$defs/nullableTimestamp"},"deletedAt":{"$ref":"#/$defs/nullableTimestamp"},"isDeleted":{"type":"boolean"},"postType":{"type":"string"},"isSystem":{"type":"boolean"},"isPinned":{"type":"boolean"},"rootId":{"type":["string","null"]},"replyCount":{"type":["integer","null"],"minimum":0},"files":{"type":"array","items":{"type":"string"}},"fileDetails":{"type":"array","items":{"$ref":"#/$defs/file"}},"attachments":{"type":"array","items":{"$ref":"#/$defs/attachment"}},"reactions":{"type":"array","items":{"$ref":"#/$defs/reaction"}},"replies":{"type":"array","items":{"$ref":"#/$defs/message"}}}},"mentionsMetadata":{"allOf":[{"$ref":"#/$defs/metadata"},{"properties":{"selection":{"properties":{"source":{"const":"mentions"}}}}},{"if":{"properties":{"completeness":{"const":"complete"}}},"then":{"properties":{"selection":{"properties":{"queryTruncated":{"const":false}}}}}},{"if":{"properties":{"completeness":{"const":"truncated"}}},"then":{"properties":{"selection":{"properties":{"queryTruncated":{"const":true}}}}}},{"if":{"properties":{"completeness":{"const":"unknown"}}},"then":{"properties":{"selection":{"properties":{"queryTruncated":{"type":"null"}}}}}}]},"mentionsHistory":{"type":"object","additionalProperties":false,"required":["channel","messages","redactions","metadata"],"properties":{"channel":{"$ref":"#/$defs/channel"},"messages":{"type":"array","items":{"$ref":"#/$defs/message"}},"redactions":{"type":"array","items":{"$ref":"#/$defs/redaction"}},"metadata":{"$ref":"#/$defs/mentionsMetadata"}}}}} From 2faee2bbe0c3358844e816afad0131e15847d443 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 16:17:33 +0300 Subject: [PATCH 034/119] feat: port direct-message read command --- internal/cli/dms.go | 352 ++++++++++++++++++++++++ internal/cli/dms_test.go | 162 +++++++++++ internal/cli/root.go | 1 + internal/mattermost/channels.go | 61 ++++ internal/mattermost/channels_test.go | 44 +++ internal/output/machine_convert.go | 3 + internal/output/machine_convert_test.go | 3 + internal/retrieval/channel.go | 15 +- internal/retrieval/channel_test.go | 17 +- internal/retrieval/dms.go | 81 ++++++ internal/retrieval/dms_test.go | 104 +++++++ internal/schema/read_test.go | 21 ++ schemas/v2/dms.schema.json | 2 +- schemas/v2/examples/dms.json | 2 +- 14 files changed, 862 insertions(+), 6 deletions(-) create mode 100644 internal/cli/dms.go create mode 100644 internal/cli/dms_test.go create mode 100644 internal/retrieval/dms.go create mode 100644 internal/retrieval/dms_test.go diff --git a/internal/cli/dms.go b/internal/cli/dms.go new file mode 100644 index 0000000..4a4bcc8 --- /dev/null +++ b/internal/cli/dms.go @@ -0,0 +1,352 @@ +package cli + +import ( + "errors" + "fmt" + "sort" + "strings" + "time" + + "github.com/spf13/cobra" + + "github.com/ardasevinc/mattermost-cli/internal/api" + "github.com/ardasevinc/mattermost-cli/internal/cursor" + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/normalization" + "github.com/ardasevinc/mattermost-cli/internal/output" + "github.com/ardasevinc/mattermost-cli/internal/retrieval" +) + +type dmsFlags struct { + users []string + limit, since, channel, cursor string +} + +func newDMsCommand(state *rootState) *cobra.Command { + flags := new(dmsFlags) + command := &cobra.Command{Use: "dms", Short: "Fetch direct messages", Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { return runDMs(cmd, state, *flags) }} + command.Flags().StringSliceVarP(&flags.users, "user", "u", nil, "filter by username (repeatable or comma-separated)") + command.Flags().StringVarP(&flags.limit, "limit", "l", "50", "maximum total seed messages across matched DMs") + command.Flags().StringVarP(&flags.since, "since", "s", "7d", "time range such as 24h, 7d, 1w, or 2m") + command.Flags().StringVarP(&flags.channel, "channel", "c", "", "specific direct-message channel ID") + command.Flags().StringVar(&flags.cursor, "cursor", "", "resume deterministic direct-message history") + return command +} + +func runDMs(cmd *cobra.Command, state *rootState, flags dmsFlags) error { + if flagChanged(cmd, "user") { + if len(flags.users) == 0 { + return invalidFailure("--user cannot be empty") + } + for _, username := range flags.users { + if strings.TrimSpace(username) == "" { + return invalidFailure("--user cannot be empty") + } + } + } + if flagChanged(cmd, "channel") && strings.TrimSpace(flags.channel) == "" { + return invalidFailure("--channel cannot be empty") + } + if flagChanged(cmd, "cursor") && strings.TrimSpace(flags.cursor) == "" { + return invalidFailure("--cursor cannot be empty") + } + limit, err := positiveInteger(flags.limit) + if err != nil { + return err + } + if flags.channel != "" && len(flags.users) != 0 { + return invalidFailure("--channel cannot be combined with --user") + } + var resume *cursor.ChannelHistory + if flags.cursor != "" { + decoded, decodeErr := cursor.DecodeChannelHistory(flags.cursor) + if decodeErr != nil { + return invalidFailure("invalid direct-message cursor") + } + resume = &decoded + if flags.channel == "" { + return invalidFailure("a cursor requires --channel for direct-message history") + } + if flagChanged(cmd, "since") { + return invalidFailure("a cursor cannot be combined with --since") + } + } + var since *int64 + if resume != nil { + since = resume.Since + } else { + value, durationErr := durationBoundary(flags.since, time.Now()) + if durationErr != nil { + return durationErr + } + since = &value + } + display, err := state.readDisplay(cmd) + if err != nil { + return err + } + runtime, err := state.runtimeFor(cmd) + if err != nil { + return err + } + if err := emitRedactionWarning(state, runtime, display.json); err != nil { + return err + } + me, err := runtime.Users.Current(cmd.Context()) + if err != nil { + return readFailure(err) + } + + channels, err := selectDMChannels(cmd, state, runtime, me, flags, display.json) + if err != nil { + return err + } + if resume != nil && (len(channels) != 1 || resume.ChannelID != channels[0].ID) { + return invalidFailure("cursor does not match the selected channel") + } + if len(channels) == 0 { + envelope, envelopeErr := output.NewDMSEnvelope(nil, output.MachineComplete) + if envelopeErr != nil { + return internalFailure(envelopeErr) + } + return state.renderRead(nil, envelope, display) + } + ids := make([]string, len(channels)) + byID := make(map[string]mattermost.Channel, len(channels)) + for index, channel := range channels { + ids[index], byID[channel.ID] = channel.ID, channel + } + options := retrieval.DMHistoryOptions{Limit: limit, Since: since} + if resume != nil { + options.Boundary = &retrieval.Boundary{CreateAt: resume.Boundary.CreateAt, ID: resume.Boundary.ID} + options.SafeBeforePostID = resume.SafeBeforePostID + } + result, err := retrieval.DMHistory(cmd.Context(), runtime.Posts, ids, options) + if err != nil { + return readFailure(err) + } + if len(result.Posts) == 0 && result.Completeness == retrieval.CompletenessUnknown && resume == nil { + return readError("Mattermost could not confirm an empty direct-message history") + } + + nextCursor := "" + if len(result.Posts) > 0 && result.Completeness != retrieval.CompletenessComplete && flags.channel != "" { + boundary := result.Posts[len(result.Posts)-1] + safeBefore := "" + for index := len(result.Posts) - 1; index >= 0; index-- { + if result.Posts[index].CreateAt > boundary.CreateAt { + safeBefore = result.Posts[index].ID + break + } + } + if safeBefore == "" && result.SafeBeforeValid && resume != nil { + safeBefore = resume.SafeBeforePostID + } + nextCursor, err = cursor.EncodeChannelHistory(cursor.ChannelHistory{Version: 1, Scope: "channel", ChannelID: channels[0].ID, + Boundary: cursor.Boundary{CreateAt: boundary.CreateAt, ID: boundary.ID}, Since: since, SafeBeforePostID: safeBefore}) + if err != nil { + return readFailure(err) + } + } else if len(result.Posts) == 0 && result.Completeness == retrieval.CompletenessUnknown && resume != nil { + nextCursor = flags.cursor + } + + groups, order := groupSearchPosts(result.Posts) + type hydratedGroup struct { + channelID string + seeds []mattermost.Post + hydrated retrieval.HydrationResult + } + hydrated := make([]hydratedGroup, 0, len(order)) + allPosts := make([]mattermost.Post, 0, len(result.Posts)) + for _, channelID := range order { + value, hydrateErr := retrieval.HydrateVisibleThreads(cmd.Context(), runtime.Posts, groups[channelID], display.threads) + if hydrateErr != nil { + return readFailure(hydrateErr) + } + hydrated = append(hydrated, hydratedGroup{channelID, groups[channelID], value}) + allPosts = append(allPosts, value.Posts...) + } + userIDs := make([]string, 0, len(hydrated)) + for _, group := range hydrated { + userIDs = append(userIDs, otherDMUserID(byID[group.channelID], me.ID)) + } + if len(result.Posts) == 0 && resume != nil && result.Completeness == retrieval.CompletenessUnknown { + userIDs = append(userIDs, otherDMUserID(channels[0], me.ID)) + } + users, err := loadReadUsersAndIDs(cmd, runtime, allPosts, userIDs) + if err != nil { + return readFailure(err) + } + sections := make([]output.MessageOutput, 0, len(hydrated)) + for _, group := range hydrated { + messages, redactions, normalizeErr := normalizeReadPostsWithUsers(runtime, group.hydrated.Posts, users, me.ID) + if normalizeErr != nil { + return readFailure(normalizeErr) + } + if display.threads { + messages = output.GroupIntoThreads(messages) + } + partner := users[otherDMUserID(byID[group.channelID], me.ID)] + channel, channelRedactions := presentedDMChannel(byID[group.channelID], partner, runtime) + redactions = append(channelRedactions, redactions...) + threads, threadRedactions := processedVisibleThreads(group.hydrated.VisibleThreads, runtime) + redactions = append(redactions, threadRedactions...) + requestedLimit := limit + sinceText := millisecondTimestamp(since) + sections = append(sections, output.MessageOutput{Channel: channel, Messages: messages, Redactions: redactions, Retrieval: output.Retrieval{ + Selection: output.Selection{Source: "recent", SelectedCount: len(group.seeds), RequestedLimit: &requestedLimit, Since: sinceText, + QueryTruncated: truncatedPointer(result.Completeness), InputCursor: stringPointer(flags.cursor), NextCursor: stringPointer(nextCursor)}, + VisibleThreads: threads, VisiblePostCount: len(group.hydrated.Posts), DeletedPostsIncluded: false, + }}) + for _, rootID := range threads.FailedRootIDs { + if err := searchWarning(state, display.json, fmt.Sprintf("warning: thread %s could only be partially hydrated\n", rootID)); err != nil { + return err + } + } + } + if len(result.Posts) == 0 && resume != nil && result.Completeness == retrieval.CompletenessUnknown { + partner := users[otherDMUserID(channels[0], me.ID)] + channel, redactions := presentedDMChannel(channels[0], partner, runtime) + requestedLimit := limit + sinceText := millisecondTimestamp(since) + sections = append(sections, output.MessageOutput{Channel: channel, Redactions: redactions, Retrieval: output.Retrieval{ + Selection: output.Selection{Source: "recent", RequestedLimit: &requestedLimit, Since: sinceText, InputCursor: stringPointer(flags.cursor), NextCursor: stringPointer(nextCursor)}, + VisibleThreads: output.VisibleThreads{Status: map[bool]string{true: "complete", false: "not_requested"}[display.threads], FailedRootIDs: []string{}}, + }}) + } + envelope, err := output.NewDMSEnvelope(sections, machineCompleteness(result.Completeness)) + if err != nil { + return internalFailure(err) + } + return state.renderRead(sections, envelope, display) +} + +func selectDMChannels(cmd *cobra.Command, state *rootState, runtime *Runtime, me mattermost.User, flags dmsFlags, machine bool) ([]mattermost.Channel, error) { + if flags.channel != "" { + channel, err := runtime.Channels.ByID(cmd.Context(), flags.channel) + if err != nil { + return nil, readFailure(err) + } + if channel.Type != "D" { + return nil, invalidFailure("selected channel is not a direct-message channel") + } + if otherDMUserID(channel, me.ID) == "" { + return nil, readFailure(mattermost.ErrInvalidChannelResponse) + } + return []mattermost.Channel{channel}, nil + } + all, err := runtime.Channels.DirectList(cmd.Context(), me.ID) + if err != nil { + return nil, readFailure(err) + } + direct := all + if len(flags.users) == 0 { + return direct, nil + } + byPartner := make(map[string]mattermost.Channel, len(direct)) + for _, channel := range direct { + byPartner[otherDMUserID(channel, me.ID)] = channel + } + selected := make(map[string]mattermost.Channel) + seenNames := make(map[string]struct{}) + for _, username := range flags.users { + key := strings.ToLower(username) + if _, duplicate := seenNames[key]; duplicate { + continue + } + seenNames[key] = struct{}{} + user, lookupErr := runtime.Users.ByUsername(cmd.Context(), username) + if lookupErr != nil { + var remote *api.APIError + if errors.As(lookupErr, &remote) && remote.Status == 404 { + if err := dmWarning(state, machine, fmt.Sprintf("warning: user @%s was not found\n", safeWarningLabel(username, runtime))); err != nil { + return nil, err + } + continue + } + return nil, readFailure(lookupErr) + } + channel, exists := byPartner[user.ID] + if !exists { + if err := dmWarning(state, machine, fmt.Sprintf("warning: no direct-message channel exists with @%s\n", safeWarningLabel(username, runtime))); err != nil { + return nil, err + } + continue + } + selected[channel.ID] = channel + } + result := make([]mattermost.Channel, 0, len(selected)) + for _, channel := range selected { + result = append(result, channel) + } + sort.Slice(result, func(i, j int) bool { return result[i].ID < result[j].ID }) + return result, nil +} + +func otherDMUserID(channel mattermost.Channel, me string) string { + parts := strings.Split(channel.Name, "__") + if channel.Type != "D" || len(parts) != 2 { + return "" + } + if parts[0] == me { + return parts[1] + } + if parts[1] == me { + return parts[0] + } + return "" +} + +func loadReadUsersAndIDs(cmd *cobra.Command, runtime *Runtime, posts []mattermost.Post, extra []string) (map[string]mattermost.User, error) { + ids := make([]string, 0, len(extra)+len(posts)) + ids = append(ids, extra...) + ids = append(ids, normalization.PostUserIDs(posts)...) + users := make(map[string]mattermost.User) + seen := make(map[string]struct{}) + unique := make([]string, 0, len(ids)) + for _, id := range ids { + if id != "" { + if _, ok := seen[id]; !ok { + seen[id] = struct{}{} + unique = append(unique, id) + } + } + } + for start := 0; start < len(unique); start += 200 { + end := start + 200 + if end > len(unique) { + end = len(unique) + } + batch, err := runtime.Users.ByIDs(cmd.Context(), unique[start:end]) + if err != nil { + return nil, err + } + for _, user := range batch { + users[user.ID] = user + } + } + return users, nil +} + +func presentedDMChannel(raw mattermost.Channel, partner mattermost.User, runtime *Runtime) (output.Channel, []output.Redaction) { + id, redactions := processLabel(raw.ID, "channel.id", runtime) + name, nameRedactions := processLabel(partner.Username, "channel.dmUsername", runtime) + return output.Channel{ID: id, Type: "dm", Name: "@" + name, MetadataStatus: "resolved"}, append(redactions, nameRedactions...) +} + +func safeWarningLabel(value string, runtime *Runtime) string { + label, _ := processLabel(value, "warning", runtime) + return label +} +func millisecondTimestamp(value *int64) *string { + if value == nil { + return nil + } + formatted := time.UnixMilli(*value).UTC().Format("2006-01-02T15:04:05.000Z") + return &formatted +} +func dmWarning(state *rootState, machine bool, warning string) error { + return searchWarning(state, machine, warning) +} diff --git a/internal/cli/dms_test.go b/internal/cli/dms_test.go new file mode 100644 index 0000000..f3e3991 --- /dev/null +++ b/internal/cli/dms_test.go @@ -0,0 +1,162 @@ +package cli + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" +) + +func TestDMsJSONDiscoversAccountWideAndAppliesGlobalLimit(t *testing.T) { + now := time.Now().UnixMilli() + var userBatches atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/users/me": + writeJSON(t, writer, `{"id":"self","username":"arda"}`) + case "/api/v4/users/self/channels": + writeJSON(t, writer, `[{"id":"dm-alice","team_id":"","type":"D","name":"alice__self","display_name":""},{"id":"dm-bob","team_id":"","type":"D","name":"bob__self","display_name":""}]`) + case "/api/v4/channels/dm-alice/posts": + post := fmt.Sprintf(`{"id":"alice-new","channel_id":"dm-alice","user_id":"alice","message":"alice","create_at":%d,"delete_at":0,"root_id":"","reply_count":0}`, now) + writeJSON(t, writer, `{"order":["alice-new"],"posts":{"alice-new":`+post+`},"has_next":false}`) + case "/api/v4/channels/dm-bob/posts": + post := fmt.Sprintf(`{"id":"bob-new","channel_id":"dm-bob","user_id":"bob","message":"bob","create_at":%d,"delete_at":0,"root_id":"","reply_count":0}`, now-1) + writeJSON(t, writer, `{"order":["bob-new"],"posts":{"bob-new":`+post+`},"has_next":false}`) + case "/api/v4/users/ids": + userBatches.Add(1) + writeJSON(t, writer, `[{"id":"alice","username":"alice"},{"id":"bob","username":"bob"}]`) + default: + t.Fatalf("unexpected request: %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + + stdout, stderr, code := executeChannel(t, server.URL, "--json", "--no-threads", "dms", "--limit", "2", "--since", "1d") + if code != 0 || stderr != "" || userBatches.Load() != 1 { + t.Fatalf("exit=%d stderr=%q user batches=%d", code, stderr, userBatches.Load()) + } + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/dms", strings.NewReader(stdout)); err != nil { + t.Fatalf("machine output did not validate: %v\n%s", err, stdout) + } + var document struct { + Schema string `json:"schema"` + Channels []struct { + Channel struct{ Name string } `json:"channel"` + Messages []struct{ ID string } `json:"messages"` + } `json:"channels"` + } + if err := json.Unmarshal([]byte(stdout), &document); err != nil { + t.Fatal(err) + } + if document.Schema != "mm/v2/dms" || len(document.Channels) != 2 || document.Channels[0].Channel.Name != "@alice" || document.Channels[1].Channel.Name != "@bob" { + t.Fatalf("unexpected document: %+v", document) + } +} + +func TestDMsRejectsInvalidCursorBeforeNetwork(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { requests.Add(1) })) + defer server.Close() + _, stderr, code := executeChannel(t, server.URL, "dms", "--channel", "dm", "--cursor", "not-a-cursor") + if code != 2 || !strings.Contains(stderr, "invalid direct-message cursor") || requests.Load() != 0 { + t.Fatalf("exit=%d stderr=%q requests=%d", code, stderr, requests.Load()) + } +} + +func TestDMsRejectsExplicitEmptyTargetFlagsBeforeRuntime(t *testing.T) { + for _, args := range [][]string{{"dms", "--user="}, {"dms", "--channel="}, {"dms", "--cursor="}} { + t.Run(strings.Join(args, "_"), func(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { requests.Add(1) })) + defer server.Close() + _, stderr, code := executeChannel(t, server.URL, args...) + if code != 2 || requests.Load() != 0 || !strings.Contains(stderr, "cannot be empty") { + t.Fatalf("exit=%d stderr=%q requests=%d", code, stderr, requests.Load()) + } + }) + } +} + +func TestDMsHydratesOnlyOutputPartnersAndReactionActors(t *testing.T) { + now := time.Now().UnixMilli() + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/users/me": + writeJSON(t, writer, `{"id":"self","username":"arda"}`) + case "/api/v4/users/self/channels": + writeJSON(t, writer, `[{"id":"dm-alice","team_id":"","type":"D","name":"alice__self","display_name":""},{"id":"dm-bob","team_id":"","type":"D","name":"bob__self","display_name":""}]`) + case "/api/v4/channels/dm-alice/posts": + post := fmt.Sprintf(`{"id":"alice-new","channel_id":"dm-alice","user_id":"alice","message":"alice","create_at":%d,"delete_at":0,"root_id":"","reply_count":0,"metadata":{"reactions":[{"user_id":"reactor","post_id":"alice-new","emoji_name":"eyes","create_at":1}]}}`, now) + writeJSON(t, writer, `{"order":["alice-new"],"posts":{"alice-new":`+post+`},"has_next":false}`) + case "/api/v4/channels/dm-bob/posts": + post := fmt.Sprintf(`{"id":"bob-old","channel_id":"dm-bob","user_id":"bob","message":"bob","create_at":%d,"delete_at":0,"root_id":"","reply_count":0}`, now-1) + writeJSON(t, writer, `{"order":["bob-old"],"posts":{"bob-old":`+post+`},"has_next":false}`) + case "/api/v4/users/ids": + var ids []string + if err := json.NewDecoder(request.Body).Decode(&ids); err != nil { + t.Fatal(err) + } + if strings.Join(ids, ",") != "alice,reactor" { + t.Fatalf("hydrated IDs = %v", ids) + } + writeJSON(t, writer, `[{"id":"alice","username":"alice"},{"id":"reactor","username":"carol"}]`) + default: + t.Fatalf("unexpected request: %s", request.URL.String()) + } + })) + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "--json", "--no-threads", "dms", "--limit", "1", "--since", "1d") + if code != 0 || stderr != "" || !strings.Contains(stdout, `"username":"carol"`) || strings.Contains(stdout, "@bob") { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } +} + +func TestDMsConfirmedEmptyDoesNotHydratePartners(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/users/me": + writeJSON(t, writer, `{"id":"self","username":"arda"}`) + case "/api/v4/users/self/channels": + writeJSON(t, writer, `[{"id":"dm","team_id":"","type":"D","name":"alice__self","display_name":""}]`) + case "/api/v4/channels/dm/posts": + writeJSON(t, writer, `{"order":[],"posts":{},"has_next":false}`) + default: + t.Fatalf("unexpected request: %s", request.URL.String()) + } + })) + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "--json", "--no-threads", "dms") + if code != 0 || stderr != "" || stdout != "{\"schema\":\"mm/v2/dms\",\"channels\":[]}\n" { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } +} + +func TestDMsUnknownEmptyFailsClosed(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/users/me": + writeJSON(t, writer, `{"id":"self","username":"arda"}`) + case "/api/v4/users/self/channels": + writeJSON(t, writer, `[{"id":"dm","team_id":"","type":"D","name":"alice__self","display_name":""}]`) + case "/api/v4/channels/dm/posts": + writeJSON(t, writer, `{"order":[],"posts":{},"has_next":true}`) + default: + t.Fatalf("unexpected request: %s", request.URL.String()) + } + })) + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "--json", "--no-threads", "dms") + if code != 3 || stdout != "" || !strings.Contains(stderr, `"code":"read_failed"`) { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } +} diff --git a/internal/cli/root.go b/internal/cli/root.go index 1a356e5..b79a3cb 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -117,6 +117,7 @@ func newRootWithState(state *rootState) *cobra.Command { cmd.PersistentFlags().BoolVar(&state.flags.noThreads, "no-threads", false, "return selected seed posts only") cmd.AddCommand(newSchemaCommand(state)) cmd.AddCommand(newChannelCommand(state)) + cmd.AddCommand(newDMsCommand(state)) cmd.AddCommand(newThreadCommand(state)) cmd.AddCommand(newSearchCommand(state)) cmd.AddCommand(newMentionsCommand(state)) diff --git a/internal/mattermost/channels.go b/internal/mattermost/channels.go index 67a5b2e..cfc875e 100644 --- a/internal/mattermost/channels.go +++ b/internal/mattermost/channels.go @@ -111,6 +111,38 @@ func (l *channelList) UnmarshalJSON(data []byte) error { return nil } +type directChannelList []Channel + +func (l *directChannelList) UnmarshalJSON(data []byte) error { + var raw []json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil || raw == nil { + return ErrInvalidChannelsResponse + } + direct := make([]Channel, 0) + for _, item := range raw { + var discriminator struct { + Type json.RawMessage `json:"type"` + } + if json.Unmarshal(item, &discriminator) != nil { + return ErrInvalidChannelsResponse + } + typeCode, ok := requiredString(discriminator.Type) + if !ok || (typeCode != "O" && typeCode != "P" && typeCode != "D" && typeCode != "G") { + return ErrInvalidChannelsResponse + } + if typeCode != "D" { + continue + } + var channel Channel + if json.Unmarshal(item, &channel) != nil { + return ErrInvalidChannelsResponse + } + direct = append(direct, channel) + } + *l = direct + return nil +} + func (s *Channels) ByID(ctx context.Context, channelID string) (Channel, error) { if strings.TrimSpace(channelID) == "" { return Channel{}, ErrInvalidChannelRequest @@ -209,6 +241,35 @@ func (s *Channels) List(ctx context.Context, userID string) ([]Channel, error) { return result, nil } +// DirectList returns only current-user-bound D channels. It deliberately does +// not resolve team membership or group membership for unrelated channel types. +func (s *Channels) DirectList(ctx context.Context, userID string) ([]Channel, error) { + if strings.TrimSpace(userID) == "" || userID == "me" { + return nil, ErrInvalidChannelRequest + } + var decoded directChannelList + if err := s.client.Get(ctx, "/users/"+url.PathEscape(userID)+"/channels", &decoded); err != nil { + return nil, err + } + seen := make(map[string]Channel) + result := make([]Channel, 0) + for _, channel := range []Channel(decoded) { + if !directChannelContains(channel.Name, userID) { + return nil, ErrInvalidChannelResponse + } + if previous, duplicate := seen[channel.ID]; duplicate { + if previous != channel { + return nil, ErrInvalidChannelsResponse + } + continue + } + seen[channel.ID] = channel + result = append(result, channel) + } + sort.Slice(result, func(i, j int) bool { return result[i].ID < result[j].ID }) + return result, nil +} + func directChannelContains(name, userID string) bool { parts := strings.Split(name, "__") return len(parts) == 2 && (parts[0] == userID || parts[1] == userID) diff --git a/internal/mattermost/channels_test.go b/internal/mattermost/channels_test.go index f0fa1bc..f849ba4 100644 --- a/internal/mattermost/channels_test.go +++ b/internal/mattermost/channels_test.go @@ -138,6 +138,50 @@ func TestChannelListDoesNotRequireTeamsForDirectOnlyDiscovery(t *testing.T) { } } +func TestDirectListIgnoresUnrelatedTeamAndGroupBindings(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{ + "/users/user/channels": `[ + {"type":"P"}, + {"type":"G"}, + {"id":"dm","team_id":"","type":"D","name":"user__other","display_name":""} + ]`, + }} + got, err := NewChannels(f).DirectList(context.Background(), "user") + if err != nil || len(got) != 1 || got[0].ID != "dm" { + t.Fatalf("channels = %#v, error = %v", got, err) + } + if !reflect.DeepEqual(f.paths, []string{"/users/user/channels"}) { + t.Fatalf("paths = %v", f.paths) + } +} + +func TestDirectListRejectsMalformedOrUnboundDirectChannels(t *testing.T) { + tests := map[string]string{ + "malformed direct identity": `[{"id":"dm","team_id":"","type":"D","display_name":""}]`, + "foreign direct identity": `[{"id":"dm","team_id":"","type":"D","name":"alice__bob","display_name":""}]`, + "conflicting duplicate": `[{"id":"dm","team_id":"","type":"D","name":"user__alice","display_name":""},{"id":"dm","team_id":"","type":"D","name":"user__bob","display_name":""}]`, + "missing discriminator": `[{"id":"dm"}]`, + "unknown discriminator": `[{"type":"X"}]`, + } + for name, payload := range tests { + t.Run(name, func(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{"/users/user/channels": payload}} + if _, err := NewChannels(f).DirectList(context.Background(), "user"); err == nil { + t.Fatal("expected direct-list validation error") + } + }) + } +} + +func TestDirectListDedupesExactDirectChannelDuplicates(t *testing.T) { + channel := `{"id":"dm","team_id":"","type":"D","name":"user__other","display_name":""}` + f := &fakeChannelTransport{responses: map[string]string{"/users/user/channels": `[` + channel + `,` + channel + `]`}} + got, err := NewChannels(f).DirectList(context.Background(), "user") + if err != nil || len(got) != 1 || got[0].ID != "dm" { + t.Fatalf("channels=%#v err=%v", got, err) + } +} + func TestChannelReadsAreRaceSafe(t *testing.T) { f := &fakeChannelTransport{responses: map[string]string{ "/channels/x": `{"id":"x","team_id":"team","type":"O","name":"general","display_name":"General"}`, diff --git a/internal/output/machine_convert.go b/internal/output/machine_convert.go index 9d987a7..ab3bcf1 100644 --- a/internal/output/machine_convert.go +++ b/internal/output/machine_convert.go @@ -110,6 +110,9 @@ func NewChannelEnvelope(value MessageOutput, completeness MachineCompleteness) ( } func NewDMSEnvelope(values []MessageOutput, completeness MachineCompleteness) (DMSEnvelope, error) { + if len(values) == 0 && completeness != MachineComplete { + return DMSEnvelope{}, fmt.Errorf("empty direct-message output requires confirmed completeness") + } histories, err := machineHistories(values, completeness, "recent", "dm", "unknown") return DMSEnvelope{Schema: "mm/v2/dms", Channels: histories}, err } diff --git a/internal/output/machine_convert_test.go b/internal/output/machine_convert_test.go index 72f7cf1..8f83f7c 100644 --- a/internal/output/machine_convert_test.go +++ b/internal/output/machine_convert_test.go @@ -297,6 +297,9 @@ func TestHistoryEnvelopeRejectsCompletenessThatContradictsQueryTruncation(t *tes if _, err := output.NewMentionsEnvelope(nil, output.MachineTruncated); err == nil { t.Fatal("truncated empty mentions envelope was accepted without metadata") } + if _, err := output.NewDMSEnvelope(nil, output.MachineUnknown); err == nil { + t.Fatal("unknown empty direct-message envelope was accepted without metadata") + } value := validOutput() value.Retrieval.Selection.Source = "search" truncated := true diff --git a/internal/retrieval/channel.go b/internal/retrieval/channel.go index ce93a38..b25f10f 100644 --- a/internal/retrieval/channel.go +++ b/internal/retrieval/channel.go @@ -36,6 +36,7 @@ type ChannelHistoryOptions struct { Since *int64 Boundary *Boundary SafeBeforePostID string + RequestBudget *int } type ChannelHistoryResult struct { @@ -67,10 +68,20 @@ func ChannelHistory(ctx context.Context, source channelPageSource, channelID str retriedWithoutAnchor := false for { + if err := ctx.Err(); err != nil { + return ChannelHistoryResult{}, err + } if pageNumber >= MaxChannelHistoryPages { uncertain = true break } + if options.RequestBudget != nil { + if *options.RequestBudget <= 0 { + uncertain = true + break + } + *options.RequestBudget = *options.RequestBudget - 1 + } page, err := source.ChannelPage(ctx, channelID, mattermost.ChannelPostsOptions{ PerPage: pageSize, Page: pageNumber, Before: activeBefore, }) @@ -135,10 +146,10 @@ func ChannelHistory(ctx context.Context, source channelPageSource, channelID str result := ChannelHistoryResult{Posts: mostRecent(byID, options.Limit), SafeBeforeValid: options.SafeBeforePostID == "" || !retriedWithoutAnchor} switch { - case len(byID) > options.Limit: - result.Completeness = CompletenessTruncated case uncertain || !exhausted: result.Completeness = CompletenessUnknown + case len(byID) > options.Limit: + result.Completeness = CompletenessTruncated default: result.Completeness = CompletenessComplete } diff --git a/internal/retrieval/channel_test.go b/internal/retrieval/channel_test.go index 5950aad..45817bd 100644 --- a/internal/retrieval/channel_test.go +++ b/internal/retrieval/channel_test.go @@ -32,7 +32,7 @@ func TestChannelHistoryUsesLimitPlusOneAndDeterministicOrder(t *testing.T) { if err != nil { t.Fatal(err) } - if requested.PerPage != 3 || result.Completeness != CompletenessTruncated || fmt.Sprint(ids(result.Posts)) != "[a b]" { + if requested.PerPage != 3 || result.Completeness != CompletenessUnknown || fmt.Sprint(ids(result.Posts)) != "[a b]" { t.Fatalf("requested=%#v result=%#v", requested, result) } } @@ -48,7 +48,7 @@ func TestChannelHistoryCompletesEqualMillisecondTiesAndBoundary(t *testing.T) { if err != nil { t.Fatal(err) } - if fmt.Sprint(ids(result.Posts)) != "[b c]" || result.Completeness != CompletenessTruncated { + if fmt.Sprint(ids(result.Posts)) != "[b c]" || result.Completeness != CompletenessUnknown { t.Fatalf("result = %#v", result) } } @@ -164,6 +164,19 @@ func TestChannelHistoryStopsAtHardPageBoundAsUnknown(t *testing.T) { } } +func TestChannelHistoryBudgetExhaustionDominatesOverLimitSameTimestampCandidates(t *testing.T) { + budget := 1 + result, err := ChannelHistory(context.Background(), pageSourceFunc(func(_ context.Context, _ string, options mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) { + return mattermost.OrderedPostsPage{ + Posts: []mattermost.Post{testPost("a", 200), testPost("b", 200), testPost("c", 200)}, + RawCount: options.PerPage, + }, nil + }), "channel", ChannelHistoryOptions{Limit: 2, RequestBudget: &budget}) + if err != nil || result.Completeness != CompletenessUnknown || fmt.Sprint(ids(result.Posts)) != "[a b]" || budget != 0 { + t.Fatalf("budget=%d result=%#v err=%v", budget, result, err) + } +} + func TestChannelHistoryRejectsValuesOutsideCursorDomain(t *testing.T) { source := pageSourceFunc(func(context.Context, string, mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) { t.Fatal("source called for invalid request") diff --git a/internal/retrieval/dms.go b/internal/retrieval/dms.go new file mode 100644 index 0000000..c8cbbb6 --- /dev/null +++ b/internal/retrieval/dms.go @@ -0,0 +1,81 @@ +package retrieval + +import ( + "context" + "errors" + "strings" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" +) + +var ErrInvalidDMHistoryRequest = errors.New("invalid direct-message history request") + +const MaxDMHistoryRequests = 100 + +type DMHistoryOptions struct { + Limit int + Since *int64 + Boundary *Boundary + SafeBeforePostID string +} + +type DMHistoryResult struct { + Posts []mattermost.Post + Completeness Completeness + SafeBeforeValid bool +} + +// DMHistory retrieves each selected direct channel independently, then caps +// the merged known candidates. Completeness is unknown if any channel could +// not be queried completely, because a global top-N cannot then be proven. +func DMHistory(ctx context.Context, source channelPageSource, channelIDs []string, options DMHistoryOptions) (DMHistoryResult, error) { + if source == nil || len(channelIDs) == 0 || options.Limit <= 0 || int64(options.Limit) > maxSafeInteger || + (options.Boundary != nil && len(channelIDs) != 1) { + return DMHistoryResult{}, ErrInvalidDMHistoryRequest + } + seenChannels := make(map[string]struct{}, len(channelIDs)) + all := make(map[string]mattermost.Post) + unknown, truncated := false, false + safeBeforeValid := true + requestBudget := MaxDMHistoryRequests + for _, channelID := range channelIDs { + if err := ctx.Err(); err != nil { + return DMHistoryResult{}, err + } + if strings.TrimSpace(channelID) == "" { + return DMHistoryResult{}, ErrInvalidDMHistoryRequest + } + if _, duplicate := seenChannels[channelID]; duplicate { + return DMHistoryResult{}, ErrInvalidDMHistoryRequest + } + seenChannels[channelID] = struct{}{} + page, err := ChannelHistory(ctx, source, channelID, ChannelHistoryOptions{ + Limit: options.Limit, Since: options.Since, Boundary: options.Boundary, SafeBeforePostID: options.SafeBeforePostID, + RequestBudget: &requestBudget, + }) + if err != nil { + return DMHistoryResult{}, err + } + if !page.SafeBeforeValid { + safeBeforeValid = false + } + unknown = unknown || page.Completeness == CompletenessUnknown + truncated = truncated || page.Completeness == CompletenessTruncated + for _, post := range page.Posts { + if post.ChannelID != channelID { + return DMHistoryResult{}, ErrInvalidDMHistoryRequest + } + if _, duplicate := all[post.ID]; duplicate { + return DMHistoryResult{}, ErrInvalidDMHistoryRequest + } + all[post.ID] = post + } + } + completeness := CompletenessComplete + if unknown { + completeness = CompletenessUnknown + } else if truncated || len(all) > options.Limit { + completeness = CompletenessTruncated + } + return DMHistoryResult{Posts: mostRecent(all, options.Limit), Completeness: completeness, SafeBeforeValid: safeBeforeValid}, nil +} diff --git a/internal/retrieval/dms_test.go b/internal/retrieval/dms_test.go new file mode 100644 index 0000000..878a492 --- /dev/null +++ b/internal/retrieval/dms_test.go @@ -0,0 +1,104 @@ +package retrieval + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" +) + +func TestDMHistoryAppliesOneGlobalDeterministicLimit(t *testing.T) { + result, err := DMHistory(context.Background(), pageSourceFunc(func(_ context.Context, channelID string, _ mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) { + posts := map[string][]mattermost.Post{ + "alice": {{ID: "alice-new", ChannelID: "alice", CreateAt: 3}, {ID: "alice-old", ChannelID: "alice", CreateAt: 1}}, + "bob": {{ID: "bob-new", ChannelID: "bob", CreateAt: 2}}, + }[channelID] + return mattermost.OrderedPostsPage{Posts: posts, RawCount: len(posts), HasNext: dmBoolPointer(false)}, nil + }), []string{"alice", "bob"}, DMHistoryOptions{Limit: 2}) + if err != nil || result.Completeness != CompletenessTruncated || fmt.Sprint(ids(result.Posts)) != "[alice-new bob-new]" { + t.Fatalf("result=%#v err=%v", result, err) + } +} + +func TestDMHistoryPreservesUnknownEmpty(t *testing.T) { + result, err := DMHistory(context.Background(), pageSourceFunc(func(context.Context, string, mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) { + truth := true + return mattermost.OrderedPostsPage{HasNext: &truth}, nil + }), []string{"dm"}, DMHistoryOptions{Limit: 2}) + if err != nil || len(result.Posts) != 0 || result.Completeness != CompletenessUnknown { + t.Fatalf("result=%#v err=%v", result, err) + } +} + +func TestDMHistorySharesOneCommandWideRequestBudget(t *testing.T) { + channels := make([]string, MaxDMHistoryRequests+1) + for index := range channels { + channels[index] = fmt.Sprintf("dm-%03d", index) + } + calls := 0 + result, err := DMHistory(context.Background(), pageSourceFunc(func(_ context.Context, channelID string, _ mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) { + calls++ + post := mattermost.Post{ID: fmt.Sprintf("post-%03d", calls), ChannelID: channelID, CreateAt: int64(calls)} + return mattermost.OrderedPostsPage{Posts: []mattermost.Post{post}, RawCount: 1, HasNext: dmBoolPointer(false)}, nil + }), channels, DMHistoryOptions{Limit: 1}) + if err != nil || calls != MaxDMHistoryRequests || result.Completeness != CompletenessUnknown || len(result.Posts) != 1 { + t.Fatalf("calls=%d result=%#v err=%v", calls, result, err) + } +} + +func TestDMHistoryUnknownDominatesEarlierTruncation(t *testing.T) { + channels := make([]string, MaxDMHistoryRequests+1) + for index := range channels { + channels[index] = fmt.Sprintf("dm-%03d", index) + } + calls := 0 + result, err := DMHistory(context.Background(), pageSourceFunc(func(_ context.Context, channelID string, _ mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) { + calls++ + if calls == 1 { + posts := []mattermost.Post{{ID: "new", ChannelID: channelID, CreateAt: 2}, {ID: "old", ChannelID: channelID, CreateAt: 1}} + return mattermost.OrderedPostsPage{Posts: posts, RawCount: len(posts), HasNext: dmBoolPointer(false)}, nil + } + return mattermost.OrderedPostsPage{HasNext: dmBoolPointer(true)}, nil + }), channels, DMHistoryOptions{Limit: 1}) + if err != nil || calls != MaxDMHistoryRequests || result.Completeness != CompletenessUnknown || fmt.Sprint(ids(result.Posts)) != "[new]" { + t.Fatalf("calls=%d result=%#v err=%v", calls, result, err) + } +} + +func TestDMHistoryBusyFirstChannelExhaustsBudgetAndLeavesGlobalSelectionUnknown(t *testing.T) { + calls := 0 + result, err := DMHistory(context.Background(), pageSourceFunc(func(_ context.Context, channelID string, options mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) { + calls++ + if channelID != "busy" { + t.Fatalf("budget-exhausted channel %q was queried", channelID) + } + post := mattermost.Post{ID: fmt.Sprintf("post-%03d", calls), ChannelID: channelID, CreateAt: int64(calls)} + return mattermost.OrderedPostsPage{Posts: []mattermost.Post{post}, RawCount: options.PerPage}, nil + }), []string{"busy", "later"}, DMHistoryOptions{Limit: 1000}) + if err != nil || calls != MaxDMHistoryRequests || result.Completeness != CompletenessUnknown || len(result.Posts) != MaxDMHistoryRequests { + t.Fatalf("calls=%d result=%#v err=%v", calls, result, err) + } +} + +func TestDMHistoryCancellationWinsBeforeBudgetShortCircuit(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + calls := 0 + _, err := DMHistory(ctx, pageSourceFunc(func(_ context.Context, channelID string, options mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) { + calls++ + if channelID != "busy" { + t.Fatalf("canceled channel %q was queried", channelID) + } + if calls == MaxDMHistoryRequests { + cancel() + } + post := mattermost.Post{ID: fmt.Sprintf("post-%03d", calls), ChannelID: channelID, CreateAt: int64(calls)} + return mattermost.OrderedPostsPage{Posts: []mattermost.Post{post}, RawCount: options.PerPage}, nil + }), []string{"busy", "later"}, DMHistoryOptions{Limit: 1000}) + if !errors.Is(err, context.Canceled) || calls != MaxDMHistoryRequests { + t.Fatalf("calls=%d err=%v", calls, err) + } +} + +func dmBoolPointer(value bool) *bool { return &value } diff --git a/internal/schema/read_test.go b/internal/schema/read_test.go index 3b821f5..8beed3e 100644 --- a/internal/schema/read_test.go +++ b/internal/schema/read_test.go @@ -235,3 +235,24 @@ func TestMentionsSchemaBindsSourceAndCompleteness(t *testing.T) { } } } + +func TestDMSSchemaBindsChannelSourceAndCompleteness(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + valid, err := fs.ReadFile(publicschemas.FS, "v2/examples/dms.json") + if err != nil { + t.Fatal(err) + } + for name, document := range map[string]string{ + "wrong channel type": strings.Replace(string(valid), `"type":"dm"`, `"type":"public"`, 1), + "wrong source": strings.Replace(string(valid), `"source":"recent"`, `"source":"search"`, 1), + "complete is truncated": strings.Replace(string(valid), `"queryTruncated":false`, `"queryTruncated":true`, 1), + "complete is unknown": strings.Replace(string(valid), `"queryTruncated":false`, `"queryTruncated":null`, 1), + } { + if err := registry.Validate("mm/v2/dms", strings.NewReader(document)); err == nil { + t.Errorf("accepted %s", name) + } + } +} diff --git a/schemas/v2/dms.schema.json b/schemas/v2/dms.schema.json index 763f1b8..9a77c6c 100644 --- a/schemas/v2/dms.schema.json +++ b/schemas/v2/dms.schema.json @@ -1 +1 @@ -{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"urn:mm:schema:v2:dms","type":"object","additionalProperties":false,"required":["schema","channels"],"properties":{"schema":{"const":"mm/v2/dms"},"channels":{"type":"array","items":{"$ref":"#/$defs/history"}}},"$defs":{"timestamp":{"type":"string","format":"date-time","pattern":"^(?:000[1-9]|00[1-9][0-9]|0[1-9][0-9]{2}|[1-9][0-9]{3})-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\\.[0-9]{3}Z$"},"nullableTimestamp":{"anyOf":[{"$ref":"#/$defs/timestamp"},{"type":"null"}]},"file":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"name":{"type":"string"},"mime":{"type":"string"},"size":{"type":"integer","minimum":0},"extension":{"type":"string"}}},"attachmentField":{"type":"object","additionalProperties":false,"properties":{"title":{"type":"string"},"value":{"type":"string"},"short":{"type":"boolean"}}},"attachment":{"type":"object","additionalProperties":false,"properties":{"fallback":{"type":"string"},"pretext":{"type":"string"},"title":{"type":"string"},"titleLink":{"type":"string"},"text":{"type":"string"},"fields":{"type":"array","items":{"$ref":"#/$defs/attachmentField"}},"footer":{"type":"string"},"footerIcon":{"type":"string"},"authorName":{"type":"string"},"authorLink":{"type":"string"},"authorIcon":{"type":"string"},"color":{"type":"string"},"imageUrl":{"type":"string"},"thumbUrl":{"type":"string"},"timestamp":{"type":"string"}}},"reactionActor":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"username":{"type":"string"}}},"reaction":{"type":"object","additionalProperties":false,"required":["emoji","count","actors"],"properties":{"emoji":{"type":"string"},"count":{"type":"integer","minimum":0},"actors":{"type":"array","items":{"$ref":"#/$defs/reactionActor"}}}},"redaction":{"type":"object","additionalProperties":false,"required":["type","masked","position"],"properties":{"type":{"type":"string"},"masked":{"type":"string"},"position":{"type":"integer","minimum":0},"field":{"type":"string"}}},"channel":{"type":"object","additionalProperties":false,"required":["id","type","name","displayName","metadataStatus"],"properties":{"id":{"type":"string"},"type":{"enum":["dm","public","private","group","unknown"]},"name":{"type":"string"},"displayName":{"type":"string"},"metadataStatus":{"enum":["resolved","unavailable"]}}},"selection":{"type":"object","additionalProperties":false,"required":["source","selectedCount","requestedLimit","since","queryTruncated","inputCursor","nextCursor"],"properties":{"source":{"enum":["recent","search","mentions","unread","thread"]},"selectedCount":{"type":"integer","minimum":0},"requestedLimit":{"type":["integer","null"],"minimum":1},"since":{"type":["string","null"]},"queryTruncated":{"type":["boolean","null"]},"inputCursor":{"type":["string","null"]},"nextCursor":{"type":["string","null"]}}},"visibleThreads":{"type":"object","additionalProperties":false,"required":["status","hydratedRootCount","failedRootIds"],"properties":{"status":{"enum":["not_requested","complete","partial"]},"hydratedRootCount":{"type":"integer","minimum":0},"failedRootIds":{"type":"array","items":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":false,"required":["completeness","selection","visibleThreads","visiblePostCount","deletedPostsIncluded"],"properties":{"completeness":{"enum":["complete","truncated","unknown"]},"selection":{"$ref":"#/$defs/selection"},"visibleThreads":{"$ref":"#/$defs/visibleThreads"},"visiblePostCount":{"type":"integer","minimum":0},"deletedPostsIncluded":{"const":false}}},"message":{"type":"object","additionalProperties":false,"required":["id","permalink","user","userId","text","timestamp","updatedAt","editedAt","deletedAt","isDeleted","postType","isSystem","isPinned","rootId","replyCount","files","fileDetails","attachments","reactions","replies"],"properties":{"id":{"type":"string"},"permalink":{"type":"string"},"user":{"type":"string"},"userId":{"type":"string"},"text":{"type":"string"},"timestamp":{"$ref":"#/$defs/timestamp"},"updatedAt":{"$ref":"#/$defs/timestamp"},"editedAt":{"$ref":"#/$defs/nullableTimestamp"},"deletedAt":{"$ref":"#/$defs/nullableTimestamp"},"isDeleted":{"type":"boolean"},"postType":{"type":"string"},"isSystem":{"type":"boolean"},"isPinned":{"type":"boolean"},"rootId":{"type":["string","null"]},"replyCount":{"type":["integer","null"],"minimum":0},"files":{"type":"array","items":{"type":"string"}},"fileDetails":{"type":"array","items":{"$ref":"#/$defs/file"}},"attachments":{"type":"array","items":{"$ref":"#/$defs/attachment"}},"reactions":{"type":"array","items":{"$ref":"#/$defs/reaction"}},"replies":{"type":"array","items":{"$ref":"#/$defs/message"}}}},"history":{"type":"object","additionalProperties":false,"required":["channel","messages","redactions","metadata"],"properties":{"channel":{"$ref":"#/$defs/channel"},"messages":{"type":"array","items":{"$ref":"#/$defs/message"}},"redactions":{"type":"array","items":{"$ref":"#/$defs/redaction"}},"metadata":{"$ref":"#/$defs/metadata"}}}}} +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"urn:mm:schema:v2:dms","type":"object","additionalProperties":false,"required":["schema","channels"],"properties":{"schema":{"const":"mm/v2/dms"},"channels":{"type":"array","items":{"$ref":"#/$defs/dmsHistory"}}},"$defs":{"timestamp":{"type":"string","format":"date-time","pattern":"^(?:000[1-9]|00[1-9][0-9]|0[1-9][0-9]{2}|[1-9][0-9]{3})-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\\.[0-9]{3}Z$"},"nullableTimestamp":{"anyOf":[{"$ref":"#/$defs/timestamp"},{"type":"null"}]},"file":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"name":{"type":"string"},"mime":{"type":"string"},"size":{"type":"integer","minimum":0},"extension":{"type":"string"}}},"attachmentField":{"type":"object","additionalProperties":false,"properties":{"title":{"type":"string"},"value":{"type":"string"},"short":{"type":"boolean"}}},"attachment":{"type":"object","additionalProperties":false,"properties":{"fallback":{"type":"string"},"pretext":{"type":"string"},"title":{"type":"string"},"titleLink":{"type":"string"},"text":{"type":"string"},"fields":{"type":"array","items":{"$ref":"#/$defs/attachmentField"}},"footer":{"type":"string"},"footerIcon":{"type":"string"},"authorName":{"type":"string"},"authorLink":{"type":"string"},"authorIcon":{"type":"string"},"color":{"type":"string"},"imageUrl":{"type":"string"},"thumbUrl":{"type":"string"},"timestamp":{"type":"string"}}},"reactionActor":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"username":{"type":"string"}}},"reaction":{"type":"object","additionalProperties":false,"required":["emoji","count","actors"],"properties":{"emoji":{"type":"string"},"count":{"type":"integer","minimum":0},"actors":{"type":"array","items":{"$ref":"#/$defs/reactionActor"}}}},"redaction":{"type":"object","additionalProperties":false,"required":["type","masked","position"],"properties":{"type":{"type":"string"},"masked":{"type":"string"},"position":{"type":"integer","minimum":0},"field":{"type":"string"}}},"channel":{"type":"object","additionalProperties":false,"required":["id","type","name","displayName","metadataStatus"],"properties":{"id":{"type":"string"},"type":{"enum":["dm","public","private","group","unknown"]},"name":{"type":"string"},"displayName":{"type":"string"},"metadataStatus":{"enum":["resolved","unavailable"]}}},"selection":{"type":"object","additionalProperties":false,"required":["source","selectedCount","requestedLimit","since","queryTruncated","inputCursor","nextCursor"],"properties":{"source":{"enum":["recent","search","mentions","unread","thread"]},"selectedCount":{"type":"integer","minimum":0},"requestedLimit":{"type":["integer","null"],"minimum":1},"since":{"type":["string","null"]},"queryTruncated":{"type":["boolean","null"]},"inputCursor":{"type":["string","null"]},"nextCursor":{"type":["string","null"]}}},"visibleThreads":{"type":"object","additionalProperties":false,"required":["status","hydratedRootCount","failedRootIds"],"properties":{"status":{"enum":["not_requested","complete","partial"]},"hydratedRootCount":{"type":"integer","minimum":0},"failedRootIds":{"type":"array","items":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":false,"required":["completeness","selection","visibleThreads","visiblePostCount","deletedPostsIncluded"],"properties":{"completeness":{"enum":["complete","truncated","unknown"]},"selection":{"$ref":"#/$defs/selection"},"visibleThreads":{"$ref":"#/$defs/visibleThreads"},"visiblePostCount":{"type":"integer","minimum":0},"deletedPostsIncluded":{"const":false}}},"message":{"type":"object","additionalProperties":false,"required":["id","permalink","user","userId","text","timestamp","updatedAt","editedAt","deletedAt","isDeleted","postType","isSystem","isPinned","rootId","replyCount","files","fileDetails","attachments","reactions","replies"],"properties":{"id":{"type":"string"},"permalink":{"type":"string"},"user":{"type":"string"},"userId":{"type":"string"},"text":{"type":"string"},"timestamp":{"$ref":"#/$defs/timestamp"},"updatedAt":{"$ref":"#/$defs/timestamp"},"editedAt":{"$ref":"#/$defs/nullableTimestamp"},"deletedAt":{"$ref":"#/$defs/nullableTimestamp"},"isDeleted":{"type":"boolean"},"postType":{"type":"string"},"isSystem":{"type":"boolean"},"isPinned":{"type":"boolean"},"rootId":{"type":["string","null"]},"replyCount":{"type":["integer","null"],"minimum":0},"files":{"type":"array","items":{"type":"string"}},"fileDetails":{"type":"array","items":{"$ref":"#/$defs/file"}},"attachments":{"type":"array","items":{"$ref":"#/$defs/attachment"}},"reactions":{"type":"array","items":{"$ref":"#/$defs/reaction"}},"replies":{"type":"array","items":{"$ref":"#/$defs/message"}}}},"dmsChannel":{"allOf":[{"$ref":"#/$defs/channel"},{"properties":{"type":{"enum":["dm","unknown"]}},"allOf":[{"if":{"properties":{"type":{"const":"dm"}}},"then":{"properties":{"metadataStatus":{"const":"resolved"}}}},{"if":{"properties":{"type":{"const":"unknown"}}},"then":{"properties":{"metadataStatus":{"const":"unavailable"}}}}]}]},"dmsMetadata":{"allOf":[{"$ref":"#/$defs/metadata"},{"properties":{"selection":{"properties":{"source":{"const":"recent"}}}}},{"if":{"properties":{"completeness":{"const":"complete"}}},"then":{"properties":{"selection":{"properties":{"queryTruncated":{"const":false}}}}}},{"if":{"properties":{"completeness":{"const":"truncated"}}},"then":{"properties":{"selection":{"properties":{"queryTruncated":{"const":true}}}}}},{"if":{"properties":{"completeness":{"const":"unknown"}}},"then":{"properties":{"selection":{"properties":{"queryTruncated":{"type":"null"}}}}}}]},"dmsHistory":{"type":"object","additionalProperties":false,"required":["channel","messages","redactions","metadata"],"properties":{"channel":{"$ref":"#/$defs/dmsChannel"},"messages":{"type":"array","items":{"$ref":"#/$defs/message"}},"redactions":{"type":"array","items":{"$ref":"#/$defs/redaction"}},"metadata":{"$ref":"#/$defs/dmsMetadata"}}}}} diff --git a/schemas/v2/examples/dms.json b/schemas/v2/examples/dms.json index 8787c9f..6a18b28 100644 --- a/schemas/v2/examples/dms.json +++ b/schemas/v2/examples/dms.json @@ -1 +1 @@ -{"schema":"mm/v2/dms","channels":[{"channel":{"id":"c1","type":"public","name":"town-square","displayName":"Town Square","metadataStatus":"resolved"},"messages":[],"redactions":[],"metadata":{"completeness":"complete","selection":{"source":"recent","selectedCount":0,"requestedLimit":50,"since":null,"queryTruncated":false,"inputCursor":null,"nextCursor":null},"visibleThreads":{"status":"not_requested","hydratedRootCount":0,"failedRootIds":[]},"visiblePostCount":0,"deletedPostsIncluded":false}}]} +{"schema":"mm/v2/dms","channels":[{"channel":{"id":"c1","type":"dm","name":"@alice","displayName":"","metadataStatus":"resolved"},"messages":[],"redactions":[],"metadata":{"completeness":"complete","selection":{"source":"recent","selectedCount":0,"requestedLimit":50,"since":null,"queryTruncated":false,"inputCursor":null,"nextCursor":null},"visibleThreads":{"status":"not_requested","hydratedRootCount":0,"failedRootIds":[]},"visiblePostCount":0,"deletedPostsIncluded":false}}]} From 7e8474d2b2dd5c4eec70fb75e86e47438db91100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 16:35:06 +0300 Subject: [PATCH 035/119] feat: port group direct-message read command --- internal/cli/group_dms.go | 214 ++++++++++++++++++++++++ internal/cli/group_dms_test.go | 132 +++++++++++++++ internal/cli/root.go | 1 + internal/mattermost/channels.go | 62 +++++++ internal/mattermost/channels_test.go | 31 ++++ internal/output/machine_convert.go | 3 + internal/output/machine_convert_test.go | 3 + internal/retrieval/group_dms.go | 13 ++ internal/retrieval/group_dms_test.go | 28 ++++ internal/schema/read_test.go | 67 ++++++++ schemas/v2/group-dms.schema.json | 2 +- 11 files changed, 555 insertions(+), 1 deletion(-) create mode 100644 internal/cli/group_dms.go create mode 100644 internal/cli/group_dms_test.go create mode 100644 internal/retrieval/group_dms.go create mode 100644 internal/retrieval/group_dms_test.go diff --git a/internal/cli/group_dms.go b/internal/cli/group_dms.go new file mode 100644 index 0000000..0d9d1b3 --- /dev/null +++ b/internal/cli/group_dms.go @@ -0,0 +1,214 @@ +package cli + +import ( + "fmt" + "strings" + "time" + + "github.com/spf13/cobra" + + "github.com/ardasevinc/mattermost-cli/internal/cursor" + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/output" + "github.com/ardasevinc/mattermost-cli/internal/retrieval" +) + +type groupDMsFlags struct{ limit, since, channel, cursor string } + +func newGroupDMsCommand(state *rootState) *cobra.Command { + flags := new(groupDMsFlags) + command := &cobra.Command{Use: "group-dms", Short: "Fetch group direct messages", Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { return runGroupDMs(cmd, state, *flags) }} + command.Flags().StringVarP(&flags.limit, "limit", "l", "50", "maximum total seed messages across matched group DMs") + command.Flags().StringVarP(&flags.since, "since", "s", "7d", "time range such as 24h, 7d, 1w, or 2m") + command.Flags().StringVarP(&flags.channel, "channel", "c", "", "specific group DM channel ID") + command.Flags().StringVar(&flags.cursor, "cursor", "", "resume deterministic group-DM history") + return command +} + +func runGroupDMs(cmd *cobra.Command, state *rootState, flags groupDMsFlags) error { + if flagChanged(cmd, "channel") && strings.TrimSpace(flags.channel) == "" { + return invalidFailure("--channel cannot be empty") + } + if flagChanged(cmd, "cursor") && strings.TrimSpace(flags.cursor) == "" { + return invalidFailure("--cursor cannot be empty") + } + limit, err := positiveInteger(flags.limit) + if err != nil { + return err + } + var resume *cursor.ChannelHistory + if flags.cursor != "" { + decoded, decodeErr := cursor.DecodeChannelHistory(flags.cursor) + if decodeErr != nil { + return invalidFailure("invalid group-DM cursor") + } + resume = &decoded + if flags.channel == "" { + return invalidFailure("a cursor requires --channel for group-DM history") + } + if decoded.ChannelID != flags.channel { + return invalidFailure("cursor does not match the selected channel") + } + if flagChanged(cmd, "since") { + return invalidFailure("a cursor cannot be combined with --since") + } + } + var since *int64 + if resume != nil { + since = resume.Since + } else { + value, durationErr := durationBoundary(flags.since, time.Now()) + if durationErr != nil { + return durationErr + } + since = &value + } + display, err := state.readDisplay(cmd) + if err != nil { + return err + } + runtime, err := state.runtimeFor(cmd) + if err != nil { + return err + } + if err := emitRedactionWarning(state, runtime, display.json); err != nil { + return err + } + me, err := runtime.Users.Current(cmd.Context()) + if err != nil { + return readFailure(err) + } + channels, err := selectGroupDMChannels(cmd, runtime, me, flags) + if err != nil { + return err + } + if resume != nil && (len(channels) != 1 || resume.ChannelID != channels[0].ID) { + return invalidFailure("cursor does not match the selected channel") + } + if len(channels) == 0 { + envelope, envelopeErr := output.NewGroupDMSEnvelope(nil, output.MachineComplete) + if envelopeErr != nil { + return internalFailure(envelopeErr) + } + return state.renderRead(nil, envelope, display) + } + ids := make([]string, len(channels)) + byID := make(map[string]mattermost.Channel, len(channels)) + for index, channel := range channels { + ids[index], byID[channel.ID] = channel.ID, channel + } + options := retrieval.GroupDMHistoryOptions{Limit: limit, Since: since} + if resume != nil { + options.Boundary = &retrieval.Boundary{CreateAt: resume.Boundary.CreateAt, ID: resume.Boundary.ID} + options.SafeBeforePostID = resume.SafeBeforePostID + } + result, err := retrieval.GroupDMHistory(cmd.Context(), runtime.Posts, ids, options) + if err != nil { + return readFailure(err) + } + if len(result.Posts) == 0 && result.Completeness == retrieval.CompletenessUnknown && resume == nil { + return readError("Mattermost could not confirm an empty group direct-message history") + } + nextCursor := "" + if len(result.Posts) > 0 && result.Completeness != retrieval.CompletenessComplete && flags.channel != "" { + boundary := result.Posts[len(result.Posts)-1] + safeBefore := "" + for index := len(result.Posts) - 1; index >= 0; index-- { + if result.Posts[index].CreateAt > boundary.CreateAt { + safeBefore = result.Posts[index].ID + break + } + } + if safeBefore == "" && result.SafeBeforeValid && resume != nil { + safeBefore = resume.SafeBeforePostID + } + nextCursor, err = cursor.EncodeChannelHistory(cursor.ChannelHistory{Version: 1, Scope: "channel", ChannelID: channels[0].ID, + Boundary: cursor.Boundary{CreateAt: boundary.CreateAt, ID: boundary.ID}, Since: since, SafeBeforePostID: safeBefore}) + if err != nil { + return readFailure(err) + } + } else if len(result.Posts) == 0 && result.Completeness == retrieval.CompletenessUnknown && resume != nil { + nextCursor = flags.cursor + } + + groups, order := groupSearchPosts(result.Posts) + type hydratedGroup struct { + channelID string + seeds []mattermost.Post + hydrated retrieval.HydrationResult + } + hydrated := make([]hydratedGroup, 0, len(order)) + allPosts := make([]mattermost.Post, 0, len(result.Posts)) + for _, channelID := range order { + value, hydrateErr := retrieval.HydrateVisibleThreads(cmd.Context(), runtime.Posts, groups[channelID], display.threads) + if hydrateErr != nil { + return readFailure(hydrateErr) + } + hydrated = append(hydrated, hydratedGroup{channelID, groups[channelID], value}) + allPosts = append(allPosts, value.Posts...) + } + users, err := loadReadUsersAndIDs(cmd, runtime, allPosts, nil) + if err != nil { + return readFailure(err) + } + sections := make([]output.MessageOutput, 0, len(hydrated)) + for _, group := range hydrated { + messages, redactions, normalizeErr := normalizeReadPostsWithUsers(runtime, group.hydrated.Posts, users, me.ID) + if normalizeErr != nil { + return readFailure(normalizeErr) + } + if display.threads { + messages = output.GroupIntoThreads(messages) + } + channel, channelRedactions := processedChannel(byID[group.channelID], runtime) + redactions = append(channelRedactions, redactions...) + threads, threadRedactions := processedVisibleThreads(group.hydrated.VisibleThreads, runtime) + redactions = append(redactions, threadRedactions...) + requestedLimit := limit + sections = append(sections, output.MessageOutput{Channel: channel, Messages: messages, Redactions: redactions, Retrieval: output.Retrieval{ + Selection: output.Selection{Source: "recent", SelectedCount: len(group.seeds), RequestedLimit: &requestedLimit, Since: millisecondTimestamp(since), + QueryTruncated: truncatedPointer(result.Completeness), InputCursor: stringPointer(flags.cursor), NextCursor: stringPointer(nextCursor)}, + VisibleThreads: threads, VisiblePostCount: len(group.hydrated.Posts), DeletedPostsIncluded: false, + }}) + for _, rootID := range threads.FailedRootIDs { + if err := searchWarning(state, display.json, fmt.Sprintf("warning: thread %s could only be partially hydrated\n", rootID)); err != nil { + return err + } + } + } + if len(result.Posts) == 0 && resume != nil && result.Completeness == retrieval.CompletenessUnknown { + channel, redactions := processedChannel(channels[0], runtime) + requestedLimit := limit + sections = append(sections, output.MessageOutput{Channel: channel, Redactions: redactions, Retrieval: output.Retrieval{ + Selection: output.Selection{Source: "recent", RequestedLimit: &requestedLimit, Since: millisecondTimestamp(since), InputCursor: stringPointer(flags.cursor), NextCursor: stringPointer(nextCursor)}, + VisibleThreads: output.VisibleThreads{Status: map[bool]string{true: "complete", false: "not_requested"}[display.threads], FailedRootIDs: []string{}}, + }}) + } + envelope, err := output.NewGroupDMSEnvelope(sections, machineCompleteness(result.Completeness)) + if err != nil { + return internalFailure(err) + } + return state.renderRead(sections, envelope, display) +} + +func selectGroupDMChannels(cmd *cobra.Command, runtime *Runtime, me mattermost.User, flags groupDMsFlags) ([]mattermost.Channel, error) { + if flags.channel != "" { + channel, err := runtime.Channels.ByID(cmd.Context(), flags.channel) + if err != nil { + return nil, readFailure(err) + } + if channel.Type != "G" { + return nil, invalidFailure("selected channel is not a group direct-message channel") + } + if _, err := runtime.Channels.Member(cmd.Context(), channel.ID, me.ID); err != nil { + return nil, readFailure(err) + } + return []mattermost.Channel{channel}, nil + } + channels, err := runtime.Channels.GroupList(cmd.Context(), me.ID) + if err != nil { + return nil, readFailure(err) + } + return channels, nil +} diff --git a/internal/cli/group_dms_test.go b/internal/cli/group_dms_test.go new file mode 100644 index 0000000..36c573f --- /dev/null +++ b/internal/cli/group_dms_test.go @@ -0,0 +1,132 @@ +package cli + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/ardasevinc/mattermost-cli/internal/cursor" + mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" +) + +func TestGroupDMsJSONDiscoversFocusedChannelsAppliesGlobalLimitAndSanitizesLabel(t *testing.T) { + now := time.Now().UnixMilli() + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/users/me": + writeJSON(t, writer, `{"id":"self","username":"arda"}`) + case "/api/v4/users/self/channels": + writeJSON(t, writer, `[{"id":"ignored","team_id":"team","type":"P","name":"private","display_name":""},{"id":"g1","team_id":"","type":"G","name":"opaque1","display_name":"Crew\u001b[31m AKIA1234567890ABCDEF"},{"id":"g2","team_id":"","type":"G","name":"opaque2","display_name":"Second"}]`) + case "/api/v4/channels/g1/posts": + post := fmt.Sprintf(`{"id":"newer","channel_id":"g1","user_id":"alice","message":"new","create_at":%d,"delete_at":0,"root_id":"","reply_count":0}`, now) + writeJSON(t, writer, `{"order":["newer"],"posts":{"newer":`+post+`},"has_next":false}`) + case "/api/v4/channels/g2/posts": + post := fmt.Sprintf(`{"id":"older","channel_id":"g2","user_id":"alice","message":"old","create_at":%d,"delete_at":0,"root_id":"","reply_count":0}`, now-1) + writeJSON(t, writer, `{"order":["older"],"posts":{"older":`+post+`},"has_next":false}`) + case "/api/v4/users/ids": + writeJSON(t, writer, `[{"id":"alice","username":"alice"}]`) + default: + t.Fatalf("unexpected request: %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "--json", "--no-threads", "group-dms", "--limit", "1", "--since", "1d") + if code != 0 || stderr != "" { + t.Fatalf("exit=%d stderr=%q stdout=%q", code, stderr, stdout) + } + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/group-dms", strings.NewReader(stdout)); err != nil { + t.Fatalf("machine output did not validate: %v\n%s", err, stdout) + } + var document struct { + Schema string `json:"schema"` + Channels []struct { + Channel struct{ Name string } `json:"channel"` + Messages []struct{ ID string } `json:"messages"` + } `json:"channels"` + } + if err := json.Unmarshal([]byte(stdout), &document); err != nil { + t.Fatal(err) + } + name := document.Channels[0].Channel.Name + if document.Schema != "mm/v2/group-dms" || len(document.Channels) != 1 || !strings.Contains(name, "AK...EF") || strings.ContainsRune(name, '\x1b') || document.Channels[0].Messages[0].ID != "newer" { + t.Fatalf("document=%+v", document) + } +} + +func TestGroupDMsExplicitChannelProvesMembershipBeforePosts(t *testing.T) { + var posts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/users/me": + writeJSON(t, writer, `{"id":"self","username":"arda"}`) + case "/api/v4/channels/group": + writeJSON(t, writer, `{"id":"group","team_id":"","type":"G","name":"opaque","display_name":"Crew"}`) + case "/api/v4/channels/group/members/self": + writer.WriteHeader(http.StatusNotFound) + writeJSON(t, writer, `{"message":"not a member"}`) + case "/api/v4/channels/group/posts": + posts.Add(1) + default: + t.Fatalf("unexpected request: %s", request.URL.String()) + } + })) + defer server.Close() + _, _, code := executeChannel(t, server.URL, "group-dms", "--channel", "group") + if code != 3 || posts.Load() != 0 { + t.Fatalf("exit=%d posts=%d", code, posts.Load()) + } +} + +func TestGroupDMsRejectsCursorFailuresBeforeNetwork(t *testing.T) { + since := time.Now().Add(-time.Hour).UnixMilli() + encoded, err := cursor.EncodeChannelHistory(cursor.ChannelHistory{Version: 1, Scope: "channel", ChannelID: "other", Boundary: cursor.Boundary{CreateAt: time.Now().UnixMilli(), ID: "post"}, Since: &since}) + if err != nil { + t.Fatal(err) + } + for _, args := range [][]string{ + {"group-dms", "--channel", "group", "--cursor", "not-a-cursor"}, + {"group-dms", "--cursor", "not-a-cursor"}, + {"group-dms", "--channel", "group", "--cursor", encoded}, + {"group-dms", "--channel="}, + {"group-dms", "--cursor="}, + } { + t.Run(strings.Join(args, "_"), func(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { requests.Add(1) })) + defer server.Close() + _, _, code := executeChannel(t, server.URL, args...) + if code != 2 || requests.Load() != 0 { + t.Fatalf("args=%v exit=%d requests=%d", args, code, requests.Load()) + } + }) + } +} + +func TestGroupDMsUnknownEmptyFailsClosed(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/users/me": + writeJSON(t, writer, `{"id":"self","username":"arda"}`) + case "/api/v4/users/self/channels": + writeJSON(t, writer, `[{"id":"group","team_id":"","type":"G","name":"opaque","display_name":"Crew"}]`) + case "/api/v4/channels/group/posts": + writeJSON(t, writer, `{"order":[],"posts":{},"has_next":true}`) + default: + t.Fatalf("unexpected request: %s", request.URL.String()) + } + })) + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "--json", "--no-threads", "group-dms") + if code != 3 || stdout != "" || !strings.Contains(stderr, `"code":"read_failed"`) { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } +} diff --git a/internal/cli/root.go b/internal/cli/root.go index b79a3cb..6dc2069 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -118,6 +118,7 @@ func newRootWithState(state *rootState) *cobra.Command { cmd.AddCommand(newSchemaCommand(state)) cmd.AddCommand(newChannelCommand(state)) cmd.AddCommand(newDMsCommand(state)) + cmd.AddCommand(newGroupDMsCommand(state)) cmd.AddCommand(newThreadCommand(state)) cmd.AddCommand(newSearchCommand(state)) cmd.AddCommand(newMentionsCommand(state)) diff --git a/internal/mattermost/channels.go b/internal/mattermost/channels.go index cfc875e..c0640af 100644 --- a/internal/mattermost/channels.go +++ b/internal/mattermost/channels.go @@ -143,6 +143,38 @@ func (l *directChannelList) UnmarshalJSON(data []byte) error { return nil } +type groupChannelList []Channel + +func (l *groupChannelList) UnmarshalJSON(data []byte) error { + var raw []json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil || raw == nil { + return ErrInvalidChannelsResponse + } + groups := make([]Channel, 0) + for _, item := range raw { + var discriminator struct { + Type json.RawMessage `json:"type"` + } + if json.Unmarshal(item, &discriminator) != nil { + return ErrInvalidChannelsResponse + } + typeCode, ok := requiredString(discriminator.Type) + if !ok || (typeCode != "O" && typeCode != "P" && typeCode != "D" && typeCode != "G") { + return ErrInvalidChannelsResponse + } + if typeCode != "G" { + continue + } + var channel Channel + if json.Unmarshal(item, &channel) != nil { + return ErrInvalidChannelsResponse + } + groups = append(groups, channel) + } + *l = groups + return nil +} + func (s *Channels) ByID(ctx context.Context, channelID string) (Channel, error) { if strings.TrimSpace(channelID) == "" { return Channel{}, ErrInvalidChannelRequest @@ -270,6 +302,36 @@ func (s *Channels) DirectList(ctx context.Context, userID string) ([]Channel, er return result, nil } +// GroupList returns only G channels from the canonical current-user channel +// listing. That authenticated, same-session endpoint is itself the membership +// proof for discovered channels; per-channel Member calls would turn one +// bounded discovery read into unbounded fan-out. Explicit channel selection +// still requires Member. Unrelated payloads are ignored after their channel +// discriminator is validated. +func (s *Channels) GroupList(ctx context.Context, userID string) ([]Channel, error) { + if strings.TrimSpace(userID) == "" || userID == "me" { + return nil, ErrInvalidChannelRequest + } + var decoded groupChannelList + if err := s.client.Get(ctx, "/users/"+url.PathEscape(userID)+"/channels", &decoded); err != nil { + return nil, err + } + seen := make(map[string]Channel) + result := make([]Channel, 0) + for _, channel := range []Channel(decoded) { + if previous, duplicate := seen[channel.ID]; duplicate { + if previous != channel { + return nil, ErrInvalidChannelsResponse + } + continue + } + seen[channel.ID] = channel + result = append(result, channel) + } + sort.Slice(result, func(i, j int) bool { return result[i].ID < result[j].ID }) + return result, nil +} + func directChannelContains(name, userID string) bool { parts := strings.Split(name, "__") return len(parts) == 2 && (parts[0] == userID || parts[1] == userID) diff --git a/internal/mattermost/channels_test.go b/internal/mattermost/channels_test.go index f849ba4..951278a 100644 --- a/internal/mattermost/channels_test.go +++ b/internal/mattermost/channels_test.go @@ -182,6 +182,37 @@ func TestDirectListDedupesExactDirectChannelDuplicates(t *testing.T) { } } +func TestGroupListUsesCanonicalListingAsBoundedMembershipProof(t *testing.T) { + group := `{"id":"group","team_id":"","type":"G","name":"opaque","display_name":"Crew"}` + f := &fakeChannelTransport{responses: map[string]string{ + "/users/user/channels": `[{"type":"P"},{"type":"D"},` + group + `,` + group + `]`, + }} + got, err := NewChannels(f).GroupList(context.Background(), "user") + if err != nil || len(got) != 1 || got[0].ID != "group" { + t.Fatalf("channels=%#v err=%v", got, err) + } + if !reflect.DeepEqual(f.paths, []string{"/users/user/channels"}) { + t.Fatalf("paths=%v", f.paths) + } +} + +func TestGroupListRejectsMalformedFocusedChannelsAndMembership(t *testing.T) { + for name, payload := range map[string]string{ + "malformed group": `[{"id":"group","team_id":"","type":"G","display_name":"Crew"}]`, + "conflicting duplicate": `[{"id":"group","team_id":"","type":"G","name":"one","display_name":""},{"id":"group","team_id":"","type":"G","name":"two","display_name":""}]`, + "foreign team binding": `[{"id":"group","team_id":"foreign","type":"G","name":"one","display_name":""}]`, + "missing discriminator": `[{"id":"ignored"}]`, + "unknown discriminator": `[{"type":"X"}]`, + } { + t.Run(name, func(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{"/users/user/channels": payload}} + if _, err := NewChannels(f).GroupList(context.Background(), "user"); err == nil { + t.Fatal("expected group-list validation error") + } + }) + } +} + func TestChannelReadsAreRaceSafe(t *testing.T) { f := &fakeChannelTransport{responses: map[string]string{ "/channels/x": `{"id":"x","team_id":"team","type":"O","name":"general","display_name":"General"}`, diff --git a/internal/output/machine_convert.go b/internal/output/machine_convert.go index ab3bcf1..8a84aa6 100644 --- a/internal/output/machine_convert.go +++ b/internal/output/machine_convert.go @@ -118,6 +118,9 @@ func NewDMSEnvelope(values []MessageOutput, completeness MachineCompleteness) (D } func NewGroupDMSEnvelope(values []MessageOutput, completeness MachineCompleteness) (GroupDMSEnvelope, error) { + if len(values) == 0 && completeness != MachineComplete { + return GroupDMSEnvelope{}, fmt.Errorf("empty group direct-message output requires confirmed completeness") + } histories, err := machineHistories(values, completeness, "recent", "group", "unknown") return GroupDMSEnvelope{Schema: "mm/v2/group-dms", Channels: histories}, err } diff --git a/internal/output/machine_convert_test.go b/internal/output/machine_convert_test.go index 8f83f7c..c4b7f73 100644 --- a/internal/output/machine_convert_test.go +++ b/internal/output/machine_convert_test.go @@ -300,6 +300,9 @@ func TestHistoryEnvelopeRejectsCompletenessThatContradictsQueryTruncation(t *tes if _, err := output.NewDMSEnvelope(nil, output.MachineUnknown); err == nil { t.Fatal("unknown empty direct-message envelope was accepted without metadata") } + if _, err := output.NewGroupDMSEnvelope(nil, output.MachineUnknown); err == nil { + t.Fatal("unknown empty group direct-message envelope was accepted without metadata") + } value := validOutput() value.Retrieval.Selection.Source = "search" truncated := true diff --git a/internal/retrieval/group_dms.go b/internal/retrieval/group_dms.go new file mode 100644 index 0000000..1f1b593 --- /dev/null +++ b/internal/retrieval/group_dms.go @@ -0,0 +1,13 @@ +package retrieval + +import "context" + +// GroupDMHistoryOptions intentionally shares the conversation-history +// contract with DMs: one command-wide request budget and one global seed cap. +type GroupDMHistoryOptions = DMHistoryOptions + +type GroupDMHistoryResult = DMHistoryResult + +func GroupDMHistory(ctx context.Context, source channelPageSource, channelIDs []string, options GroupDMHistoryOptions) (GroupDMHistoryResult, error) { + return DMHistory(ctx, source, channelIDs, options) +} diff --git a/internal/retrieval/group_dms_test.go b/internal/retrieval/group_dms_test.go new file mode 100644 index 0000000..854c1d5 --- /dev/null +++ b/internal/retrieval/group_dms_test.go @@ -0,0 +1,28 @@ +package retrieval + +import ( + "context" + "fmt" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" +) + +func TestGroupDMHistoryUsesGlobalDeterministicCapAndUnknownDominance(t *testing.T) { + channels := make([]string, MaxDMHistoryRequests+1) + for index := range channels { + channels[index] = fmt.Sprintf("group-%03d", index) + } + calls := 0 + result, err := GroupDMHistory(context.Background(), pageSourceFunc(func(_ context.Context, channelID string, _ mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) { + calls++ + if calls == 1 { + posts := []mattermost.Post{{ID: "new", ChannelID: channelID, CreateAt: 2}, {ID: "old", ChannelID: channelID, CreateAt: 1}} + return mattermost.OrderedPostsPage{Posts: posts, RawCount: len(posts), HasNext: dmBoolPointer(false)}, nil + } + return mattermost.OrderedPostsPage{HasNext: dmBoolPointer(true)}, nil + }), channels, GroupDMHistoryOptions{Limit: 1}) + if err != nil || calls != MaxDMHistoryRequests || result.Completeness != CompletenessUnknown || fmt.Sprint(ids(result.Posts)) != "[new]" { + t.Fatalf("calls=%d result=%#v err=%v", calls, result, err) + } +} diff --git a/internal/schema/read_test.go b/internal/schema/read_test.go index 8beed3e..eaf8301 100644 --- a/internal/schema/read_test.go +++ b/internal/schema/read_test.go @@ -87,6 +87,73 @@ func TestReadSchemaRejectsInvalidThreadTimestamps(t *testing.T) { } } +func TestGroupDMsSchemaEnforcesCommandSpecificBindings(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + valid, err := fs.ReadFile(publicschemas.FS, "v2/examples/group-dms.json") + if err != nil { + t.Fatal(err) + } + decode := func(t *testing.T) map[string]any { + t.Helper() + var document map[string]any + if err := json.Unmarshal(valid, &document); err != nil { + t.Fatal(err) + } + return document + } + validateInvalid := func(t *testing.T, document map[string]any) { + t.Helper() + raw, err := json.Marshal(document) + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/group-dms", strings.NewReader(string(raw))); err == nil { + t.Fatalf("invalid group-DM document accepted: %s", raw) + } + } + history := func(document map[string]any) map[string]any { + return document["channels"].([]any)[0].(map[string]any) + } + + for name, mutate := range map[string]func(map[string]any){ + "non-group channel": func(document map[string]any) { + history(document)["channel"].(map[string]any)["type"] = "dm" + }, + "resolved unknown channel": func(document map[string]any) { + channel := history(document)["channel"].(map[string]any) + channel["type"], channel["metadataStatus"] = "unknown", "resolved" + }, + "unavailable group channel": func(document map[string]any) { + history(document)["channel"].(map[string]any)["metadataStatus"] = "unavailable" + }, + "non-recent source": func(document map[string]any) { + history(document)["metadata"].(map[string]any)["selection"].(map[string]any)["source"] = "search" + }, + "complete with true truncation": func(document map[string]any) { + history(document)["metadata"].(map[string]any)["selection"].(map[string]any)["queryTruncated"] = true + }, + "truncated with false truncation": func(document map[string]any) { + metadata := history(document)["metadata"].(map[string]any) + metadata["completeness"] = "truncated" + metadata["selection"].(map[string]any)["queryTruncated"] = false + }, + "unknown with boolean truncation": func(document map[string]any) { + metadata := history(document)["metadata"].(map[string]any) + metadata["completeness"] = "unknown" + metadata["selection"].(map[string]any)["queryTruncated"] = false + }, + } { + t.Run(name, func(t *testing.T) { + document := decode(t) + mutate(document) + validateInvalid(t, document) + }) + } +} + func TestThreadSchemaEnforcesRootAndUnboundShape(t *testing.T) { registry, err := Load() if err != nil { diff --git a/schemas/v2/group-dms.schema.json b/schemas/v2/group-dms.schema.json index 372a8b0..deb44b9 100644 --- a/schemas/v2/group-dms.schema.json +++ b/schemas/v2/group-dms.schema.json @@ -1 +1 @@ -{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"urn:mm:schema:v2:group-dms","type":"object","additionalProperties":false,"required":["schema","channels"],"properties":{"schema":{"const":"mm/v2/group-dms"},"channels":{"type":"array","items":{"$ref":"#/$defs/history"}}},"$defs":{"timestamp":{"type":"string","format":"date-time","pattern":"^(?:000[1-9]|00[1-9][0-9]|0[1-9][0-9]{2}|[1-9][0-9]{3})-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\\.[0-9]{3}Z$"},"nullableTimestamp":{"anyOf":[{"$ref":"#/$defs/timestamp"},{"type":"null"}]},"file":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"name":{"type":"string"},"mime":{"type":"string"},"size":{"type":"integer","minimum":0},"extension":{"type":"string"}}},"attachmentField":{"type":"object","additionalProperties":false,"properties":{"title":{"type":"string"},"value":{"type":"string"},"short":{"type":"boolean"}}},"attachment":{"type":"object","additionalProperties":false,"properties":{"fallback":{"type":"string"},"pretext":{"type":"string"},"title":{"type":"string"},"titleLink":{"type":"string"},"text":{"type":"string"},"fields":{"type":"array","items":{"$ref":"#/$defs/attachmentField"}},"footer":{"type":"string"},"footerIcon":{"type":"string"},"authorName":{"type":"string"},"authorLink":{"type":"string"},"authorIcon":{"type":"string"},"color":{"type":"string"},"imageUrl":{"type":"string"},"thumbUrl":{"type":"string"},"timestamp":{"type":"string"}}},"reactionActor":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"username":{"type":"string"}}},"reaction":{"type":"object","additionalProperties":false,"required":["emoji","count","actors"],"properties":{"emoji":{"type":"string"},"count":{"type":"integer","minimum":0},"actors":{"type":"array","items":{"$ref":"#/$defs/reactionActor"}}}},"redaction":{"type":"object","additionalProperties":false,"required":["type","masked","position"],"properties":{"type":{"type":"string"},"masked":{"type":"string"},"position":{"type":"integer","minimum":0},"field":{"type":"string"}}},"channel":{"type":"object","additionalProperties":false,"required":["id","type","name","displayName","metadataStatus"],"properties":{"id":{"type":"string"},"type":{"enum":["dm","public","private","group","unknown"]},"name":{"type":"string"},"displayName":{"type":"string"},"metadataStatus":{"enum":["resolved","unavailable"]}}},"selection":{"type":"object","additionalProperties":false,"required":["source","selectedCount","requestedLimit","since","queryTruncated","inputCursor","nextCursor"],"properties":{"source":{"enum":["recent","search","mentions","unread","thread"]},"selectedCount":{"type":"integer","minimum":0},"requestedLimit":{"type":["integer","null"],"minimum":1},"since":{"type":["string","null"]},"queryTruncated":{"type":["boolean","null"]},"inputCursor":{"type":["string","null"]},"nextCursor":{"type":["string","null"]}}},"visibleThreads":{"type":"object","additionalProperties":false,"required":["status","hydratedRootCount","failedRootIds"],"properties":{"status":{"enum":["not_requested","complete","partial"]},"hydratedRootCount":{"type":"integer","minimum":0},"failedRootIds":{"type":"array","items":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":false,"required":["completeness","selection","visibleThreads","visiblePostCount","deletedPostsIncluded"],"properties":{"completeness":{"enum":["complete","truncated","unknown"]},"selection":{"$ref":"#/$defs/selection"},"visibleThreads":{"$ref":"#/$defs/visibleThreads"},"visiblePostCount":{"type":"integer","minimum":0},"deletedPostsIncluded":{"const":false}}},"message":{"type":"object","additionalProperties":false,"required":["id","permalink","user","userId","text","timestamp","updatedAt","editedAt","deletedAt","isDeleted","postType","isSystem","isPinned","rootId","replyCount","files","fileDetails","attachments","reactions","replies"],"properties":{"id":{"type":"string"},"permalink":{"type":"string"},"user":{"type":"string"},"userId":{"type":"string"},"text":{"type":"string"},"timestamp":{"$ref":"#/$defs/timestamp"},"updatedAt":{"$ref":"#/$defs/timestamp"},"editedAt":{"$ref":"#/$defs/nullableTimestamp"},"deletedAt":{"$ref":"#/$defs/nullableTimestamp"},"isDeleted":{"type":"boolean"},"postType":{"type":"string"},"isSystem":{"type":"boolean"},"isPinned":{"type":"boolean"},"rootId":{"type":["string","null"]},"replyCount":{"type":["integer","null"],"minimum":0},"files":{"type":"array","items":{"type":"string"}},"fileDetails":{"type":"array","items":{"$ref":"#/$defs/file"}},"attachments":{"type":"array","items":{"$ref":"#/$defs/attachment"}},"reactions":{"type":"array","items":{"$ref":"#/$defs/reaction"}},"replies":{"type":"array","items":{"$ref":"#/$defs/message"}}}},"history":{"type":"object","additionalProperties":false,"required":["channel","messages","redactions","metadata"],"properties":{"channel":{"$ref":"#/$defs/channel"},"messages":{"type":"array","items":{"$ref":"#/$defs/message"}},"redactions":{"type":"array","items":{"$ref":"#/$defs/redaction"}},"metadata":{"$ref":"#/$defs/metadata"}}}}} +{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"urn:mm:schema:v2:group-dms","type":"object","additionalProperties":false,"required":["schema","channels"],"properties":{"schema":{"const":"mm/v2/group-dms"},"channels":{"type":"array","items":{"$ref":"#/$defs/groupDmsHistory"}}},"$defs":{"timestamp":{"type":"string","format":"date-time","pattern":"^(?:000[1-9]|00[1-9][0-9]|0[1-9][0-9]{2}|[1-9][0-9]{3})-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\\.[0-9]{3}Z$"},"nullableTimestamp":{"anyOf":[{"$ref":"#/$defs/timestamp"},{"type":"null"}]},"file":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"name":{"type":"string"},"mime":{"type":"string"},"size":{"type":"integer","minimum":0},"extension":{"type":"string"}}},"attachmentField":{"type":"object","additionalProperties":false,"properties":{"title":{"type":"string"},"value":{"type":"string"},"short":{"type":"boolean"}}},"attachment":{"type":"object","additionalProperties":false,"properties":{"fallback":{"type":"string"},"pretext":{"type":"string"},"title":{"type":"string"},"titleLink":{"type":"string"},"text":{"type":"string"},"fields":{"type":"array","items":{"$ref":"#/$defs/attachmentField"}},"footer":{"type":"string"},"footerIcon":{"type":"string"},"authorName":{"type":"string"},"authorLink":{"type":"string"},"authorIcon":{"type":"string"},"color":{"type":"string"},"imageUrl":{"type":"string"},"thumbUrl":{"type":"string"},"timestamp":{"type":"string"}}},"reactionActor":{"type":"object","additionalProperties":false,"required":["id"],"properties":{"id":{"type":"string"},"username":{"type":"string"}}},"reaction":{"type":"object","additionalProperties":false,"required":["emoji","count","actors"],"properties":{"emoji":{"type":"string"},"count":{"type":"integer","minimum":0},"actors":{"type":"array","items":{"$ref":"#/$defs/reactionActor"}}}},"redaction":{"type":"object","additionalProperties":false,"required":["type","masked","position"],"properties":{"type":{"type":"string"},"masked":{"type":"string"},"position":{"type":"integer","minimum":0},"field":{"type":"string"}}},"channel":{"type":"object","additionalProperties":false,"required":["id","type","name","displayName","metadataStatus"],"properties":{"id":{"type":"string"},"type":{"enum":["dm","public","private","group","unknown"]},"name":{"type":"string"},"displayName":{"type":"string"},"metadataStatus":{"enum":["resolved","unavailable"]}}},"selection":{"type":"object","additionalProperties":false,"required":["source","selectedCount","requestedLimit","since","queryTruncated","inputCursor","nextCursor"],"properties":{"source":{"enum":["recent","search","mentions","unread","thread"]},"selectedCount":{"type":"integer","minimum":0},"requestedLimit":{"type":["integer","null"],"minimum":1},"since":{"type":["string","null"]},"queryTruncated":{"type":["boolean","null"]},"inputCursor":{"type":["string","null"]},"nextCursor":{"type":["string","null"]}}},"visibleThreads":{"type":"object","additionalProperties":false,"required":["status","hydratedRootCount","failedRootIds"],"properties":{"status":{"enum":["not_requested","complete","partial"]},"hydratedRootCount":{"type":"integer","minimum":0},"failedRootIds":{"type":"array","items":{"type":"string"}}}},"metadata":{"type":"object","additionalProperties":false,"required":["completeness","selection","visibleThreads","visiblePostCount","deletedPostsIncluded"],"properties":{"completeness":{"enum":["complete","truncated","unknown"]},"selection":{"$ref":"#/$defs/selection"},"visibleThreads":{"$ref":"#/$defs/visibleThreads"},"visiblePostCount":{"type":"integer","minimum":0},"deletedPostsIncluded":{"const":false}}},"message":{"type":"object","additionalProperties":false,"required":["id","permalink","user","userId","text","timestamp","updatedAt","editedAt","deletedAt","isDeleted","postType","isSystem","isPinned","rootId","replyCount","files","fileDetails","attachments","reactions","replies"],"properties":{"id":{"type":"string"},"permalink":{"type":"string"},"user":{"type":"string"},"userId":{"type":"string"},"text":{"type":"string"},"timestamp":{"$ref":"#/$defs/timestamp"},"updatedAt":{"$ref":"#/$defs/timestamp"},"editedAt":{"$ref":"#/$defs/nullableTimestamp"},"deletedAt":{"$ref":"#/$defs/nullableTimestamp"},"isDeleted":{"type":"boolean"},"postType":{"type":"string"},"isSystem":{"type":"boolean"},"isPinned":{"type":"boolean"},"rootId":{"type":["string","null"]},"replyCount":{"type":["integer","null"],"minimum":0},"files":{"type":"array","items":{"type":"string"}},"fileDetails":{"type":"array","items":{"$ref":"#/$defs/file"}},"attachments":{"type":"array","items":{"$ref":"#/$defs/attachment"}},"reactions":{"type":"array","items":{"$ref":"#/$defs/reaction"}},"replies":{"type":"array","items":{"$ref":"#/$defs/message"}}}},"groupDmsChannel":{"allOf":[{"$ref":"#/$defs/channel"},{"properties":{"type":{"enum":["group","unknown"]}},"allOf":[{"if":{"properties":{"type":{"const":"group"}}},"then":{"properties":{"metadataStatus":{"const":"resolved"}}}},{"if":{"properties":{"type":{"const":"unknown"}}},"then":{"properties":{"metadataStatus":{"const":"unavailable"}}}}]}]},"groupDmsMetadata":{"allOf":[{"$ref":"#/$defs/metadata"},{"properties":{"selection":{"properties":{"source":{"const":"recent"}}}}},{"if":{"properties":{"completeness":{"const":"complete"}}},"then":{"properties":{"selection":{"properties":{"queryTruncated":{"const":false}}}}}},{"if":{"properties":{"completeness":{"const":"truncated"}}},"then":{"properties":{"selection":{"properties":{"queryTruncated":{"const":true}}}}}},{"if":{"properties":{"completeness":{"const":"unknown"}}},"then":{"properties":{"selection":{"properties":{"queryTruncated":{"type":"null"}}}}}}]},"groupDmsHistory":{"type":"object","additionalProperties":false,"required":["channel","messages","redactions","metadata"],"properties":{"channel":{"$ref":"#/$defs/groupDmsChannel"},"messages":{"type":"array","items":{"$ref":"#/$defs/message"}},"redactions":{"type":"array","items":{"$ref":"#/$defs/redaction"}},"metadata":{"$ref":"#/$defs/groupDmsMetadata"}}}}} From 2d932d8374b67f56f2ccb86ab5579986dd2c8455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 17:37:42 +0300 Subject: [PATCH 036/119] feat: add secure config command --- internal/cli/config.go | 200 ++++++++++++++++++ internal/cli/config_test.go | 304 ++++++++++++++++++++++++++++ internal/cli/root.go | 5 +- internal/cli/root_test.go | 2 +- internal/cli/runtime.go | 109 +++++++--- internal/config/config.go | 3 + internal/config/secure_unix.go | 73 ++++++- internal/config/secure_unix_test.go | 172 ++++++++++++++++ internal/output/machine.go | 22 +- schemas/v2/config.schema.json | 102 ++++++++++ schemas/v2/examples/config.json | 1 + 11 files changed, 962 insertions(+), 31 deletions(-) create mode 100644 internal/cli/config.go create mode 100644 internal/cli/config_test.go create mode 100644 schemas/v2/config.schema.json create mode 100644 schemas/v2/examples/config.json diff --git a/internal/cli/config.go b/internal/cli/config.go new file mode 100644 index 0000000..9ccd101 --- /dev/null +++ b/internal/cli/config.go @@ -0,0 +1,200 @@ +package cli + +import ( + "strings" + + "github.com/spf13/cobra" + + "github.com/ardasevinc/mattermost-cli/internal/config" + "github.com/ardasevinc/mattermost-cli/internal/output" + "github.com/ardasevinc/mattermost-cli/internal/presentation" +) + +type configFlags struct { + path bool + init bool +} + +func newConfigCommand(state *rootState) *cobra.Command { + var flags configFlags + command := &cobra.Command{ + Use: "config", + Short: "Inspect or initialize configuration", + Args: cobra.NoArgs, + PreRunE: func(cmd *cobra.Command, _ []string) error { + if flags.path && flags.init { + return invalidFailure("--path and --init cannot be used together") + } + _, err := state.redactOption(cmd) + return err + }, + RunE: func(_ *cobra.Command, _ []string) error { + paths, err := state.configPaths() + if err != nil { + return err + } + if flags.path { + status := state.presentConfigStatus(output.ConfigEnvelope{ + Schema: "mm/v2/config", Action: "path", SelectedPath: paths.ConfigPath, + }) + return writeConfigStatus(state, status) + } + + action := "status" + var created *bool + if flags.init { + action = "init" + value, initErr := config.Init(paths.ConfigPath) + if initErr != nil { + return configFailure(initErr.Error()) + } + created = &value + } + file := config.Load(paths) + status := state.presentConfigStatus(configMachineStatus(action, file, created)) + if file.Error != "" || file.Unsafe != "" || (file.InsecurePermissions && file.Config.Token != "") { + state.setSemanticExit(3) + } + return writeConfigStatus(state, status) + }, + } + command.Flags().BoolVar(&flags.path, "path", false, "print the selected v2 config path") + command.Flags().BoolVar(&flags.init, "init", false, "create a secure config template without overwriting") + return command +} + +func (s *rootState) configPaths() (config.Paths, error) { + home, err := s.deps.homeDir() + if err != nil { + return config.Paths{}, configFailure("could not resolve the home directory") + } + paths, err := config.ResolvePaths(home, s.deps.lookupEnv) + if err != nil { + return config.Paths{}, configFailure(err.Error()) + } + return paths, nil +} + +func configMachineStatus(action string, file config.FileState, created *bool) output.ConfigEnvelope { + migration := string(file.Migration) + if migration == "" { + migration = string(config.MigrationNone) + } + readStatus, parseStatus, permissions := "ok", "ok", "secure" + if !file.Exists { + readStatus, parseStatus, permissions = "missing", "not_attempted", "not_applicable" + } else if file.Error == config.FileErrorRead { + readStatus, parseStatus = "error", "not_attempted" + permissions = "unknown" + } else if file.Error == config.FileErrorParse { + parseStatus = "error" + } + if file.InsecurePermissions { + permissions = "insecure" + } + var readPath *string + if file.Exists { + readPathValue := file.ReadPath + readPath = &readPathValue + } + var unsafeReason *string + if file.Unsafe != "" { + value := string(file.Unsafe) + unsafeReason = &value + } + var warning *string + if value := file.Warning(); value != "" { + warning = &value + } + return output.ConfigEnvelope{ + Schema: "mm/v2/config", Action: action, SelectedPath: file.SelectedPath, ReadPath: readPath, + Migration: pointer(migration), Exists: pointer(file.Exists), URLConfigured: pointer(file.Config.URL != ""), + TokenConfigured: pointer(file.Config.Token != ""), Permissions: pointer(permissions), ReadStatus: pointer(readStatus), + ParseStatus: pointer(parseStatus), UnsafeReason: unsafeReason, Created: created, Warning: warning, + } +} + +func (s *rootState) presentConfigStatus(status output.ConfigEnvelope) output.ConfigEnvelope { + status.SelectedPath = s.presentConfigLabel(status.SelectedPath) + if status.ReadPath != nil { + value := s.presentConfigLabel(*status.ReadPath) + status.ReadPath = &value + } + if status.Warning != nil { + value := s.presentConfigLabel(*status.Warning) + status.Warning = &value + } + return status +} + +func (s *rootState) presentConfigLabel(value string) string { + processed := presentation.PreprocessWithOptions(value, presentation.Options{ + Credentials: s.credentials, DisableHeuristics: s.disableHeuristics, + }) + return presentation.SanitizeLabel(processed.Text) +} + +func writeConfigStatus(state *rootState, status output.ConfigEnvelope) error { + if state.flags.json { + if _, err := output.WriteMachineJSON(state.streams.out, status); err != nil { + return outputError{err: err} + } + return nil + } + return writeConfigHuman(state, status) +} + +func writeConfigHuman(state *rootState, status output.ConfigEnvelope) error { + if status.Action == "path" { + return writeAll(state.streams.out, []byte(status.SelectedPath+"\n")) + } + if status.Warning != nil { + if err := writeAll(state.streams.err, []byte("warning: "+*status.Warning+"\n")); err != nil { + return err + } + } + if status.Action == "init" { + verb := "Config file already exists: " + if status.Created != nil && *status.Created { + verb = "Created config file: " + } + return writeAll(state.streams.out, []byte(verb+status.SelectedPath+"\n")) + } + readPath := "none" + if status.ReadPath != nil { + readPath = *status.ReadPath + } + unsafe := "none" + if status.UnsafeReason != nil { + unsafe = *status.UnsafeReason + } + lines := []string{ + "Selected config path: " + status.SelectedPath, + "Effective read path: " + readPath, + "Migration: " + valueOr(status.Migration, "none"), + "Exists: " + yesNo(valueOr(status.Exists, false)), + "URL configured: " + yesNo(valueOr(status.URLConfigured, false)), + "Token configured: " + yesNo(valueOr(status.TokenConfigured, false)), + "Permissions: " + valueOr(status.Permissions, "not_applicable"), + "Read status: " + valueOr(status.ReadStatus, "not_attempted"), + "Parse status: " + valueOr(status.ParseStatus, "not_attempted"), + "Unsafe reason: " + unsafe, + } + return writeAll(state.streams.out, []byte(strings.Join(lines, "\n")+"\n")) +} + +func pointer[T any](value T) *T { return &value } + +func valueOr[T any](value *T, fallback T) T { + if value != nil { + return *value + } + return fallback +} + +func yesNo(value bool) string { + if value { + return "yes" + } + return "no" +} diff --git a/internal/cli/config_test.go b/internal/cli/config_test.go new file mode 100644 index 0000000..ea5a4ca --- /dev/null +++ b/internal/cli/config_test.go @@ -0,0 +1,304 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" +) + +func TestConfigStatusIsOfflineAndNeverEmitsValues(t *testing.T) { + home := t.TempDir() + const token = "config-status-opaque-token" + const server = "https://private-mm.example/team" + writeFile(t, filepath.Join(home, ".config", "mattermost-cli", "config.toml"), "url = \""+server+"\"\ntoken = \""+token+"\"\n", 0o600) + t.Setenv("HOME", home) + t.Setenv("MM_URL", "") + t.Setenv("MM_TOKEN", "") + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "config"}, strings.NewReader(""), &stdout, &stderr) + if code != 0 || stderr.Len() != 0 { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + if strings.Contains(stdout.String(), token) || strings.Contains(stdout.String(), server) { + t.Fatalf("config status leaked a configured value: %q", stdout.String()) + } + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/config", bytes.NewReader(stdout.Bytes())); err != nil { + t.Fatalf("machine config document invalid: %v; %s", err, stdout.String()) + } + var document map[string]any + if err := json.Unmarshal(stdout.Bytes(), &document); err != nil { + t.Fatal(err) + } + if document["urlConfigured"] != true || document["tokenConfigured"] != true || document["readStatus"] != "ok" { + t.Fatalf("machine status = %#v", document) + } +} + +func TestConfigPathAndInitUseSelectedXDGPathWithoutCredentials(t *testing.T) { + home, xdg := t.TempDir(), filepath.Join(t.TempDir(), "xdg") + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", xdg) + t.Setenv("MM_URL", "") + t.Setenv("MM_TOKEN", "") + want := filepath.Join(xdg, "mattermost-cli", "config.toml") + + run := func(args ...string) (int, string, string) { + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), args, strings.NewReader(""), &stdout, &stderr) + return code, stdout.String(), stderr.String() + } + if code, stdout, stderr := run("config", "--path"); code != 0 || stdout != want+"\n" || stderr != "" { + t.Fatalf("path exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } + if code, stdout, stderr := run("config", "--init"); code != 0 || !strings.Contains(stdout, "Created config file") || stderr != "" { + t.Fatalf("init exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } + info, err := os.Stat(want) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("initialized config info=%v err=%v", info, err) + } + if err := os.WriteFile(want, []byte("url = \"keep me\"\n"), 0o600); err != nil { + t.Fatal(err) + } + if code, stdout, stderr := run("config", "--init"); code != 0 || !strings.Contains(stdout, "already exists") || stderr != "" { + t.Fatalf("second init exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } + if data, err := os.ReadFile(want); err != nil || string(data) != "url = \"keep me\"\n" { + t.Fatalf("second init changed file: %q %v", data, err) + } +} + +func TestConfigReportsLegacyFallbackAndUnsafeDiagnostics(t *testing.T) { + home, xdg := t.TempDir(), filepath.Join(t.TempDir(), "xdg") + legacy := filepath.Join(home, ".config", "mattermost-cli", "config.toml") + writeFile(t, legacy, "broken = [", 0o644) + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", xdg) + t.Setenv("MM_TOKEN", "") + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"config"}, strings.NewReader(""), &stdout, &stderr) + if code != 3 || !strings.Contains(stderr.String(), "reading legacy config") || + !strings.Contains(stdout.String(), "Migration: legacy_fallback") || + !strings.Contains(stdout.String(), "Permissions: insecure") || + !strings.Contains(stdout.String(), "Parse status: error") { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } +} + +func TestConfigFlagsAreMutuallyExclusiveAndMachineErrorsAreIsolated(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + for _, args := range [][]string{{"config", "--path", "--init"}, {"config", "--path="}} { + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), append([]string{"--json"}, args...), strings.NewReader(""), &stdout, &stderr) + if code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), `"schema":"mm/v2/error"`) { + t.Fatalf("args=%q exit=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String()) + } + } +} + +func TestConfigRedactionFlagsShareGlobalValidationAndSemantics(t *testing.T) { + const token = "config-redaction-owned-token" + const heuristic = "AKIAIOSFODNN7EXAMPLE" + home := filepath.Join(t.TempDir(), token+"-"+heuristic) + if err := os.Mkdir(home, 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("HOME", home) + t.Setenv("MM_TOKEN", token) + + for _, flag := range []string{"--no-redact", "--redact=false"} { + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", flag, "config", "--path"}, strings.NewReader(""), &stdout, &stderr) + if code != 0 || stderr.Len() != 0 || strings.Contains(stdout.String(), token) || !strings.Contains(stdout.String(), heuristic) || !strings.Contains(stdout.String(), "mattermost_credential") { + t.Fatalf("flag=%s exit=%d stdout=%q stderr=%q", flag, code, stdout.String(), stderr.String()) + } + } + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "config", "--path"}, strings.NewReader(""), &stdout, &stderr) + if code != 0 || strings.Contains(stdout.String(), token) || strings.Contains(stdout.String(), heuristic) { + t.Fatalf("default redaction exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + + for _, machine := range []bool{false, true} { + args := []string{"--redact", "--no-redact", "config"} + if machine { + args = append([]string{"--json"}, args...) + } + stdout.Reset() + stderr.Reset() + code = Execute(context.Background(), args, strings.NewReader(""), &stdout, &stderr) + if code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "cannot be used together") { + t.Fatalf("machine=%v exit=%d stdout=%q stderr=%q", machine, code, stdout.String(), stderr.String()) + } + } +} + +func TestConfigInsecureStoredTokenMatchesRuntimeSeverity(t *testing.T) { + for _, test := range []struct { + name, body string + wantExit int + }{ + {name: "stored token", body: `token = "stored-private-token"`, wantExit: 3}, + {name: "tokenless", body: `url = "https://example.com"`, wantExit: 0}, + } { + t.Run(test.name, func(t *testing.T) { + home := t.TempDir() + writeFile(t, filepath.Join(home, ".config", "mattermost-cli", "config.toml"), test.body, 0o644) + t.Setenv("HOME", home) + for _, machine := range []bool{false, true} { + args := []string{"config"} + if machine { + args = append([]string{"--json"}, args...) + } + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), args, strings.NewReader(""), &stdout, &stderr) + if code != test.wantExit || stdout.Len() == 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), "insecure") || strings.Contains(stdout.String(), "stored-private-token") { + t.Fatalf("machine=%v exit=%d stdout=%q stderr=%q", machine, code, stdout.String(), stderr.String()) + } + } + }) + } +} + +func TestShortTokenFormsHaveParityAndNeverLeak(t *testing.T) { + for _, args := range [][]string{{"-t", "short-separated-secret"}, {"-tshort-attached-secret"}, {"-t=short-equals-secret"}, {"-rtgrouped-secret"}, {"-rt=grouped-equals-secret"}} { + secret := earlyTokens(args)[0] + args = append(args, secret) + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), args, strings.NewReader(""), &stdout, &stderr) + if code != 2 || strings.Contains(stderr.String(), secret) || !strings.Contains(stderr.String(), "mattermost_credential") { + t.Fatalf("args=%q exit=%d stderr=%q", args, code, stderr.String()) + } + } +} + +func TestConfigPresentationSanitizesPathsAndWarningsEvenWithoutRedaction(t *testing.T) { + const token = "path-owned-opaque-token" + hostile := token + "\x1b]0;evil\a" + "\u202e" + home := filepath.Join(t.TempDir(), hostile) + if err := os.Mkdir(home, 0o700); err != nil { + t.Fatal(err) + } + legacy := filepath.Join(home, ".config", "mattermost-cli", "config.toml") + writeFile(t, legacy, "url = \"https://example.com\"\n", 0o600) + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, "selected")) + t.Setenv("MM_TOKEN", token) + + for _, args := range [][]string{{"--no-redact", "config"}, {"--json", "--no-redact", "config"}, {"--json", "--no-redact", "config", "--path"}} { + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), args, strings.NewReader(""), &stdout, &stderr) + combined := stdout.String() + stderr.String() + if code != 0 || strings.Contains(combined, token) || strings.ContainsRune(combined, '\x1b') || strings.ContainsRune(combined, '\a') || strings.ContainsRune(combined, '\u202e') { + t.Fatalf("args=%q exit=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String()) + } + if !strings.Contains(combined, "mattermost_credential") || !strings.Contains(combined, `\u001b`) { + t.Fatalf("args=%q lacked safe provenance: %q", args, combined) + } + } +} + +func TestConfigDiagnosticDocumentsUseHandledExitThree(t *testing.T) { + tests := []struct { + name string + setup func(t *testing.T, path string) + }{ + {name: "parse", setup: func(t *testing.T, path string) { writeFile(t, path, "broken = [", 0o600) }}, + {name: "read", setup: func(t *testing.T, path string) { + if err := os.MkdirAll(path, 0o700); err != nil { + t.Fatal(err) + } + }}, + {name: "unsafe", setup: func(t *testing.T, path string) { + target := filepath.Join(filepath.Dir(path), "target.toml") + writeFile(t, target, "url = \"https://example.com\"", 0o600) + if err := os.Symlink(target, path); err != nil { + t.Fatal(err) + } + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + home := t.TempDir() + path := filepath.Join(home, ".config", "mattermost-cli", "config.toml") + test.setup(t, path) + t.Setenv("HOME", home) + for _, machine := range []bool{false, true} { + args := []string{"config"} + if machine { + args = append([]string{"--json"}, args...) + } + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), args, strings.NewReader(""), &stdout, &stderr) + if code != 3 || stdout.Len() == 0 || stderr.Len() != 0 || strings.Contains(stderr.String(), "error:") { + t.Fatalf("machine=%v exit=%d stdout=%q stderr=%q", machine, code, stdout.String(), stderr.String()) + } + if machine { + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/config", bytes.NewReader(stdout.Bytes())); err != nil { + t.Fatalf("invalid document: %v: %s", err, stdout.String()) + } + if strings.Contains(stdout.String(), "mm/v2/error") { + t.Fatalf("appended error document: %q", stdout.String()) + } + } + } + }) + } +} + +func TestConfigPathDoesNotInspectSelectedFile(t *testing.T) { + home := t.TempDir() + path := filepath.Join(home, ".config", "mattermost-cli", "config.toml") + target := filepath.Join(filepath.Dir(path), "target.toml") + writeFile(t, target, "broken = [", 0o600) + if err := os.Symlink(target, path); err != nil { + t.Fatal(err) + } + t.Setenv("HOME", home) + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "config", "--path"}, strings.NewReader(""), &stdout, &stderr) + if code != 0 || stderr.Len() != 0 { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + var document map[string]any + if err := json.Unmarshal(stdout.Bytes(), &document); err != nil { + t.Fatal(err) + } + for _, field := range []string{"readPath", "migration", "exists", "readStatus", "parseStatus", "unsafeReason"} { + if document[field] != nil { + t.Fatalf("path document %s=%v, want null: %#v", field, document[field], document) + } + } +} + +func TestConfigSchemaRejectsContradictoryActionStates(t *testing.T) { + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + contradictions := []string{ + `{"schema":"mm/v2/config","action":"path","selectedPath":"/tmp/config","readPath":"/tmp/config","migration":null,"exists":null,"urlConfigured":null,"tokenConfigured":null,"permissions":null,"readStatus":null,"parseStatus":null,"unsafeReason":null,"created":null,"warning":null}`, + `{"schema":"mm/v2/config","action":"status","selectedPath":"/tmp/config","readPath":null,"migration":"none","exists":false,"urlConfigured":true,"tokenConfigured":false,"permissions":"not_applicable","readStatus":"missing","parseStatus":"not_attempted","unsafeReason":null,"created":null,"warning":null}`, + `{"schema":"mm/v2/config","action":"init","selectedPath":"/tmp/config","readPath":null,"migration":"none","exists":false,"urlConfigured":false,"tokenConfigured":false,"permissions":"not_applicable","readStatus":"missing","parseStatus":"not_attempted","unsafeReason":null,"created":null,"warning":null}`, + `{"schema":"mm/v2/config","action":"status","selectedPath":"/tmp/config","readPath":"/tmp/config","migration":"none","exists":true,"urlConfigured":false,"tokenConfigured":false,"permissions":"secure","readStatus":"ok","parseStatus":"not_attempted","unsafeReason":"type","created":null,"warning":null}`, + } + for _, document := range contradictions { + if err := registry.Validate("mm/v2/config", strings.NewReader(document)); err == nil { + t.Fatalf("contradictory document accepted: %s", document) + } + } +} diff --git a/internal/cli/root.go b/internal/cli/root.go index 6dc2069..120d538 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -63,7 +63,7 @@ func Execute(ctx context.Context, args []string, in io.Reader, out, errOut io.Wr return 3 } } - return 0 + return state.semanticExitCode() } func machineErrorCode(err error) string { @@ -106,7 +106,7 @@ func newRootWithState(state *rootState) *cobra.Command { cmd.SetOut(s.out) cmd.SetErr(s.err) cmd.PersistentFlags().StringVar(&state.flags.url, "url", "", "Mattermost server URL") - cmd.PersistentFlags().StringVar(&state.flags.token, "token", "", "Mattermost personal access token") + cmd.PersistentFlags().StringVarP(&state.flags.token, "token", "t", "", "Mattermost personal access token") cmd.PersistentFlags().BoolVar(&state.flags.redact, "redact", true, "redact detected secrets") cmd.PersistentFlags().BoolVar(&state.flags.noRedact, "no-redact", false, "disable heuristic secret redaction") cmd.PersistentFlags().BoolVar(&state.flags.json, "json", false, "output a versioned JSON document") @@ -116,6 +116,7 @@ func newRootWithState(state *rootState) *cobra.Command { cmd.PersistentFlags().BoolVar(&state.flags.threads, "threads", true, "show visible thread structure") cmd.PersistentFlags().BoolVar(&state.flags.noThreads, "no-threads", false, "return selected seed posts only") cmd.AddCommand(newSchemaCommand(state)) + cmd.AddCommand(newConfigCommand(state)) cmd.AddCommand(newChannelCommand(state)) cmd.AddCommand(newDMsCommand(state)) cmd.AddCommand(newGroupDMsCommand(state)) diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index 93ff9ae..7cb9195 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -57,7 +57,7 @@ func TestSchemaList(t *testing.T) { if code != 0 { t.Fatalf("exit code = %d, want 0; stderr = %q", code, stderr.String()) } - if got, want := stdout.String(), "mm/v2/channel\nmm/v2/dms\nmm/v2/error\nmm/v2/group-dms\nmm/v2/mentions\nmm/v2/search\nmm/v2/thread\n"; got != want { + if got, want := stdout.String(), "mm/v2/channel\nmm/v2/config\nmm/v2/dms\nmm/v2/error\nmm/v2/group-dms\nmm/v2/mentions\nmm/v2/search\nmm/v2/thread\n"; got != want { t.Fatalf("stdout = %q, want %q", got, want) } } diff --git a/internal/cli/runtime.go b/internal/cli/runtime.go index 1d78eb8..3fb6495 100644 --- a/internal/cli/runtime.go +++ b/internal/cli/runtime.go @@ -61,14 +61,16 @@ type rootState struct { deps dependencies flags runtimeFlags - mu sync.Mutex - runtime *Runtime - runtimeErr error - resolved bool - warned bool - releases []func() - credentials []string - pendingWarnings []string + mu sync.Mutex + runtime *Runtime + runtimeErr error + resolved bool + warned bool + releases []func() + credentials []string + pendingWarnings []string + semanticExit int + disableHeuristics bool } type runtimeFlags struct { @@ -151,21 +153,10 @@ func (s *rootState) runtimeFor(cmd *cobra.Command) (*Runtime, error) { s.warned = true } - var redact *bool - redactFlag := cmd.Flags().Lookup("redact") - noRedactFlag := cmd.Flags().Lookup("no-redact") - redactChanged := redactFlag != nil && redactFlag.Changed - noRedactChanged := noRedactFlag != nil && noRedactFlag.Changed - if redactChanged && noRedactChanged { - s.runtimeErr = invalidFailure("--redact and --no-redact cannot be used together") - return nil, s.runtimeErr - } - if redactChanged { - value := s.flags.redact - redact = &value - } else if noRedactChanged { - value := !s.flags.noRedact - redact = &value + redact, err := s.redactOption(cmd) + if err != nil { + s.runtimeErr = err + return nil, err } resolved := config.Resolve(config.Options{URL: s.flags.url, Token: s.flags.token, Redact: redact}, s.deps.lookupEnv, file) if resolved.URL == "" || resolved.Token == "" { @@ -191,6 +182,26 @@ func (s *rootState) runtimeFor(cmd *cobra.Command) (*Runtime, error) { return s.runtime, nil } +func (s *rootState) redactOption(cmd *cobra.Command) (*bool, error) { + redactFlag := cmd.Flags().Lookup("redact") + noRedactFlag := cmd.Flags().Lookup("no-redact") + redactChanged := redactFlag != nil && redactFlag.Changed + noRedactChanged := noRedactFlag != nil && noRedactFlag.Changed + if redactChanged && noRedactChanged { + return nil, invalidFailure("--redact and --no-redact cannot be used together") + } + var redact *bool + if redactChanged { + value := s.flags.redact + redact = &value + } else if noRedactChanged { + value := !s.flags.noRedact + redact = &value + } + s.disableHeuristics = redact != nil && !*redact + return redact, nil +} + func (s *rootState) flushMachineWarnings() error { s.mu.Lock() warnings := strings.Join(s.pendingWarnings, "") @@ -208,6 +219,20 @@ func (s *rootState) queueMachineWarning(message string) { s.mu.Unlock() } +func (s *rootState) setSemanticExit(code int) { + s.mu.Lock() + defer s.mu.Unlock() + if code > s.semanticExit { + s.semanticExit = code + } +} + +func (s *rootState) semanticExitCode() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.semanticExit +} + type errorClass uint8 const ( @@ -278,17 +303,53 @@ func exitCode(err error) int { func earlyTokens(args []string) []string { var tokens []string - for index, arg := range args { + for index := 0; index < len(args); index++ { + arg := args[index] if arg == "--token" && index+1 < len(args) { tokens = append(tokens, args[index+1]) + index++ + continue } if strings.HasPrefix(arg, "--token=") { tokens = append(tokens, strings.TrimPrefix(arg, "--token=")) + continue + } + if value, next, ok := shortTokenValue(args, index); ok { + tokens = append(tokens, value) + if next { + index++ + } } } return tokens } +func shortTokenValue(args []string, index int) (value string, consumedNext, ok bool) { + arg := args[index] + if !strings.HasPrefix(arg, "-") || strings.HasPrefix(arg, "--") || len(arg) < 2 { + return "", false, false + } + shorthand := strings.TrimPrefix(arg, "-") + for position := 0; position < len(shorthand); position++ { + switch shorthand[position] { + case 'r': + continue + case 't': + value := strings.TrimPrefix(shorthand[position+1:], "=") + if value != "" { + return value, false, true + } + if index+1 < len(args) { + return args[index+1], true, true + } + return "", false, false + default: + return "", false, false + } + } + return "", false, false +} + func bestEffortFileToken(deps dependencies) string { home, err := deps.homeDir() if err != nil { diff --git a/internal/config/config.go b/internal/config/config.go index a97b30a..5f4c9c7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -140,6 +140,9 @@ func Init(path string) (bool, error) { _ = os.Remove(path) } }() + if err := file.Chmod(0o600); err != nil { + return false, fmt.Errorf("could not secure config file") + } written, err := io.WriteString(file, Template) if err != nil { return false, fmt.Errorf("could not write config template") diff --git a/internal/config/secure_unix.go b/internal/config/secure_unix.go index a2b71e7..d91589e 100644 --- a/internal/config/secure_unix.go +++ b/internal/config/secure_unix.go @@ -13,6 +13,8 @@ import ( var errUnsafePath = errors.New("unsafe config path") +var configDirectoryFchmodat = unix.Fchmodat + func openConfigFile(path string) (*os.File, os.FileInfo, UnsafeReason, error) { directory, name, unsafe, err := walkConfigParent(path, false) if err != nil { @@ -88,17 +90,28 @@ func walkConfigParent(path string, create bool) (int, string, UnsafeReason, erro func openConfigDirectory(parent int, name string, boundary, create bool) (int, bool, UnsafeReason, error) { var entry unix.Stat_t err := unix.Fstatat(parent, name, &entry, unix.AT_SYMLINK_NOFOLLOW) + created := false if errors.Is(err, unix.ENOENT) && create { - if err := unix.Mkdirat(parent, name, 0o700); err != nil && !errors.Is(err, unix.EEXIST) { - return -1, boundary, "", err + mkdirErr := unix.Mkdirat(parent, name, 0o700) + if mkdirErr == nil { + created = true + } else if !errors.Is(mkdirErr, unix.EEXIST) { + return -1, boundary, "", mkdirErr } err = unix.Fstatat(parent, name, &entry, unix.AT_SYMLINK_NOFOLLOW) } if err != nil { return -1, boundary, "", err } - isSymlink := entry.Mode&unix.S_IFMT == unix.S_IFLNK currentUser := uint32(os.Geteuid()) + if created { + secured, unsafe, secureErr := secureCreatedConfigDirectory(parent, name, entry, currentUser, boundary) + if secureErr != nil { + return -1, boundary, unsafe, secureErr + } + entry = secured + } + isSymlink := entry.Mode&unix.S_IFMT == unix.S_IFLNK if isSymlink && (boundary || (currentUser != 0 && entry.Uid == currentUser)) { return -1, boundary, UnsafeType, errUnsafePath } @@ -120,10 +133,28 @@ func openConfigDirectory(parent int, name string, boundary, create bool) (int, b _ = unix.Close(fd) return -1, boundary, UnsafeType, errUnsafePath } + if !isSymlink && (entry.Dev != opened.Dev || entry.Ino != opened.Ino) { + _ = unix.Close(fd) + return -1, boundary, UnsafeChanged, errUnsafePath + } if opened.Uid != 0 && opened.Uid != currentUser { _ = unix.Close(fd) return -1, boundary, UnsafeOwnership, errUnsafePath } + if created { + if err := unix.Fchmod(fd, 0o700); err != nil { + _ = unix.Close(fd) + return -1, boundary, "", err + } + if err := unix.Fstat(fd, &opened); err != nil { + _ = unix.Close(fd) + return -1, boundary, "", err + } + if opened.Mode&0o777 != 0o700 { + _ = unix.Close(fd) + return -1, boundary, UnsafeUnsupported, errUnsafePath + } + } nextBoundary := boundary || (opened.Uid == currentUser && (currentUser != 0 || opened.Mode&0o077 == 0)) if nextBoundary && opened.Mode&0o022 != 0 { _ = unix.Close(fd) @@ -132,6 +163,42 @@ func openConfigDirectory(parent int, name string, boundary, create bool) (int, b return fd, nextBoundary, "", nil } +func secureCreatedConfigDirectory(parent int, name string, expected unix.Stat_t, currentUser uint32, trustedParent bool) (unix.Stat_t, UnsafeReason, error) { + if expected.Mode&unix.S_IFMT != unix.S_IFDIR || expected.Uid != currentUser { + return unix.Stat_t{}, UnsafeChanged, errUnsafePath + } + err := configDirectoryFchmodat(parent, name, 0o700, unix.AT_SYMLINK_NOFOLLOW) + if errors.Is(err, unix.EOPNOTSUPP) || errors.Is(err, unix.ENOTSUP) { + if !trustedParent { + return unix.Stat_t{}, UnsafeUnsupported, errUnsafePath + } + var beforeFallback unix.Stat_t + if statErr := unix.Fstatat(parent, name, &beforeFallback, unix.AT_SYMLINK_NOFOLLOW); statErr != nil { + return unix.Stat_t{}, "", statErr + } + if !sameCreatedConfigDirectory(expected, beforeFallback, currentUser) { + return unix.Stat_t{}, UnsafeChanged, errUnsafePath + } + err = configDirectoryFchmodat(parent, name, 0o700, 0) + } + if err != nil { + return unix.Stat_t{}, "", err + } + var secured unix.Stat_t + if err := unix.Fstatat(parent, name, &secured, unix.AT_SYMLINK_NOFOLLOW); err != nil { + return unix.Stat_t{}, "", err + } + if !sameCreatedConfigDirectory(expected, secured, currentUser) { + return unix.Stat_t{}, UnsafeChanged, errUnsafePath + } + return secured, "", nil +} + +func sameCreatedConfigDirectory(expected, actual unix.Stat_t, currentUser uint32) bool { + return expected.Dev == actual.Dev && expected.Ino == actual.Ino && + actual.Mode&unix.S_IFMT == unix.S_IFDIR && actual.Uid == currentUser +} + func classifyOpenError(err error) (UnsafeReason, error) { if errors.Is(err, unix.ELOOP) || errors.Is(err, unix.ENOTDIR) { return UnsafeType, errUnsafePath diff --git a/internal/config/secure_unix_test.go b/internal/config/secure_unix_test.go index 8a909bd..50c46d9 100644 --- a/internal/config/secure_unix_test.go +++ b/internal/config/secure_unix_test.go @@ -3,6 +3,8 @@ package config import ( + "errors" + "os" "path/filepath" "testing" "time" @@ -10,6 +12,176 @@ import ( "golang.org/x/sys/unix" ) +func TestInitEnforcesModeDespiteRestrictiveUmask(t *testing.T) { + home := filepath.Join(t.TempDir(), "fresh-home") + if err := os.Mkdir(home, 0o700); err != nil { + t.Fatal(err) + } + configDirectory := filepath.Join(home, ".config", "mattermost-cli") + path := filepath.Join(configDirectory, "config.toml") + previous := unix.Umask(0o777) + t.Cleanup(func() { unix.Umask(previous) }) + created, err := Init(path) + unix.Umask(previous) + if err != nil || !created { + t.Fatalf("Init() = (%v, %v)", created, err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Fatalf("mode = %#o, want 0600", got) + } + for _, directory := range []string{filepath.Join(home, ".config"), configDirectory} { + info, err := os.Stat(directory) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o700 { + t.Fatalf("directory %q mode = %#o, want 0700", directory, got) + } + } +} + +func TestInitFallsBackWhenNoFollowDirectoryChmodIsUnsupportedUnderTrustedParent(t *testing.T) { + original := configDirectoryFchmodat + noFollowCalls, fallbackCalls := 0, 0 + configDirectoryFchmodat = func(dirfd int, path string, mode uint32, flags int) error { + if flags == unix.AT_SYMLINK_NOFOLLOW { + noFollowCalls++ + return unix.ENOTSUP + } + fallbackCalls++ + return original(dirfd, path, mode, flags) + } + t.Cleanup(func() { configDirectoryFchmodat = original }) + + home := filepath.Join(t.TempDir(), "fresh-home") + if err := os.Mkdir(home, 0o700); err != nil { + t.Fatal(err) + } + directory := filepath.Join(home, ".config", "mattermost-cli") + previous := unix.Umask(0o777) + t.Cleanup(func() { unix.Umask(previous) }) + created, err := Init(filepath.Join(directory, "config.toml")) + unix.Umask(previous) + if err != nil || !created { + t.Fatalf("Init() = (%v, %v)", created, err) + } + if noFollowCalls != 2 || fallbackCalls != 2 { + t.Fatalf("chmod calls no-follow=%d fallback=%d, want 2/2", noFollowCalls, fallbackCalls) + } + for _, path := range []string{filepath.Join(home, ".config"), directory} { + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o700 { + t.Fatalf("directory %q mode = %#o, want 0700", path, got) + } + } +} + +func TestUnsupportedNoFollowChmodDoesNotFallbackUnderUntrustedParent(t *testing.T) { + original := configDirectoryFchmodat + fallbackCalls := 0 + configDirectoryFchmodat = func(dirfd int, path string, mode uint32, flags int) error { + if flags == unix.AT_SYMLINK_NOFOLLOW { + return unix.ENOTSUP + } + fallbackCalls++ + return original(dirfd, path, mode, flags) + } + t.Cleanup(func() { configDirectoryFchmodat = original }) + + parentPath := t.TempDir() + if err := os.Chmod(parentPath, 0o777); err != nil { + t.Fatal(err) + } + parent, err := unix.Open(parentPath, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = unix.Close(parent) }) + if err := unix.Mkdirat(parent, "child", 0o700); err != nil { + t.Fatal(err) + } + var expected unix.Stat_t + if err := unix.Fstatat(parent, "child", &expected, unix.AT_SYMLINK_NOFOLLOW); err != nil { + t.Fatal(err) + } + + _, unsafe, err := secureCreatedConfigDirectory(parent, "child", expected, uint32(os.Geteuid()), false) + if !errors.Is(err, errUnsafePath) || unsafe != UnsafeUnsupported { + t.Fatalf("secureCreatedConfigDirectory() unsafe=%q err=%v", unsafe, err) + } + if fallbackCalls != 0 { + t.Fatalf("flags-free fallback called %d times under untrusted parent", fallbackCalls) + } +} + +func TestFallbackRejectsInjectedSwapEvenWithinTrustedBoundary(t *testing.T) { + original := configDirectoryFchmodat + configDirectoryFchmodat = func(dirfd int, path string, mode uint32, flags int) error { + if flags == unix.AT_SYMLINK_NOFOLLOW { + return unix.ENOTSUP + } + if err := unix.Renameat(dirfd, path, dirfd, "original"); err != nil { + return err + } + if err := unix.Mkdirat(dirfd, path, 0o700); err != nil { + return err + } + return original(dirfd, path, mode, flags) + } + t.Cleanup(func() { configDirectoryFchmodat = original }) + + parentPath := t.TempDir() + parent, err := unix.Open(parentPath, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = unix.Close(parent) }) + if err := unix.Mkdirat(parent, "child", 0o700); err != nil { + t.Fatal(err) + } + var expected unix.Stat_t + if err := unix.Fstatat(parent, "child", &expected, unix.AT_SYMLINK_NOFOLLOW); err != nil { + t.Fatal(err) + } + + _, unsafe, err := secureCreatedConfigDirectory(parent, "child", expected, uint32(os.Geteuid()), true) + if !errors.Is(err, errUnsafePath) || unsafe != UnsafeChanged { + t.Fatalf("secureCreatedConfigDirectory() unsafe=%q err=%v, want changed", unsafe, err) + } +} + +func TestInitDoesNotFallbackForOtherDirectoryChmodErrors(t *testing.T) { + original := configDirectoryFchmodat + fallbackCalls := 0 + configDirectoryFchmodat = func(dirfd int, path string, mode uint32, flags int) error { + if flags == unix.AT_SYMLINK_NOFOLLOW { + return unix.EPERM + } + fallbackCalls++ + return original(dirfd, path, mode, flags) + } + t.Cleanup(func() { configDirectoryFchmodat = original }) + + path := filepath.Join(t.TempDir(), "fresh-home", ".config", "mattermost-cli", "config.toml") + created, err := Init(path) + if err == nil || created { + t.Fatalf("Init() = (%v, %v), want fail-closed chmod error", created, err) + } + if fallbackCalls != 0 { + t.Fatalf("flags-free fallback called %d times for EPERM", fallbackCalls) + } + if _, statErr := os.Stat(path); !os.IsNotExist(statErr) { + t.Fatalf("config file survived failed init: %v", statErr) + } +} + func TestLoadRejectsFIFOWithoutBlocking(t *testing.T) { path := filepath.Join(t.TempDir(), "config.toml") if err := unix.Mkfifo(path, 0o600); err != nil { diff --git a/internal/output/machine.go b/internal/output/machine.go index f40049c..091127b 100644 --- a/internal/output/machine.go +++ b/internal/output/machine.go @@ -123,6 +123,23 @@ type ErrorEnvelope struct { Recovery string `json:"recovery"` } +type ConfigEnvelope struct { + Schema string `json:"schema"` + Action string `json:"action"` + SelectedPath string `json:"selectedPath"` + ReadPath *string `json:"readPath"` + Migration *string `json:"migration"` + Exists *bool `json:"exists"` + URLConfigured *bool `json:"urlConfigured"` + TokenConfigured *bool `json:"tokenConfigured"` + Permissions *string `json:"permissions"` + ReadStatus *string `json:"readStatus"` + ParseStatus *string `json:"parseStatus"` + UnsafeReason *string `json:"unsafeReason"` + Created *bool `json:"created"` + Warning *string `json:"warning"` +} + type MachineDocument interface{ machineDocument() } func (DMSEnvelope) machineDocument() {} @@ -132,6 +149,7 @@ func (ThreadEnvelope) machineDocument() {} func (SearchEnvelope) machineDocument() {} func (MentionsEnvelope) machineDocument() {} func (ErrorEnvelope) machineDocument() {} +func (ConfigEnvelope) machineDocument() {} type wireMessage struct { ID string `json:"id"` @@ -247,6 +265,8 @@ func canonicalMachineDocument(document MachineDocument) (any, error) { }{value.Schema, canonicalHistories(value.Results)}, nil case ErrorEnvelope: return value, nil + case ConfigEnvelope: + return value, nil default: return nil, fmt.Errorf("unsupported machine document type %T", document) } @@ -417,7 +437,7 @@ const ( func preflightMachineDocument(document MachineDocument) error { switch document.(type) { - case DMSEnvelope, GroupDMSEnvelope, ChannelEnvelope, ThreadEnvelope, SearchEnvelope, MentionsEnvelope, ErrorEnvelope: + case DMSEnvelope, GroupDMSEnvelope, ChannelEnvelope, ThreadEnvelope, SearchEnvelope, MentionsEnvelope, ErrorEnvelope, ConfigEnvelope: default: return fmt.Errorf("unsupported machine document type %T", document) } diff --git a/schemas/v2/config.schema.json b/schemas/v2/config.schema.json new file mode 100644 index 0000000..9b3ea9c --- /dev/null +++ b/schemas/v2/config.schema.json @@ -0,0 +1,102 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:config", + "title": "mm v2 configuration status", + "type": "object", + "additionalProperties": false, + "required": ["schema", "action", "selectedPath", "readPath", "migration", "exists", "urlConfigured", "tokenConfigured", "permissions", "readStatus", "parseStatus", "unsafeReason", "created", "warning"], + "properties": { + "schema": { "const": "mm/v2/config" }, + "action": { "enum": ["status", "path", "init"] }, + "selectedPath": { "type": "string", "minLength": 1 }, + "readPath": { "type": ["string", "null"], "minLength": 1 }, + "migration": { "type": ["string", "null"], "enum": [null, "none", "legacy_fallback", "legacy_ignored"] }, + "exists": { "type": ["boolean", "null"] }, + "urlConfigured": { "type": ["boolean", "null"] }, + "tokenConfigured": { "type": ["boolean", "null"] }, + "permissions": { "type": ["string", "null"], "enum": [null, "secure", "insecure", "unknown", "not_applicable"] }, + "readStatus": { "type": ["string", "null"], "enum": [null, "ok", "missing", "error"] }, + "parseStatus": { "type": ["string", "null"], "enum": [null, "ok", "not_attempted", "error"] }, + "unsafeReason": { "type": ["string", "null"], "enum": [null, "type", "changed", "ownership", "unsupported"] }, + "created": { "type": ["boolean", "null"] }, + "warning": { "type": ["string", "null"], "minLength": 1 } + }, + "allOf": [ + { + "if": { "properties": { "action": { "const": "path" } }, "required": ["action"] }, + "then": { "properties": { "readPath": { "const": null }, "migration": { "const": null }, "exists": { "const": null }, "urlConfigured": { "const": null }, "tokenConfigured": { "const": null }, "permissions": { "const": null }, "readStatus": { "const": null }, "parseStatus": { "const": null }, "unsafeReason": { "const": null }, "created": { "const": null }, "warning": { "const": null } } } + }, + { + "if": { "properties": { "action": { "const": "status" } }, "required": ["action"] }, + "then": { "properties": { "created": { "const": null } } } + }, + { + "if": { "properties": { "action": { "const": "init" } }, "required": ["action"] }, + "then": { "properties": { "created": { "type": "boolean" } } } + }, + { + "if": { "properties": { "action": { "enum": ["status", "init"] } }, "required": ["action"] }, + "then": { + "properties": { "migration": { "type": "string" }, "exists": { "type": "boolean" }, "urlConfigured": { "type": "boolean" }, "tokenConfigured": { "type": "boolean" }, "permissions": { "type": "string" }, "readStatus": { "type": "string" }, "parseStatus": { "type": "string" } }, + "allOf": [ + { + "if": { "properties": { "exists": { "const": false } }, "required": ["exists"] }, + "then": { "properties": { "readPath": { "const": null }, "urlConfigured": { "const": false }, "tokenConfigured": { "const": false }, "permissions": { "const": "not_applicable" }, "readStatus": { "const": "missing" }, "parseStatus": { "const": "not_attempted" }, "unsafeReason": { "const": null } } } + }, + { + "if": { "properties": { "exists": { "const": true } }, "required": ["exists"] }, + "then": { "properties": { "readPath": { "type": "string" } } } + }, + { + "if": { "properties": { "readStatus": { "const": "error" } }, "required": ["readStatus"] }, + "then": { "properties": { "exists": { "const": true }, "urlConfigured": { "const": false }, "tokenConfigured": { "const": false }, "permissions": { "const": "unknown" }, "parseStatus": { "const": "not_attempted" } } } + }, + { + "if": { "properties": { "readStatus": { "const": "missing" } }, "required": ["readStatus"] }, + "then": { "properties": { "exists": { "const": false } } } + }, + { + "if": { "properties": { "readStatus": { "const": "ok" } }, "required": ["readStatus"] }, + "then": { "properties": { "exists": { "const": true }, "parseStatus": { "enum": ["ok", "error"] } } } + }, + { + "if": { "properties": { "parseStatus": { "const": "not_attempted" } }, "required": ["parseStatus"] }, + "then": { "properties": { "readStatus": { "enum": ["missing", "error"] } } } + }, + { + "if": { "properties": { "unsafeReason": { "type": "string" } }, "required": ["unsafeReason"] }, + "then": { "properties": { "readStatus": { "const": "error" } } } + }, + { + "if": { "properties": { "permissions": { "const": "unknown" } }, "required": ["permissions"] }, + "then": { "properties": { "readStatus": { "const": "error" } } } + }, + { + "if": { "properties": { "permissions": { "const": "not_applicable" } }, "required": ["permissions"] }, + "then": { "properties": { "readStatus": { "const": "missing" } } } + }, + { + "if": { "properties": { "permissions": { "enum": ["secure", "insecure"] } }, "required": ["permissions"] }, + "then": { "properties": { "readStatus": { "const": "ok" } } } + }, + { + "if": { "properties": { "parseStatus": { "const": "error" } }, "required": ["parseStatus"] }, + "then": { "properties": { "exists": { "const": true }, "urlConfigured": { "const": false }, "tokenConfigured": { "const": false }, "readStatus": { "const": "ok" }, "permissions": { "enum": ["secure", "insecure"] }, "unsafeReason": { "const": null } } } + }, + { + "if": { "properties": { "parseStatus": { "const": "ok" } }, "required": ["parseStatus"] }, + "then": { "properties": { "exists": { "const": true }, "readStatus": { "const": "ok" }, "permissions": { "enum": ["secure", "insecure"] }, "unsafeReason": { "const": null } } } + }, + { + "if": { "properties": { "migration": { "const": "none" } }, "required": ["migration"] }, + "then": { "properties": { "warning": { "const": null } } } + }, + { + "if": { "properties": { "migration": { "enum": ["legacy_fallback", "legacy_ignored"] } }, "required": ["migration"] }, + "then": { "properties": { "warning": { "type": "string" } } } + } + ] + } + } + ] +} diff --git a/schemas/v2/examples/config.json b/schemas/v2/examples/config.json new file mode 100644 index 0000000..a3c4428 --- /dev/null +++ b/schemas/v2/examples/config.json @@ -0,0 +1 @@ +{"schema":"mm/v2/config","action":"status","selectedPath":"/home/agent/.config/mattermost-cli/config.toml","readPath":null,"migration":"none","exists":false,"urlConfigured":false,"tokenConfigured":false,"permissions":"not_applicable","readStatus":"missing","parseStatus":"not_attempted","unsafeReason":null,"created":null,"warning":null} From 56f08962cb63daf604d5a874656fba2036410043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 18:09:53 +0300 Subject: [PATCH 037/119] fix: align identity list service models --- internal/mattermost/channels.go | 38 ++++++--- internal/mattermost/channels_test.go | 112 +++++++++++++++++++-------- internal/mattermost/teams.go | 5 +- internal/mattermost/teams_test.go | 32 +++++++- 4 files changed, 138 insertions(+), 49 deletions(-) diff --git a/internal/mattermost/channels.go b/internal/mattermost/channels.go index c0640af..a14ec4c 100644 --- a/internal/mattermost/channels.go +++ b/internal/mattermost/channels.go @@ -20,11 +20,13 @@ type channelTransport interface { } type Channel struct { - ID string - TeamID string - Type string - Name string - DisplayName string + ID string + TeamID string + Type string + Name string + DisplayName string + LastPostAt int64 + TotalMsgCount int64 } func (c *Channel) UnmarshalJSON(data []byte) error { @@ -34,6 +36,8 @@ func (c *Channel) UnmarshalJSON(data []byte) error { Type json.RawMessage `json:"type"` Name json.RawMessage `json:"name"` DisplayName json.RawMessage `json:"display_name"` + LastPostAt json.RawMessage `json:"last_post_at"` + TotalCount json.RawMessage `json:"total_msg_count"` } if err := json.Unmarshal(data, &raw); err != nil { return ErrInvalidChannelResponse @@ -43,7 +47,9 @@ func (c *Channel) UnmarshalJSON(data []byte) error { name, nameOK := requiredString(raw.Name) teamID, teamOK := strictString(raw.TeamID) displayName, displayOK := strictString(raw.DisplayName) - if !idOK || !typeOK || !nameOK || !teamOK || !displayOK { + lastPostAt, lastPostOK := optionalNonNegativeInt64(raw.LastPostAt) + totalCount, totalCountOK := optionalNonNegativeInt64(raw.TotalCount) + if !idOK || !typeOK || !nameOK || !teamOK || !displayOK || !lastPostOK || !totalCountOK { return ErrInvalidChannelResponse } if typeCode != "O" && typeCode != "P" && typeCode != "D" && typeCode != "G" { @@ -62,10 +68,21 @@ func (c *Channel) UnmarshalJSON(data []byte) error { return ErrInvalidChannelResponse } } - *c = Channel{ID: id, TeamID: teamID, Type: typeCode, Name: name, DisplayName: displayName} + *c = Channel{ID: id, TeamID: teamID, Type: typeCode, Name: name, DisplayName: displayName, LastPostAt: lastPostAt, TotalMsgCount: totalCount} return nil } +func optionalNonNegativeInt64(raw json.RawMessage) (int64, bool) { + if len(raw) == 0 { + return 0, true + } + var value int64 + if string(raw) == "null" || json.Unmarshal(raw, &value) != nil || value < 0 { + return 0, false + } + return value, true +} + func strictString(raw json.RawMessage) (string, bool) { var value string if len(raw) == 0 || json.Unmarshal(raw, &value) != nil { @@ -253,6 +270,9 @@ func (s *Channels) List(ctx context.Context, userID string) ([]Channel, error) { continue } seen[channel.ID] = channel + // The canonical current-user listing is the bounded membership proof + // for discovered G channels, as it is in GroupList. Explicit group + // reads continue to prove membership through Member. switch channel.Type { case "O", "P": if !membership.contains(channel.TeamID) { @@ -262,10 +282,6 @@ func (s *Channels) List(ctx context.Context, userID string) ([]Channel, error) { if !directChannelContains(channel.Name, userID) { return nil, ErrInvalidChannelResponse } - case "G": - if _, err := s.Member(ctx, channel.ID, userID); err != nil { - return nil, err - } } result = append(result, channel) } diff --git a/internal/mattermost/channels_test.go b/internal/mattermost/channels_test.go index 951278a..c0e3498 100644 --- a/internal/mattermost/channels_test.go +++ b/internal/mattermost/channels_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "reflect" + "strings" "sync" "testing" ) @@ -15,7 +16,10 @@ type fakeChannelTransport struct { paths []string } -func (f *fakeChannelTransport) Get(_ context.Context, path string, out any) error { +func (f *fakeChannelTransport) Get(ctx context.Context, path string, out any) error { + if err := ctx.Err(); err != nil { + return err + } f.mu.Lock() defer f.mu.Unlock() f.paths = append(f.paths, path) @@ -26,10 +30,42 @@ func (f *fakeChannelTransport) Get(_ context.Context, path string, out any) erro return json.Unmarshal([]byte(payload), out) } +func TestChannelMetadataRequiresExactBoundedJSONIntegers(t *testing.T) { + valid := `{"id":"x","team_id":"team","type":"O","name":"general","display_name":"General","last_post_at":9223372036854775807,"total_msg_count":0}` + got, err := NewChannels(&fakeChannelTransport{responses: map[string]string{"/channels/x": valid}}).ByID(context.Background(), "x") + if err != nil || got.LastPostAt != int64(9223372036854775807) || got.TotalMsgCount != 0 { + t.Fatalf("channel = %+v, error = %v", got, err) + } + + badValues := []string{`null`, `"1"`, `1.5`, `1e3`, `-1`, `9223372036854775808`} + for _, field := range []string{"last_post_at", "total_msg_count"} { + for _, value := range badValues { + payload := `{"id":"x","team_id":"team","type":"O","name":"general","display_name":"General","last_post_at":0,"total_msg_count":0}` + if field == "last_post_at" { + payload = strings.Replace(payload, `"last_post_at":0`, `"last_post_at":`+value, 1) + } else { + payload = strings.Replace(payload, `"total_msg_count":0`, `"total_msg_count":`+value, 1) + } + f := &fakeChannelTransport{responses: map[string]string{"/channels/x": payload}} + if _, err := NewChannels(f).ByID(context.Background(), "x"); !errors.Is(err, ErrInvalidChannelResponse) { + t.Fatalf("%s=%s: error = %v", field, value, err) + } + } + } +} + +func TestChannelMetadataDefaultsOnlyWhenAbsent(t *testing.T) { + payload := `{"id":"x","team_id":"team","type":"O","name":"general","display_name":"General"}` + got, err := NewChannels(&fakeChannelTransport{responses: map[string]string{"/channels/x": payload}}).ByID(context.Background(), "x") + if err != nil || got.LastPostAt != 0 || got.TotalMsgCount != 0 { + t.Fatalf("channel = %+v, error = %v", got, err) + } +} + func TestChannelLookupsEncodeAndRequireExactIdentity(t *testing.T) { f := &fakeChannelTransport{responses: map[string]string{ - "/channels/channel%2Fone": `{"id":"channel/one","team_id":"team/one","type":"P","name":"release/name","display_name":"Release"}`, - "/teams/team%2Fone/channels/name/release%2Fname": `{"id":"channel/one","team_id":"team/one","type":"P","name":"release/name","display_name":"Release"}`, + "/channels/channel%2Fone": `{"id":"channel/one","team_id":"team/one","type":"P","name":"release/name","display_name":"Release","last_post_at":7,"total_msg_count":8}`, + "/teams/team%2Fone/channels/name/release%2Fname": `{"id":"channel/one","team_id":"team/one","type":"P","name":"release/name","display_name":"Release","last_post_at":7,"total_msg_count":8}`, }} channels := NewChannels(f) if _, err := channels.ByID(context.Background(), "channel/one"); err != nil { @@ -45,12 +81,12 @@ func TestChannelLookupsEncodeAndRequireExactIdentity(t *testing.T) { func TestChannelDecodingFailsClosedForRequiredShape(t *testing.T) { bad := []string{ - `null`, `{}`, `{"id":"remote-secret","team_id":"","type":"X","name":"x","display_name":""}`, - `{"id":"x","team_id":"","type":"O","name":"x","display_name":""}`, - `{"id":"x","team_id":"team","type":"G","name":"x","display_name":""}`, - `{"id":"x","team_id":" ","type":"G","name":"x","display_name":""}`, - `{"id":"x","team_id":" ","type":"D","name":"a__b","display_name":""}`, - `{"id":"x","team_id":"","type":"D","name":"alice","display_name":""}`, + `null`, `{}`, `{"id":"remote-secret","team_id":"","type":"X","name":"x","display_name":"","last_post_at":0,"total_msg_count":0}`, + `{"id":"x","team_id":"","type":"O","name":"x","display_name":"","last_post_at":0,"total_msg_count":0}`, + `{"id":"x","team_id":"team","type":"G","name":"x","display_name":"","last_post_at":0,"total_msg_count":0}`, + `{"id":"x","team_id":" ","type":"G","name":"x","display_name":"","last_post_at":0,"total_msg_count":0}`, + `{"id":"x","team_id":" ","type":"D","name":"a__b","display_name":"","last_post_at":0,"total_msg_count":0}`, + `{"id":"x","team_id":"","type":"D","name":"alice","display_name":"","last_post_at":0,"total_msg_count":0}`, } for _, payload := range bad { f := &fakeChannelTransport{responses: map[string]string{"/channels/x": payload}} @@ -68,12 +104,11 @@ func TestChannelListBindsTeamsAndDirectAndGroupParticipants(t *testing.T) { f := &fakeChannelTransport{responses: map[string]string{ "/users/user/teams": `[{"id":"team","name":"core","display_name":"Core","type":"O"}]`, "/users/user/channels": `[ - {"id":"public","team_id":"team","type":"O","name":"general","display_name":"General"}, - {"id":"dm","team_id":"","type":"D","name":"user__alice","display_name":""}, - {"id":"group","team_id":"","type":"G","name":"opaque","display_name":"Crew"}, - {"id":"public","team_id":"team","type":"O","name":"general","display_name":"General"} + {"id":"public","team_id":"team","type":"O","name":"general","display_name":"General","last_post_at":11,"total_msg_count":12}, + {"id":"dm","team_id":"","type":"D","name":"user__alice","display_name":"","last_post_at":0,"total_msg_count":0}, + {"id":"group","team_id":"","type":"G","name":"opaque","display_name":"Crew","last_post_at":1,"total_msg_count":2}, + {"id":"public","team_id":"team","type":"O","name":"general","display_name":"General","last_post_at":11,"total_msg_count":12} ]`, - "/channels/group/members/user": `{"channel_id":"group","user_id":"user","roles":"channel_user"}`, }} got, err := NewChannels(f).List(context.Background(), "user") if err != nil { @@ -82,6 +117,12 @@ func TestChannelListBindsTeamsAndDirectAndGroupParticipants(t *testing.T) { if ids := []string{got[0].ID, got[1].ID, got[2].ID}; !reflect.DeepEqual(ids, []string{"dm", "group", "public"}) { t.Fatalf("IDs = %v", ids) } + if got[2].LastPostAt != 11 || got[2].TotalMsgCount != 12 { + t.Fatalf("channel metadata was not preserved: %+v", got[2]) + } + if !reflect.DeepEqual(f.paths, []string{"/users/user/channels", "/users/user/teams"}) { + t.Fatalf("account-wide discovery fanned out: %v", f.paths) + } for _, path := range f.paths { if path == "/channels/direct" { t.Fatal("read path attempted channel creation") @@ -91,9 +132,9 @@ func TestChannelListBindsTeamsAndDirectAndGroupParticipants(t *testing.T) { func TestChannelListRejectsIncompleteBindings(t *testing.T) { for name, payload := range map[string]string{ - "foreign team": `[{"id":"x","team_id":"other","type":"P","name":"private","display_name":""}]`, - "foreign direct participant": `[{"id":"x","team_id":"","type":"D","name":"alice__bob","display_name":""}]`, - "conflicting duplicate": `[{"id":"x","team_id":"team","type":"O","name":"one","display_name":""},{"id":"x","team_id":"team","type":"O","name":"two","display_name":""}]`, + "foreign team": `[{"id":"x","team_id":"other","type":"P","name":"private","display_name":"","last_post_at":0,"total_msg_count":0}]`, + "foreign direct participant": `[{"id":"x","team_id":"","type":"D","name":"alice__bob","display_name":"","last_post_at":0,"total_msg_count":0}]`, + "conflicting duplicate": `[{"id":"x","team_id":"team","type":"O","name":"one","display_name":"","last_post_at":0,"total_msg_count":0},{"id":"x","team_id":"team","type":"O","name":"two","display_name":"","last_post_at":0,"total_msg_count":0}]`, } { t.Run(name, func(t *testing.T) { f := &fakeChannelTransport{responses: map[string]string{ @@ -106,13 +147,6 @@ func TestChannelListRejectsIncompleteBindings(t *testing.T) { } }) } - f := &fakeChannelTransport{responses: map[string]string{ - "/users/user/channels": `[{"id":"group","team_id":"","type":"G","name":"opaque","display_name":""}]`, - "/channels/group/members/user": `{"channel_id":"other","user_id":"user"}`, - }} - if _, err := NewChannels(f).List(context.Background(), "user"); !errors.Is(err, ErrInvalidChannelResponse) { - t.Fatalf("group binding error = %v", err) - } } func TestChannelListAndMemberRejectAlias(t *testing.T) { @@ -125,9 +159,21 @@ func TestChannelListAndMemberRejectAlias(t *testing.T) { } } +func TestChannelReadsPropagateCancellationWithoutFanout(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + f := &fakeChannelTransport{responses: map[string]string{"/users/user/channels": `[]`}} + if _, err := NewChannels(f).List(ctx, "user"); !errors.Is(err, context.Canceled) { + t.Fatalf("List error = %v", err) + } + if len(f.paths) != 0 { + t.Fatalf("canceled read reached transport paths: %v", f.paths) + } +} + func TestChannelListDoesNotRequireTeamsForDirectOnlyDiscovery(t *testing.T) { f := &fakeChannelTransport{responses: map[string]string{ - "/users/user/channels": `[{"id":"dm","team_id":"","type":"D","name":"user__other","display_name":""}]`, + "/users/user/channels": `[{"id":"dm","team_id":"","type":"D","name":"user__other","display_name":"","last_post_at":0,"total_msg_count":0}]`, }} got, err := NewChannels(f).List(context.Background(), "user") if err != nil || len(got) != 1 || got[0].ID != "dm" { @@ -143,7 +189,7 @@ func TestDirectListIgnoresUnrelatedTeamAndGroupBindings(t *testing.T) { "/users/user/channels": `[ {"type":"P"}, {"type":"G"}, - {"id":"dm","team_id":"","type":"D","name":"user__other","display_name":""} + {"id":"dm","team_id":"","type":"D","name":"user__other","display_name":"","last_post_at":0,"total_msg_count":0} ]`, }} got, err := NewChannels(f).DirectList(context.Background(), "user") @@ -158,8 +204,8 @@ func TestDirectListIgnoresUnrelatedTeamAndGroupBindings(t *testing.T) { func TestDirectListRejectsMalformedOrUnboundDirectChannels(t *testing.T) { tests := map[string]string{ "malformed direct identity": `[{"id":"dm","team_id":"","type":"D","display_name":""}]`, - "foreign direct identity": `[{"id":"dm","team_id":"","type":"D","name":"alice__bob","display_name":""}]`, - "conflicting duplicate": `[{"id":"dm","team_id":"","type":"D","name":"user__alice","display_name":""},{"id":"dm","team_id":"","type":"D","name":"user__bob","display_name":""}]`, + "foreign direct identity": `[{"id":"dm","team_id":"","type":"D","name":"alice__bob","display_name":"","last_post_at":0,"total_msg_count":0}]`, + "conflicting duplicate": `[{"id":"dm","team_id":"","type":"D","name":"user__alice","display_name":"","last_post_at":0,"total_msg_count":0},{"id":"dm","team_id":"","type":"D","name":"user__bob","display_name":"","last_post_at":0,"total_msg_count":0}]`, "missing discriminator": `[{"id":"dm"}]`, "unknown discriminator": `[{"type":"X"}]`, } @@ -174,7 +220,7 @@ func TestDirectListRejectsMalformedOrUnboundDirectChannels(t *testing.T) { } func TestDirectListDedupesExactDirectChannelDuplicates(t *testing.T) { - channel := `{"id":"dm","team_id":"","type":"D","name":"user__other","display_name":""}` + channel := `{"id":"dm","team_id":"","type":"D","name":"user__other","display_name":"","last_post_at":0,"total_msg_count":0}` f := &fakeChannelTransport{responses: map[string]string{"/users/user/channels": `[` + channel + `,` + channel + `]`}} got, err := NewChannels(f).DirectList(context.Background(), "user") if err != nil || len(got) != 1 || got[0].ID != "dm" { @@ -183,7 +229,7 @@ func TestDirectListDedupesExactDirectChannelDuplicates(t *testing.T) { } func TestGroupListUsesCanonicalListingAsBoundedMembershipProof(t *testing.T) { - group := `{"id":"group","team_id":"","type":"G","name":"opaque","display_name":"Crew"}` + group := `{"id":"group","team_id":"","type":"G","name":"opaque","display_name":"Crew","last_post_at":0,"total_msg_count":0}` f := &fakeChannelTransport{responses: map[string]string{ "/users/user/channels": `[{"type":"P"},{"type":"D"},` + group + `,` + group + `]`, }} @@ -199,8 +245,8 @@ func TestGroupListUsesCanonicalListingAsBoundedMembershipProof(t *testing.T) { func TestGroupListRejectsMalformedFocusedChannelsAndMembership(t *testing.T) { for name, payload := range map[string]string{ "malformed group": `[{"id":"group","team_id":"","type":"G","display_name":"Crew"}]`, - "conflicting duplicate": `[{"id":"group","team_id":"","type":"G","name":"one","display_name":""},{"id":"group","team_id":"","type":"G","name":"two","display_name":""}]`, - "foreign team binding": `[{"id":"group","team_id":"foreign","type":"G","name":"one","display_name":""}]`, + "conflicting duplicate": `[{"id":"group","team_id":"","type":"G","name":"one","display_name":"","last_post_at":0,"total_msg_count":0},{"id":"group","team_id":"","type":"G","name":"two","display_name":"","last_post_at":0,"total_msg_count":0}]`, + "foreign team binding": `[{"id":"group","team_id":"foreign","type":"G","name":"one","display_name":"","last_post_at":0,"total_msg_count":0}]`, "missing discriminator": `[{"id":"ignored"}]`, "unknown discriminator": `[{"type":"X"}]`, } { @@ -215,7 +261,7 @@ func TestGroupListRejectsMalformedFocusedChannelsAndMembership(t *testing.T) { func TestChannelReadsAreRaceSafe(t *testing.T) { f := &fakeChannelTransport{responses: map[string]string{ - "/channels/x": `{"id":"x","team_id":"team","type":"O","name":"general","display_name":"General"}`, + "/channels/x": `{"id":"x","team_id":"team","type":"O","name":"general","display_name":"General","last_post_at":0,"total_msg_count":0}`, }} channels := NewChannels(f) var wg sync.WaitGroup diff --git a/internal/mattermost/teams.go b/internal/mattermost/teams.go index 1a0c5d1..9f6bd7d 100644 --- a/internal/mattermost/teams.go +++ b/internal/mattermost/teams.go @@ -60,12 +60,11 @@ func (t *Team) UnmarshalJSON(data []byte) error { } id, idOK := requiredString(raw.ID) name, nameOK := requiredString(raw.Name) - displayName, displayOK := strictString(raw.DisplayName) typeCode, typeOK := requiredString(raw.Type) - if !idOK || !nameOK || !displayOK || !typeOK || (typeCode != "O" && typeCode != "I") { + if !idOK || !nameOK || !typeOK || (typeCode != "O" && typeCode != "I") { return ErrInvalidTeamResponse } - *t = Team{ID: id, Name: name, DisplayName: displayName, Type: typeCode} + *t = Team{ID: id, Name: name, DisplayName: optionalString(raw.DisplayName), Type: typeCode} return nil } diff --git a/internal/mattermost/teams_test.go b/internal/mattermost/teams_test.go index ae5f256..65c8d56 100644 --- a/internal/mattermost/teams_test.go +++ b/internal/mattermost/teams_test.go @@ -15,7 +15,10 @@ type fakeTeamTransport struct { paths []string } -func (f *fakeTeamTransport) Get(_ context.Context, path string, out any) error { +func (f *fakeTeamTransport) Get(ctx context.Context, path string, out any) error { + if err := ctx.Err(); err != nil { + return err + } f.mu.Lock() defer f.mu.Unlock() f.paths = append(f.paths, path) @@ -42,7 +45,7 @@ func TestTeamsListValidatesEncodesSortsAndNarrows(t *testing.T) { } func TestTeamsFailClosedWithoutReflectingRemoteValues(t *testing.T) { - for _, payload := range []string{`null`, `{}`, `[{"id":"remote-secret","name":"x","display_name":"X","type":"X"}]`, `[{"id":"a","name":"x","display_name":7,"type":"O"}]`, `[{"id":"a","name":"x","display_name":"X","type":"O"},{"id":"a","name":"y","display_name":"Y","type":"I"}]`} { + for _, payload := range []string{`null`, `{}`, `[{"id":"remote-secret","name":"x","display_name":"X","type":"X"}]`, `[{"id":"a","name":"x","display_name":"X","type":"O"},{"id":"a","name":"y","display_name":"Y","type":"I"}]`} { _, err := NewTeams(&fakeTeamTransport{payload: payload}).List(context.Background(), "user") if !errors.Is(err, ErrInvalidTeamResponse) && !errors.Is(err, ErrInvalidTeamsResponse) { t.Fatalf("payload %s: error = %v", payload, err) @@ -53,6 +56,31 @@ func TestTeamsFailClosedWithoutReflectingRemoteValues(t *testing.T) { } } +func TestTeamsTreatDisplayNameAsOptionalPresentationData(t *testing.T) { + f := &fakeTeamTransport{payload: `[{"id":"a","name":"core","type":"O"},{"id":"b","name":"eng","display_name":null,"type":"I"},{"id":"c","name":"ops","display_name":7,"type":"O"}]`} + got, err := NewTeams(f).List(context.Background(), "user") + if err != nil { + t.Fatal(err) + } + for _, team := range got.Items() { + if team.DisplayName != "" { + t.Fatalf("display name retained malformed optional data: %+v", team) + } + } +} + +func TestTeamsPropagateCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + f := &fakeTeamTransport{payload: `[]`} + if _, err := NewTeams(f).List(ctx, "user"); !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v", err) + } + if len(f.paths) != 0 { + t.Fatalf("canceled read reached transport paths: %v", f.paths) + } +} + func TestResolveRequiresUniqueCompleteSelection(t *testing.T) { teams := NewTeams(&fakeTeamTransport{payload: `[{"id":"a","name":"core","display_name":"Shared","type":"O"},{"id":"b","name":"eng","display_name":"Shared","type":"I"}]`}) if _, err := teams.Resolve(context.Background(), "user", ""); !errors.Is(err, ErrAmbiguousTeam) { From b6726b35fc6fdfea45ae63b42ee90427ebd17ae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 18:14:23 +0300 Subject: [PATCH 038/119] docs: refresh Go parity status --- docs/V1_PARITY_MATRIX.md | 74 ++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/docs/V1_PARITY_MATRIX.md b/docs/V1_PARITY_MATRIX.md index 58ed81a..e32c96e 100644 --- a/docs/V1_PARITY_MATRIX.md +++ b/docs/V1_PARITY_MATRIX.md @@ -17,40 +17,40 @@ This is the removal gate for the TypeScript implementation. A row may become `ve | Surface | v1 behavior | v2 disposition | Status | | --- | --- | --- | --- | -| binary | `mm` | same public name at cutover | oracle | -| `--help` | Commander help | Cobra help, artifact-smoked | oracle | -| `--version` | package version | Go build metadata, artifact-smoked | oracle | -| `-t`, `--token` | credential CLI override | preserve | oracle | -| `--url` | server URL CLI override | preserve | oracle | +| binary | `mm` | same public name at cutover | scaffolded | +| `--help` | Commander help | Cobra help, artifact-smoked | scaffolded | +| `--version` | package version | Go build metadata, artifact-smoked | scaffolded | +| `-t`, `--token` | credential CLI override | preserve | scaffolded | +| `--url` | server URL CLI override | preserve | scaffolded | | `--json` | command JSON; watch JSONL | replace with schema-identified `mm/v2` JSON/JSONL | intentionally_changed | -| `--no-color` | disables ANSI, not output-format selection | preserve | oracle | -| `-r`, `--relative` | relative timestamps | preserve | oracle | -| `--no-relative` | absolute timestamps | preserve | oracle | -| agent relative default | `is-ai-agent` enables relative output | preserve semantically with Go detection | oracle | -| `--redact` | enable heuristic redaction | preserve | oracle | -| `--no-redact` | disable heuristic redaction, never active-token masking | preserve | oracle | -| `--threads` | hydrate complete visible threads | preserve | oracle | -| `--no-threads` | selected seeds only except `thread` | preserve | oracle | -| numeric validation | canonical positive safe integer only | preserve bounded canonical integer validation | oracle | -| duration validation | `^\d+[hdwm]$` | preserve | oracle | +| `--no-color` | disables ANSI, not output-format selection | preserve | scaffolded | +| `-r`, `--relative` | relative timestamps | preserve | scaffolded | +| `--no-relative` | absolute timestamps | preserve | scaffolded | +| agent relative default | `is-ai-agent` enables relative output | preserve semantically with Go detection | scaffolded | +| `--redact` | enable heuristic redaction | preserve | scaffolded | +| `--no-redact` | disable heuristic redaction, never active-token masking | preserve | scaffolded | +| `--threads` | hydrate complete visible threads | preserve | scaffolded | +| `--no-threads` | selected seeds only except `thread` | preserve | scaffolded | +| numeric validation | canonical positive safe integer only | preserve bounded canonical integer validation | scaffolded | +| duration validation | `^\d+[hdwm]$` | preserve | scaffolded | | URL normalization | WHATWG normalization plus custom loopback test | preserve safe canonicalization; reject transport-ambiguous IPv4/backslash forms | intentionally_changed | ## Configuration | Surface | v1 behavior | v2 disposition | Status | | --- | --- | --- | --- | -| default path | `~/.config/mattermost-cli/config.toml` | preserve on every OS | oracle | +| default path | `~/.config/mattermost-cli/config.toml` | preserve on every OS | scaffolded | | XDG path | ignored | use absolute `$XDG_CONFIG_HOME`, mandatory read-only v1 fallback | intentionally_changed | -| `url` | TOML server URL | preserve | oracle | -| `token` | TOML PAT | preserve | oracle | -| `redact` | TOML default | preserve | oracle | -| `mention_names` | trimmed non-empty string array | preserve | oracle | -| `MM_URL` | URL env override | preserve | oracle | -| `MM_TOKEN` | token env override | preserve | oracle | -| `MM_REDACT` | `false` disables; other defined values enable | preserve | oracle | -| precedence | CLI, env, file, defaults | preserve | oracle | -| init | non-overwriting, mode `0600` | preserve at selected v2 path | oracle | -| permissions | diagnose group/other access; token exposure can be fatal | preserve/fail closed | oracle | +| `url` | TOML server URL | preserve | scaffolded | +| `token` | TOML PAT | preserve | scaffolded | +| `redact` | TOML default | preserve | scaffolded | +| `mention_names` | trimmed non-empty string array | preserve | scaffolded | +| `MM_URL` | URL env override | preserve | scaffolded | +| `MM_TOKEN` | token env override | preserve | scaffolded | +| `MM_REDACT` | `false` disables; other defined values enable | preserve | scaffolded | +| precedence | CLI, env, file, defaults | preserve | scaffolded | +| init | non-overwriting, mode `0600` | preserve at selected v2 path | scaffolded | +| permissions | diagnose group/other access; token exposure can be fatal | preserve/fail closed | scaffolded | | state path | none | XDG state with `~/.local/state` fallback | intentionally_changed | ## Read, diagnostic, and watch commands @@ -58,17 +58,17 @@ This is the removal gate for the TypeScript implementation. A row may become `ve | Command | Flags/defaults | Required v2 behavior | Status | | --- | --- | --- | --- | | `doctor` | global flags | same read-only readiness checks; `mm/v2/doctor` | oracle | -| `config` | `--path`, `--init` | preserve plus deterministic XDG migration warning | oracle | -| `whoami` | global flags | narrow validated identity | oracle | -| `teams` | global flags | validated deterministic teams | oracle | -| `users [query]` | `--team`, `-l`/`--limit 20` | preserve exact directory semantics | oracle | -| `channels` | `--type all`; `dm/public/private/group/all` | preserve account-wide dedupe/team identity | oracle | -| `dms` | repeated `-u`/`--user`; `-l`/`--limit 50`; `-s`/`--since 7d`; `-c`/`--channel`; `--cursor` | preserve exact D-channel and aggregate semantics | oracle | -| `group-dms` | `-l`/`--limit 50`; `-s`/`--since 7d`; `-c`/`--channel`; `--cursor` | preserve exact G-channel and aggregate semantics | oracle | -| `channel ` | `--team`; `-l`/`--limit 50`; `-s`/`--since 7d`; `--cursor` | preserve team-aware resolution | oracle | -| `thread ` | always hydrate | preserve | oracle | -| `search ` | `--team`; `-l`/`--limit 50` | preserve bounded search/completeness | oracle | -| `mentions` | `--team`; `-l`/`--limit 50`; optional `-s`/`--since`; `--channel` | preserve aliases and resolution | oracle | +| `config` | `--path`, `--init` | preserve plus deterministic XDG migration warning | scaffolded | +| `whoami` | global flags | narrow validated identity | scaffolded | +| `teams` | global flags | validated deterministic teams | scaffolded | +| `users [query]` | `--team`, `-l`/`--limit 20` | preserve exact directory semantics | scaffolded | +| `channels` | `--type all`; `dm/public/private/group/all` | preserve account-wide dedupe/team identity | scaffolded | +| `dms` | repeated `-u`/`--user`; `-l`/`--limit 50`; `-s`/`--since 7d`; `-c`/`--channel`; `--cursor` | preserve exact D-channel and aggregate semantics | scaffolded | +| `group-dms` | `-l`/`--limit 50`; `-s`/`--since 7d`; `-c`/`--channel`; `--cursor` | preserve exact G-channel and aggregate semantics | scaffolded | +| `channel ` | `--team`; `-l`/`--limit 50`; `-s`/`--since 7d`; `--cursor` | preserve team-aware resolution | scaffolded | +| `thread ` | always hydrate | preserve | scaffolded | +| `search ` | `--team`; `-l`/`--limit 50` | preserve bounded search/completeness | scaffolded | +| `mentions` | `--team`; `-l`/`--limit 50`; optional `-s`/`--since`; `--channel` | preserve aliases and resolution | scaffolded | | `unread` | `--team`; `--peek` | preserve metrics/sorting/fail-closed empty | oracle | | `watch [channel]` | `--team`; `--dm` | preserve auth, heartbeat, reconnect, gap diagnostics | oracle | From 82991cb2b8cb65e1496e3a8c0603f9222c1400b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 18:38:42 +0300 Subject: [PATCH 039/119] feat: add unread membership model --- internal/mattermost/channels.go | 117 ++++++++++++++++++++- internal/mattermost/channels_test.go | 152 +++++++++++++++++++++++++++ 2 files changed, 268 insertions(+), 1 deletion(-) diff --git a/internal/mattermost/channels.go b/internal/mattermost/channels.go index a14ec4c..dca4325 100644 --- a/internal/mattermost/channels.go +++ b/internal/mattermost/channels.go @@ -27,6 +27,8 @@ type Channel struct { DisplayName string LastPostAt int64 TotalMsgCount int64 + + totalMsgCountPresent bool } func (c *Channel) UnmarshalJSON(data []byte) error { @@ -68,7 +70,10 @@ func (c *Channel) UnmarshalJSON(data []byte) error { return ErrInvalidChannelResponse } } - *c = Channel{ID: id, TeamID: teamID, Type: typeCode, Name: name, DisplayName: displayName, LastPostAt: lastPostAt, TotalMsgCount: totalCount} + *c = Channel{ + ID: id, TeamID: teamID, Type: typeCode, Name: name, DisplayName: displayName, + LastPostAt: lastPostAt, TotalMsgCount: totalCount, totalMsgCountPresent: len(raw.TotalCount) != 0, + } return nil } @@ -113,6 +118,55 @@ func (m *ChannelMember) UnmarshalJSON(data []byte) error { return nil } +type UnreadMember struct { + ChannelID string + UserID string + MsgCount int64 + MentionCount int64 + LastViewedAt int64 +} + +func (m *UnreadMember) UnmarshalJSON(data []byte) error { + var raw struct { + ChannelID json.RawMessage `json:"channel_id"` + UserID json.RawMessage `json:"user_id"` + MsgCount json.RawMessage `json:"msg_count"` + Mentions json.RawMessage `json:"mention_count"` + ViewedAt json.RawMessage `json:"last_viewed_at"` + } + if json.Unmarshal(data, &raw) != nil { + return ErrInvalidChannelResponse + } + channelID, channelOK := requiredString(raw.ChannelID) + userID, userOK := requiredString(raw.UserID) + msgCount, msgOK := requiredNonNegativeInt64(raw.MsgCount) + mentionCount, mentionOK := requiredNonNegativeInt64(raw.Mentions) + lastViewedAt, viewedOK := requiredNonNegativeInt64(raw.ViewedAt) + if !channelOK || !userOK || !msgOK || !mentionOK || !viewedOK { + return ErrInvalidChannelResponse + } + *m = UnreadMember{ChannelID: channelID, UserID: userID, MsgCount: msgCount, MentionCount: mentionCount, LastViewedAt: lastViewedAt} + return nil +} + +func requiredNonNegativeInt64(raw json.RawMessage) (int64, bool) { + if len(raw) == 0 { + return 0, false + } + return optionalNonNegativeInt64(raw) +} + +type unreadMemberList []UnreadMember + +func (l *unreadMemberList) UnmarshalJSON(data []byte) error { + var members []UnreadMember + if err := json.Unmarshal(data, &members); err != nil || members == nil { + return ErrInvalidChannelsResponse + } + *l = members + return nil +} + type Channels struct{ client channelTransport } func NewChannels(client channelTransport) *Channels { return &Channels{client: client} } @@ -237,6 +291,52 @@ func (s *Channels) Member(ctx context.Context, channelID, userID string) (Channe return member, nil } +func (s *Channels) UnreadMember(ctx context.Context, channelID, userID string) (UnreadMember, error) { + if strings.TrimSpace(channelID) == "" || strings.TrimSpace(userID) == "" || userID == "me" { + return UnreadMember{}, ErrInvalidChannelRequest + } + var member UnreadMember + path := "/channels/" + url.PathEscape(channelID) + "/members/" + url.PathEscape(userID) + if err := s.client.Get(ctx, path, &member); err != nil { + return UnreadMember{}, err + } + if member.ChannelID != channelID || member.UserID != userID { + return UnreadMember{}, ErrInvalidChannelResponse + } + return member, nil +} + +// TeamMembers returns the bounded, complete membership snapshot used for team +// unread metrics. The endpoint is authoritative for the requested team, while +// every returned row is still bound to the requested canonical user ID. +func (s *Channels) TeamMembers(ctx context.Context, userID, teamID string) ([]UnreadMember, error) { + if !canonicalChannelRequestID(userID) || !canonicalChannelRequestID(teamID) || userID == "me" { + return nil, ErrInvalidChannelRequest + } + var decoded unreadMemberList + path := "/users/" + url.PathEscape(userID) + "/teams/" + url.PathEscape(teamID) + "/channels/members" + if err := s.client.Get(ctx, path, &decoded); err != nil { + return nil, err + } + members := []UnreadMember(decoded) + seen := make(map[string]struct{}, len(members)) + for _, member := range members { + if member.UserID != userID || !canonicalChannelRequestID(member.ChannelID) { + return nil, ErrInvalidChannelsResponse + } + if _, duplicate := seen[member.ChannelID]; duplicate { + return nil, ErrInvalidChannelsResponse + } + seen[member.ChannelID] = struct{}{} + } + sort.Slice(members, func(i, j int) bool { return members[i].ChannelID < members[j].ChannelID }) + return members, nil +} + +func canonicalChannelRequestID(value string) bool { + return value != "" && value == strings.TrimSpace(value) +} + // List returns the authenticated user's channels with every identity binding // checked before any result is released. Team membership is fetched through // the same transport so proof cannot be mixed across sessions or servers. @@ -289,6 +389,21 @@ func (s *Channels) List(ctx context.Context, userID string) ([]Channel, error) { return result, nil } +// ListForUnread preserves List's bounded identity proof while refusing to +// interpret an absent total_msg_count as a provably empty unread state. +func (s *Channels) ListForUnread(ctx context.Context, userID string) ([]Channel, error) { + channels, err := s.List(ctx, userID) + if err != nil { + return nil, err + } + for _, channel := range channels { + if !channel.totalMsgCountPresent { + return nil, ErrInvalidChannelsResponse + } + } + return channels, nil +} + // DirectList returns only current-user-bound D channels. It deliberately does // not resolve team membership or group membership for unrelated channel types. func (s *Channels) DirectList(ctx context.Context, userID string) ([]Channel, error) { diff --git a/internal/mattermost/channels_test.go b/internal/mattermost/channels_test.go index c0e3498..b8e25f1 100644 --- a/internal/mattermost/channels_test.go +++ b/internal/mattermost/channels_test.go @@ -159,6 +159,128 @@ func TestChannelListAndMemberRejectAlias(t *testing.T) { } } +func TestChannelMemberIsProofOnlyAndIgnoresMetricFields(t *testing.T) { + responses := map[string]string{ + "/channels/channel/members/user": `{"channel_id":"channel","user_id":"user"}`, + } + got, err := NewChannels(&fakeChannelTransport{responses: responses}).Member(context.Background(), "channel", "user") + if err != nil || got.ChannelID != "channel" || got.UserID != "user" { + t.Fatalf("member = %+v, error = %v", got, err) + } + payload := `{"channel_id":"channel","user_id":"user","msg_count":-1,"mention_count":-1,"last_viewed_at":-1}` + f := &fakeChannelTransport{responses: map[string]string{"/channels/channel/members/user": payload}} + if _, err := NewChannels(f).Member(context.Background(), "channel", "user"); err != nil { + t.Fatalf("sanitized proof-only member error = %v", err) + } +} + +func TestUnreadMemberRequiresCompleteStrictMetrics(t *testing.T) { + valid := `{"channel_id":"channel","user_id":"user","msg_count":1,"mention_count":2,"last_viewed_at":3}` + f := &fakeChannelTransport{responses: map[string]string{"/channels/channel/members/user": valid}} + got, err := NewChannels(f).UnreadMember(context.Background(), "channel", "user") + if err != nil || got.MsgCount != 1 || got.MentionCount != 2 || got.LastViewedAt != 3 { + t.Fatalf("member = %+v, error = %v", got, err) + } + for name, payload := range map[string]string{ + "missing": `{"channel_id":"channel","user_id":"user"}`, + "partial": `{"channel_id":"channel","user_id":"user","msg_count":1}`, + "negative": `{"channel_id":"channel","user_id":"user","msg_count":-1,"mention_count":0,"last_viewed_at":0}`, + } { + t.Run(name, func(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{"/channels/channel/members/user": payload}} + if _, err := NewChannels(f).UnreadMember(context.Background(), "channel", "user"); !errors.Is(err, ErrInvalidChannelResponse) { + t.Fatalf("error = %v", err) + } + }) + } +} + +func TestTeamMembersReturnsExactBoundedSortedSnapshot(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{ + "/users/user%2Fone/teams/team%2Fone/channels/members": `[ + {"channel_id":"z","user_id":"user/one","msg_count":4,"mention_count":2,"last_viewed_at":9}, + {"channel_id":"a","user_id":"user/one","msg_count":1,"mention_count":0,"last_viewed_at":3} + ]`, + }} + got, err := NewChannels(f).TeamMembers(context.Background(), "user/one", "team/one") + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || got[0].ChannelID != "a" || got[1].ChannelID != "z" || got[1].MentionCount != 2 { + t.Fatalf("members = %+v", got) + } + if !reflect.DeepEqual(f.paths, []string{"/users/user%2Fone/teams/team%2Fone/channels/members"}) { + t.Fatalf("paths = %v", f.paths) + } +} + +func TestTeamMembersAcceptsEmptyCompleteSnapshot(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{ + "/users/user/teams/team/channels/members": `[]`, + }} + got, err := NewChannels(f).TeamMembers(context.Background(), "user", "team") + if err != nil || got == nil || len(got) != 0 { + t.Fatalf("members = %#v, error = %v", got, err) + } +} + +func TestTeamMembersRequiresCompleteBoundSnapshot(t *testing.T) { + tests := map[string]string{ + "null list": `null`, + "object": `{}`, + "missing metric": `[{"channel_id":"a","user_id":"user","msg_count":1,"mention_count":0}]`, + "foreign user": `[{"channel_id":"a","user_id":"other","msg_count":1,"mention_count":0,"last_viewed_at":0}]`, + "noncanonical channel": `[{"channel_id":" a","user_id":"user","msg_count":1,"mention_count":0,"last_viewed_at":0}]`, + "duplicate channel": `[{"channel_id":"a","user_id":"user","msg_count":1,"mention_count":0,"last_viewed_at":0},{"channel_id":"a","user_id":"user","msg_count":1,"mention_count":0,"last_viewed_at":0}]`, + } + for name, payload := range tests { + t.Run(name, func(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{"/users/user/teams/team/channels/members": payload}} + _, err := NewChannels(f).TeamMembers(context.Background(), "user", "team") + if !errors.Is(err, ErrInvalidChannelsResponse) { + t.Fatalf("error = %v", err) + } + if err != nil && strings.Contains(err.Error(), "other") { + t.Fatalf("error reflected remote data: %v", err) + } + }) + } +} + +func TestTeamMembersRejectsNoncanonicalRequestsAndCancellation(t *testing.T) { + channels := NewChannels(&fakeChannelTransport{}) + for _, ids := range [][2]string{{"", "team"}, {"me", "team"}, {" user", "team"}, {"user", "team "}} { + if _, err := channels.TeamMembers(context.Background(), ids[0], ids[1]); !errors.Is(err, ErrInvalidChannelRequest) { + t.Fatalf("ids=%q: error = %v", ids, err) + } + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + f := &fakeChannelTransport{responses: map[string]string{"/users/user/teams/team/channels/members": `[]`}} + if _, err := NewChannels(f).TeamMembers(ctx, "user", "team"); !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v", err) + } + if len(f.paths) != 0 { + t.Fatalf("canceled read reached transport: %v", f.paths) + } +} + +func TestTeamMembersIsRaceSafe(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{ + "/users/user/teams/team/channels/members": `[{"channel_id":"a","user_id":"user","msg_count":1,"mention_count":0,"last_viewed_at":0}]`, + }} + channels := NewChannels(f) + var wg sync.WaitGroup + for range 40 { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = channels.TeamMembers(context.Background(), "user", "team") + }() + } + wg.Wait() +} + func TestChannelReadsPropagateCancellationWithoutFanout(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() @@ -184,6 +306,36 @@ func TestChannelListDoesNotRequireTeamsForDirectOnlyDiscovery(t *testing.T) { } } +func TestListForUnreadRequiresPresentTotalsWithoutFanout(t *testing.T) { + for name, payload := range map[string]string{ + "missing": `[{"id":"remote-secret","team_id":"","type":"D","name":"user__other","display_name":""}]`, + "present": `[{"id":"dm","team_id":"","type":"D","name":"user__other","display_name":"","total_msg_count":0}]`, + } { + t.Run(name, func(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{"/users/user/channels": payload}} + got, err := NewChannels(f).ListForUnread(context.Background(), "user") + if name == "missing" { + if !errors.Is(err, ErrInvalidChannelsResponse) || strings.Contains(err.Error(), "remote-secret") { + t.Fatalf("channels = %#v, error = %v", got, err) + } + } else if err != nil || len(got) != 1 || got[0].TotalMsgCount != 0 { + t.Fatalf("channels = %#v, error = %v", got, err) + } + if !reflect.DeepEqual(f.paths, []string{"/users/user/channels"}) { + t.Fatalf("paths = %v", f.paths) + } + }) + } +} + +func TestListForUnreadAcceptsEmptyCompleteSnapshot(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{"/users/user/channels": `[]`}} + got, err := NewChannels(f).ListForUnread(context.Background(), "user") + if err != nil || got == nil || len(got) != 0 { + t.Fatalf("channels = %#v, error = %v", got, err) + } +} + func TestDirectListIgnoresUnrelatedTeamAndGroupBindings(t *testing.T) { f := &fakeChannelTransport{responses: map[string]string{ "/users/user/channels": `[ From c9c99bc6ba06a6c0da7f86fcd83cb9088eac4d2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 19:04:47 +0300 Subject: [PATCH 040/119] feat: add doctor command --- internal/cli/doctor.go | 120 +++++++++++++++++++++++ internal/cli/doctor_test.go | 160 +++++++++++++++++++++++++++++++ internal/cli/root.go | 1 + internal/cli/root_test.go | 2 +- internal/output/machine.go | 148 +++++++++++++++++++++++++++- internal/output/machine_test.go | 55 +++++++++++ internal/schema/registry_test.go | 26 +++++ schemas/v2/doctor.schema.json | 152 +++++++++++++++++++++++++++++ schemas/v2/examples/doctor.json | 1 + 9 files changed, 663 insertions(+), 2 deletions(-) create mode 100644 internal/cli/doctor.go create mode 100644 internal/cli/doctor_test.go create mode 100644 schemas/v2/doctor.schema.json create mode 100644 schemas/v2/examples/doctor.json diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go new file mode 100644 index 0000000..9233a3a --- /dev/null +++ b/internal/cli/doctor.go @@ -0,0 +1,120 @@ +package cli + +import ( + "strconv" + "strings" + + "github.com/spf13/cobra" + + "github.com/ardasevinc/mattermost-cli/internal/config" + "github.com/ardasevinc/mattermost-cli/internal/doctor" + "github.com/ardasevinc/mattermost-cli/internal/output" + "github.com/ardasevinc/mattermost-cli/internal/presentation" +) + +func newDoctorCommand(state *rootState) *cobra.Command { + var redactOverride *bool + return &cobra.Command{ + Use: "doctor", + Short: "Check configuration, server health, and authentication", + Args: cobra.NoArgs, + PreRunE: func(cmd *cobra.Command, _ []string) error { + var err error + redactOverride, err = state.redactOption(cmd) + return err + }, + RunE: func(cmd *cobra.Command, _ []string) error { + resolved, warning := state.resolveForDoctor(redactOverride) + if warning != "" { + if state.flags.json { + state.queueMachineWarning("warning: " + warning + "\n") + } else if err := writeAll(state.streams.err, []byte("warning: "+warning+"\n")); err != nil { + return err + } + } + report := doctor.Run(cmd.Context(), resolved, func(baseURL, token string) (doctor.Transport, func(), error) { + client, err := state.deps.newClient(baseURL, token) + if err != nil { + return nil, nil, err + } + return client, client.Close, nil + }) + if !report.OK { + state.setSemanticExit(3) + } + return writeDoctorReport(state, report) + }, + } +} + +func (s *rootState) resolveForDoctor(redactOverride *bool) (config.Resolved, string) { + var file config.FileState + paths, err := s.configPaths() + if err != nil { + file.Error = config.FileErrorRead + } else { + file = config.Load(paths) + } + if file.Config.Token != "" { + s.releases = append(s.releases, presentation.ActiveCredentials.Register(file.Config.Token)) + s.credentials = append(s.credentials, file.Config.Token) + } + resolved := config.Resolve(config.Options{URL: s.flags.url, Token: s.flags.token, Redact: redactOverride}, s.deps.lookupEnv, file) + s.disableHeuristics = !resolved.Redact + return resolved, s.presentConfigLabel(file.Warning()) +} + +func writeDoctorReport(state *rootState, report doctor.Report) error { + if state.flags.json { + checks := make([]output.DoctorCheck, len(report.Checks)) + for index, check := range report.Checks { + checks[index] = output.DoctorCheck{Name: check.Name, Status: string(check.Status), Message: check.Message, Details: check.Details} + } + document, err := output.NewDoctorEnvelope(report.OK, checks) + if err != nil { + return outputError{err: err} + } + if _, err := output.WriteMachineJSON(state.streams.out, document); err != nil { + return outputError{err: err} + } + return nil + } + lines := make([]string, len(report.Checks)) + for index, check := range report.Checks { + details := doctorDetails(check) + lines[index] = formatDoctorStatus(check.Status) + " " + check.Name + ": " + check.Message + details + } + return writeAll(state.streams.out, []byte(strings.Join(lines, "\n")+"\n")) +} + +func doctorDetails(check doctor.Check) string { + if len(check.Details) == 0 { + return "" + } + keys := []string{"urlSource", "tokenSource", "status", "databaseStatus", "filestoreStatus", "httpStatus", "id", "username"} + values := make([]string, 0, len(check.Details)) + for _, key := range keys { + if value, ok := check.Details[key]; ok { + values = append(values, key+"="+presentation.SanitizeLabel(strings.TrimSpace(toString(value)))) + } + } + return " (" + strings.Join(values, ", ") + ")" +} + +func toString(value any) string { + switch value := value.(type) { + case string: + return value + case config.Source: + return string(value) + case int: + return strconv.Itoa(value) + default: + return "unknown" + } +} + +func formatDoctorStatus(status doctor.Status) string { + value := string(status) + return value + strings.Repeat(" ", 7-len(value)) +} diff --git a/internal/cli/doctor_test.go b/internal/cli/doctor_test.go new file mode 100644 index 0000000..c3b9ee2 --- /dev/null +++ b/internal/cli/doctor_test.go @@ -0,0 +1,160 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" +) + +func TestDoctorMachineReportUsesPublicPingAndAuthenticatedIdentity(t *testing.T) { + const token = "doctor-active-token" + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + calls.Add(1) + switch request.URL.Path { + case "/api/v4/system/ping": + if request.Header.Get("Authorization") != "" || request.URL.Query().Get("get_server_status") != "true" { + t.Fatalf("public ping carried auth or omitted status query") + } + writeJSON(t, w, `{"status":"OK","database_status":"OK","filestore_status":"OK"}`) + case "/api/v4/users/me": + if request.Header.Get("Authorization") != "Bearer "+token { + t.Fatalf("identity authorization = %q", request.Header.Get("Authorization")) + } + writeJSON(t, w, `{"id":"user-id","username":"arda","email":"`+token+`"}`) + default: + t.Fatalf("unexpected request %s", request.URL.String()) + } + })) + defer server.Close() + + t.Setenv("HOME", t.TempDir()) + t.Setenv("MM_URL", "") + t.Setenv("MM_TOKEN", "") + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "--url", server.URL, "--token", token, "doctor"}, strings.NewReader(""), &stdout, &stderr) + if code != 0 || stderr.Len() != 0 || calls.Load() != 2 || strings.Contains(stdout.String(), token) { + t.Fatalf("exit=%d calls=%d stdout=%q stderr=%q", code, calls.Load(), stdout.String(), stderr.String()) + } + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/doctor", bytes.NewReader(stdout.Bytes())); err != nil { + t.Fatalf("doctor report failed schema validation: %v\n%s", err, stdout.String()) + } +} + +func TestDoctorIncompleteConfigurationEmitsReportThenHandledExit(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + t.Setenv("MM_URL", "") + t.Setenv("MM_TOKEN", "") + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "doctor"}, strings.NewReader(""), &stdout, &stderr) + if code != 3 || stderr.Len() != 0 || strings.Contains(stderr.String(), "mm/v2/error") { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + var report struct { + Schema string `json:"schema"` + OK bool `json:"ok"` + Checks []struct{ Name, Status string } `json:"checks"` + } + if err := json.Unmarshal(stdout.Bytes(), &report); err != nil { + t.Fatal(err) + } + if report.Schema != "mm/v2/doctor" || report.OK || len(report.Checks) != 3 || report.Checks[0].Name != "configuration" || report.Checks[1].Name != "server" || report.Checks[2].Name != "authentication" { + t.Fatalf("report = %+v", report) + } +} + +func TestDoctorMigrationWarningDoesNotCorruptMachineReport(t *testing.T) { + home, xdg := t.TempDir(), filepath.Join(t.TempDir(), "xdg\x1b]8;;bad\x07") + legacy := filepath.Join(home, ".config", "mattermost-cli", "config.toml") + writeFile(t, legacy, "url = \"http://127.0.0.1:1\"\n", 0o600) + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", xdg) + t.Setenv("MM_URL", "") + t.Setenv("MM_TOKEN", "") + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "doctor"}, strings.NewReader(""), &stdout, &stderr) + if code != 3 || !strings.Contains(stderr.String(), "warning:") || strings.Contains(stderr.String(), "\x1b") || strings.Contains(stdout.String(), "warning:") { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + if !json.Valid(stdout.Bytes()) { + t.Fatalf("invalid report: %q", stdout.String()) + } +} + +func TestDoctorRejectsArgumentsAndReportsWriterFailure(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + var stdout, stderr bytes.Buffer + if code := Execute(context.Background(), []string{"doctor", "extra"}, strings.NewReader(""), &stdout, &stderr); code != 2 || stdout.Len() != 0 { + t.Fatalf("argument exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + stderr.Reset() + if code := Execute(context.Background(), []string{"--json", "doctor"}, strings.NewReader(""), shortWriter{}, &stderr); code != 3 || strings.Contains(stderr.String(), "mm/v2/error") { + t.Fatalf("short writer exit=%d stderr=%q", code, stderr.String()) + } +} + +func TestDoctorHonorsEnvAndFileRedactionFalseWithoutExposingActiveCredential(t *testing.T) { + const token = "doctor-owned-active-credential" + const probe = "ghp_abcdefghijklmnopqrstuvwxyz1234567890" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/system/ping": + writeJSON(t, w, `{"status":"OK","database_status":"`+probe+`","filestore_status":"OK"}`) + case "/api/v4/users/me": + writeJSON(t, w, `{"id":"`+token+`","username":"`+token+probe+`"}`) + default: + t.Fatalf("unexpected request %s", request.URL.String()) + } + })) + defer server.Close() + + t.Run("environment", func(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + t.Setenv("MM_REDACT", "false") + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "--url", server.URL, "--token", token, "doctor"}, strings.NewReader(""), &stdout, &stderr) + assertDoctorHeuristicsDisabled(t, code, stdout.String(), stderr.String(), probe, token) + }) + + t.Run("file", func(t *testing.T) { + previous, present := os.LookupEnv("MM_REDACT") + if err := os.Unsetenv("MM_REDACT"); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if present { + _ = os.Setenv("MM_REDACT", previous) + } else { + _ = os.Unsetenv("MM_REDACT") + } + }) + home := t.TempDir() + writeFile(t, filepath.Join(home, ".config", "mattermost-cli", "config.toml"), "url = \""+server.URL+"\"\ntoken = \""+token+"\"\nredact = false\n", 0o600) + t.Setenv("HOME", home) + t.Setenv("MM_URL", "") + t.Setenv("MM_TOKEN", "") + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "doctor"}, strings.NewReader(""), &stdout, &stderr) + assertDoctorHeuristicsDisabled(t, code, stdout.String(), stderr.String(), probe, token) + }) +} + +func assertDoctorHeuristicsDisabled(t *testing.T, code int, stdout, stderr, probe, token string) { + t.Helper() + if code != 3 || stderr != "" || !strings.Contains(stdout, probe) || strings.Contains(stdout, token) || !strings.Contains(stdout, "REDACTED") { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } +} diff --git a/internal/cli/root.go b/internal/cli/root.go index 120d538..b56e348 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -117,6 +117,7 @@ func newRootWithState(state *rootState) *cobra.Command { cmd.PersistentFlags().BoolVar(&state.flags.noThreads, "no-threads", false, "return selected seed posts only") cmd.AddCommand(newSchemaCommand(state)) cmd.AddCommand(newConfigCommand(state)) + cmd.AddCommand(newDoctorCommand(state)) cmd.AddCommand(newChannelCommand(state)) cmd.AddCommand(newDMsCommand(state)) cmd.AddCommand(newGroupDMsCommand(state)) diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index 7cb9195..28bf2b6 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -57,7 +57,7 @@ func TestSchemaList(t *testing.T) { if code != 0 { t.Fatalf("exit code = %d, want 0; stderr = %q", code, stderr.String()) } - if got, want := stdout.String(), "mm/v2/channel\nmm/v2/config\nmm/v2/dms\nmm/v2/error\nmm/v2/group-dms\nmm/v2/mentions\nmm/v2/search\nmm/v2/thread\n"; got != want { + if got, want := stdout.String(), "mm/v2/channel\nmm/v2/config\nmm/v2/dms\nmm/v2/doctor\nmm/v2/error\nmm/v2/group-dms\nmm/v2/mentions\nmm/v2/search\nmm/v2/thread\n"; got != want { t.Fatalf("stdout = %q, want %q", got, want) } } diff --git a/internal/output/machine.go b/internal/output/machine.go index 091127b..5d245e0 100644 --- a/internal/output/machine.go +++ b/internal/output/machine.go @@ -140,6 +140,27 @@ type ConfigEnvelope struct { Warning *string `json:"warning"` } +type DoctorCheck struct { + Name string `json:"name"` + Status string `json:"status"` + Message string `json:"message"` + Details map[string]any `json:"details,omitempty"` +} + +type DoctorEnvelope struct { + Schema string `json:"schema"` + OK bool `json:"ok"` + Checks []DoctorCheck `json:"checks"` +} + +func NewDoctorEnvelope(ok bool, checks []DoctorCheck) (DoctorEnvelope, error) { + document := DoctorEnvelope{Schema: "mm/v2/doctor", OK: ok, Checks: cloneSlice(checks)} + if err := validateDoctorEnvelope(document); err != nil { + return DoctorEnvelope{}, err + } + return document, nil +} + type MachineDocument interface{ machineDocument() } func (DMSEnvelope) machineDocument() {} @@ -150,6 +171,7 @@ func (SearchEnvelope) machineDocument() {} func (MentionsEnvelope) machineDocument() {} func (ErrorEnvelope) machineDocument() {} func (ConfigEnvelope) machineDocument() {} +func (DoctorEnvelope) machineDocument() {} type wireMessage struct { ID string `json:"id"` @@ -267,6 +289,8 @@ func canonicalMachineDocument(document MachineDocument) (any, error) { return value, nil case ConfigEnvelope: return value, nil + case DoctorEnvelope: + return value, nil default: return nil, fmt.Errorf("unsupported machine document type %T", document) } @@ -437,10 +461,15 @@ const ( func preflightMachineDocument(document MachineDocument) error { switch document.(type) { - case DMSEnvelope, GroupDMSEnvelope, ChannelEnvelope, ThreadEnvelope, SearchEnvelope, MentionsEnvelope, ErrorEnvelope, ConfigEnvelope: + case DMSEnvelope, GroupDMSEnvelope, ChannelEnvelope, ThreadEnvelope, SearchEnvelope, MentionsEnvelope, ErrorEnvelope, ConfigEnvelope, DoctorEnvelope: default: return fmt.Errorf("unsupported machine document type %T", document) } + if doctorDocument, ok := document.(DoctorEnvelope); ok { + if err := validateDoctorEnvelope(doctorDocument); err != nil { + return err + } + } contentBudget := int64(MaxMachineDocumentBytes) valueBudget := machinePreflightMaxValues stack := make(map[preflightVisit]bool) @@ -450,6 +479,123 @@ func preflightMachineDocument(document MachineDocument) error { return nil } +func validateDoctorEnvelope(document DoctorEnvelope) error { + if document.Schema != "mm/v2/doctor" || len(document.Checks) != 3 { + return errors.New("invalid doctor document shape") + } + names := [3]string{"configuration", "server", "authentication"} + hasFailure := false + for index, check := range document.Checks { + if check.Name != names[index] || !doctorStatusValid(check.Status) || !safeDoctorString(check.Message) { + return errors.New("invalid doctor check presentation") + } + if check.Status == "fail" { + hasFailure = true + } + if err := validateDoctorDetails(index, check.Status, check.Details); err != nil { + return err + } + } + if document.OK == hasFailure { + return errors.New("doctor ok does not match check statuses") + } + return nil +} + +func doctorStatusValid(status string) bool { + return status == "pass" || status == "warn" || status == "fail" || status == "skipped" +} + +func validateDoctorDetails(index int, status string, details map[string]any) error { + switch index { + case 0: + if status == "skipped" || len(details) != 2 || !doctorSource(details["urlSource"]) || !doctorSource(details["tokenSource"]) { + return errors.New("invalid doctor configuration details") + } + case 1: + if status == "skipped" { + if len(details) != 0 { + return errors.New("skipped doctor server check has details") + } + return nil + } + if status == "pass" || status == "warn" { + if !doctorHealthDetails(details) { + return errors.New("doctor server health details are invalid") + } + return nil + } + if len(details) != 0 && !doctorHealthDetails(details) && !doctorHTTPDetails(details) { + return errors.New("failed doctor server details are invalid") + } + case 2: + if status == "warn" { + return errors.New("doctor authentication cannot warn") + } + if status == "skipped" { + if len(details) != 0 { + return errors.New("skipped doctor authentication check has details") + } + return nil + } + if status == "pass" { + if len(details) != 2 || !safeDoctorValue(details["id"]) || !safeDoctorValue(details["username"]) { + return errors.New("doctor authentication identity is invalid") + } + return nil + } + if len(details) != 0 && !doctorHTTPDetails(details) { + return errors.New("failed doctor authentication details are invalid") + } + } + return nil +} + +func doctorHealthDetails(details map[string]any) bool { + if len(details) != 3 { + return false + } + return safeDoctorValue(details["status"]) && safeDoctorValue(details["databaseStatus"]) && safeDoctorValue(details["filestoreStatus"]) +} + +func doctorHTTPDetails(details map[string]any) bool { + if len(details) != 1 { + return false + } + status, ok := details["httpStatus"].(int) + return ok && status >= 100 && status <= 599 +} + +func doctorSource(value any) bool { + text, ok := doctorString(value) + return ok && (text == "cli" || text == "env" || text == "file" || text == "missing") +} + +func safeDoctorValue(value any) bool { + text, ok := doctorString(value) + return ok && safeDoctorString(text) +} + +func doctorString(value any) (string, bool) { + reflected := reflect.ValueOf(value) + if !reflected.IsValid() || reflected.Kind() != reflect.String { + return "", false + } + return reflected.String(), true +} + +func safeDoctorString(value string) bool { + if value == "" || !utf8.ValidString(value) { + return false + } + for _, current := range value { + if current < 0x20 || (current >= 0x7f && current <= 0x9f) || current == 0x202a || current == 0x202b || current == 0x202c || current == 0x202d || current == 0x202e || current == 0x2066 || current == 0x2067 || current == 0x2068 || current == 0x2069 { + return false + } + } + return true +} + type preflightVisit struct { typeName reflect.Type pointer uintptr diff --git a/internal/output/machine_test.go b/internal/output/machine_test.go index d0a5b8a..c163d61 100644 --- a/internal/output/machine_test.go +++ b/internal/output/machine_test.go @@ -24,6 +24,61 @@ func TestWriteMachineJSONWireContract(t *testing.T) { } } +func TestDoctorEnvelopeConstructorAndPreflightRejectContradictions(t *testing.T) { + valid := []DoctorCheck{ + {Name: "configuration", Status: "pass", Message: "credentials resolved", Details: map[string]any{"urlSource": "cli", "tokenSource": "env"}}, + {Name: "server", Status: "warn", Message: "incomplete health", Details: map[string]any{"status": "OK", "databaseStatus": "OK", "filestoreStatus": "unknown"}}, + {Name: "authentication", Status: "pass", Message: "authenticated", Details: map[string]any{"id": "id", "username": "arda"}}, + } + document, err := NewDoctorEnvelope(true, valid) + if err != nil { + t.Fatal(err) + } + if _, err := WriteMachineJSON(io.Discard, document); err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + ok bool + mutate func([]DoctorCheck) + }{ + {"ok with failure", true, func(checks []DoctorCheck) { checks[1].Status = "fail" }}, + {"false without failure", false, func([]DoctorCheck) {}}, + {"wrong order", true, func(checks []DoctorCheck) { checks[0].Name = "server" }}, + {"invalid status", true, func(checks []DoctorCheck) { checks[1].Status = "healthy" }}, + {"skipped with details", false, func(checks []DoctorCheck) { checks[1].Status = "skipped" }}, + {"pass without health", true, func(checks []DoctorCheck) { checks[1].Details = nil }}, + {"auth pass wrong type", true, func(checks []DoctorCheck) { checks[2].Details["id"] = 42 }}, + {"hostile message", true, func(checks []DoctorCheck) { checks[2].Message = "bad\x1b[2J" }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + checks := cloneDoctorChecks(valid) + test.mutate(checks) + if _, err := NewDoctorEnvelope(test.ok, checks); err == nil { + t.Fatal("constructor accepted contradictory doctor document") + } + writer := &shortMachineWriter{} + if _, err := WriteMachineJSON(writer, DoctorEnvelope{Schema: "mm/v2/doctor", OK: test.ok, Checks: checks}); err == nil || writer.calls != 0 { + t.Fatalf("preflight err=%v writes=%d", err, writer.calls) + } + }) + } +} + +func cloneDoctorChecks(checks []DoctorCheck) []DoctorCheck { + result := make([]DoctorCheck, len(checks)) + for index, check := range checks { + result[index] = check + result[index].Details = make(map[string]any, len(check.Details)) + for key, value := range check.Details { + result[index].Details[key] = value + } + } + return result +} + type shortMachineWriter struct{ calls int } func (w *shortMachineWriter) Write(p []byte) (int, error) { w.calls++; return len(p) - 1, nil } diff --git a/internal/schema/registry_test.go b/internal/schema/registry_test.go index dfdac26..72850c3 100644 --- a/internal/schema/registry_test.go +++ b/internal/schema/registry_test.go @@ -102,3 +102,29 @@ func TestErrorSchemaBindsCodesToExitClassesAndRequiresRecovery(t *testing.T) { } } } + +func TestDoctorSchemaRejectsSemanticContradictions(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + configuration := `{"name":"configuration","status":"pass","message":"credentials resolved","details":{"urlSource":"cli","tokenSource":"env"}}` + health := `{"name":"server","status":"pass","message":"healthy","details":{"status":"OK","databaseStatus":"OK","filestoreStatus":"OK"}}` + auth := `{"name":"authentication","status":"pass","message":"authenticated","details":{"id":"id","username":"arda"}}` + documents := []string{ + `{"schema":"mm/v2/doctor","ok":true,"checks":[` + configuration + `,{"name":"server","status":"fail","message":"failed"},` + auth + `]}`, + `{"schema":"mm/v2/doctor","ok":false,"checks":[` + configuration + `,` + health + `,` + auth + `]}`, + `{"schema":"mm/v2/doctor","ok":true,"checks":[` + configuration + `,{"name":"server","status":"pass","message":"healthy"},` + auth + `]}`, + `{"schema":"mm/v2/doctor","ok":false,"checks":[` + configuration + `,{"name":"server","status":"skipped","message":"missing","details":{"httpStatus":503}},{"name":"authentication","status":"fail","message":"failed"}]}`, + `{"schema":"mm/v2/doctor","ok":true,"checks":[` + configuration + `,` + health + `,{"name":"authentication","status":"pass","message":"authenticated","details":{"httpStatus":200}}]}`, + `{"schema":"mm/v2/doctor","ok":false,"checks":[` + configuration + `,` + health + `,{"name":"authentication","status":"fail","message":"failed","details":{"id":"id","username":"arda"}}]}`, + `{"schema":"mm/v2/doctor","ok":true,"checks":[{"name":"configuration","status":"pass","message":"bad\u001bvalue","details":{"urlSource":"cli","tokenSource":"env"}},` + health + `,` + auth + `]}`, + `{"schema":"mm/v2/doctor","ok":true,"checks":[` + configuration + `,{"name":"server","status":"pass","message":"healthy","details":{"status":"OK","databaseStatus":"bad\u202evalue","filestoreStatus":"OK"}},` + auth + `]}`, + `{"schema":"mm/v2/doctor","ok":true,"checks":[` + configuration + `,` + health + `,{"name":"authentication","status":"pass","message":"authenticated","details":{"id":"id","username":"bad\u0085value"}}]}`, + } + for _, document := range documents { + if err := registry.Validate("mm/v2/doctor", strings.NewReader(document)); err == nil { + t.Fatalf("doctor schema accepted contradiction: %s", document) + } + } +} diff --git a/schemas/v2/doctor.schema.json b/schemas/v2/doctor.schema.json new file mode 100644 index 0000000..a42eb96 --- /dev/null +++ b/schemas/v2/doctor.schema.json @@ -0,0 +1,152 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:doctor", + "title": "mm v2 doctor report", + "type": "object", + "additionalProperties": false, + "required": ["schema", "ok", "checks"], + "properties": { + "schema": { "const": "mm/v2/doctor" }, + "ok": { "type": "boolean" }, + "checks": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "prefixItems": [ + { "$ref": "#/$defs/configuration" }, + { "$ref": "#/$defs/server" }, + { "$ref": "#/$defs/authentication" } + ], + "items": false + } + }, + "allOf": [ + { + "if": { "properties": { "ok": { "const": true } }, "required": ["ok"] }, + "then": { "properties": { "checks": { "not": { "contains": { "properties": { "status": { "const": "fail" } }, "required": ["status"] } } } } } + }, + { + "if": { "properties": { "ok": { "const": false } }, "required": ["ok"] }, + "then": { "properties": { "checks": { "contains": { "properties": { "status": { "const": "fail" } }, "required": ["status"] }, "minContains": 1 } } } + } + ], + "$defs": { + "base": { + "type": "object", + "required": ["name", "status", "message"], + "properties": { + "name": { "type": "string" }, + "status": { "enum": ["pass", "warn", "fail", "skipped"] }, + "message": { "$ref": "#/$defs/safeString" } + } + }, + "configuration": { + "allOf": [ + { "$ref": "#/$defs/base" }, + { + "type": "object", + "additionalProperties": false, + "required": ["name", "status", "message", "details"], + "properties": { + "name": { "const": "configuration" }, + "status": { "enum": ["pass", "warn", "fail"] }, + "message": { "$ref": "#/$defs/safeString" }, + "details": { + "type": "object", + "additionalProperties": false, + "required": ["urlSource", "tokenSource"], + "properties": { + "urlSource": { "enum": ["cli", "env", "file", "missing"] }, + "tokenSource": { "enum": ["cli", "env", "file", "missing"] } + } + } + } + } + ] + }, + "server": { + "allOf": [ + { "$ref": "#/$defs/base" }, + { + "type": "object", + "additionalProperties": false, + "required": ["name", "status", "message"], + "properties": { + "name": { "const": "server" }, + "status": { "enum": ["pass", "warn", "fail", "skipped"] }, + "message": { "$ref": "#/$defs/safeString" }, + "details": { "$ref": "#/$defs/serverDetails" } + }, + "allOf": [ + { + "if": { "properties": { "status": { "enum": ["pass", "warn"] } }, "required": ["status"] }, + "then": { "required": ["details"], "properties": { "details": { "$ref": "#/$defs/health" } } } + }, + { + "if": { "properties": { "status": { "const": "skipped" } }, "required": ["status"] }, + "then": { "not": { "required": ["details"] } } + } + ] + } + ] + }, + "serverDetails": { + "oneOf": [ + { "$ref": "#/$defs/health" }, + { "$ref": "#/$defs/httpStatus" } + ] + }, + "health": { + "type": "object", "additionalProperties": false, + "required": ["status", "databaseStatus", "filestoreStatus"], + "properties": { "status": { "$ref": "#/$defs/safeString" }, "databaseStatus": { "$ref": "#/$defs/safeString" }, "filestoreStatus": { "$ref": "#/$defs/safeString" } } + }, + "authentication": { + "allOf": [ + { "$ref": "#/$defs/base" }, + { + "type": "object", + "additionalProperties": false, + "required": ["name", "status", "message"], + "properties": { + "name": { "const": "authentication" }, + "status": { "enum": ["pass", "fail", "skipped"] }, + "message": { "$ref": "#/$defs/safeString" }, + "details": { + "oneOf": [ + { "type": "object", "additionalProperties": false, "required": ["id", "username"], "properties": { "id": { "$ref": "#/$defs/safeString" }, "username": { "$ref": "#/$defs/safeString" } } }, + { "$ref": "#/$defs/httpStatus" } + ] + } + }, + "allOf": [ + { + "if": { "properties": { "status": { "const": "pass" } }, "required": ["status"] }, + "then": { + "required": ["details"], + "properties": { "details": { "type": "object", "additionalProperties": false, "required": ["id", "username"], "properties": { "id": { "$ref": "#/$defs/safeString" }, "username": { "$ref": "#/$defs/safeString" } } } } + } + }, + { + "if": { "properties": { "status": { "const": "skipped" } }, "required": ["status"] }, + "then": { "not": { "required": ["details"] } } + }, + { + "if": { "properties": { "status": { "const": "fail" } }, "required": ["status"] }, + "then": { "properties": { "details": { "$ref": "#/$defs/httpStatus" } } } + } + ] + } + ] + }, + "httpStatus": { + "type": "object", "additionalProperties": false, + "required": ["httpStatus"], "properties": { "httpStatus": { "type": "integer", "minimum": 100, "maximum": 599 } } + }, + "safeString": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\x{0000}-\\x{001F}\\x{007F}-\\x{009F}\\x{202A}-\\x{202E}\\x{2066}-\\x{2069}]*$" + } + } +} diff --git a/schemas/v2/examples/doctor.json b/schemas/v2/examples/doctor.json new file mode 100644 index 0000000..d81190e --- /dev/null +++ b/schemas/v2/examples/doctor.json @@ -0,0 +1 @@ +{"schema":"mm/v2/doctor","ok":true,"checks":[{"name":"configuration","status":"pass","message":"credentials resolved","details":{"tokenSource":"env","urlSource":"cli"}},{"name":"server","status":"pass","message":"server is healthy","details":{"databaseStatus":"OK","filestoreStatus":"OK","status":"OK"}},{"name":"authentication","status":"pass","message":"authenticated","details":{"id":"user-id","username":"arda"}}]} From d65fc48f160121e8435d43e37b51a22cdf95341a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 19:09:00 +0300 Subject: [PATCH 041/119] feat: add identity list schemas --- internal/cli/root_test.go | 2 +- internal/schema/identity_test.go | 101 ++++++++++++++++++++++++++++++ schemas/v2/channels.schema.json | 59 +++++++++++++++++ schemas/v2/examples/channels.json | 1 + schemas/v2/examples/teams.json | 1 + schemas/v2/examples/users.json | 1 + schemas/v2/examples/whoami.json | 1 + schemas/v2/teams.schema.json | 31 +++++++++ schemas/v2/users.schema.json | 44 +++++++++++++ schemas/v2/whoami.schema.json | 32 ++++++++++ 10 files changed, 272 insertions(+), 1 deletion(-) create mode 100644 internal/schema/identity_test.go create mode 100644 schemas/v2/channels.schema.json create mode 100644 schemas/v2/examples/channels.json create mode 100644 schemas/v2/examples/teams.json create mode 100644 schemas/v2/examples/users.json create mode 100644 schemas/v2/examples/whoami.json create mode 100644 schemas/v2/teams.schema.json create mode 100644 schemas/v2/users.schema.json create mode 100644 schemas/v2/whoami.schema.json diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index 28bf2b6..65cfe80 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -57,7 +57,7 @@ func TestSchemaList(t *testing.T) { if code != 0 { t.Fatalf("exit code = %d, want 0; stderr = %q", code, stderr.String()) } - if got, want := stdout.String(), "mm/v2/channel\nmm/v2/config\nmm/v2/dms\nmm/v2/doctor\nmm/v2/error\nmm/v2/group-dms\nmm/v2/mentions\nmm/v2/search\nmm/v2/thread\n"; got != want { + if got, want := stdout.String(), "mm/v2/channel\nmm/v2/channels\nmm/v2/config\nmm/v2/dms\nmm/v2/doctor\nmm/v2/error\nmm/v2/group-dms\nmm/v2/mentions\nmm/v2/search\nmm/v2/teams\nmm/v2/thread\nmm/v2/users\nmm/v2/whoami\n"; got != want { t.Fatalf("stdout = %q, want %q", got, want) } } diff --git a/internal/schema/identity_test.go b/internal/schema/identity_test.go new file mode 100644 index 0000000..860651b --- /dev/null +++ b/internal/schema/identity_test.go @@ -0,0 +1,101 @@ +package schema + +import ( + "strings" + "testing" +) + +func TestIdentitySchemasRejectMissingAndContradictoryFields(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + tests := []struct { + name string + schemaID string + document string + }{ + { + name: "whoami optional field must be explicit", + schemaID: "mm/v2/whoami", + document: `{"schema":"mm/v2/whoami","data":{"id":"u","username":"arda","nickname":null,"roles":[]}}`, + }, + { + name: "whoami roles are strings", + schemaID: "mm/v2/whoami", + document: `{"schema":"mm/v2/whoami","data":{"id":"u","username":"arda","displayName":null,"nickname":null,"roles":[7]}}`, + }, + { + name: "whoami absent optional value is null", + schemaID: "mm/v2/whoami", + document: `{"schema":"mm/v2/whoami","data":{"id":"u","username":"arda","displayName":"","nickname":null,"roles":[]}}`, + }, + { + name: "team display name must be explicit", + schemaID: "mm/v2/teams", + document: `{"schema":"mm/v2/teams","teams":[{"id":"t","name":"core","type":"open"}]}`, + }, + { + name: "unknown team type", + schemaID: "mm/v2/teams", + document: `{"schema":"mm/v2/teams","teams":[{"id":"t","name":"core","displayName":null,"type":"private"}]}`, + }, + { + name: "team absent display name is null", + schemaID: "mm/v2/teams", + document: `{"schema":"mm/v2/teams","teams":[{"id":"t","name":"core","displayName":"","type":"open"}]}`, + }, + { + name: "users require retrieval", + schemaID: "mm/v2/users", + document: `{"schema":"mm/v2/users","users":[]}`, + }, + { + name: "users preserve unknown truncation only as null", + schemaID: "mm/v2/users", + document: `{"schema":"mm/v2/users","users":[],"retrieval":{"selectedCount":0,"requestedLimit":20,"query":null,"teamId":null,"truncated":"unknown"}}`, + }, + { + name: "users absent query is null", + schemaID: "mm/v2/users", + document: `{"schema":"mm/v2/users","users":[],"retrieval":{"selectedCount":0,"requestedLimit":20,"query":"","teamId":null,"truncated":false}}`, + }, + { + name: "direct channel cannot carry a team", + schemaID: "mm/v2/channels", + document: `{"schema":"mm/v2/channels","channels":[{"id":"c","type":"dm","name":"@arda","displayName":null,"team":{"id":"t","name":"core","displayName":null},"lastPost":null,"messageCount":0}]}`, + }, + { + name: "direct channel has no separate display name", + schemaID: "mm/v2/channels", + document: `{"schema":"mm/v2/channels","channels":[{"id":"c","type":"dm","name":"@arda","displayName":"Arda","team":null,"lastPost":null,"messageCount":0}]}`, + }, + { + name: "public channel requires proven team", + schemaID: "mm/v2/channels", + document: `{"schema":"mm/v2/channels","channels":[{"id":"c","type":"public","name":"town-square","displayName":null,"team":null,"lastPost":null,"messageCount":0}]}`, + }, + { + name: "channel absent display name is null", + schemaID: "mm/v2/channels", + document: `{"schema":"mm/v2/channels","channels":[{"id":"c","type":"public","name":"town-square","displayName":"","team":{"id":"t","name":"core","displayName":null},"lastPost":null,"messageCount":0}]}`, + }, + { + name: "last post is canonical UTC milliseconds", + schemaID: "mm/v2/channels", + document: `{"schema":"mm/v2/channels","channels":[{"id":"c","type":"group","name":"group","displayName":null,"team":null,"lastPost":"2026-07-16T10:20:30Z","messageCount":0}]}`, + }, + { + name: "message count is nonnegative", + schemaID: "mm/v2/channels", + document: `{"schema":"mm/v2/channels","channels":[{"id":"c","type":"group","name":"group","displayName":null,"team":null,"lastPost":null,"messageCount":-1}]}`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if err := registry.Validate(test.schemaID, strings.NewReader(test.document)); err == nil { + t.Fatalf("accepted contradictory %s document: %s", test.schemaID, test.document) + } + }) + } +} diff --git a/schemas/v2/channels.schema.json b/schemas/v2/channels.schema.json new file mode 100644 index 0000000..5e172f2 --- /dev/null +++ b/schemas/v2/channels.schema.json @@ -0,0 +1,59 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:channels", + "title": "mm v2 account channel directory", + "type": "object", + "additionalProperties": false, + "required": ["schema", "channels"], + "properties": { + "schema": { "const": "mm/v2/channels" }, + "channels": { + "type": "array", + "items": { "$ref": "#/$defs/channel" } + } + }, + "$defs": { + "timestamp": { + "type": "string", + "format": "date-time", + "pattern": "^(?:000[1-9]|00[1-9][0-9]|0[1-9][0-9]{2}|[1-9][0-9]{3})-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\\.[0-9]{3}Z$" + }, + "team": { + "type": "object", + "additionalProperties": false, + "required": ["id", "name", "displayName"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "name": { "type": "string", "minLength": 1 }, + "displayName": { "$ref": "#/$defs/nullableNonEmptyString" } + } + }, + "channel": { + "type": "object", + "additionalProperties": false, + "required": ["id", "type", "name", "displayName", "team", "lastPost", "messageCount"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "type": { "enum": ["dm", "public", "private", "group"] }, + "name": { "type": "string", "minLength": 1 }, + "displayName": { "$ref": "#/$defs/nullableNonEmptyString" }, + "team": { "anyOf": [{ "$ref": "#/$defs/team" }, { "type": "null" }] }, + "lastPost": { "anyOf": [{ "$ref": "#/$defs/timestamp" }, { "type": "null" }] }, + "messageCount": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 } + }, + "allOf": [ + { + "if": { "properties": { "type": { "enum": ["public", "private"] } }, "required": ["type"] }, + "then": { "properties": { "team": { "$ref": "#/$defs/team" } } } + }, + { + "if": { "properties": { "type": { "enum": ["dm", "group"] } }, "required": ["type"] }, + "then": { "properties": { "team": { "type": "null" }, "displayName": { "type": "null" } } } + } + ] + }, + "nullableNonEmptyString": { + "anyOf": [{ "type": "string", "minLength": 1 }, { "type": "null" }] + } + } +} diff --git a/schemas/v2/examples/channels.json b/schemas/v2/examples/channels.json new file mode 100644 index 0000000..a7f6157 --- /dev/null +++ b/schemas/v2/examples/channels.json @@ -0,0 +1 @@ +{"schema":"mm/v2/channels","channels":[{"id":"channel-id","type":"public","name":"town-square","displayName":"Town Square","team":{"id":"team-id","name":"core","displayName":"Core"},"lastPost":"2026-07-16T10:20:30.000Z","messageCount":42}]} diff --git a/schemas/v2/examples/teams.json b/schemas/v2/examples/teams.json new file mode 100644 index 0000000..287b6c0 --- /dev/null +++ b/schemas/v2/examples/teams.json @@ -0,0 +1 @@ +{"schema":"mm/v2/teams","teams":[{"id":"team-id","name":"core","displayName":"Core","type":"open"}]} diff --git a/schemas/v2/examples/users.json b/schemas/v2/examples/users.json new file mode 100644 index 0000000..8df5276 --- /dev/null +++ b/schemas/v2/examples/users.json @@ -0,0 +1 @@ +{"schema":"mm/v2/users","users":[{"id":"user-id","username":"alice","displayName":"Alice A","nickname":null}],"retrieval":{"selectedCount":1,"requestedLimit":20,"query":null,"teamId":null,"truncated":false}} diff --git a/schemas/v2/examples/whoami.json b/schemas/v2/examples/whoami.json new file mode 100644 index 0000000..fa62439 --- /dev/null +++ b/schemas/v2/examples/whoami.json @@ -0,0 +1 @@ +{"schema":"mm/v2/whoami","data":{"id":"user-id","username":"arda","displayName":"Arda Sevinc","nickname":null,"roles":["system_user","team_user"]}} diff --git a/schemas/v2/teams.schema.json b/schemas/v2/teams.schema.json new file mode 100644 index 0000000..521c4bd --- /dev/null +++ b/schemas/v2/teams.schema.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:teams", + "title": "mm v2 team membership", + "type": "object", + "additionalProperties": false, + "required": ["schema", "teams"], + "properties": { + "schema": { "const": "mm/v2/teams" }, + "teams": { + "type": "array", + "items": { "$ref": "#/$defs/team" } + } + }, + "$defs": { + "team": { + "type": "object", + "additionalProperties": false, + "required": ["id", "name", "displayName", "type"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "name": { "type": "string", "minLength": 1 }, + "displayName": { "$ref": "#/$defs/nullableNonEmptyString" }, + "type": { "enum": ["open", "invite_only"] } + } + }, + "nullableNonEmptyString": { + "anyOf": [{ "type": "string", "minLength": 1 }, { "type": "null" }] + } + } +} diff --git a/schemas/v2/users.schema.json b/schemas/v2/users.schema.json new file mode 100644 index 0000000..2084092 --- /dev/null +++ b/schemas/v2/users.schema.json @@ -0,0 +1,44 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:users", + "title": "mm v2 user directory", + "type": "object", + "additionalProperties": false, + "required": ["schema", "users", "retrieval"], + "properties": { + "schema": { "const": "mm/v2/users" }, + "users": { + "type": "array", + "items": { "$ref": "#/$defs/user" } + }, + "retrieval": { "$ref": "#/$defs/retrieval" } + }, + "$defs": { + "user": { + "type": "object", + "additionalProperties": false, + "required": ["id", "username", "displayName", "nickname"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "username": { "type": "string", "minLength": 1 }, + "displayName": { "$ref": "#/$defs/nullableNonEmptyString" }, + "nickname": { "$ref": "#/$defs/nullableNonEmptyString" } + } + }, + "retrieval": { + "type": "object", + "additionalProperties": false, + "required": ["selectedCount", "requestedLimit", "query", "teamId", "truncated"], + "properties": { + "selectedCount": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, + "requestedLimit": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, + "query": { "$ref": "#/$defs/nullableNonEmptyString" }, + "teamId": { "$ref": "#/$defs/nullableNonEmptyString" }, + "truncated": { "type": ["boolean", "null"] } + } + }, + "nullableNonEmptyString": { + "anyOf": [{ "type": "string", "minLength": 1 }, { "type": "null" }] + } + } +} diff --git a/schemas/v2/whoami.schema.json b/schemas/v2/whoami.schema.json new file mode 100644 index 0000000..3aeb1ed --- /dev/null +++ b/schemas/v2/whoami.schema.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:whoami", + "title": "mm v2 authenticated identity", + "type": "object", + "additionalProperties": false, + "required": ["schema", "data"], + "properties": { + "schema": { "const": "mm/v2/whoami" }, + "data": { "$ref": "#/$defs/identity" } + }, + "$defs": { + "identity": { + "type": "object", + "additionalProperties": false, + "required": ["id", "username", "displayName", "nickname", "roles"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "username": { "type": "string", "minLength": 1 }, + "displayName": { "$ref": "#/$defs/nullableNonEmptyString" }, + "nickname": { "$ref": "#/$defs/nullableNonEmptyString" }, + "roles": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + } + } + }, + "nullableNonEmptyString": { + "anyOf": [{ "type": "string", "minLength": 1 }, { "type": "null" }] + } + } +} From 81bb31c49f3b8f0bbca0ccaf66cdb25c0571274d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 19:09:29 +0300 Subject: [PATCH 042/119] feat: add bounded unread retrieval --- internal/retrieval/unread.go | 252 ++++++++++++++++++++++++ internal/retrieval/unread_test.go | 310 ++++++++++++++++++++++++++++++ 2 files changed, 562 insertions(+) create mode 100644 internal/retrieval/unread.go create mode 100644 internal/retrieval/unread_test.go diff --git a/internal/retrieval/unread.go b/internal/retrieval/unread.go new file mode 100644 index 0000000..72267f7 --- /dev/null +++ b/internal/retrieval/unread.go @@ -0,0 +1,252 @@ +package retrieval + +import ( + "context" + "errors" + "sort" + "strings" + "sync" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" +) + +const ( + MaxUnreadMembershipFallbacks = 100 + MaxUnreadPeekRequests = 100 + MaxUnreadPeekConcurrency = 8 +) + +var ( + ErrInvalidUnreadRequest = errors.New("invalid unread request") + ErrIncompleteUnreadResult = errors.New("Mattermost did not provide a complete unread result") +) + +type currentUserSource interface { + Current(context.Context) (mattermost.User, error) +} + +type teamResolveSource interface { + Resolve(context.Context, string, string) (mattermost.Team, error) +} + +type unreadChannelSource interface { + ListForUnread(context.Context, string) ([]mattermost.Channel, error) + TeamMembers(context.Context, string, string) ([]mattermost.UnreadMember, error) + UnreadMember(context.Context, string, string) (mattermost.UnreadMember, error) +} + +type UnreadOptions struct { + TeamSelector string + PeekLimit int + PeekRequestBudget int + PeekConcurrency int +} + +type UnreadEntry struct { + Channel mattermost.Channel + UnreadCount int64 + MentionCount int64 + LastViewedAt int64 + Peek []mattermost.Post + PeekState Completeness +} + +type UnreadResult struct { + User mattermost.User + Team mattermost.Team + Entries []UnreadEntry +} + +// Unread builds one complete unread snapshot. Team membership metrics are the +// primary bounded source; only D/G channels absent from that team-scoped +// snapshot use exact per-channel membership reads. +func Unread( + ctx context.Context, + users currentUserSource, + teams teamResolveSource, + channels unreadChannelSource, + posts channelPageSource, + options UnreadOptions, +) (UnreadResult, error) { + if users == nil || teams == nil || channels == nil || !validUnreadOptions(options, posts) { + return UnreadResult{}, ErrInvalidUnreadRequest + } + if err := ctx.Err(); err != nil { + return UnreadResult{}, err + } + user, err := users.Current(ctx) + if err != nil { + return UnreadResult{}, err + } + team, err := teams.Resolve(ctx, user.ID, options.TeamSelector) + if err != nil { + return UnreadResult{}, err + } + allChannels, err := channels.ListForUnread(ctx, user.ID) + if err != nil { + return UnreadResult{}, err + } + candidates := make([]mattermost.Channel, 0, len(allChannels)) + for _, channel := range allChannels { + if channel.Type == "D" || channel.Type == "G" || ((channel.Type == "O" || channel.Type == "P") && channel.TeamID == team.ID) { + candidates = append(candidates, channel) + } + } + members, err := channels.TeamMembers(ctx, user.ID, team.ID) + if err != nil { + return UnreadResult{}, err + } + memberByChannel := make(map[string]mattermost.UnreadMember, len(members)) + for _, member := range members { + if member.UserID != user.ID || strings.TrimSpace(member.ChannelID) != member.ChannelID || member.ChannelID == "" || + member.MsgCount < 0 || member.MentionCount < 0 || member.LastViewedAt < 0 { + return UnreadResult{}, ErrIncompleteUnreadResult + } + if _, duplicate := memberByChannel[member.ChannelID]; duplicate { + return UnreadResult{}, ErrIncompleteUnreadResult + } + memberByChannel[member.ChannelID] = member + } + missingDirect := 0 + for _, channel := range candidates { + if _, ok := memberByChannel[channel.ID]; !ok && (channel.Type == "D" || channel.Type == "G") { + missingDirect++ + } + } + if missingDirect > MaxUnreadMembershipFallbacks { + return UnreadResult{}, ErrIncompleteUnreadResult + } + + entries := make([]UnreadEntry, 0, len(candidates)) + for _, channel := range candidates { + if err := ctx.Err(); err != nil { + return UnreadResult{}, err + } + member, ok := memberByChannel[channel.ID] + if !ok && (channel.Type == "D" || channel.Type == "G") { + member, err = channels.UnreadMember(ctx, channel.ID, user.ID) + if err != nil { + return UnreadResult{}, err + } + if member.ChannelID != channel.ID || member.UserID != user.ID || member.MsgCount < 0 || member.MentionCount < 0 || member.LastViewedAt < 0 { + return UnreadResult{}, ErrIncompleteUnreadResult + } + ok = true + } + if !ok { + return UnreadResult{}, ErrIncompleteUnreadResult + } + unreadCount := int64(0) + if channel.TotalMsgCount > member.MsgCount { + unreadCount = channel.TotalMsgCount - member.MsgCount + } + if unreadCount == 0 { + continue + } + entries = append(entries, UnreadEntry{ + Channel: channel, UnreadCount: unreadCount, MentionCount: member.MentionCount, + LastViewedAt: member.LastViewedAt, Peek: []mattermost.Post{}, PeekState: CompletenessComplete, + }) + } + sortUnreadEntries(entries) + if options.PeekLimit > 0 && len(entries) > 0 { + if err := addUnreadPeek(ctx, posts, entries, options); err != nil { + return UnreadResult{}, err + } + } + return UnreadResult{User: user, Team: team, Entries: entries}, nil +} + +func validUnreadOptions(options UnreadOptions, posts channelPageSource) bool { + if strings.TrimSpace(options.TeamSelector) != options.TeamSelector || options.PeekLimit < 0 || int64(options.PeekLimit) > maxSafeInteger || + options.PeekRequestBudget < 0 || options.PeekRequestBudget > MaxUnreadPeekRequests || + options.PeekConcurrency < 0 || options.PeekConcurrency > MaxUnreadPeekConcurrency { + return false + } + return options.PeekLimit == 0 || posts != nil +} + +func sortUnreadEntries(entries []UnreadEntry) { + sort.Slice(entries, func(i, j int) bool { + if entries[i].MentionCount != entries[j].MentionCount { + return entries[i].MentionCount > entries[j].MentionCount + } + if entries[i].UnreadCount != entries[j].UnreadCount { + return entries[i].UnreadCount > entries[j].UnreadCount + } + return entries[i].Channel.ID < entries[j].Channel.ID + }) +} + +func addUnreadPeek(ctx context.Context, posts channelPageSource, entries []UnreadEntry, options UnreadOptions) error { + budget := options.PeekRequestBudget + if budget == 0 { + budget = MaxUnreadPeekRequests + } + if len(entries) > budget { + return ErrIncompleteUnreadResult + } + concurrency := options.PeekConcurrency + if concurrency == 0 { + concurrency = MaxUnreadPeekConcurrency + } + if concurrency > len(entries) { + concurrency = len(entries) + } + peekCtx, cancel := context.WithCancel(ctx) + defer cancel() + jobs := make(chan int) + var wg sync.WaitGroup + var firstErr error + var failOnce sync.Once + fail := func(err error) { + if err == nil { + return + } + failOnce.Do(func() { + firstErr = err + cancel() + }) + } + for range concurrency { + wg.Add(1) + go func() { + defer wg.Done() + for index := range jobs { + requestBudget := 1 + since := entries[index].LastViewedAt + result, err := ChannelHistory(peekCtx, posts, entries[index].Channel.ID, ChannelHistoryOptions{ + Limit: options.PeekLimit, Since: &since, RequestBudget: &requestBudget, + }) + if err != nil { + fail(err) + return + } + if result.Completeness == CompletenessUnknown { + fail(ErrIncompleteUnreadResult) + return + } + entries[index].Peek = result.Posts + entries[index].PeekState = result.Completeness + } + }() + } + +launch: + for i := range entries { + select { + case jobs <- i: + case <-peekCtx.Done(): + break launch + } + } + close(jobs) + wg.Wait() + if firstErr != nil { + return firstErr + } + if err := ctx.Err(); err != nil { + return err + } + return nil +} diff --git a/internal/retrieval/unread_test.go b/internal/retrieval/unread_test.go new file mode 100644 index 0000000..bfb815d --- /dev/null +++ b/internal/retrieval/unread_test.go @@ -0,0 +1,310 @@ +package retrieval + +import ( + "context" + "errors" + "reflect" + "sync" + "sync/atomic" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" +) + +type unreadUsersFake struct { + user mattermost.User + err error +} + +func (f unreadUsersFake) Current(context.Context) (mattermost.User, error) { return f.user, f.err } + +type unreadTeamsFake struct { + team mattermost.Team + userID string + selectTeam string +} + +func (f *unreadTeamsFake) Resolve(_ context.Context, userID, selector string) (mattermost.Team, error) { + f.userID, f.selectTeam = userID, selector + return f.team, nil +} + +type unreadChannelsFake struct { + channels []mattermost.Channel + members []mattermost.UnreadMember + fallback map[string]mattermost.UnreadMember + mu sync.Mutex + calls []string +} + +func (f *unreadChannelsFake) ListForUnread(ctx context.Context, userID string) ([]mattermost.Channel, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, "list:"+userID) + return append([]mattermost.Channel(nil), f.channels...), nil +} +func (f *unreadChannelsFake) TeamMembers(ctx context.Context, userID, teamID string) ([]mattermost.UnreadMember, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, "team:"+userID+":"+teamID) + return append([]mattermost.UnreadMember(nil), f.members...), nil +} +func (f *unreadChannelsFake) UnreadMember(ctx context.Context, channelID, userID string) (mattermost.UnreadMember, error) { + if err := ctx.Err(); err != nil { + return mattermost.UnreadMember{}, err + } + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, "fallback:"+channelID+":"+userID) + member, ok := f.fallback[channelID] + if !ok { + return mattermost.UnreadMember{}, ErrIncompleteUnreadResult + } + return member, nil +} + +type unreadPostsFake struct { + mu sync.Mutex + pages map[string]mattermost.OrderedPostsPage + calls int + active int + maxActive int +} + +type blockingUnreadPosts struct { + started chan string + exited atomic.Int32 + calls atomic.Int32 + failID string + failErr error + barrier chan struct{} + once sync.Once +} + +func (f *blockingUnreadPosts) ChannelPage(ctx context.Context, channelID string, _ mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) { + f.calls.Add(1) + f.started <- channelID + if f.barrier != nil { + if f.calls.Load() >= 2 { + f.once.Do(func() { close(f.barrier) }) + } + select { + case <-f.barrier: + case <-ctx.Done(): + f.exited.Add(1) + return mattermost.OrderedPostsPage{}, ctx.Err() + } + } + if channelID == f.failID { + f.exited.Add(1) + return mattermost.OrderedPostsPage{}, f.failErr + } + <-ctx.Done() + f.exited.Add(1) + return mattermost.OrderedPostsPage{}, ctx.Err() +} + +func (f *unreadPostsFake) ChannelPage(ctx context.Context, channelID string, _ mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) { + if err := ctx.Err(); err != nil { + return mattermost.OrderedPostsPage{}, err + } + f.mu.Lock() + f.calls++ + f.active++ + if f.active > f.maxActive { + f.maxActive = f.active + } + page, ok := f.pages[channelID] + f.active-- + f.mu.Unlock() + if !ok { + return mattermost.OrderedPostsPage{}, errors.New("missing page") + } + return page, nil +} + +func unreadFixture() (unreadUsersFake, *unreadTeamsFake, *unreadChannelsFake) { + users := unreadUsersFake{user: mattermost.User{ID: "user", Username: "arda"}} + teams := &unreadTeamsFake{team: mattermost.Team{ID: "team", Name: "core", Type: "O"}} + channels := &unreadChannelsFake{ + channels: []mattermost.Channel{ + {ID: "z", TeamID: "team", Type: "O", TotalMsgCount: 8}, + {ID: "a", TeamID: "team", Type: "P", TotalMsgCount: 8}, + {ID: "dm", Type: "D", TotalMsgCount: 4}, + {ID: "other", TeamID: "other-team", Type: "O", TotalMsgCount: 99}, + }, + members: []mattermost.UnreadMember{ + {ChannelID: "z", UserID: "user", MsgCount: 6, MentionCount: 1, LastViewedAt: 10}, + {ChannelID: "a", UserID: "user", MsgCount: 6, MentionCount: 1, LastViewedAt: 11}, + }, + fallback: map[string]mattermost.UnreadMember{"dm": {ChannelID: "dm", UserID: "user", MsgCount: 1, MentionCount: 2, LastViewedAt: 12}}, + } + return users, teams, channels +} + +func TestUnreadResolvesExactScopeFallsBackOnlyForDirectAndSortsDeterministically(t *testing.T) { + users, teams, channels := unreadFixture() + got, err := Unread(context.Background(), users, teams, channels, nil, UnreadOptions{TeamSelector: "core"}) + if err != nil { + t.Fatal(err) + } + if teams.userID != "user" || teams.selectTeam != "core" { + t.Fatalf("resolve = %q %q", teams.userID, teams.selectTeam) + } + ids := []string{got.Entries[0].Channel.ID, got.Entries[1].Channel.ID, got.Entries[2].Channel.ID} + if !reflect.DeepEqual(ids, []string{"dm", "a", "z"}) { + t.Fatalf("IDs = %v", ids) + } + if !reflect.DeepEqual(channels.calls, []string{"list:user", "team:user:team", "fallback:dm:user"}) { + t.Fatalf("calls = %v", channels.calls) + } +} + +func TestUnreadHandlesMemberAheadOfTotalAsCaughtUp(t *testing.T) { + users, teams, channels := unreadFixture() + channels.members[0].MsgCount = 99 + channels.channels = channels.channels[:1] + got, err := Unread(context.Background(), users, teams, channels, nil, UnreadOptions{}) + if err != nil || len(got.Entries) != 0 { + t.Fatalf("result = %+v, error = %v", got, err) + } +} + +func TestUnreadRejectsMalformedDuplicateAndMissingMetrics(t *testing.T) { + for name, mutate := range map[string]func(*unreadChannelsFake){ + "duplicate": func(f *unreadChannelsFake) { f.members = append(f.members, f.members[0]) }, + "foreign user": func(f *unreadChannelsFake) { f.members[0].UserID = "other" }, + "negative metric": func(f *unreadChannelsFake) { f.members[0].MentionCount = -1 }, + "missing selected member": func(f *unreadChannelsFake) { f.members = f.members[1:] }, + } { + t.Run(name, func(t *testing.T) { + users, teams, channels := unreadFixture() + mutate(channels) + _, err := Unread(context.Background(), users, teams, channels, nil, UnreadOptions{}) + if !errors.Is(err, ErrIncompleteUnreadResult) { + t.Fatalf("error = %v", err) + } + }) + } +} + +func TestUnreadRejectsMalformedFallbackBinding(t *testing.T) { + users, teams, channels := unreadFixture() + channels.fallback["dm"] = mattermost.UnreadMember{ChannelID: "other", UserID: "user", MsgCount: 1} + _, err := Unread(context.Background(), users, teams, channels, nil, UnreadOptions{}) + if !errors.Is(err, ErrIncompleteUnreadResult) { + t.Fatalf("error = %v", err) + } +} + +func TestUnreadEmptySnapshotIsComplete(t *testing.T) { + users, teams, channels := unreadFixture() + channels.channels, channels.members = []mattermost.Channel{}, []mattermost.UnreadMember{} + got, err := Unread(context.Background(), users, teams, channels, nil, UnreadOptions{}) + if err != nil || got.Entries == nil || len(got.Entries) != 0 { + t.Fatalf("result = %#v, error = %v", got, err) + } +} + +func TestUnreadPeekUsesGlobalBudgetAndFailsClosedOnPartialUnknown(t *testing.T) { + users, teams, channels := unreadFixture() + posts := &unreadPostsFake{pages: map[string]mattermost.OrderedPostsPage{ + "dm": {Posts: []mattermost.Post{{ID: "p1", ChannelID: "dm", CreateAt: 20}}, RawCount: 1, HasNext: dmBoolPointer(false)}, + "a": {Incomplete: true}, + "z": {Posts: []mattermost.Post{{ID: "p2", ChannelID: "z", CreateAt: 20}}, RawCount: 1, HasNext: dmBoolPointer(false)}, + }} + _, err := Unread(context.Background(), users, teams, channels, posts, UnreadOptions{PeekLimit: 1, PeekRequestBudget: 3, PeekConcurrency: 2}) + if !errors.Is(err, ErrIncompleteUnreadResult) { + t.Fatalf("error = %v", err) + } + if posts.calls > 3 || posts.maxActive > 2 { + t.Fatalf("calls=%d maxActive=%d", posts.calls, posts.maxActive) + } + + users, teams, channels = unreadFixture() + posts = &unreadPostsFake{pages: map[string]mattermost.OrderedPostsPage{}} + _, err = Unread(context.Background(), users, teams, channels, posts, UnreadOptions{PeekLimit: 1, PeekRequestBudget: 2}) + if !errors.Is(err, ErrIncompleteUnreadResult) || posts.calls != 0 { + t.Fatalf("error=%v calls=%d", err, posts.calls) + } +} + +func TestUnreadPeekReturnsOnlyCompleteBoundedResults(t *testing.T) { + users, teams, channels := unreadFixture() + channels.channels = channels.channels[:1] + channels.members = channels.members[:1] + posts := &unreadPostsFake{pages: map[string]mattermost.OrderedPostsPage{ + "z": {Posts: []mattermost.Post{{ID: "p", ChannelID: "z", CreateAt: 20}}, RawCount: 1, HasNext: dmBoolPointer(false)}, + }} + got, err := Unread(context.Background(), users, teams, channels, posts, UnreadOptions{PeekLimit: 1, PeekRequestBudget: 1, PeekConcurrency: 1}) + if err != nil || len(got.Entries) != 1 || len(got.Entries[0].Peek) != 1 || got.Entries[0].PeekState != CompletenessComplete { + t.Fatalf("result = %+v, error = %v", got, err) + } +} + +func TestUnreadBoundsMembershipFallbackBeforeFanout(t *testing.T) { + users, teams, channels := unreadFixture() + channels.channels = make([]mattermost.Channel, MaxUnreadMembershipFallbacks+1) + channels.members = []mattermost.UnreadMember{} + for i := range channels.channels { + channels.channels[i] = mattermost.Channel{ID: string(rune('a' + i)), Type: "G", TotalMsgCount: 1} + } + _, err := Unread(context.Background(), users, teams, channels, nil, UnreadOptions{}) + if !errors.Is(err, ErrIncompleteUnreadResult) { + t.Fatalf("error = %v", err) + } + if !reflect.DeepEqual(channels.calls, []string{"list:user", "team:user:team"}) { + t.Fatalf("calls = %v", channels.calls) + } +} + +func TestUnreadCancellationStopsBeforeReads(t *testing.T) { + users, teams, channels := unreadFixture() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := Unread(ctx, users, teams, channels, nil, UnreadOptions{}) + if !errors.Is(err, context.Canceled) || len(channels.calls) != 0 { + t.Fatalf("error=%v calls=%v", err, channels.calls) + } +} + +func TestUnreadPeekCallerCancellationJoinsAllStartedWorkers(t *testing.T) { + users, teams, channels := unreadFixture() + posts := &blockingUnreadPosts{started: make(chan string, 3)} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + _, err := Unread(ctx, users, teams, channels, posts, UnreadOptions{PeekLimit: 1, PeekRequestBudget: 3, PeekConcurrency: 2}) + done <- err + }() + <-posts.started + <-posts.started + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v", err) + } + if got := posts.exited.Load(); got != posts.calls.Load() || got != 2 { + t.Fatalf("calls=%d exited=%d", posts.calls.Load(), got) + } +} + +func TestUnreadPeekFirstFailureCancelsSiblingsAndQueuedWork(t *testing.T) { + users, teams, channels := unreadFixture() + sentinel := errors.New("peek failed") + posts := &blockingUnreadPosts{ + started: make(chan string, 3), failID: "dm", failErr: sentinel, barrier: make(chan struct{}), + } + _, err := Unread(context.Background(), users, teams, channels, posts, UnreadOptions{PeekLimit: 1, PeekRequestBudget: 3, PeekConcurrency: 2}) + if !errors.Is(err, sentinel) { + t.Fatalf("error = %v", err) + } + if calls, exited := posts.calls.Load(), posts.exited.Load(); calls != 2 || exited != calls { + t.Fatalf("calls=%d exited=%d", calls, exited) + } +} From c2d8beb426423e485d3b9e2fdcc7caabe3b4e490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 19:46:12 +0300 Subject: [PATCH 043/119] feat: add identity list output models --- internal/output/identity_machine.go | 444 +++++++++++++++++++++++ internal/output/identity_machine_test.go | 285 +++++++++++++++ internal/output/machine.go | 14 +- 3 files changed, 742 insertions(+), 1 deletion(-) create mode 100644 internal/output/identity_machine.go create mode 100644 internal/output/identity_machine_test.go diff --git a/internal/output/identity_machine.go b/internal/output/identity_machine.go new file mode 100644 index 0000000..d236198 --- /dev/null +++ b/internal/output/identity_machine.go @@ -0,0 +1,444 @@ +package output + +import ( + "errors" + "reflect" + "sort" + "strings" + "time" + "unicode" + + "github.com/ardasevinc/mattermost-cli/internal/presentation" +) + +const MaxSafeMachineInteger int64 = 9007199254740991 + +type RawIdentity struct { + ID, Username, DisplayName, Nickname string + Roles []string +} +type RawTeam struct{ ID, Name, DisplayName, Type string } +type RawUser struct{ ID, Username, DisplayName, Nickname string } +type RawChannel struct { + ID, Type, Name, DisplayName, DirectUsername, TeamID string + Team *RawTeam + LastPostAt, TotalMsgCount int64 +} +type UsersRetrievalProof struct { + RequestedLimit, ProbeCount int64 + Query, TeamID string +} + +type Identity struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName *string `json:"displayName"` + Nickname *string `json:"nickname"` + Roles []string `json:"roles"` +} +type TeamItem struct { + ID string `json:"id"` + Name string `json:"name"` + DisplayName *string `json:"displayName"` + Type string `json:"type"` +} +type UserItem struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName *string `json:"displayName"` + Nickname *string `json:"nickname"` +} +type UsersRetrieval struct { + SelectedCount int64 `json:"selectedCount"` + RequestedLimit int64 `json:"requestedLimit"` + Query *string `json:"query"` + TeamID *string `json:"teamId"` + Truncated *bool `json:"truncated"` +} +type ChannelTeam struct { + ID string `json:"id"` + Name string `json:"name"` + DisplayName *string `json:"displayName"` +} +type ChannelItem struct { + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + DisplayName *string `json:"displayName"` + Team *ChannelTeam `json:"team"` + LastPost *MillisTime `json:"lastPost"` + MessageCount int64 `json:"messageCount"` +} + +type WhoAmIEnvelope struct { + Schema string `json:"schema"` + Data Identity `json:"data"` + proof *WhoAmIEnvelope +} +type TeamsEnvelope struct { + Schema string `json:"schema"` + Teams []TeamItem `json:"teams"` + proof *TeamsEnvelope +} +type UsersEnvelope struct { + Schema string `json:"schema"` + Users []UserItem `json:"users"` + Retrieval UsersRetrieval `json:"retrieval"` + proof *UsersEnvelope +} +type ChannelsEnvelope struct { + Schema string `json:"schema"` + Channels []ChannelItem `json:"channels"` + proof *ChannelsEnvelope +} + +func NewWhoAmIEnvelope(raw RawIdentity, options presentation.Options) (WhoAmIEnvelope, error) { + if !rawRequired(raw.ID) || !rawRequired(raw.Username) { + return WhoAmIEnvelope{}, errors.New("invalid raw identity") + } + roles := make([]string, len(raw.Roles)) + for i, role := range raw.Roles { + if !rawRequired(role) { + return WhoAmIEnvelope{}, errors.New("invalid raw identity role") + } + roles[i] = present(role, options) + } + doc := WhoAmIEnvelope{Schema: "mm/v2/whoami", Data: Identity{ID: present(raw.ID, options), Username: present(raw.Username, options), DisplayName: presentNullable(raw.DisplayName, options), Nickname: presentNullable(raw.Nickname, options), Roles: roles}} + doc.proof = cloneWhoAmI(&doc) + if err := validateIdentityDocument(doc); err != nil { + return WhoAmIEnvelope{}, err + } + return doc, nil +} + +func NewTeamsEnvelope(raw []RawTeam, options presentation.Options) (TeamsEnvelope, error) { + values := append([]RawTeam(nil), raw...) + sort.Slice(values, func(i, j int) bool { + if values[i].Name != values[j].Name { + return values[i].Name < values[j].Name + } + return values[i].ID < values[j].ID + }) + teams := make([]TeamItem, len(values)) + seen := map[string]bool{} + for i, value := range values { + if !rawRequired(value.ID) || !rawRequired(value.Name) || seen[value.ID] { + return TeamsEnvelope{}, errors.New("invalid raw team") + } + seen[value.ID] = true + kind := map[string]string{"O": "open", "I": "invite_only"}[value.Type] + if kind == "" { + return TeamsEnvelope{}, errors.New("invalid raw team type") + } + teams[i] = TeamItem{ID: present(value.ID, options), Name: present(value.Name, options), DisplayName: presentNullable(value.DisplayName, options), Type: kind} + } + doc := TeamsEnvelope{Schema: "mm/v2/teams", Teams: teams} + doc.proof = cloneTeamsEnvelope(&doc) + if err := validateIdentityDocument(doc); err != nil { + return TeamsEnvelope{}, err + } + return doc, nil +} + +func NewUsersEnvelope(raw []RawUser, proof UsersRetrievalProof, options presentation.Options) (UsersEnvelope, error) { + values := append([]RawUser(nil), raw...) + sort.Slice(values, func(i, j int) bool { + if values[i].Username != values[j].Username { + return values[i].Username < values[j].Username + } + return values[i].ID < values[j].ID + }) + users := make([]UserItem, len(values)) + seen := map[string]bool{} + for i, value := range values { + if !rawRequired(value.ID) || !rawRequired(value.Username) || seen[value.ID] { + return UsersEnvelope{}, errors.New("invalid raw user") + } + seen[value.ID] = true + users[i] = UserItem{ID: present(value.ID, options), Username: present(value.Username, options), DisplayName: presentNullable(value.DisplayName, options), Nickname: presentNullable(value.Nickname, options)} + } + retrieval, err := deriveRetrieval(int64(len(users)), proof, options) + if err != nil { + return UsersEnvelope{}, err + } + doc := UsersEnvelope{Schema: "mm/v2/users", Users: users, Retrieval: retrieval} + doc.proof = cloneUsersEnvelope(&doc) + if err := validateIdentityDocument(doc); err != nil { + return UsersEnvelope{}, err + } + return doc, nil +} + +func NewChannelsEnvelope(raw []RawChannel, options presentation.Options) (ChannelsEnvelope, error) { + values := append([]RawChannel(nil), raw...) + sort.Slice(values, func(i, j int) bool { + left, right := normalizedLastPostAt(values[i].LastPostAt), normalizedLastPostAt(values[j].LastPostAt) + if left != right { + return left > right + } + return values[i].ID < values[j].ID + }) + channels := make([]ChannelItem, len(values)) + seen := map[string]bool{} + for i, value := range values { + if !rawRequired(value.ID) || !rawRequired(value.Name) || seen[value.ID] || !safeCount(value.TotalMsgCount) { + return ChannelsEnvelope{}, errors.New("invalid raw channel") + } + seen[value.ID] = true + kind := map[string]string{"D": "dm", "O": "public", "P": "private", "G": "group"}[value.Type] + if kind == "" { + return ChannelsEnvelope{}, errors.New("invalid raw channel type") + } + item := ChannelItem{ID: present(value.ID, options), Type: kind, Name: present(value.Name, options), MessageCount: value.TotalMsgCount} + item.LastPost = presentTimestamp(normalizedLastPostAt(value.LastPostAt)) + if value.Type == "D" { + if !rawRequired(value.DirectUsername) || value.Team != nil || value.TeamID != "" { + return ChannelsEnvelope{}, errors.New("invalid direct channel") + } + item.Name = "@" + present(value.DirectUsername, options) + } else if value.Type == "G" { + if value.Team != nil || value.TeamID != "" { + return ChannelsEnvelope{}, errors.New("invalid group channel") + } + if strings.TrimSpace(value.DisplayName) != "" { + item.Name = present(value.DisplayName, options) + } + } else { + if value.Team == nil || !rawRequired(value.TeamID) || value.TeamID != value.Team.ID || !rawRequired(value.Team.ID) || !rawRequired(value.Team.Name) { + return ChannelsEnvelope{}, errors.New("invalid team channel") + } + item.DisplayName = presentNullable(value.DisplayName, options) + item.Team = &ChannelTeam{ID: present(value.Team.ID, options), Name: present(value.Team.Name, options), DisplayName: presentNullable(value.Team.DisplayName, options)} + } + channels[i] = item + } + doc := ChannelsEnvelope{Schema: "mm/v2/channels", Channels: channels} + doc.proof = cloneChannelsEnvelope(&doc) + if err := validateIdentityDocument(doc); err != nil { + return ChannelsEnvelope{}, err + } + return doc, nil +} + +func deriveRetrieval(selected int64, proof UsersRetrievalProof, options presentation.Options) (UsersRetrieval, error) { + proof.Query = strings.TrimSpace(proof.Query) + ceiling := int64(200) + if proof.Query != "" { + ceiling = 1000 + } + if proof.RequestedLimit < 1 || !safeCount(proof.RequestedLimit) || selected < 0 || selected > proof.RequestedLimit || proof.ProbeCount < selected || proof.ProbeCount > ceiling { + return UsersRetrieval{}, errors.New("invalid retrieval proof") + } + var truncated *bool + falseValue := false + trueValue := true + if proof.RequestedLimit < ceiling && selected == proof.RequestedLimit && proof.ProbeCount == selected+1 { + truncated = &trueValue + } else if proof.RequestedLimit >= ceiling && selected == ceiling && proof.ProbeCount == ceiling { + truncated = nil + } else if proof.ProbeCount == selected { + truncated = &falseValue + } else { + return UsersRetrieval{}, errors.New("contradictory retrieval proof") + } + return UsersRetrieval{SelectedCount: selected, RequestedLimit: proof.RequestedLimit, Query: presentNullable(proof.Query, options), TeamID: presentNullable(proof.TeamID, options), Truncated: truncated}, nil +} + +func validateIdentityDocument(document MachineDocument) error { + switch value := document.(type) { + case WhoAmIEnvelope: + if value.Schema != "mm/v2/whoami" || value.proof == nil || !reflect.DeepEqual(withoutWhoProof(value), withoutWhoProof(*value.proof)) { + return errors.New("invalid whoami document") + } + if !validIdentity(value.Data) { + return errors.New("invalid whoami data") + } + case TeamsEnvelope: + if value.Schema != "mm/v2/teams" || value.Teams == nil || value.proof == nil || !reflect.DeepEqual(withoutTeamsProof(value), withoutTeamsProof(*value.proof)) { + return errors.New("invalid teams document") + } + for _, item := range value.Teams { + if !validTeamItem(item) { + return errors.New("invalid team item") + } + } + case UsersEnvelope: + if value.Schema != "mm/v2/users" || value.Users == nil || value.proof == nil || value.Retrieval.SelectedCount != int64(len(value.Users)) || !reflect.DeepEqual(withoutUsersProof(value), withoutUsersProof(*value.proof)) { + return errors.New("invalid users document") + } + if !validUsersRetrieval(value.Retrieval) { + return errors.New("invalid users retrieval") + } + for _, item := range value.Users { + if !validUserItem(item) { + return errors.New("invalid user item") + } + } + case ChannelsEnvelope: + if value.Schema != "mm/v2/channels" || value.Channels == nil || value.proof == nil || !reflect.DeepEqual(withoutChannelsProof(value), withoutChannelsProof(*value.proof)) { + return errors.New("invalid channels document") + } + for _, item := range value.Channels { + if !validChannelItem(item) { + return errors.New("invalid channel item") + } + } + default: + return errors.New("not an identity document") + } + return nil +} + +func validTeamItem(value TeamItem) bool { + return safeText(value.ID) && safeText(value.Name) && validNullable(value.DisplayName) && (value.Type == "open" || value.Type == "invite_only") +} +func validUserItem(value UserItem) bool { + return safeText(value.ID) && safeText(value.Username) && validNullable(value.DisplayName) && validNullable(value.Nickname) +} +func validUsersRetrieval(value UsersRetrieval) bool { + return safeCount(value.SelectedCount) && value.RequestedLimit >= 1 && safeCount(value.RequestedLimit) && validNullable(value.Query) && validNullable(value.TeamID) +} +func validChannelItem(value ChannelItem) bool { + if !safeText(value.ID) || !safeText(value.Name) || !safeCount(value.MessageCount) || !validNullable(value.DisplayName) || !validMillisPointer(value.LastPost) { + return false + } + switch value.Type { + case "dm", "group": + return value.Team == nil && value.DisplayName == nil + case "public", "private": + return value.Team != nil && safeText(value.Team.ID) && safeText(value.Team.Name) && validNullable(value.Team.DisplayName) + default: + return false + } +} + +func validIdentity(value Identity) bool { + if !safeText(value.ID) || !safeText(value.Username) || !validNullable(value.DisplayName) || !validNullable(value.Nickname) || value.Roles == nil { + return false + } + for _, role := range value.Roles { + if !safeText(role) { + return false + } + } + return true +} +func safeText(value string) bool { return strings.TrimSpace(value) != "" && safeDoctorString(value) } +func validNullable(value *string) bool { return value == nil || safeText(*value) } +func rawRequired(value string) bool { + for _, current := range value { + if !unicode.IsSpace(current) && !rawControlOrBidi(current) { + return true + } + } + return false +} +func rawControlOrBidi(current rune) bool { + // Keep the bidi set aligned with presentation.unsafeControl. Required raw + // identity values need a meaningful rune beyond anything presentation will + // make visible as an unsafe directional control. + return current < 0x20 || (current >= 0x7f && current <= 0x9f) || + current == 0x061c || current == 0x200e || current == 0x200f || + (current >= 0x202a && current <= 0x202e) || (current >= 0x2066 && current <= 0x2069) +} +func safeCount(value int64) bool { return value >= 0 && value <= MaxSafeMachineInteger } +func validMillisPointer(value *MillisTime) bool { + if value == nil { + return true + } + _, offset := value.Zone() + return offset == 0 && value.Year() >= 1 && value.Year() <= 9999 && value.Nanosecond()%int(time.Millisecond) == 0 +} +func present(value string, options presentation.Options) string { + return presentation.SanitizeLabel(presentation.PreprocessWithOptions(value, options).Text) +} +func presentNullable(value string, options presentation.Options) *string { + if strings.TrimSpace(value) == "" { + return nil + } + result := present(value, options) + return &result +} +func presentTimestamp(milliseconds int64) *MillisTime { + if milliseconds <= 0 { + return nil + } + stamp := MillisTime{Time: time.UnixMilli(milliseconds).UTC()} + if !validMillisPointer(&stamp) { + return nil + } + return &stamp +} +func normalizedLastPostAt(milliseconds int64) int64 { + if presentTimestamp(milliseconds) == nil { + return 0 + } + return milliseconds +} + +func cloneString(value *string) *string { + if value == nil { + return nil + } + copy := *value + return © +} +func cloneBool(value *bool) *bool { + if value == nil { + return nil + } + copy := *value + return © +} +func cloneMillis(value *MillisTime) *MillisTime { + if value == nil { + return nil + } + copy := *value + return © +} +func cloneIdentity(value Identity) Identity { + value.DisplayName = cloneString(value.DisplayName) + value.Nickname = cloneString(value.Nickname) + value.Roles = cloneSlice(value.Roles) + return value +} +func cloneWhoAmI(value *WhoAmIEnvelope) *WhoAmIEnvelope { + copy := WhoAmIEnvelope{Schema: value.Schema, Data: cloneIdentity(value.Data)} + return © +} +func withoutWhoProof(value WhoAmIEnvelope) WhoAmIEnvelope { value.proof = nil; return value } +func cloneTeamsEnvelope(value *TeamsEnvelope) *TeamsEnvelope { + copy := TeamsEnvelope{Schema: value.Schema, Teams: cloneSlice(value.Teams)} + for i := range copy.Teams { + copy.Teams[i].DisplayName = cloneString(copy.Teams[i].DisplayName) + } + return © +} +func withoutTeamsProof(value TeamsEnvelope) TeamsEnvelope { value.proof = nil; return value } +func cloneUsersEnvelope(value *UsersEnvelope) *UsersEnvelope { + copy := UsersEnvelope{Schema: value.Schema, Users: cloneSlice(value.Users), Retrieval: value.Retrieval} + for i := range copy.Users { + copy.Users[i].DisplayName = cloneString(copy.Users[i].DisplayName) + copy.Users[i].Nickname = cloneString(copy.Users[i].Nickname) + } + copy.Retrieval.Query = cloneString(copy.Retrieval.Query) + copy.Retrieval.TeamID = cloneString(copy.Retrieval.TeamID) + copy.Retrieval.Truncated = cloneBool(copy.Retrieval.Truncated) + return © +} +func withoutUsersProof(value UsersEnvelope) UsersEnvelope { value.proof = nil; return value } +func cloneChannelsEnvelope(value *ChannelsEnvelope) *ChannelsEnvelope { + copy := ChannelsEnvelope{Schema: value.Schema, Channels: cloneSlice(value.Channels)} + for i := range copy.Channels { + copy.Channels[i].DisplayName = cloneString(copy.Channels[i].DisplayName) + copy.Channels[i].LastPost = cloneMillis(copy.Channels[i].LastPost) + if copy.Channels[i].Team != nil { + team := *copy.Channels[i].Team + team.DisplayName = cloneString(team.DisplayName) + copy.Channels[i].Team = &team + } + } + return © +} +func withoutChannelsProof(value ChannelsEnvelope) ChannelsEnvelope { value.proof = nil; return value } diff --git a/internal/output/identity_machine_test.go b/internal/output/identity_machine_test.go new file mode 100644 index 0000000..0681179 --- /dev/null +++ b/internal/output/identity_machine_test.go @@ -0,0 +1,285 @@ +package output_test + +import ( + "bytes" + "fmt" + "strings" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/output" + "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/internal/schema" +) + +func TestIdentityDocumentsGoldenSchemaAndCredentialPresentation(t *testing.T) { + options := presentation.Options{Credentials: []string{"token-secret"}, DisableHeuristics: true} + values := []struct { + id string + doc output.MachineDocument + want string + }{ + {"mm/v2/whoami", mustWho(t, output.RawIdentity{ID: "u", Username: "arda", Roles: nil}, options), `{"schema":"mm/v2/whoami","data":{"id":"u","username":"arda","displayName":null,"nickname":null,"roles":[]}}`}, + {"mm/v2/teams", mustTeams(t, []output.RawTeam{{ID: "b", Name: "z", Type: "I"}, {ID: "a", Name: "a", DisplayName: "token-secret", Type: "O"}}, options), `{"schema":"mm/v2/teams","teams":[{"id":"a","name":"a","displayName":"[REDACTED:mattermost_credential]","type":"open"},{"id":"b","name":"z","displayName":null,"type":"invite_only"}]}`}, + {"mm/v2/users", mustUsers(t, nil, output.UsersRetrievalProof{RequestedLimit: 20}, options), `{"schema":"mm/v2/users","users":[],"retrieval":{"selectedCount":0,"requestedLimit":20,"query":null,"teamId":null,"truncated":false}}`}, + {"mm/v2/channels", mustChannels(t, []output.RawChannel{{ID: "old", Type: "G", Name: "group", LastPostAt: 0}, {ID: "new", Type: "D", Name: "raw", DirectUsername: "arda", LastPostAt: 1784197230123, TotalMsgCount: 7}}, options), `{"schema":"mm/v2/channels","channels":[{"id":"new","type":"dm","name":"@arda","displayName":null,"team":null,"lastPost":"2026-07-16T10:20:30.123Z","messageCount":7},{"id":"old","type":"group","name":"group","displayName":null,"team":null,"lastPost":null,"messageCount":0}]}`}, + } + registry, err := schema.Load() + if err != nil { + t.Fatal(err) + } + for _, test := range values { + t.Run(test.id, func(t *testing.T) { + var wire bytes.Buffer + if _, err := output.WriteMachineJSON(&wire, test.doc); err != nil { + t.Fatal(err) + } + if got := strings.TrimSuffix(wire.String(), "\n"); got != test.want { + t.Fatalf("got %s\nwant %s", got, test.want) + } + if err := registry.Validate(test.id, bytes.NewReader(wire.Bytes())); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestDirectNilAndMutationFailBeforeWrite(t *testing.T) { + direct := []output.MachineDocument{output.WhoAmIEnvelope{Schema: "mm/v2/whoami"}, output.TeamsEnvelope{Schema: "mm/v2/teams"}, output.UsersEnvelope{Schema: "mm/v2/users"}, output.ChannelsEnvelope{Schema: "mm/v2/channels"}} + for i, doc := range direct { + w := &countingWriter{} + if _, err := output.WriteMachineJSON(w, doc); err == nil || w.calls != 0 { + t.Fatalf("direct %d err=%v writes=%d", i, err, w.calls) + } + } + doc := mustTeams(t, []output.RawTeam{{ID: "t", Name: "core", Type: "O"}}, presentation.Options{}) + doc.Teams = nil + w := &countingWriter{} + if _, err := output.WriteMachineJSON(w, doc); err == nil || w.calls != 0 { + t.Fatalf("mutation err=%v writes=%d", err, w.calls) + } +} + +func TestRawPresentationCannotBeFalselyBoundAndCollisionsRemainDeterministic(t *testing.T) { + options := presentation.Options{Credentials: []string{"secret-a", "secret-b"}, DisableHeuristics: true} + doc, err := output.NewUsersEnvelope([]output.RawUser{{ID: "secret-b", Username: "z"}, {ID: "secret-a", Username: "a"}}, output.UsersRetrievalProof{RequestedLimit: 2, ProbeCount: 2}, options) + if err != nil { + t.Fatal(err) + } + if doc.Users[0].Username != "a" || doc.Users[0].ID != "[REDACTED:mattermost_credential]" || doc.Users[1].ID != doc.Users[0].ID { + t.Fatalf("presentation/order=%+v", doc.Users) + } + doc.Users[0].ID = "unrelated" + w := &countingWriter{} + if _, err := output.WriteMachineJSON(w, doc); err == nil || w.calls != 0 { + t.Fatalf("false binding survived err=%v writes=%d", err, w.calls) + } +} + +func TestDeterministicRawOrdering(t *testing.T) { + teams := mustTeams(t, []output.RawTeam{{ID: "b", Name: "same", Type: "O"}, {ID: "a", Name: "same", Type: "O"}}, presentation.Options{}) + if teams.Teams[0].ID != "a" { + t.Fatalf("teams=%+v", teams.Teams) + } + users := mustUsers(t, []output.RawUser{{ID: "b", Username: "same"}, {ID: "a", Username: "same"}}, output.UsersRetrievalProof{RequestedLimit: 2, ProbeCount: 2}, presentation.Options{}) + if users.Users[0].ID != "a" { + t.Fatalf("users=%+v", users.Users) + } + channels := mustChannels(t, []output.RawChannel{{ID: "z", Type: "G", Name: "z", LastPostAt: 0}, {ID: "b", Type: "G", Name: "b", LastPostAt: 1}, {ID: "a", Type: "G", Name: "a", LastPostAt: 1}}, presentation.Options{}) + if channels.Channels[0].ID != "a" || channels.Channels[1].ID != "b" || channels.Channels[2].ID != "z" { + t.Fatalf("channels=%+v", channels.Channels) + } +} + +func TestUsersRetrievalTruthTable(t *testing.T) { + users := func(n int) []output.RawUser { + result := make([]output.RawUser, n) + for i := range result { + value := fmt.Sprintf("user-%04d", i) + result[i] = output.RawUser{ID: value, Username: value} + } + return result + } + tests := []struct { + name string + n int + proof output.UsersRetrievalProof + want *bool + ok bool + }{{"empty false", 0, output.UsersRetrievalProof{RequestedLimit: 20}, boolp(false), true}, {"proved more", 2, output.UsersRetrievalProof{RequestedLimit: 2, ProbeCount: 3}, boolp(true), true}, {"list ceiling unknown", 200, output.UsersRetrievalProof{RequestedLimit: 300, ProbeCount: 200}, nil, true}, {"query ceiling unknown", 1000, output.UsersRetrievalProof{RequestedLimit: 1000, ProbeCount: 1000, Query: "q"}, nil, true}, {"cannot claim true at ceiling", 200, output.UsersRetrievalProof{RequestedLimit: 200, ProbeCount: 200}, nil, true}, {"selected over limit", 2, output.UsersRetrievalProof{RequestedLimit: 1, ProbeCount: 2}, nil, false}, {"unsupported probe gap", 1, output.UsersRetrievalProof{RequestedLimit: 20, ProbeCount: 2}, nil, false}} + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + doc, err := output.NewUsersEnvelope(users(test.n), test.proof, presentation.Options{}) + if (err == nil) != test.ok { + t.Fatalf("err=%v", err) + } + if err == nil && !equalBool(doc.Retrieval.Truncated, test.want) { + t.Fatalf("truncated=%v want=%v", doc.Retrieval.Truncated, test.want) + } + }) + } +} + +func TestWhitespaceAndDMLabelsRejected(t *testing.T) { + if _, err := output.NewWhoAmIEnvelope(output.RawIdentity{ID: " ", Username: "a"}, presentation.Options{}); err == nil { + t.Fatal("accepted whitespace ID") + } + users, err := output.NewUsersEnvelope(nil, output.UsersRetrievalProof{RequestedLimit: 20, Query: " ", TeamID: "\t"}, presentation.Options{}) + if err != nil || users.Retrieval.Query != nil || users.Retrieval.TeamID != nil { + t.Fatalf("blank optional retrieval values were not null: %+v err=%v", users.Retrieval, err) + } + for _, username := range []string{"", " ", "\t"} { + if _, err := output.NewChannelsEnvelope([]output.RawChannel{{ID: "d", Type: "D", Name: "raw", DirectUsername: username}}, presentation.Options{}); err == nil { + t.Fatalf("accepted DM username %q", username) + } + } + doc := mustChannels(t, []output.RawChannel{{ID: "d", Type: "D", Name: "raw", DirectUsername: "arda"}}, presentation.Options{}) + if doc.Channels[0].Name != "@arda" || doc.Channels[0].DisplayName != nil || doc.Channels[0].Team != nil { + t.Fatalf("dm=%+v", doc.Channels[0]) + } +} + +func TestIdentityPresentationMakesControlsVisibleAndInvalidTimestampsNull(t *testing.T) { + doc := mustTeams(t, []output.RawTeam{{ID: "t\nvalue", Name: "core\tname", DisplayName: "bad\u202ename", Type: "O"}}, presentation.Options{}) + if doc.Teams[0].ID != `t\nvalue` || doc.Teams[0].Name != `core\tname` || *doc.Teams[0].DisplayName != `bad\u202ename` { + t.Fatalf("labels were not made visible: %+v", doc.Teams[0]) + } + channels := mustChannels(t, []output.RawChannel{{ID: "negative", Type: "G", Name: "g", LastPostAt: -1}, {ID: "overflow", Type: "G", Name: "g", LastPostAt: 1 << 62}}, presentation.Options{}) + for _, channel := range channels.Channels { + if channel.LastPost != nil { + t.Fatalf("invalid timestamp retained: %+v", channel) + } + } +} + +func TestChannelTimestampNormalizationPrecedesOrdering(t *testing.T) { + doc := mustChannels(t, []output.RawChannel{ + {ID: "overflow", Type: "G", Name: "overflow", LastPostAt: 1 << 62}, + {ID: "valid", Type: "G", Name: "valid", LastPostAt: 1784197230123}, + {ID: "negative", Type: "G", Name: "negative", LastPostAt: -1}, + {ID: "zero", Type: "G", Name: "zero"}, + }, presentation.Options{}) + want := []string{"valid", "negative", "overflow", "zero"} + for index, id := range want { + if doc.Channels[index].ID != id { + t.Fatalf("channels=%+v", doc.Channels) + } + } + if doc.Channels[0].LastPost == nil { + t.Fatal("valid timestamp became null") + } + for _, channel := range doc.Channels[1:] { + if channel.LastPost != nil { + t.Fatalf("invalid timestamp retained: %+v", channel) + } + } +} + +func TestRawChannelTeamBindingAndGroupDisplayFallback(t *testing.T) { + team := &output.RawTeam{ID: "team", Name: "core", Type: "O"} + if _, err := output.NewChannelsEnvelope([]output.RawChannel{{ID: "c", Type: "O", Name: "general", TeamID: "other", Team: team}}, presentation.Options{}); err == nil { + t.Fatal("accepted mismatched raw team binding") + } + doc := mustChannels(t, []output.RawChannel{{ID: "display", Type: "G", Name: "opaque", DisplayName: "Crew"}, {ID: "fallback", Type: "G", Name: "opaque"}}, presentation.Options{}) + if doc.Channels[0].Name != "Crew" || doc.Channels[0].DisplayName != nil || doc.Channels[1].Name != "opaque" { + t.Fatalf("groups=%+v", doc.Channels) + } +} + +func TestUsersQueryIsTrimmedBeforeCeilingAndPresentation(t *testing.T) { + doc := mustUsers(t, nil, output.UsersRetrievalProof{RequestedLimit: 20, Query: " dev "}, presentation.Options{}) + if doc.Retrieval.Query == nil || *doc.Retrieval.Query != "dev" || doc.Retrieval.Truncated == nil || *doc.Retrieval.Truncated { + t.Fatalf("retrieval=%+v", doc.Retrieval) + } +} + +func TestWhoAmIRawRequiredFieldsRejectWhitespaceControls(t *testing.T) { + for _, raw := range []output.RawIdentity{{ID: "\n", Username: "arda"}, {ID: "u", Username: "\t"}, {ID: "u", Username: "arda", Roles: []string{"\n\t"}}} { + if _, err := output.NewWhoAmIEnvelope(raw, presentation.Options{}); err == nil { + t.Fatalf("accepted raw identity %+v", raw) + } + } +} + +func TestRawRequiredRejectsControlOnlyAcrossIdentityConsumers(t *testing.T) { + for _, hazard := range []string{"\x00", "\x1b", "\u0085", "\u061c", "\u200e", "\u200f", "\u202e", "\u2066"} { + t.Run(fmt.Sprintf("%x", []byte(hazard)), func(t *testing.T) { + for _, raw := range []output.RawIdentity{{ID: hazard, Username: "arda"}, {ID: "u", Username: hazard}, {ID: "u", Username: "arda", Roles: []string{hazard}}} { + if _, err := output.NewWhoAmIEnvelope(raw, presentation.Options{}); err == nil { + t.Fatalf("accepted identity %+v", raw) + } + } + if _, err := output.NewTeamsEnvelope([]output.RawTeam{{ID: hazard, Name: "core", Type: "O"}}, presentation.Options{}); err == nil { + t.Fatal("accepted team ID") + } + if _, err := output.NewTeamsEnvelope([]output.RawTeam{{ID: "t", Name: hazard, Type: "O"}}, presentation.Options{}); err == nil { + t.Fatal("accepted team name") + } + if _, err := output.NewUsersEnvelope([]output.RawUser{{ID: "u", Username: hazard}}, output.UsersRetrievalProof{RequestedLimit: 1, ProbeCount: 1}, presentation.Options{}); err == nil { + t.Fatal("accepted username") + } + if _, err := output.NewChannelsEnvelope([]output.RawChannel{{ID: hazard, Type: "G", Name: "group"}}, presentation.Options{}); err == nil { + t.Fatal("accepted channel ID") + } + if _, err := output.NewChannelsEnvelope([]output.RawChannel{{ID: "g", Type: "G", Name: hazard}}, presentation.Options{}); err == nil { + t.Fatal("accepted channel name") + } + if _, err := output.NewChannelsEnvelope([]output.RawChannel{{ID: "d", Type: "D", Name: "raw", DirectUsername: hazard}}, presentation.Options{}); err == nil { + t.Fatal("accepted DM username") + } + }) + } +} + +func TestRawRequiredAllowsEmbeddedControlsAndSanitizesThem(t *testing.T) { + doc := mustWho(t, output.RawIdentity{ID: "u\x00id", Username: "ar\x1bda", Roles: []string{"system\u202euser"}}, presentation.Options{}) + if doc.Data.ID != `u\u0000id` || doc.Data.Username != `ar\u001bda` || doc.Data.Roles[0] != `system\u202euser` { + t.Fatalf("identity=%+v", doc.Data) + } + dm := mustChannels(t, []output.RawChannel{{ID: "d", Type: "D", Name: "raw", DirectUsername: "ar\u0085da"}}, presentation.Options{}) + if dm.Channels[0].Name != `@ar\u0085da` { + t.Fatalf("DM label=%q", dm.Channels[0].Name) + } +} + +type countingWriter struct{ calls int } + +func (w *countingWriter) Write(p []byte) (int, error) { w.calls++; return len(p), nil } +func boolp(v bool) *bool { return &v } +func equalBool(a, b *bool) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return *a == *b +} +func mustWho(t *testing.T, v output.RawIdentity, o presentation.Options) output.WhoAmIEnvelope { + t.Helper() + d, e := output.NewWhoAmIEnvelope(v, o) + if e != nil { + t.Fatal(e) + } + return d +} +func mustTeams(t *testing.T, v []output.RawTeam, o presentation.Options) output.TeamsEnvelope { + t.Helper() + d, e := output.NewTeamsEnvelope(v, o) + if e != nil { + t.Fatal(e) + } + return d +} +func mustUsers(t *testing.T, v []output.RawUser, p output.UsersRetrievalProof, o presentation.Options) output.UsersEnvelope { + t.Helper() + d, e := output.NewUsersEnvelope(v, p, o) + if e != nil { + t.Fatal(e) + } + return d +} +func mustChannels(t *testing.T, v []output.RawChannel, o presentation.Options) output.ChannelsEnvelope { + t.Helper() + d, e := output.NewChannelsEnvelope(v, o) + if e != nil { + t.Fatal(e) + } + return d +} diff --git a/internal/output/machine.go b/internal/output/machine.go index 5d245e0..b0ee97f 100644 --- a/internal/output/machine.go +++ b/internal/output/machine.go @@ -172,6 +172,10 @@ func (MentionsEnvelope) machineDocument() {} func (ErrorEnvelope) machineDocument() {} func (ConfigEnvelope) machineDocument() {} func (DoctorEnvelope) machineDocument() {} +func (WhoAmIEnvelope) machineDocument() {} +func (TeamsEnvelope) machineDocument() {} +func (UsersEnvelope) machineDocument() {} +func (ChannelsEnvelope) machineDocument() {} type wireMessage struct { ID string `json:"id"` @@ -291,6 +295,8 @@ func canonicalMachineDocument(document MachineDocument) (any, error) { return value, nil case DoctorEnvelope: return value, nil + case WhoAmIEnvelope, TeamsEnvelope, UsersEnvelope, ChannelsEnvelope: + return value, nil default: return nil, fmt.Errorf("unsupported machine document type %T", document) } @@ -461,7 +467,7 @@ const ( func preflightMachineDocument(document MachineDocument) error { switch document.(type) { - case DMSEnvelope, GroupDMSEnvelope, ChannelEnvelope, ThreadEnvelope, SearchEnvelope, MentionsEnvelope, ErrorEnvelope, ConfigEnvelope, DoctorEnvelope: + case DMSEnvelope, GroupDMSEnvelope, ChannelEnvelope, ThreadEnvelope, SearchEnvelope, MentionsEnvelope, ErrorEnvelope, ConfigEnvelope, DoctorEnvelope, WhoAmIEnvelope, TeamsEnvelope, UsersEnvelope, ChannelsEnvelope: default: return fmt.Errorf("unsupported machine document type %T", document) } @@ -470,6 +476,12 @@ func preflightMachineDocument(document MachineDocument) error { return err } } + switch document.(type) { + case WhoAmIEnvelope, TeamsEnvelope, UsersEnvelope, ChannelsEnvelope: + if err := validateIdentityDocument(document); err != nil { + return err + } + } contentBudget := int64(MaxMachineDocumentBytes) valueBudget := machinePreflightMaxValues stack := make(map[preflightVisit]bool) From d8f73dcc4da387f996c255494c87f56d2accd0a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 20:21:50 +0300 Subject: [PATCH 044/119] feat: add identity list commands --- internal/cli/identity.go | 364 +++++++++++++++++++++++++++ internal/cli/identity_test.go | 206 +++++++++++++++ internal/cli/root.go | 4 + internal/mattermost/channels.go | 76 +++++- internal/mattermost/channels_test.go | 55 ++++ 5 files changed, 696 insertions(+), 9 deletions(-) create mode 100644 internal/cli/identity.go create mode 100644 internal/cli/identity_test.go diff --git a/internal/cli/identity.go b/internal/cli/identity.go new file mode 100644 index 0000000..c1d99da --- /dev/null +++ b/internal/cli/identity.go @@ -0,0 +1,364 @@ +package cli + +import ( + "bytes" + "fmt" + "sort" + "strings" + "time" + + "github.com/spf13/cobra" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/output" + "github.com/ardasevinc/mattermost-cli/internal/presentation" +) + +func newWhoAmICommand(state *rootState) *cobra.Command { + return &cobra.Command{Use: "whoami", Short: "Show the authenticated identity", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { return runWhoAmI(cmd, state) }} +} +func newTeamsCommand(state *rootState) *cobra.Command { + return &cobra.Command{Use: "teams", Short: "List team memberships", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { return runTeams(cmd, state) }} +} + +type usersFlags struct{ limit, team string } + +func newUsersCommand(state *rootState) *cobra.Command { + flags := new(usersFlags) + command := &cobra.Command{Use: "users [query]", Short: "List users", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { return runUsers(cmd, state, *flags, args) }} + command.Flags().StringVarP(&flags.limit, "limit", "l", "20", "maximum users") + command.Flags().StringVar(&flags.team, "team", "", "exact team name or display name") + return command +} + +type channelsFlags struct{ kind string } + +func newChannelsCommand(state *rootState) *cobra.Command { + flags := new(channelsFlags) + command := &cobra.Command{Use: "channels", Short: "List account channels", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { return runChannels(cmd, state, *flags) }} + command.Flags().StringVar(&flags.kind, "type", "all", "channel type: all, dm, public, private, or group") + return command +} + +func identityOptions(runtime *Runtime) presentation.Options { + return presentation.Options{Credentials: []string{runtime.Config.Token}, DisableHeuristics: !runtime.Config.Redact} +} +func rawIdentity(user mattermost.User) output.RawIdentity { + display := strings.TrimSpace(strings.Join([]string{user.FirstName, user.LastName}, " ")) + return output.RawIdentity{ID: user.ID, Username: user.Username, DisplayName: display, Nickname: user.Nickname, Roles: strings.Fields(user.Roles)} +} +func rawTeams(values []mattermost.Team) []output.RawTeam { + result := make([]output.RawTeam, len(values)) + for i, value := range values { + result[i] = output.RawTeam{ID: value.ID, Name: value.Name, DisplayName: value.DisplayName, Type: value.Type} + } + return result +} +func rawUsers(values []mattermost.User) []output.RawUser { + result := make([]output.RawUser, len(values)) + for i, value := range values { + display := strings.TrimSpace(strings.Join([]string{value.FirstName, value.LastName}, " ")) + result[i] = output.RawUser{ID: value.ID, Username: value.Username, DisplayName: display, Nickname: value.Nickname} + } + return result +} + +func runWhoAmI(cmd *cobra.Command, state *rootState) error { + runtime, err := state.runtimeFor(cmd) + if err != nil { + return err + } + if err = emitRedactionWarning(state, runtime, state.flags.json); err != nil { + return err + } + user, err := runtime.Users.Current(cmd.Context()) + if err != nil { + return readFailure(err) + } + document, err := output.NewWhoAmIEnvelope(rawIdentity(user), identityOptions(runtime)) + if err != nil { + return readFailure(err) + } + if state.flags.json { + return writeIdentityMachine(state, document) + } + labels := []string{} + if document.Data.DisplayName != nil { + labels = append(labels, *document.Data.DisplayName) + } + if document.Data.Nickname != nil { + labels = append(labels, "aka "+*document.Data.Nickname) + } + suffix := "" + if len(labels) > 0 { + suffix = " (" + strings.Join(labels, ", ") + ")" + } + roles := "none" + if len(document.Data.Roles) > 0 { + roles = strings.Join(document.Data.Roles, ", ") + } + return writeAll(state.streams.out, []byte(fmt.Sprintf("@%s%s [%s]\nRoles: %s\n", document.Data.Username, suffix, document.Data.ID, roles))) +} + +func runTeams(cmd *cobra.Command, state *rootState) error { + runtime, err := state.runtimeFor(cmd) + if err != nil { + return err + } + if err = emitRedactionWarning(state, runtime, state.flags.json); err != nil { + return err + } + me, err := runtime.Users.Current(cmd.Context()) + if err != nil { + return readFailure(err) + } + membership, err := runtime.Teams.List(cmd.Context(), me.ID) + if err != nil { + return readFailure(err) + } + document, err := output.NewTeamsEnvelope(rawTeams(membership.Items()), identityOptions(runtime)) + if err != nil { + return readFailure(err) + } + if state.flags.json { + return writeIdentityMachine(state, document) + } + if len(document.Teams) == 0 { + return writeAll(state.streams.out, []byte("No teams found.\n")) + } + var text strings.Builder + for _, team := range document.Teams { + display := "" + if team.DisplayName != nil && *team.DisplayName != team.Name { + display = " (" + *team.DisplayName + ")" + } + fmt.Fprintf(&text, "%s%s [%s] %s\n", team.Name, display, team.ID, team.Type) + } + return writeAll(state.streams.out, []byte(text.String())) +} + +func runUsers(cmd *cobra.Command, state *rootState, flags usersFlags, args []string) error { + limit, err := positiveInteger(flags.limit) + if err != nil { + return err + } + if limit > 1000 { + return invalidFailure("--limit exceeds the users endpoint ceiling of 1000") + } + if flagChanged(cmd, "team") && strings.TrimSpace(flags.team) == "" { + return invalidFailure("--team cannot be empty") + } + query := "" + if len(args) > 0 { + query = strings.TrimSpace(args[0]) + } + runtime, err := state.runtimeFor(cmd) + if err != nil { + return err + } + if err = emitRedactionWarning(state, runtime, state.flags.json); err != nil { + return err + } + teamID := "" + if flags.team != "" { + me, currentErr := runtime.Users.Current(cmd.Context()) + if currentErr != nil { + return readFailure(currentErr) + } + team, resolutionErr := runtime.Teams.Resolve(cmd.Context(), me.ID, flags.team) + if resolutionErr != nil { + return readFailure(resolutionErr) + } + teamID = team.ID + } + result, err := runtime.Users.Directory(cmd.Context(), query, teamID, limit) + if err != nil { + return readFailure(err) + } + probe := int64(len(result.Users)) + if result.Truncated == nil { + if query == "" { + probe = 200 + } else { + probe = 1000 + } + } else if *result.Truncated { + probe++ + } + document, err := output.NewUsersEnvelope(rawUsers(result.Users), output.UsersRetrievalProof{RequestedLimit: int64(limit), ProbeCount: probe, Query: query, TeamID: teamID}, identityOptions(runtime)) + if err != nil { + return readFailure(err) + } + if state.flags.json { + return writeIdentityMachine(state, document) + } + if len(document.Users) == 0 { + return writeAll(state.streams.out, []byte("No users found.\n")) + } + var text strings.Builder + for _, user := range document.Users { + labels := []string{} + if user.DisplayName != nil { + labels = append(labels, *user.DisplayName) + } + if user.Nickname != nil { + labels = append(labels, "aka "+*user.Nickname) + } + suffix := "" + if len(labels) > 0 { + suffix = " (" + strings.Join(labels, ", ") + ")" + } + fmt.Fprintf(&text, "@%s%s [%s]\n", user.Username, suffix, user.ID) + } + coverage := "unknown" + if document.Retrieval.Truncated != nil { + coverage = map[bool]string{true: "truncated", false: "complete"}[*document.Retrieval.Truncated] + } + fmt.Fprintf(&text, "Showing %d of up to %d users (coverage: %s).\n", len(document.Users), limit, coverage) + return writeAll(state.streams.out, []byte(text.String())) +} + +func runChannels(cmd *cobra.Command, state *rootState, flags channelsFlags) error { + valid := map[string]string{"all": "", "dm": "D", "public": "O", "private": "P", "group": "G"} + typeCode, ok := valid[flags.kind] + if !ok { + return invalidFailure("--type must be one of: all, dm, public, private, group") + } + display, err := state.readDisplay(cmd) + if err != nil { + return err + } + runtime, err := state.runtimeFor(cmd) + if err != nil { + return err + } + if err = emitRedactionWarning(state, runtime, display.json); err != nil { + return err + } + me, err := runtime.Users.Current(cmd.Context()) + if err != nil { + return readFailure(err) + } + selectedTypes := []string{"O", "P", "D", "G"} + if typeCode != "" { + selectedTypes = []string{typeCode} + } + selection, err := runtime.Channels.ListSelected(cmd.Context(), me.ID, selectedTypes...) + if err != nil { + return readFailure(err) + } + channels := selection.Channels + teams := map[string]mattermost.Team{} + for _, team := range selection.Membership.Items() { + teams[team.ID] = team + } + peers := map[string]mattermost.User{} + peerIDs := []string{} + seenPeers := map[string]bool{} + for _, channel := range channels { + if channel.Type != "D" { + continue + } + parts := strings.Split(channel.Name, "__") + other := parts[0] + if other == me.ID { + other = parts[1] + } + if !seenPeers[other] { + seenPeers[other] = true + peerIDs = append(peerIDs, other) + } + } + sort.Strings(peerIDs) + for start := 0; start < len(peerIDs); start += 200 { + end := start + 200 + if end > len(peerIDs) { + end = len(peerIDs) + } + users, userErr := runtime.Users.ByIDs(cmd.Context(), peerIDs[start:end]) + if userErr != nil { + return readFailure(userErr) + } + for _, user := range users { + peers[user.ID] = user + } + } + raw := make([]output.RawChannel, len(channels)) + for i, channel := range channels { + value := output.RawChannel{ID: channel.ID, Type: channel.Type, Name: channel.Name, DisplayName: channel.DisplayName, TeamID: channel.TeamID, LastPostAt: channel.LastPostAt, TotalMsgCount: channel.TotalMsgCount} + if channel.Type == "D" { + parts := strings.Split(channel.Name, "__") + other := parts[0] + if other == me.ID { + other = parts[1] + } + value.DirectUsername = peers[other].Username + } else if channel.Type == "O" || channel.Type == "P" { + team, exists := teams[channel.TeamID] + if !exists { + return readError("Mattermost returned incomplete team metadata for a channel") + } + rawTeam := output.RawTeam{ID: team.ID, Name: team.Name, DisplayName: team.DisplayName, Type: team.Type} + value.Team = &rawTeam + } + raw[i] = value + } + document, err := output.NewChannelsEnvelope(raw, identityOptions(runtime)) + if err != nil { + return readFailure(err) + } + if display.json { + return writeIdentityMachine(state, document) + } + return renderHumanChannels(state, document, display.relative) +} + +func writeIdentityMachine(state *rootState, document output.MachineDocument) error { + var wire bytes.Buffer + if _, err := output.WriteMachineJSON(&wire, document); err != nil { + return readFailure(err) + } + return writeAll(state.streams.out, wire.Bytes()) +} +func renderHumanChannels(state *rootState, document output.ChannelsEnvelope, relative bool) error { + if len(document.Channels) == 0 { + return writeAll(state.streams.out, []byte("\nTotal: 0 channels\n")) + } + labels := map[string]string{"public": "Public Channels", "private": "Private Channels", "group": "Group Messages", "dm": "Direct Messages"} + order := []string{"public", "private", "group", "dm"} + formatter := output.NewDateFormatter(time.Now, time.Local) + var text strings.Builder + for _, kind := range order { + items := []output.ChannelItem{} + for _, item := range document.Channels { + if item.Type == kind { + items = append(items, item) + } + } + if len(items) == 0 { + continue + } + fmt.Fprintf(&text, "\n%s:\n\n", labels[kind]) + for _, item := range items { + last := "never" + if item.LastPost != nil { + if relative { + last = formatter.FormatRelativeTime(item.LastPost.Time) + } else { + last = formatter.FormatDate(item.LastPost.Time, true) + } + } + label := item.Name + if item.Team != nil { + label = item.Team.Name + "/#" + item.Name + } + display := "" + if item.DisplayName != nil { + display = " (" + *item.DisplayName + ")" + } + fmt.Fprintf(&text, " %-25s%-25s [%s] %d msgs, last: %s\n", label, display, item.ID, item.MessageCount, last) + } + } + fmt.Fprintf(&text, "\nTotal: %d channels\n", len(document.Channels)) + return writeAll(state.streams.out, []byte(text.String())) +} diff --git a/internal/cli/identity_test.go b/internal/cli/identity_test.go new file mode 100644 index 0000000..b8552b6 --- /dev/null +++ b/internal/cli/identity_test.go @@ -0,0 +1,206 @@ +package cli + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/output" + "github.com/ardasevinc/mattermost-cli/internal/schema" +) + +func TestIdentityCommandsEmitStrictMachineSchemasAndHumanSemantics(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v4/users/me": + writeJSON(t, w, `{"id":"u","username":"arda","first_name":"Arda","last_name":"Sevinc","roles":"system_user"}`) + case "/api/v4/users/u/teams": + writeJSON(t, w, `[{"id":"t","name":"core","display_name":"Core Team","type":"O"}]`) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + registry, err := schema.Load() + if err != nil { + t.Fatal(err) + } + for _, command := range []string{"whoami", "teams"} { + stdout, stderr, code := executeChannel(t, server.URL, "--json", command) + if code != 0 || stderr != "" { + t.Fatalf("%s exit=%d stderr=%q", command, code, stderr) + } + if err := registry.Validate("mm/v2/"+command, strings.NewReader(stdout)); err != nil { + t.Fatalf("%s schema: %v", command, err) + } + } + stdout, stderr, code := executeChannel(t, server.URL, "whoami") + if code != 0 || stderr != "" || stdout != "@arda (Arda Sevinc) [u]\nRoles: system_user\n" { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } + stdout, stderr, code = executeChannel(t, server.URL, "teams") + if code != 0 || stderr != "" || stdout != "core (Core Team) [t] open\n" { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } +} + +func TestUsersValidatesLimitBeforeNetworkAndProvesTriState(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if r.URL.Path != "/api/v4/users/search" { + t.Fatalf("unexpected %s", r.URL.Path) + } + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatal(err) + } + if body["term"] != "dev" || body["limit"] != float64(3) { + t.Fatalf("body=%v", body) + } + writeJSON(t, w, `[{"id":"b","username":"z"},{"id":"a","username":"a"},{"id":"c","username":"more"}]`) + })) + defer server.Close() + _, _, code := executeChannel(t, server.URL, "users", "--limit", "0") + if code != 2 || requests.Load() != 0 { + t.Fatalf("exit=%d requests=%d", code, requests.Load()) + } + stdout, stderr, code := executeChannel(t, server.URL, "--json", "users", " dev ", "--limit", "2") + if code != 0 || stderr != "" { + t.Fatalf("exit=%d stderr=%q", code, stderr) + } + if !strings.Contains(stdout, `"query":"dev"`) || !strings.Contains(stdout, `"truncated":true`) || strings.Index(stdout, `"username":"a"`) > strings.Index(stdout, `"username":"z"`) { + t.Fatalf("stdout=%s", stdout) + } +} + +func TestChannelsFiltersBeforeHydrationAndBatchesDMPeers(t *testing.T) { + var teamCalls, userBatchCalls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v4/users/me": + writeJSON(t, w, `{"id":"user","username":"me"}`) + case "/api/v4/users/user/channels": + writeJSON(t, w, `[{"type":"O"},{"id":"d","team_id":"","type":"D","name":"user__peer","display_name":"","last_post_at":7,"total_msg_count":2}]`) + case "/api/v4/users/ids": + userBatchCalls.Add(1) + writeJSON(t, w, `[{"id":"peer","username":"bob"}]`) + case "/api/v4/users/user/teams": + teamCalls.Add(1) + t.Fatal("D-only filter must not read teams") + default: + http.NotFound(w, r) + } + })) + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "--json", "channels", "--type", "dm") + if code != 0 || stderr != "" { + t.Fatalf("exit=%d stderr=%q", code, stderr) + } + if teamCalls.Load() != 0 || userBatchCalls.Load() != 1 || !strings.Contains(stdout, `"name":"@bob"`) { + t.Fatalf("teams=%d users=%d stdout=%s", teamCalls.Load(), userBatchCalls.Load(), stdout) + } +} + +func TestChannelsGroupFilterUsesNoTeamOrUserHydrationAndEmptyIsStrict(t *testing.T) { + var unrelated atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v4/users/me": + writeJSON(t, w, `{"id":"user","username":"me"}`) + case "/api/v4/users/user/channels": + writeJSON(t, w, `[]`) + default: + unrelated.Add(1) + http.NotFound(w, r) + } + })) + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "--json", "channels", "--type", "group") + if code != 0 || stderr != "" || unrelated.Load() != 0 || stdout != `{"schema":"mm/v2/channels","channels":[]}`+"\n" { + t.Fatalf("exit=%d unrelated=%d stdout=%q stderr=%q", code, unrelated.Load(), stdout, stderr) + } +} + +func TestChannelsPublicFilterHydratesOnlyCompleteTeams(t *testing.T) { + var teamCalls, userCalls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v4/users/me": + writeJSON(t, w, `{"id":"user","username":"me"}`) + case "/api/v4/users/user/channels": + writeJSON(t, w, `[{"id":"d","team_id":"","type":"D","name":"user__peer","display_name":""},{"id":"c","team_id":"t","type":"O","name":"general","display_name":"General"}]`) + case "/api/v4/users/user/teams": + teamCalls.Add(1) + writeJSON(t, w, `[{"id":"t","name":"core","display_name":"Core","type":"O"}]`) + case "/api/v4/users/ids": + userCalls.Add(1) + t.Fatal("public filter must not hydrate D peers") + default: + http.NotFound(w, r) + } + })) + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "--json", "channels", "--type", "public") + if code != 0 || stderr != "" || teamCalls.Load() != 1 || userCalls.Load() != 0 || !strings.Contains(stdout, `"team":{"id":"t","name":"core"`) { + t.Fatalf("exit=%d teams=%d users=%d stdout=%s stderr=%q", code, teamCalls.Load(), userCalls.Load(), stdout, stderr) + } +} + +func TestIdentityPresentationMasksCredentialsAndFailureKeepsStdoutEmpty(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v4/users/me" { + writeJSON(t, w, `{"id":"u","username":"test-token\u001b[2J"}`) + return + } + http.Error(w, "remote secret", 500) + })) + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "--json", "whoami") + if code != 0 || stderr != "" || strings.Contains(stdout, "test-token") || strings.ContainsRune(stdout, '\x1b') || !strings.Contains(stdout, "[REDACTED:mattermost_credential]") { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } + stdout, stderr, code = executeChannel(t, server.URL, "--json", "teams") + if code != 3 || stdout != "" || !strings.Contains(stderr, `"code":"read_failed"`) { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } +} + +func TestRemoteBindingFailuresAreReadFailedMachineErrors(t *testing.T) { + registry, err := schema.Load() + if err != nil { + t.Fatal(err) + } + tests := []struct { + name, command string + handler http.HandlerFunc + }{ + {"hostile identity", "whoami", func(w http.ResponseWriter, r *http.Request) { writeJSON(t, w, `{"id":"u","username":"\u202e"}`) }}, + {"duplicate teams", "teams", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v4/users/me" { + writeJSON(t, w, `{"id":"u","username":"a"}`) + return + } + writeJSON(t, w, `[{"id":"t","name":"a","type":"O"},{"id":"t","name":"b","type":"O"}]`) + }}, + {"oversized identity", "whoami", func(w http.ResponseWriter, r *http.Request) { + writeJSON(t, w, `{"id":"u","username":"`+strings.Repeat("x", output.MaxMachineDocumentBytes)+`"}`) + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(test.handler) + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "--json", test.command) + if code != 3 || stdout != "" || !strings.Contains(stderr, `"code":"read_failed"`) { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } + if err := registry.Validate("mm/v2/error", strings.NewReader(stderr)); err != nil { + t.Fatalf("error schema: %v", err) + } + }) + } +} diff --git a/internal/cli/root.go b/internal/cli/root.go index b56e348..a652503 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -118,6 +118,10 @@ func newRootWithState(state *rootState) *cobra.Command { cmd.AddCommand(newSchemaCommand(state)) cmd.AddCommand(newConfigCommand(state)) cmd.AddCommand(newDoctorCommand(state)) + cmd.AddCommand(newWhoAmICommand(state)) + cmd.AddCommand(newTeamsCommand(state)) + cmd.AddCommand(newUsersCommand(state)) + cmd.AddCommand(newChannelsCommand(state)) cmd.AddCommand(newChannelCommand(state)) cmd.AddCommand(newDMsCommand(state)) cmd.AddCommand(newGroupDMsCommand(state)) diff --git a/internal/mattermost/channels.go b/internal/mattermost/channels.go index dca4325..9b94100 100644 --- a/internal/mattermost/channels.go +++ b/internal/mattermost/channels.go @@ -182,6 +182,46 @@ func (l *channelList) UnmarshalJSON(data []byte) error { return nil } +type selectedChannelList struct { + wanted map[string]bool + channels []Channel +} + +func (l *selectedChannelList) UnmarshalJSON(data []byte) error { + var rows []json.RawMessage + if err := json.Unmarshal(data, &rows); err != nil || rows == nil { + return ErrInvalidChannelsResponse + } + channels := make([]Channel, 0) + for _, row := range rows { + var discriminator struct { + Type json.RawMessage `json:"type"` + } + if json.Unmarshal(row, &discriminator) != nil { + return ErrInvalidChannelsResponse + } + typeCode, ok := requiredString(discriminator.Type) + if !ok || (typeCode != "O" && typeCode != "P" && typeCode != "D" && typeCode != "G") { + return ErrInvalidChannelsResponse + } + if !l.wanted[typeCode] { + continue + } + var channel Channel + if json.Unmarshal(row, &channel) != nil { + return ErrInvalidChannelsResponse + } + channels = append(channels, channel) + } + l.channels = channels + return nil +} + +type ChannelSelection struct { + Channels []Channel + Membership TeamMembership +} + type directChannelList []Channel func (l *directChannelList) UnmarshalJSON(data []byte) error { @@ -341,21 +381,39 @@ func canonicalChannelRequestID(value string) bool { // checked before any result is released. Team membership is fetched through // the same transport so proof cannot be mixed across sessions or servers. func (s *Channels) List(ctx context.Context, userID string) ([]Channel, error) { + selection, err := s.ListSelected(ctx, userID, "O", "P", "D", "G") + if err != nil { + return nil, err + } + return selection.Channels, nil +} + +// ListSelected validates every row's discriminator before fully decoding only +// the requested channel types. The returned team snapshot is the exact proof +// used to bind selected O/P channels and is empty when no selected O/P exists. +func (s *Channels) ListSelected(ctx context.Context, userID string, types ...string) (ChannelSelection, error) { if strings.TrimSpace(userID) == "" || userID == "me" { - return nil, ErrInvalidChannelRequest + return ChannelSelection{}, ErrInvalidChannelRequest + } + wanted := make(map[string]bool, len(types)) + for _, typeCode := range types { + if typeCode != "O" && typeCode != "P" && typeCode != "D" && typeCode != "G" { + return ChannelSelection{}, ErrInvalidChannelRequest + } + wanted[typeCode] = true } - var decoded channelList + var decoded = selectedChannelList{wanted: wanted} if err := s.client.Get(ctx, "/users/"+url.PathEscape(userID)+"/channels", &decoded); err != nil { - return nil, err + return ChannelSelection{}, err } - channels := []Channel(decoded) + channels := decoded.channels var membership TeamMembership for _, channel := range channels { if channel.Type == "O" || channel.Type == "P" { var err error membership, err = NewTeams(s.client).List(ctx, userID) if err != nil { - return nil, err + return ChannelSelection{}, err } break } @@ -365,7 +423,7 @@ func (s *Channels) List(ctx context.Context, userID string) ([]Channel, error) { for _, channel := range channels { if previous, duplicate := seen[channel.ID]; duplicate { if previous != channel { - return nil, ErrInvalidChannelsResponse + return ChannelSelection{}, ErrInvalidChannelsResponse } continue } @@ -376,17 +434,17 @@ func (s *Channels) List(ctx context.Context, userID string) ([]Channel, error) { switch channel.Type { case "O", "P": if !membership.contains(channel.TeamID) { - return nil, ErrInvalidChannelsResponse + return ChannelSelection{}, ErrInvalidChannelsResponse } case "D": if !directChannelContains(channel.Name, userID) { - return nil, ErrInvalidChannelResponse + return ChannelSelection{}, ErrInvalidChannelResponse } } result = append(result, channel) } sort.Slice(result, func(i, j int) bool { return result[i].ID < result[j].ID }) - return result, nil + return ChannelSelection{Channels: result, Membership: membership}, nil } // ListForUnread preserves List's bounded identity proof while refusing to diff --git a/internal/mattermost/channels_test.go b/internal/mattermost/channels_test.go index b8e25f1..f697b01 100644 --- a/internal/mattermost/channels_test.go +++ b/internal/mattermost/channels_test.go @@ -306,6 +306,61 @@ func TestChannelListDoesNotRequireTeamsForDirectOnlyDiscovery(t *testing.T) { } } +func TestListSelectedFiltersBeforeOneExactTeamProof(t *testing.T) { + public := `{"id":"public","team_id":"team","type":"O","name":"general","display_name":"General"}` + f := &fakeChannelTransport{responses: map[string]string{ + "/users/user/channels": `[` + public + `,{"type":"P","malformed":true},{"id":"dm","team_id":"","type":"D","name":"user__other","display_name":""}]`, + "/users/user/teams": `[{"id":"team","name":"core","display_name":"Core","type":"O"}]`, + }} + selection, err := NewChannels(f).ListSelected(context.Background(), "user", "O") + if err != nil || len(selection.Channels) != 1 || selection.Channels[0].ID != "public" || len(selection.Membership.Items()) != 1 { + t.Fatalf("selection=%+v err=%v", selection, err) + } + if !reflect.DeepEqual(f.paths, []string{"/users/user/channels", "/users/user/teams"}) { + t.Fatalf("paths=%v", f.paths) + } +} + +func TestListSelectedEmptyOppositeFilterDoesNotReadTeams(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{"/users/user/channels": `[{"type":"P","malformed":true},{"id":"d","team_id":"","type":"D","name":"user__other","display_name":""}]`}} + selection, err := NewChannels(f).ListSelected(context.Background(), "user", "O") + if err != nil || selection.Channels == nil || len(selection.Channels) != 0 || len(selection.Membership.Items()) != 0 { + t.Fatalf("selection=%+v err=%v", selection, err) + } + if !reflect.DeepEqual(f.paths, []string{"/users/user/channels"}) { + t.Fatalf("paths=%v", f.paths) + } +} + +func TestListSelectedRejectsMalformedDiscardedDiscriminatorAndSelectedDuplicates(t *testing.T) { + for name, payload := range map[string]string{ + "missing discriminator": `[{"id":"discarded"}]`, + "unknown discriminator": `[{"type":"X"}]`, + "conflicting selected duplicate": `[{"id":"d","team_id":"","type":"D","name":"user__a","display_name":""},{"id":"d","team_id":"","type":"D","name":"user__b","display_name":""}]`, + } { + t.Run(name, func(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{"/users/user/channels": payload}} + if _, err := NewChannels(f).ListSelected(context.Background(), "user", "D"); err == nil { + t.Fatal("expected validation error") + } + }) + } +} + +func TestListSelectedAllUsesOneTeamProof(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{ + "/users/user/channels": `[{"id":"o","team_id":"team","type":"O","name":"general","display_name":""},{"id":"d","team_id":"","type":"D","name":"user__other","display_name":""}]`, + "/users/user/teams": `[{"id":"team","name":"core","type":"O"}]`, + }} + selection, err := NewChannels(f).ListSelected(context.Background(), "user", "O", "P", "D", "G") + if err != nil || len(selection.Channels) != 2 || len(selection.Membership.Items()) != 1 { + t.Fatalf("selection=%+v err=%v", selection, err) + } + if !reflect.DeepEqual(f.paths, []string{"/users/user/channels", "/users/user/teams"}) { + t.Fatalf("paths=%v", f.paths) + } +} + func TestListForUnreadRequiresPresentTotalsWithoutFanout(t *testing.T) { for name, payload := range map[string]string{ "missing": `[{"id":"remote-secret","team_id":"","type":"D","name":"user__other","display_name":""}]`, From 9e85d0d03a34fef8179f4927ff1afa43b34e4faa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 20:38:07 +0300 Subject: [PATCH 045/119] feat: add unread output contracts --- internal/cli/root_test.go | 2 +- internal/output/machine.go | 13 +- internal/output/unread.go | 400 +++++++++++++++ internal/output/unread_test.go | 248 ++++++++++ internal/schema/unread_test.go | 66 +++ schemas/v2/examples/unread.json | 1 + schemas/v2/unread.schema.json | 831 ++++++++++++++++++++++++++++++++ 7 files changed, 1559 insertions(+), 2 deletions(-) create mode 100644 internal/output/unread.go create mode 100644 internal/output/unread_test.go create mode 100644 internal/schema/unread_test.go create mode 100644 schemas/v2/examples/unread.json create mode 100644 schemas/v2/unread.schema.json diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index 65cfe80..f0f4072 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -57,7 +57,7 @@ func TestSchemaList(t *testing.T) { if code != 0 { t.Fatalf("exit code = %d, want 0; stderr = %q", code, stderr.String()) } - if got, want := stdout.String(), "mm/v2/channel\nmm/v2/channels\nmm/v2/config\nmm/v2/dms\nmm/v2/doctor\nmm/v2/error\nmm/v2/group-dms\nmm/v2/mentions\nmm/v2/search\nmm/v2/teams\nmm/v2/thread\nmm/v2/users\nmm/v2/whoami\n"; got != want { + if got, want := stdout.String(), "mm/v2/channel\nmm/v2/channels\nmm/v2/config\nmm/v2/dms\nmm/v2/doctor\nmm/v2/error\nmm/v2/group-dms\nmm/v2/mentions\nmm/v2/search\nmm/v2/teams\nmm/v2/thread\nmm/v2/unread\nmm/v2/users\nmm/v2/whoami\n"; got != want { t.Fatalf("stdout = %q, want %q", got, want) } } diff --git a/internal/output/machine.go b/internal/output/machine.go index b0ee97f..5c69bef 100644 --- a/internal/output/machine.go +++ b/internal/output/machine.go @@ -176,6 +176,7 @@ func (WhoAmIEnvelope) machineDocument() {} func (TeamsEnvelope) machineDocument() {} func (UsersEnvelope) machineDocument() {} func (ChannelsEnvelope) machineDocument() {} +func (UnreadEnvelope) machineDocument() {} type wireMessage struct { ID string `json:"id"` @@ -297,6 +298,8 @@ func canonicalMachineDocument(document MachineDocument) (any, error) { return value, nil case WhoAmIEnvelope, TeamsEnvelope, UsersEnvelope, ChannelsEnvelope: return value, nil + case UnreadEnvelope: + return value, nil default: return nil, fmt.Errorf("unsupported machine document type %T", document) } @@ -467,7 +470,7 @@ const ( func preflightMachineDocument(document MachineDocument) error { switch document.(type) { - case DMSEnvelope, GroupDMSEnvelope, ChannelEnvelope, ThreadEnvelope, SearchEnvelope, MentionsEnvelope, ErrorEnvelope, ConfigEnvelope, DoctorEnvelope, WhoAmIEnvelope, TeamsEnvelope, UsersEnvelope, ChannelsEnvelope: + case DMSEnvelope, GroupDMSEnvelope, ChannelEnvelope, ThreadEnvelope, SearchEnvelope, MentionsEnvelope, ErrorEnvelope, ConfigEnvelope, DoctorEnvelope, WhoAmIEnvelope, TeamsEnvelope, UsersEnvelope, ChannelsEnvelope, UnreadEnvelope: default: return fmt.Errorf("unsupported machine document type %T", document) } @@ -482,6 +485,11 @@ func preflightMachineDocument(document MachineDocument) error { return err } } + if unread, ok := document.(UnreadEnvelope); ok { + if err := validateUnreadEnvelope(unread); err != nil { + return err + } + } contentBudget := int64(MaxMachineDocumentBytes) valueBudget := machinePreflightMaxValues stack := make(map[preflightVisit]bool) @@ -636,6 +644,9 @@ func consumeMachineBudget(value reflect.Value, contentBudget *int64, valueBudget } case reflect.Struct: for index := 0; index < value.NumField(); index++ { + if value.Type().Field(index).PkgPath != "" { + continue + } if err := consumeMachineBudget(value.Field(index), contentBudget, valueBudget, stack, depth+1); err != nil { return err } diff --git a/internal/output/unread.go b/internal/output/unread.go new file mode 100644 index 0000000..6b4b41a --- /dev/null +++ b/internal/output/unread.go @@ -0,0 +1,400 @@ +package output + +import ( + "errors" + "fmt" + "reflect" + "sort" + "strings" + + "github.com/ardasevinc/mattermost-cli/internal/presentation" +) + +type RawUnreadItem struct { + Channel RawChannel + UnreadCount int64 + MentionCount int64 + LastViewedAt int64 +} + +type UnreadItem struct { + Channel ChannelItem `json:"channel"` + UnreadCount int64 `json:"unreadCount"` + MentionCount int64 `json:"mentionCount"` + LastViewedAt *MillisTime `json:"lastViewedAt"` +} + +type UnreadData struct { + Unread []UnreadItem `json:"unread"` + Peek []MachineHistory `json:"peek"` +} + +type UnreadEnvelope struct { + Schema string `json:"schema"` + Data UnreadData `json:"data"` + proof *UnreadData +} + +type UnreadProof struct { + // PeekLimit is nil when peek was not requested. A non-nil value requires + // one full history per unread summary channel, including confirmed empties. + PeekLimit *int +} + +func NewUnreadEnvelope(raw []RawUnreadItem, peek []MessageOutput, proof UnreadProof, options presentation.Options) (UnreadEnvelope, error) { + values := append([]RawUnreadItem(nil), raw...) + sort.Slice(values, func(i, j int) bool { + if values[i].MentionCount != values[j].MentionCount { + return values[i].MentionCount > values[j].MentionCount + } + if values[i].UnreadCount != values[j].UnreadCount { + return values[i].UnreadCount > values[j].UnreadCount + } + return values[i].Channel.ID < values[j].Channel.ID + }) + if proof.PeekLimit == nil && len(peek) != 0 { + return UnreadEnvelope{}, errors.New("peek histories supplied without a request") + } + if proof.PeekLimit != nil && (*proof.PeekLimit < 1 || int64(*proof.PeekLimit) > MaxSafeMachineInteger || len(peek) != len(values)) { + return UnreadEnvelope{}, errors.New("incomplete unread peek histories") + } + options.Credentials = append(append([]string(nil), options.Credentials...), presentation.ActiveCredentials.Values()...) + items := make([]UnreadItem, len(values)) + seen := make(map[string]struct{}, len(values)) + for i, value := range values { + if !rawRequired(value.Channel.ID) || value.UnreadCount < 1 || !safeCount(value.UnreadCount) || !safeCount(value.MentionCount) || value.LastViewedAt < 0 { + return UnreadEnvelope{}, errors.New("invalid raw unread item") + } + if _, duplicate := seen[value.Channel.ID]; duplicate { + return UnreadEnvelope{}, errors.New("duplicate raw unread channel") + } + seen[value.Channel.ID] = struct{}{} + channel, err := NewChannelsEnvelope([]RawChannel{value.Channel}, options) + if err != nil { + return UnreadEnvelope{}, err + } + viewed := presentTimestamp(value.LastViewedAt) + if value.LastViewedAt > 0 && viewed == nil { + return UnreadEnvelope{}, errors.New("invalid unread last viewed timestamp") + } + items[i] = UnreadItem{Channel: channel.Channels[0], UnreadCount: value.UnreadCount, MentionCount: value.MentionCount, LastViewedAt: viewed} + } + histories := make([]MachineHistory, 0, len(peek)) + for i, value := range peek { + if err := validateUnreadPeekOutput(value, items[i], *proof.PeekLimit, options); err != nil { + return UnreadEnvelope{}, err + } + complete := MachineComplete + if *value.Retrieval.Selection.QueryTruncated { + complete = MachineTruncated + } + history, err := MachineHistoryFromOutput(value, complete) + if err != nil { + return UnreadEnvelope{}, fmt.Errorf("invalid unread peek history: %w", err) + } + histories = append(histories, history) + } + document := UnreadEnvelope{Schema: "mm/v2/unread", Data: UnreadData{Unread: items, Peek: histories}} + if err := preflightUnreadCandidate(document); err != nil { + return UnreadEnvelope{}, err + } + proofData := cloneUnreadData(document.Data) + document.proof = &proofData + return document, nil +} + +// MessageOutput is the established post normalization and hydration boundary. +// This constructor validates and snapshots it; it deliberately does not create +// a second raw-message presentation pipeline. +func validateUnreadPeekOutput(value MessageOutput, item UnreadItem, limit int, options presentation.Options) error { + selection := value.Retrieval.Selection + displayName := "" + if item.Channel.DisplayName != nil { + displayName = *item.Channel.DisplayName + } + if value.Channel.ID != item.Channel.ID || value.Channel.Type != item.Channel.Type || value.Channel.Name != item.Channel.Name || + value.Channel.DisplayName != displayName || value.Channel.MetadataStatus != "resolved" || selection.Source != "unread" || + selection.RequestedLimit == nil || *selection.RequestedLimit != limit || int64(*selection.RequestedLimit) > MaxSafeMachineInteger || selection.QueryTruncated == nil || selection.SelectedCount < 0 || + selection.SelectedCount > limit || selection.InputCursor != nil || selection.NextCursor != nil || value.Retrieval.VisibleThreads.Status != "not_requested" || + value.Retrieval.VisibleThreads.HydratedRootCount != 0 || len(value.Retrieval.VisibleThreads.FailedRootIDs) != 0 || value.Retrieval.DeletedPostsIncluded { + return errors.New("unbound unread peek history") + } + wantSince := nullableMillisString(item.LastViewedAt) + visible, validGraph := validateUnreadMessageGraph(value.Messages) + if !reflect.DeepEqual(selection.Since, wantSince) || !validGraph || visible != selection.SelectedCount || visible != value.Retrieval.VisiblePostCount || + !safePresentedOutput(value, options) { + return errors.New("invalid unread peek presentation") + } + return nil +} + +func nullableMillisString(value *MillisTime) *string { + if value == nil { + return nil + } + text := value.UTC().Format("2006-01-02T15:04:05.000Z") + return &text +} + +func validateUnreadMessageGraph(messages []Message) (int, bool) { + presented := make(map[string]string) + canonical := make(map[string]string) + count := 0 + for _, message := range messages { + if message.ID == "" || message.CanonicalID == "" || (message.RootID == "") != (message.CanonicalRootID == "") || (len(message.Replies) > 0 && (message.RootID != "" || message.CanonicalRootID != "")) { + return 0, false + } + if prior, duplicate := presented[message.ID]; duplicate || (prior != "" && prior != message.CanonicalID) { + return 0, false + } + if prior, duplicate := canonical[message.CanonicalID]; duplicate || (prior != "" && prior != message.ID) { + return 0, false + } + presented[message.ID], canonical[message.CanonicalID] = message.CanonicalID, message.ID + count++ + for _, reply := range message.Replies { + if len(reply.Replies) != 0 || reply.ID == "" || reply.CanonicalID == "" || reply.RootID != message.ID || reply.CanonicalRootID != message.CanonicalID { + return 0, false + } + if _, duplicate := presented[reply.ID]; duplicate { + return 0, false + } + if _, duplicate := canonical[reply.CanonicalID]; duplicate { + return 0, false + } + presented[reply.ID], canonical[reply.CanonicalID] = reply.CanonicalID, reply.ID + count++ + } + } + return count, true +} + +func safePresentedOutput(value MessageOutput, options presentation.Options) bool { + label := func(text string) bool { return safePresentedString(text, false, options) } + multiline := func(text string) bool { return safePresentedString(text, true, options) } + for _, text := range []string{value.Channel.ID, value.Channel.Type, value.Channel.Name, value.Channel.DisplayName, value.Channel.MetadataStatus} { + if !label(text) { + return false + } + } + selection := value.Retrieval.Selection + for _, pointer := range []*string{selection.Since, selection.InputCursor, selection.NextCursor} { + if pointer != nil && !label(*pointer) { + return false + } + } + for _, id := range value.Retrieval.VisibleThreads.FailedRootIDs { + if !label(id) { + return false + } + } + for _, redaction := range value.Redactions { + if !label(redaction.Type) || !label(redaction.Masked) || !label(redaction.Field) { + return false + } + } + var messageSafe func(Message) bool + messageSafe = func(message Message) bool { + for _, text := range []string{message.ID, message.Permalink, message.User, message.UserID, message.PostType, message.RootID} { + if !label(text) { + return false + } + } + if !multiline(message.Text) { + return false + } + for _, id := range message.Files { + if !label(id) { + return false + } + } + for _, file := range message.FileDetails { + for _, text := range []string{file.ID, file.Name, file.MIME, file.Extension} { + if !label(text) { + return false + } + } + } + for _, attachment := range message.Attachments { + for _, text := range []string{attachment.TitleLink, attachment.FooterIcon, attachment.AuthorLink, attachment.AuthorIcon, attachment.Color, attachment.ImageURL, attachment.ThumbURL, attachment.Timestamp} { + if !label(text) { + return false + } + } + for _, text := range []string{attachment.Fallback, attachment.Pretext, attachment.Title, attachment.Text, attachment.Footer, attachment.AuthorName} { + if !multiline(text) { + return false + } + } + for _, field := range attachment.Fields { + if !multiline(field.Title) || !multiline(field.Value) { + return false + } + } + } + for _, reaction := range message.Reactions { + if !label(reaction.Emoji) { + return false + } + for _, actor := range reaction.Actors { + if !label(actor.ID) || !label(actor.Username) { + return false + } + } + } + for _, reply := range message.Replies { + if !messageSafe(reply) { + return false + } + } + return true + } + for _, message := range value.Messages { + if !messageSafe(message) { + return false + } + } + return true +} + +func safePresentedString(value string, allowMultiline bool, options presentation.Options) bool { + if value == "" { + return true + } + if allowMultiline { + if presentation.SanitizeControls(value) != value { + return false + } + } else if presentation.SanitizeLabel(value) != value { + return false + } + exact := presentation.PreprocessWithOptions(value, presentation.Options{Credentials: options.Credentials, DisableHeuristics: true}) + return exact.Text == value +} + +func validateUnreadEnvelope(document UnreadEnvelope) error { + if document.Schema != "mm/v2/unread" || document.Data.Unread == nil || document.Data.Peek == nil || document.proof == nil { + return errors.New("invalid unread document") + } + if !reflect.DeepEqual(document.Data, *document.proof) { + return errors.New("mutated unread document") + } + return nil +} + +func preflightUnreadCandidate(document UnreadEnvelope) error { + contentBudget := int64(MaxMachineDocumentBytes) + valueBudget := machinePreflightMaxValues + return consumeMachineBudget(reflect.ValueOf(document), &contentBudget, &valueBudget, make(map[preflightVisit]bool), 0) +} + +func cloneUnreadData(value UnreadData) UnreadData { + unread := cloneSlice(value.Unread) + for i := range unread { + unread[i].Channel.DisplayName = clonePointer(unread[i].Channel.DisplayName) + unread[i].Channel.Team = cloneChannelTeam(unread[i].Channel.Team) + unread[i].Channel.LastPost = clonePointer(unread[i].Channel.LastPost) + unread[i].LastViewedAt = clonePointer(unread[i].LastViewedAt) + } + peek := make([]MachineHistory, len(value.Peek)) + for i := range value.Peek { + peek[i] = cloneMachineHistory(value.Peek[i]) + } + return UnreadData{Unread: unread, Peek: peek} +} + +func cloneChannelTeam(value *ChannelTeam) *ChannelTeam { + if value == nil { + return nil + } + copy := *value + copy.DisplayName = clonePointer(value.DisplayName) + return © +} + +func cloneMachineHistory(value MachineHistory) MachineHistory { + messages := make([]MachineMessage, len(value.Messages)) + for i := range value.Messages { + messages[i] = cloneMachineMessage(value.Messages[i]) + } + return MachineHistory{Channel: value.Channel, Messages: messages, Redactions: cloneSlice(value.Redactions), Metadata: MachineMetadata{ + Completeness: value.Metadata.Completeness, Selection: cloneSelection(value.Metadata.Selection), VisibleThreads: cloneVisibleThreads(value.Metadata.VisibleThreads), + VisiblePostCount: value.Metadata.VisiblePostCount, DeletedPostsIncluded: value.Metadata.DeletedPostsIncluded, + }} +} + +func cloneMachineMessage(value MachineMessage) MachineMessage { + copy := value + copy.EditedAt, copy.DeletedAt, copy.RootID, copy.ReplyCount = clonePointer(value.EditedAt), clonePointer(value.DeletedAt), clonePointer(value.RootID), clonePointer(value.ReplyCount) + copy.Files, copy.FileDetails = cloneSlice(value.Files), cloneSlice(value.FileDetails) + for i := range copy.FileDetails { + copy.FileDetails[i].Size = clonePointer(value.FileDetails[i].Size) + } + copy.Attachments = cloneSlice(value.Attachments) + for i := range copy.Attachments { + copy.Attachments[i].Fields = cloneSlice(value.Attachments[i].Fields) + for j := range copy.Attachments[i].Fields { + copy.Attachments[i].Fields[j].Short = clonePointer(value.Attachments[i].Fields[j].Short) + } + } + copy.Reactions = cloneSlice(value.Reactions) + for i := range copy.Reactions { + copy.Reactions[i].Actors = cloneSlice(value.Reactions[i].Actors) + } + copy.Replies = make([]MachineMessage, len(value.Replies)) + for i := range value.Replies { + copy.Replies[i] = cloneMachineMessage(value.Replies[i]) + } + return copy +} + +func FormatUnreadPretty(document UnreadEnvelope, peek []MessageOutput, dates DateFormatter, options PrettyOptions) (string, error) { + return formatUnreadHuman(document, peek, FormatPretty(peek, dates, options)) +} + +func FormatUnreadMarkdown(document UnreadEnvelope, peek []MessageOutput, dates DateFormatter, options MarkdownOptions) (string, error) { + return formatUnreadHuman(document, peek, FormatMarkdown(peek, dates, options)) +} + +func formatUnreadHuman(document UnreadEnvelope, peek []MessageOutput, renderedPeek string) (string, error) { + if err := validateUnreadEnvelope(document); err != nil { + return "", err + } + if len(document.Data.Peek) == 0 { + if len(peek) != 0 { + return "", errors.New("peek does not match unread document") + } + } else { + if len(peek) != len(document.Data.Peek) { + return "", errors.New("peek does not match unread document") + } + for i := range peek { + complete := document.Data.Peek[i].Metadata.Completeness + converted, err := MachineHistoryFromOutput(peek[i], complete) + if err != nil || !reflect.DeepEqual(converted, document.Data.Peek[i]) { + return "", errors.New("peek does not match unread document") + } + } + } + if len(document.Data.Unread) == 0 { + return "All caught up!", nil + } + lines := []string{"Unread Channels:", ""} + for _, item := range document.Data.Unread { + label := item.Channel.Name + if item.Channel.Type == "public" || item.Channel.Type == "private" { + label = "#" + label + } + summary := fmt.Sprintf("%d unread", item.UnreadCount) + if item.MentionCount > 0 { + summary += fmt.Sprintf(", %d mentions", item.MentionCount) + } + lines = append(lines, fmt.Sprintf(" %-32s %s", label, summary)) + } + lines = append(lines, "", fmt.Sprintf("Total: %d channels with unread messages", len(document.Data.Unread))) + result := strings.Join(lines, "\n") + if renderedPeek != "" { + result += "\n\n" + renderedPeek + } + return result, nil +} diff --git a/internal/output/unread_test.go b/internal/output/unread_test.go new file mode 100644 index 0000000..999fe0b --- /dev/null +++ b/internal/output/unread_test.go @@ -0,0 +1,248 @@ +package output_test + +import ( + "bytes" + "strings" + "testing" + "time" + + "github.com/ardasevinc/mattermost-cli/internal/output" + "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/internal/schema" +) + +func TestUnreadMachineGoldenSchemaOrderingAndActiveCredential(t *testing.T) { + release := presentation.ActiveCredentials.Register("active-token") + defer release() + raw := []output.RawUnreadItem{ + {Channel: output.RawChannel{ID: "z", Type: "G", Name: "z", TotalMsgCount: 4}, UnreadCount: 2, MentionCount: 1}, + {Channel: output.RawChannel{ID: "a", Type: "G", Name: "active-token\u202e", TotalMsgCount: 4}, UnreadCount: 2, MentionCount: 1}, + } + doc, err := output.NewUnreadEnvelope(raw, nil, output.UnreadProof{}, presentation.Options{DisableHeuristics: true}) + if err != nil { + t.Fatal(err) + } + var wire bytes.Buffer + if _, err := output.WriteMachineJSON(&wire, doc); err != nil { + t.Fatal(err) + } + want := `{"schema":"mm/v2/unread","data":{"unread":[{"channel":{"id":"a","type":"group","name":"[REDACTED:mattermost_credential]\\u202e","displayName":null,"team":null,"lastPost":null,"messageCount":4},"unreadCount":2,"mentionCount":1,"lastViewedAt":null},{"channel":{"id":"z","type":"group","name":"z","displayName":null,"team":null,"lastPost":null,"messageCount":4},"unreadCount":2,"mentionCount":1,"lastViewedAt":null}],"peek":[]}}` + "\n" + if wire.String() != want { + t.Fatalf("got %s\nwant %s", wire.String(), want) + } + registry, err := schema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/unread", bytes.NewReader(wire.Bytes())); err != nil { + t.Fatal(err) + } +} + +func TestUnreadPeekRequiresOneExactCompleteHistoryPerSummary(t *testing.T) { + limit, no := 2, false + since := "1970-01-01T00:00:01.000Z" + peek := []output.MessageOutput{{ + Channel: output.Channel{ID: "c", Type: "group", Name: "crew", MetadataStatus: "resolved"}, + Messages: []output.Message{}, Redactions: []output.Redaction{}, + Retrieval: output.Retrieval{ + Selection: output.Selection{Source: "unread", SelectedCount: 0, RequestedLimit: &limit, Since: &since, QueryTruncated: &no}, + VisibleThreads: output.VisibleThreads{Status: "not_requested", FailedRootIDs: []string{}}, + }, + }} + raw := []output.RawUnreadItem{{Channel: output.RawChannel{ID: "c", Type: "G", Name: "crew"}, UnreadCount: 1, LastViewedAt: 1000}} + doc, err := output.NewUnreadEnvelope(raw, peek, output.UnreadProof{PeekLimit: &limit}, presentation.Options{}) + if err != nil { + t.Fatal(err) + } + if len(doc.Data.Peek) != 1 || doc.Data.Peek[0].Messages == nil || doc.Data.Peek[0].Metadata.Completeness != output.MachineComplete { + t.Fatalf("peek = %+v", doc.Data.Peek) + } + for name, mutate := range map[string]func([]output.MessageOutput){ + "unknown": func(v []output.MessageOutput) { v[0].Retrieval.Selection.QueryTruncated = nil }, + "wrong channel": func(v []output.MessageOutput) { v[0].Channel.ID = "other" }, + "unsafe": func(v []output.MessageOutput) { v[0].Channel.Name = "bad\u202e" }, + "wrong since": func(v []output.MessageOutput) { v[0].Retrieval.Selection.Since = nil }, + } { + t.Run(name, func(t *testing.T) { + copy := append([]output.MessageOutput(nil), peek...) + mutate(copy) + w := &countingWriter{} + bad, err := output.NewUnreadEnvelope(raw, copy, output.UnreadProof{PeekLimit: &limit}, presentation.Options{}) + if err == nil { + _, err = output.WriteMachineJSON(w, bad) + } + if err == nil || w.calls != 0 { + t.Fatalf("error=%v writes=%d", err, w.calls) + } + }) + } +} + +func TestUnreadPeekAllowsMarkdownMultilineAndRejectsNestedCredentialsAndControls(t *testing.T) { + release := presentation.ActiveCredentials.Register("active-token") + defer release() + limit, complete := 2, false + message := output.Message{ID: "p", CanonicalID: "p", Permalink: "https://mm.test/_redirect/pl/p", User: "arda", UserID: "u", Text: "**short**\n\tlong markdown line", Timestamp: time.Unix(2, 0), UpdatedAt: time.Unix(2, 0), Files: []string{}, FileDetails: []output.File{}, Attachments: []output.Attachment{}, Reactions: []output.Reaction{}, Replies: []output.Message{}} + peek := output.MessageOutput{Channel: output.Channel{ID: "c", Type: "group", Name: "crew", MetadataStatus: "resolved"}, Messages: []output.Message{message}, Redactions: []output.Redaction{}, Retrieval: output.Retrieval{Selection: output.Selection{Source: "unread", SelectedCount: 1, RequestedLimit: &limit, QueryTruncated: &complete}, VisibleThreads: output.VisibleThreads{Status: "not_requested", FailedRootIDs: []string{}}, VisiblePostCount: 1}} + raw := []output.RawUnreadItem{{Channel: output.RawChannel{ID: "c", Type: "G", Name: "crew"}, UnreadCount: 1}} + if _, err := output.NewUnreadEnvelope(raw, []output.MessageOutput{peek}, output.UnreadProof{PeekLimit: &limit}, presentation.Options{}); err != nil { + t.Fatalf("valid markdown: %v", err) + } + + for name, mutate := range map[string]func(*output.MessageOutput){ + "nested credential": func(v *output.MessageOutput) { + v.Messages[0].Attachments = []output.Attachment{{Fields: []output.AttachmentField{{Title: "key", Value: "active-token"}}}} + }, + "bidi": func(v *output.MessageOutput) { v.Messages[0].Text = "bad\u061cvalue" }, + "duplicate identity": func(v *output.MessageOutput) { + v.Messages = append(v.Messages, v.Messages[0]) + v.Retrieval.Selection.SelectedCount = 2 + v.Retrieval.VisiblePostCount = 2 + }, + "forged metadata": func(v *output.MessageOutput) { v.Channel.Name = "other" }, + } { + t.Run(name, func(t *testing.T) { + copy := peek + copy.Messages = append([]output.Message(nil), peek.Messages...) + mutate(©) + if _, err := output.NewUnreadEnvelope(raw, []output.MessageOutput{copy}, output.UnreadProof{PeekLimit: &limit}, presentation.Options{}); err == nil { + t.Fatal("invalid peek accepted") + } + }) + } +} + +func TestUnreadMessageGraphRejectsCanonicalCollisionsAndContradictoryRoots(t *testing.T) { + limit, complete := 4, false + base := output.Message{ID: "p", CanonicalID: "cp", Permalink: "https://mm.test/_redirect/pl/p", User: "arda", UserID: "u", Text: "root", Timestamp: time.Unix(2, 0), UpdatedAt: time.Unix(2, 0), Files: []string{}, FileDetails: []output.File{}, Attachments: []output.Attachment{}, Reactions: []output.Reaction{}, Replies: []output.Message{}} + makePeek := func(messages []output.Message) output.MessageOutput { + return output.MessageOutput{Channel: output.Channel{ID: "c", Type: "group", Name: "crew", MetadataStatus: "resolved"}, Messages: messages, Redactions: []output.Redaction{}, Retrieval: output.Retrieval{Selection: output.Selection{Source: "unread", SelectedCount: len(messages), RequestedLimit: &limit, QueryTruncated: &complete}, VisibleThreads: output.VisibleThreads{Status: "not_requested", FailedRootIDs: []string{}}, VisiblePostCount: len(messages)}} + } + raw := []output.RawUnreadItem{{Channel: output.RawChannel{ID: "c", Type: "G", Name: "crew"}, UnreadCount: 1}} + for name, messages := range map[string][]output.Message{ + "canonical collision": {base, func() output.Message { v := base; v.ID = "q"; return v }()}, + "contradictory top root": {func() output.Message { v := base; v.RootID = "p"; return v }()}, + "parent canonical reuse": {func() output.Message { + v := base + v.Replies = []output.Message{{ID: "r", CanonicalID: "cp", RootID: "p", CanonicalRootID: "cp", Permalink: "https://mm.test/r", User: "arda", UserID: "u", Text: "reply", Timestamp: time.Unix(3, 0), UpdatedAt: time.Unix(3, 0), Files: []string{}, FileDetails: []output.File{}, Attachments: []output.Attachment{}, Reactions: []output.Reaction{}, Replies: []output.Message{}}} + return v + }()}, + "nonroot parent with child": {func() output.Message { + v := base + v.RootID, v.CanonicalRootID = "x", "cx" + child := base + child.ID, child.CanonicalID, child.RootID, child.CanonicalRootID = "r", "cr", "p", "cp" + v.Replies = []output.Message{child} + return v + }()}, + } { + t.Run(name, func(t *testing.T) { + peek := makePeek(messages) + flattened := len(messages) + for _, message := range messages { + flattened += len(message.Replies) + } + peek.Retrieval.Selection.SelectedCount, peek.Retrieval.VisiblePostCount = flattened, flattened + if _, err := output.NewUnreadEnvelope(raw, []output.MessageOutput{peek}, output.UnreadProof{PeekLimit: &limit}, presentation.Options{}); err == nil { + t.Fatal("invalid graph accepted") + } + }) + } +} + +func TestUnreadAttachmentMultilineNormalizationContract(t *testing.T) { + limit, complete := 1, false + message := output.Message{ID: "p", CanonicalID: "p", Permalink: "https://mm.test/p", User: "arda", UserID: "u", Text: "body", Timestamp: time.Unix(2, 0), UpdatedAt: time.Unix(2, 0), Files: []string{}, FileDetails: []output.File{}, Reactions: []output.Reaction{}, Replies: []output.Message{}, Attachments: []output.Attachment{{Title: "title\ncontinued", AuthorName: "author\tname", Fields: []output.AttachmentField{{Title: "field\nname", Value: "line 1\nline 2"}}}}} + peek := output.MessageOutput{Channel: output.Channel{ID: "c", Type: "group", Name: "crew", MetadataStatus: "resolved"}, Messages: []output.Message{message}, Redactions: []output.Redaction{}, Retrieval: output.Retrieval{Selection: output.Selection{Source: "unread", SelectedCount: 1, RequestedLimit: &limit, QueryTruncated: &complete}, VisibleThreads: output.VisibleThreads{Status: "not_requested", FailedRootIDs: []string{}}, VisiblePostCount: 1}} + raw := []output.RawUnreadItem{{Channel: output.RawChannel{ID: "c", Type: "G", Name: "crew"}, UnreadCount: 1}} + doc, err := output.NewUnreadEnvelope(raw, []output.MessageOutput{peek}, output.UnreadProof{PeekLimit: &limit}, presentation.Options{}) + if err != nil { + t.Fatal(err) + } + field := doc.Data.Peek[0].Messages[0].Attachments[0] + if field.Title != "title\ncontinued" || field.AuthorName != "author\tname" || field.Fields[0].Title != "field\nname" { + t.Fatalf("attachment = %+v", field) + } +} + +func TestUnreadAttachmentFieldShortMutationDoesNotAlterProof(t *testing.T) { + limit, complete, short := 1, false, true + message := output.Message{ID: "p", CanonicalID: "p", Permalink: "https://mm.test/p", User: "arda", UserID: "u", Text: "body", Timestamp: time.Unix(2, 0), UpdatedAt: time.Unix(2, 0), Files: []string{}, FileDetails: []output.File{}, Reactions: []output.Reaction{}, Replies: []output.Message{}, Attachments: []output.Attachment{{Fields: []output.AttachmentField{{Title: "field", Value: "value", Short: &short}}}}} + peek := output.MessageOutput{Channel: output.Channel{ID: "c", Type: "group", Name: "crew", MetadataStatus: "resolved"}, Messages: []output.Message{message}, Redactions: []output.Redaction{}, Retrieval: output.Retrieval{Selection: output.Selection{Source: "unread", SelectedCount: 1, RequestedLimit: &limit, QueryTruncated: &complete}, VisibleThreads: output.VisibleThreads{Status: "not_requested", FailedRootIDs: []string{}}, VisiblePostCount: 1}} + raw := []output.RawUnreadItem{{Channel: output.RawChannel{ID: "c", Type: "G", Name: "crew"}, UnreadCount: 1}} + doc, err := output.NewUnreadEnvelope(raw, []output.MessageOutput{peek}, output.UnreadProof{PeekLimit: &limit}, presentation.Options{}) + if err != nil { + t.Fatal(err) + } + live := doc.Data.Peek[0].Messages[0].Attachments[0].Fields[0].Short + *live = false + w := &countingWriter{} + if _, err := output.WriteMachineJSON(w, doc); err == nil || w.calls != 0 { + t.Fatalf("error=%v writes=%d", err, w.calls) + } +} + +func TestUnreadStructuralSealUsesMachineBudgetOnce(t *testing.T) { + large := strings.Repeat("x", 3<<20) + doc, err := output.NewUnreadEnvelope([]output.RawUnreadItem{{Channel: output.RawChannel{ID: "c", Type: "G", Name: large}, UnreadCount: 1}}, nil, output.UnreadProof{}, presentation.Options{}) + if err != nil { + t.Fatalf("near-limit constructor: %v", err) + } + var wire bytes.Buffer + if _, err := output.WriteMachineJSON(&wire, doc); err != nil { + t.Fatalf("near-limit write: %v", err) + } + if _, err := output.NewUnreadEnvelope([]output.RawUnreadItem{{Channel: output.RawChannel{ID: "c", Type: "G", Name: strings.Repeat("x", 5<<20)}, UnreadCount: 1}}, nil, output.UnreadProof{}, presentation.Options{}); err == nil { + t.Fatal("oversized candidate accepted") + } +} + +func TestUnreadPeekLimitMustBeSafeInteger(t *testing.T) { + if int64(^uint(0)>>1) <= output.MaxSafeMachineInteger { + t.Skip("int is not wider than safe machine integer") + } + limit := int(output.MaxSafeMachineInteger + 1) + _, err := output.NewUnreadEnvelope(nil, nil, output.UnreadProof{PeekLimit: &limit}, presentation.Options{}) + if err == nil { + t.Fatal("unsafe peek limit accepted") + } +} + +func TestUnreadMutationAndDirectConstructionAreZeroWrite(t *testing.T) { + doc, err := output.NewUnreadEnvelope([]output.RawUnreadItem{{Channel: output.RawChannel{ID: "c", Type: "G", Name: "crew"}, UnreadCount: 1}}, nil, output.UnreadProof{}, presentation.Options{}) + if err != nil { + t.Fatal(err) + } + doc.Data.Unread[0].UnreadCount = 7 + for _, value := range []output.MachineDocument{doc, output.UnreadEnvelope{Schema: "mm/v2/unread"}} { + w := &countingWriter{} + if _, err := output.WriteMachineJSON(w, value); err == nil || w.calls != 0 { + t.Fatalf("error=%v writes=%d", err, w.calls) + } + } +} + +func TestUnreadHumanGoldenAndAllCaughtUp(t *testing.T) { + doc, err := output.NewUnreadEnvelope([]output.RawUnreadItem{{Channel: output.RawChannel{ID: "c", Type: "G", Name: "crew"}, UnreadCount: 4, MentionCount: 2}}, nil, output.UnreadProof{}, presentation.Options{}) + if err != nil { + t.Fatal(err) + } + dates := output.NewDateFormatter(func() time.Time { return time.Unix(0, 0) }, time.UTC) + got, err := output.FormatUnreadPretty(doc, nil, dates, output.PrettyOptions{}) + if err != nil { + t.Fatal(err) + } + want := "Unread Channels:\n\n crew 4 unread, 2 mentions\n\nTotal: 1 channels with unread messages" + if got != want { + t.Fatalf("got %q\nwant %q", got, want) + } + empty, err := output.NewUnreadEnvelope(nil, nil, output.UnreadProof{}, presentation.Options{}) + if err != nil { + t.Fatal(err) + } + got, err = output.FormatUnreadMarkdown(empty, nil, dates, output.MarkdownOptions{}) + if err != nil || strings.TrimSpace(got) != "All caught up!" { + t.Fatalf("empty=%q error=%v", got, err) + } +} diff --git a/internal/schema/unread_test.go b/internal/schema/unread_test.go new file mode 100644 index 0000000..820b8b8 --- /dev/null +++ b/internal/schema/unread_test.go @@ -0,0 +1,66 @@ +package schema + +import ( + "bytes" + "io/fs" + "strings" + "testing" + + publicschemas "github.com/ardasevinc/mattermost-cli/schemas" +) + +func TestUnreadSchemaRequiresNonNullArraysAndHonestHistory(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + for _, document := range []string{ + `{"schema":"mm/v2/unread","data":{"unread":null,"peek":[]}}`, + `{"schema":"mm/v2/unread","data":{"unread":[],"peek":null}}`, + `{"schema":"mm/v2/unread","data":{"unread":[{"channel":{},"unreadCount":0,"mentionCount":-1,"lastViewedAt":null}],"peek":[]}}`, + } { + if err := registry.Validate("mm/v2/unread", strings.NewReader(document)); err == nil { + t.Fatalf("accepted %s", document) + } + } +} + +func TestUnreadSchemaRejectsEveryPeekRestrictionMutation(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + valid, err := fs.ReadFile(publicschemas.FS, "v2/examples/unread.json") + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/unread", bytes.NewReader(valid)); err != nil { + t.Fatal(err) + } + mutations := map[string][2]string{ + "unresolved": {`"metadataStatus":"resolved"`, `"metadataStatus":"unavailable"`}, + "unknown channel": {`"type":"public","name":"town-square","displayName":"Town Square","metadataStatus":"resolved"`, `"type":"unknown","name":"town-square","displayName":"Town Square","metadataStatus":"unavailable"`}, + "missing limit": {`"requestedLimit":2`, `"requestedLimit":null`}, + "unsafe limit": {`"requestedLimit":2`, `"requestedLimit":9007199254740992`}, + "bad since": {`"since":"2026-07-16T00:00:00.000Z"`, `"since":"not-a-time"`}, + "input cursor": {`"inputCursor":null`, `"inputCursor":"cursor"`}, + "next cursor": {`"nextCursor":null`, `"nextCursor":"cursor"`}, + "unknown truncation": {`"queryTruncated":false`, `"queryTruncated":null`}, + "thread status": {`"status":"not_requested"`, `"status":"complete"`}, + "hydrated": {`"hydratedRootCount":0`, `"hydratedRootCount":1`}, + "failed root": {`"failedRootIds":[]`, `"failedRootIds":["p1"]`}, + "unknown completeness": {`"completeness":"complete"`, `"completeness":"unknown"`}, + "contradictory complete": {`"queryTruncated":false`, `"queryTruncated":true`}, + } + for name, replacement := range mutations { + t.Run(name, func(t *testing.T) { + document := strings.Replace(string(valid), replacement[0], replacement[1], 1) + if document == string(valid) { + t.Fatal("mutation did not apply") + } + if err := registry.Validate("mm/v2/unread", strings.NewReader(document)); err == nil { + t.Fatal("mutation accepted") + } + }) + } +} diff --git a/schemas/v2/examples/unread.json b/schemas/v2/examples/unread.json new file mode 100644 index 0000000..1b7bda1 --- /dev/null +++ b/schemas/v2/examples/unread.json @@ -0,0 +1 @@ +{"schema":"mm/v2/unread","data":{"unread":[{"channel":{"id":"c1","type":"public","name":"town-square","displayName":"Town Square","team":{"id":"t1","name":"core","displayName":"Core"},"lastPost":"2026-07-16T01:02:03.456Z","messageCount":4},"unreadCount":1,"mentionCount":1,"lastViewedAt":"2026-07-16T00:00:00.000Z"}],"peek":[{"channel":{"id":"c1","type":"public","name":"town-square","displayName":"Town Square","metadataStatus":"resolved"},"messages":[{"id":"p1","permalink":"https://mm.example/team/pl/p1","user":"arda","userId":"u1","text":"hello","timestamp":"2026-07-16T01:02:03.456Z","updatedAt":"2026-07-16T01:02:03.456Z","editedAt":null,"deletedAt":null,"isDeleted":false,"postType":"","isSystem":false,"isPinned":false,"rootId":null,"replyCount":0,"files":[],"fileDetails":[],"attachments":[],"reactions":[],"replies":[]}],"redactions":[],"metadata":{"completeness":"complete","selection":{"source":"unread","selectedCount":1,"requestedLimit":2,"since":"2026-07-16T00:00:00.000Z","queryTruncated":false,"inputCursor":null,"nextCursor":null},"visibleThreads":{"status":"not_requested","hydratedRootCount":0,"failedRootIds":[]},"visiblePostCount":1,"deletedPostsIncluded":false}}]}} diff --git a/schemas/v2/unread.schema.json b/schemas/v2/unread.schema.json new file mode 100644 index 0000000..158f0ad --- /dev/null +++ b/schemas/v2/unread.schema.json @@ -0,0 +1,831 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:unread", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "data" + ], + "properties": { + "schema": { + "const": "mm/v2/unread" + }, + "data": { + "type": "object", + "additionalProperties": false, + "required": [ + "unread", + "peek" + ], + "properties": { + "unread": { + "type": "array", + "items": { + "$ref": "#/$defs/unreadItem" + } + }, + "peek": { + "type": "array", + "items": { + "$ref": "#/$defs/unreadHistory" + } + } + } + } + }, + "$defs": { + "timestamp": { + "type": "string", + "format": "date-time", + "pattern": "^(?:000[1-9]|00[1-9][0-9]|0[1-9][0-9]{2}|[1-9][0-9]{3})-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])T(?:[01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\\.[0-9]{3}Z$" + }, + "nullableTimestamp": { + "anyOf": [ + { + "$ref": "#/$defs/timestamp" + }, + { + "type": "null" + } + ] + }, + "file": { + "type": "object", + "additionalProperties": false, + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "mime": { + "type": "string" + }, + "size": { + "type": "integer", + "minimum": 0 + }, + "extension": { + "type": "string" + } + } + }, + "attachmentField": { + "type": "object", + "additionalProperties": false, + "properties": { + "title": { + "type": "string" + }, + "value": { + "type": "string" + }, + "short": { + "type": "boolean" + } + } + }, + "attachment": { + "type": "object", + "additionalProperties": false, + "properties": { + "fallback": { + "type": "string" + }, + "pretext": { + "type": "string" + }, + "title": { + "type": "string" + }, + "titleLink": { + "type": "string" + }, + "text": { + "type": "string" + }, + "fields": { + "type": "array", + "items": { + "$ref": "#/$defs/attachmentField" + } + }, + "footer": { + "type": "string" + }, + "footerIcon": { + "type": "string" + }, + "authorName": { + "type": "string" + }, + "authorLink": { + "type": "string" + }, + "authorIcon": { + "type": "string" + }, + "color": { + "type": "string" + }, + "imageUrl": { + "type": "string" + }, + "thumbUrl": { + "type": "string" + }, + "timestamp": { + "type": "string" + } + } + }, + "reactionActor": { + "type": "object", + "additionalProperties": false, + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string" + }, + "username": { + "type": "string" + } + } + }, + "reaction": { + "type": "object", + "additionalProperties": false, + "required": [ + "emoji", + "count", + "actors" + ], + "properties": { + "emoji": { + "type": "string" + }, + "count": { + "type": "integer", + "minimum": 0 + }, + "actors": { + "type": "array", + "items": { + "$ref": "#/$defs/reactionActor" + } + } + } + }, + "redaction": { + "type": "object", + "additionalProperties": false, + "required": [ + "type", + "masked", + "position" + ], + "properties": { + "type": { + "type": "string" + }, + "masked": { + "type": "string" + }, + "position": { + "type": "integer", + "minimum": 0 + }, + "field": { + "type": "string" + } + } + }, + "channel": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "type", + "name", + "displayName", + "metadataStatus" + ], + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": [ + "dm", + "public", + "private", + "group", + "unknown" + ] + }, + "name": { + "type": "string" + }, + "displayName": { + "type": "string" + }, + "metadataStatus": { + "enum": [ + "resolved", + "unavailable" + ] + } + } + }, + "selection": { + "type": "object", + "additionalProperties": false, + "required": [ + "source", + "selectedCount", + "requestedLimit", + "since", + "queryTruncated", + "inputCursor", + "nextCursor" + ], + "properties": { + "source": { + "enum": [ + "recent", + "search", + "mentions", + "unread", + "thread" + ] + }, + "selectedCount": { + "type": "integer", + "minimum": 0 + }, + "requestedLimit": { + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "since": { + "type": [ + "string", + "null" + ] + }, + "queryTruncated": { + "type": [ + "boolean", + "null" + ] + }, + "inputCursor": { + "type": [ + "string", + "null" + ] + }, + "nextCursor": { + "type": [ + "string", + "null" + ] + } + } + }, + "visibleThreads": { + "type": "object", + "additionalProperties": false, + "required": [ + "status", + "hydratedRootCount", + "failedRootIds" + ], + "properties": { + "status": { + "enum": [ + "not_requested", + "complete", + "partial" + ] + }, + "hydratedRootCount": { + "type": "integer", + "minimum": 0 + }, + "failedRootIds": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "metadata": { + "type": "object", + "additionalProperties": false, + "required": [ + "completeness", + "selection", + "visibleThreads", + "visiblePostCount", + "deletedPostsIncluded" + ], + "properties": { + "completeness": { + "enum": [ + "complete", + "truncated", + "unknown" + ] + }, + "selection": { + "$ref": "#/$defs/selection" + }, + "visibleThreads": { + "$ref": "#/$defs/visibleThreads" + }, + "visiblePostCount": { + "type": "integer", + "minimum": 0 + }, + "deletedPostsIncluded": { + "const": false + } + } + }, + "message": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "permalink", + "user", + "userId", + "text", + "timestamp", + "updatedAt", + "editedAt", + "deletedAt", + "isDeleted", + "postType", + "isSystem", + "isPinned", + "rootId", + "replyCount", + "files", + "fileDetails", + "attachments", + "reactions", + "replies" + ], + "properties": { + "id": { + "type": "string" + }, + "permalink": { + "type": "string" + }, + "user": { + "type": "string" + }, + "userId": { + "type": "string" + }, + "text": { + "type": "string" + }, + "timestamp": { + "$ref": "#/$defs/timestamp" + }, + "updatedAt": { + "$ref": "#/$defs/timestamp" + }, + "editedAt": { + "$ref": "#/$defs/nullableTimestamp" + }, + "deletedAt": { + "$ref": "#/$defs/nullableTimestamp" + }, + "isDeleted": { + "type": "boolean" + }, + "postType": { + "type": "string" + }, + "isSystem": { + "type": "boolean" + }, + "isPinned": { + "type": "boolean" + }, + "rootId": { + "type": [ + "string", + "null" + ] + }, + "replyCount": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "files": { + "type": "array", + "items": { + "type": "string" + } + }, + "fileDetails": { + "type": "array", + "items": { + "$ref": "#/$defs/file" + } + }, + "attachments": { + "type": "array", + "items": { + "$ref": "#/$defs/attachment" + } + }, + "reactions": { + "type": "array", + "items": { + "$ref": "#/$defs/reaction" + } + }, + "replies": { + "type": "array", + "items": { + "$ref": "#/$defs/message" + } + } + } + }, + "history": { + "type": "object", + "additionalProperties": false, + "required": [ + "channel", + "messages", + "redactions", + "metadata" + ], + "properties": { + "channel": { + "$ref": "#/$defs/channel" + }, + "messages": { + "type": "array", + "items": { + "$ref": "#/$defs/message" + } + }, + "redactions": { + "type": "array", + "items": { + "$ref": "#/$defs/redaction" + } + }, + "metadata": { + "$ref": "#/$defs/metadata" + } + } + }, + "unreadItem": { + "type": "object", + "additionalProperties": false, + "required": [ + "channel", + "unreadCount", + "mentionCount", + "lastViewedAt" + ], + "properties": { + "channel": { + "$ref": "#/$defs/directoryChannel" + }, + "unreadCount": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "mentionCount": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "lastViewedAt": { + "anyOf": [ + { + "$ref": "#/$defs/timestamp" + }, + { + "type": "null" + } + ] + } + } + }, + "directoryChannel": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "type", + "name", + "displayName", + "team", + "lastPost", + "messageCount" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "type": { + "enum": [ + "dm", + "public", + "private", + "group" + ] + }, + "name": { + "type": "string", + "minLength": 1 + }, + "displayName": { + "$ref": "#/$defs/nullableNonEmptyString" + }, + "team": { + "anyOf": [ + { + "$ref": "#/$defs/team" + }, + { + "type": "null" + } + ] + }, + "lastPost": { + "anyOf": [ + { + "$ref": "#/$defs/timestamp" + }, + { + "type": "null" + } + ] + }, + "messageCount": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "allOf": [ + { + "if": { + "properties": { + "type": { + "enum": [ + "public", + "private" + ] + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "team": { + "$ref": "#/$defs/team" + } + } + } + }, + { + "if": { + "properties": { + "type": { + "enum": [ + "dm", + "group" + ] + } + }, + "required": [ + "type" + ] + }, + "then": { + "properties": { + "team": { + "type": "null" + }, + "displayName": { + "type": "null" + } + } + } + } + ] + }, + "unreadHistory": { + "allOf": [ + { + "$ref": "#/$defs/history" + }, + { + "properties": { + "channel": { + "properties": { + "type": { + "enum": [ + "dm", + "public", + "private", + "group" + ] + }, + "metadataStatus": { + "const": "resolved" + } + } + }, + "metadata": { + "properties": { + "selection": { + "properties": { + "source": { + "const": "unread" + }, + "requestedLimit": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "since": { + "anyOf": [ + { + "$ref": "#/$defs/timestamp" + }, + { + "type": "null" + } + ] + }, + "queryTruncated": { + "type": "boolean" + }, + "inputCursor": { + "type": "null" + }, + "nextCursor": { + "type": "null" + } + } + }, + "visibleThreads": { + "properties": { + "status": { + "const": "not_requested" + }, + "hydratedRootCount": { + "const": 0 + }, + "failedRootIds": { + "maxItems": 0 + } + } + }, + "deletedPostsIncluded": { + "const": false + } + } + } + } + }, + { + "properties": { + "metadata": { + "properties": { + "completeness": { + "enum": [ + "complete", + "truncated" + ] + } + } + } + } + }, + { + "if": { + "properties": { + "metadata": { + "properties": { + "completeness": { + "const": "complete" + } + } + } + } + }, + "then": { + "properties": { + "metadata": { + "properties": { + "selection": { + "properties": { + "queryTruncated": { + "const": false + } + } + } + } + } + } + } + }, + { + "if": { + "properties": { + "metadata": { + "properties": { + "completeness": { + "const": "truncated" + } + } + } + } + }, + "then": { + "properties": { + "metadata": { + "properties": { + "selection": { + "properties": { + "queryTruncated": { + "const": true + } + } + } + } + } + } + } + } + ] + }, + "team": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "name", + "displayName" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "name": { + "type": "string", + "minLength": 1 + }, + "displayName": { + "$ref": "#/$defs/nullableNonEmptyString" + } + } + }, + "nullableNonEmptyString": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "null" + } + ] + } + } +} From bcf26a318aea39aecfb999904cd69989badd7ec7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 20:52:35 +0300 Subject: [PATCH 046/119] feat: add unread command --- internal/cli/root.go | 1 + internal/cli/unread.go | 156 +++++++++++++++++++++ internal/cli/unread_test.go | 272 ++++++++++++++++++++++++++++++++++++ 3 files changed, 429 insertions(+) create mode 100644 internal/cli/unread.go create mode 100644 internal/cli/unread_test.go diff --git a/internal/cli/root.go b/internal/cli/root.go index a652503..1d3032b 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -128,6 +128,7 @@ func newRootWithState(state *rootState) *cobra.Command { cmd.AddCommand(newThreadCommand(state)) cmd.AddCommand(newSearchCommand(state)) cmd.AddCommand(newMentionsCommand(state)) + cmd.AddCommand(newUnreadCommand(state)) return cmd } diff --git a/internal/cli/unread.go b/internal/cli/unread.go new file mode 100644 index 0000000..1840d73 --- /dev/null +++ b/internal/cli/unread.go @@ -0,0 +1,156 @@ +package cli + +import ( + "bytes" + "sort" + "strings" + "time" + + "github.com/spf13/cobra" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/output" + "github.com/ardasevinc/mattermost-cli/internal/retrieval" +) + +type unreadFlags struct { + team string + peek string +} + +func newUnreadCommand(state *rootState) *cobra.Command { + flags := new(unreadFlags) + command := &cobra.Command{Use: "unread", Short: "Show unread channels", Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { return runUnread(cmd, state, *flags) }} + command.Flags().StringVar(&flags.team, "team", "", "exact team name or display name") + command.Flags().StringVar(&flags.peek, "peek", "", "maximum messages to preview per unread channel") + return command +} + +func runUnread(cmd *cobra.Command, state *rootState, flags unreadFlags) error { + if flagChanged(cmd, "team") && (strings.TrimSpace(flags.team) == "" || strings.TrimSpace(flags.team) != flags.team) { + return invalidFailure("--team must be a non-empty exact selector") + } + peekLimit := 0 + var err error + if flagChanged(cmd, "peek") { + peekLimit, err = positiveInteger(flags.peek) + if err != nil { + return invalidFailure("--peek must be a positive number") + } + } + display, err := state.readDisplay(cmd) + if err != nil { + return err + } + runtime, err := state.runtimeFor(cmd) + if err != nil { + return err + } + if err := emitRedactionWarning(state, runtime, display.json); err != nil { + return err + } + result, err := retrieval.Unread(cmd.Context(), runtime.Users, runtime.Teams, runtime.Channels, runtime.Posts, retrieval.UnreadOptions{TeamSelector: flags.team, PeekLimit: peekLimit}) + if err != nil { + return readFailure(err) + } + + allPosts := make([]mattermost.Post, 0) + peerIDs := make([]string, 0) + for _, entry := range result.Entries { + allPosts = append(allPosts, entry.Peek...) + if entry.Channel.Type == "D" { + peerID := otherDMUserID(entry.Channel, result.User.ID) + if peerID == "" { + return readError("Mattermost returned an invalid direct channel identity") + } + peerIDs = append(peerIDs, peerID) + } + } + sort.Strings(peerIDs) + users, err := loadReadUsersAndIDs(cmd, runtime, allPosts, peerIDs) + if err != nil { + return readFailure(err) + } + + raw := make([]output.RawUnreadItem, len(result.Entries)) + sections := make([]output.MessageOutput, 0, len(result.Entries)) + for i, entry := range result.Entries { + channel := entry.Channel + rawChannel := output.RawChannel{ID: channel.ID, Type: channel.Type, Name: channel.Name, DisplayName: channel.DisplayName, TeamID: channel.TeamID, LastPostAt: channel.LastPostAt, TotalMsgCount: channel.TotalMsgCount} + var presented output.Channel + var channelRedactions []output.Redaction + switch channel.Type { + case "D": + peer := users[otherDMUserID(channel, result.User.ID)] + if peer.ID == "" { + return readError("Mattermost returned incomplete direct-message identity metadata") + } + rawChannel.DirectUsername = peer.Username + presented, channelRedactions = presentedDMChannel(channel, peer, runtime) + case "G": + presented, channelRedactions = processedChannel(channel, runtime) + case "O", "P": + if channel.TeamID != result.Team.ID { + return readError("Mattermost returned a channel outside the selected team") + } + team := output.RawTeam{ID: result.Team.ID, Name: result.Team.Name, DisplayName: result.Team.DisplayName, Type: result.Team.Type} + rawChannel.Team = &team + presented, channelRedactions = processedChannel(channel, runtime) + default: + return readError("Mattermost returned an unsupported unread channel type") + } + raw[i] = output.RawUnreadItem{Channel: rawChannel, UnreadCount: entry.UnreadCount, MentionCount: entry.MentionCount, LastViewedAt: entry.LastViewedAt} + if peekLimit == 0 { + continue + } + if entry.PeekState == retrieval.CompletenessUnknown { + return readError("Mattermost could not prove a complete unread preview") + } + messages, redactions, normalizeErr := normalizeReadPostsWithUsers(runtime, entry.Peek, users, result.User.ID) + if normalizeErr != nil { + return readFailure(normalizeErr) + } + redactions = append(channelRedactions, redactions...) + limit := peekLimit + sections = append(sections, output.MessageOutput{Channel: presented, Messages: messages, Redactions: redactions, Retrieval: output.Retrieval{ + Selection: output.Selection{Source: "unread", SelectedCount: len(entry.Peek), RequestedLimit: &limit, Since: unreadSince(entry.LastViewedAt), QueryTruncated: truncatedPointer(entry.PeekState)}, + VisibleThreads: output.VisibleThreads{Status: "not_requested", FailedRootIDs: []string{}}, VisiblePostCount: len(entry.Peek), DeletedPostsIncluded: false, + }}) + } + + proof := output.UnreadProof{} + if peekLimit > 0 { + proof.PeekLimit = &peekLimit + } + document, err := output.NewUnreadEnvelope(raw, sections, proof, identityOptions(runtime)) + if err != nil { + return readFailure(err) + } + if display.json { + var wire bytes.Buffer + if _, err := output.WriteMachineJSON(&wire, document); err != nil { + return readFailure(err) + } + return writeAll(state.streams.out, wire.Bytes()) + } + dates := output.NewDateFormatter(time.Now, time.Local) + var rendered string + if state.deps.stdoutTTY() { + rendered, err = output.FormatUnreadPretty(document, sections, dates, output.PrettyOptions{Color: display.color, Relative: display.relative}) + } else { + rendered, err = output.FormatUnreadMarkdown(document, sections, dates, output.MarkdownOptions{Relative: display.relative}) + } + if err != nil { + return readFailure(err) + } + return writeAll(state.streams.out, []byte(rendered+"\n")) +} + +func unreadSince(milliseconds int64) *string { + if milliseconds == 0 { + return nil + } + value := time.UnixMilli(milliseconds).UTC().Format("2006-01-02T15:04:05.000Z") + return &value +} diff --git a/internal/cli/unread_test.go b/internal/cli/unread_test.go new file mode 100644 index 0000000..bffc52a --- /dev/null +++ b/internal/cli/unread_test.go @@ -0,0 +1,272 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" +) + +func TestUnreadValidatesFlagsBeforeNetwork(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { requests.Add(1) })) + defer server.Close() + for _, args := range [][]string{{"unread", "--peek", "0"}, {"unread", "--peek", "nope"}, {"unread", "--team", " "}} { + _, _, code := executeChannel(t, server.URL, args...) + if code != 2 || requests.Load() != 0 { + t.Fatalf("args=%v exit=%d requests=%d", args, code, requests.Load()) + } + } +} + +func TestUnreadMachineVerticalSliceBindsAllChannelTypesAndPeekOrder(t *testing.T) { + var teamCalls, postCalls, userCalls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v4/users/me": + writeJSON(t, w, `{"id":"u","username":"me"}`) + case "/api/v4/users/u/teams": + teamCalls.Add(1) + writeJSON(t, w, `[{"id":"t","name":"core","display_name":"Core","type":"O"}]`) + case "/api/v4/users/u/channels": + writeJSON(t, w, `[ + {"id":"d","team_id":"","type":"D","name":"u__peer","display_name":"","total_msg_count":4}, + {"id":"g","team_id":"","type":"G","name":"opaque","display_name":"Crew","total_msg_count":3}, + {"id":"o","team_id":"t","type":"O","name":"general","display_name":"General","total_msg_count":2}, + {"id":"p","team_id":"t","type":"P","name":"private","display_name":"Private","total_msg_count":2}]`) + case "/api/v4/users/u/teams/t/channels/members": + writeJSON(t, w, `[ + {"channel_id":"d","user_id":"u","msg_count":1,"mention_count":3,"last_viewed_at":0}, + {"channel_id":"g","user_id":"u","msg_count":1,"mention_count":2,"last_viewed_at":0}, + {"channel_id":"o","user_id":"u","msg_count":1,"mention_count":1,"last_viewed_at":0}, + {"channel_id":"p","user_id":"u","msg_count":1,"mention_count":0,"last_viewed_at":0}]`) + case "/api/v4/channels/d/posts": + postCalls.Add(1) + post := `{"id":"pd","channel_id":"d","user_id":"peer","message":"**short**\nlong markdown","create_at":2,"update_at":2,"delete_at":0,"root_id":"","reply_count":0}` + writeJSON(t, w, `{"order":["pd"],"posts":{"pd":`+post+`},"has_next":false}`) + case "/api/v4/channels/g/posts", "/api/v4/channels/o/posts", "/api/v4/channels/p/posts": + postCalls.Add(1) + writeJSON(t, w, `{"order":[],"posts":{},"has_next":false}`) + case "/api/v4/users/ids": + userCalls.Add(1) + writeJSON(t, w, `[{"id":"peer","username":"bob"}]`) + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.String()) + } + })) + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "--json", "unread", "--team", "core", "--peek", "2") + if code != 0 || stderr != "" { + t.Fatalf("exit=%d stderr=%q stdout=%s", code, stderr, stdout) + } + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/unread", strings.NewReader(stdout)); err != nil { + t.Fatalf("schema: %v\n%s", err, stdout) + } + var document struct { + Data struct { + Unread []struct { + Channel struct{ ID, Type, Name string } `json:"channel"` + Mention int64 `json:"mentionCount"` + } `json:"unread"` + Peek []struct { + Channel struct{ ID string } `json:"channel"` + Messages []struct{ Text string } `json:"messages"` + } `json:"peek"` + } `json:"data"` + } + if err := json.Unmarshal([]byte(stdout), &document); err != nil { + t.Fatal(err) + } + ids := make([]string, len(document.Data.Unread)) + for i := range ids { + ids[i] = document.Data.Unread[i].Channel.ID + if document.Data.Peek[i].Channel.ID != ids[i] { + t.Fatalf("peek order mismatch") + } + } + if strings.Join(ids, ",") != "d,g,o,p" || len(document.Data.Peek) != 4 || len(document.Data.Peek[1].Messages) != 0 || document.Data.Unread[0].Channel.Name != "@bob" || !strings.Contains(document.Data.Peek[0].Messages[0].Text, "\n") { + t.Fatalf("document=%+v", document) + } + if teamCalls.Load() != 2 || postCalls.Load() != 4 || userCalls.Load() != 1 { + t.Fatalf("teams=%d posts=%d users=%d", teamCalls.Load(), postCalls.Load(), userCalls.Load()) + } +} + +func TestUnreadEmptyMachineAndHumanAreExact(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v4/users/me": + writeJSON(t, w, `{"id":"u","username":"me"}`) + case "/api/v4/users/u/teams": + writeJSON(t, w, `[{"id":"t","name":"core","display_name":"Core","type":"O"}]`) + case "/api/v4/users/u/channels", "/api/v4/users/u/teams/t/channels/members": + writeJSON(t, w, `[]`) + default: + t.Fatalf("unexpected %s", r.URL.Path) + } + })) + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "--json", "unread") + if code != 0 || stderr != "" || stdout != `{"schema":"mm/v2/unread","data":{"unread":[],"peek":[]}}`+"\n" { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } + stdout, stderr, code = executeChannel(t, server.URL, "unread") + if code != 0 || stderr != "" || stdout != "All caught up!\n" { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } +} + +func TestUnreadPipeUsesSummaryThenMarkdown(t *testing.T) { + server := singleUnreadServer(t, "hello") + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "unread", "--peek", "1") + if code != 0 || stderr != "" || !strings.HasPrefix(stdout, "Unread Channels:\n") || !strings.Contains(stdout, "\n\n## Group DM: Crew") || !strings.Contains(stdout, "hello") { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } +} + +func TestUnreadTTYUsesExistingPrettySections(t *testing.T) { + server := singleUnreadServer(t, "hello") + defer server.Close() + setChannelEnvironment(t, server.URL) + var stdout, stderr bytes.Buffer + state := &rootState{streams: streams{in: strings.NewReader(""), out: &stdout, err: &stderr}, deps: defaultDependencies(&stdout)} + state.deps.stdoutTTY = func() bool { return true } + command := newRootWithState(state) + command.SetArgs([]string{"--no-color", "unread", "--peek", "1"}) + err := command.ExecuteContext(context.Background()) + state.close() + if err != nil || stderr.Len() != 0 || !strings.Contains(stdout.String(), "\n\nGroup DM: Crew") || strings.Contains(stdout.String(), "## Group DM") { + t.Fatalf("error=%v stdout=%q stderr=%q", err, stdout.String(), stderr.String()) + } +} + +func TestUnreadIncompletePeekFailsWithEmptyMachineStdout(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v4/users/me": + writeJSON(t, w, `{"id":"u","username":"me"}`) + case "/api/v4/users/u/teams": + writeJSON(t, w, `[{"id":"t","name":"core","display_name":"Core","type":"O"}]`) + case "/api/v4/users/u/channels": + writeJSON(t, w, `[{"id":"g","team_id":"","type":"G","name":"opaque","display_name":"Crew","total_msg_count":2}]`) + case "/api/v4/users/u/teams/t/channels/members": + writeJSON(t, w, `[{"channel_id":"g","user_id":"u","msg_count":1,"mention_count":0,"last_viewed_at":0}]`) + case "/api/v4/channels/g/posts": + writeJSON(t, w, `{"order":[],"posts":{},"has_next":true}`) + default: + t.Fatalf("unexpected %s", r.URL.Path) + } + })) + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "--json", "unread", "--peek", "1") + if code == 0 || stdout != "" || stderr == "" { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } +} + +func TestUnreadMachineWriterFailure(t *testing.T) { + server := singleUnreadServer(t, "hello") + defer server.Close() + setChannelEnvironment(t, server.URL) + var stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "unread"}, strings.NewReader(""), zeroErrorWriter{}, &stderr) + if code != 3 || !strings.Contains(stderr.String(), `"code":"internal"`) { + t.Fatalf("exit=%d stderr=%q", code, stderr.String()) + } +} + +func TestUnreadNestedCredentialIsNormalizedBeforeEnvelope(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v4/users/me": + writeJSON(t, w, `{"id":"u","username":"me"}`) + case "/api/v4/users/u/teams": + writeJSON(t, w, `[{"id":"t","name":"core","display_name":"Core","type":"O"}]`) + case "/api/v4/users/u/channels": + writeJSON(t, w, `[{"id":"g","team_id":"","type":"G","name":"opaque","display_name":"Crew","total_msg_count":2}]`) + case "/api/v4/users/u/teams/t/channels/members": + writeJSON(t, w, `[{"channel_id":"g","user_id":"u","msg_count":1,"mention_count":0,"last_viewed_at":0}]`) + case "/api/v4/channels/g/posts": + post := `{"id":"p","channel_id":"g","user_id":"u","message":"safe","create_at":2,"update_at":2,"delete_at":0,"root_id":"","reply_count":0,"props":{"attachments":[{"fields":[{"title":"secret","value":"test-token","short":true}]}]}}` + writeJSON(t, w, `{"order":["p"],"posts":{"p":`+post+`},"has_next":false}`) + case "/api/v4/users/ids": + writeJSON(t, w, `[{"id":"u","username":"me"}]`) + default: + t.Fatalf("unexpected %s", r.URL.Path) + } + })) + defer server.Close() + stdout, stderr, code := executeChannel(t, server.URL, "--json", "unread", "--peek", "1") + if code != 0 || stderr != "" || strings.Contains(stdout, "test-token") || !strings.Contains(stdout, "[REDACTED:mattermost_credential]") { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout, stderr) + } +} + +func TestUnreadCancellationPropagatesWithoutOutput(t *testing.T) { + started := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v4/users/me": + writeJSON(t, w, `{"id":"u","username":"me"}`) + case "/api/v4/users/u/teams": + writeJSON(t, w, `[{"id":"t","name":"core","display_name":"Core","type":"O"}]`) + case "/api/v4/users/u/channels": + writeJSON(t, w, `[{"id":"g","team_id":"","type":"G","name":"opaque","display_name":"Crew","total_msg_count":2}]`) + case "/api/v4/users/u/teams/t/channels/members": + writeJSON(t, w, `[{"channel_id":"g","user_id":"u","msg_count":1,"mention_count":0,"last_viewed_at":0}]`) + case "/api/v4/channels/g/posts": + close(started) + <-r.Context().Done() + default: + t.Fatalf("unexpected %s", r.URL.Path) + } + })) + defer server.Close() + setChannelEnvironment(t, server.URL) + ctx, cancel := context.WithCancel(context.Background()) + var stdout, stderr bytes.Buffer + done := make(chan int, 1) + go func() { + done <- Execute(ctx, []string{"--json", "unread", "--peek", "1"}, strings.NewReader(""), &stdout, &stderr) + }() + <-started + cancel() + if code := <-done; code == 0 || stdout.Len() != 0 || stderr.Len() == 0 { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } +} + +func singleUnreadServer(t *testing.T, message string) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v4/users/me": + writeJSON(t, w, `{"id":"u","username":"me"}`) + case "/api/v4/users/u/teams": + writeJSON(t, w, `[{"id":"t","name":"core","display_name":"Core","type":"O"}]`) + case "/api/v4/users/u/channels": + writeJSON(t, w, `[{"id":"g","team_id":"","type":"G","name":"opaque","display_name":"Crew","total_msg_count":2}]`) + case "/api/v4/users/u/teams/t/channels/members": + writeJSON(t, w, `[{"channel_id":"g","user_id":"u","msg_count":1,"mention_count":0,"last_viewed_at":0}]`) + case "/api/v4/channels/g/posts": + post := fmt.Sprintf(`{"id":"p","channel_id":"g","user_id":"u","message":%q,"create_at":2,"update_at":2,"delete_at":0,"root_id":"","reply_count":0}`, message) + writeJSON(t, w, `{"order":["p"],"posts":{"p":`+post+`},"has_next":false}`) + case "/api/v4/users/ids": + writeJSON(t, w, `[{"id":"u","username":"me"}]`) + default: + t.Fatalf("unexpected %s", r.URL.Path) + } + })) +} From a65de9de74f3760118140dc94df771a2e48409fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 21:33:39 +0300 Subject: [PATCH 047/119] feat: add watch protocol foundation --- go.mod | 1 + go.sum | 2 + internal/cli/root_test.go | 2 +- internal/mattermost/watch.go | 526 ++++++++++++++++++++++ internal/mattermost/watch_test.go | 286 ++++++++++++ internal/output/watch.go | 251 +++++++++++ internal/output/watch_test.go | 155 +++++++ internal/schema/watch_test.go | 35 ++ internal/transport/websocket.go | 60 +++ internal/transport/websocket_test.go | 15 + schemas/v2/examples/watch-diagnostic.json | 1 + schemas/v2/examples/watch-event.json | 1 + schemas/v2/watch-diagnostic.schema.json | 12 + schemas/v2/watch-event.schema.json | 33 ++ 14 files changed, 1379 insertions(+), 1 deletion(-) create mode 100644 internal/mattermost/watch.go create mode 100644 internal/mattermost/watch_test.go create mode 100644 internal/output/watch.go create mode 100644 internal/output/watch_test.go create mode 100644 internal/schema/watch_test.go create mode 100644 internal/transport/websocket.go create mode 100644 internal/transport/websocket_test.go create mode 100644 schemas/v2/examples/watch-diagnostic.json create mode 100644 schemas/v2/examples/watch-event.json create mode 100644 schemas/v2/watch-diagnostic.schema.json create mode 100644 schemas/v2/watch-event.schema.json diff --git a/go.mod b/go.mod index 8a522b5..6c082b5 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require github.com/pelletier/go-toml/v2 v2.4.3 require golang.org/x/net v0.57.0 require ( + github.com/coder/websocket v1.8.15 golang.org/x/sys v0.47.0 golang.org/x/text v0.40.0 ) diff --git a/go.sum b/go.sum index 8799a06..d70ba59 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= +github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index f0f4072..29113ef 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -57,7 +57,7 @@ func TestSchemaList(t *testing.T) { if code != 0 { t.Fatalf("exit code = %d, want 0; stderr = %q", code, stderr.String()) } - if got, want := stdout.String(), "mm/v2/channel\nmm/v2/channels\nmm/v2/config\nmm/v2/dms\nmm/v2/doctor\nmm/v2/error\nmm/v2/group-dms\nmm/v2/mentions\nmm/v2/search\nmm/v2/teams\nmm/v2/thread\nmm/v2/unread\nmm/v2/users\nmm/v2/whoami\n"; got != want { + if got, want := stdout.String(), "mm/v2/channel\nmm/v2/channels\nmm/v2/config\nmm/v2/dms\nmm/v2/doctor\nmm/v2/error\nmm/v2/group-dms\nmm/v2/mentions\nmm/v2/search\nmm/v2/teams\nmm/v2/thread\nmm/v2/unread\nmm/v2/users\nmm/v2/watch-diagnostic\nmm/v2/watch-event\nmm/v2/whoami\n"; got != want { t.Fatalf("stdout = %q, want %q", got, want) } } diff --git a/internal/mattermost/watch.go b/internal/mattermost/watch.go new file mode 100644 index 0000000..56f4a69 --- /dev/null +++ b/internal/mattermost/watch.go @@ -0,0 +1,526 @@ +package mattermost + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net/url" + "strconv" + "strings" + "time" + + "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/internal/serverurl" + "github.com/ardasevinc/mattermost-cli/internal/transport" +) + +const MaxWatchFrameBytes = 1 << 20 +const MaxSafeSequence int64 = 9007199254740991 + +var ( + ErrMalformedWatchFrame = errors.New("malformed WebSocket frame") + ErrOversizedWatchFrame = errors.New("WebSocket frame exceeds 1048576 bytes") + ErrWatchAuthentication = errors.New("WebSocket authentication failed") + ErrWatchSink = errors.New("watch output failed") + ErrWatchRetries = errors.New("WebSocket reconnect limit reached") + ErrInvalidWatchOptions = errors.New("invalid watch options") +) + +type WatchPost struct { + ID, ChannelID, UserID, Message, RootID, ChannelName, SenderName string + CreateAt int64 + FileIDs []string +} +type Sequence struct { + ConnectionID string + Number int64 +} +type WatchDiagnostic struct { + Type, Message, PreviousID, CurrentID string + Timestamp time.Time + Backfill, Fatal bool + Expected, Received *int64 + Attempt *int + Delay *time.Duration +} +type WatchSink interface { + Post(WatchPost, Sequence) error + Diagnostic(WatchDiagnostic) error +} + +type WatchOptions struct { + URL, Token, ChannelID string + Sink WatchSink + Dial transport.DialWebSocket + HandshakeTimeout, HeartbeatInterval, HeartbeatTimeout, CloseTimeout time.Duration + MaxReconnects int + Random func() float64 + NewTimer func(time.Duration) WatchTimer + Now func() time.Time +} + +type WatchTimer interface { + C() <-chan time.Time + Stop() bool +} +type realWatchTimer struct{ timer *time.Timer } + +func (timer realWatchTimer) C() <-chan time.Time { return timer.timer.C } +func (timer realWatchTimer) Stop() bool { return timer.timer.Stop() } + +type wireFrame struct { + Event, Status string + Data json.RawMessage + Seq, Reply *int64 + Error json.RawMessage +} + +func decodeFrame(data []byte) (wireFrame, error) { + if len(data) > MaxWatchFrameBytes { + return wireFrame{}, ErrOversizedWatchFrame + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.UseNumber() + var raw map[string]json.RawMessage + if err := decoder.Decode(&raw); err != nil || raw == nil { + return wireFrame{}, ErrMalformedWatchFrame + } + if err := requireEOF(decoder); err != nil { + return wireFrame{}, ErrMalformedWatchFrame + } + var frame wireFrame + for key, target := range map[string]any{"event": &frame.Event, "status": &frame.Status, "data": &frame.Data, "seq": &frame.Seq, "seq_reply": &frame.Reply, "error": &frame.Error} { + if value, ok := raw[key]; ok && json.Unmarshal(value, target) != nil { + return wireFrame{}, ErrMalformedWatchFrame + } + } + if frame.Seq != nil && (*frame.Seq < 0 || *frame.Seq >= MaxSafeSequence) || frame.Reply != nil && (*frame.Reply < 0 || *frame.Reply > MaxSafeSequence) { + return wireFrame{}, ErrMalformedWatchFrame + } + return frame, nil +} + +func requireEOF(decoder *json.Decoder) error { + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + return errors.New("trailing JSON") + } + return nil +} + +func parsePost(frame wireFrame) (WatchPost, bool) { + if frame.Event != "posted" { + return WatchPost{}, false + } + var data map[string]json.RawMessage + if json.Unmarshal(frame.Data, &data) != nil { + return WatchPost{}, false + } + var encoded, channelName, senderName string + if json.Unmarshal(data["post"], &encoded) != nil || json.Unmarshal(data["channel_name"], &channelName) != nil || json.Unmarshal(data["sender_name"], &senderName) != nil { + return WatchPost{}, false + } + decoder := json.NewDecoder(strings.NewReader(encoded)) + decoder.UseNumber() + var raw map[string]json.RawMessage + if decoder.Decode(&raw) != nil || raw == nil || requireEOF(decoder) != nil { + return WatchPost{}, false + } + var post WatchPost + fields := map[string]*string{"id": &post.ID, "channel_id": &post.ChannelID, "user_id": &post.UserID, "message": &post.Message, "root_id": &post.RootID} + for key, target := range fields { + value, ok := raw[key] + if !ok || json.Unmarshal(value, target) != nil { + return WatchPost{}, false + } + } + var timestamp json.Number + if value, ok := raw["create_at"]; !ok || json.Unmarshal(value, ×tamp) != nil { + return WatchPost{}, false + } + parsed, err := strconv.ParseInt(timestamp.String(), 10, 64) + if err != nil || parsed < 0 || parsed > 253402300799999 { + return WatchPost{}, false + } + post.CreateAt = parsed + if value, ok := raw["file_ids"]; !ok || json.Unmarshal(value, &post.FileIDs) != nil || post.FileIDs == nil { + return WatchPost{}, false + } + for _, id := range post.FileIDs { + if id == "" { + return WatchPost{}, false + } + } + if post.ID == "" || post.ChannelID == "" || post.UserID == "" { + return WatchPost{}, false + } + post.ChannelName, post.SenderName = channelName, senderName + return post, true +} + +type watchState struct { + connectionID string + nextServer, nextAction int64 + authenticated, hello bool +} +type readResult struct { + type_ transport.MessageType + data []byte + err error +} + +func Watch(ctx context.Context, options WatchOptions) error { + if err := validateWatchOptions(&options); err != nil { + return err + } + release := presentation.ActiveCredentials.Register(options.Token) + defer release() + state := watchState{nextAction: 1} + dedupe := NewPostDeduplicator(1000) + outages := 0 + for { + if outages > options.MaxReconnects { + return ErrWatchRetries + } + if outages > 0 { + random := options.Random() + if math.IsNaN(random) || random < 0 || random > 1 { + return ErrInvalidWatchOptions + } + delay := jitterBackoff(outages, random) + attempt := outages + if err := emitDiagnostic(options, WatchDiagnostic{Type: "reconnect", Message: "WebSocket disconnected; reconnecting without REST backfill.", Attempt: &attempt, Delay: &delay}); err != nil { + return err + } + timer := options.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C(): + } + } + stable, err := runConnection(ctx, options, &state, dedupe) + if errors.Is(err, ErrWatchAuthentication) || errors.Is(err, ErrWatchSink) || ctx.Err() != nil { + return err + } + if stable { + outages = 1 + } else { + outages++ + } + if err := emitDiagnostic(options, WatchDiagnostic{Type: "disconnected", Message: "WebSocket disconnected; live events may be missing.", Fatal: false}); err != nil { + return err + } + } +} + +func runConnection(ctx context.Context, options WatchOptions, state *watchState, dedupe *PostDeduplicator) (bool, error) { + target, err := watchURL(options.URL, state.connectionID, state.nextServer) + if err != nil { + return false, err + } + dialCtx, cancelDial := context.WithTimeout(ctx, options.HandshakeTimeout) + conn, err := options.Dial(dialCtx, target) + cancelDial() + if err != nil { + return false, errors.New("WebSocket connection failed") + } + conn.SetReadLimit(MaxWatchFrameBytes) + readCtx, cancelRead := context.WithCancel(ctx) + reads := make(chan readResult) + readerDone := make(chan struct{}) + go func() { + defer close(readerDone) + for { + type_, data, err := conn.Read(readCtx) + select { + case reads <- readResult{type_, data, err}: + case <-readCtx.Done(): + return + } + if err != nil { + return + } + } + }() + defer func() { + cancelRead() + closeCtx, cancel := context.WithTimeout(context.Background(), options.CloseTimeout) + _ = conn.Close(closeCtx) + cancel() + _ = conn.CloseNow() + <-readerDone + }() + if state.nextAction > MaxSafeSequence { + return false, errors.New("WebSocket action sequence exhausted") + } + authSeq := state.nextAction + state.nextAction++ + auth, _ := json.Marshal(map[string]any{"seq": authSeq, "action": "authentication_challenge", "data": map[string]string{"token": options.Token}}) + writeCtx, cancelWrite := context.WithTimeout(ctx, options.HandshakeTimeout) + err = conn.Write(writeCtx, transport.MessageText, auth) + cancelWrite() + if err != nil { + return false, errors.New("WebSocket handshake failed") + } + state.authenticated, state.hello = false, false + handshakeDeadline := options.Now().Add(options.HandshakeTimeout) + heartbeatAt := time.Time{} + pongDeadline := time.Time{} + var pendingPing int64 = -1 + stable := false + for { + deadline := handshakeDeadline + if state.authenticated && state.hello { + if !pongDeadline.IsZero() { + deadline = pongDeadline + } else { + deadline = heartbeatAt + } + } + wait := deadline.Sub(options.Now()) + if wait < 0 { + wait = 0 + } + timer := options.NewTimer(wait) + select { + case <-ctx.Done(): + timer.Stop() + return stable, ctx.Err() + case <-timer.C(): + if !state.authenticated || !state.hello { + return false, errors.New("WebSocket handshake failed") + } + if !pongDeadline.IsZero() { + return stable, errors.New("WebSocket heartbeat failed") + } + if state.nextAction > MaxSafeSequence { + return stable, errors.New("WebSocket action sequence exhausted") + } + pendingPing = state.nextAction + state.nextAction++ + ping, _ := json.Marshal(map[string]any{"seq": pendingPing, "action": "ping", "data": map[string]any{}}) + pingCtx, cancel := context.WithTimeout(ctx, options.HeartbeatTimeout) + err := conn.Write(pingCtx, transport.MessageText, ping) + cancel() + if err != nil { + return stable, errors.New("WebSocket heartbeat failed") + } + pongDeadline = options.Now().Add(options.HeartbeatTimeout) + case result := <-reads: + timer.Stop() + if result.err != nil { + return stable, result.err + } + if result.type_ != transport.MessageText { + if err := emitDiagnostic(options, WatchDiagnostic{Type: "malformed", Message: "Binary WebSocket frame skipped."}); err != nil { + return false, err + } + continue + } + frame, err := decodeFrame(result.data) + if err != nil { + message := "Malformed WebSocket frame skipped." + if errors.Is(err, ErrOversizedWatchFrame) { + message = ErrOversizedWatchFrame.Error() + } + if sinkErr := emitDiagnostic(options, WatchDiagnostic{Type: "malformed", Message: message}); sinkErr != nil { + return false, sinkErr + } + continue + } + if frame.Status == "FAIL" { + if !state.authenticated && frame.Reply != nil && *frame.Reply == authSeq { + return false, ErrWatchAuthentication + } + return stable, errors.New("WebSocket request failed") + } + if frame.Status == "OK" && frame.Reply != nil { + if *frame.Reply == authSeq { + state.authenticated = true + } + if *frame.Reply == pendingPing && !pongDeadline.IsZero() { + pongDeadline = time.Time{} + stable = true + heartbeatAt = options.Now().Add(options.HeartbeatInterval) + } + } + if frame.Event == "" { + continue + } + if frame.Seq == nil { + if err := emitDiagnostic(options, WatchDiagnostic{Type: "malformed", Message: "WebSocket event without sequence skipped."}); err != nil { + return false, err + } + continue + } + if frame.Event == "hello" { + id, ok := connectionID(frame.Data) + if !ok { + if err := emitDiagnostic(options, WatchDiagnostic{Type: "malformed", Message: "Malformed WebSocket hello skipped."}); err != nil { + return false, err + } + continue + } + if state.connectionID != "" && id != state.connectionID { + expected, received := state.nextServer, *frame.Seq + if err := emitDiagnostic(options, WatchDiagnostic{Type: "connection_changed", Message: "WebSocket connection changed; live events may be missing; no REST backfill was attempted.", PreviousID: state.connectionID, CurrentID: id, Expected: &expected, Received: &received}); err != nil { + return false, err + } + state.connectionID, state.nextServer = id, 0 + if *frame.Seq != 0 { + return false, errors.New("WebSocket sequence mismatch") + } + } + if *frame.Seq != state.nextServer { + expected, received := state.nextServer, *frame.Seq + if err := emitDiagnostic(options, WatchDiagnostic{Type: "sequence_gap", Message: "WebSocket sequence mismatch; live events may be missing; no REST backfill was attempted.", Expected: &expected, Received: &received}); err != nil { + return false, err + } + return false, errors.New("WebSocket sequence mismatch") + } + state.connectionID, state.nextServer, state.hello = id, *frame.Seq+1, true + } else { + if *frame.Seq != state.nextServer { + expected, received := state.nextServer, *frame.Seq + if err := emitDiagnostic(options, WatchDiagnostic{Type: "sequence_gap", Message: "WebSocket sequence mismatch; frame suppressed; no REST backfill was attempted.", Expected: &expected, Received: &received}); err != nil { + return false, err + } + return stable, errors.New("WebSocket sequence mismatch") + } + state.nextServer++ + } + if state.authenticated && state.hello && heartbeatAt.IsZero() { + heartbeatAt = options.Now().Add(options.HeartbeatInterval) + } + if !state.authenticated || !state.hello || frame.Event != "posted" { + continue + } + post, ok := parsePost(frame) + if !ok { + if err := emitDiagnostic(options, WatchDiagnostic{Type: "malformed", Message: "Malformed WebSocket post payload skipped."}); err != nil { + return false, err + } + continue + } + if options.ChannelID != "" && post.ChannelID != options.ChannelID || !dedupe.Add(post.ID) { + continue + } + if err := options.Sink.Post(post, Sequence{ConnectionID: state.connectionID, Number: *frame.Seq}); err != nil { + return false, ErrWatchSink + } + } + } +} + +func validateWatchOptions(options *WatchOptions) error { + if _, err := serverurl.Normalize(options.URL); err != nil || options.Token == "" || options.Sink == nil { + return ErrInvalidWatchOptions + } + if options.Dial == nil { + options.Dial = transport.Dial + } + if options.HandshakeTimeout == 0 { + options.HandshakeTimeout = 15 * time.Second + } + if options.HeartbeatInterval == 0 { + options.HeartbeatInterval = 30 * time.Second + } + if options.HeartbeatTimeout == 0 { + options.HeartbeatTimeout = 10 * time.Second + } + if options.CloseTimeout == 0 { + options.CloseTimeout = time.Second + } + if options.MaxReconnects == 0 { + options.MaxReconnects = 8 + } + if options.Random == nil { + options.Random = func() float64 { return .5 } + } + if options.Now == nil { + options.Now = time.Now + } + if options.NewTimer == nil { + options.NewTimer = func(duration time.Duration) WatchTimer { return realWatchTimer{time.NewTimer(duration)} } + } + if options.HandshakeTimeout <= 0 || options.HeartbeatInterval <= 0 || options.HeartbeatTimeout <= 0 || options.CloseTimeout <= 0 || options.MaxReconnects < 0 { + return ErrInvalidWatchOptions + } + random := options.Random() + if math.IsNaN(random) || random < 0 || random > 1 { + return ErrInvalidWatchOptions + } + return nil +} + +func watchURL(raw, connectionID string, next int64) (string, error) { + normalized, err := serverurl.Normalize(raw) + if err != nil { + return "", err + } + parsed, _ := url.Parse(normalized) + if parsed.Scheme == "https" { + parsed.Scheme = "wss" + } else { + parsed.Scheme = "ws" + } + parsed.Path = strings.TrimRight(parsed.Path, "/") + "/api/v4/websocket" + if connectionID != "" { + query := parsed.Query() + query.Set("connection_id", connectionID) + query.Set("sequence_number", fmt.Sprint(next)) + parsed.RawQuery = query.Encode() + } + return parsed.String(), nil +} +func connectionID(data json.RawMessage) (string, bool) { + var value struct { + ConnectionID string `json:"connection_id"` + } + err := json.Unmarshal(data, &value) + return value.ConnectionID, err == nil && value.ConnectionID != "" +} +func emitDiagnostic(options WatchOptions, diagnostic WatchDiagnostic) error { + diagnostic.Timestamp = options.Now().UTC() + diagnostic.Backfill = false + if err := options.Sink.Diagnostic(diagnostic); err != nil { + return ErrWatchSink + } + return nil +} +func jitterBackoff(attempt int, random float64) time.Duration { + base := time.Second * time.Duration(1< 30*time.Second { + base = 30 * time.Second + } + return time.Duration(float64(base) * (0.8 + 0.4*random)) +} + +type PostDeduplicator struct { + capacity int + order []string + seen map[string]struct{} +} + +func NewPostDeduplicator(capacity int) *PostDeduplicator { + return &PostDeduplicator{capacity: capacity, seen: make(map[string]struct{}, capacity)} +} +func (set *PostDeduplicator) Add(id string) bool { + if _, ok := set.seen[id]; ok { + return false + } + if len(set.order) == set.capacity { + delete(set.seen, set.order[0]) + copy(set.order, set.order[1:]) + set.order = set.order[:len(set.order)-1] + } + set.order = append(set.order, id) + set.seen[id] = struct{}{} + return true +} diff --git a/internal/mattermost/watch_test.go b/internal/mattermost/watch_test.go new file mode 100644 index 0000000..397c235 --- /dev/null +++ b/internal/mattermost/watch_test.go @@ -0,0 +1,286 @@ +package mattermost + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/internal/transport" +) + +type recordingSink struct { + mu sync.Mutex + posts []WatchPost + sequences []Sequence + diagnostics []WatchDiagnostic + err error +} +type stoppingDiagnosticSink struct{ recordingSink } + +func (s *stoppingDiagnosticSink) Diagnostic(value WatchDiagnostic) error { + _ = s.recordingSink.Diagnostic(value) + if value.Type == "sequence_gap" { + return errors.New("stop") + } + return nil +} + +func (s *recordingSink) Post(post WatchPost, sequence Sequence) error { + s.mu.Lock() + defer s.mu.Unlock() + s.posts = append(s.posts, post) + s.sequences = append(s.sequences, sequence) + return s.err +} +func (s *recordingSink) Diagnostic(value WatchDiagnostic) error { + s.mu.Lock() + defer s.mu.Unlock() + s.diagnostics = append(s.diagnostics, value) + return s.err +} + +type fakeSocket struct { + reads chan readResult + writes [][]byte + mu sync.Mutex + closed chan struct{} + once sync.Once +} +type instantTimer struct{ channel chan time.Time } + +func newInstantTimer() WatchTimer { + channel := make(chan time.Time, 1) + channel <- time.Time{} + return instantTimer{channel} +} +func (timer instantTimer) C() <-chan time.Time { return timer.channel } +func (instantTimer) Stop() bool { return true } + +func newFakeSocket() *fakeSocket { + return &fakeSocket{reads: make(chan readResult, 16), closed: make(chan struct{})} +} +func (s *fakeSocket) Read(ctx context.Context) (transport.MessageType, []byte, error) { + select { + case <-ctx.Done(): + return 0, nil, ctx.Err() + case value := <-s.reads: + return value.type_, value.data, value.err + } +} +func (s *fakeSocket) Write(_ context.Context, _ transport.MessageType, data []byte) error { + s.mu.Lock() + defer s.mu.Unlock() + s.writes = append(s.writes, append([]byte(nil), data...)) + return nil +} +func (s *fakeSocket) Close(context.Context) error { s.once.Do(func() { close(s.closed) }); return nil } +func (s *fakeSocket) CloseNow() error { s.once.Do(func() { close(s.closed) }); return nil } +func (*fakeSocket) SetReadLimit(int64) {} + +func TestDecodeFrameRejectsMalformedTrailingOversizedNegativeAndScalar(t *testing.T) { + for _, data := range [][]byte{[]byte("{"), []byte(`{} {}`), []byte(`null`), []byte(`{"seq":-1}`)} { + if _, err := decodeFrame(data); !errors.Is(err, ErrMalformedWatchFrame) { + t.Fatalf("%q: %v", data, err) + } + } + if _, err := decodeFrame([]byte(strings.Repeat("x", MaxWatchFrameBytes+1))); !errors.Is(err, ErrOversizedWatchFrame) { + t.Fatal(err) + } + if _, err := decodeFrame([]byte(fmt.Sprintf(`{"seq":%d}`, MaxSafeSequence-1))); err != nil { + t.Fatal(err) + } + if _, err := decodeFrame([]byte(fmt.Sprintf(`{"seq":%d}`, MaxSafeSequence))); !errors.Is(err, ErrMalformedWatchFrame) { + t.Fatalf("headroom boundary=%v", err) + } + if _, err := decodeFrame([]byte(fmt.Sprintf(`{"seq_reply":%d}`, MaxSafeSequence))); err != nil { + t.Fatal(err) + } +} +func TestParsePostPresenceTrailerAndTimestamp(t *testing.T) { + good := wireFrame{Event: "posted", Data: []byte(`{"post":"{\"id\":\"p\",\"channel_id\":\"c\",\"user_id\":\"u\",\"message\":\"m\",\"root_id\":\"\",\"create_at\":1,\"file_ids\":[]}","channel_name":"town","sender_name":"arda"}`)} + if post, ok := parsePost(good); !ok || post.ID != "p" { + t.Fatalf("%#v %v", post, ok) + } + for _, encoded := range []string{`{}`, `{"id":"p","channel_id":"c","user_id":"u","message":"m","root_id":"","create_at":1e3,"file_ids":[]}`, `{"id":"p","channel_id":"c","user_id":"u","message":"m","root_id":"","create_at":253402300800000,"file_ids":[]}{}`} { + bad := good + bad.Data = []byte(`{"post":` + strconvQuote(encoded) + `,"channel_name":"x","sender_name":"y"}`) + if _, ok := parsePost(bad); ok { + t.Fatalf("accepted %s", encoded) + } + } +} +func strconvQuote(value string) string { return `"` + strings.ReplaceAll(value, `"`, `\"`) + `"` } +func TestWatchAuthWireInterleavingDuplicateSequenceAndCancellation(t *testing.T) { + socket := newFakeSocket() + sink := &recordingSink{} + socket.reads <- readResult{type_: transport.MessageText, data: []byte(`{"event":"hello","seq":0,"data":{"connection_id":"one"}}`)} + socket.reads <- readResult{type_: transport.MessageText, data: []byte(`{"event":"posted","seq":1,"data":{"post":"{\"id\":\"early\",\"channel_id\":\"c\",\"user_id\":\"u\",\"message\":\"m\",\"root_id\":\"\",\"create_at\":1,\"file_ids\":[]}","channel_name":"town","sender_name":"arda"}}`)} + socket.reads <- readResult{type_: transport.MessageText, data: []byte(`{"status":"OK","seq_reply":1}`)} + socket.reads <- readResult{type_: transport.MessageText, data: []byte(`{"event":"posted","seq":2,"data":{"post":"{\"id\":\"p\",\"channel_id\":\"c\",\"user_id\":\"u\",\"message\":\"m\",\"root_id\":\"\",\"create_at\":1,\"file_ids\":[]}","channel_name":"town","sender_name":"arda"}}`)} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- Watch(ctx, WatchOptions{URL: "https://mm.example.com", Token: "secret", Sink: sink, Dial: func(context.Context, string) (transport.WebSocket, error) { return socket, nil }}) + }() + time.Sleep(10 * time.Millisecond) + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + if len(sink.posts) != 1 || sink.posts[0].ID != "p" || sink.sequences[0].Number != 2 { + t.Fatalf("posts=%#v", sink.posts) + } + socket.mu.Lock() + wire := string(socket.writes[0]) + socket.mu.Unlock() + if wire != `{"action":"authentication_challenge","data":{"token":"secret"},"seq":1}` { + t.Fatalf("auth=%s", wire) + } +} +func TestWatchSinkErrorIsTerminalWithoutReconnect(t *testing.T) { + socket := newFakeSocket() + sink := &recordingSink{err: errors.New("hostile")} + socket.reads <- readResult{type_: transport.MessageBinary, data: []byte("x")} + dials := 0 + err := Watch(context.Background(), WatchOptions{URL: "https://mm.example.com", Token: "secret", Sink: sink, Dial: func(context.Context, string) (transport.WebSocket, error) { dials++; return socket, nil }}) + if !errors.Is(err, ErrWatchSink) || dials != 1 || strings.Contains(err.Error(), "hostile") { + t.Fatalf("%v %d", err, dials) + } +} +func TestWatchAuthenticationFailureIsFatalAndNeverReflected(t *testing.T) { + socket := newFakeSocket() + sink := &recordingSink{} + credential := "secret-token" + socket.reads <- readResult{type_: transport.MessageText, data: []byte(`{"status":"FAIL","seq_reply":1,"error":{"message":"secret-token invalid token"}}`)} + dials := 0 + err := Watch(context.Background(), WatchOptions{URL: "https://mm.example.com", Token: credential, Sink: sink, Dial: func(context.Context, string) (transport.WebSocket, error) { dials++; return socket, nil }}) + if !errors.Is(err, ErrWatchAuthentication) || dials != 1 || strings.Contains(err.Error(), credential) { + t.Fatalf("%v %d", err, dials) + } +} +func TestWatchValidationPrecedesCredentialRegistrationAndDial(t *testing.T) { + before := presentation.ActiveCredentials.Values() + dials := 0 + err := Watch(context.Background(), WatchOptions{URL: "https://not-loopback.invalid", Token: "secret", Sink: &recordingSink{}, HandshakeTimeout: -1, Dial: func(context.Context, string) (transport.WebSocket, error) { dials++; return nil, nil }}) + if !errors.Is(err, ErrInvalidWatchOptions) || dials != 0 || len(presentation.ActiveCredentials.Values()) != len(before) { + t.Fatalf("%v dials=%d", err, dials) + } +} +func TestWatchURLResumeAndJitter(t *testing.T) { + got, err := watchURL("https://mm.example.com/base", "connection", 7) + if err != nil || got != "wss://mm.example.com/base/api/v4/websocket?connection_id=connection&sequence_number=7" { + t.Fatalf("%q %v", got, err) + } + if jitterBackoff(1, 0) != 800*time.Millisecond || jitterBackoff(99, 1) != 36*time.Second { + t.Fatal("jitter bounds") + } +} +func TestWatchForwardAndStaleSequencesAreSuppressed(t *testing.T) { + for _, sequence := range []int{0, 2} { + t.Run(strconv.Itoa(sequence), func(t *testing.T) { + socket := newFakeSocket() + sink := &stoppingDiagnosticSink{} + socket.reads <- readResult{type_: transport.MessageText, data: []byte(`{"event":"hello","seq":0,"data":{"connection_id":"one"}}`)} + socket.reads <- readResult{type_: transport.MessageText, data: []byte(`{"status":"OK","seq_reply":1}`)} + posted := fmt.Sprintf(`{"event":"posted","seq":%d,"data":{"post":"{\"id\":\"p\",\"channel_id\":\"c\",\"user_id\":\"u\",\"message\":\"m\",\"root_id\":\"\",\"create_at\":1,\"file_ids\":[]}","channel_name":"town","sender_name":"arda"}}`, sequence) + socket.reads <- readResult{type_: transport.MessageText, data: []byte(posted)} + err := Watch(context.Background(), WatchOptions{URL: "https://mm.example.com", Token: "secret", Sink: sink, Dial: func(context.Context, string) (transport.WebSocket, error) { return socket, nil }}) + if !errors.Is(err, ErrWatchSink) || len(sink.posts) != 0 || len(sink.diagnostics) != 1 || sink.diagnostics[0].Expected == nil || *sink.diagnostics[0].Expected != 1 { + t.Fatalf("err=%v posts=%v diagnostics=%#v", err, sink.posts, sink.diagnostics) + } + }) + } +} +func TestWatchRetryCapUsesInjectedBackoff(t *testing.T) { + sink := &recordingSink{} + dials := 0 + waits := []time.Duration{} + err := Watch(context.Background(), WatchOptions{URL: "https://mm.example.com", Token: "secret", Sink: sink, MaxReconnects: 2, Random: func() float64 { return .5 }, NewTimer: func(d time.Duration) WatchTimer { waits = append(waits, d); return newInstantTimer() }, Dial: func(context.Context, string) (transport.WebSocket, error) { + dials++ + return nil, errors.New("hostile remote") + }}) + if !errors.Is(err, ErrWatchRetries) || dials != 3 || len(waits) != 2 || waits[0] != time.Second || waits[1] != 2*time.Second { + t.Fatalf("err=%v dials=%d waits=%v", err, dials, waits) + } +} +func TestRapidAuthenticatedDropsRemainBoundedUntilPongStability(t *testing.T) { + sink := &recordingSink{} + dials := 0 + err := Watch(context.Background(), WatchOptions{URL: "https://mm.example.com", Token: "secret", Sink: sink, MaxReconnects: 2, Random: func() float64 { return .5 }, NewTimer: func(duration time.Duration) WatchTimer { + if duration < 10*time.Second { + return newInstantTimer() + } + return instantTimer{make(chan time.Time)} + }, Dial: func(context.Context, string) (transport.WebSocket, error) { + socket := newFakeSocket() + sequence := dials + auth := dials + 1 + dials++ + socket.reads <- readResult{type_: transport.MessageText, data: []byte(fmt.Sprintf(`{"event":"hello","seq":%d,"data":{"connection_id":"one"}}`, sequence))} + socket.reads <- readResult{type_: transport.MessageText, data: []byte(fmt.Sprintf(`{"status":"OK","seq_reply":%d}`, auth))} + socket.reads <- readResult{err: errors.New("drop")} + return socket, nil + }}) + if !errors.Is(err, ErrWatchRetries) || dials != 3 { + t.Fatalf("err=%v dials=%d", err, dials) + } +} + +func TestActionHeartbeatExactWireWrongReplyAndTimeout(t *testing.T) { + socket := newFakeSocket() + sink := &recordingSink{} + socket.reads <- readResult{type_: transport.MessageText, data: []byte(`{"event":"hello","seq":0,"data":{"connection_id":"one"}}`)} + socket.reads <- readResult{type_: transport.MessageText, data: []byte(`{"status":"OK","seq_reply":1}`)} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- Watch(ctx, WatchOptions{URL: "https://mm.example.com", Token: "secret", Sink: sink, HeartbeatInterval: 2 * time.Millisecond, HeartbeatTimeout: 3 * time.Millisecond, Dial: func(context.Context, string) (transport.WebSocket, error) { return socket, nil }}) + }() + deadline := time.Now().Add(time.Second) + for { + socket.mu.Lock() + count := len(socket.writes) + socket.mu.Unlock() + if count >= 2 { + break + } + if time.Now().After(deadline) { + t.Fatal("ping not written") + } + time.Sleep(time.Millisecond) + } + socket.mu.Lock() + ping := string(socket.writes[1]) + socket.mu.Unlock() + if ping != `{"action":"ping","data":{},"seq":2}` { + t.Fatalf("ping=%s", ping) + } + socket.reads <- readResult{type_: transport.MessageText, data: []byte(`{"status":"OK","seq_reply":999}`)} + for { + sink.mu.Lock() + found := false + for _, diagnostic := range sink.diagnostics { + if diagnostic.Type == "disconnected" { + found = true + } + } + sink.mu.Unlock() + if found { + break + } + if time.Now().After(deadline) { + t.Fatal("wrong pong prevented timeout") + } + time.Sleep(time.Millisecond) + } + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatal(err) + } +} diff --git a/internal/output/watch.go b/internal/output/watch.go new file mode 100644 index 0000000..42526c0 --- /dev/null +++ b/internal/output/watch.go @@ -0,0 +1,251 @@ +package output + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "reflect" + "time" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/presentation" +) + +const MaxWatchLineBytes = 1 << 20 + +var ErrPartialWatchLine = errors.New("partial JSONL write; stream is no longer recoverable") +var ErrInvalidWatchDocument = errors.New("invalid watch document") +var ErrWatchOutput = errors.New("watch output unavailable") + +type WatchDocument interface { + watchDocument() + valid() bool +} +type WatchSequence struct { + ConnectionID string `json:"connectionId"` + Number int64 `json:"number"` +} +type watchEvent struct { + Schema string `json:"schema"` + Type string `json:"type"` + Sequence WatchSequence `json:"sequence"` + PostID string `json:"postId"` + ChannelID string `json:"channelId"` + ChannelName string `json:"channelName"` + SenderID string `json:"senderId"` + Sender string `json:"sender"` + Message string `json:"message"` + Timestamp MillisTime `json:"timestamp"` + RootID *string `json:"rootId"` + FileIDs []string `json:"fileIds"` + Redactions []presentation.Redaction `json:"redactions"` + seal bool +} +type watchDiagnostic struct { + Schema string `json:"schema"` + Type string `json:"type"` + Timestamp MillisTime `json:"timestamp"` + Message string `json:"message"` + Backfill bool `json:"backfill"` + Fatal bool `json:"fatal"` + Redactions []presentation.Redaction `json:"redactions"` + Attempt *int `json:"attempt,omitempty"` + DelayMS *int64 `json:"delayMs,omitempty"` + Expected *int64 `json:"expected,omitempty"` + Received *int64 `json:"received,omitempty"` + PreviousID *string `json:"previousConnectionId,omitempty"` + CurrentID *string `json:"currentConnectionId,omitempty"` + seal bool +} + +func (watchEvent) watchDocument() {} +func (watchDiagnostic) watchDocument() {} +func (value watchEvent) valid() bool { + if !value.seal || value.Schema != "mm/v2/watch-event" || value.Type != "posted" || value.Sequence.ConnectionID == "" || value.Sequence.Number < 0 || value.Sequence.Number > mattermost.MaxSafeSequence || value.PostID == "" || value.ChannelID == "" || value.SenderID == "" || value.Timestamp.Time.IsZero() || value.FileIDs == nil || value.Redactions == nil || value.RootID != nil && *value.RootID == "" { + return false + } + for _, id := range value.FileIDs { + if id == "" { + return false + } + } + return true +} +func (value watchDiagnostic) valid() bool { + if !value.seal || value.Schema != "mm/v2/watch-diagnostic" || value.Message == "" || value.Timestamp.Time.IsZero() || value.Backfill || value.Redactions == nil { + return false + } + switch value.Type { + case "reconnect": + return !value.Fatal && value.Attempt != nil && *value.Attempt > 0 && value.DelayMS != nil && *value.DelayMS >= 0 && value.Expected == nil && value.Received == nil && value.PreviousID == nil && value.CurrentID == nil + case "sequence_gap": + return !value.Fatal && value.Expected != nil && *value.Expected >= 0 && *value.Expected <= mattermost.MaxSafeSequence && value.Received != nil && *value.Received >= 0 && *value.Received <= mattermost.MaxSafeSequence && value.Attempt == nil && value.DelayMS == nil && value.PreviousID == nil && value.CurrentID == nil + case "connection_changed": + return !value.Fatal && value.Expected != nil && *value.Expected >= 0 && *value.Expected <= mattermost.MaxSafeSequence && value.Received != nil && *value.Received >= 0 && *value.Received <= mattermost.MaxSafeSequence && value.PreviousID != nil && *value.PreviousID != "" && value.CurrentID != nil && *value.CurrentID != "" && value.Attempt == nil && value.DelayMS == nil + case "malformed", "disconnected": + return !value.Fatal && value.Attempt == nil && value.DelayMS == nil && value.Expected == nil && value.Received == nil && value.PreviousID == nil && value.CurrentID == nil + case "terminal": + return value.Fatal && value.Attempt == nil && value.DelayMS == nil && value.Expected == nil && value.Received == nil && value.PreviousID == nil && value.CurrentID == nil + default: + return false + } +} + +func NewWatchEvent(post mattermost.WatchPost, sequence mattermost.Sequence, disableHeuristics bool) (WatchDocument, error) { + if post.ID == "" || post.ChannelID == "" || post.UserID == "" || sequence.ConnectionID == "" || sequence.Number < 0 || sequence.Number > mattermost.MaxSafeSequence || post.CreateAt < 0 || post.FileIDs == nil || rawWatchSize(post, sequence) > MaxWatchLineBytes { + return nil, ErrInvalidWatchDocument + } + credentials := presentation.ActiveCredentials.Values() + clean := func(value string, label bool, field string) (string, []presentation.Redaction) { + if label { + value = presentation.SanitizeLabel(value) + } + result := presentation.PreprocessWithOptions(value, presentation.Options{Credentials: credentials, DisableHeuristics: disableHeuristics}) + text := result.Text + for i := range result.Redactions { + result.Redactions[i].Field = field + } + return text, result.Redactions + } + postID, r := clean(post.ID, true, "watch.postId") + redactions := append([]presentation.Redaction{}, r...) + channelID, r := clean(post.ChannelID, true, "watch.channelId") + redactions = append(redactions, r...) + channelName, r := clean(post.ChannelName, true, "watch.channelName") + redactions = append(redactions, r...) + senderID, r := clean(post.UserID, true, "watch.senderId") + redactions = append(redactions, r...) + sender, r := clean(post.SenderName, true, "watch.sender") + redactions = append(redactions, r...) + message, r := clean(post.Message, false, "watch.message") + redactions = append(redactions, r...) + connectionID, r := clean(sequence.ConnectionID, true, "watch.sequence.connectionId") + redactions = append(redactions, r...) + files := make([]string, len(post.FileIDs)) + for i, id := range post.FileIDs { + if id == "" { + return nil, ErrInvalidWatchDocument + } + files[i], r = clean(id, true, "watch.fileId") + redactions = append(redactions, r...) + } + var root *string + if post.RootID != "" { + value, rr := clean(post.RootID, true, "watch.rootId") + root = &value + redactions = append(redactions, rr...) + } + document := watchEvent{Schema: "mm/v2/watch-event", Type: "posted", Sequence: WatchSequence{connectionID, sequence.Number}, PostID: postID, ChannelID: channelID, ChannelName: channelName, SenderID: senderID, Sender: sender, Message: message, Timestamp: MillisTime{Time: time.UnixMilli(post.CreateAt).UTC()}, RootID: root, FileIDs: files, Redactions: redactions, seal: true} + if !document.valid() { + return nil, ErrInvalidWatchDocument + } + return document, nil +} + +func NewWatchDiagnostic(value mattermost.WatchDiagnostic) (WatchDocument, error) { + if len(value.Message)+len(value.PreviousID)+len(value.CurrentID) > MaxWatchLineBytes { + return nil, ErrInvalidWatchDocument + } + credentials := presentation.ActiveCredentials.Values() + clean := func(text, field string) (string, []presentation.Redaction) { + text = presentation.SanitizeLabel(text) + result := presentation.PreprocessWithOptions(text, presentation.Options{Credentials: credentials}) + for i := range result.Redactions { + result.Redactions[i].Field = field + } + return result.Text, result.Redactions + } + message, redactions := clean(value.Message, "watch.diagnostic.message") + document := watchDiagnostic{Schema: "mm/v2/watch-diagnostic", Type: value.Type, Timestamp: MillisTime{Time: value.Timestamp.UTC()}, Message: message, Backfill: false, Fatal: value.Fatal, Redactions: redactions, Attempt: value.Attempt, Expected: value.Expected, Received: value.Received, seal: true} + if value.Delay != nil { + ms := value.Delay.Milliseconds() + document.DelayMS = &ms + } + if value.PreviousID != "" { + cleaned, reds := clean(value.PreviousID, "watch.diagnostic.previousConnectionId") + document.PreviousID = &cleaned + document.Redactions = append(document.Redactions, reds...) + } + if value.CurrentID != "" { + cleaned, reds := clean(value.CurrentID, "watch.diagnostic.currentConnectionId") + document.CurrentID = &cleaned + document.Redactions = append(document.Redactions, reds...) + } + if !document.valid() { + return nil, ErrInvalidWatchDocument + } + return document, nil +} + +type limitedBuffer struct { + bytes.Buffer + limit int +} + +func (writer *limitedBuffer) Write(data []byte) (int, error) { + if writer.Len()+len(data) > writer.limit { + return 0, errors.New("JSONL document exceeds limit") + } + return writer.Buffer.Write(data) +} +func WriteWatchDocument(writer io.Writer, document WatchDocument) error { + if nilableWriter(writer) { + return ErrWatchOutput + } + if document == nil || !document.valid() { + return ErrInvalidWatchDocument + } + buffer := limitedBuffer{limit: MaxWatchLineBytes} + encoder := json.NewEncoder(&buffer) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(document); err != nil { + return err + } + written, err := writer.Write(buffer.Bytes()) + if written != buffer.Len() { + return ErrPartialWatchLine + } + return err +} + +func nilableWriter(writer io.Writer) bool { + if writer == nil { + return true + } + value := reflect.ValueOf(writer) + switch value.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice, reflect.UnsafePointer: + return value.IsNil() + default: + return false + } +} + +func rawWatchSize(post mattermost.WatchPost, sequence mattermost.Sequence) int { + total := len(post.ID) + len(post.ChannelID) + len(post.UserID) + len(post.Message) + len(post.RootID) + len(post.ChannelName) + len(post.SenderName) + len(sequence.ConnectionID) + for _, id := range post.FileIDs { + total += len(id) + } + return total +} + +type JSONLWatchSink struct { + Events, Diagnostics io.Writer + DisableHeuristics bool +} + +func (sink JSONLWatchSink) Post(post mattermost.WatchPost, sequence mattermost.Sequence) error { + document, err := NewWatchEvent(post, sequence, sink.DisableHeuristics) + if err != nil { + return err + } + return WriteWatchDocument(sink.Events, document) +} +func (sink JSONLWatchSink) Diagnostic(value mattermost.WatchDiagnostic) error { + document, err := NewWatchDiagnostic(value) + if err != nil { + return err + } + return WriteWatchDocument(sink.Diagnostics, document) +} diff --git a/internal/output/watch_test.go b/internal/output/watch_test.go new file mode 100644 index 0000000..126c3f2 --- /dev/null +++ b/internal/output/watch_test.go @@ -0,0 +1,155 @@ +package output + +import ( + "bytes" + "encoding/json" + "errors" + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/presentation" + "io" + "strings" + "testing" + "time" +) + +type shortWatchWriter struct{} +type nilMapWriter map[string]string + +func (nilMapWriter) Write([]byte) (int, error) { panic("nil map writer invoked") } + +type nilSliceWriter []byte + +func (nilSliceWriter) Write([]byte) (int, error) { panic("nil slice writer invoked") } + +func (shortWatchWriter) Write(value []byte) (int, error) { return len(value) - 1, nil } +func TestWatchEventSealedSanitizedMultilineAndAtomic(t *testing.T) { + credential := "credential-secret" + release := presentation.ActiveCredentials.Register(credential) + defer release() + document, err := NewWatchEvent(mattermost.WatchPost{ID: "p\x1b", ChannelID: "c", UserID: "u", Message: "**markdown**\n" + credential, CreateAt: 1, FileIDs: []string{}}, mattermost.Sequence{ConnectionID: "one", Number: 2}, true) + if err != nil { + t.Fatal(err) + } + var output bytes.Buffer + if err = WriteWatchDocument(&output, document); err != nil { + t.Fatal(err) + } + if strings.Contains(output.String(), credential) || strings.ContainsRune(output.String(), '\x1b') || !strings.Contains(output.String(), `**markdown**\n`) { + t.Fatalf("%q", output.String()) + } + if err = WriteWatchDocument(shortWatchWriter{}, document); !errors.Is(err, ErrPartialWatchLine) { + t.Fatal(err) + } +} +func TestWatchDocumentRejectsInvalidAndOversized(t *testing.T) { + if err := WriteWatchDocument(&bytes.Buffer{}, nil); !errors.Is(err, ErrInvalidWatchDocument) { + t.Fatal(err) + } + if _, err := NewWatchEvent(mattermost.WatchPost{ID: "p", ChannelID: "c", UserID: "u", Message: strings.Repeat("x", MaxWatchLineBytes), CreateAt: 1, FileIDs: []string{}}, mattermost.Sequence{ConnectionID: "one", Number: 1}, true); !errors.Is(err, ErrInvalidWatchDocument) { + t.Fatalf("oversize error=%v", err) + } +} +func TestWriteWatchDocumentRejectsNilWriters(t *testing.T) { + document, err := NewWatchEvent(mattermost.WatchPost{ID: "p", ChannelID: "c", UserID: "u", CreateAt: 1, FileIDs: []string{}}, mattermost.Sequence{ConnectionID: "one", Number: 1}, true) + if err != nil { + t.Fatal(err) + } + for _, writer := range []io.Writer{nil, (*bytes.Buffer)(nil), nilMapWriter(nil), nilSliceWriter(nil)} { + if err := WriteWatchDocument(writer, document); !errors.Is(err, ErrWatchOutput) { + t.Fatalf("error=%v", err) + } + } +} +func TestWatchPresentationOwnsCredentialInConnectionAndDiagnostic(t *testing.T) { + credential := "connection-secret" + release := presentation.ActiveCredentials.Register(credential) + defer release() + document, err := NewWatchEvent(mattermost.WatchPost{ID: "p", ChannelID: "c", UserID: "u", CreateAt: 1, FileIDs: []string{}}, mattermost.Sequence{ConnectionID: credential + "\n", Number: mattermost.MaxSafeSequence}, true) + if err != nil { + t.Fatal(err) + } + var out bytes.Buffer + if err := WriteWatchDocument(&out, document); err != nil { + t.Fatal(err) + } + if strings.Contains(out.String(), credential) || !strings.Contains(out.String(), "watch.sequence.connectionId") { + t.Fatalf("%s", out.String()) + } + diagnostic, err := NewWatchDiagnostic(mattermost.WatchDiagnostic{Type: "connection_changed", Timestamp: time.UnixMilli(1), Message: "changed " + credential, PreviousID: credential, CurrentID: "new\n", Expected: pointerInt64(1), Received: pointerInt64(0)}) + if err != nil { + t.Fatal(err) + } + out.Reset() + if err := WriteWatchDocument(&out, diagnostic); err != nil { + t.Fatal(err) + } + if strings.Contains(out.String(), credential) || !strings.Contains(out.String(), `"redactions"`) { + t.Fatalf("%s", out.String()) + } +} +func pointerInt64(value int64) *int64 { return &value } +func TestWatchSequenceSafeIntegerBoundary(t *testing.T) { + post := mattermost.WatchPost{ID: "p", ChannelID: "c", UserID: "u", CreateAt: 1, FileIDs: []string{}} + if _, err := NewWatchEvent(post, mattermost.Sequence{ConnectionID: "c", Number: mattermost.MaxSafeSequence}, true); err != nil { + t.Fatal(err) + } + if _, err := NewWatchEvent(post, mattermost.Sequence{ConnectionID: "c", Number: mattermost.MaxSafeSequence + 1}, true); !errors.Is(err, ErrInvalidWatchDocument) { + t.Fatal(err) + } +} + +func TestWatchLabelRedactionPositionsMatchEmittedText(t *testing.T) { + credential := "credential-position-secret" + release := presentation.ActiveCredentials.Register(credential) + defer release() + document, err := NewWatchEvent(mattermost.WatchPost{ID: "\n\t" + credential, ChannelID: "c", UserID: "u", Message: "line one\n" + credential, CreateAt: 1, FileIDs: []string{}}, mattermost.Sequence{ConnectionID: "\n\t" + credential, Number: 1}, true) + if err != nil { + t.Fatal(err) + } + var output bytes.Buffer + if err := WriteWatchDocument(&output, document); err != nil { + t.Fatal(err) + } + var decoded struct { + PostID string `json:"postId"` + Message string `json:"message"` + Sequence WatchSequence `json:"sequence"` + Redactions []presentation.Redaction `json:"redactions"` + } + if err := json.Unmarshal(output.Bytes(), &decoded); err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(decoded.PostID, `\n\t`) || !strings.HasPrefix(decoded.Sequence.ConnectionID, `\n\t`) || !strings.HasPrefix(decoded.Message, "line one\n") { + t.Fatalf("presentation=%#v", decoded) + } + positions := map[string]int{} + for _, redaction := range decoded.Redactions { + positions[redaction.Field] = redaction.Position + } + if positions["watch.postId"] != 4 || positions["watch.sequence.connectionId"] != 4 || positions["watch.message"] != 9 { + t.Fatalf("positions=%v", positions) + } + expected := pointerInt64(1) + received := pointerInt64(0) + diagnostic, err := NewWatchDiagnostic(mattermost.WatchDiagnostic{Type: "connection_changed", Timestamp: time.UnixMilli(1), Message: "\n\t" + credential, PreviousID: "\n\t" + credential, CurrentID: "new", Expected: expected, Received: received}) + if err != nil { + t.Fatal(err) + } + output.Reset() + if err := WriteWatchDocument(&output, diagnostic); err != nil { + t.Fatal(err) + } + var diagnosticDecoded struct { + Redactions []presentation.Redaction `json:"redactions"` + } + if err := json.Unmarshal(output.Bytes(), &diagnosticDecoded); err != nil { + t.Fatal(err) + } + for _, redaction := range diagnosticDecoded.Redactions { + if redaction.Field == "watch.diagnostic.message" || redaction.Field == "watch.diagnostic.previousConnectionId" { + if redaction.Position != 4 { + t.Fatalf("diagnostic redaction=%#v", redaction) + } + } + } +} diff --git a/internal/schema/watch_test.go b/internal/schema/watch_test.go new file mode 100644 index 0000000..365a27c --- /dev/null +++ b/internal/schema/watch_test.go @@ -0,0 +1,35 @@ +package schema + +import ( + "strings" + "testing" +) + +func TestWatchSchemasAreStrictAndDiscriminatorBound(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + validEvent := `{"schema":"mm/v2/watch-event","type":"posted","sequence":{"connectionId":"c","number":1},"postId":"p","channelId":"ch","channelName":"town","senderId":"u","sender":"arda","message":"hello\nworld","timestamp":"1970-01-01T00:00:00.001Z","rootId":null,"fileIds":[],"redactions":[]}` + if err := registry.Validate("mm/v2/watch-event", strings.NewReader(validEvent)); err != nil { + t.Fatal(err) + } + for _, mutation := range []string{ + strings.Replace(validEvent, `"number":1`, `"number":-1`, 1), + strings.Replace(validEvent, `"fileIds":[]`, `"fileIds":[""]`, 1), + strings.Replace(validEvent, `"redactions":[]`, `"redactions":[],"unknown":true`, 1), + } { + if err := registry.Validate("mm/v2/watch-event", strings.NewReader(mutation)); err == nil { + t.Fatalf("accepted mutation %s", mutation) + } + } + validDiagnostic := `{"schema":"mm/v2/watch-diagnostic","type":"reconnect","timestamp":"2026-07-16T00:00:00.000Z","message":"retry","backfill":false,"fatal":false,"redactions":[],"attempt":1,"delayMs":1000}` + if err := registry.Validate("mm/v2/watch-diagnostic", strings.NewReader(validDiagnostic)); err != nil { + t.Fatal(err) + } + for _, mutation := range []string{strings.Replace(validDiagnostic, `"backfill":false`, `"backfill":true`, 1), strings.Replace(validDiagnostic, `,"attempt":1,"delayMs":1000`, `,"expected":1,"received":2`, 1)} { + if err := registry.Validate("mm/v2/watch-diagnostic", strings.NewReader(mutation)); err == nil { + t.Fatalf("accepted mutation %s", mutation) + } + } +} diff --git a/internal/transport/websocket.go b/internal/transport/websocket.go new file mode 100644 index 0000000..3816abb --- /dev/null +++ b/internal/transport/websocket.go @@ -0,0 +1,60 @@ +package transport + +import ( + "context" + "net/http" + + "github.com/coder/websocket" +) + +type MessageType int + +const ( + MessageText MessageType = MessageType(websocket.MessageText) + MessageBinary MessageType = MessageType(websocket.MessageBinary) +) + +type WebSocket interface { + Read(context.Context) (MessageType, []byte, error) + Write(context.Context, MessageType, []byte) error + Close(context.Context) error + CloseNow() error + SetReadLimit(int64) +} + +type socketAdapter struct{ conn *websocket.Conn } + +func (socketAdapter socketAdapter) Read(ctx context.Context) (MessageType, []byte, error) { + type_, data, err := socketAdapter.conn.Read(ctx) + return MessageType(type_), data, err +} +func (socketAdapter socketAdapter) Write(ctx context.Context, type_ MessageType, data []byte) error { + return socketAdapter.conn.Write(ctx, websocket.MessageType(type_), data) +} +func (socketAdapter socketAdapter) Close(ctx context.Context) error { + done := make(chan error, 1) + go func() { done <- socketAdapter.conn.Close(websocket.StatusNormalClosure, "") }() + select { + case err := <-done: + return err + case <-ctx.Done(): + _ = socketAdapter.conn.CloseNow() + <-done + return ctx.Err() + } +} +func (socketAdapter socketAdapter) CloseNow() error { return socketAdapter.conn.CloseNow() } +func (socketAdapter socketAdapter) SetReadLimit(limit int64) { socketAdapter.conn.SetReadLimit(limit) } + +type DialWebSocket func(context.Context, string) (WebSocket, error) + +func Dial(ctx context.Context, target string) (WebSocket, error) { + conn, response, err := websocket.Dial(ctx, target, &websocket.DialOptions{HTTPClient: http.DefaultClient}) + if response != nil && response.Body != nil { + _ = response.Body.Close() + } + if err != nil { + return nil, err + } + return socketAdapter{conn: conn}, nil +} diff --git a/internal/transport/websocket_test.go b/internal/transport/websocket_test.go new file mode 100644 index 0000000..d060b93 --- /dev/null +++ b/internal/transport/websocket_test.go @@ -0,0 +1,15 @@ +package transport + +import ( + "context" + "testing" +) + +func TestMessageTypesRemainDistinct(t *testing.T) { + if MessageText == MessageBinary { + t.Fatal("text and binary WebSocket messages collapsed") + } + if _, err := Dial(context.Background(), "://invalid"); err == nil { + t.Fatal("invalid dial target accepted") + } +} diff --git a/schemas/v2/examples/watch-diagnostic.json b/schemas/v2/examples/watch-diagnostic.json new file mode 100644 index 0000000..667f604 --- /dev/null +++ b/schemas/v2/examples/watch-diagnostic.json @@ -0,0 +1 @@ +{"schema":"mm/v2/watch-diagnostic","type":"reconnect","timestamp":"2026-07-16T00:00:00.000Z","message":"WebSocket disconnected; reconnecting without REST backfill.","backfill":false,"fatal":false,"redactions":[],"attempt":1,"delayMs":1000} diff --git a/schemas/v2/examples/watch-event.json b/schemas/v2/examples/watch-event.json new file mode 100644 index 0000000..fc2431a --- /dev/null +++ b/schemas/v2/examples/watch-event.json @@ -0,0 +1 @@ +{"schema":"mm/v2/watch-event","type":"posted","sequence":{"connectionId":"connection-1","number":1},"postId":"post-1","channelId":"channel-1","channelName":"town-square","senderId":"user-1","sender":"arda","message":"hello","timestamp":"1970-01-01T00:00:00.001Z","rootId":null,"fileIds":[],"redactions":[]} diff --git a/schemas/v2/watch-diagnostic.schema.json b/schemas/v2/watch-diagnostic.schema.json new file mode 100644 index 0000000..2f04c07 --- /dev/null +++ b/schemas/v2/watch-diagnostic.schema.json @@ -0,0 +1,12 @@ +{ + "$schema":"https://json-schema.org/draft/2020-12/schema","$id":"urn:mm:schema:v2:watch-diagnostic","title":"mm v2 watch diagnostic","type":"object","additionalProperties":false, + "required":["schema","type","timestamp","message","backfill","fatal","redactions"], + "properties":{"schema":{"const":"mm/v2/watch-diagnostic"},"type":{"enum":["reconnect","disconnected","sequence_gap","connection_changed","malformed","terminal"]},"timestamp":{"type":"string","format":"date-time","pattern":"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]{3}Z$"},"message":{"type":"string","minLength":1},"backfill":{"const":false},"fatal":{"type":"boolean"},"redactions":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["type","masked","position","field"],"properties":{"type":{"type":"string","minLength":1},"masked":{"type":"string"},"position":{"type":"integer","minimum":0},"field":{"type":"string","minLength":1}}}},"attempt":{"type":"integer","minimum":1},"delayMs":{"type":"integer","minimum":0},"expected":{"type":"integer","minimum":0,"maximum":9007199254740991},"received":{"type":"integer","minimum":0,"maximum":9007199254740991},"previousConnectionId":{"type":"string","minLength":1},"currentConnectionId":{"type":"string","minLength":1}}, + "allOf":[ + {"if":{"properties":{"type":{"const":"reconnect"}}},"then":{"required":["attempt","delayMs"],"properties":{"fatal":{"const":false}},"not":{"anyOf":[{"required":["expected"]},{"required":["received"]},{"required":["previousConnectionId"]},{"required":["currentConnectionId"]}]}}}, + {"if":{"properties":{"type":{"const":"sequence_gap"}}},"then":{"required":["expected","received"],"properties":{"fatal":{"const":false}},"not":{"anyOf":[{"required":["attempt"]},{"required":["delayMs"]},{"required":["previousConnectionId"]},{"required":["currentConnectionId"]}]}}}, + {"if":{"properties":{"type":{"const":"connection_changed"}}},"then":{"required":["expected","received","previousConnectionId","currentConnectionId"],"properties":{"fatal":{"const":false}},"not":{"anyOf":[{"required":["attempt"]},{"required":["delayMs"]}]}}}, + {"if":{"properties":{"type":{"enum":["disconnected","malformed"]}}},"then":{"properties":{"fatal":{"const":false}},"not":{"anyOf":[{"required":["attempt"]},{"required":["delayMs"]},{"required":["expected"]},{"required":["received"]},{"required":["previousConnectionId"]},{"required":["currentConnectionId"]}]}}}, + {"if":{"properties":{"type":{"const":"terminal"}}},"then":{"properties":{"fatal":{"const":true}},"not":{"anyOf":[{"required":["attempt"]},{"required":["delayMs"]},{"required":["expected"]},{"required":["received"]},{"required":["previousConnectionId"]},{"required":["currentConnectionId"]}]}}} + ] +} diff --git a/schemas/v2/watch-event.schema.json b/schemas/v2/watch-event.schema.json new file mode 100644 index 0000000..4636de6 --- /dev/null +++ b/schemas/v2/watch-event.schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:watch-event", + "title": "mm v2 watch event", + "type": "object", + "additionalProperties": false, + "required": ["schema", "type", "sequence", "postId", "channelId", "channelName", "senderId", "sender", "message", "timestamp", "rootId", "fileIds", "redactions"], + "properties": { + "schema": { "const": "mm/v2/watch-event" }, + "type": { "const": "posted" }, + "sequence": { "type": "object", "additionalProperties": false, "required": ["connectionId", "number"], "properties": { "connectionId": { "type": "string", "minLength": 1 }, "number": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 } } }, + "postId": { "type": "string", "minLength": 1 }, + "channelId": { "type": "string", "minLength": 1 }, + "channelName": { "type": "string" }, + "senderId": { "type": "string", "minLength": 1 }, + "sender": { "type": "string" }, + "message": { "type": "string" }, + "timestamp": { "type": "string", "format": "date-time", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]{3}Z$" }, + "rootId": { "type": ["string", "null"], "minLength": 1 }, + "fileIds": { "type": "array", "items": { "type": "string", "minLength": 1 } }, + "redactions": { + "type": "array", + "items": { + "type": "object", "additionalProperties": false, + "required": ["type", "masked", "position", "field"], + "properties": { + "type": { "type": "string", "minLength": 1 }, "masked": { "type": "string" }, + "position": { "type": "integer", "minimum": 0 }, "field": { "type": "string", "minLength": 1 } + } + } + } + } +} From 8a56be2a756944a0b877afa7c095443034ade9c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 22:14:38 +0300 Subject: [PATCH 048/119] feat: add watch command --- cmd/mm/main.go | 4 +- cmd/mm/signal_other.go | 27 +++ cmd/mm/signal_unix.go | 26 ++ cmd/mm/signal_unix_test.go | 30 +++ internal/cli/read.go | 2 +- internal/cli/root.go | 9 + internal/cli/runtime.go | 35 ++- internal/cli/watch.go | 258 ++++++++++++++++++++ internal/cli/watch_test.go | 308 ++++++++++++++++++++++++ internal/mattermost/watch.go | 12 +- internal/output/watch.go | 31 ++- internal/output/watch_test.go | 36 +++ internal/schema/watch_test.go | 22 +- schemas/v2/watch-diagnostic.schema.json | 16 +- 14 files changed, 789 insertions(+), 27 deletions(-) create mode 100644 internal/cli/watch.go create mode 100644 internal/cli/watch_test.go diff --git a/cmd/mm/main.go b/cmd/mm/main.go index b600837..2c07ccd 100644 --- a/cmd/mm/main.go +++ b/cmd/mm/main.go @@ -9,5 +9,7 @@ import ( func main() { handleBrokenPipe() - os.Exit(cli.Execute(context.Background(), os.Args[1:], os.Stdin, os.Stdout, os.Stderr)) + ctx, stop := commandContext(context.Background()) + defer stop() + os.Exit(cli.Execute(ctx, os.Args[1:], os.Stdin, os.Stdout, os.Stderr)) } diff --git a/cmd/mm/signal_other.go b/cmd/mm/signal_other.go index 1af0e8c..eb32ef4 100644 --- a/cmd/mm/signal_other.go +++ b/cmd/mm/signal_other.go @@ -2,4 +2,31 @@ package main +import ( + "context" + "os" + "os/signal" + "sync" + + "github.com/ardasevinc/mattermost-cli/internal/cli" +) + func handleBrokenPipe() {} +func commandContext(parent context.Context) (context.Context, func()) { + ctx, cancel := context.WithCancelCause(parent) + notifications := make(chan os.Signal, 1) + stopped := make(chan struct{}) + signal.Notify(notifications, os.Interrupt) + var once sync.Once + go func() { + select { + case <-parent.Done(): + cancel(context.Cause(parent)) + case <-notifications: + cancel(cli.ErrSignalCancellation) + case <-stopped: + cancel(context.Canceled) + } + }() + return ctx, func() { once.Do(func() { signal.Stop(notifications); close(stopped) }) } +} diff --git a/cmd/mm/signal_unix.go b/cmd/mm/signal_unix.go index 6f0f11a..9e40a4b 100644 --- a/cmd/mm/signal_unix.go +++ b/cmd/mm/signal_unix.go @@ -3,9 +3,13 @@ package main import ( + "context" "os" "os/signal" + "sync" "syscall" + + "github.com/ardasevinc/mattermost-cli/internal/cli" ) var brokenPipeSignals = make(chan os.Signal, 1) @@ -13,3 +17,25 @@ var brokenPipeSignals = make(chan os.Signal, 1) func handleBrokenPipe() { signal.Notify(brokenPipeSignals, syscall.SIGPIPE) } + +func commandContext(parent context.Context) (context.Context, func()) { + return causedSignalContext(parent, os.Interrupt, syscall.SIGTERM) +} +func causedSignalContext(parent context.Context, signals ...os.Signal) (context.Context, func()) { + ctx, cancel := context.WithCancelCause(parent) + notifications := make(chan os.Signal, 1) + stopped := make(chan struct{}) + signal.Notify(notifications, signals...) + var once sync.Once + go func() { + select { + case <-parent.Done(): + cancel(context.Cause(parent)) + case <-notifications: + cancel(cli.ErrSignalCancellation) + case <-stopped: + cancel(context.Canceled) + } + }() + return ctx, func() { once.Do(func() { signal.Stop(notifications); close(stopped) }) } +} diff --git a/cmd/mm/signal_unix_test.go b/cmd/mm/signal_unix_test.go index 3f1306c..2088cf6 100644 --- a/cmd/mm/signal_unix_test.go +++ b/cmd/mm/signal_unix_test.go @@ -10,7 +10,9 @@ import ( "os/exec" "os/signal" "strings" + "syscall" "testing" + "time" "github.com/ardasevinc/mattermost-cli/internal/cli" ) @@ -63,3 +65,31 @@ func TestBrokenPipeHandlerIsNotInheritedByChild(t *testing.T) { t.Fatalf("child inherited handled SIGPIPE: %q", output) } } + +func TestCommandContextStopsWithParent(t *testing.T) { + parent, cancel := context.WithCancel(context.Background()) + ctx, stop := commandContext(parent) + cancel() + defer stop() + select { + case <-ctx.Done(): + case <-time.After(time.Second): + t.Fatal("command context did not stop") + } +} + +func TestCommandContextRecordsSignalCause(t *testing.T) { + ctx, stop := commandContext(context.Background()) + defer stop() + if err := syscall.Kill(os.Getpid(), syscall.SIGTERM); err != nil { + t.Fatal(err) + } + select { + case <-ctx.Done(): + if !errors.Is(context.Cause(ctx), cli.ErrSignalCancellation) { + t.Fatalf("cause=%v", context.Cause(ctx)) + } + case <-time.After(time.Second): + t.Fatal("signal did not cancel command context") + } +} diff --git a/internal/cli/read.go b/internal/cli/read.go index e6ddff6..0e2a4d6 100644 --- a/internal/cli/read.go +++ b/internal/cli/read.go @@ -281,7 +281,7 @@ func emitRedactionWarning(s *rootState, runtime *Runtime, machine bool) error { } warning := "warning: secret redaction is disabled; output may contain secrets\n" if machine { - s.queueMachineWarning(warning) + s.queueTypedMachineWarning("redaction_disabled", strings.TrimSuffix(warning, "\n")) return nil } return writeAll(s.streams.err, []byte(warning)) diff --git a/internal/cli/root.go b/internal/cli/root.go index 1d3032b..efe7a05 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -37,6 +37,14 @@ func Execute(ctx context.Context, args []string, in io.Reader, out, errOut io.Wr cmd := newRootWithState(state) cmd.SetArgs(args) if err := cmd.ExecuteContext(ctx); err != nil { + var corrupted watchOutputFailure + if errors.As(err, &corrupted) { + return 3 + } + var terminal watchTerminalFailure + if errors.As(err, &terminal) { + return 3 + } message := presentation.SanitizeLabel(presentation.Preprocess(err.Error(), state.credentials).Text) code := exitCode(err) if state.flags.json && trackedOut.BytesWritten() > 0 { @@ -129,6 +137,7 @@ func newRootWithState(state *rootState) *cobra.Command { cmd.AddCommand(newSearchCommand(state)) cmd.AddCommand(newMentionsCommand(state)) cmd.AddCommand(newUnreadCommand(state)) + cmd.AddCommand(newWatchCommand(state)) return cmd } diff --git a/internal/cli/runtime.go b/internal/cli/runtime.go index 3fb6495..aeb972a 100644 --- a/internal/cli/runtime.go +++ b/internal/cli/runtime.go @@ -1,6 +1,7 @@ package cli import ( + "context" "errors" "os" "strings" @@ -22,6 +23,7 @@ type dependencies struct { homeDir func() (string, error) newClient clientFactory stdoutTTY func() bool + watch func(context.Context, mattermost.WatchOptions) error } func defaultDependencies(out any) dependencies { @@ -37,6 +39,7 @@ func defaultDependencies(out any) dependencies { info, err := file.Stat() return err == nil && info.Mode()&os.ModeCharDevice != 0 }, + watch: mattermost.Watch, } } @@ -68,7 +71,7 @@ type rootState struct { warned bool releases []func() credentials []string - pendingWarnings []string + pendingWarnings []machineWarning semanticExit int disableHeuristics bool } @@ -143,7 +146,7 @@ func (s *rootState) runtimeFor(cmd *cobra.Command) (*Runtime, error) { if warning := file.Warning(); warning != "" && !s.warned { warning = presentation.SanitizeLabel(presentation.Preprocess(warning, s.credentials).Text) if s.flags.json { - s.pendingWarnings = append(s.pendingWarnings, "warning: "+warning+"\n") + s.pendingWarnings = append(s.pendingWarnings, machineWarning{code: "configuration_warning", message: "warning: " + warning}) } else { if err := writeAll(s.streams.err, []byte("warning: "+warning+"\n")); err != nil { s.runtimeErr = err @@ -204,20 +207,40 @@ func (s *rootState) redactOption(cmd *cobra.Command) (*bool, error) { func (s *rootState) flushMachineWarnings() error { s.mu.Lock() - warnings := strings.Join(s.pendingWarnings, "") + items := s.pendingWarnings s.pendingWarnings = nil s.mu.Unlock() - if warnings == "" { + if len(items) == 0 { return nil } - return writeAll(s.streams.err, []byte(warnings)) + var warnings strings.Builder + for _, item := range items { + warnings.WriteString(item.message) + warnings.WriteByte('\n') + } + return writeAll(s.streams.err, []byte(warnings.String())) } func (s *rootState) queueMachineWarning(message string) { s.mu.Lock() - s.pendingWarnings = append(s.pendingWarnings, message) + s.pendingWarnings = append(s.pendingWarnings, machineWarning{code: "configuration_warning", message: strings.TrimSuffix(message, "\n")}) + s.mu.Unlock() +} + +type machineWarning struct{ code, message string } + +func (s *rootState) queueTypedMachineWarning(code, message string) { + s.mu.Lock() + s.pendingWarnings = append(s.pendingWarnings, machineWarning{code, message}) s.mu.Unlock() } +func (s *rootState) takeMachineWarnings() []machineWarning { + s.mu.Lock() + defer s.mu.Unlock() + items := append([]machineWarning(nil), s.pendingWarnings...) + s.pendingWarnings = nil + return items +} func (s *rootState) setSemanticExit(code int) { s.mu.Lock() diff --git a/internal/cli/watch.go b/internal/cli/watch.go new file mode 100644 index 0000000..0b6904c --- /dev/null +++ b/internal/cli/watch.go @@ -0,0 +1,258 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + "time" + + "github.com/spf13/cobra" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/output" + "github.com/ardasevinc/mattermost-cli/internal/presentation" +) + +type watchFlags struct{ team, dm string } + +type watchOutputFailure struct{} + +func (watchOutputFailure) Error() string { return "watch output failed" } + +type watchTerminalFailure struct{} + +func (watchTerminalFailure) Error() string { return "watch terminated" } + +var ErrSignalCancellation = errors.New("command canceled by signal") + +func newWatchCommand(state *rootState) *cobra.Command { + flags := new(watchFlags) + command := &cobra.Command{Use: "watch [channel]", Short: "Watch posted events in a channel or direct message", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { + channel := "" + if len(args) == 1 { + channel = args[0] + } + err := runWatch(cmd, state, channel, *flags) + if err != nil && errors.Is(context.Cause(cmd.Context()), ErrSignalCancellation) { + if !watchStreamFailed(state, err) { + state.takeMachineWarnings() + return nil + } + } + return err + }} + command.Flags().StringVar(&flags.team, "team", "", "team name (auto-detected for one team)") + command.Flags().StringVar(&flags.dm, "dm", "", "direct-message username") + return command +} + +func runWatch(cmd *cobra.Command, state *rootState, channelName string, flags watchFlags) error { + channelSet := strings.TrimSpace(channelName) != "" + dmSet := strings.TrimSpace(flags.dm) != "" + if channelSet == dmSet { + return invalidFailure("provide exactly one channel name or --dm username") + } + if flagChanged(cmd, "dm") && !dmSet { + return invalidFailure("--dm cannot be empty") + } + if flagChanged(cmd, "team") && strings.TrimSpace(flags.team) == "" { + return invalidFailure("--team cannot be empty") + } + if dmSet && flagChanged(cmd, "team") { + return invalidFailure("--team cannot be combined with --dm") + } + display, err := state.readDisplay(cmd) + if err != nil { + return err + } + runtime, err := state.runtimeFor(cmd) + if err != nil { + return err + } + me, err := runtime.Users.Current(cmd.Context()) + if err != nil { + return readFailure(err) + } + var channel mattermost.Channel + knownSenders := map[string]string{me.ID: me.Username} + targetLabel := "" + if dmSet { + username := strings.TrimPrefix(flags.dm, "@") + partner, lookupErr := runtime.Users.ByUsername(cmd.Context(), username) + if lookupErr != nil { + return readFailure(lookupErr) + } + if partner.ID == me.ID { + return invalidFailure("cannot watch a direct message with the current user") + } + knownSenders[partner.ID] = partner.Username + channels, listErr := runtime.Channels.DirectList(cmd.Context(), me.ID) + if listErr != nil { + return readFailure(listErr) + } + for _, candidate := range channels { + parts := strings.Split(candidate.Name, "__") + if len(parts) == 2 && ((parts[0] == me.ID && parts[1] == partner.ID) || (parts[1] == me.ID && parts[0] == partner.ID)) { + if channel.ID != "" && channel.ID != candidate.ID { + return readError("Mattermost returned ambiguous direct-message channels") + } + channel = candidate + } + } + if channel.ID == "" { + return readError("direct-message channel was not found") + } + targetLabel = "DMs with @" + presentWatchLabel(username, runtime.Config.Token, state.disableHeuristics) + } else { + team, teamErr := runtime.Teams.Resolve(cmd.Context(), me.ID, flags.team) + if teamErr != nil { + return readFailure(teamErr) + } + channel, err = runtime.Channels.ByName(cmd.Context(), team.ID, channelName) + if err != nil { + return readFailure(err) + } + if _, err = runtime.Channels.Member(cmd.Context(), channel.ID, me.ID); err != nil { + return readFailure(err) + } + targetLabel = "#" + presentWatchLabel(channel.Name, runtime.Config.Token, state.disableHeuristics) + } + if err := emitWatchRedactionWarning(state, runtime, display.json); err != nil { + return err + } + var sink mattermost.WatchSink + if display.json { + sink = output.JSONLWatchSink{Events: state.streams.out, Diagnostics: state.streams.err, DisableHeuristics: state.disableHeuristics} + } else { + sink = &humanWatchSink{out: state.streams.out, err: state.streams.err, token: runtime.Config.Token, disableHeuristics: state.disableHeuristics, color: display.color && runtime.StdoutTTY} + if err := writeWatchLine(state.streams.err, "Watching "+targetLabel+" (Ctrl+C to stop)\n"); err != nil { + return watchOutputFailure{} + } + } + sink = &watchSenderBindingSink{next: sink, known: knownSenders} + if display.json { + for _, warning := range state.takeMachineWarnings() { + if err := sink.Diagnostic(mattermost.WatchDiagnostic{Type: "warning", Code: warning.code, Recovery: "none", Timestamp: time.Now().UTC(), Message: warning.message}); err != nil { + return watchOutputFailure{} + } + } + } + if cmd.Context().Err() != nil && errors.Is(context.Cause(cmd.Context()), ErrSignalCancellation) { + return nil + } + err = state.deps.watch(cmd.Context(), mattermost.WatchOptions{URL: runtime.Config.URL, Token: runtime.Config.Token, ChannelID: channel.ID, Sink: sink}) + if err == nil { + return nil + } + if errors.Is(err, mattermost.ErrWatchSink) { + return watchOutputFailure{} + } + if errors.Is(err, context.Canceled) && errors.Is(context.Cause(cmd.Context()), ErrSignalCancellation) { + return nil + } + code, recovery, message := "watch_failed", "none", "WebSocket watch failed." + switch { + case errors.Is(err, mattermost.ErrWatchAuthentication): + code, recovery, message = "authentication", "check_token", "WebSocket authentication failed; check the configured token." + case errors.Is(err, mattermost.ErrWatchRetries): + code, recovery, message = "reconnect_exhausted", "retry_later", "WebSocket reconnect limit reached; retry later." + case errors.Is(err, mattermost.ErrInvalidWatchOptions): + code, message = "invalid_options", "WebSocket watch configuration is invalid." + case errors.Is(err, context.Canceled): + code, message = "canceled", "WebSocket watch was canceled by the caller." + } + if sinkErr := sink.Diagnostic(mattermost.WatchDiagnostic{Type: "terminal", Code: code, Recovery: recovery, Timestamp: time.Now().UTC(), Message: message, Fatal: true}); sinkErr != nil { + return watchOutputFailure{} + } + return watchTerminalFailure{} +} + +type humanWatchSink struct { + out, err io.Writer + token string + disableHeuristics bool + color bool +} + +func (s *humanWatchSink) Post(post mattermost.WatchPost, _ mattermost.Sequence) error { + sender := presentWatchLabel(post.SenderName, s.token, s.disableHeuristics) + message := presentWatchMessage(post.Message, s.token, s.disableHeuristics) + return writeWatchLine(s.out, output.FormatWatchHumanLine(time.UnixMilli(post.CreateAt), sender, message, s.color)+"\n") +} + +type watchSenderBindingSink struct { + next mattermost.WatchSink + known map[string]string +} + +func (s *watchSenderBindingSink) Post(post mattermost.WatchPost, sequence mattermost.Sequence) error { + if canonical, ok := s.known[post.UserID]; ok { + post.SenderName = canonical + } else if post.SenderName == "" { + post.SenderName = "unknown" + } + return s.next.Post(post, sequence) +} + +func (s *watchSenderBindingSink) Diagnostic(value mattermost.WatchDiagnostic) error { + return s.next.Diagnostic(value) +} + +func emitWatchRedactionWarning(state *rootState, runtime *Runtime, machine bool) error { + if runtime.Config.Redact { + return nil + } + const warning = "Warning: Secret redaction is disabled. Output may contain secrets." + if machine { + state.queueTypedMachineWarning("redaction_disabled", warning) + return nil + } + return writeAll(state.streams.err, []byte(warning+"\n")) +} + +func watchStreamFailed(state *rootState, err error) bool { + var outputFailure outputError + var watchFailure watchOutputFailure + if errors.As(err, &outputFailure) || errors.As(err, &watchFailure) { + return true + } + tracker, ok := state.streams.out.(interface{ Failed() bool }) + return ok && tracker.Failed() +} +func (s *humanWatchSink) Diagnostic(value mattermost.WatchDiagnostic) error { + message := "WebSocket diagnostic." + switch value.Type { + case "reconnect": + message = fmt.Sprintf("WebSocket disconnected; reconnecting in %dms (attempt %d); no REST backfill.", value.Delay.Milliseconds(), *value.Attempt) + case "sequence_gap": + message = fmt.Sprintf("Warning: WebSocket sequence gap detected (expected %d, received %d); live events may be missing; no REST backfill.", *value.Expected, *value.Received) + case "connection_changed": + message = "Warning: WebSocket connection changed; live events may be missing; no REST backfill." + case "malformed": + message = "Warning: Malformed WebSocket event skipped." + case "disconnected": + message = "WebSocket disconnected; live events may be missing." + case "warning": + message = value.Message + case "terminal": + message = value.Message + } + return writeWatchLine(s.err, message+"\n") +} + +func presentWatchLabel(value, token string, disable bool) string { + value = presentation.SanitizeLabel(value) + return presentation.PreprocessWithOptions(value, presentation.Options{Credentials: []string{token}, DisableHeuristics: disable}).Text +} +func presentWatchMessage(value, token string, disable bool) string { + return presentation.PreprocessWithOptions(value, presentation.Options{Credentials: []string{token}, DisableHeuristics: disable}).Text +} +func writeWatchLine(writer io.Writer, value string) error { + written, err := writer.Write([]byte(value)) + if err != nil || written != len(value) { + return io.ErrShortWrite + } + return nil +} diff --git a/internal/cli/watch_test.go b/internal/cli/watch_test.go new file mode 100644 index 0000000..53f3226 --- /dev/null +++ b/internal/cli/watch_test.go @@ -0,0 +1,308 @@ +package cli + +import ( + "bytes" + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/presentation" + mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" +) + +func TestWatchSelectorsFailBeforeNetwork(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { requests.Add(1) })) + defer server.Close() + for _, args := range [][]string{{"watch"}, {"watch", "town", "--dm", "bob"}, {"watch", "--dm", "bob", "--team", "main"}, {"watch", "--dm="}, {"watch", "town", "--team="}} { + _, _, code := executeChannel(t, server.URL, args...) + if code != 2 || requests.Load() != 0 { + t.Fatalf("args=%v code=%d requests=%d", args, code, requests.Load()) + } + } +} + +func TestWatchMachineChannelResolutionAndSplit(t *testing.T) { + server := watchChannelServer(t) + defer server.Close() + watch := func(_ context.Context, options mattermost.WatchOptions) error { + if options.ChannelID != "channel1" { + t.Fatalf("channel=%s", options.ChannelID) + } + release := presentation.ActiveCredentials.Register(options.Token) + defer release() + if err := options.Sink.Post(mattermost.WatchPost{ID: "post", ChannelID: "channel1", UserID: "user1", SenderName: "spoofed", Message: "line one\nline two test-token", CreateAt: 1, FileIDs: []string{}}, mattermost.Sequence{ConnectionID: "connection", Number: 1}); err != nil { + return mattermost.ErrWatchSink + } + attempt := 1 + delay := time.Second + return options.Sink.Diagnostic(mattermost.WatchDiagnostic{Type: "reconnect", Timestamp: time.UnixMilli(1), Message: "retry", Attempt: &attempt, Delay: &delay}) + } + stdout, stderr, err := runWatchCommand(t, server.URL, watch, context.Background(), "--json", "watch", "town", "--team", "main") + if err != nil { + t.Fatal(err) + } + registry, _ := mmSchema.Load() + if err := registry.Validate("mm/v2/watch-event", strings.NewReader(stdout)); err != nil { + t.Fatalf("event schema: %v\n%s", err, stdout) + } + if err := registry.Validate("mm/v2/watch-diagnostic", strings.NewReader(stderr)); err != nil { + t.Fatalf("diagnostic schema: %v\n%s", err, stderr) + } + if !strings.Contains(stdout, `line one\nline two`) || !strings.Contains(stdout, `"sender":"arda"`) || strings.Contains(stdout, "spoofed") || strings.Contains(stdout, "test-token") || strings.Contains(stderr, "Watching") { + t.Fatalf("stdout=%q stderr=%q", stdout, stderr) + } +} + +func TestWatchDMResolutionUsesReadOnlyExactDirectListAndRejectsSelf(t *testing.T) { + var methods []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + methods = append(methods, r.Method+" "+r.URL.Path) + switch r.URL.Path { + case "/api/v4/users/me": + writeJSON(t, w, `{"id":"user1","username":"arda"}`) + case "/api/v4/users/username/bob": + writeJSON(t, w, `{"id":"bob","username":"bob"}`) + case "/api/v4/users/user1/channels": + writeJSON(t, w, `[{"id":"dm","team_id":"","type":"D","name":"bob__user1","display_name":""}]`) + default: + t.Fatalf("unexpected %s", r.URL.Path) + } + })) + defer server.Close() + called := false + _, _, err := runWatchCommand(t, server.URL, func(_ context.Context, options mattermost.WatchOptions) error { + called = true + if options.ChannelID != "dm" { + t.Fatal(options.ChannelID) + } + return nil + }, context.Background(), "watch", "--dm", "bob") + if err != nil || !called { + t.Fatalf("err=%v called=%v", err, called) + } + for _, method := range methods { + if strings.HasPrefix(method, "POST ") { + t.Fatalf("mutation: %v", methods) + } + } +} + +func TestWatchHumanPresentationCancellationAndTerminalOwnership(t *testing.T) { + server := watchChannelServer(t) + defer server.Close() + ctx, cancel := context.WithCancelCause(context.Background()) + watch := func(ctx context.Context, options mattermost.WatchOptions) error { + _ = options.Sink.Post(mattermost.WatchPost{ID: "p", ChannelID: "channel1", UserID: "u", SenderName: "", Message: "hello\n world", CreateAt: 0, FileIDs: []string{}}, mattermost.Sequence{ConnectionID: "c", Number: 1}) + cancel(ErrSignalCancellation) + <-ctx.Done() + return ctx.Err() + } + stdout, stderr, err := runWatchCommand(t, server.URL, watch, ctx, "--no-color", "watch", "town", "--team", "main") + if err != nil || !strings.Contains(stdout, "unknown: hello world") || !strings.Contains(stderr, "Watching #town") { + t.Fatalf("err=%v stdout=%q stderr=%q", err, stdout, stderr) + } + _, machineErr, err := runWatchCommand(t, server.URL, func(context.Context, mattermost.WatchOptions) error { return mattermost.ErrWatchAuthentication }, context.Background(), "--json", "watch", "town", "--team", "main") + if _, ok := err.(watchTerminalFailure); !ok { + t.Fatalf("err=%T %v", err, err) + } + if strings.Count(machineErr, "\n") != 1 || !strings.Contains(machineErr, `"type":"terminal"`) || strings.Contains(machineErr, "mm/v2/error") { + t.Fatalf("stderr=%q", machineErr) + } +} + +func TestWatchMachineWriterFailureIsTerminalWithoutSecondObject(t *testing.T) { + server := watchChannelServer(t) + defer server.Close() + setChannelEnvironment(t, server.URL) + var stderr bytes.Buffer + state := &rootState{streams: streams{in: strings.NewReader(""), out: shortWriter{}, err: &stderr}, deps: defaultDependencies(shortWriter{})} + state.deps.watch = func(_ context.Context, options mattermost.WatchOptions) error { + if options.Sink.Post(mattermost.WatchPost{ID: "p", ChannelID: "channel1", UserID: "u", CreateAt: 1, FileIDs: []string{}}, mattermost.Sequence{ConnectionID: "c", Number: 1}) != nil { + return mattermost.ErrWatchSink + } + return nil + } + command := newRootWithState(state) + command.SetArgs([]string{"--json", "watch", "town", "--team", "main"}) + err := command.ExecuteContext(context.Background()) + state.close() + if _, ok := err.(watchOutputFailure); !ok || stderr.Len() != 0 { + t.Fatalf("err=%T stderr=%q", err, stderr.String()) + } +} + +func TestWatchMachineWarningsAreJSONLAndConsumed(t *testing.T) { + server := watchChannelServer(t) + defer server.Close() + watch := func(_ context.Context, options mattermost.WatchOptions) error { + if err := options.Sink.Post(mattermost.WatchPost{ID: "post", ChannelID: "channel1", UserID: "u2", SenderName: "arda", Message: "credential test-token", CreateAt: 1, FileIDs: []string{}}, mattermost.Sequence{ConnectionID: "connection", Number: 1}); err != nil { + return mattermost.ErrWatchSink + } + return nil + } + stdout, stderr, err := runWatchCommand(t, server.URL, watch, context.Background(), "--json", "--no-redact", "watch", "town", "--team", "main") + if err != nil || !strings.Contains(stdout, `"type":"posted"`) || strings.Contains(stdout, "test-token") { + t.Fatalf("err=%v stdout=%q", err, stdout) + } + lines := strings.Split(strings.TrimSpace(stderr), "\n") + if len(lines) != 1 || !strings.Contains(lines[0], `"type":"warning"`) || !strings.Contains(lines[0], `"code":"redaction_disabled"`) || strings.Contains(stderr, "warning: warning:") { + t.Fatalf("stderr=%q", stderr) + } + registry, _ := mmSchema.Load() + if err := registry.Validate("mm/v2/watch-diagnostic", strings.NewReader(lines[0])); err != nil { + t.Fatal(err) + } +} + +func TestWatchHumanNoRedactWarningMatchesFrozenV1Text(t *testing.T) { + server := watchChannelServer(t) + defer server.Close() + _, stderr, err := runWatchCommand(t, server.URL, func(context.Context, mattermost.WatchOptions) error { return nil }, context.Background(), "--no-redact", "watch", "town", "--team", "main") + if err != nil || !strings.HasPrefix(stderr, "Warning: Secret redaction is disabled. Output may contain secrets.\nWatching #town (Ctrl+C to stop)\n") { + t.Fatalf("err=%v stderr=%q", err, stderr) + } +} + +func TestWatchConsumesQueuedConfigurationWarningBeforeStreaming(t *testing.T) { + server := watchChannelServer(t) + defer server.Close() + setChannelEnvironment(t, server.URL) + var stdout, stderr bytes.Buffer + state := &rootState{streams: streams{in: strings.NewReader(""), out: &stdout, err: &stderr}, deps: defaultDependencies(&stdout)} + state.queueTypedMachineWarning("configuration_warning", "legacy configuration path in use") + state.deps.watch = func(context.Context, mattermost.WatchOptions) error { return nil } + command := newRootWithState(state) + command.SetArgs([]string{"--json", "watch", "town", "--team", "main"}) + err := command.ExecuteContext(context.Background()) + state.close() + if err != nil || !strings.Contains(stderr.String(), `"code":"configuration_warning"`) || strings.Contains(stderr.String(), "legacy configuration path in use\nwarning:") { + t.Fatalf("err=%v stderr=%q", err, stderr.String()) + } + if len(state.takeMachineWarnings()) != 0 { + t.Fatal("warning was not consumed") + } +} + +func TestWatchTerminalClassificationsAreClosed(t *testing.T) { + server := watchChannelServer(t) + defer server.Close() + tests := []struct { + err error + code, recovery string + }{{mattermost.ErrWatchAuthentication, "authentication", "check_token"}, {mattermost.ErrWatchRetries, "reconnect_exhausted", "retry_later"}, {mattermost.ErrInvalidWatchOptions, "invalid_options", "none"}, {context.Canceled, "canceled", "none"}, {errors.New("hostile remote token"), "watch_failed", "none"}} + for _, test := range tests { + _, stderr, err := runWatchCommand(t, server.URL, func(context.Context, mattermost.WatchOptions) error { return test.err }, context.Background(), "--json", "watch", "town", "--team", "main") + if _, ok := err.(watchTerminalFailure); !ok || !strings.Contains(stderr, `"code":"`+test.code+`"`) || !strings.Contains(stderr, `"recovery":"`+test.recovery+`"`) || strings.Contains(stderr, "hostile remote token") { + t.Fatalf("case=%s err=%T stderr=%q", test.code, err, stderr) + } + } +} + +func TestWatchHumanTTYColorAndCanonicalSenderFallback(t *testing.T) { + server := watchChannelServer(t) + defer server.Close() + for _, test := range []struct { + args []string + color bool + }{{[]string{"watch", "town", "--team", "main"}, true}, {[]string{"--no-color", "watch", "town", "--team", "main"}, false}} { + setChannelEnvironment(t, server.URL) + var stdout, stderr bytes.Buffer + state := &rootState{streams: streams{in: strings.NewReader(""), out: &stdout, err: &stderr}, deps: defaultDependencies(&stdout)} + state.deps.stdoutTTY = func() bool { return true } + state.deps.watch = func(_ context.Context, options mattermost.WatchOptions) error { + return options.Sink.Post(mattermost.WatchPost{ID: "p", ChannelID: "channel1", UserID: "user1", SenderName: "spoofed", Message: "a\tb\u00a0c\ufeffd\u0085e", CreateAt: 0, FileIDs: []string{}}, mattermost.Sequence{ConnectionID: "c", Number: 1}) + } + command := newRootWithState(state) + command.SetArgs(test.args) + err := command.ExecuteContext(context.Background()) + state.close() + if err != nil || !strings.Contains(stdout.String(), "arda") || strings.Contains(stdout.String(), "spoofed") || !strings.Contains(stdout.String(), `a b c d\u0085e`) || (strings.Contains(stdout.String(), "\x1b[") != test.color) { + t.Fatalf("color=%v err=%v stdout=%q", test.color, err, stdout.String()) + } + } +} + +func TestWatchSignalDoesNotSuppressRuntimeWarningOutputFailure(t *testing.T) { + home := t.TempDir() + xdg := t.TempDir() + writeRuntimeConfig(t, home, "url = \"https://example.com\"\ntoken = \"token\"\n") + lookup := func(key string) (string, bool) { + if key == "XDG_CONFIG_HOME" { + return xdg, true + } + return "", false + } + var stdout bytes.Buffer + state := &rootState{streams: streams{in: strings.NewReader(""), out: &stdout, err: shortWriter{}}, deps: defaultDependencies(&stdout)} + state.deps.homeDir = func() (string, error) { return home, nil } + state.deps.lookupEnv = lookup + ctx, cancel := context.WithCancelCause(context.Background()) + cancel(ErrSignalCancellation) + command := newRootWithState(state) + command.SetArgs([]string{"watch", "town"}) + err := command.ExecuteContext(ctx) + state.close() + var outputFailure outputError + if !errors.As(err, &outputFailure) { + t.Fatalf("err=%T %v", err, err) + } +} + +func TestWatchSignalCauseDuringResolutionIsClean(t *testing.T) { + started := make(chan struct{}, 1) + server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + select { + case started <- struct{}{}: + default: + } + <-r.Context().Done() + })) + defer server.Close() + ctx, cancel := context.WithCancelCause(context.Background()) + done := make(chan error, 1) + go func() { + _, _, err := runWatchCommand(t, server.URL, func(context.Context, mattermost.WatchOptions) error { return errors.New("watch unexpectedly started") }, ctx, "--json", "watch", "town") + done <- err + }() + <-started + cancel(ErrSignalCancellation) + if err := <-done; err != nil { + t.Fatalf("err=%v", err) + } +} + +func runWatchCommand(t *testing.T, serverURL string, watch func(context.Context, mattermost.WatchOptions) error, ctx context.Context, args ...string) (string, string, error) { + t.Helper() + setChannelEnvironment(t, serverURL) + var stdout, stderr bytes.Buffer + state := &rootState{streams: streams{in: strings.NewReader(""), out: &stdout, err: &stderr}, deps: defaultDependencies(&stdout)} + state.deps.watch = watch + command := newRootWithState(state) + command.SetArgs(args) + err := command.ExecuteContext(ctx) + state.close() + return stdout.String(), stderr.String(), err +} +func watchChannelServer(t *testing.T) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v4/users/me": + writeJSON(t, w, `{"id":"user1","username":"arda"}`) + case "/api/v4/users/user1/teams": + writeJSON(t, w, `[{"id":"team1","name":"main","display_name":"Main","type":"O"}]`) + case "/api/v4/teams/team1/channels/name/town": + writeJSON(t, w, `{"id":"channel1","team_id":"team1","type":"O","name":"town","display_name":"Town"}`) + case "/api/v4/channels/channel1/members/user1": + writeJSON(t, w, `{"channel_id":"channel1","user_id":"user1"}`) + default: + t.Fatalf("unexpected %s %s", r.Method, r.URL.Path) + } + })) +} diff --git a/internal/mattermost/watch.go b/internal/mattermost/watch.go index 56f4a69..9233af0 100644 --- a/internal/mattermost/watch.go +++ b/internal/mattermost/watch.go @@ -40,12 +40,12 @@ type Sequence struct { Number int64 } type WatchDiagnostic struct { - Type, Message, PreviousID, CurrentID string - Timestamp time.Time - Backfill, Fatal bool - Expected, Received *int64 - Attempt *int - Delay *time.Duration + Type, Code, Recovery, Message, PreviousID, CurrentID string + Timestamp time.Time + Backfill, Fatal bool + Expected, Received *int64 + Attempt *int + Delay *time.Duration } type WatchSink interface { Post(WatchPost, Sequence) error diff --git a/internal/output/watch.go b/internal/output/watch.go index 42526c0..2c33a1e 100644 --- a/internal/output/watch.go +++ b/internal/output/watch.go @@ -6,6 +6,7 @@ import ( "errors" "io" "reflect" + "strings" "time" "github.com/ardasevinc/mattermost-cli/internal/mattermost" @@ -45,6 +46,8 @@ type watchEvent struct { type watchDiagnostic struct { Schema string `json:"schema"` Type string `json:"type"` + Code string `json:"code,omitempty"` + Recovery string `json:"recovery,omitempty"` Timestamp MillisTime `json:"timestamp"` Message string `json:"message"` Backfill bool `json:"backfill"` @@ -77,16 +80,20 @@ func (value watchDiagnostic) valid() bool { return false } switch value.Type { + case "warning": + return !value.Fatal && (value.Code == "configuration_warning" || value.Code == "redaction_disabled") && value.Recovery == "none" && value.Attempt == nil && value.DelayMS == nil && value.Expected == nil && value.Received == nil && value.PreviousID == nil && value.CurrentID == nil case "reconnect": - return !value.Fatal && value.Attempt != nil && *value.Attempt > 0 && value.DelayMS != nil && *value.DelayMS >= 0 && value.Expected == nil && value.Received == nil && value.PreviousID == nil && value.CurrentID == nil + return value.Code == "" && value.Recovery == "" && !value.Fatal && value.Attempt != nil && *value.Attempt > 0 && value.DelayMS != nil && *value.DelayMS >= 0 && value.Expected == nil && value.Received == nil && value.PreviousID == nil && value.CurrentID == nil case "sequence_gap": - return !value.Fatal && value.Expected != nil && *value.Expected >= 0 && *value.Expected <= mattermost.MaxSafeSequence && value.Received != nil && *value.Received >= 0 && *value.Received <= mattermost.MaxSafeSequence && value.Attempt == nil && value.DelayMS == nil && value.PreviousID == nil && value.CurrentID == nil + return value.Code == "" && value.Recovery == "" && !value.Fatal && value.Expected != nil && *value.Expected >= 0 && *value.Expected <= mattermost.MaxSafeSequence && value.Received != nil && *value.Received >= 0 && *value.Received <= mattermost.MaxSafeSequence && value.Attempt == nil && value.DelayMS == nil && value.PreviousID == nil && value.CurrentID == nil case "connection_changed": - return !value.Fatal && value.Expected != nil && *value.Expected >= 0 && *value.Expected <= mattermost.MaxSafeSequence && value.Received != nil && *value.Received >= 0 && *value.Received <= mattermost.MaxSafeSequence && value.PreviousID != nil && *value.PreviousID != "" && value.CurrentID != nil && *value.CurrentID != "" && value.Attempt == nil && value.DelayMS == nil + return value.Code == "" && value.Recovery == "" && !value.Fatal && value.Expected != nil && *value.Expected >= 0 && *value.Expected <= mattermost.MaxSafeSequence && value.Received != nil && *value.Received >= 0 && *value.Received <= mattermost.MaxSafeSequence && value.PreviousID != nil && *value.PreviousID != "" && value.CurrentID != nil && *value.CurrentID != "" && value.Attempt == nil && value.DelayMS == nil case "malformed", "disconnected": - return !value.Fatal && value.Attempt == nil && value.DelayMS == nil && value.Expected == nil && value.Received == nil && value.PreviousID == nil && value.CurrentID == nil + return value.Code == "" && value.Recovery == "" && !value.Fatal && value.Attempt == nil && value.DelayMS == nil && value.Expected == nil && value.Received == nil && value.PreviousID == nil && value.CurrentID == nil case "terminal": - return value.Fatal && value.Attempt == nil && value.DelayMS == nil && value.Expected == nil && value.Received == nil && value.PreviousID == nil && value.CurrentID == nil + validCode := value.Code == "authentication" || value.Code == "reconnect_exhausted" || value.Code == "canceled" || value.Code == "invalid_options" || value.Code == "watch_failed" + validRecovery := (value.Code == "authentication" && value.Recovery == "check_token") || (value.Code == "reconnect_exhausted" && value.Recovery == "retry_later") || ((value.Code == "canceled" || value.Code == "invalid_options" || value.Code == "watch_failed") && value.Recovery == "none") + return value.Fatal && validCode && validRecovery && value.Attempt == nil && value.DelayMS == nil && value.Expected == nil && value.Received == nil && value.PreviousID == nil && value.CurrentID == nil default: return false } @@ -157,7 +164,7 @@ func NewWatchDiagnostic(value mattermost.WatchDiagnostic) (WatchDocument, error) return result.Text, result.Redactions } message, redactions := clean(value.Message, "watch.diagnostic.message") - document := watchDiagnostic{Schema: "mm/v2/watch-diagnostic", Type: value.Type, Timestamp: MillisTime{Time: value.Timestamp.UTC()}, Message: message, Backfill: false, Fatal: value.Fatal, Redactions: redactions, Attempt: value.Attempt, Expected: value.Expected, Received: value.Received, seal: true} + document := watchDiagnostic{Schema: "mm/v2/watch-diagnostic", Type: value.Type, Code: value.Code, Recovery: value.Recovery, Timestamp: MillisTime{Time: value.Timestamp.UTC()}, Message: message, Backfill: false, Fatal: value.Fatal, Redactions: redactions, Attempt: value.Attempt, Expected: value.Expected, Received: value.Received, seal: true} if value.Delay != nil { ms := value.Delay.Milliseconds() document.DelayMS = &ms @@ -235,6 +242,18 @@ type JSONLWatchSink struct { DisableHeuristics bool } +func FormatWatchHumanLine(timestamp time.Time, sender, message string, color bool) string { + message = strings.Join(strings.Fields(strings.ReplaceAll(message, "\ufeff", " ")), " ") + if message == "" { + message = "[empty message]" + } + stamp := "[" + timestamp.Format("15:04") + "]" + if color { + return dim(stamp) + " " + userColor(sender) + ": " + message + } + return stamp + " " + sender + ": " + message +} + func (sink JSONLWatchSink) Post(post mattermost.WatchPost, sequence mattermost.Sequence) error { document, err := NewWatchEvent(post, sequence, sink.DisableHeuristics) if err != nil { diff --git a/internal/output/watch_test.go b/internal/output/watch_test.go index 126c3f2..ee32958 100644 --- a/internal/output/watch_test.go +++ b/internal/output/watch_test.go @@ -98,6 +98,42 @@ func TestWatchSequenceSafeIntegerBoundary(t *testing.T) { } } +func TestFormatWatchHumanLineMatchesJSWhitespaceAndColor(t *testing.T) { + stamp := time.Date(2026, 1, 1, 3, 4, 0, 0, time.UTC) + plain := FormatWatchHumanLine(stamp, "😀", "a\tb\u00a0c\ufeffd\n e", false) + if plain != "[03:04] 😀: a b c d e" { + t.Fatalf("plain=%q", plain) + } + colored := FormatWatchHumanLine(stamp, "😀", "x", true) + if colored != "\x1b[2m[03:04]\x1b[0m \x1b[32m😀\x1b[0m: x" { + t.Fatalf("color=%q", colored) + } +} + +func TestWatchWarningAndTerminalConstructorsAreClosed(t *testing.T) { + now := time.UnixMilli(1) + for _, value := range []mattermost.WatchDiagnostic{{Type: "warning", Code: "configuration_warning", Recovery: "none", Timestamp: now, Message: "warning"}, {Type: "terminal", Code: "authentication", Recovery: "check_token", Timestamp: now, Message: "failed", Fatal: true}} { + if _, err := NewWatchDiagnostic(value); err != nil { + t.Fatal(err) + } + } + for _, value := range []mattermost.WatchDiagnostic{{Type: "warning", Code: "authentication", Recovery: "none", Timestamp: now, Message: "bad"}, {Type: "terminal", Code: "authentication", Recovery: "none", Timestamp: now, Message: "bad", Fatal: true}, {Type: "terminal", Code: "redaction_disabled", Recovery: "none", Timestamp: now, Message: "bad", Fatal: true}} { + if _, err := NewWatchDiagnostic(value); !errors.Is(err, ErrInvalidWatchDocument) { + t.Fatalf("accepted %#v", value) + } + } + attempt := 1 + delay := time.Second + for _, value := range []mattermost.WatchDiagnostic{ + {Type: "reconnect", Code: "watch_failed", Timestamp: now, Message: "retry", Attempt: &attempt, Delay: &delay}, + {Type: "reconnect", Recovery: "none", Timestamp: now, Message: "retry", Attempt: &attempt, Delay: &delay}, + } { + if _, err := NewWatchDiagnostic(value); !errors.Is(err, ErrInvalidWatchDocument) { + t.Fatalf("accepted ordinary diagnostic metadata %#v", value) + } + } +} + func TestWatchLabelRedactionPositionsMatchEmittedText(t *testing.T) { credential := "credential-position-secret" release := presentation.ActiveCredentials.Register(credential) diff --git a/internal/schema/watch_test.go b/internal/schema/watch_test.go index 365a27c..744dbb6 100644 --- a/internal/schema/watch_test.go +++ b/internal/schema/watch_test.go @@ -27,9 +27,29 @@ func TestWatchSchemasAreStrictAndDiscriminatorBound(t *testing.T) { if err := registry.Validate("mm/v2/watch-diagnostic", strings.NewReader(validDiagnostic)); err != nil { t.Fatal(err) } - for _, mutation := range []string{strings.Replace(validDiagnostic, `"backfill":false`, `"backfill":true`, 1), strings.Replace(validDiagnostic, `,"attempt":1,"delayMs":1000`, `,"expected":1,"received":2`, 1)} { + for _, mutation := range []string{strings.Replace(validDiagnostic, `"backfill":false`, `"backfill":true`, 1), strings.Replace(validDiagnostic, `,"attempt":1,"delayMs":1000`, `,"expected":1,"received":2`, 1), strings.Replace(validDiagnostic, `"timestamp":`, `"code":"watch_failed","timestamp":`, 1), strings.Replace(validDiagnostic, `"timestamp":`, `"recovery":"none","timestamp":`, 1)} { if err := registry.Validate("mm/v2/watch-diagnostic", strings.NewReader(mutation)); err == nil { t.Fatalf("accepted mutation %s", mutation) } } } + +func TestWatchWarningAndTerminalDiagnosticVariantsAreExact(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + warning := `{"schema":"mm/v2/watch-diagnostic","type":"warning","code":"redaction_disabled","recovery":"none","timestamp":"2026-07-16T00:00:00.000Z","message":"warning","backfill":false,"fatal":false,"redactions":[]}` + terminal := `{"schema":"mm/v2/watch-diagnostic","type":"terminal","code":"authentication","recovery":"check_token","timestamp":"2026-07-16T00:00:00.000Z","message":"failed","backfill":false,"fatal":true,"redactions":[]}` + for _, value := range []string{warning, terminal} { + if err := registry.Validate("mm/v2/watch-diagnostic", strings.NewReader(value)); err != nil { + t.Fatal(err) + } + } + mutations := []string{strings.Replace(warning, `"recovery":"none"`, `"recovery":"retry_later"`, 1), strings.Replace(warning, `"fatal":false`, `"fatal":true`, 1), strings.Replace(terminal, `"recovery":"check_token"`, `"recovery":"none"`, 1), strings.Replace(terminal, `"code":"authentication"`, `"code":"redaction_disabled"`, 1), strings.Replace(terminal, `"redactions":[]`, `"redactions":[],"attempt":1`, 1)} + for _, value := range mutations { + if err := registry.Validate("mm/v2/watch-diagnostic", strings.NewReader(value)); err == nil { + t.Fatalf("accepted %s", value) + } + } +} diff --git a/schemas/v2/watch-diagnostic.schema.json b/schemas/v2/watch-diagnostic.schema.json index 2f04c07..596baef 100644 --- a/schemas/v2/watch-diagnostic.schema.json +++ b/schemas/v2/watch-diagnostic.schema.json @@ -1,12 +1,16 @@ { "$schema":"https://json-schema.org/draft/2020-12/schema","$id":"urn:mm:schema:v2:watch-diagnostic","title":"mm v2 watch diagnostic","type":"object","additionalProperties":false, "required":["schema","type","timestamp","message","backfill","fatal","redactions"], - "properties":{"schema":{"const":"mm/v2/watch-diagnostic"},"type":{"enum":["reconnect","disconnected","sequence_gap","connection_changed","malformed","terminal"]},"timestamp":{"type":"string","format":"date-time","pattern":"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]{3}Z$"},"message":{"type":"string","minLength":1},"backfill":{"const":false},"fatal":{"type":"boolean"},"redactions":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["type","masked","position","field"],"properties":{"type":{"type":"string","minLength":1},"masked":{"type":"string"},"position":{"type":"integer","minimum":0},"field":{"type":"string","minLength":1}}}},"attempt":{"type":"integer","minimum":1},"delayMs":{"type":"integer","minimum":0},"expected":{"type":"integer","minimum":0,"maximum":9007199254740991},"received":{"type":"integer","minimum":0,"maximum":9007199254740991},"previousConnectionId":{"type":"string","minLength":1},"currentConnectionId":{"type":"string","minLength":1}}, + "properties":{"schema":{"const":"mm/v2/watch-diagnostic"},"type":{"enum":["reconnect","disconnected","sequence_gap","connection_changed","malformed","warning","terminal"]},"code":{"enum":["configuration_warning","redaction_disabled","authentication","reconnect_exhausted","canceled","invalid_options","watch_failed"]},"recovery":{"enum":["none","check_token","retry_later"]},"timestamp":{"type":"string","format":"date-time","pattern":"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]{3}Z$"},"message":{"type":"string","minLength":1},"backfill":{"const":false},"fatal":{"type":"boolean"},"redactions":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["type","masked","position","field"],"properties":{"type":{"type":"string","minLength":1},"masked":{"type":"string"},"position":{"type":"integer","minimum":0},"field":{"type":"string","minLength":1}}}},"attempt":{"type":"integer","minimum":1},"delayMs":{"type":"integer","minimum":0},"expected":{"type":"integer","minimum":0,"maximum":9007199254740991},"received":{"type":"integer","minimum":0,"maximum":9007199254740991},"previousConnectionId":{"type":"string","minLength":1},"currentConnectionId":{"type":"string","minLength":1}}, "allOf":[ - {"if":{"properties":{"type":{"const":"reconnect"}}},"then":{"required":["attempt","delayMs"],"properties":{"fatal":{"const":false}},"not":{"anyOf":[{"required":["expected"]},{"required":["received"]},{"required":["previousConnectionId"]},{"required":["currentConnectionId"]}]}}}, - {"if":{"properties":{"type":{"const":"sequence_gap"}}},"then":{"required":["expected","received"],"properties":{"fatal":{"const":false}},"not":{"anyOf":[{"required":["attempt"]},{"required":["delayMs"]},{"required":["previousConnectionId"]},{"required":["currentConnectionId"]}]}}}, - {"if":{"properties":{"type":{"const":"connection_changed"}}},"then":{"required":["expected","received","previousConnectionId","currentConnectionId"],"properties":{"fatal":{"const":false}},"not":{"anyOf":[{"required":["attempt"]},{"required":["delayMs"]}]}}}, - {"if":{"properties":{"type":{"enum":["disconnected","malformed"]}}},"then":{"properties":{"fatal":{"const":false}},"not":{"anyOf":[{"required":["attempt"]},{"required":["delayMs"]},{"required":["expected"]},{"required":["received"]},{"required":["previousConnectionId"]},{"required":["currentConnectionId"]}]}}}, - {"if":{"properties":{"type":{"const":"terminal"}}},"then":{"properties":{"fatal":{"const":true}},"not":{"anyOf":[{"required":["attempt"]},{"required":["delayMs"]},{"required":["expected"]},{"required":["received"]},{"required":["previousConnectionId"]},{"required":["currentConnectionId"]}]}}} + {"if":{"properties":{"type":{"const":"reconnect"}}},"then":{"required":["attempt","delayMs"],"properties":{"fatal":{"const":false}},"not":{"anyOf":[{"required":["code"]},{"required":["recovery"]},{"required":["expected"]},{"required":["received"]},{"required":["previousConnectionId"]},{"required":["currentConnectionId"]}]}}}, + {"if":{"properties":{"type":{"const":"sequence_gap"}}},"then":{"required":["expected","received"],"properties":{"fatal":{"const":false}},"not":{"anyOf":[{"required":["code"]},{"required":["recovery"]},{"required":["attempt"]},{"required":["delayMs"]},{"required":["previousConnectionId"]},{"required":["currentConnectionId"]}]}}}, + {"if":{"properties":{"type":{"const":"connection_changed"}}},"then":{"required":["expected","received","previousConnectionId","currentConnectionId"],"properties":{"fatal":{"const":false}},"not":{"anyOf":[{"required":["code"]},{"required":["recovery"]},{"required":["attempt"]},{"required":["delayMs"]}]}}}, + {"if":{"properties":{"type":{"enum":["disconnected","malformed"]}}},"then":{"properties":{"fatal":{"const":false}},"not":{"anyOf":[{"required":["code"]},{"required":["recovery"]},{"required":["attempt"]},{"required":["delayMs"]},{"required":["expected"]},{"required":["received"]},{"required":["previousConnectionId"]},{"required":["currentConnectionId"]}]}}}, + {"if":{"properties":{"type":{"const":"warning"}}},"then":{"required":["code","recovery"],"properties":{"code":{"enum":["configuration_warning","redaction_disabled"]},"recovery":{"const":"none"},"fatal":{"const":false}},"not":{"anyOf":[{"required":["attempt"]},{"required":["delayMs"]},{"required":["expected"]},{"required":["received"]},{"required":["previousConnectionId"]},{"required":["currentConnectionId"]}]}}}, + {"if":{"properties":{"type":{"const":"terminal"}}},"then":{"required":["code","recovery"],"properties":{"fatal":{"const":true},"code":{"enum":["authentication","reconnect_exhausted","canceled","invalid_options","watch_failed"]}},"not":{"anyOf":[{"required":["attempt"]},{"required":["delayMs"]},{"required":["expected"]},{"required":["received"]},{"required":["previousConnectionId"]},{"required":["currentConnectionId"]}]}}}, + {"if":{"properties":{"type":{"const":"terminal"},"code":{"const":"authentication"}}},"then":{"properties":{"recovery":{"const":"check_token"}}}}, + {"if":{"properties":{"type":{"const":"terminal"},"code":{"const":"reconnect_exhausted"}}},"then":{"properties":{"recovery":{"const":"retry_later"}}}}, + {"if":{"properties":{"type":{"const":"terminal"},"code":{"enum":["canceled","invalid_options","watch_failed"]}}},"then":{"properties":{"recovery":{"const":"none"}}}} ] } From 160c72b5f76b70d03def8d391be5b81ab03549de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 23:28:00 +0300 Subject: [PATCH 049/119] feat: add secure stage store foundation --- go.mod | 10 + go.sum | 49 ++ internal/stagestore/doctor.go | 108 ++++ internal/stagestore/fs_darwin.go | 23 + internal/stagestore/fs_linux.go | 13 + internal/stagestore/fs_types.go | 24 + internal/stagestore/paths.go | 35 ++ internal/stagestore/schema.go | 72 +++ internal/stagestore/secure_other.go | 24 + internal/stagestore/secure_unix.go | 563 +++++++++++++++++ internal/stagestore/store.go | 363 +++++++++++ internal/stagestore/store_test.go | 926 ++++++++++++++++++++++++++++ 12 files changed, 2210 insertions(+) create mode 100644 internal/stagestore/doctor.go create mode 100644 internal/stagestore/fs_darwin.go create mode 100644 internal/stagestore/fs_linux.go create mode 100644 internal/stagestore/fs_types.go create mode 100644 internal/stagestore/paths.go create mode 100644 internal/stagestore/schema.go create mode 100644 internal/stagestore/secure_other.go create mode 100644 internal/stagestore/secure_unix.go create mode 100644 internal/stagestore/store.go create mode 100644 internal/stagestore/store_test.go diff --git a/go.mod b/go.mod index 6c082b5..f3a561a 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,8 @@ require github.com/pelletier/go-toml/v2 v2.4.3 require golang.org/x/net v0.57.0 +require modernc.org/sqlite v1.53.0 + require ( github.com/coder/websocket v1.8.15 golang.org/x/sys v0.47.0 @@ -17,6 +19,14 @@ require ( ) require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/spf13/pflag v1.0.9 // indirect + modernc.org/libc v1.73.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect ) diff --git a/go.sum b/go.sum index d70ba59..61efc7c 100644 --- a/go.sum +++ b/go.sum @@ -3,10 +3,24 @@ github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6p github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= @@ -15,10 +29,45 @@ github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiT github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c= +modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws= +modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc= +modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA= +modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M= +modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/stagestore/doctor.go b/internal/stagestore/doctor.go new file mode 100644 index 0000000..4cfaf96 --- /dev/null +++ b/internal/stagestore/doctor.go @@ -0,0 +1,108 @@ +package stagestore + +import ( + "context" + "errors" + "os" + "strconv" +) + +const doctorRowLimit = 20 + +type MigrationStatus struct { + Applied, Latest int + Valid bool +} + +type DoctorReport struct { + Exists, FilesystemSafe bool + ApplicationID int + Integrity []string + ForeignKeyIssues int + ForeignKeyRows []string + Migrations MigrationStatus + JournalMode string + Synchronous, SecureDelete int + ForeignKeys, TrustedSchema, QueryOnly, WALFallback bool + IntegrityTruncated, ForeignKeyTruncated bool + PermissionModelLimitations []string +} + +// Doctor inspects an existing store without creating, migrating, or changing it. +func Doctor(ctx context.Context, path string) (DoctorReport, error) { + report := DoctorReport{PermissionModelLimitations: permissionModelLimitations()} + if !platformSupported() { + return report, errors.New("stage store: unsupported platform") + } + if err := validateDatabasePath(path); err != nil { + return report, err + } + if _, err := os.Lstat(path); errors.Is(err, os.ErrNotExist) { + return report, nil + } else if err != nil { + return report, errors.New("stage store: database unavailable") + } + report.Exists = true + if err := validateExisting(path); err != nil { + return report, err + } + report.FilesystemSafe = true + s, err := OpenReadOnly(ctx, path) + if err != nil { + return report, err + } + defer s.Close() + if err := s.db.QueryRowContext(ctx, "PRAGMA application_id").Scan(&report.ApplicationID); err != nil { + return report, localError(err) + } + report.JournalMode, report.WALFallback = s.journalMode, s.journalFallback + var foreignKeys, trustedSchema, queryOnly int + for query, target := range map[string]any{"PRAGMA synchronous": &report.Synchronous, "PRAGMA secure_delete": &report.SecureDelete, "PRAGMA foreign_keys": &foreignKeys, "PRAGMA trusted_schema": &trustedSchema, "PRAGMA query_only": &queryOnly} { + if err := s.db.QueryRowContext(ctx, query).Scan(target); err != nil { + return report, localError(err) + } + } + report.ForeignKeys, report.TrustedSchema, report.QueryOnly = foreignKeys == 1, trustedSchema == 1, queryOnly == 1 + rows, err := s.db.QueryContext(ctx, "PRAGMA integrity_check("+strconv.Itoa(doctorRowLimit+1)+")") + if err != nil { + return report, localError(err) + } + for rows.Next() { + var value string + if err := rows.Scan(&value); err != nil { + rows.Close() + return report, localError(err) + } + if len(report.Integrity) == doctorRowLimit { + report.IntegrityTruncated = true + continue + } + report.Integrity = append(report.Integrity, value) + } + if err := rows.Close(); err != nil { + return report, localError(err) + } + rows, err = s.db.QueryContext(ctx, "SELECT `table`, rowid, parent, fkid FROM pragma_foreign_key_check LIMIT "+strconv.Itoa(doctorRowLimit+1)) + if err != nil { + return report, localError(err) + } + for rows.Next() { + var table string + var rowid, parent, fkid any + if err := rows.Scan(&table, &rowid, &parent, &fkid); err != nil { + rows.Close() + return report, localError(err) + } + report.ForeignKeyIssues++ + if len(report.ForeignKeyRows) == doctorRowLimit { + report.ForeignKeyTruncated = true + continue + } + report.ForeignKeyRows = append(report.ForeignKeyRows, table) + } + if err := rows.Close(); err != nil { + return report, localError(err) + } + report.Migrations = MigrationStatus{Applied: len(migrations), Latest: migrations[len(migrations)-1].version, Valid: true} + return report, nil +} diff --git a/internal/stagestore/fs_darwin.go b/internal/stagestore/fs_darwin.go new file mode 100644 index 0000000..ef6f4ea --- /dev/null +++ b/internal/stagestore/fs_darwin.go @@ -0,0 +1,23 @@ +//go:build darwin + +package stagestore + +import ( + "strings" + + "golang.org/x/sys/unix" +) + +func localFilesystemAllowed(fd int) bool { + var stat unix.Statfs_t + if unix.Fstatfs(fd, &stat) != nil || stat.Flags&unix.MNT_LOCAL == 0 { + return false + } + name := strings.TrimRight(string(stat.Fstypename[:]), "\x00") + switch name { + case "nfs", "smbfs", "webdav", "osxfuse", "macfuse", "fusefs", "afpfs", "autofs": + return false + default: + return true + } +} diff --git a/internal/stagestore/fs_linux.go b/internal/stagestore/fs_linux.go new file mode 100644 index 0000000..2ab5c2f --- /dev/null +++ b/internal/stagestore/fs_linux.go @@ -0,0 +1,13 @@ +//go:build linux + +package stagestore + +import "golang.org/x/sys/unix" + +func localFilesystemAllowed(fd int) bool { + var stat unix.Statfs_t + if unix.Fstatfs(fd, &stat) != nil { + return false + } + return linuxFilesystemTypeAllowed(uint64(stat.Type)) +} diff --git a/internal/stagestore/fs_types.go b/internal/stagestore/fs_types.go new file mode 100644 index 0000000..2ca9616 --- /dev/null +++ b/internal/stagestore/fs_types.go @@ -0,0 +1,24 @@ +package stagestore + +func linuxFilesystemTypeAllowed(kind uint64) bool { + // Explicit local filesystems only. Unknown types fail closed. + switch kind { + case 0xEF53, // ext2/3/4 + 0x58465342, // XFS + 0x9123683E, // btrfs + 0x794C7630, // overlayfs + 0x01021994, // tmpfs + 0x858458F6, // ramfs + 0x2FC12FC1, // ZFS + 0xF2F52010, // f2fs + 0xCA451A4E, // bcachefs + 0x00003434, // NILFS2 + 0x24051905, // UBIFS + 0x3153464A, // JFS + 0x52654973, // ReiserFS + 0x00011954: // UFS + return true + default: + return false + } +} diff --git a/internal/stagestore/paths.go b/internal/stagestore/paths.go new file mode 100644 index 0000000..670f30d --- /dev/null +++ b/internal/stagestore/paths.go @@ -0,0 +1,35 @@ +package stagestore + +import ( + "errors" + "fmt" + "path/filepath" +) + +const DatabaseFilename = "stages.sqlite3" + +func validateDatabasePath(path string) error { + if !filepath.IsAbs(path) || filepath.Clean(path) != path || filepath.Base(path) != DatabaseFilename { + return errors.New("stage store: database path must name the canonical database") + } + return nil +} + +type LookupEnv func(string) (string, bool) + +type Paths struct { + StateDir string + DBPath string +} + +func ResolvePaths(home string, lookup LookupEnv) (Paths, error) { + if !filepath.IsAbs(home) { + return Paths{}, fmt.Errorf("stage store: home directory must be absolute") + } + root := filepath.Join(home, ".local", "state") + if value, ok := lookup("XDG_STATE_HOME"); ok && filepath.IsAbs(value) { + root = value + } + dir := filepath.Join(root, "mattermost-cli") + return Paths{StateDir: dir, DBPath: filepath.Join(dir, DatabaseFilename)}, nil +} diff --git a/internal/stagestore/schema.go b/internal/stagestore/schema.go new file mode 100644 index 0000000..cc8d088 --- /dev/null +++ b/internal/stagestore/schema.go @@ -0,0 +1,72 @@ +package stagestore + +import ( + "crypto/sha256" + "encoding/hex" +) + +type migration struct { + version int + name string + sql string +} + +func (m migration) checksum() string { + sum := sha256.Sum256([]byte(m.name + "\x00" + m.sql)) + return hex.EncodeToString(sum[:]) +} + +var migrations = []migration{{version: 1, name: "core-stage-state", sql: ` +CREATE TABLE stages ( + id TEXT PRIMARY KEY NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + operation TEXT NOT NULL CHECK (operation IN ('create_post','reply','edit_post','delete_post','react','unreact','resolve_dm','resolve_group_dm')), + server_url TEXT NOT NULL, + server_id TEXT, + user_id TEXT NOT NULL, + lifecycle TEXT NOT NULL CHECK (lifecycle IN ('open','applying','completed','canceled','expired','pruned')), + recovery TEXT NOT NULL CHECK (recovery IN ('none','resume_partial','force_unknown','forbidden')), + current_revision INTEGER NOT NULL CHECK (current_revision > 0), + current_revision_state TEXT NOT NULL DEFAULT 'current' CHECK (current_revision_state = 'current'), + FOREIGN KEY (id, current_revision, current_revision_state) REFERENCES stage_revisions(stage_id, revision, state) DEFERRABLE INITIALLY DEFERRED +) STRICT; +CREATE TABLE stage_revisions ( + stage_id TEXT NOT NULL REFERENCES stages(id) ON DELETE CASCADE, + revision INTEGER NOT NULL CHECK (revision > 0), + state TEXT NOT NULL CHECK (state IN ('current','superseded')), + created_at TEXT NOT NULL, + semantic_digest BLOB NOT NULL CHECK (length(semantic_digest) = 32), + body BLOB, + destination_json TEXT NOT NULL CHECK (json_valid(destination_json)), + plan_json TEXT NOT NULL CHECK (json_valid(plan_json)), + PRIMARY KEY (stage_id, revision), + UNIQUE (stage_id, revision, state) +) STRICT; +CREATE UNIQUE INDEX one_current_stage_revision ON stage_revisions(stage_id) WHERE state = 'current'; +CREATE TABLE stage_attachments ( + stage_id TEXT NOT NULL, + revision INTEGER NOT NULL, + ordinal INTEGER NOT NULL CHECK (ordinal >= 0), + supplied_path TEXT NOT NULL, + canonical_path TEXT NOT NULL, + remote_filename TEXT NOT NULL, + byte_length INTEGER NOT NULL CHECK (byte_length >= 0), + media_type TEXT, + content_digest BLOB NOT NULL CHECK (length(content_digest) = 32), + PRIMARY KEY (stage_id, revision, ordinal), + FOREIGN KEY (stage_id, revision) REFERENCES stage_revisions(stage_id, revision) ON DELETE CASCADE +) STRICT; +CREATE TABLE request_replays ( + server_url TEXT NOT NULL, + user_id TEXT NOT NULL, + request_id TEXT NOT NULL, + request_schema TEXT NOT NULL, + semantic_digest BLOB NOT NULL CHECK (length(semantic_digest) = 32), + stage_id TEXT NOT NULL REFERENCES stages(id) ON DELETE CASCADE, + revision INTEGER NOT NULL CHECK (revision > 0), + created_at TEXT NOT NULL, + PRIMARY KEY (server_url, user_id, request_id), + FOREIGN KEY (stage_id, revision) REFERENCES stage_revisions(stage_id, revision) +) STRICT; +`}} diff --git a/internal/stagestore/secure_other.go b/internal/stagestore/secure_other.go new file mode 100644 index 0000000..64546d4 --- /dev/null +++ b/internal/stagestore/secure_other.go @@ -0,0 +1,24 @@ +//go:build !darwin && !linux + +package stagestore + +import ( + "context" + "fmt" +) + +func prepareWritable(context.Context, string) (bool, bool, func(), error) { + return false, false, nil, fmt.Errorf("stage store: unsupported platform") +} +func lockExisting(context.Context, string) (func(), error) { + return nil, fmt.Errorf("stage store: unsupported platform") +} +func validateExisting(string) error { return fmt.Errorf("stage store: unsupported platform") } +func validateSidecars(string) error { return fmt.Errorf("stage store: unsupported platform") } +func validateImmutableRead(string) error { return fmt.Errorf("stage store: unsupported platform") } +func removeFailedNewDatabase(string) {} +func clearBootstrapPending(string) error { return fmt.Errorf("stage store: unsupported platform") } +func platformSupported() bool { return false } +func permissionModelLimitations() []string { + return []string{"secure stage storage is unsupported on this platform"} +} diff --git a/internal/stagestore/secure_unix.go b/internal/stagestore/secure_unix.go new file mode 100644 index 0000000..980ed53 --- /dev/null +++ b/internal/stagestore/secure_unix.go @@ -0,0 +1,563 @@ +//go:build darwin || linux + +package stagestore + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "time" + + "golang.org/x/sys/unix" +) + +const ( + bootstrapLockFilename = ".stages.bootstrap.lock" + bootstrapPendingFilename = ".stages.bootstrap.pending" + bootstrapPendingContent = "mattermost-cli-v2-bootstrap\n" +) + +var filesystemAllowed = localFilesystemAllowed +var stateDirectoryFchmodat = unix.Fchmodat +var pendingWrite = unix.Write + +func prepareWritable(ctx context.Context, path string) (bool, bool, func(), error) { + dirfd, name, err := walkParent(path, true) + if err != nil { + return false, false, nil, err + } + defer unix.Close(dirfd) + if !filesystemAllowed(dirfd) { + return false, false, nil, ErrUnsafeFilesystem + } + lockCreated := true + lockfd, err := unix.Openat(dirfd, bootstrapLockFilename, unix.O_RDWR|unix.O_CREAT|unix.O_EXCL|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0o600) + if errors.Is(err, unix.EEXIST) { + lockCreated = false + if recoverErr := recoverPrivateRegularAt(dirfd, bootstrapLockFilename, 0o600); recoverErr != nil { + return false, false, nil, recoverErr + } + lockfd, err = unix.Openat(dirfd, bootstrapLockFilename, unix.O_RDWR|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0) + } + if err != nil { + return false, false, nil, errors.New("stage store: bootstrap lock unavailable") + } + if lockCreated { + if err := unix.Fchmod(lockfd, 0o600); err != nil { + unix.Close(lockfd) + return false, false, nil, errors.New("stage store: cannot secure bootstrap lock") + } + } + if err := validateFD(lockfd, 0o600); err != nil { + unix.Close(lockfd) + return false, false, nil, err + } + if err := flockContext(ctx, lockfd); err != nil { + unix.Close(lockfd) + return false, false, nil, err + } + databaseExists, err := entryExistsAt(dirfd, name) + if err != nil { + unix.Close(lockfd) + return false, false, nil, err + } + pendingExists, err := entryExistsAt(dirfd, bootstrapPendingFilename) + if err != nil { + unix.Close(lockfd) + return false, false, nil, err + } + if pendingExists { + if err := validatePendingAt(dirfd); err != nil { + if databaseExists { + unix.Close(lockfd) + return false, false, nil, err + } + if unlinkErr := unix.Unlinkat(dirfd, bootstrapPendingFilename, 0); unlinkErr != nil { + unix.Close(lockfd) + return false, false, nil, errors.New("stage store: cannot replace bootstrap marker") + } + if syncErr := unix.Fsync(dirfd); syncErr != nil { + unix.Close(lockfd) + return false, false, nil, errors.New("stage store: cannot sync state directory") + } + pendingExists = false + } + } + if !databaseExists && !pendingExists { + if err := createPendingAt(dirfd); err != nil { + unix.Close(lockfd) + return false, false, nil, err + } + pendingExists = true + } + createdNow := false + if !databaseExists { + dbfd, err := unix.Openat(dirfd, name, unix.O_RDWR|unix.O_CREAT|unix.O_EXCL|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0o600) + if err != nil { + unix.Close(lockfd) + return false, false, nil, errors.New("stage store: cannot create database") + } + createdNow = true + if err := unix.Fchmod(dbfd, 0o600); err != nil { + unix.Close(dbfd) + unix.Close(lockfd) + return false, false, nil, errors.New("stage store: cannot secure database") + } + unix.Close(dbfd) + } else if pendingExists { + if err := recoverPrivateRegularAt(dirfd, name, 0o600); err != nil { + unix.Close(lockfd) + return false, false, nil, err + } + } + if err := validateAt(dirfd, name, 0o600); err != nil { + unix.Close(lockfd) + return false, false, nil, err + } + if err := validateSidecarsAt(dirfd, name); err != nil { + unix.Close(lockfd) + return false, false, nil, err + } + return pendingExists, createdNow, func() { _ = unix.Flock(lockfd, unix.LOCK_UN); _ = unix.Close(lockfd) }, nil +} + +func entryExistsAt(dirfd int, name string) (bool, error) { + var stat unix.Stat_t + err := unix.Fstatat(dirfd, name, &stat, unix.AT_SYMLINK_NOFOLLOW) + if errors.Is(err, unix.ENOENT) { + return false, nil + } + if err != nil { + return false, errors.New("stage store: cannot inspect private state") + } + return true, nil +} + +func createPendingAt(dirfd int) error { + fd, err := unix.Openat(dirfd, bootstrapPendingFilename, unix.O_WRONLY|unix.O_CREAT|unix.O_EXCL|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0o600) + if err != nil { + return errors.New("stage store: cannot create bootstrap marker") + } + succeeded := false + defer func() { + _ = unix.Close(fd) + if !succeeded { + _ = unix.Unlinkat(dirfd, bootstrapPendingFilename, 0) + _ = unix.Fsync(dirfd) + } + }() + if err := unix.Fchmod(fd, 0o600); err != nil { + return errors.New("stage store: cannot secure bootstrap marker") + } + remaining := []byte(bootstrapPendingContent) + for len(remaining) > 0 { + n, err := pendingWrite(fd, remaining) + if err != nil || n <= 0 || n > len(remaining) { + return errors.New("stage store: cannot write bootstrap marker") + } + remaining = remaining[n:] + } + if err := unix.Fsync(fd); err != nil { + return errors.New("stage store: cannot sync bootstrap marker") + } + if err := unix.Fsync(dirfd); err != nil { + return errors.New("stage store: cannot sync state directory") + } + succeeded = true + return nil +} + +func validatePendingAt(dirfd int) error { + fd, err := unix.Openat(dirfd, bootstrapPendingFilename, unix.O_RDONLY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0) + if err != nil { + return errors.New("stage store: bootstrap marker unavailable") + } + defer unix.Close(fd) + if err := validateFD(fd, 0o600); err != nil { + return err + } + buf := make([]byte, len(bootstrapPendingContent)+1) + n, err := unix.Read(fd, buf) + if err != nil || string(buf[:n]) != bootstrapPendingContent { + return errors.New("stage store: invalid bootstrap marker") + } + return nil +} + +func walkParent(path string, create bool) (int, string, error) { + if !filepath.IsAbs(path) { + return -1, "", errors.New("stage store: database path must be absolute") + } + parts := strings.Split(strings.TrimPrefix(filepath.Clean(path), string(filepath.Separator)), string(filepath.Separator)) + if len(parts) < 2 || parts[len(parts)-1] == "" { + return -1, "", errors.New("stage store: unsafe state path") + } + fd, err := unix.Open(string(filepath.Separator), unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) + if err != nil { + return -1, "", errors.New("stage store: cannot inspect state path") + } + boundary := false + for i, part := range parts[:len(parts)-1] { + var entry unix.Stat_t + err = unix.Fstatat(fd, part, &entry, unix.AT_SYMLINK_NOFOLLOW) + created := false + if errors.Is(err, unix.ENOENT) && create { + err = unix.Mkdirat(fd, part, 0o700) + if errors.Is(err, unix.EACCES) { + if recoverErr := recoverPrivateDirectoryFD(fd); recoverErr == nil { + err = unix.Mkdirat(fd, part, 0o700) + } + } + if err != nil && !errors.Is(err, unix.EEXIST) { + unix.Close(fd) + return -1, "", errors.New("stage store: cannot create state directory") + } + err = unix.Fstatat(fd, part, &entry, unix.AT_SYMLINK_NOFOLLOW) + created = true + } + if err != nil { + unix.Close(fd) + return -1, "", errors.New("stage store: state directory unavailable") + } + if created { + entry, err = secureCreatedDirectory(fd, part, entry, boundary) + if err != nil { + unix.Close(fd) + return -1, "", err + } + } else if create && i == len(parts)-2 && entry.Mode&unix.S_IFMT == unix.S_IFDIR && entry.Uid == uint32(os.Geteuid()) { + entry, err = recoverPrivateDirectoryAt(fd, part, entry) + if err != nil { + unix.Close(fd) + return -1, "", err + } + } + isLink := entry.Mode&unix.S_IFMT == unix.S_IFLNK + if isLink && !symlinkAllowedBeforeBoundary(entry.Uid, boundary) { + unix.Close(fd) + return -1, "", errors.New("stage store: unsafe state path") + } + flags := unix.O_RDONLY | unix.O_DIRECTORY | unix.O_CLOEXEC + if !isLink { + flags |= unix.O_NOFOLLOW + } + next, openErr := unix.Openat(fd, part, flags, 0) + if create && errors.Is(openErr, unix.EACCES) && !isLink && entry.Mode&unix.S_IFMT == unix.S_IFDIR && entry.Uid == uint32(os.Geteuid()) { + if recovered, recoverErr := recoverPrivateDirectoryAt(fd, part, entry); recoverErr == nil { + entry = recovered + next, openErr = unix.Openat(fd, part, flags, 0) + } + } + unix.Close(fd) + if openErr != nil { + return -1, "", errors.New("stage store: unsafe state path") + } + fd = next + var opened unix.Stat_t + if unix.Fstat(fd, &opened) != nil || opened.Mode&unix.S_IFMT != unix.S_IFDIR || (opened.Uid != 0 && opened.Uid != uint32(os.Geteuid())) { + unix.Close(fd) + return -1, "", errors.New("stage store: unsafe state path") + } + if created { + if err := unix.Fchmod(fd, 0o700); err != nil { + unix.Close(fd) + return -1, "", errors.New("stage store: cannot secure state directory") + } + if err := unix.Fstat(fd, &opened); err != nil { + unix.Close(fd) + return -1, "", errors.New("stage store: cannot inspect state directory") + } + } + if !isLink && (entry.Dev != opened.Dev || entry.Ino != opened.Ino) { + unix.Close(fd) + return -1, "", errors.New("stage store: state path changed") + } + if !ancestorPermissionsAllowed(opened.Uid, uint32(opened.Mode), boundary) { + unix.Close(fd) + return -1, "", errors.New("stage store: unsafe state path permissions") + } + if opened.Uid == uint32(os.Geteuid()) { + boundary = true + if opened.Mode&0o022 != 0 { + unix.Close(fd) + return -1, "", errors.New("stage store: unsafe state path permissions") + } + } + if created && !hasExactMode(uint32(opened.Mode), unix.S_IFDIR, 0o700) { + unix.Close(fd) + return -1, "", errors.New("stage store: unsupported permission semantics") + } + if i == len(parts)-2 && (opened.Uid != uint32(os.Geteuid()) || !hasExactMode(uint32(opened.Mode), unix.S_IFDIR, 0o700)) { + unix.Close(fd) + return -1, "", errors.New("stage store: unsafe state directory") + } + } + return fd, parts[len(parts)-1], nil +} + +func secureCreatedDirectory(parent int, name string, expected unix.Stat_t, trustedParent bool) (unix.Stat_t, error) { + euid := uint32(os.Geteuid()) + if expected.Mode&unix.S_IFMT != unix.S_IFDIR || expected.Uid != euid { + return unix.Stat_t{}, errors.New("stage store: state directory changed") + } + err := stateDirectoryFchmodat(parent, name, 0o700, unix.AT_SYMLINK_NOFOLLOW) + if errors.Is(err, unix.EOPNOTSUPP) || errors.Is(err, unix.ENOTSUP) { + if !trustedParent { + return unix.Stat_t{}, errors.New("stage store: unsupported permission semantics") + } + var before unix.Stat_t + if statErr := unix.Fstatat(parent, name, &before, unix.AT_SYMLINK_NOFOLLOW); statErr != nil || !sameDirectory(expected, before, euid) { + return unix.Stat_t{}, errors.New("stage store: state directory changed") + } + err = stateDirectoryFchmodat(parent, name, 0o700, 0) + } + if err != nil { + return unix.Stat_t{}, errors.New("stage store: cannot secure state directory") + } + var secured unix.Stat_t + if err := unix.Fstatat(parent, name, &secured, unix.AT_SYMLINK_NOFOLLOW); err != nil || !sameDirectory(expected, secured, euid) || !hasExactMode(uint32(secured.Mode), unix.S_IFDIR, 0o700) { + return unix.Stat_t{}, errors.New("stage store: state directory changed") + } + return secured, nil +} + +func recoverPrivateDirectoryAt(parent int, name string, expected unix.Stat_t) (unix.Stat_t, error) { + if hasExactMode(uint32(expected.Mode), unix.S_IFDIR, 0o700) { + return expected, nil + } + if uint32(expected.Mode)&^uint32(unix.S_IFMT|0o700) != 0 { + return unix.Stat_t{}, errors.New("stage store: unsafe state directory") + } + if err := unix.Fchmodat(parent, name, 0o700, 0); err != nil { + return unix.Stat_t{}, errors.New("stage store: cannot recover state directory") + } + var recovered unix.Stat_t + if err := unix.Fstatat(parent, name, &recovered, unix.AT_SYMLINK_NOFOLLOW); err != nil || !sameDirectory(expected, recovered, uint32(os.Geteuid())) || uint32(recovered.Mode)&0o777 != 0o700 { + return unix.Stat_t{}, errors.New("stage store: state directory changed") + } + return recovered, nil +} + +func recoverPrivateDirectoryFD(fd int) error { + var stat unix.Stat_t + if err := unix.Fstat(fd, &stat); err != nil { + return errors.New("stage store: cannot inspect state directory") + } + mode := uint32(stat.Mode) + if mode&unix.S_IFMT != unix.S_IFDIR || stat.Uid != uint32(os.Geteuid()) || mode&^uint32(unix.S_IFMT|0o700) != 0 { + return errors.New("stage store: unsafe state directory") + } + if err := unix.Fchmod(fd, 0o700); err != nil { + return errors.New("stage store: cannot recover state directory") + } + return nil +} + +func sameDirectory(expected, actual unix.Stat_t, euid uint32) bool { + return expected.Dev == actual.Dev && expected.Ino == actual.Ino && actual.Mode&unix.S_IFMT == unix.S_IFDIR && actual.Uid == euid +} + +func hasExactMode(mode, kind, permissions uint32) bool { + return mode&unix.S_IFMT == kind && mode&0o777 == permissions && mode&^uint32(unix.S_IFMT|permissions) == 0 +} + +func symlinkAllowedBeforeBoundary(uid uint32, boundary bool) bool { + return !boundary && uid == 0 +} + +func ancestorPermissionsAllowed(uid, mode uint32, boundary bool) bool { + if boundary || uid != 0 || mode&0o022 == 0 { + return true + } + return mode&unix.S_ISVTX != 0 +} + +func flockContext(ctx context.Context, fd int) error { + deadline := time.Now().Add(busyMillis * time.Millisecond) + for { + err := unix.Flock(fd, unix.LOCK_EX|unix.LOCK_NB) + if err == nil { + return nil + } + if !errors.Is(err, unix.EWOULDBLOCK) || !time.Now().Before(deadline) { + return ErrBusy + } + timer := time.NewTimer(10 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } +} + +func lockExisting(ctx context.Context, path string) (func(), error) { + dirfd, _, err := walkParent(path, false) + if err != nil { + return nil, err + } + defer unix.Close(dirfd) + lockfd, err := unix.Openat(dirfd, bootstrapLockFilename, unix.O_RDWR|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0) + if err != nil { + return nil, errors.New("stage store: bootstrap lock unavailable") + } + if err := validateFD(lockfd, 0o600); err != nil { + unix.Close(lockfd) + return nil, err + } + if err := flockContext(ctx, lockfd); err != nil { + unix.Close(lockfd) + return nil, err + } + return func() { _ = unix.Flock(lockfd, unix.LOCK_UN); _ = unix.Close(lockfd) }, nil +} + +func validateExisting(path string) error { + dirfd, name, err := walkParent(path, false) + if err != nil { + return err + } + defer unix.Close(dirfd) + if !filesystemAllowed(dirfd) { + return ErrUnsafeFilesystem + } + if err := validateAt(dirfd, name, 0o600); err != nil { + return err + } + return validateSidecarsAt(dirfd, name) +} + +func validateAt(dirfd int, name string, mode uint32) error { + fd, err := unix.Openat(dirfd, name, unix.O_RDONLY|unix.O_CLOEXEC|unix.O_NOFOLLOW|unix.O_NONBLOCK, 0) + if err != nil { + return errors.New("stage store: private file unavailable") + } + defer unix.Close(fd) + return validateFD(fd, mode) +} + +func recoverPrivateRegularAt(dirfd int, name string, desired uint32) error { + var stat unix.Stat_t + if err := unix.Fstatat(dirfd, name, &stat, unix.AT_SYMLINK_NOFOLLOW); err != nil { + return errors.New("stage store: private file unavailable") + } + mode := uint32(stat.Mode) + if mode&unix.S_IFMT != unix.S_IFREG || stat.Uid != uint32(os.Geteuid()) || stat.Nlink != 1 || mode&^uint32(unix.S_IFMT|desired) != 0 { + return errors.New("stage store: unsafe private file") + } + if mode&0o777 == desired { + return nil + } + if err := unix.Fchmodat(dirfd, name, desired, 0); err != nil { + return errors.New("stage store: cannot recover private file") + } + return validateAt(dirfd, name, desired) +} + +func validateFD(fd int, mode uint32) error { + var stat unix.Stat_t + if err := unix.Fstat(fd, &stat); err != nil { + return errors.New("stage store: cannot inspect private file") + } + if !hasExactMode(uint32(stat.Mode), unix.S_IFREG, mode) || stat.Uid != uint32(os.Geteuid()) || stat.Nlink != 1 { + return errors.New("stage store: unsafe private file") + } + return nil +} + +func validateSidecars(path string) error { + dirfd, name, err := walkParent(path, false) + if err != nil { + return err + } + defer unix.Close(dirfd) + return validateSidecarsAt(dirfd, name) +} + +func validateImmutableRead(path string) error { + dirfd, name, err := walkParent(path, false) + if err != nil { + return err + } + defer unix.Close(dirfd) + for _, suffix := range []string{"-wal", "-shm", "-journal"} { + var stat unix.Stat_t + err := unix.Fstatat(dirfd, name+suffix, &stat, unix.AT_SYMLINK_NOFOLLOW) + if err == nil { + return errors.New("stage store: active journal prevents immutable inspection") + } + if !errors.Is(err, unix.ENOENT) { + return errors.New("stage store: cannot inspect database sidecar") + } + } + return nil +} + +func validateSidecarsAt(dirfd int, name string) error { + for _, suffix := range []string{"-wal", "-shm", "-journal"} { + var stat unix.Stat_t + err := unix.Fstatat(dirfd, name+suffix, &stat, unix.AT_SYMLINK_NOFOLLOW) + if errors.Is(err, unix.ENOENT) { + continue + } + if err != nil { + return errors.New("stage store: cannot inspect database sidecar") + } + if err := validateAt(dirfd, name+suffix, 0o600); err != nil { + return errors.New("stage store: unsafe database sidecar") + } + } + return nil +} + +func removeFailedNewDatabase(path string) { + dirfd, name, err := walkParent(path, false) + if err != nil { + return + } + defer unix.Close(dirfd) + if validateAt(dirfd, name, 0o600) != nil { + return + } + for _, suffix := range []string{"-wal", "-shm", "-journal", ""} { + _ = unix.Unlinkat(dirfd, name+suffix, 0) + } +} + +func clearBootstrapPending(path string) error { + dirfd, _, err := walkParent(path, false) + if err != nil { + return err + } + defer unix.Close(dirfd) + exists, err := entryExistsAt(dirfd, bootstrapPendingFilename) + if err != nil || !exists { + return err + } + if err := validatePendingAt(dirfd); err != nil { + return err + } + if err := unix.Unlinkat(dirfd, bootstrapPendingFilename, 0); err != nil { + return errors.New("stage store: cannot clear bootstrap marker") + } + if err := unix.Fsync(dirfd); err != nil { + return errors.New("stage store: cannot sync state directory") + } + return nil +} + +// SQLite ultimately resolves the validated absolute path itself. Descriptor-relative +// checks before and after open close accidental races, but cannot defend against a +// malicious same-UID process continuously swapping entries. Same-UID hostile +// processes and extended ACLs unavailable through x/sys are outside this boundary. +func platformSupported() bool { return true } + +func permissionModelLimitations() []string { + return []string{ + "same-UID path replacement is outside the threat boundary", + "extended ACL verification is unavailable without cgo", + } +} diff --git a/internal/stagestore/store.go b/internal/stagestore/store.go new file mode 100644 index 0000000..d88bb7a --- /dev/null +++ b/internal/stagestore/store.go @@ -0,0 +1,363 @@ +package stagestore + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net/url" + "strconv" + "strings" + "sync" + "time" + + sqlite3 "modernc.org/sqlite" +) + +const ( + applicationID = 0x4d4d5632 + busyMillis = 5000 + driverBusyMS = 25 +) + +var ( + ErrBusy = errors.New("stage store: database busy") + ErrIdentity = errors.New("stage store: database identity mismatch") + ErrMigration = errors.New("stage store: migration state invalid") + ErrUnsafeFilesystem = errors.New("stage store: unsafe filesystem") + setJournalMode = configureWAL +) + +type Store struct { + db *sql.DB + path string + journalMode string + journalFallback bool + unlock func() + closeOnce sync.Once + closeErr error +} + +func Open(ctx context.Context, path string) (*Store, error) { + if err := validateDatabasePath(path); err != nil { + return nil, err + } + bootstrap, createdNow, unlock, err := prepareWritable(ctx, path) + if err != nil { + return nil, err + } + db, err := sql.Open("sqlite", sqliteURI(path, false)) + if err != nil { + unlock() + return nil, localError(err) + } + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + s := &Store{db: db, path: path, unlock: unlock} + if err := s.initializeBounded(ctx, bootstrap); err != nil { + _ = db.Close() + if createdNow { + removeFailedNewDatabase(path) + } + unlock() + return nil, err + } + if err := s.verifyIdentityAndMigrations(ctx); err != nil { + _ = db.Close() + if createdNow { + removeFailedNewDatabase(path) + } + unlock() + return nil, err + } + if err := clearBootstrapPending(path); err != nil { + _ = db.Close() + unlock() + return nil, err + } + if err := validateSidecars(path); err != nil { + _ = db.Close() + unlock() + return nil, err + } + return s, nil +} + +func (s *Store) initializeBounded(ctx context.Context, created bool) error { + deadline := time.Now().Add(busyMillis * time.Millisecond) + for { + err := s.initialize(ctx, created) + if !errors.Is(err, ErrBusy) || !time.Now().Before(deadline) { + return err + } + timer := time.NewTimer(10 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } +} + +func OpenReadOnly(ctx context.Context, path string) (*Store, error) { + if err := validateDatabasePath(path); err != nil { + return nil, err + } + unlock, err := lockExisting(ctx, path) + if err != nil { + return nil, err + } + failed := true + defer func() { + if failed { + unlock() + } + }() + if err := validateExisting(path); err != nil { + return nil, err + } + if err := validateImmutableRead(path); err != nil { + return nil, err + } + db, err := sql.Open("sqlite", sqliteURI(path, true)) + if err != nil { + return nil, localError(err) + } + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + s := &Store{db: db, path: path, unlock: unlock} + var queryOnly int + if err := db.QueryRowContext(ctx, "PRAGMA query_only").Scan(&queryOnly); err != nil || queryOnly != 1 { + _ = db.Close() + return nil, fmt.Errorf("stage store: read-only guard unavailable") + } + if err := s.verifyIdentityAndMigrations(ctx); err != nil { + _ = db.Close() + return nil, err + } + if err := db.QueryRowContext(ctx, "PRAGMA journal_mode").Scan(&s.journalMode); err != nil { + _ = db.Close() + return nil, localError(err) + } + s.journalMode = strings.ToLower(s.journalMode) + if s.journalMode != "wal" && !isLocalRollbackJournal(s.journalMode) { + _ = db.Close() + return nil, errors.New("stage store: unsupported journal mode") + } + s.journalFallback = s.journalMode != "wal" + if err := validateExisting(path); err != nil { + _ = db.Close() + return nil, err + } + if err := validateImmutableRead(path); err != nil { + _ = db.Close() + return nil, err + } + failed = false + return s, nil +} + +func (s *Store) Close() error { + s.closeOnce.Do(func() { + s.closeErr = s.db.Close() + if s.unlock != nil { + s.unlock() + } + }) + return s.closeErr +} + +func sqliteURI(path string, readOnly bool) string { + u := &url.URL{Scheme: "file", Path: path} + q := u.Query() + q.Set("_dqs", "0") + if readOnly { + q.Set("mode", "ro") + q.Set("immutable", "1") + q.Add("_pragma", "query_only(1)") + } else { + q.Set("mode", "rw") + q.Set("_txlock", "immediate") + } + q.Add("_pragma", "busy_timeout("+strconv.Itoa(driverBusyMS)+")") + q.Add("_pragma", "foreign_keys(1)") + q.Add("_pragma", "trusted_schema(0)") + q.Add("_pragma", "secure_delete(FAST)") + q.Add("_pragma", "synchronous(FULL)") + u.RawQuery = q.Encode() + return u.String() +} + +func (s *Store) initialize(ctx context.Context, created bool) error { + for _, statement := range []string{ + "PRAGMA foreign_keys = ON", "PRAGMA trusted_schema = OFF", + "PRAGMA secure_delete = FAST", "PRAGMA synchronous = FULL", + } { + if _, err := s.db.ExecContext(ctx, statement); err != nil { + return localError(err) + } + } + mode, err := setJournalMode(ctx, s.db) + if err != nil { + return localError(err) + } + s.journalMode = mode + s.journalMode = strings.ToLower(s.journalMode) + if s.journalMode != "wal" { + if !isLocalRollbackJournal(s.journalMode) { + return fmt.Errorf("stage store: unsupported journal mode") + } + s.journalFallback = true + } + return s.migrate(ctx, created) +} + +func configureWAL(ctx context.Context, db *sql.DB) (string, error) { + var mode string + err := db.QueryRowContext(ctx, "PRAGMA journal_mode = WAL").Scan(&mode) + return mode, err +} + +func isLocalRollbackJournal(mode string) bool { + return mode == "delete" || mode == "truncate" || mode == "persist" +} + +func (s *Store) migrate(ctx context.Context, created bool) error { + conn, err := s.db.Conn(ctx) + if err != nil { + return localError(err) + } + defer conn.Close() + if _, err = conn.ExecContext(ctx, "BEGIN IMMEDIATE"); err != nil { + return localError(err) + } + committed := false + defer func() { + if !committed { + _, _ = conn.ExecContext(context.Background(), "ROLLBACK") + } + }() + var id int + if err := conn.QueryRowContext(ctx, "PRAGMA application_id").Scan(&id); err != nil { + return localError(err) + } + if created { + if id != 0 && id != applicationID { + return ErrIdentity + } + if id == 0 { + var schemaObjects, userVersion int + if err := conn.QueryRowContext(ctx, "SELECT count(*) FROM sqlite_schema").Scan(&schemaObjects); err != nil { + return localError(err) + } + if err := conn.QueryRowContext(ctx, "PRAGMA user_version").Scan(&userVersion); err != nil { + return localError(err) + } + if schemaObjects != 0 || userVersion != 0 { + return ErrIdentity + } + if _, err := conn.ExecContext(ctx, fmt.Sprintf("PRAGMA application_id = %d", applicationID)); err != nil { + return localError(err) + } + } + } else if id != applicationID { + return ErrIdentity + } + if _, err = conn.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations ( +version INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, checksum TEXT NOT NULL, applied_at TEXT NOT NULL +) STRICT`); err != nil { + return localError(err) + } + rows, err := conn.QueryContext(ctx, "SELECT version, name, checksum FROM schema_migrations ORDER BY version") + if err != nil { + return localError(err) + } + type applied struct { + version int + name, checksum string + } + var got []applied + for rows.Next() { + var a applied + if err := rows.Scan(&a.version, &a.name, &a.checksum); err != nil { + rows.Close() + return localError(err) + } + got = append(got, a) + } + if err := rows.Close(); err != nil { + return localError(err) + } + if len(got) > len(migrations) { + return ErrMigration + } + for i, a := range got { + m := migrations[i] + if a.version != m.version || a.name != m.name || a.checksum != m.checksum() { + return ErrMigration + } + } + for _, m := range migrations[len(got):] { + if _, err = conn.ExecContext(ctx, m.sql); err != nil { + return fmt.Errorf("%w: apply failed", ErrMigration) + } + if _, err = conn.ExecContext(ctx, "INSERT INTO schema_migrations(version,name,checksum,applied_at) VALUES(?,?,?,?)", m.version, m.name, m.checksum(), time.Now().UTC().Format(time.RFC3339Nano)); err != nil { + return localError(err) + } + } + if _, err = conn.ExecContext(ctx, "COMMIT"); err != nil { + return localError(err) + } + committed = true + return nil +} + +func (s *Store) verifyIdentityAndMigrations(ctx context.Context) error { + var id int + if err := s.db.QueryRowContext(ctx, "PRAGMA application_id").Scan(&id); err != nil { + return localError(err) + } + if id != applicationID { + return ErrIdentity + } + rows, err := s.db.QueryContext(ctx, "SELECT version,name,checksum FROM schema_migrations ORDER BY version") + if err != nil { + return ErrMigration + } + defer rows.Close() + i := 0 + for rows.Next() { + if i >= len(migrations) { + return ErrMigration + } + var version int + var name, checksum string + if err := rows.Scan(&version, &name, &checksum); err != nil { + return localError(err) + } + m := migrations[i] + if version != m.version || name != m.name || checksum != m.checksum() { + return ErrMigration + } + i++ + } + if i != len(migrations) || rows.Err() != nil { + return ErrMigration + } + return nil +} + +func localError(err error) error { + if err == nil { + return nil + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + var sqliteErr *sqlite3.Error + if errors.As(err, &sqliteErr) && (sqliteErr.Code()&0xff == 5 || sqliteErr.Code()&0xff == 6) { + return ErrBusy + } + return fmt.Errorf("stage store: local database failure") +} diff --git a/internal/stagestore/store_test.go b/internal/stagestore/store_test.go new file mode 100644 index 0000000..b4d0467 --- /dev/null +++ b/internal/stagestore/store_test.go @@ -0,0 +1,926 @@ +//go:build darwin || linux + +package stagestore + +import ( + "context" + "crypto/sha256" + "database/sql" + "errors" + "os" + "path/filepath" + "reflect" + "sync" + "testing" + "time" + + "golang.org/x/sys/unix" +) + +func testPath(t *testing.T) string { + t.Helper() + return filepath.Join(t.TempDir(), "state", "mattermost-cli", DatabaseFilename) +} + +func TestResolvePaths(t *testing.T) { + paths, err := ResolvePaths("/home/test", func(key string) (string, bool) { return "/state root/?x", key == "XDG_STATE_HOME" }) + if err != nil || paths.DBPath != "/state root/?x/mattermost-cli/stages.sqlite3" { + t.Fatalf("paths=%#v err=%v", paths, err) + } + paths, err = ResolvePaths("/home/test", func(string) (string, bool) { return "relative", true }) + if err != nil || paths.StateDir != "/home/test/.local/state/mattermost-cli" { + t.Fatalf("fallback=%#v err=%v", paths, err) + } +} + +func TestCanonicalDatabasePathOnly(t *testing.T) { + root := t.TempDir() + for _, path := range []string{ + filepath.Join(root, "other.sqlite3"), + filepath.Join(root, "nested", "..", DatabaseFilename), + "relative/" + DatabaseFilename, + } { + if _, err := Open(context.Background(), path); err == nil { + t.Fatalf("accepted path %q", path) + } + } +} + +func TestOpenInitializesAndReopens(t *testing.T) { + ctx, path := context.Background(), testPath(t) + s, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + var id, foreignKeys, trusted, secureDelete, synchronous int + var journal string + checks := []struct { + query string + target any + }{{"PRAGMA application_id", &id}, {"PRAGMA foreign_keys", &foreignKeys}, {"PRAGMA trusted_schema", &trusted}, {"PRAGMA secure_delete", &secureDelete}, {"PRAGMA synchronous", &synchronous}, {"PRAGMA journal_mode", &journal}} + for _, check := range checks { + if err := s.db.QueryRow(check.query).Scan(check.target); err != nil { + t.Fatal(err) + } + } + if id != applicationID || foreignKeys != 1 || trusted != 0 || secureDelete != 2 || synchronous != 2 || journal == "" { + t.Fatalf("id=%x fk=%d trusted=%d delete=%d sync=%d journal=%s", id, foreignKeys, trusted, secureDelete, synchronous, journal) + } + s.db.SetConnMaxLifetime(time.Nanosecond) + time.Sleep(time.Millisecond) + if err := s.db.QueryRow("PRAGMA secure_delete").Scan(&secureDelete); err != nil { + t.Fatal(err) + } + if err := s.db.QueryRow("PRAGMA synchronous").Scan(&synchronous); err != nil { + t.Fatal(err) + } + if secureDelete != 2 || synchronous != 2 { + t.Fatalf("replacement connection delete=%d sync=%d", secureDelete, synchronous) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + for _, item := range []struct { + path string + mode os.FileMode + }{{filepath.Dir(path), 0o700}, {path, 0o600}} { + info, err := os.Stat(item.path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != item.mode { + t.Fatalf("%s mode=%o", item.path, info.Mode().Perm()) + } + } + s, err = Open(ctx, path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + var count int + if err := s.db.QueryRow("SELECT count(*) FROM schema_migrations").Scan(&count); err != nil || count != 1 { + t.Fatalf("count=%d err=%v", count, err) + } +} + +func TestReadOnlyAbsentDoesNotCreate(t *testing.T) { + path := testPath(t) + if _, err := OpenReadOnly(context.Background(), path); err == nil { + t.Fatal("expected error") + } + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("stat=%v", err) + } + report, err := Doctor(context.Background(), path) + if err != nil || report.Exists { + t.Fatalf("report=%#v err=%v", report, err) + } +} + +func createPendingMarkerForTest(t *testing.T, path string) { + t.Helper() + dirfd, _, err := walkParent(path, true) + if err != nil { + t.Fatal(err) + } + defer unix.Close(dirfd) + if err := createPendingAt(dirfd); err != nil { + t.Fatal(err) + } +} + +func TestAdoptsNonzeroBootstrapResidueWithMarker(t *testing.T) { + path := testPath(t) + createPendingMarkerForTest(t, path) + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite", sqliteURI(path, false)) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec("VACUUM"); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Size() == 0 { + t.Fatal("VACUUM left an empty database file") + } + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + var id int + if err := s.db.QueryRow("PRAGMA application_id").Scan(&id); err != nil || id != applicationID { + t.Fatalf("id=%x err=%v", id, err) + } +} + +func TestRejectsMarkerBackedForeignSchema(t *testing.T) { + path := testPath(t) + createPendingMarkerForTest(t, path) + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite", sqliteURI(path, false)) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec("CREATE TABLE foreign_schema(value TEXT)"); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + if _, err := Open(context.Background(), path); !errors.Is(err, ErrIdentity) { + t.Fatalf("error=%v", err) + } +} + +func TestStaleBootstrapMarkerAfterCommitHeals(t *testing.T) { + path := testPath(t) + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + createPendingMarkerForTest(t, path) + s, err = Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(filepath.Dir(path), bootstrapPendingFilename)); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("marker stat=%v", err) + } +} + +func TestLinuxFilesystemTypeWhitelist(t *testing.T) { + for name, kind := range map[string]uint64{ + "ext": 0xEF53, "bcachefs": 0xCA451A4E, "nilfs2": 0x3434, "ubifs": 0x24051905, + } { + if !linuxFilesystemTypeAllowed(kind) { + t.Fatalf("%s filesystem rejected", name) + } + } + if linuxFilesystemTypeAllowed(0x6969) { + t.Fatal("NFS filesystem accepted") + } + if linuxFilesystemTypeAllowed(0xDEADBEEF) { + t.Fatal("unknown filesystem accepted") + } +} + +func TestCreatedDirectoryNoFollowFallbackRequiresTrustedParent(t *testing.T) { + root := t.TempDir() + fd, err := unix.Open(root, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC, 0) + if err != nil { + t.Fatal(err) + } + defer unix.Close(fd) + if err := unix.Mkdirat(fd, "child", 0o700); err != nil { + t.Fatal(err) + } + var expected unix.Stat_t + if err := unix.Fstatat(fd, "child", &expected, unix.AT_SYMLINK_NOFOLLOW); err != nil { + t.Fatal(err) + } + old := stateDirectoryFchmodat + stateDirectoryFchmodat = func(parent int, name string, mode uint32, flags int) error { + if flags != 0 { + return unix.ENOTSUP + } + return unix.Fchmodat(parent, name, mode, flags) + } + t.Cleanup(func() { stateDirectoryFchmodat = old }) + if _, err := secureCreatedDirectory(fd, "child", expected, false); err == nil { + t.Fatal("unsafe fallback accepted") + } + if _, err := secureCreatedDirectory(fd, "child", expected, true); err != nil { + t.Fatal(err) + } +} + +func TestPendingMarkerShortWriteCompletesExactly(t *testing.T) { + old := pendingWrite + first := true + pendingWrite = func(fd int, value []byte) (int, error) { + if first { + first = false + return unix.Write(fd, value[:len(value)/2]) + } + return unix.Write(fd, value) + } + t.Cleanup(func() { pendingWrite = old }) + s, err := Open(context.Background(), testPath(t)) + if err != nil { + t.Fatal(err) + } + defer s.Close() +} + +func TestPartialPendingMarkerWithoutDatabaseRecovers(t *testing.T) { + path := testPath(t) + dirfd, _, err := walkParent(path, true) + if err != nil { + t.Fatal(err) + } + unix.Close(dirfd) + marker := filepath.Join(filepath.Dir(path), bootstrapPendingFilename) + if err := os.WriteFile(marker, []byte("partial"), 0o600); err != nil { + t.Fatal(err) + } + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + defer s.Close() +} + +func TestPartialPendingMarkerWithDatabaseRejects(t *testing.T) { + path := testPath(t) + dirfd, _, err := walkParent(path, true) + if err != nil { + t.Fatal(err) + } + unix.Close(dirfd) + if err := os.WriteFile(filepath.Join(filepath.Dir(path), bootstrapPendingFilename), []byte("partial"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatal(err) + } + if _, err := Open(context.Background(), path); err == nil { + t.Fatal("accepted partial marker beside database") + } +} + +func TestSymlinkAdmissionBeforeBoundary(t *testing.T) { + if !symlinkAllowedBeforeBoundary(0, false) { + t.Fatal("root-owned pre-boundary symlink rejected") + } + if symlinkAllowedBeforeBoundary(uint32(os.Geteuid()), false) && os.Geteuid() != 0 { + t.Fatal("user-owned pre-boundary symlink accepted") + } + if symlinkAllowedBeforeBoundary(0, true) { + t.Fatal("root-owned symlink accepted after boundary") + } + if symlinkAllowedBeforeBoundary(12345, false) { + t.Fatal("foreign-owned pre-boundary symlink accepted") + } +} + +func TestRootOwnedAncestorPermissions(t *testing.T) { + if !ancestorPermissionsAllowed(0, uint32(unix.S_IFDIR|0o755), false) { + t.Fatal("ordinary root-owned ancestor rejected") + } + if !ancestorPermissionsAllowed(0, uint32(unix.S_IFDIR|unix.S_ISVTX|0o777), false) { + t.Fatal("sticky root-owned ancestor rejected") + } + if ancestorPermissionsAllowed(0, uint32(unix.S_IFDIR|0o777), false) { + t.Fatal("non-sticky world-writable root-owned ancestor accepted") + } + if ancestorPermissionsAllowed(0, uint32(unix.S_IFDIR|0o775), false) { + t.Fatal("non-sticky group-writable root-owned ancestor accepted") + } +} + +func TestRejectsNonEmptyZeroIdentityDatabase(t *testing.T) { + path := testPath(t) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite", sqliteURI(path, false)) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec("CREATE TABLE foreign_database(value TEXT)"); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + if err := os.Chmod(path, 0o600); err != nil { + t.Fatal(err) + } + if _, err := Open(context.Background(), path); !errors.Is(err, ErrIdentity) { + t.Fatalf("error=%v", err) + } +} + +func TestRestrictiveUmaskStillCreatesExactModes(t *testing.T) { + path := testPath(t) + old := unix.Umask(0o777) + t.Cleanup(func() { unix.Umask(old) }) + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + for _, item := range []struct { + path string + mode os.FileMode + }{{filepath.Dir(path), 0o700}, {path, 0o600}, {filepath.Join(filepath.Dir(path), bootstrapLockFilename), 0o600}} { + info, err := os.Stat(item.path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != item.mode { + t.Fatalf("%s mode=%o", item.path, info.Mode().Perm()) + } + } +} + +func TestRecoversPrivateCreationResidue(t *testing.T) { + t.Run("state directory", func(t *testing.T) { + path := testPath(t) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Chmod(filepath.Dir(path), 0); err != nil { + t.Fatal(err) + } + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + }) + + t.Run("intermediate directory", func(t *testing.T) { + root := t.TempDir() + intermediate := filepath.Join(root, "state") + if err := os.Mkdir(intermediate, 0o700); err != nil { + t.Fatal(err) + } + if err := os.Chmod(intermediate, 0); err != nil { + t.Fatal(err) + } + path := filepath.Join(intermediate, "mattermost-cli", DatabaseFilename) + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + }) + + t.Run("bootstrap lock", func(t *testing.T) { + path := testPath(t) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + lock := filepath.Join(filepath.Dir(path), bootstrapLockFilename) + if err := os.WriteFile(lock, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chmod(lock, 0); err != nil { + t.Fatal(err) + } + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + }) + + t.Run("marker-backed database", func(t *testing.T) { + path := testPath(t) + createPendingMarkerForTest(t, path) + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chmod(path, 0); err != nil { + t.Fatal(err) + } + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + }) +} + +func TestReadOnlyDoesNotRecoverDirectoryMode(t *testing.T) { + path := testPath(t) + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + if err := os.Chmod(filepath.Dir(path), 0o500); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(filepath.Dir(path), 0o700) }) + if _, err := OpenReadOnly(context.Background(), path); err == nil { + t.Fatal("read-only open accepted noncanonical directory mode") + } + info, err := os.Stat(filepath.Dir(path)) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o500 { + t.Fatalf("read-only open mutated directory mode to %o", info.Mode().Perm()) + } +} + +func TestRejectsSpecialBitsDuringPrivateFileRecovery(t *testing.T) { + path := testPath(t) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + lock := filepath.Join(filepath.Dir(path), bootstrapLockFilename) + if err := os.WriteFile(lock, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := unix.Chmod(lock, unix.S_ISUID|0o600); err != nil { + t.Skipf("setuid mode unavailable: %v", err) + } + if _, err := Open(context.Background(), path); err == nil { + t.Fatal("accepted setuid bootstrap lock") + } +} + +func TestExactModeRejectsSpecialBits(t *testing.T) { + if !hasExactMode(unix.S_IFREG|0o600, unix.S_IFREG, 0o600) { + t.Fatal("exact regular mode rejected") + } + if hasExactMode(unix.S_IFREG|unix.S_ISUID|0o600, unix.S_IFREG, 0o600) { + t.Fatal("setuid regular mode accepted") + } + if hasExactMode(unix.S_IFDIR|unix.S_ISGID|0o700, unix.S_IFDIR, 0o700) { + t.Fatal("setgid directory mode accepted") + } + if hasExactMode(unix.S_IFDIR|unix.S_ISVTX|0o700, unix.S_IFDIR, 0o700) { + t.Fatal("sticky directory mode accepted") + } +} + +func TestRejectsIdentityAndMigrationDrift(t *testing.T) { + ctx, path := context.Background(), testPath(t) + s, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec("PRAGMA application_id=42"); err != nil { + t.Fatal(err) + } + s.Close() + if _, err := Open(ctx, path); !errors.Is(err, ErrIdentity) { + t.Fatalf("identity=%v", err) + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + s, err = Open(ctx, path) + if err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec("UPDATE schema_migrations SET checksum='wrong'"); err != nil { + t.Fatal(err) + } + s.Close() + if _, err := OpenReadOnly(ctx, path); !errors.Is(err, ErrMigration) { + t.Fatalf("migration=%v", err) + } +} + +func TestMigrationFailureIsAtomicAndUnknownIsRejected(t *testing.T) { + ctx, path := context.Background(), testPath(t) + s, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + s.Close() + original := migrations + t.Cleanup(func() { migrations = original }) + migrations = append(append([]migration{}, migrations...), migration{version: 2, name: "broken", sql: "CREATE TABLE should_rollback(x); SELECT no_such_function();"}) + if _, err := Open(ctx, path); !errors.Is(err, ErrMigration) { + t.Fatalf("failure=%v", err) + } + migrations = original + s, err = OpenReadOnly(ctx, path) + if err != nil { + t.Fatal(err) + } + var count int + if err := s.db.QueryRow("SELECT count(*) FROM sqlite_master WHERE name='should_rollback'").Scan(&count); err != nil || count != 0 { + t.Fatalf("partial table count=%d err=%v", count, err) + } + s.Close() + s, err = Open(ctx, path) + if err != nil { + t.Fatal(err) + } + _, err = s.db.Exec("INSERT INTO schema_migrations(version,name,checksum,applied_at) VALUES(2,'future','x','x')") + if err != nil { + t.Fatal(err) + } + s.Close() + if _, err := OpenReadOnly(ctx, path); !errors.Is(err, ErrMigration) { + t.Fatalf("unknown=%v", err) + } +} + +func TestRejectsUnsafeFiles(t *testing.T) { + ctx := context.Background() + t.Run("mode", func(t *testing.T) { + path := testPath(t) + s, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + s.Close() + os.Chmod(path, 0o644) + if _, err := Open(ctx, path); err == nil { + t.Fatal("accepted mode") + } + }) + t.Run("hardlink", func(t *testing.T) { + path := testPath(t) + s, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + s.Close() + if err := os.Link(path, path+".link"); err != nil { + t.Skip(err) + } + if _, err := Open(ctx, path); err == nil { + t.Fatal("accepted hardlink") + } + }) + t.Run("symlink", func(t *testing.T) { + root, real := t.TempDir(), filepath.Join(t.TempDir(), "real") + if err := os.Mkdir(real, 0o700); err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "link") + if err := os.Symlink(real, link); err != nil { + t.Fatal(err) + } + if _, err := Open(ctx, filepath.Join(link, DatabaseFilename)); err == nil { + t.Fatal("accepted symlink") + } + }) + t.Run("sidecar mode", func(t *testing.T) { + path := testPath(t) + s, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + s.Close() + if err := os.WriteFile(path+"-journal", nil, 0o644); err != nil { + t.Fatal(err) + } + if _, err := Open(ctx, path); err == nil { + t.Fatal("accepted unsafe sidecar") + } + }) + t.Run("bootstrap lock mode", func(t *testing.T) { + path := testPath(t) + s, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + if err := os.Chmod(filepath.Join(filepath.Dir(path), bootstrapLockFilename), 0o644); err != nil { + t.Fatal(err) + } + if _, err := Open(ctx, path); err == nil { + t.Fatal("accepted unsafe bootstrap lock") + } + }) +} + +func TestNetworkFilesystemSeam(t *testing.T) { + old := filesystemAllowed + filesystemAllowed = func(int) bool { return false } + t.Cleanup(func() { filesystemAllowed = old }) + if _, err := Open(context.Background(), testPath(t)); !errors.Is(err, ErrUnsafeFilesystem) { + t.Fatalf("error=%v", err) + } +} + +func TestRollbackJournalFallbackSeam(t *testing.T) { + old := setJournalMode + setJournalMode = func(context.Context, *sql.DB) (string, error) { return "delete", nil } + t.Cleanup(func() { setJournalMode = old }) + s, err := Open(context.Background(), testPath(t)) + if err != nil { + t.Fatal(err) + } + defer s.Close() + if !s.journalFallback || s.journalMode != "delete" { + t.Fatalf("mode=%s fallback=%v", s.journalMode, s.journalFallback) + } +} + +func TestBootstrapLockHonorsCancellation(t *testing.T) { + path := testPath(t) + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + s.Close() + f, err := os.OpenFile(filepath.Join(filepath.Dir(path), bootstrapLockFilename), os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + defer f.Close() + if err := unix.Flock(int(f.Fd()), unix.LOCK_EX); err != nil { + t.Fatal(err) + } + defer unix.Flock(int(f.Fd()), unix.LOCK_UN) + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) + defer cancel() + _, err = Open(ctx, path) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("error=%v", err) + } +} + +func TestSQLiteBusyHonorsCancellation(t *testing.T) { + path := testPath(t) + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + blocker, err := sql.Open("sqlite", sqliteURI(path, false)) + if err != nil { + t.Fatal(err) + } + defer blocker.Close() + if _, err := blocker.Exec("BEGIN IMMEDIATE"); err != nil { + t.Fatal(err) + } + defer blocker.Exec("ROLLBACK") + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Millisecond) + defer cancel() + started := time.Now() + _, err = Open(ctx, path) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("error=%v", err) + } + if elapsed := time.Since(started); elapsed > 500*time.Millisecond { + t.Fatalf("cancellation took %s", elapsed) + } +} + +type fileSnapshot struct { + exists bool + mode os.FileMode + size int64 + modTime time.Time + digest [32]byte +} + +func snapshotStoreFiles(t *testing.T, path string) map[string]fileSnapshot { + t.Helper() + result := make(map[string]fileSnapshot) + for _, suffix := range []string{"", "-wal", "-shm", "-journal"} { + name := path + suffix + info, err := os.Stat(name) + if errors.Is(err, os.ErrNotExist) { + result[suffix] = fileSnapshot{} + continue + } + if err != nil { + t.Fatal(err) + } + content, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + result[suffix] = fileSnapshot{exists: true, mode: info.Mode(), size: info.Size(), modTime: info.ModTime(), digest: sha256.Sum256(content)} + } + return result +} + +func TestReadOnlyAndDoctorDoNotMutateStoreFiles(t *testing.T) { + ctx, path := context.Background(), testPath(t) + writer, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + if _, err := writer.db.Exec("CREATE TABLE readonly_probe(value TEXT)"); err != nil { + t.Fatal(err) + } + if _, err := writer.db.Exec("INSERT INTO readonly_probe VALUES('visible-in-wal')"); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + before := snapshotStoreFiles(t, path) + ro, err := OpenReadOnly(ctx, path) + if err != nil { + t.Fatal(err) + } + var value string + if err := ro.db.QueryRow("SELECT value FROM readonly_probe").Scan(&value); err != nil { + t.Fatal(err) + } + if value != "visible-in-wal" { + t.Fatalf("value=%q", value) + } + if err := ro.Close(); err != nil { + t.Fatal(err) + } + if _, err := Doctor(ctx, path); err != nil { + t.Fatal(err) + } + after := snapshotStoreFiles(t, path) + if !reflect.DeepEqual(before, after) { + t.Fatalf("store files changed during read-only inspection: before=%#v after=%#v", before, after) + } +} + +func TestReadOnlyRejectsActiveWALWithoutMutation(t *testing.T) { + ctx, path := context.Background(), testPath(t) + writer, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + defer writer.Close() + if _, err := writer.db.Exec("CREATE TABLE active_wal(value TEXT)"); err != nil { + t.Fatal(err) + } + before := snapshotStoreFiles(t, path) + waitCtx, cancel := context.WithTimeout(ctx, 25*time.Millisecond) + defer cancel() + if _, err := OpenReadOnly(waitCtx, path); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("error=%v", err) + } + after := snapshotStoreFiles(t, path) + if !reflect.DeepEqual(before, after) { + t.Fatal("active WAL changed during rejected read-only open") + } +} + +func TestCurrentRevisionMustBeCurrent(t *testing.T) { + path := testPath(t) + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + tx, err := s.db.Begin() + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(`INSERT INTO stages(id,created_at,updated_at,operation,server_url,user_id,lifecycle,recovery,current_revision) VALUES('s','x','x','create_post','https://x','u','open','none',1)`) + if err != nil { + t.Fatal(err) + } + _, err = tx.Exec(`INSERT INTO stage_revisions(stage_id,revision,state,created_at,semantic_digest,destination_json,plan_json) VALUES('s',1,'superseded','x',zeroblob(32),'{}','{}')`) + if err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err == nil { + t.Fatal("accepted superseded current revision") + } +} + +func TestDoctorReportsForeignKeyViolation(t *testing.T) { + ctx, path := context.Background(), testPath(t) + s, err := Open(ctx, path) + if err != nil { + t.Fatal(err) + } + s.Close() + db, err := sql.Open("sqlite", sqliteURI(path, false)) + if err != nil { + t.Fatal(err) + } + _, err = db.Exec(`PRAGMA foreign_keys=OFF; +INSERT INTO stages(id,created_at,updated_at,operation,server_url,user_id,lifecycle,recovery,current_revision) VALUES('s','x','x','create_post','https://x','u','open','none',1); +INSERT INTO stage_revisions(stage_id,revision,state,created_at,semantic_digest,destination_json,plan_json) VALUES('s',1,'current','x',zeroblob(32),'{}','{}'); +INSERT INTO request_replays(server_url,user_id,request_id,request_schema,semantic_digest,stage_id,revision,created_at) VALUES('https://x','u','r','v',zeroblob(32),'s',2,'x')`) + if err != nil { + t.Fatal(err) + } + db.Close() + report, err := Doctor(ctx, path) + if err != nil { + t.Fatal(err) + } + if report.ForeignKeyIssues != 1 || len(report.ForeignKeyRows) != 1 { + t.Fatalf("report=%#v", report) + } +} + +func TestDoctorRejectsCorruptDatabase(t *testing.T) { + path := testPath(t) + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + s.Close() + f, err := os.OpenFile(path, os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + if _, err := f.WriteAt(make([]byte, 32), 100); err != nil { + f.Close() + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + if _, err := Doctor(context.Background(), path); err == nil { + t.Fatal("doctor accepted corrupt database") + } +} + +func TestConcurrentOpenAndDoctor(t *testing.T) { + path := testPath(t) + const n = 4 + var wg sync.WaitGroup + wg.Add(n) + errs := make(chan error, n) + for range n { + go func() { + defer wg.Done() + s, err := Open(context.Background(), path) + if err == nil { + err = s.Close() + } + errs <- err + }() + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatal(err) + } + } + report, err := Doctor(context.Background(), path) + if err != nil { + t.Fatal(err) + } + if !report.Exists || !report.FilesystemSafe || report.ApplicationID != applicationID || !report.QueryOnly || len(report.Integrity) == 0 || !report.Migrations.Valid { + t.Fatalf("report=%#v", report) + } +} From 172fa557ccd40bee3cbbfbca92801372a6bb2024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 23:33:06 +0300 Subject: [PATCH 050/119] feat: add bounded message input --- internal/messageinput/input.go | 77 +++++++++++++++++++++++++++++ internal/messageinput/input_test.go | 75 ++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 internal/messageinput/input.go create mode 100644 internal/messageinput/input_test.go diff --git a/internal/messageinput/input.go b/internal/messageinput/input.go new file mode 100644 index 0000000..7450e29 --- /dev/null +++ b/internal/messageinput/input.go @@ -0,0 +1,77 @@ +package messageinput + +import ( + "errors" + "io" + "unicode" + "unicode/utf8" +) + +const ( + MaxBytes = 65_535 + MaxCharacters = 16_383 +) + +var ( + ErrEmpty = errors.New("message cannot be empty") + ErrInvalidUTF8 = errors.New("message must be valid UTF-8") + ErrTooManyBytes = errors.New("message exceeds 65535 UTF-8 bytes") + ErrTooManyRunes = errors.New("message exceeds 16383 Unicode characters") + ErrRead = errors.New("could not read message input") +) + +// Read consumes at most one byte beyond Mattermost's verified message limit. +// The returned bytes are an exact copy of the caller's valid UTF-8 input. +func Read(input io.Reader) ([]byte, error) { + if input == nil { + return nil, ErrRead + } + data, err := io.ReadAll(io.LimitReader(input, MaxBytes+1)) + if err != nil { + return nil, ErrRead + } + if len(data) > MaxBytes { + return nil, ErrTooManyBytes + } + if err := Validate(data); err != nil { + return nil, err + } + return data, nil +} + +func Validate(message []byte) error { + if len(message) > MaxBytes { + return ErrTooManyBytes + } + if !utf8.Valid(message) { + return ErrInvalidUTF8 + } + if utf8.RuneCount(message) > MaxCharacters { + return ErrTooManyRunes + } + if whitespaceOnly(message) { + return ErrEmpty + } + return nil +} + +func whitespaceOnly(message []byte) bool { + if len(message) == 0 { + return true + } + for _, character := range string(message) { + if !ecmaScriptWhitespace(character) { + return false + } + } + return true +} + +func ecmaScriptWhitespace(character rune) bool { + switch character { + case '\t', '\n', '\v', '\f', '\r', ' ', '\u00a0', '\u2028', '\u2029', '\ufeff': + return true + default: + return unicode.Is(unicode.Zs, character) + } +} diff --git a/internal/messageinput/input_test.go b/internal/messageinput/input_test.go new file mode 100644 index 0000000..81e28eb --- /dev/null +++ b/internal/messageinput/input_test.go @@ -0,0 +1,75 @@ +package messageinput + +import ( + "bytes" + "errors" + "io" + "strings" + "testing" + "unicode/utf8" +) + +func TestReadPreservesExactMarkdown(t *testing.T) { + short := []byte("# release notes\n\n- **bold** `code`\n- [link](https://example.com/?a=1&b=2)\n\n> final 🌍\n") + long := []byte(strings.Repeat("## section 🌍\n\n- **bold** `code`\n\n", 470)) + if utf8.RuneCount(long) < 15_500 || utf8.RuneCount(long) >= MaxCharacters || len(long) >= MaxBytes { + t.Fatalf("invalid long fixture: bytes=%d characters=%d", len(long), utf8.RuneCount(long)) + } + for name, value := range map[string][]byte{"short": short, "long": long} { + t.Run(name, func(t *testing.T) { + chunks := io.MultiReader( + bytes.NewReader(value[:min(17, len(value))]), + bytes.NewReader(value[min(17, len(value)):min(4099, len(value))]), + bytes.NewReader(value[min(4099, len(value)):]), + ) + got, err := Read(chunks) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, value) { + t.Fatal("message bytes changed") + } + }) + } +} + +func TestValidateMessageBounds(t *testing.T) { + if err := Validate([]byte(strings.Repeat("a", MaxCharacters))); err != nil { + t.Fatal(err) + } + for name, test := range map[string]struct { + value []byte + want error + }{ + "empty": {nil, ErrEmpty}, + "ascii whitespace": {[]byte(" \t\r\n\v\f"), ErrEmpty}, + "ecmascript whitespace": {[]byte("\u00a0\u1680\u2028\u2029\ufeff"), ErrEmpty}, + "invalid utf8": {[]byte{0xc3, 0x28}, ErrInvalidUTF8}, + "characters": {[]byte(strings.Repeat("a", MaxCharacters+1)), ErrTooManyRunes}, + "bytes": {[]byte(strings.Repeat("x", MaxBytes+1)), ErrTooManyBytes}, + } { + t.Run(name, func(t *testing.T) { + if err := Validate(test.value); !errors.Is(err, test.want) { + t.Fatalf("error=%v want=%v", err, test.want) + } + }) + } +} + +func TestReadFailsAtStreamingByteBoundary(t *testing.T) { + input := io.MultiReader(bytes.NewReader(bytes.Repeat([]byte{'x'}, MaxBytes)), strings.NewReader("x")) + if _, err := Read(input); !errors.Is(err, ErrTooManyBytes) { + t.Fatalf("error=%v", err) + } +} + +func TestReadHidesPhysicalInputFailure(t *testing.T) { + physical := errors.New("hostile reader detail") + if _, err := Read(failingReader{err: physical}); !errors.Is(err, ErrRead) || errors.Is(err, physical) || strings.Contains(err.Error(), physical.Error()) { + t.Fatalf("error=%v", err) + } +} + +type failingReader struct{ err error } + +func (r failingReader) Read([]byte) (int, error) { return 0, r.err } From 146575df17fb7e682398fa7606258d2d099bc4db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Thu, 16 Jul 2026 23:49:11 +0300 Subject: [PATCH 051/119] fix: align message input oracle --- internal/messageinput/input.go | 12 ++-- internal/messageinput/input_test.go | 104 +++++++++++++++++++++++++++- 2 files changed, 107 insertions(+), 9 deletions(-) diff --git a/internal/messageinput/input.go b/internal/messageinput/input.go index 7450e29..7045218 100644 --- a/internal/messageinput/input.go +++ b/internal/messageinput/input.go @@ -27,12 +27,12 @@ func Read(input io.Reader) ([]byte, error) { return nil, ErrRead } data, err := io.ReadAll(io.LimitReader(input, MaxBytes+1)) - if err != nil { - return nil, ErrRead - } if len(data) > MaxBytes { return nil, ErrTooManyBytes } + if err != nil { + return nil, ErrRead + } if err := Validate(data); err != nil { return nil, err } @@ -46,12 +46,12 @@ func Validate(message []byte) error { if !utf8.Valid(message) { return ErrInvalidUTF8 } - if utf8.RuneCount(message) > MaxCharacters { - return ErrTooManyRunes - } if whitespaceOnly(message) { return ErrEmpty } + if utf8.RuneCount(message) > MaxCharacters { + return ErrTooManyRunes + } return nil } diff --git a/internal/messageinput/input_test.go b/internal/messageinput/input_test.go index 81e28eb..a919fd0 100644 --- a/internal/messageinput/input_test.go +++ b/internal/messageinput/input_test.go @@ -2,7 +2,9 @@ package messageinput import ( "bytes" + "crypto/sha256" "errors" + "fmt" "io" "strings" "testing" @@ -10,9 +12,15 @@ import ( ) func TestReadPreservesExactMarkdown(t *testing.T) { - short := []byte("# release notes\n\n- **bold** `code`\n- [link](https://example.com/?a=1&b=2)\n\n> final 🌍\n") - long := []byte(strings.Repeat("## section 🌍\n\n- **bold** `code`\n\n", 470)) - if utf8.RuneCount(long) < 15_500 || utf8.RuneCount(long) >= MaxCharacters || len(long) >= MaxBytes { + short := []byte(shortMarkdown) + long := []byte(longMarkdown()) + if got := fmt.Sprintf("%x", sha256.Sum256(short)); got != "f723745ae7f1c3c716d20ea724a383fc9777a34ffc73ffd1d5999977bd8fadf9" { + t.Fatalf("short fixture drifted: %s", got) + } + if got := fmt.Sprintf("%x", sha256.Sum256(long)); got != "b81aca78af1a5273db3da59748a1d06f6bcbaac885badf56fb7edf9fa809aa30" { + t.Fatalf("long fixture drifted: %s", got) + } + if utf8.RuneCount(long) != 15_666 || len(long) != 15_841 { t.Fatalf("invalid long fixture: bytes=%d characters=%d", len(long), utf8.RuneCount(long)) } for name, value := range map[string][]byte{"short": short, "long": long} { @@ -33,6 +41,40 @@ func TestReadPreservesExactMarkdown(t *testing.T) { } } +const shortMarkdown = `# Release ready ✅ + +**Status:** shipped with [runbook](https://example.test/runbook). + +- [x] API healthy +- [x] migrations applied + +> Verify the canary before broad rollout. + +` + "```ts\nconst ready = true\n```\n" + +func longMarkdown() string { + message := "# Extended deployment report 🌍\n\nThis intentionally large fixture exercises structured Markdown without relying on generated prose.\n" + for section := 1; utf8.RuneCountInString(message) < 15_500; section++ { + message += fmt.Sprintf(` +## Service %d + +| Check | Result | Detail | +| --- | --- | --- | +| health | ✅ | [probe](https://example.test/health/%d) | +| queue | ✅ | **drained** | + +- [x] deploy completed +- [x] metrics reviewed +- [ ] observe for 30 minutes + +> Service %d remained inside its latency budget. + +`+"```json\n"+`{"service":%d,"status":"healthy","regions":["eu","us"],"rollback":false} +`+"```\n", section, section, section, section) + } + return message + "\n---\n\nEnd of report. _Keep this final newline._\n" +} + func TestValidateMessageBounds(t *testing.T) { if err := Validate([]byte(strings.Repeat("a", MaxCharacters))); err != nil { t.Fatal(err) @@ -47,6 +89,7 @@ func TestValidateMessageBounds(t *testing.T) { "invalid utf8": {[]byte{0xc3, 0x28}, ErrInvalidUTF8}, "characters": {[]byte(strings.Repeat("a", MaxCharacters+1)), ErrTooManyRunes}, "bytes": {[]byte(strings.Repeat("x", MaxBytes+1)), ErrTooManyBytes}, + "oversized whitespace": {[]byte(strings.Repeat(" ", MaxCharacters+1)), ErrEmpty}, } { t.Run(name, func(t *testing.T) { if err := Validate(test.value); !errors.Is(err, test.want) { @@ -56,6 +99,24 @@ func TestValidateMessageBounds(t *testing.T) { } } +func TestUnicodeBoundariesAndNegativeWhitespace(t *testing.T) { + maximum := []byte(strings.Repeat("🌍", MaxCharacters)) + if len(maximum) != 65_532 { + t.Fatalf("maximum non-BMP fixture bytes=%d", len(maximum)) + } + if err := Validate(maximum); err != nil { + t.Fatal(err) + } + if err := Validate([]byte(strings.Repeat("🌍", MaxCharacters+1))); !errors.Is(err, ErrTooManyBytes) { + t.Fatalf("overflow error=%v", err) + } + for _, value := range []string{"\u0085", "\u180e", "\u200b"} { + if err := Validate([]byte(value)); err != nil { + t.Fatalf("non-ECMAScript whitespace %U rejected: %v", []rune(value)[0], err) + } + } +} + func TestReadFailsAtStreamingByteBoundary(t *testing.T) { input := io.MultiReader(bytes.NewReader(bytes.Repeat([]byte{'x'}, MaxBytes)), strings.NewReader("x")) if _, err := Read(input); !errors.Is(err, ErrTooManyBytes) { @@ -70,6 +131,43 @@ func TestReadHidesPhysicalInputFailure(t *testing.T) { } } +func TestReadPrefersConfirmedOverflowToSameReadFailure(t *testing.T) { + physical := errors.New("late read failure") + reader := dataErrorReader{data: bytes.Repeat([]byte{'x'}, MaxBytes+1), err: physical} + if _, err := Read(&reader); !errors.Is(err, ErrTooManyBytes) || errors.Is(err, physical) { + t.Fatalf("error=%v", err) + } +} + +func FuzzValidate(f *testing.F) { + for _, seed := range [][]byte{nil, []byte("hello 🌍\n"), []byte{0xc3, 0x28}, bytes.Repeat([]byte{'x'}, MaxBytes+1)} { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, value []byte) { + err := Validate(value) + if err == nil && (len(value) > MaxBytes || !utf8.Valid(value) || utf8.RuneCount(value) > MaxCharacters || whitespaceOnly(value)) { + t.Fatalf("accepted invalid input: bytes=%d", len(value)) + } + }) +} + type failingReader struct{ err error } func (r failingReader) Read([]byte) (int, error) { return 0, r.err } + +type dataErrorReader struct { + data []byte + err error +} + +func (r *dataErrorReader) Read(destination []byte) (int, error) { + if len(r.data) == 0 { + return 0, r.err + } + n := copy(destination, r.data) + r.data = r.data[n:] + if len(r.data) == 0 { + return n, r.err + } + return n, nil +} From 4d94e5b8ee568453fa0a16eb8469ac560a52389f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 00:02:38 +0300 Subject: [PATCH 052/119] feat: add offline stage lifecycle --- internal/stagestore/doc.go | 7 + internal/stagestore/domain.go | 837 +++++++++++++++++++++++++++++ internal/stagestore/domain_test.go | 470 ++++++++++++++++ internal/stagestore/schema.go | 20 + internal/stagestore/store_test.go | 6 +- 5 files changed, 1337 insertions(+), 3 deletions(-) create mode 100644 internal/stagestore/doc.go create mode 100644 internal/stagestore/domain.go create mode 100644 internal/stagestore/domain_test.go diff --git a/internal/stagestore/doc.go b/internal/stagestore/doc.go new file mode 100644 index 0000000..f1b73b6 --- /dev/null +++ b/internal/stagestore/doc.go @@ -0,0 +1,7 @@ +// Package stagestore persists already validated, normalized mutation plans. +// +// It is intentionally not the active-credential validation boundary. CreateInput +// and ReviseInput are persistence records for the future staging service, which +// must construct them only after credential scanning and target resolution. CLI +// and transport packages must not call Store mutation methods directly. +package stagestore diff --git a/internal/stagestore/domain.go b/internal/stagestore/domain.go new file mode 100644 index 0000000..718e67d --- /dev/null +++ b/internal/stagestore/domain.go @@ -0,0 +1,837 @@ +package stagestore + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "database/sql" + "encoding/base64" + "encoding/json" + "errors" + "io" + "strings" + "sync" + "time" + "unicode" + "unicode/utf8" + + "github.com/ardasevinc/mattermost-cli/internal/messageinput" + "github.com/ardasevinc/mattermost-cli/internal/serverurl" +) + +const ( + maxIdentityBytes = 4096 + maxJSONBytes = 1 << 20 + maxRequestID = 256 + maxListLimit = 100 + maxAttachments = 100 + maxFilenameBytes = 255 + maxMediaTypeBytes = 256 +) + +var ( + ErrConflict = errors.New("stage store: request conflict") + ErrNotFound = errors.New("stage store: stage not found") + ErrNotEligible = errors.New("stage store: lifecycle transition not allowed") + ErrInvalid = errors.New("stage store: invalid stage input") + commitHook struct { + sync.RWMutex + fn func() + } +) + +type Operation string + +const ( + CreatePost Operation = "create_post" + Reply Operation = "reply" + EditPost Operation = "edit_post" + DeletePost Operation = "delete_post" + React Operation = "react" + Unreact Operation = "unreact" + ResolveDM Operation = "resolve_dm" + ResolveGroupDM Operation = "resolve_group_dm" +) + +type Lifecycle string +type Recovery string + +const ( + LifecycleOpen Lifecycle = "open" + LifecycleApplying Lifecycle = "applying" + LifecycleCompleted Lifecycle = "completed" + LifecycleCanceled Lifecycle = "canceled" + LifecycleExpired Lifecycle = "expired" + LifecyclePruned Lifecycle = "pruned" + RecoveryNone Recovery = "none" + RecoveryPartial Recovery = "resume_partial" + RecoveryUnknown Recovery = "force_unknown" + RecoveryForbidden Recovery = "forbidden" +) + +type Attachment struct { + SuppliedPath string `json:"suppliedPath"` + CanonicalPath string `json:"canonicalPath"` + RemoteFilename string `json:"remoteFilename"` + ByteLength int64 `json:"byteLength"` + MediaType string `json:"mediaType,omitempty"` + ContentDigest [32]byte `json:"contentDigest"` +} +type RevisionContent struct { + Body []byte + Destination json.RawMessage + Plan json.RawMessage + Attachments []Attachment +} +type Composition struct { + Body []byte `json:"body,omitempty"` + Attachments []Attachment `json:"attachments,omitempty"` +} +type CreateInput struct { + RequestID string + Operation Operation + ServerURL, ServerID, UserID string + Content RevisionContent +} +type ReviseInput struct { + StageID, RequestID string + ExpectedRevision int64 + ExpectedDigest [32]byte + Revive bool + Composition Composition +} +type CancelInput struct { + StageID, RequestID string + ExpectedRevision int64 + ExpectedDigest [32]byte +} + +type StageSummary struct { + ID string `json:"id"` + ServerURL string `json:"serverUrl"` + ServerID string `json:"serverId,omitempty"` + UserID string `json:"userId"` + Operation Operation `json:"operation"` + Lifecycle Lifecycle `json:"lifecycle"` + Recovery Recovery `json:"recovery"` + Revision int64 `json:"revision"` + SemanticDigest [32]byte `json:"semanticDigest"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} +type StageDetail struct { + StageSummary + RevisionCreatedAt time.Time + Body []byte + Destination, Plan json.RawMessage + Attachments []Attachment +} +type MutationResult struct { + Schema string `json:"schema"` + Action string `json:"action"` + Stage StageSummary `json:"stage"` + Revived bool `json:"revived"` + RecordedAt time.Time `json:"recordedAt"` + Replay bool `json:"-"` +} +type ListOptions struct{ Limit int } + +func (s *Store) Create(ctx context.Context, in CreateInput) (MutationResult, error) { + if err := ctx.Err(); err != nil { + return MutationResult{}, err + } + content, err := normalizeContent(in.Operation, in.Content) + if err != nil || !validOperation(in.Operation) || !canonicalServerURL(in.ServerURL) || !bounded(in.UserID, maxIdentityBytes) || (in.ServerID != "" && !bounded(in.ServerID, maxIdentityBytes)) || !validRequestID(in.RequestID) { + return MutationResult{}, ErrInvalid + } + semantic := semanticDigest(in.Operation, in.ServerURL, in.ServerID, in.UserID, content) + requestDigest := digestValue(struct { + Operation Operation `json:"operation"` + Semantic [32]byte `json:"semanticDigest"` + }{in.Operation, semantic}) + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return MutationResult{}, localError(err) + } + defer tx.Rollback() + if in.RequestID != "" { + if result, found, e := loadReplay(ctx, tx, in.ServerURL, in.UserID, in.RequestID, "mm/v2/stage-request", requestDigest); e != nil { + return MutationResult{}, e + } else if found { + return result, nil + } + } + id, err := newStageID() + if err != nil { + return MutationResult{}, errors.New("stage store: random identity unavailable") + } + now := time.Now().UTC() + stamp := formatTime(now) + if _, err = tx.ExecContext(ctx, `INSERT INTO stages(id,created_at,updated_at,operation,server_url,server_id,user_id,lifecycle,recovery,current_revision) VALUES(?,?,?,?,?,?,?,?,?,1)`, id, stamp, stamp, in.Operation, in.ServerURL, nullable(in.ServerID), in.UserID, LifecycleOpen, RecoveryNone); err != nil { + return MutationResult{}, localError(err) + } + if _, err = tx.ExecContext(ctx, `INSERT INTO stage_revisions(stage_id,revision,state,created_at,semantic_digest,body,destination_json,plan_json) VALUES(?,1,'current',?,?,?,?,?)`, id, stamp, semantic[:], nullableBytes(content.Body), string(content.Destination), string(content.Plan)); err != nil { + return MutationResult{}, localError(err) + } + if err = insertAttachments(ctx, tx, id, 1, content.Attachments); err != nil { + return MutationResult{}, err + } + summary := StageSummary{id, in.ServerURL, in.ServerID, in.UserID, in.Operation, LifecycleOpen, RecoveryNone, 1, semantic, now, now} + result := MutationResult{"mm/v2/stage-mutation-receipt", "create", summary, false, now, false} + if err = persistReplay(ctx, tx, in.ServerURL, in.UserID, in.RequestID, "mm/v2/stage-request", requestDigest, result, stamp); err != nil { + return MutationResult{}, err + } + if err = tx.Commit(); err != nil { + return MutationResult{}, localError(err) + } + runCommitHook() + return result, nil +} + +func (s *Store) Revise(ctx context.Context, in ReviseInput) (MutationResult, error) { + if err := ctx.Err(); err != nil { + return MutationResult{}, err + } + if !bounded(in.StageID, maxIdentityBytes) || in.ExpectedRevision < 1 || !validRequestID(in.RequestID) { + return MutationResult{}, ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return MutationResult{}, localError(err) + } + defer tx.Rollback() + base, err := scanCurrent(ctx, tx, in.StageID) + if err != nil { + return MutationResult{}, err + } + composition, err := normalizeComposition(base.Operation, in.Composition) + if err != nil { + return MutationResult{}, ErrInvalid + } + content := RevisionContent{composition.Body, bytes.Clone(base.Destination), bytes.Clone(base.Plan), composition.Attachments} + requestDigest := digestValue(struct { + Action, StageID string + ExpectedRevision int64 + ExpectedDigest [32]byte + Revive bool + Composition Composition + }{"revise", in.StageID, in.ExpectedRevision, in.ExpectedDigest, in.Revive, composition}) + if in.RequestID != "" { + if result, found, e := loadReplay(ctx, tx, base.ServerURL, base.UserID, in.RequestID, "mm/v2/stage-revise-request", requestDigest); e != nil { + return MutationResult{}, e + } else if found { + return result, nil + } + } + if base.Revision != in.ExpectedRevision || base.SemanticDigest != in.ExpectedDigest { + return MutationResult{}, ErrConflict + } + recovery := base.Recovery + if in.Revive { + if base.Lifecycle != LifecycleExpired || base.Recovery != RecoveryForbidden { + return MutationResult{}, ErrNotEligible + } + recovery = RecoveryNone + } else if base.Lifecycle != LifecycleOpen || base.Recovery == RecoveryForbidden { + return MutationResult{}, ErrNotEligible + } + next := base.Revision + 1 + semantic := semanticDigest(base.Operation, base.ServerURL, base.ServerID, base.UserID, content) + now := time.Now().UTC() + stamp := formatTime(now) + resultSQL, err := tx.ExecContext(ctx, `UPDATE stage_revisions SET state='superseded' WHERE stage_id=? AND revision=? AND state='current' AND semantic_digest=?`, in.StageID, in.ExpectedRevision, in.ExpectedDigest[:]) + if err != nil { + return MutationResult{}, localError(err) + } + if !oneRow(resultSQL) { + return MutationResult{}, ErrConflict + } + if _, err = tx.ExecContext(ctx, `INSERT INTO stage_revisions(stage_id,revision,state,created_at,semantic_digest,body,destination_json,plan_json) VALUES(?,?,'current',?,?,?,?,?)`, in.StageID, next, stamp, semantic[:], nullableBytes(content.Body), string(content.Destination), string(content.Plan)); err != nil { + return MutationResult{}, localError(err) + } + if err = insertAttachments(ctx, tx, in.StageID, next, content.Attachments); err != nil { + return MutationResult{}, err + } + resultSQL, err = tx.ExecContext(ctx, `UPDATE stages SET updated_at=?,lifecycle='open',recovery=?,current_revision=? WHERE id=? AND current_revision=? AND lifecycle=? AND recovery=?`, stamp, recovery, next, in.StageID, in.ExpectedRevision, base.Lifecycle, base.Recovery) + if err != nil { + return MutationResult{}, localError(err) + } + if !oneRow(resultSQL) { + return MutationResult{}, ErrConflict + } + summary := StageSummary{in.StageID, base.ServerURL, base.ServerID, base.UserID, base.Operation, LifecycleOpen, recovery, next, semantic, base.CreatedAt, now} + result := MutationResult{"mm/v2/stage-mutation-receipt", "revise", summary, in.Revive, now, false} + if err = persistReplay(ctx, tx, base.ServerURL, base.UserID, in.RequestID, "mm/v2/stage-revise-request", requestDigest, result, stamp); err != nil { + return MutationResult{}, err + } + if err = tx.Commit(); err != nil { + return MutationResult{}, localError(err) + } + runCommitHook() + return result, nil +} + +func (s *Store) Cancel(ctx context.Context, in CancelInput) (MutationResult, error) { + if err := ctx.Err(); err != nil { + return MutationResult{}, err + } + if !bounded(in.StageID, maxIdentityBytes) || in.ExpectedRevision < 1 || !validRequestID(in.RequestID) { + return MutationResult{}, ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return MutationResult{}, localError(err) + } + defer tx.Rollback() + base, err := scanCurrent(ctx, tx, in.StageID) + if err != nil { + return MutationResult{}, err + } + digest := digestValue(struct { + Action, StageID string + ExpectedRevision int64 + ExpectedDigest [32]byte + }{"cancel", in.StageID, in.ExpectedRevision, in.ExpectedDigest}) + if in.RequestID != "" { + if result, found, e := loadReplay(ctx, tx, base.ServerURL, base.UserID, in.RequestID, "mm/v2/stage-cancel-request", digest); e != nil { + return MutationResult{}, e + } else if found { + return result, nil + } + } + if base.Revision != in.ExpectedRevision || base.SemanticDigest != in.ExpectedDigest { + return MutationResult{}, ErrConflict + } + if base.Lifecycle != LifecycleOpen { + return MutationResult{}, ErrNotEligible + } + now := time.Now().UTC() + stamp := formatTime(now) + resultSQL, err := tx.ExecContext(ctx, `UPDATE stages SET lifecycle='canceled',recovery='forbidden',updated_at=? WHERE id=? AND current_revision=? AND lifecycle='open' AND EXISTS(SELECT 1 FROM stage_revisions WHERE stage_id=? AND revision=? AND state='current' AND semantic_digest=?)`, stamp, in.StageID, in.ExpectedRevision, in.StageID, in.ExpectedRevision, in.ExpectedDigest[:]) + if err != nil { + return MutationResult{}, localError(err) + } + if !oneRow(resultSQL) { + return MutationResult{}, ErrConflict + } + summary := base.StageSummary + summary.Lifecycle = LifecycleCanceled + summary.Recovery = RecoveryForbidden + summary.UpdatedAt = now + result := MutationResult{"mm/v2/stage-mutation-receipt", "cancel", summary, false, now, false} + if err = persistReplay(ctx, tx, base.ServerURL, base.UserID, in.RequestID, "mm/v2/stage-cancel-request", digest, result, stamp); err != nil { + return MutationResult{}, err + } + if err = tx.Commit(); err != nil { + return MutationResult{}, localError(err) + } + runCommitHook() + return result, nil +} + +func (s *Store) Show(ctx context.Context, id string) (StageDetail, error) { + if !bounded(id, maxIdentityBytes) { + return StageDetail{}, ErrInvalid + } + detail, err := scanDetail(s.db.QueryRowContext(ctx, currentDetailSQL, id)) + if err != nil { + return detail, err + } + detail.Attachments, err = readAttachments(ctx, s.db, id, detail.Revision) + return detail, err +} +func (s *Store) List(ctx context.Context, o ListOptions) ([]StageSummary, error) { + limit := o.Limit + if limit == 0 { + limit = 50 + } + if limit < 1 || limit > maxListLimit { + return nil, ErrInvalid + } + rows, err := s.db.QueryContext(ctx, `SELECT s.id,s.server_url,coalesce(s.server_id,''),s.user_id,s.operation,s.lifecycle,s.recovery,r.revision,r.semantic_digest,s.created_at,s.updated_at FROM stages s JOIN stage_revisions r ON r.stage_id=s.id AND r.revision=s.current_revision ORDER BY s.updated_at DESC,s.id ASC LIMIT ?`, limit) + if err != nil { + return nil, localError(err) + } + defer rows.Close() + out := make([]StageSummary, 0) + for rows.Next() { + v, e := scanSummary(rows) + if e != nil { + return nil, e + } + out = append(out, v) + } + if err = rows.Err(); err != nil { + return nil, localError(err) + } + return out, nil +} + +const currentDetailSQL = `SELECT s.id,s.server_url,coalesce(s.server_id,''),s.user_id,s.operation,s.lifecycle,s.recovery,r.revision,r.semantic_digest,s.created_at,s.updated_at,r.created_at,r.body,r.destination_json,r.plan_json FROM stages s JOIN stage_revisions r ON r.stage_id=s.id AND r.revision=s.current_revision WHERE s.id=?` + +type rowScanner interface{ Scan(...any) error } + +func scanSummary(row rowScanner) (StageSummary, error) { + var v StageSummary + var digest []byte + var created, updated string + err := row.Scan(&v.ID, &v.ServerURL, &v.ServerID, &v.UserID, &v.Operation, &v.Lifecycle, &v.Recovery, &v.Revision, &digest, &created, &updated) + if errors.Is(err, sql.ErrNoRows) { + return v, ErrNotFound + } + if err != nil { + return v, localError(err) + } + if len(digest) != 32 { + return v, localError(errors.New("digest")) + } + copy(v.SemanticDigest[:], digest) + if v.CreatedAt, err = parseTime(created); err != nil { + return v, err + } + if v.UpdatedAt, err = parseTime(updated); err != nil { + return v, err + } + return v, nil +} +func scanDetail(row rowScanner) (StageDetail, error) { + var v StageDetail + var digest, body []byte + var created, updated, revCreated, destination, plan string + err := row.Scan(&v.ID, &v.ServerURL, &v.ServerID, &v.UserID, &v.Operation, &v.Lifecycle, &v.Recovery, &v.Revision, &digest, &created, &updated, &revCreated, &body, &destination, &plan) + if errors.Is(err, sql.ErrNoRows) { + return v, ErrNotFound + } + if err != nil { + return v, localError(err) + } + if len(digest) != 32 { + return v, localError(errors.New("digest")) + } + copy(v.SemanticDigest[:], digest) + if v.CreatedAt, err = parseTime(created); err != nil { + return v, err + } + if v.UpdatedAt, err = parseTime(updated); err != nil { + return v, err + } + if v.RevisionCreatedAt, err = parseTime(revCreated); err != nil { + return v, err + } + v.Body = bytes.Clone(body) + v.Destination = json.RawMessage(destination) + v.Plan = json.RawMessage(plan) + return v, nil +} +func scanCurrent(ctx context.Context, tx *sql.Tx, id string) (StageDetail, error) { + return scanDetail(tx.QueryRowContext(ctx, currentDetailSQL, id)) +} + +type semanticContent struct { + Body []byte `json:"body,omitempty"` + Destination json.RawMessage `json:"destination"` + Plan json.RawMessage `json:"plan"` + Attachments []Attachment `json:"attachments,omitempty"` +} + +func (v RevisionContent) semantic() semanticContent { + return semanticContent{v.Body, v.Destination, v.Plan, v.Attachments} +} +func semanticDigest(op Operation, server, serverID, user string, c RevisionContent) [32]byte { + return digestValue(struct { + Operation Operation `json:"operation"` + ServerURL string `json:"serverUrl"` + ServerID string `json:"serverId,omitempty"` + UserID string `json:"userId"` + Content semanticContent `json:"content"` + }{op, server, serverID, user, c.semantic()}) +} +func normalizeContent(op Operation, v RevisionContent) (RevisionContent, error) { + destination, err := canonicalObject(v.Destination) + if err != nil { + return v, err + } + plan, err := canonicalObject(v.Plan) + if err != nil { + return v, err + } + v.Destination, v.Plan = destination, plan + composition, err := normalizeComposition(op, Composition{v.Body, v.Attachments}) + if err != nil { + return v, err + } + v.Body, v.Attachments = composition.Body, composition.Attachments + return v, nil +} +func normalizeComposition(op Operation, v Composition) (Composition, error) { + v.Body = bytes.Clone(v.Body) + v.Attachments = append([]Attachment(nil), v.Attachments...) + if len(v.Attachments) > maxAttachments { + return v, ErrInvalid + } + switch op { + case CreatePost, Reply, EditPost: + if err := messageinput.Validate(v.Body); err != nil { + return v, ErrInvalid + } + default: + if len(v.Body) != 0 { + return v, ErrInvalid + } + } + if op != CreatePost && op != Reply && len(v.Attachments) > 0 { + return v, ErrInvalid + } + for _, a := range v.Attachments { + if !boundedMetadata(a.SuppliedPath, maxIdentityBytes) || !boundedMetadata(a.CanonicalPath, maxIdentityBytes) || !boundedMetadata(a.RemoteFilename, maxFilenameBytes) || a.ByteLength < 0 || a.ContentDigest == ([32]byte{}) || (a.MediaType != "" && !boundedMetadata(a.MediaType, maxMediaTypeBytes)) { + return v, ErrInvalid + } + } + return v, nil +} + +func canonicalObject(raw []byte) (json.RawMessage, error) { + if len(raw) == 0 || len(raw) > maxJSONBytes || !utf8.Valid(raw) || !validJSONStringEscapes(raw) { + return nil, ErrInvalid + } + d := json.NewDecoder(bytes.NewReader(raw)) + d.UseNumber() + value, err := decodeUnique(d) + if err != nil { + return nil, ErrInvalid + } + if _, ok := value.(map[string]any); !ok { + return nil, ErrInvalid + } + if err = d.Decode(new(any)); !errors.Is(err, io.EOF) { + return nil, ErrInvalid + } + var out bytes.Buffer + e := json.NewEncoder(&out) + e.SetEscapeHTML(false) + if e.Encode(value) != nil { + return nil, ErrInvalid + } + return restoreLineSeparators(bytes.TrimSuffix(out.Bytes(), []byte("\n"))), nil +} +func decodeUnique(d *json.Decoder) (any, error) { + token, err := d.Token() + if err != nil { + return nil, err + } + switch token := token.(type) { + case json.Delim: + switch token { + case '{': + m := map[string]any{} + for d.More() { + keyToken, e := d.Token() + if e != nil { + return nil, e + } + key, ok := keyToken.(string) + if !ok { + return nil, ErrInvalid + } + if _, exists := m[key]; exists { + return nil, ErrInvalid + } + value, e := decodeUnique(d) + if e != nil { + return nil, e + } + m[key] = value + } + _, err = d.Token() + return m, err + case '[': + a := []any{} + for d.More() { + v, e := decodeUnique(d) + if e != nil { + return nil, e + } + a = append(a, v) + } + _, err = d.Token() + return a, err + default: + return nil, ErrInvalid + } + default: + return token, nil + } +} +func validJSONStringEscapes(raw []byte) bool { + in := false + for i := 0; i < len(raw); i++ { + if !in { + if raw[i] == '"' { + in = true + } + continue + } + if raw[i] == '"' { + in = false + continue + } + if raw[i] != '\\' { + continue + } + i++ + if i >= len(raw) { + return false + } + if raw[i] != 'u' { + continue + } + if i+4 >= len(raw) { + return false + } + code, ok := hex4(raw[i+1 : i+5]) + if !ok { + return false + } + i += 4 + if code >= 0xD800 && code <= 0xDBFF { + if i+6 >= len(raw) || raw[i+1] != '\\' || raw[i+2] != 'u' { + return false + } + low, ok := hex4(raw[i+3 : i+7]) + if !ok || low < 0xDC00 || low > 0xDFFF { + return false + } + i += 6 + } else if code >= 0xDC00 && code <= 0xDFFF { + return false + } + } + return !in +} +func hex4(v []byte) (uint16, bool) { + var out uint16 + for _, c := range v { + out <<= 4 + switch { + case c >= '0' && c <= '9': + out += uint16(c - '0') + case c >= 'a' && c <= 'f': + out += uint16(c - 'a' + 10) + case c >= 'A' && c <= 'F': + out += uint16(c - 'A' + 10) + default: + return 0, false + } + } + return out, true +} + +func insertAttachments(ctx context.Context, tx *sql.Tx, stage string, revision int64, values []Attachment) error { + for i, a := range values { + if _, err := tx.ExecContext(ctx, `INSERT INTO stage_attachments(stage_id,revision,ordinal,supplied_path,canonical_path,remote_filename,byte_length,media_type,content_digest) VALUES(?,?,?,?,?,?,?,?,?)`, stage, revision, i, a.SuppliedPath, a.CanonicalPath, a.RemoteFilename, a.ByteLength, nullable(a.MediaType), a.ContentDigest[:]); err != nil { + return localError(err) + } + } + return nil +} + +type queryer interface { + QueryContext(context.Context, string, ...any) (*sql.Rows, error) +} + +func readAttachments(ctx context.Context, q queryer, stage string, revision int64) ([]Attachment, error) { + rows, err := q.QueryContext(ctx, `SELECT supplied_path,canonical_path,remote_filename,byte_length,coalesce(media_type,''),content_digest FROM stage_attachments WHERE stage_id=? AND revision=? ORDER BY ordinal`, stage, revision) + if err != nil { + return nil, localError(err) + } + defer rows.Close() + out := make([]Attachment, 0) + for rows.Next() { + var a Attachment + var digest []byte + if err = rows.Scan(&a.SuppliedPath, &a.CanonicalPath, &a.RemoteFilename, &a.ByteLength, &a.MediaType, &digest); err != nil { + return nil, localError(err) + } + if len(digest) != 32 { + return nil, localError(errors.New("digest")) + } + copy(a.ContentDigest[:], digest) + out = append(out, a) + } + return out, localError(rows.Err()) +} + +func loadReplay(ctx context.Context, tx *sql.Tx, server, user, id, schema string, digest [32]byte) (MutationResult, bool, error) { + var storedSchema, raw, created string + var stored []byte + err := tx.QueryRowContext(ctx, `SELECT request_schema,request_digest,result_json,created_at FROM local_requests WHERE server_url=? AND user_id=? AND request_id=?`, server, user, id).Scan(&storedSchema, &stored, &raw, &created) + if errors.Is(err, sql.ErrNoRows) { + return MutationResult{}, false, nil + } + if err != nil { + return MutationResult{}, false, localError(err) + } + if storedSchema != schema || !bytes.Equal(stored, digest[:]) { + return MutationResult{}, false, ErrConflict + } + var result MutationResult + decoder := json.NewDecoder(strings.NewReader(raw)) + decoder.DisallowUnknownFields() + recordedAt, timeErr := parseTime(created) + if decoder.Decode(&result) != nil || decoder.Decode(new(any)) != io.EOF || timeErr != nil || !result.RecordedAt.Equal(recordedAt) || !validReplayResult(result, schema, server, user) { + return MutationResult{}, false, localError(errors.New("receipt")) + } + result.Replay = true + return result, true, nil +} +func validReplayResult(result MutationResult, requestSchema, server, user string) bool { + action := map[string]string{"mm/v2/stage-request": "create", "mm/v2/stage-revise-request": "revise", "mm/v2/stage-cancel-request": "cancel"}[requestSchema] + stage := result.Stage + return action != "" && result.Schema == "mm/v2/stage-mutation-receipt" && result.Action == action && stage.ServerURL == server && stage.UserID == user && + bounded(stage.ID, maxIdentityBytes) && validOperation(stage.Operation) && stage.Revision > 0 && stage.SemanticDigest != ([32]byte{}) && validLifecycle(stage.Lifecycle) && validRecovery(stage.Recovery) && + !stage.CreatedAt.IsZero() && !stage.UpdatedAt.IsZero() && !result.RecordedAt.IsZero() && !stage.UpdatedAt.Before(stage.CreatedAt) && !result.RecordedAt.Before(stage.UpdatedAt) +} +func validLifecycle(v Lifecycle) bool { + switch v { + case LifecycleOpen, LifecycleApplying, LifecycleCompleted, LifecycleCanceled, LifecycleExpired, LifecyclePruned: + return true + } + return false +} +func validRecovery(v Recovery) bool { + switch v { + case RecoveryNone, RecoveryPartial, RecoveryUnknown, RecoveryForbidden: + return true + } + return false +} +func persistReplay(ctx context.Context, tx *sql.Tx, server, user, id, schema string, digest [32]byte, result MutationResult, stamp string) error { + if id == "" { + return nil + } + raw, err := marshalCanonical(result) + if err != nil { + return localError(err) + } + _, err = tx.ExecContext(ctx, `INSERT INTO local_requests(server_url,user_id,request_id,request_schema,request_digest,result_json,created_at) VALUES(?,?,?,?,?,?,?)`, server, user, id, schema, digest[:], string(raw), stamp) + return localError(err) +} +func digestValue(v any) [32]byte { raw, _ := marshalCanonical(v); return sha256.Sum256(raw) } +func marshalCanonical(v any) ([]byte, error) { + var out bytes.Buffer + encoder := json.NewEncoder(&out) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(v); err != nil { + return nil, err + } + return restoreLineSeparators(bytes.TrimSuffix(out.Bytes(), []byte("\n"))), nil +} +func restoreLineSeparators(data []byte) []byte { + var out bytes.Buffer + out.Grow(len(data)) + for i := 0; i < len(data); { + if i+6 <= len(data) && data[i] == '\\' && (bytes.Equal(data[i:i+6], []byte(`\u2028`)) || bytes.Equal(data[i:i+6], []byte(`\u2029`))) { + slashes := 0 + for cursor := i - 1; cursor >= 0 && data[cursor] == '\\'; cursor-- { + slashes++ + } + if slashes%2 == 0 { + if data[i+5] == '8' { + out.WriteString("\u2028") + } else { + out.WriteString("\u2029") + } + i += 6 + continue + } + } + out.WriteByte(data[i]) + i++ + } + return out.Bytes() +} +func newStageID() (string, error) { + v := make([]byte, 24) + if _, err := rand.Read(v); err != nil { + return "", err + } + return "stg_" + base64.RawURLEncoding.EncodeToString(v), nil +} +func validOperation(v Operation) bool { + switch v { + case CreatePost, Reply, EditPost, DeletePost, React, Unreact, ResolveDM, ResolveGroupDM: + return true + } + return false +} +func bounded(v string, max int) bool { + return v != "" && len(v) <= max && utf8.ValidString(v) && strings.TrimSpace(v) == v +} +func boundedMetadata(v string, max int) bool { + if !bounded(v, max) { + return false + } + for _, r := range v { + if unicode.IsControl(r) { + return false + } + } + return true +} +func validRequestID(v string) bool { + if v == "" { + return true + } + if len(v) > maxRequestID || !requestChar(v[0], true) { + return false + } + for i := 1; i < len(v); i++ { + if !requestChar(v[i], false) { + return false + } + } + return true +} +func requestChar(c byte, first bool) bool { + if c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= '0' && c <= '9' { + return true + } + return !first && strings.ContainsRune("._~:-", rune(c)) +} +func canonicalServerURL(v string) bool { + if !bounded(v, maxIdentityBytes) { + return false + } + normalized, err := serverurl.Normalize(v) + return err == nil && normalized == v +} +func nullable(v string) any { + if v == "" { + return nil + } + return v +} +func nullableBytes(v []byte) any { + if v == nil { + return nil + } + return v +} +func parseTime(v string) (time.Time, error) { + t, err := time.Parse(time.RFC3339Nano, v) + if err != nil { + return t, localError(err) + } + return t, nil +} +func formatTime(v time.Time) string { return v.UTC().Format("2006-01-02T15:04:05.000000000Z") } +func oneRow(v sql.Result) bool { n, err := v.RowsAffected(); return err == nil && n == 1 } +func runCommitHook() { + commitHook.RLock() + fn := commitHook.fn + commitHook.RUnlock() + if fn != nil { + fn() + } +} diff --git a/internal/stagestore/domain_test.go b/internal/stagestore/domain_test.go new file mode 100644 index 0000000..9f0169c --- /dev/null +++ b/internal/stagestore/domain_test.go @@ -0,0 +1,470 @@ +//go:build darwin || linux + +package stagestore + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "reflect" + "sync" + "testing" + "time" +) + +func openDomainStore(t *testing.T) *Store { + t.Helper() + s, err := Open(context.Background(), testPath(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + return s +} +func attachment(name string) Attachment { + return Attachment{"/tmp/" + name, "/private/tmp/" + name, name, 3, "text/plain", sha256.Sum256([]byte(name))} +} +func createInput(request, body string) CreateInput { + return CreateInput{request, CreatePost, "https://mattermost.example/api/v4", "server-1", "user-1", RevisionContent{[]byte(body), json.RawMessage(`{"kind":"O","channelId":"channel-1"}`), json.RawMessage(`{"steps":[{"kind":"create_post"}]}`), []Attachment{attachment("a.txt"), attachment("b.txt")}}} +} +func reviseInput(stage StageSummary, request, body string) ReviseInput { + content := createInput("", body).Content + return ReviseInput{stage.ID, request, stage.Revision, stage.SemanticDigest, false, Composition{content.Body, content.Attachments}} +} + +func TestRevisePreservesImmutableDestinationAndPlan(t *testing.T) { + s := openDomainStore(t) + input := createInput("", "one") + input.Content.Destination = json.RawMessage(`{ "binding": {"postId":"post-1"}, "channelId":"channel-1" }`) + input.Content.Plan = json.RawMessage(`{ "steps": [{"kind":"edit","postId":"post-1"}], "targetVersion":7 }`) + created, err := s.Create(context.Background(), input) + if err != nil { + t.Fatal(err) + } + before, err := s.Show(context.Background(), created.Stage.ID) + if err != nil { + t.Fatal(err) + } + revised, err := s.Revise(context.Background(), reviseInput(created.Stage, "revise-binding", "two")) + if err != nil { + t.Fatal(err) + } + after, err := s.Show(context.Background(), revised.Stage.ID) + if err != nil { + t.Fatal(err) + } + if string(after.Destination) != string(before.Destination) || string(after.Plan) != string(before.Plan) { + t.Fatalf("binding changed: destination %s -> %s, plan %s -> %s", before.Destination, after.Destination, before.Plan, after.Plan) + } +} + +func TestCreateShowListPrivacyAttachmentsAndCanonicalDigest(t *testing.T) { + s := openDomainStore(t) + created, err := s.Create(context.Background(), createInput("create-1", "private body\n")) + if err != nil { + t.Fatal(err) + } + if created.Action != "create" || created.Replay || created.Stage.Revision != 1 { + t.Fatalf("receipt=%#v", created) + } + if _, ok := reflect.TypeOf(MutationResult{}).FieldByName("Body"); ok { + t.Fatal("receipt can carry body") + } + if _, ok := reflect.TypeOf(StageSummary{}).FieldByName("Body"); ok { + t.Fatal("summary can carry body") + } + shown, err := s.Show(context.Background(), created.Stage.ID) + if err != nil { + t.Fatal(err) + } + if string(shown.Body) != "private body\n" || string(shown.Destination) != `{"channelId":"channel-1","kind":"O"}` || len(shown.Attachments) != 2 || shown.Attachments[0].RemoteFilename != "a.txt" || shown.Attachments[1].RemoteFilename != "b.txt" { + t.Fatalf("shown=%#v", shown) + } + list, err := s.List(context.Background(), ListOptions{}) + if err != nil || len(list) != 1 || list[0].SemanticDigest != shown.SemanticDigest { + t.Fatalf("list=%#v err=%v", list, err) + } + reordered := createInput("", "private body\n") + reordered.Content.Destination = json.RawMessage(`{"channelId":"channel-1","kind":"O"}`) + other, err := s.Create(context.Background(), reordered) + if err != nil || other.Stage.SemanticDigest != created.Stage.SemanticDigest { + t.Fatalf("canonical digest differs: %x %x err=%v", other.Stage.SemanticDigest, created.Stage.SemanticDigest, err) + } + third, err := s.Create(context.Background(), createInput("", "newer")) + if err != nil { + t.Fatal(err) + } + const tied = "2026-01-01T00:00:00.000000000Z" + if _, err = s.db.Exec(`UPDATE stages SET updated_at=? WHERE id IN (?,?)`, tied, created.Stage.ID, third.Stage.ID); err != nil { + t.Fatal(err) + } + list, err = s.List(context.Background(), ListOptions{}) + if err != nil { + t.Fatal(err) + } + var tiedIDs []string + for _, item := range list { + if item.ID == created.Stage.ID || item.ID == third.Stage.ID { + tiedIDs = append(tiedIDs, item.ID) + } + } + if len(tiedIDs) != 2 || tiedIDs[0] > tiedIDs[1] { + t.Fatalf("nondeterministic tie order: %v", tiedIDs) + } +} + +func TestImmutableReplaySnapshots(t *testing.T) { + s := openDomainStore(t) + first, err := s.Create(context.Background(), createInput("same.request:1", "one")) + if err != nil { + t.Fatal(err) + } + revised, err := s.Revise(context.Background(), reviseInput(first.Stage, "revise-1", "two")) + if err != nil { + t.Fatal(err) + } + replay, err := s.Create(context.Background(), createInput("same.request:1", "one")) + if err != nil || !replay.Replay || replay.Stage.Revision != 1 || replay.Stage.Lifecycle != LifecycleOpen || replay.RecordedAt != first.RecordedAt { + t.Fatalf("replay=%#v err=%v", replay, err) + } + if _, err = s.Create(context.Background(), createInput("same.request:1", "changed")); !errors.Is(err, ErrConflict) { + t.Fatalf("conflict=%v", err) + } + cancel, err := s.Cancel(context.Background(), CancelInput{revised.Stage.ID, "cancel-1", revised.Stage.Revision, revised.Stage.SemanticDigest}) + if err != nil { + t.Fatal(err) + } + cancelReplay, err := s.Cancel(context.Background(), CancelInput{revised.Stage.ID, "cancel-1", revised.Stage.Revision, revised.Stage.SemanticDigest}) + if err != nil || !cancelReplay.Replay || cancelReplay.Stage.Lifecycle != LifecycleCanceled || cancelReplay.RecordedAt != cancel.RecordedAt { + t.Fatalf("cancel replay=%#v err=%v", cancelReplay, err) + } + var count int + if err = s.db.QueryRow(`SELECT count(*) FROM local_requests`).Scan(&count); err != nil || count != 3 { + t.Fatalf("requests=%d err=%v", count, err) + } +} + +func TestReviewedStateCASAndApplying(t *testing.T) { + s := openDomainStore(t) + created, _ := s.Create(context.Background(), createInput("", "one")) + editorA := reviseInput(created.Stage, "a", "two") + editorB := reviseInput(created.Stage, "b", "three") + revised, err := s.Revise(context.Background(), editorA) + if err != nil { + t.Fatal(err) + } + if _, err = s.Revise(context.Background(), editorB); !errors.Is(err, ErrConflict) { + t.Fatalf("two-editor=%v", err) + } + staleCancel := CancelInput{revised.Stage.ID, "cancel-stale", 1, created.Stage.SemanticDigest} + if _, err = s.Cancel(context.Background(), staleCancel); !errors.Is(err, ErrConflict) { + t.Fatalf("stale cancel=%v", err) + } + if _, err = s.db.Exec(`UPDATE stages SET lifecycle='applying' WHERE id=?`, revised.Stage.ID); err != nil { + t.Fatal(err) + } + if _, err = s.Revise(context.Background(), reviseInput(revised.Stage, "applying", "four")); !errors.Is(err, ErrNotEligible) { + t.Fatalf("applying revise=%v", err) + } + if _, err = s.Cancel(context.Background(), CancelInput{revised.Stage.ID, "applying-cancel", revised.Stage.Revision, revised.Stage.SemanticDigest}); !errors.Is(err, ErrNotEligible) { + t.Fatalf("applying cancel=%v", err) + } +} + +func TestSimultaneousMutationCAS(t *testing.T) { + t.Run("revise revise", func(t *testing.T) { + s := openDomainStore(t) + created, _ := s.Create(context.Background(), createInput("", "one")) + start := make(chan struct{}) + results := make(chan error, 2) + var wg sync.WaitGroup + for _, in := range []ReviseInput{reviseInput(created.Stage, "race-a", "two"), reviseInput(created.Stage, "race-b", "three")} { + wg.Add(1) + go func(in ReviseInput) { + defer wg.Done() + <-start + _, err := s.Revise(context.Background(), in) + results <- err + }(in) + } + close(start) + wg.Wait() + close(results) + success, conflict := 0, 0 + for err := range results { + if err == nil { + success++ + } else if errors.Is(err, ErrConflict) { + conflict++ + } else { + t.Fatalf("unexpected error=%v", err) + } + } + if success != 1 || conflict != 1 { + t.Fatalf("success=%d conflict=%d", success, conflict) + } + }) + t.Run("revise cancel", func(t *testing.T) { + s := openDomainStore(t) + created, _ := s.Create(context.Background(), createInput("", "one")) + start := make(chan struct{}) + results := make(chan error, 2) + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + <-start + _, err := s.Revise(context.Background(), reviseInput(created.Stage, "race-revise", "two")) + results <- err + }() + go func() { + defer wg.Done() + <-start + _, err := s.Cancel(context.Background(), CancelInput{created.Stage.ID, "race-cancel", created.Stage.Revision, created.Stage.SemanticDigest}) + results <- err + }() + close(start) + wg.Wait() + close(results) + success, refused := 0, 0 + for err := range results { + if err == nil { + success++ + } else if errors.Is(err, ErrConflict) || errors.Is(err, ErrNotEligible) { + refused++ + } else { + t.Fatalf("unexpected error=%v", err) + } + } + if success != 1 || refused != 1 { + t.Fatalf("success=%d refused=%d", success, refused) + } + }) +} + +func TestReviveOnlyLegalExpiredForbidden(t *testing.T) { + s := openDomainStore(t) + created, _ := s.Create(context.Background(), createInput("", "one")) + if _, err := s.db.Exec(`UPDATE stages SET lifecycle='expired',recovery='forbidden' WHERE id=?`, created.Stage.ID); err != nil { + t.Fatal(err) + } + input := reviseInput(created.Stage, "revive", "two") + input.Revive = true + revived, err := s.Revise(context.Background(), input) + if err != nil || !revived.Revived || revived.Stage.Lifecycle != LifecycleOpen || revived.Stage.Recovery != RecoveryNone { + t.Fatalf("revived=%#v err=%v", revived, err) + } + other, _ := s.Create(context.Background(), createInput("", "x")) + if _, err := s.db.Exec(`UPDATE stages SET lifecycle='expired',recovery='force_unknown' WHERE id=?`, other.Stage.ID); err != nil { + t.Fatal(err) + } + illegal := reviseInput(other.Stage, "illegal", "y") + illegal.Revive = true + if _, err = s.Revise(context.Background(), illegal); !errors.Is(err, ErrNotEligible) { + t.Fatalf("illegal revive=%v", err) + } + partial, _ := s.Create(context.Background(), createInput("", "p")) + if _, err := s.db.Exec(`UPDATE stages SET recovery='resume_partial' WHERE id=?`, partial.Stage.ID); err != nil { + t.Fatal(err) + } + normal := reviseInput(partial.Stage, "partial", "q") + got, err := s.Revise(context.Background(), normal) + if err != nil || got.Stage.Recovery != RecoveryPartial { + t.Fatalf("monotonic=%#v err=%v", got, err) + } +} + +func TestOperationContentApplicability(t *testing.T) { + s := openDomainStore(t) + for name, test := range map[string]CreateInput{"empty create": createInput("", " \n"), "edit attachments": func() CreateInput { v := createInput("", "body"); v.Operation = EditPost; return v }(), "delete body": func() CreateInput { + v := createInput("", "body") + v.Operation = DeletePost + v.Content.Attachments = nil + return v + }()} { + t.Run(name, func(t *testing.T) { + if _, err := s.Create(context.Background(), test); !errors.Is(err, ErrInvalid) { + t.Fatalf("err=%v", err) + } + }) + } + validDelete := createInput("", "body") + validDelete.Operation = DeletePost + validDelete.Content.Body = nil + validDelete.Content.Attachments = nil + if _, err := s.Create(context.Background(), validDelete); err != nil { + t.Fatal(err) + } +} + +func TestStrictCanonicalObjects(t *testing.T) { + s := openDomainStore(t) + for name, raw := range map[string]string{"array": `[]`, "duplicate": `{"a":1,"a":2}`, "nested duplicate": `{"x":{"a":1,"a":2}}`, "high surrogate": `{"x":"\ud800"}`, "low surrogate": `{"x":"\udc00"}`, "invalid utf8": string([]byte{'{', '"', 'x', '"', ':', '"', 0xff, '"', '}'})} { + t.Run(name, func(t *testing.T) { + v := createInput("", "body") + v.Content.Destination = json.RawMessage(raw) + if _, err := s.Create(context.Background(), v); !errors.Is(err, ErrInvalid) { + t.Fatalf("err=%v", err) + } + }) + } + valid := createInput("", "body") + valid.Content.Destination = json.RawMessage(`{"emoji":"\ud83d\ude00"}`) + if _, err := s.Create(context.Background(), valid); err != nil { + t.Fatal(err) + } + literal := createInput("", "body") + literal.Content.Destination = json.RawMessage("{\"line\":\"\u2028\"}") + escaped := createInput("", "body") + escaped.Content.Destination = json.RawMessage(`{"line":"\u2028"}`) + one, err := s.Create(context.Background(), literal) + if err != nil { + t.Fatal(err) + } + two, err := s.Create(context.Background(), escaped) + if err != nil || one.Stage.SemanticDigest != two.Stage.SemanticDigest { + t.Fatalf("line separator canonicalization differs: %x %x err=%v", one.Stage.SemanticDigest, two.Stage.SemanticDigest, err) + } +} + +func TestBoundsRequestIDOrderingAndCommitCancellation(t *testing.T) { + s := openDomainStore(t) + bad := createInput("-bad", "body") + if _, err := s.Create(context.Background(), bad); !errors.Is(err, ErrInvalid) { + t.Fatalf("request id=%v", err) + } + tooMany := createInput("", "body") + tooMany.Content.Attachments = make([]Attachment, maxAttachments+1) + if _, err := s.Create(context.Background(), tooMany); !errors.Is(err, ErrInvalid) { + t.Fatalf("attachments=%v", err) + } + for name, mutate := range map[string]func(*CreateInput){ + "zero digest": func(v *CreateInput) { v.Content.Attachments[0].ContentDigest = [32]byte{} }, + "path control": func(v *CreateInput) { v.Content.Attachments[0].SuppliedPath = "/tmp/a\x00b" }, + "filename control": func(v *CreateInput) { v.Content.Attachments[0].RemoteFilename = "a\nb.txt" }, + } { + t.Run(name, func(t *testing.T) { + v := createInput("", "body") + mutate(&v) + if _, err := s.Create(context.Background(), v); !errors.Is(err, ErrInvalid) { + t.Fatalf("err=%v", err) + } + }) + } + if _, err := s.List(context.Background(), ListOptions{maxListLimit + 1}); !errors.Is(err, ErrInvalid) { + t.Fatalf("list=%v", err) + } + preCanceled, stop := context.WithCancel(context.Background()) + stop() + if _, err := s.Create(preCanceled, createInput("never", "body")); !errors.Is(err, context.Canceled) { + t.Fatalf("pre-canceled create=%v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + commitHook.Lock() + commitHook.fn = cancel + commitHook.Unlock() + t.Cleanup(func() { commitHook.Lock(); commitHook.fn = nil; commitHook.Unlock() }) + receipt, err := s.Create(ctx, createInput("durable", "body")) + if err != nil || ctx.Err() != context.Canceled { + t.Fatalf("receipt=%#v err=%v ctx=%v", receipt, err, ctx.Err()) + } + replay, err := s.Create(context.Background(), createInput("durable", "body")) + if err != nil || !replay.Replay || replay.Stage.ID != receipt.Stage.ID { + t.Fatalf("durable replay=%#v err=%v", replay, err) + } + if _, err := s.db.Exec(`UPDATE local_requests SET request_schema='changed' WHERE request_id='durable'`); err == nil { + t.Fatal("immutable receipt update succeeded") + } +} + +func TestReplayReceiptValidationFailsClosed(t *testing.T) { + base := MutationResult{"mm/v2/stage-mutation-receipt", "create", StageSummary{ + ID: "stg_valid", ServerURL: "https://mattermost.example/api/v4", UserID: "user-1", Operation: CreatePost, + Lifecycle: LifecycleOpen, Recovery: RecoveryNone, Revision: 1, SemanticDigest: sha256.Sum256([]byte("stage")), + CreatedAt: mustTime(t, "2026-01-01T00:00:00Z"), UpdatedAt: mustTime(t, "2026-01-01T00:00:00Z"), + }, false, mustTime(t, "2026-01-01T00:00:00Z"), false} + if !validReplayResult(base, "mm/v2/stage-request", base.Stage.ServerURL, base.Stage.UserID) { + t.Fatal("valid receipt rejected") + } + for name, mutate := range map[string]func(*MutationResult){ + "schema": func(v *MutationResult) { v.Schema = "wrong" }, "action": func(v *MutationResult) { v.Action = "cancel" }, + "digest": func(v *MutationResult) { v.Stage.SemanticDigest = [32]byte{} }, "timestamp": func(v *MutationResult) { v.RecordedAt = v.Stage.CreatedAt.Add(-1) }, + } { + t.Run(name, func(t *testing.T) { + v := base + mutate(&v) + if validReplayResult(v, "mm/v2/stage-request", base.Stage.ServerURL, base.Stage.UserID) { + t.Fatal("corrupt receipt accepted") + } + }) + } +} +func mustTime(t *testing.T, value string) time.Time { + t.Helper() + v, err := time.Parse(time.RFC3339, value) + if err != nil { + t.Fatal(err) + } + return v +} + +func TestV1RequestReplayMigratesToConflictTombstone(t *testing.T) { + original := migrations + migrations = append([]migration(nil), migrations[:1]...) + t.Cleanup(func() { migrations = original }) + path := testPath(t) + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + tx, err := s.db.Begin() + if err != nil { + t.Fatal(err) + } + digest := sha256.Sum256([]byte("legacy")) + const stamp = "2026-01-01T00:00:00Z" + if _, err = tx.Exec(`INSERT INTO stages(id,created_at,updated_at,operation,server_url,user_id,lifecycle,recovery,current_revision) VALUES('legacy-stage',?,?,'create_post','https://mattermost.example/api/v4','user-1','open','none',1)`, stamp, stamp); err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(`INSERT INTO stage_revisions(stage_id,revision,state,created_at,semantic_digest,body,destination_json,plan_json) VALUES('legacy-stage',1,'current',?,?,?,'{}','{}')`, stamp, digest[:], []byte("body")); err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(`INSERT INTO request_replays(server_url,user_id,request_id,request_schema,semantic_digest,stage_id,revision,created_at) VALUES('https://mattermost.example/api/v4','user-1','legacy-id','old',?,'legacy-stage',1,?)`, digest[:], stamp); err != nil { + t.Fatal(err) + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + _ = s.Close() + migrations = original + s, err = Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + var schema string + if err = s.db.QueryRow(`SELECT request_schema FROM local_requests WHERE request_id='legacy-id'`).Scan(&schema); err != nil || schema != "mm/v2/legacy-request-conflict" { + t.Fatalf("schema=%q err=%v", schema, err) + } + in := createInput("legacy-id", "body") + if _, err = s.Create(context.Background(), in); !errors.Is(err, ErrConflict) { + t.Fatalf("legacy reuse=%v", err) + } +} + +func FuzzCanonicalObject(f *testing.F) { + for _, seed := range [][]byte{[]byte(`{}`), []byte(`{"b":2,"a":1}`), []byte(`{"a":1,"a":2}`), []byte(`[]`), []byte(`{"x":"\ud800"}`)} { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, raw []byte) { + canonical, err := canonicalObject(raw) + if err != nil { + return + } + again, err := canonicalObject(canonical) + if err != nil || !reflect.DeepEqual(canonical, again) { + t.Fatalf("canonicalization unstable: %q -> %q err=%v", canonical, again, err) + } + }) +} diff --git a/internal/stagestore/schema.go b/internal/stagestore/schema.go index cc8d088..2aa44c0 100644 --- a/internal/stagestore/schema.go +++ b/internal/stagestore/schema.go @@ -69,4 +69,24 @@ CREATE TABLE request_replays ( PRIMARY KEY (server_url, user_id, request_id), FOREIGN KEY (stage_id, revision) REFERENCES stage_revisions(stage_id, revision) ) STRICT; +`}, {version: 2, name: "immutable-local-request-receipts", sql: ` +CREATE TABLE local_requests ( + server_url TEXT NOT NULL, + user_id TEXT NOT NULL, + request_id TEXT NOT NULL, + request_schema TEXT NOT NULL, + request_digest BLOB NOT NULL CHECK (length(request_digest) = 32), + result_json TEXT NOT NULL CHECK (json_valid(result_json)), + created_at TEXT NOT NULL, + PRIMARY KEY (server_url, user_id, request_id) +) STRICT; +INSERT INTO local_requests(server_url,user_id,request_id,request_schema,request_digest,result_json,created_at) +SELECT server_url,user_id,request_id,'mm/v2/legacy-request-conflict',semantic_digest,'{}',created_at FROM request_replays; +CREATE TRIGGER local_requests_immutable_update BEFORE UPDATE ON local_requests BEGIN SELECT RAISE(ABORT, 'local request receipts are immutable'); END; +CREATE TRIGGER local_requests_immutable_delete BEFORE DELETE ON local_requests BEGIN SELECT RAISE(ABORT, 'local request receipts are immutable'); END; +CREATE TRIGGER stage_revision_binding_immutable BEFORE INSERT ON stage_revisions +WHEN NEW.revision > 1 AND EXISTS(SELECT 1 FROM stage_revisions WHERE stage_id=NEW.stage_id) + AND (NEW.destination_json != (SELECT destination_json FROM stage_revisions WHERE stage_id=NEW.stage_id ORDER BY revision LIMIT 1) + OR NEW.plan_json != (SELECT plan_json FROM stage_revisions WHERE stage_id=NEW.stage_id ORDER BY revision LIMIT 1)) +BEGIN SELECT RAISE(ABORT, 'stage destination and plan are immutable'); END; `}} diff --git a/internal/stagestore/store_test.go b/internal/stagestore/store_test.go index b4d0467..ba08143 100644 --- a/internal/stagestore/store_test.go +++ b/internal/stagestore/store_test.go @@ -98,7 +98,7 @@ func TestOpenInitializesAndReopens(t *testing.T) { } defer s.Close() var count int - if err := s.db.QueryRow("SELECT count(*) FROM schema_migrations").Scan(&count); err != nil || count != 1 { + if err := s.db.QueryRow("SELECT count(*) FROM schema_migrations").Scan(&count); err != nil || count != len(migrations) { t.Fatalf("count=%d err=%v", count, err) } } @@ -549,7 +549,7 @@ func TestMigrationFailureIsAtomicAndUnknownIsRejected(t *testing.T) { s.Close() original := migrations t.Cleanup(func() { migrations = original }) - migrations = append(append([]migration{}, migrations...), migration{version: 2, name: "broken", sql: "CREATE TABLE should_rollback(x); SELECT no_such_function();"}) + migrations = append(append([]migration{}, migrations...), migration{version: len(migrations) + 1, name: "broken", sql: "CREATE TABLE should_rollback(x); SELECT no_such_function();"}) if _, err := Open(ctx, path); !errors.Is(err, ErrMigration) { t.Fatalf("failure=%v", err) } @@ -567,7 +567,7 @@ func TestMigrationFailureIsAtomicAndUnknownIsRejected(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = s.db.Exec("INSERT INTO schema_migrations(version,name,checksum,applied_at) VALUES(2,'future','x','x')") + _, err = s.db.Exec("INSERT INTO schema_migrations(version,name,checksum,applied_at) VALUES(?,'future','x','x')", len(migrations)+1) if err != nil { t.Fatal(err) } From fd17ed389377494ea2f046e07640b7332ec3f330 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 00:09:40 +0300 Subject: [PATCH 053/119] feat: add stage store inspection --- internal/cli/root.go | 1 + internal/cli/root_test.go | 2 +- internal/cli/store.go | 237 ++++++++++++++++ internal/cli/store_test.go | 316 ++++++++++++++++++++++ internal/output/bounded_json_test.go | 56 ++++ internal/output/machine.go | 9 +- internal/schema/registry_test.go | 22 ++ internal/stagestore/migrations.go | 17 ++ schemas/v2/examples/store-doctor.json | 1 + schemas/v2/examples/store-migrations.json | 1 + schemas/v2/store-doctor.schema.json | 74 +++++ schemas/v2/store-migrations.schema.json | 20 ++ 12 files changed, 754 insertions(+), 2 deletions(-) create mode 100644 internal/cli/store.go create mode 100644 internal/cli/store_test.go create mode 100644 internal/output/bounded_json_test.go create mode 100644 internal/stagestore/migrations.go create mode 100644 schemas/v2/examples/store-doctor.json create mode 100644 schemas/v2/examples/store-migrations.json create mode 100644 schemas/v2/store-doctor.schema.json create mode 100644 schemas/v2/store-migrations.schema.json diff --git a/internal/cli/root.go b/internal/cli/root.go index efe7a05..ece11f9 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -124,6 +124,7 @@ func newRootWithState(state *rootState) *cobra.Command { cmd.PersistentFlags().BoolVar(&state.flags.threads, "threads", true, "show visible thread structure") cmd.PersistentFlags().BoolVar(&state.flags.noThreads, "no-threads", false, "return selected seed posts only") cmd.AddCommand(newSchemaCommand(state)) + cmd.AddCommand(newStoreCommand(state)) cmd.AddCommand(newConfigCommand(state)) cmd.AddCommand(newDoctorCommand(state)) cmd.AddCommand(newWhoAmICommand(state)) diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index 29113ef..e020655 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -57,7 +57,7 @@ func TestSchemaList(t *testing.T) { if code != 0 { t.Fatalf("exit code = %d, want 0; stderr = %q", code, stderr.String()) } - if got, want := stdout.String(), "mm/v2/channel\nmm/v2/channels\nmm/v2/config\nmm/v2/dms\nmm/v2/doctor\nmm/v2/error\nmm/v2/group-dms\nmm/v2/mentions\nmm/v2/search\nmm/v2/teams\nmm/v2/thread\nmm/v2/unread\nmm/v2/users\nmm/v2/watch-diagnostic\nmm/v2/watch-event\nmm/v2/whoami\n"; got != want { + if got, want := stdout.String(), "mm/v2/channel\nmm/v2/channels\nmm/v2/config\nmm/v2/dms\nmm/v2/doctor\nmm/v2/error\nmm/v2/group-dms\nmm/v2/mentions\nmm/v2/search\nmm/v2/store-doctor\nmm/v2/store-migrations\nmm/v2/teams\nmm/v2/thread\nmm/v2/unread\nmm/v2/users\nmm/v2/watch-diagnostic\nmm/v2/watch-event\nmm/v2/whoami\n"; got != want { t.Fatalf("stdout = %q, want %q", got, want) } } diff --git a/internal/cli/store.go b/internal/cli/store.go new file mode 100644 index 0000000..5af1335 --- /dev/null +++ b/internal/cli/store.go @@ -0,0 +1,237 @@ +package cli + +import ( + "fmt" + "slices" + "strconv" + "strings" + "unicode" + + "github.com/spf13/cobra" + + "github.com/ardasevinc/mattermost-cli/internal/config" + "github.com/ardasevinc/mattermost-cli/internal/output" + "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +type storeDoctorEnvelope struct { + Schema string `json:"schema"` + Path string `json:"path"` + Report storeDoctorReport `json:"report"` +} + +type storeDoctorReport struct { + Exists bool `json:"exists"` + FilesystemSafe *bool `json:"filesystemSafe"` + ApplicationID *int `json:"applicationId"` + Integrity []string `json:"integrity"` + IntegrityTruncated *bool `json:"integrityTruncated"` + ForeignKeyIssues *int `json:"foreignKeyIssues"` + ForeignKeyRows []string `json:"foreignKeyRows"` + ForeignKeyTruncated *bool `json:"foreignKeyTruncated"` + Migrations storeMigrationStatus `json:"migrations"` + JournalMode *string `json:"journalMode"` + Synchronous *int `json:"synchronous"` + SecureDelete *int `json:"secureDelete"` + ForeignKeys *bool `json:"foreignKeys"` + TrustedSchema *bool `json:"trustedSchema"` + QueryOnly *bool `json:"queryOnly"` + WALFallback *bool `json:"walFallback"` + PermissionModelLimitations []string `json:"permissionModelLimitations"` +} + +type storeMigrationStatus struct { + Applied *int `json:"applied"` + Latest int `json:"latest"` + Valid *bool `json:"valid"` +} + +type storeMigrationsEnvelope struct { + Schema string `json:"schema"` + Latest int `json:"latest"` + Migrations []storeMigrationDescriptor `json:"migrations"` +} + +type storeMigrationDescriptor struct { + Version int `json:"version"` + Name string `json:"name"` + Checksum string `json:"checksum"` +} + +func newStoreCommand(state *rootState) *cobra.Command { + command := &cobra.Command{Use: "store", Short: "Inspect local stage storage", Args: cobra.NoArgs, + PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { return resolveStoreRedaction(state, cmd) }, + RunE: func(cmd *cobra.Command, _ []string) error { + if state.flags.json { + return invalidFailure("--json requires a store inspection subcommand") + } + return cmd.Help() + }} + command.AddCommand(&cobra.Command{ + Use: "doctor", Short: "Inspect local stage storage without changing it", Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + paths, err := storePaths(state) + if err != nil { + return err + } + report, err := stagestore.Doctor(cmd.Context(), paths.DBPath) + if err != nil { + return readFailure(err) + } + if report.Exists && (!report.Migrations.Valid || len(report.Integrity) != 1 || report.Integrity[0] != "ok" || report.IntegrityTruncated || report.ForeignKeyIssues > 0) { + state.setSemanticExit(6) + } + return writeStoreDoctor(state, paths.DBPath, report) + }, + }) + command.AddCommand(&cobra.Command{ + Use: "migrations", Short: "List migrations compiled into mm", Args: cobra.NoArgs, + RunE: func(_ *cobra.Command, _ []string) error { return writeStoreMigrations(state, stagestore.Migrations()) }, + }) + return command +} + +func resolveStoreRedaction(state *rootState, cmd *cobra.Command) error { + override, err := state.redactOption(cmd) + if err != nil { + return err + } + var file config.FileState + home, homeErr := state.deps.homeDir() + if homeErr == nil { + if paths, pathErr := config.ResolvePaths(home, state.deps.lookupEnv); pathErr == nil { + file = config.Load(paths) + if file.Config.Token != "" && !slices.Contains(state.credentials, file.Config.Token) { + state.releases = append(state.releases, presentation.ActiveCredentials.Register(file.Config.Token)) + state.credentials = append(state.credentials, file.Config.Token) + } + file.Config.Token = "" + file.Config.URL = "" + } + } + resolved := config.Resolve(config.Options{Redact: override}, state.deps.lookupEnv, file) + state.disableHeuristics = !resolved.Redact + return nil +} + +func storePaths(state *rootState) (stagestore.Paths, error) { + home, err := state.deps.homeDir() + if err != nil { + return stagestore.Paths{}, configFailure("could not resolve the home directory") + } + paths, err := stagestore.ResolvePaths(home, func(key string) (string, bool) { return state.deps.lookupEnv(key) }) + if err != nil { + return stagestore.Paths{}, configFailure(err.Error()) + } + return paths, nil +} + +func safeStoreValue(state *rootState, value string) string { + return presentation.SanitizeLabel(presentation.PreprocessWithOptions(value, presentation.Options{Credentials: state.credentials, DisableHeuristics: state.disableHeuristics}).Text) +} + +func writeStoreDoctor(state *rootState, path string, report stagestore.DoctorReport) error { + safe := func(values []string) []string { + result := make([]string, len(values)) + for i, value := range values { + result[i] = safeStoreValue(state, value) + } + return result + } + compiled := stagestore.Migrations() + latest := compiled[len(compiled)-1].Version + document := storeDoctorEnvelope{Schema: "mm/v2/store-doctor", Path: safeStoreValue(state, path), Report: storeDoctorReport{Exists: report.Exists, + Migrations: storeMigrationStatus{Latest: latest}, PermissionModelLimitations: safe(report.PermissionModelLimitations)}} + if report.Exists { + document.Report.FilesystemSafe, document.Report.ApplicationID = pointer(report.FilesystemSafe), pointer(report.ApplicationID) + document.Report.Integrity, document.Report.IntegrityTruncated = safe(report.Integrity), pointer(report.IntegrityTruncated) + document.Report.ForeignKeyIssues, document.Report.ForeignKeyRows, document.Report.ForeignKeyTruncated = pointer(report.ForeignKeyIssues), safe(report.ForeignKeyRows), pointer(report.ForeignKeyTruncated) + document.Report.Migrations.Applied, document.Report.Migrations.Valid = pointer(report.Migrations.Applied), pointer(report.Migrations.Valid) + journal := safeStoreValue(state, report.JournalMode) + document.Report.JournalMode = &journal + document.Report.Synchronous, document.Report.SecureDelete = pointer(report.Synchronous), pointer(report.SecureDelete) + document.Report.ForeignKeys, document.Report.TrustedSchema, document.Report.QueryOnly, document.Report.WALFallback = pointer(report.ForeignKeys), pointer(report.TrustedSchema), pointer(report.QueryOnly), pointer(report.WALFallback) + } + if state.flags.json { + return writeStoreJSON(state, document) + } + lines := []string{"path: " + document.Path, "exists: " + strconv.FormatBool(document.Report.Exists), "compiled latest migration: " + strconv.Itoa(document.Report.Migrations.Latest)} + if report.Exists { + lines = append(lines, "filesystem safe: "+strconv.FormatBool(report.FilesystemSafe), "integrity: "+strings.Join(document.Report.Integrity, ", "), + "integrity truncated: "+strconv.FormatBool(report.IntegrityTruncated), "foreign key issues: "+strconv.Itoa(report.ForeignKeyIssues), + "foreign key rows truncated: "+strconv.FormatBool(report.ForeignKeyTruncated), + "migrations: "+strconv.Itoa(report.Migrations.Applied)+"/"+strconv.Itoa(report.Migrations.Latest)+" valid="+strconv.FormatBool(report.Migrations.Valid), + "journal mode: "+safeStoreValue(state, report.JournalMode)+" fallback="+strconv.FormatBool(report.WALFallback), + "guards: foreign_keys="+strconv.FormatBool(report.ForeignKeys)+" trusted_schema="+strconv.FormatBool(report.TrustedSchema)+" query_only="+strconv.FormatBool(report.QueryOnly), + "durability: synchronous="+strconv.Itoa(report.Synchronous)+" secure_delete="+strconv.Itoa(report.SecureDelete)) + } else { + lines = append(lines, "store facts: not applicable (store absent)") + } + for _, limitation := range document.Report.PermissionModelLimitations { + lines = append(lines, "limitation: "+limitation) + } + return writeAll(state.streams.out, []byte(strings.Join(lines, "\n")+"\n")) +} + +func writeStoreMigrations(state *rootState, migrations []stagestore.MigrationInfo) error { + if len(migrations) == 0 { + return internalFailure(fmt.Errorf("compiled migration set is empty")) + } + document := storeMigrationsEnvelope{Schema: "mm/v2/store-migrations", Migrations: make([]storeMigrationDescriptor, len(migrations))} + for i, item := range migrations { + if item.Version != i+1 || (i > 0 && item.Version <= migrations[i-1].Version) { + return internalFailure(fmt.Errorf("compiled migration order is invalid")) + } + if !safeMigrationName(item.Name) || !lowerHexDigest(item.Checksum) { + return internalFailure(fmt.Errorf("compiled migration metadata is invalid")) + } + for _, credential := range state.credentials { + if credential != "" && (strings.Contains(item.Name, credential) || strings.Contains(item.Checksum, credential)) { + return internalFailure(fmt.Errorf("compiled migration metadata conflicts with an active credential")) + } + } + document.Migrations[i] = storeMigrationDescriptor{Version: item.Version, Name: item.Name, Checksum: item.Checksum} + document.Latest = item.Version + } + if state.flags.json { + return writeStoreJSON(state, document) + } + lines := []string{"latest: " + strconv.Itoa(document.Latest)} + for _, item := range document.Migrations { + lines = append(lines, strconv.Itoa(item.Version)+" "+item.Name+" "+item.Checksum) + } + return writeAll(state.streams.out, []byte(strings.Join(lines, "\n")+"\n")) +} + +func safeMigrationName(value string) bool { + if value == "" { + return false + } + for _, r := range value { + if r > unicode.MaxASCII || !(unicode.IsLower(r) || unicode.IsDigit(r) || r == '-') { + return false + } + } + return true +} + +func lowerHexDigest(value string) bool { + if len(value) != 64 { + return false + } + for _, r := range value { + if !(r >= '0' && r <= '9') && !(r >= 'a' && r <= 'f') { + return false + } + } + return true +} + +func writeStoreJSON(state *rootState, document any) error { + _, err := output.WriteBoundedJSON(state.streams.out, document) + if err != nil { + return outputError{err: err} + } + return nil +} diff --git a/internal/cli/store_test.go b/internal/cli/store_test.go new file mode 100644 index 0000000..951feee --- /dev/null +++ b/internal/cli/store_test.go @@ -0,0 +1,316 @@ +//go:build darwin || linux + +package cli + +import ( + "bytes" + "context" + "database/sql" + "os" + "path/filepath" + "strings" + "testing" + + mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +func TestStoreDoctorAbsentIsReadOnlyAndSchemaValid(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_STATE_HOME", filepath.Join(home, "hostile\x1bstate")) + t.Setenv("MM_URL", "not-a-url") + t.Setenv("MM_TOKEN", "unused-token") + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "store", "doctor"}, strings.NewReader(""), &stdout, &stderr) + if code != 0 || stderr.Len() != 0 { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + if strings.ContainsRune(stdout.String(), '\x1b') || !strings.Contains(stdout.String(), `\\u001b`) { + t.Fatalf("path was not safely presented: %q", stdout.String()) + } + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/store-doctor", bytes.NewReader(stdout.Bytes())); err != nil { + t.Fatalf("schema: %v\n%s", err, stdout.String()) + } + for _, fact := range []string{`"filesystemSafe":null`, `"applicationId":null`, `"integrity":null`, `"applied":null`, `"latest":2`, `"valid":null`, `"journalMode":null`} { + if !strings.Contains(stdout.String(), fact) { + t.Fatalf("absent report omitted %s: %s", fact, stdout.String()) + } + } + if _, err := os.Stat(filepath.Join(home, "hostile\x1bstate")); !os.IsNotExist(err) { + t.Fatalf("inspection created state: %v", err) + } +} + +func TestBareStoreRejectsJSONWithMachineError(t *testing.T) { + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "store"}, strings.NewReader(""), &stdout, &stderr) + if code != 2 || stdout.Len() != 0 { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/error", bytes.NewReader(stderr.Bytes())); err != nil { + t.Fatalf("error schema: %v\n%s", err, stderr.String()) + } +} + +func TestStoreDoctorExistingDoesNotMutate(t *testing.T) { + home := t.TempDir() + stateRoot := filepath.Join(home, "state") + paths, err := stagestore.ResolvePaths(home, func(key string) (string, bool) { return stateRoot, key == "XDG_STATE_HOME" }) + if err != nil { + t.Fatal(err) + } + store, err := stagestore.Open(context.Background(), paths.DBPath) + if err != nil { + t.Fatal(err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + before, err := os.Stat(paths.DBPath) + if err != nil { + t.Fatal(err) + } + t.Setenv("HOME", home) + t.Setenv("XDG_STATE_HOME", stateRoot) + t.Setenv("MM_URL", "") + t.Setenv("MM_TOKEN", "") + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "store", "doctor"}, strings.NewReader(""), &stdout, &stderr) + after, err := os.Stat(paths.DBPath) + if err != nil { + t.Fatal(err) + } + if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), `"exists":true`) || !before.ModTime().Equal(after.ModTime()) || before.Size() != after.Size() { + t.Fatalf("exit=%d stdout=%q stderr=%q before=%v after=%v", code, stdout.String(), stderr.String(), before, after) + } +} + +func TestStoreMigrationsIsOfflineAndSchemaValid(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + t.Setenv("MM_URL", "not-a-url") + t.Setenv("MM_TOKEN", "unused-token") + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "store", "migrations"}, strings.NewReader(""), &stdout, &stderr) + if code != 0 || stderr.Len() != 0 { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/store-migrations", bytes.NewReader(stdout.Bytes())); err != nil { + t.Fatalf("schema: %v\n%s", err, stdout.String()) + } + want := "{\"schema\":\"mm/v2/store-migrations\",\"latest\":2,\"migrations\":[{\"version\":1,\"name\":\"core-stage-state\",\"checksum\":\"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f\"},{\"version\":2,\"name\":\"immutable-local-request-receipts\",\"checksum\":\"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c\"}]}\n" + if stdout.String() != want { + t.Fatalf("stdout does not match golden migration contract: %q", stdout.String()) + } +} + +func TestStoreDoctorUnsafeStateIsMachineError(t *testing.T) { + home := t.TempDir() + stateRoot := filepath.Join(home, "state") + db := filepath.Join(stateRoot, "mattermost-cli", stagestore.DatabaseFilename) + if err := os.MkdirAll(filepath.Dir(db), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(db, []byte("not sqlite"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("HOME", home) + t.Setenv("XDG_STATE_HOME", stateRoot) + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "store", "doctor"}, strings.NewReader(""), &stdout, &stderr) + if code != 3 || stdout.Len() != 0 || !strings.Contains(stderr.String(), `"code":"read_failed"`) { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } +} + +func TestStoreDoctorUnhealthyIntegrityReturnsStateConflictExit(t *testing.T) { + home := t.TempDir() + stateRoot := filepath.Join(home, "state") + paths, err := stagestore.ResolvePaths(home, func(key string) (string, bool) { return stateRoot, key == "XDG_STATE_HOME" }) + if err != nil { + t.Fatal(err) + } + store, err := stagestore.Open(context.Background(), paths.DBPath) + if err != nil { + t.Fatal(err) + } + if err := store.Close(); err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite", paths.DBPath) + if err != nil { + t.Fatal(err) + } + _, err = db.Exec(`PRAGMA foreign_keys=OFF; INSERT INTO stage_attachments(stage_id,revision,ordinal,supplied_path,canonical_path,remote_filename,byte_length,content_digest) VALUES('missing',1,0,'a','a','a',0,zeroblob(32))`) + if err != nil { + _ = db.Close() + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + t.Setenv("HOME", home) + t.Setenv("XDG_STATE_HOME", stateRoot) + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "store", "doctor"}, strings.NewReader(""), &stdout, &stderr) + if code != 6 || stderr.Len() != 0 || !strings.Contains(stdout.String(), `"foreignKeyIssues":1`) { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } +} + +func TestStoreNoRedactDisablesOnlyHeuristics(t *testing.T) { + home := t.TempDir() + const credential = "active-mm-credential-123456" + const heuristic = "AKIAIOSFODNN7EXAMPLE" + t.Setenv("HOME", home) + t.Setenv("MM_TOKEN", credential) + t.Setenv("XDG_STATE_HOME", filepath.Join(home, heuristic, credential)) + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "--no-redact", "store", "doctor"}, strings.NewReader(""), &stdout, &stderr) + if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), heuristic) || strings.Contains(stdout.String(), credential) || !strings.Contains(stdout.String(), "REDACTED") { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } +} + +func TestStoreDoctorResolvesRedactionFromEnvironmentAndFileOffline(t *testing.T) { + const heuristic = "AKIAIOSFODNN7EXAMPLE" + for _, test := range []struct { + name string + env bool + }{{"environment", true}, {"file", false}} { + t.Run(test.name, func(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_STATE_HOME", filepath.Join(home, heuristic)) + if test.env { + t.Setenv("MM_REDACT", "false") + } else { + previous, present := os.LookupEnv("MM_REDACT") + if err := os.Unsetenv("MM_REDACT"); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if present { + _ = os.Setenv("MM_REDACT", previous) + } else { + _ = os.Unsetenv("MM_REDACT") + } + }) + writeFile(t, filepath.Join(home, ".config", "mattermost-cli", "config.toml"), "redact = false\ntoken = \"core-stage-state\"\n", 0o600) + } + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "store", "doctor"}, strings.NewReader(""), &stdout, &stderr) + if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), heuristic) { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + }) + } +} + +func TestStoreMigrationsActiveCredentialCollisionFailsBeforeOutput(t *testing.T) { + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "--token", "core-stage-state", "store", "migrations"}, strings.NewReader(""), &stdout, &stderr) + if code != 3 || stdout.Len() != 0 || strings.Contains(stderr.String(), "core-stage-state") { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/error", bytes.NewReader(stderr.Bytes())); err != nil { + t.Fatalf("error schema: %v\n%s", err, stderr.String()) + } +} + +func TestStoreMigrationsTreatsReadFileTokenAsActiveCredential(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("MM_TOKEN", "") + writeFile(t, filepath.Join(home, ".config", "mattermost-cli", "config.toml"), "token = \"core-stage-state\"\n", 0o600) + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "store", "migrations"}, strings.NewReader(""), &stdout, &stderr) + if code != 3 || stdout.Len() != 0 || strings.Contains(stderr.String(), "core-stage-state") { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } +} + +func TestStoreDoctorMasksConfiguredTokenReadForPreference(t *testing.T) { + home := t.TempDir() + const token = "configured-mm-token-123456" + t.Setenv("HOME", home) + t.Setenv("XDG_STATE_HOME", filepath.Join(home, token)) + t.Setenv("MM_TOKEN", "") + writeFile(t, filepath.Join(home, ".config", "mattermost-cli", "config.toml"), "redact = false\ntoken = \""+token+"\"\n", 0o600) + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "store", "doctor"}, strings.NewReader(""), &stdout, &stderr) + if code != 0 || stderr.Len() != 0 || strings.Contains(stdout.String(), token) || !strings.Contains(stdout.String(), "REDACTED") { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } +} + +func TestStoreArgValidationMasksConfiguredTokenBeforePreRun(t *testing.T) { + home := t.TempDir() + const token = "configured-mm-token-arg-123456" + t.Setenv("HOME", home) + t.Setenv("MM_TOKEN", "") + writeFile(t, filepath.Join(home, ".config", "mattermost-cli", "config.toml"), "token = \""+token+"\"\n", 0o600) + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"--json", "store", token}, strings.NewReader(""), &stdout, &stderr) + if code != 2 || stdout.Len() != 0 || strings.Contains(stderr.String(), token) || !strings.Contains(stderr.String(), "REDACTED") { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/error", bytes.NewReader(stderr.Bytes())); err != nil { + t.Fatalf("error schema: %v\n%s", err, stderr.String()) + } +} + +func TestStoreMigrationRuntimeRejectsInvalidOrderBeforeWrite(t *testing.T) { + var stdout, stderr bytes.Buffer + state := &rootState{streams: streams{out: &stdout, err: &stderr}} + err := writeStoreMigrations(state, []stagestore.MigrationInfo{{Version: 2, Name: "later", Checksum: strings.Repeat("a", 64)}}) + if err == nil || stdout.Len() != 0 { + t.Fatalf("err=%v stdout=%q", err, stdout.String()) + } +} + +func TestStoreMigrationRuntimeRejectsUnsafeMetadata(t *testing.T) { + for _, migration := range []stagestore.MigrationInfo{{Version: 1, Name: "bad name", Checksum: strings.Repeat("a", 64)}, {Version: 1, Name: "safe-name", Checksum: strings.Repeat("G", 64)}} { + var stdout bytes.Buffer + state := &rootState{streams: streams{out: &stdout}} + if err := writeStoreMigrations(state, []stagestore.MigrationInfo{migration}); err == nil || stdout.Len() != 0 { + t.Fatalf("migration=%+v err=%v stdout=%q", migration, err, stdout.String()) + } + } +} + +func TestStoreHumanOutputStatesBounds(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + t.Setenv("XDG_STATE_HOME", "") + var stdout, stderr bytes.Buffer + if code := Execute(context.Background(), []string{"store", "doctor"}, strings.NewReader(""), &stdout, &stderr); code != 0 || stderr.Len() != 0 { + t.Fatalf("exit=%d stderr=%q", code, stderr.String()) + } + if !strings.Contains(stdout.String(), "exists: false") { + t.Fatalf("stdout=%q", stdout.String()) + } + stdout.Reset() + if code := Execute(context.Background(), []string{"store", "migrations"}, strings.NewReader(""), &stdout, &stderr); code != 0 || !strings.Contains(stdout.String(), "latest: 2\n1 core-stage-state ") { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } +} diff --git a/internal/output/bounded_json_test.go b/internal/output/bounded_json_test.go new file mode 100644 index 0000000..45d3b84 --- /dev/null +++ b/internal/output/bounded_json_test.go @@ -0,0 +1,56 @@ +package output + +import ( + "bytes" + "io" + "strings" + "testing" +) + +func TestWriteBoundedJSONCanonicalLiterals(t *testing.T) { + var output bytes.Buffer + value := struct { + Text string `json:"text"` + }{Text: "<>&\u2028\u2029"} + if _, err := WriteBoundedJSON(&output, value); err != nil { + t.Fatal(err) + } + if got, want := output.String(), "{\"text\":\"<>&\u2028\u2029\"}\n"; got != want { + t.Fatalf("bytes=%q want=%q", got, want) + } +} + +func TestWriteBoundedJSONOversizeFailsBeforeWrite(t *testing.T) { + w := &countingWriter{} + _, err := WriteBoundedJSON(w, struct { + Text string `json:"text"` + }{Text: strings.Repeat("x", MaxMachineDocumentBytes)}) + if err == nil || w.calls != 0 || w.bytes != 0 { + t.Fatalf("err=%v calls=%d bytes=%d", err, w.calls, w.bytes) + } +} + +func TestWriteBoundedJSONShortWriteIsNotRetried(t *testing.T) { + w := &countingWriter{short: true} + _, err := WriteBoundedJSON(w, struct { + OK bool `json:"ok"` + }{true}) + if err != io.ErrShortWrite || w.calls != 1 { + t.Fatalf("err=%v calls=%d", err, w.calls) + } +} + +type countingWriter struct { + calls, bytes int + short bool +} + +func (w *countingWriter) Write(value []byte) (int, error) { + w.calls++ + n := len(value) + if w.short { + n-- + } + w.bytes += n + return n, nil +} diff --git a/internal/output/machine.go b/internal/output/machine.go index 5c69bef..4333777 100644 --- a/internal/output/machine.go +++ b/internal/output/machine.go @@ -373,11 +373,18 @@ func WriteMachineJSON(w io.Writer, value MachineDocument) (int, error) { if err != nil { return 0, err } + return WriteBoundedJSON(w, wireValue) +} + +// WriteBoundedJSON emits canonical machine JSON using the same byte policy as +// WriteMachineJSON. It is for versioned machine documents whose concrete type +// is owned outside output's closed MachineDocument set. +func WriteBoundedJSON(w io.Writer, value any) (int, error) { buffer := boundedBuffer{limit: MaxMachineDocumentBytes} wire := separatorWriter{destination: &buffer} encoder := json.NewEncoder(&wire) encoder.SetEscapeHTML(false) - if err := encoder.Encode(wireValue); err != nil { + if err := encoder.Encode(value); err != nil { return 0, err } if err := wire.flush(); err != nil { diff --git a/internal/schema/registry_test.go b/internal/schema/registry_test.go index 72850c3..14dd6c8 100644 --- a/internal/schema/registry_test.go +++ b/internal/schema/registry_test.go @@ -128,3 +128,25 @@ func TestDoctorSchemaRejectsSemanticContradictions(t *testing.T) { } } } + +func TestStoreSchemasRejectSemanticContradictions(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + absent := `{"schema":"mm/v2/store-doctor","path":"/tmp/stages.sqlite3","report":{"exists":false,"filesystemSafe":false,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":2,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}}` + issueWithoutRow := `{"schema":"mm/v2/store-doctor","path":"/tmp/stages.sqlite3","report":{"exists":true,"filesystemSafe":true,"applicationId":1296913970,"integrity":["ok"],"integrityTruncated":false,"foreignKeyIssues":1,"foreignKeyRows":[],"foreignKeyTruncated":false,"migrations":{"applied":2,"latest":2,"valid":true},"journalMode":"wal","synchronous":2,"secureDelete":2,"foreignKeys":true,"trustedSchema":false,"queryOnly":true,"walFallback":false,"permissionModelLimitations":[]}}` + contradictoryWAL := `{"schema":"mm/v2/store-doctor","path":"/tmp/stages.sqlite3","report":{"exists":true,"filesystemSafe":true,"applicationId":1296913970,"integrity":["ok"],"integrityTruncated":false,"foreignKeyIssues":0,"foreignKeyRows":[],"foreignKeyTruncated":false,"migrations":{"applied":2,"latest":2,"valid":true},"journalMode":"wal","synchronous":2,"secureDelete":2,"foreignKeys":true,"trustedSchema":false,"queryOnly":true,"walFallback":true,"permissionModelLimitations":[]}}` + twentyRows := `"row",` + strings.Repeat(`"row",`, 18) + `"row"` + unmarkedTruncation := `{"schema":"mm/v2/store-doctor","path":"/tmp/stages.sqlite3","report":{"exists":true,"filesystemSafe":true,"applicationId":1296913970,"integrity":["ok"],"integrityTruncated":false,"foreignKeyIssues":21,"foreignKeyRows":[` + twentyRows + `],"foreignKeyTruncated":false,"migrations":{"applied":2,"latest":2,"valid":true},"journalMode":"wal","synchronous":2,"secureDelete":2,"foreignKeys":true,"trustedSchema":false,"queryOnly":true,"walFallback":false,"permissionModelLimitations":[]}}` + mixedIntegrity := `{"schema":"mm/v2/store-doctor","path":"/tmp/stages.sqlite3","report":{"exists":true,"filesystemSafe":true,"applicationId":1296913970,"integrity":["ok","damaged"],"integrityTruncated":false,"foreignKeyIssues":0,"foreignKeyRows":[],"foreignKeyTruncated":false,"migrations":{"applied":2,"latest":2,"valid":true},"journalMode":"wal","synchronous":2,"secureDelete":2,"foreignKeys":true,"trustedSchema":false,"queryOnly":true,"walFallback":false,"permissionModelLimitations":[]}}` + wrongLatest := `{"schema":"mm/v2/store-migrations","latest":1,"migrations":[{"version":1,"name":"core-stage-state","checksum":"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"},{"version":2,"name":"immutable-local-request-receipts","checksum":"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"}]}` + wrongOrder := `{"schema":"mm/v2/store-migrations","latest":2,"migrations":[{"version":2,"name":"immutable-local-request-receipts","checksum":"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"},{"version":1,"name":"core-stage-state","checksum":"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"}]}` + for id, documents := range map[string][]string{"mm/v2/store-doctor": {absent, issueWithoutRow, contradictoryWAL, unmarkedTruncation, mixedIntegrity}, "mm/v2/store-migrations": {wrongLatest, wrongOrder}} { + for _, document := range documents { + if err := registry.Validate(id, strings.NewReader(document)); err == nil { + t.Fatalf("%s accepted contradiction: %s", id, document) + } + } + } +} diff --git a/internal/stagestore/migrations.go b/internal/stagestore/migrations.go new file mode 100644 index 0000000..1d3e092 --- /dev/null +++ b/internal/stagestore/migrations.go @@ -0,0 +1,17 @@ +package stagestore + +// MigrationInfo is immutable metadata for one migration compiled into mm. +type MigrationInfo struct { + Version int + Name string + Checksum string +} + +// Migrations returns the ordered migration set without opening local state. +func Migrations() []MigrationInfo { + result := make([]MigrationInfo, len(migrations)) + for i, item := range migrations { + result[i] = MigrationInfo{Version: item.version, Name: item.name, Checksum: item.checksum()} + } + return result +} diff --git a/schemas/v2/examples/store-doctor.json b/schemas/v2/examples/store-doctor.json new file mode 100644 index 0000000..fc442cb --- /dev/null +++ b/schemas/v2/examples/store-doctor.json @@ -0,0 +1 @@ +{"schema":"mm/v2/store-doctor","path":"/home/arda/.local/state/mattermost-cli/stages.sqlite3","report":{"exists":false,"filesystemSafe":null,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":2,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}} diff --git a/schemas/v2/examples/store-migrations.json b/schemas/v2/examples/store-migrations.json new file mode 100644 index 0000000..9cf9d2a --- /dev/null +++ b/schemas/v2/examples/store-migrations.json @@ -0,0 +1 @@ +{"schema":"mm/v2/store-migrations","latest":2,"migrations":[{"version":1,"name":"core-stage-state","checksum":"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"},{"version":2,"name":"immutable-local-request-receipts","checksum":"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"}]} diff --git a/schemas/v2/store-doctor.schema.json b/schemas/v2/store-doctor.schema.json new file mode 100644 index 0000000..2b310ac --- /dev/null +++ b/schemas/v2/store-doctor.schema.json @@ -0,0 +1,74 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:store-doctor", + "title": "mm v2 local store diagnostic", + "type": "object", + "additionalProperties": false, + "required": ["schema", "path", "report"], + "properties": { + "schema": { "const": "mm/v2/store-doctor" }, + "path": { "$ref": "#/$defs/safeString" }, + "report": { + "type": "object", "additionalProperties": false, + "required": ["exists", "filesystemSafe", "applicationId", "integrity", "integrityTruncated", "foreignKeyIssues", "foreignKeyRows", "foreignKeyTruncated", "migrations", "journalMode", "synchronous", "secureDelete", "foreignKeys", "trustedSchema", "queryOnly", "walFallback", "permissionModelLimitations"], + "properties": { + "exists": { "type": "boolean" }, "filesystemSafe": {}, "applicationId": {}, "integrity": {}, "integrityTruncated": {}, + "foreignKeyIssues": {}, "foreignKeyRows": {}, "foreignKeyTruncated": {}, "journalMode": {}, "synchronous": {}, "secureDelete": {}, + "foreignKeys": {}, "trustedSchema": {}, "queryOnly": {}, "walFallback": {}, + "migrations": { "type": "object", "additionalProperties": false, "required": ["applied", "latest", "valid"], "properties": { "applied": {}, "latest": { "const": 2 }, "valid": {} } }, + "permissionModelLimitations": { "type": "array", "items": { "$ref": "#/$defs/safeString" } } + }, + "allOf": [ + { "if": { "properties": { "exists": { "const": false } }, "required": ["exists"] }, "then": { "properties": { + "filesystemSafe": { "const": null }, "applicationId": { "const": null }, "integrity": { "const": null }, "integrityTruncated": { "const": null }, + "foreignKeyIssues": { "const": null }, "foreignKeyRows": { "const": null }, "foreignKeyTruncated": { "const": null }, "journalMode": { "const": null }, + "synchronous": { "const": null }, "secureDelete": { "const": null }, "foreignKeys": { "const": null }, "trustedSchema": { "const": null }, "queryOnly": { "const": null }, "walFallback": { "const": null }, + "migrations": { "properties": { "applied": { "const": null }, "latest": { "const": 2 }, "valid": { "const": null } } } + } } }, + { "if": { "properties": { "exists": { "const": true } }, "required": ["exists"] }, "then": { "properties": { + "filesystemSafe": { "const": true }, "applicationId": { "const": 1296913970 }, "integrity": { "$ref": "#/$defs/nonemptyBoundedStrings" }, "integrityTruncated": { "type": "boolean" }, + "foreignKeyIssues": { "type": "integer", "minimum": 0, "maximum": 21 }, "foreignKeyRows": { "$ref": "#/$defs/boundedStrings" }, "foreignKeyTruncated": { "type": "boolean" }, + "journalMode": { "enum": ["wal", "delete", "truncate", "persist"] }, "synchronous": { "type": "integer" }, "secureDelete": { "type": "integer" }, + "foreignKeys": { "const": true }, "trustedSchema": { "const": false }, "queryOnly": { "const": true }, "walFallback": { "type": "boolean" }, + "migrations": { "properties": { "applied": { "const": 2 }, "latest": { "const": 2 }, "valid": { "const": true } } } + }, "allOf": [ + { "oneOf": [ + { "properties": { "integrity": { "const": ["ok"] }, "integrityTruncated": { "const": false } } }, + { "properties": { "integrity": { "type": "array", "minItems": 1, "maxItems": 20, "items": { "allOf": [{ "$ref": "#/$defs/safeString" }, { "not": { "const": "ok" } }] } } } } + ] }, + { "if": { "properties": { "integrityTruncated": { "const": true } }, "required": ["integrityTruncated"] }, "then": { "properties": { "integrity": { "minItems": 20, "maxItems": 20 } } } }, + { "if": { "properties": { "foreignKeyTruncated": { "const": true } }, "required": ["foreignKeyTruncated"] }, "then": { "properties": { "foreignKeyIssues": { "const": 21 }, "foreignKeyRows": { "minItems": 20, "maxItems": 20 } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 0 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "maxItems": 0 }, "foreignKeyTruncated": { "const": false } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 1 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "minItems": 1, "maxItems": 1 }, "foreignKeyTruncated": { "const": false } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 2 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "minItems": 2, "maxItems": 2 }, "foreignKeyTruncated": { "const": false } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 3 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "minItems": 3, "maxItems": 3 }, "foreignKeyTruncated": { "const": false } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 4 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "minItems": 4, "maxItems": 4 }, "foreignKeyTruncated": { "const": false } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 5 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "minItems": 5, "maxItems": 5 }, "foreignKeyTruncated": { "const": false } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 6 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "minItems": 6, "maxItems": 6 }, "foreignKeyTruncated": { "const": false } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 7 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "minItems": 7, "maxItems": 7 }, "foreignKeyTruncated": { "const": false } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 8 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "minItems": 8, "maxItems": 8 }, "foreignKeyTruncated": { "const": false } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 9 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "minItems": 9, "maxItems": 9 }, "foreignKeyTruncated": { "const": false } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 10 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "minItems": 10, "maxItems": 10 }, "foreignKeyTruncated": { "const": false } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 11 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "minItems": 11, "maxItems": 11 }, "foreignKeyTruncated": { "const": false } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 12 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "minItems": 12, "maxItems": 12 }, "foreignKeyTruncated": { "const": false } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 13 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "minItems": 13, "maxItems": 13 }, "foreignKeyTruncated": { "const": false } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 14 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "minItems": 14, "maxItems": 14 }, "foreignKeyTruncated": { "const": false } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 15 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "minItems": 15, "maxItems": 15 }, "foreignKeyTruncated": { "const": false } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 16 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "minItems": 16, "maxItems": 16 }, "foreignKeyTruncated": { "const": false } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 17 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "minItems": 17, "maxItems": 17 }, "foreignKeyTruncated": { "const": false } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 18 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "minItems": 18, "maxItems": 18 }, "foreignKeyTruncated": { "const": false } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 19 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "minItems": 19, "maxItems": 19 }, "foreignKeyTruncated": { "const": false } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 20 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "minItems": 20, "maxItems": 20 }, "foreignKeyTruncated": { "const": false } } } }, + { "if": { "properties": { "foreignKeyIssues": { "const": 21 } }, "required": ["foreignKeyIssues"] }, "then": { "properties": { "foreignKeyRows": { "minItems": 20, "maxItems": 20 }, "foreignKeyTruncated": { "const": true } } } }, + { "if": { "properties": { "walFallback": { "const": true } }, "required": ["walFallback"] }, "then": { "properties": { "journalMode": { "enum": ["delete", "truncate", "persist"] } } } }, + { "if": { "properties": { "walFallback": { "const": false } }, "required": ["walFallback"] }, "then": { "properties": { "journalMode": { "const": "wal" } } } } + ] } } + ] + } + }, + "$defs": { + "safeString": { "type": "string", "minLength": 1, "pattern": "^[^\\x{0000}-\\x{001F}\\x{007F}-\\x{009F}\\x{202A}-\\x{202E}\\x{2066}-\\x{2069}]*$" }, + "boundedStrings": { "type": "array", "maxItems": 20, "items": { "$ref": "#/$defs/safeString" } }, + "nonemptyBoundedStrings": { "type": "array", "minItems": 1, "maxItems": 20, "items": { "$ref": "#/$defs/safeString" } } + } +} diff --git a/schemas/v2/store-migrations.schema.json b/schemas/v2/store-migrations.schema.json new file mode 100644 index 0000000..8c33e6f --- /dev/null +++ b/schemas/v2/store-migrations.schema.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:store-migrations", + "title": "mm v2 compiled store migrations", + "type": "object", "additionalProperties": false, + "required": ["schema", "latest", "migrations"], + "properties": { + "schema": { "const": "mm/v2/store-migrations" }, + "latest": { "const": 2 }, + "migrations": { + "type": "array", "minItems": 2, "maxItems": 2, + "prefixItems": [{ "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { + "version": { "const": 1 }, "name": { "const": "core-stage-state" }, "checksum": { "const": "e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f" } + } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { + "version": { "const": 2 }, "name": { "const": "immutable-local-request-receipts" }, "checksum": { "const": "ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c" } + } }], + "items": false + } + } +} From cc5669d5873c4268518b2d5ed7baf866cf531d7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 00:27:01 +0300 Subject: [PATCH 054/119] feat: add mutation target reads --- internal/mattermost/channels.go | 38 +++++++++++++ internal/mattermost/channels_test.go | 65 ++++++++++++++++++++- internal/mattermost/posts.go | 54 ++++++++++++++++++ internal/mattermost/posts_test.go | 84 ++++++++++++++++++++++++++++ 4 files changed, 239 insertions(+), 2 deletions(-) diff --git a/internal/mattermost/channels.go b/internal/mattermost/channels.go index 9b94100..0243768 100644 --- a/internal/mattermost/channels.go +++ b/internal/mattermost/channels.go @@ -491,6 +491,44 @@ func (s *Channels) DirectList(ctx context.Context, userID string) ([]Channel, er return result, nil } +// ExistingDirect finds the unique existing D channel for the exact current +// user and peer pair. DirectList is the canonical, read-only membership proof; +// absence is distinct from an invalid or ambiguous response. +func (s *Channels) ExistingDirect(ctx context.Context, currentUserID, peerID string) (Channel, bool, error) { + if !canonicalChannelRequestID(currentUserID) || !canonicalChannelRequestID(peerID) || currentUserID == "me" || peerID == "me" { + return Channel{}, false, ErrInvalidChannelRequest + } + var decoded directChannelList + if err := s.client.Get(ctx, "/users/"+url.PathEscape(currentUserID)+"/channels", &decoded); err != nil { + return Channel{}, false, err + } + wantedForward := currentUserID + "__" + peerID + wantedReverse := peerID + "__" + currentUserID + seen := make(map[string]Channel) + var match Channel + found := false + for _, channel := range []Channel(decoded) { + if !directChannelContains(channel.Name, currentUserID) { + return Channel{}, false, ErrInvalidChannelResponse + } + if previous, duplicate := seen[channel.ID]; duplicate && previous != channel { + return Channel{}, false, ErrInvalidChannelsResponse + } + seen[channel.ID] = channel + if channel.Name != wantedForward && channel.Name != wantedReverse { + continue + } + if !canonicalChannelRequestID(channel.ID) { + return Channel{}, false, ErrInvalidChannelsResponse + } + if found { + return Channel{}, false, ErrInvalidChannelsResponse + } + match, found = channel, true + } + return match, found, nil +} + // GroupList returns only G channels from the canonical current-user channel // listing. That authenticated, same-session endpoint is itself the membership // proof for discovered channels; per-channel Member calls would turn one diff --git a/internal/mattermost/channels_test.go b/internal/mattermost/channels_test.go index f697b01..d329ecf 100644 --- a/internal/mattermost/channels_test.go +++ b/internal/mattermost/channels_test.go @@ -435,6 +435,65 @@ func TestDirectListDedupesExactDirectChannelDuplicates(t *testing.T) { } } +func TestExistingDirectFindsExactPairInEitherOrder(t *testing.T) { + for _, name := range []string{"user__peer", "peer__user"} { + t.Run(name, func(t *testing.T) { + payload := `[{"id":"other","team_id":"","type":"D","name":"user__someone","display_name":""},{"id":"wanted","team_id":"","type":"D","name":"` + name + `","display_name":""}]` + f := &fakeChannelTransport{responses: map[string]string{"/users/user/channels": payload}} + got, found, err := NewChannels(f).ExistingDirect(context.Background(), "user", "peer") + if err != nil || !found || got.ID != "wanted" { + t.Fatalf("channel=%#v found=%v error=%v", got, found, err) + } + if !reflect.DeepEqual(f.paths, []string{"/users/user/channels"}) { + t.Fatalf("paths=%v", f.paths) + } + }) + } +} + +func TestExistingDirectReturnsCleanNone(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{"/users/user/channels": `[{"id":"other","team_id":"","type":"D","name":"user__someone","display_name":""}]`}} + got, found, err := NewChannels(f).ExistingDirect(context.Background(), "user", "peer") + if err != nil || found || got != (Channel{}) { + t.Fatalf("channel=%#v found=%v error=%v", got, found, err) + } +} + +func TestExistingDirectPreservesCanonicalSelfDM(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{"/users/user/channels": `[{"id":"self","team_id":"","type":"D","name":"user__user","display_name":""}]`}} + got, found, err := NewChannels(f).ExistingDirect(context.Background(), "user", "user") + if err != nil || !found || got.ID != "self" { + t.Fatalf("channel=%#v found=%v error=%v", got, found, err) + } +} + +func TestExistingDirectRejectsNonCanonicalMatchingChannelID(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{"/users/user/channels": `[{"id":" bad ","team_id":"","type":"D","name":"user__peer","display_name":""}]`}} + if _, _, err := NewChannels(f).ExistingDirect(context.Background(), "user", "peer"); !errors.Is(err, ErrInvalidChannelsResponse) { + t.Fatalf("error=%v", err) + } +} + +func TestExistingDirectRejectsSelfInvalidAndDuplicateMatches(t *testing.T) { + for _, ids := range [][2]string{{"me", "peer"}, {"user", "me"}, {" user", "peer"}, {"user", ""}} { + f := &fakeChannelTransport{responses: map[string]string{}} + if _, _, err := NewChannels(f).ExistingDirect(context.Background(), ids[0], ids[1]); !errors.Is(err, ErrInvalidChannelRequest) || len(f.paths) != 0 { + t.Fatalf("ids=%q error=%v paths=%v", ids, err, f.paths) + } + } + for name, payload := range map[string]string{ + "identical row": `[{"id":"one","team_id":"","type":"D","name":"user__peer","display_name":""},{"id":"one","team_id":"","type":"D","name":"user__peer","display_name":""}]`, + "distinct IDs": `[{"id":"one","team_id":"","type":"D","name":"user__peer","display_name":""},{"id":"two","team_id":"","type":"D","name":"peer__user","display_name":""}]`, + } { + t.Run(name, func(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{"/users/user/channels": payload}} + if _, _, err := NewChannels(f).ExistingDirect(context.Background(), "user", "peer"); !errors.Is(err, ErrInvalidChannelsResponse) { + t.Fatalf("error=%v", err) + } + }) + } +} + func TestGroupListUsesCanonicalListingAsBoundedMembershipProof(t *testing.T) { group := `{"id":"group","team_id":"","type":"G","name":"opaque","display_name":"Crew","last_post_at":0,"total_msg_count":0}` f := &fakeChannelTransport{responses: map[string]string{ @@ -468,13 +527,15 @@ func TestGroupListRejectsMalformedFocusedChannelsAndMembership(t *testing.T) { func TestChannelReadsAreRaceSafe(t *testing.T) { f := &fakeChannelTransport{responses: map[string]string{ - "/channels/x": `{"id":"x","team_id":"team","type":"O","name":"general","display_name":"General","last_post_at":0,"total_msg_count":0}`, + "/channels/x": `{"id":"x","team_id":"team","type":"O","name":"general","display_name":"General","last_post_at":0,"total_msg_count":0}`, + "/users/user/channels": `[{"id":"dm","team_id":"","type":"D","name":"user__peer","display_name":""}]`, }} channels := NewChannels(f) var wg sync.WaitGroup for range 40 { - wg.Add(1) + wg.Add(2) go func() { defer wg.Done(); _, _ = channels.ByID(context.Background(), "x") }() + go func() { defer wg.Done(); _, _, _ = channels.ExistingDirect(context.Background(), "user", "peer") }() } wg.Wait() } diff --git a/internal/mattermost/posts.go b/internal/mattermost/posts.go index 494679a..4b6afe5 100644 --- a/internal/mattermost/posts.go +++ b/internal/mattermost/posts.go @@ -579,6 +579,60 @@ type Posts struct{ client postTransport } func NewPosts(client postTransport) *Posts { return &Posts{client: client} } +type canonicalSinglePost struct{ Post Post } + +func (p *canonicalSinglePost) UnmarshalJSON(data []byte) error { + var raw struct { + ID json.RawMessage `json:"id"` + ChannelID json.RawMessage `json:"channel_id"` + UserID json.RawMessage `json:"user_id"` + Message json.RawMessage `json:"message"` + CreateAt json.RawMessage `json:"create_at"` + UpdateAt json.RawMessage `json:"update_at"` + DeleteAt json.RawMessage `json:"delete_at"` + RootID json.RawMessage `json:"root_id"` + } + if json.Unmarshal(data, &raw) != nil { + return ErrInvalidPostResponse + } + _, idOK := safePostID(raw.ID) + _, channelOK := safePostID(raw.ChannelID) + _, userOK := safePostID(raw.UserID) + _, messageOK := strictString(raw.Message) + createAt, createOK := nonnegativeInteger(raw.CreateAt) + updateAt, updateOK := nonnegativeInteger(raw.UpdateAt) + deleteAt, deleteOK := nonnegativeInteger(raw.DeleteAt) + rootID, rootOK := strictString(raw.RootID) + rootShapeOK := rootOK && (rootID == "" || isSafePostID(rootID)) + if !idOK || !channelOK || !userOK || !messageOK || !createOK || createAt == 0 || createAt > maxDateMilliseconds || + !updateOK || updateAt == 0 || updateAt > maxDateMilliseconds || !deleteOK || deleteAt != 0 || !rootShapeOK { + return ErrInvalidPostResponse + } + var post Post + if json.Unmarshal(data, &post) != nil { + return ErrInvalidPostResponse + } + p.Post = post + return nil +} + +// ByID returns one exact, live post. The response must carry the canonical +// identity fields needed to bind later operations to the requested post. +func (s *Posts) ByID(ctx context.Context, postID string) (Post, error) { + if !isSafePostID(postID) { + return Post{}, ErrInvalidPostsRequest + } + var decoded canonicalSinglePost + if err := s.client.Get(ctx, "/posts/"+url.PathEscape(postID), &decoded); err != nil { + return Post{}, err + } + post := decoded.Post + if post.ID != postID { + return Post{}, ErrInvalidPostResponse + } + return post, nil +} + func (s *Posts) SearchPage(ctx context.Context, teamID string, options SearchPageOptions) (SearchPage, error) { if strings.TrimSpace(teamID) == "" || strings.TrimSpace(options.Terms) == "" || options.Page < 0 || options.PerPage <= 0 || options.PerPage > MaxSearchPage { return SearchPage{}, ErrInvalidPostsRequest diff --git a/internal/mattermost/posts_test.go b/internal/mattermost/posts_test.go index 5579bbc..a26178c 100644 --- a/internal/mattermost/posts_test.go +++ b/internal/mattermost/posts_test.go @@ -6,6 +6,7 @@ import ( "errors" "reflect" "strings" + "sync" "testing" ) @@ -28,6 +29,89 @@ func (f searchTransportFunc) PostRead(ctx context.Context, path string, body, ou return f(ctx, path, body, out) } +func TestPostByIDBuildsExactGETAndRequiresCanonicalLivePost(t *testing.T) { + var gotPath string + api := NewPosts(postTransportFunc(func(_ context.Context, path string, out any) error { + gotPath = path + return json.Unmarshal([]byte(`{"id":"post","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":""}`), out) + })) + post, err := api.ByID(context.Background(), "post") + if err != nil || post.ID != "post" || post.ChannelID != "channel" || post.UserID != "author" { + t.Fatalf("post=%#v error=%v", post, err) + } + if gotPath != "/posts/post" { + t.Fatalf("path=%q", gotPath) + } +} + +func TestPostByIDRejectsInvalidRequestMismatchMalformedAndDeleted(t *testing.T) { + for _, id := range []string{"", " post", "post ", "slash/id", "nonascii-é"} { + called := false + api := NewPosts(postTransportFunc(func(context.Context, string, any) error { called = true; return nil })) + if _, err := api.ByID(context.Background(), id); !errors.Is(err, ErrInvalidPostsRequest) || called { + t.Fatalf("id=%q error=%v called=%v", id, err, called) + } + } + for name, payload := range map[string]string{ + "mismatch": `{"id":"other","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":""}`, + "malformed": `{"id":"post","channel_id":"channel","user_id":"author","create_at":1,"update_at":1,"delete_at":0,"root_id":""}`, + "missing author": `{"id":"post","channel_id":"channel","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":""}`, + "missing update": `{"id":"post","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"delete_at":0,"root_id":""}`, + "oversized update": `{"id":"post","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":8640000000000001,"delete_at":0,"root_id":""}`, + "whitespace channel": `{"id":"post","channel_id":" channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":""}`, + "whitespace author": `{"id":"post","channel_id":"channel","user_id":"author ","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":""}`, + "missing root": `{"id":"post","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0}`, + "wrong-type root": `{"id":"post","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":7}`, + "unsafe root": `{"id":"post","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":"bad/root"}`, + "deleted": `{"id":"post","channel_id":"channel","user_id":"author","message":"stale","create_at":1,"update_at":1,"delete_at":2,"root_id":""}`, + } { + t.Run(name, func(t *testing.T) { + api := NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { + return json.Unmarshal([]byte(payload), out) + })) + if _, err := api.ByID(context.Background(), "post"); !errors.Is(err, ErrInvalidPostResponse) { + t.Fatalf("error=%v", err) + } + }) + } +} + +func TestPostByIDPropagatesCancellationAndTransportErrors(t *testing.T) { + sentinel := errors.New("transport failed") + api := NewPosts(postTransportFunc(func(_ context.Context, _ string, _ any) error { return sentinel })) + if _, err := api.ByID(context.Background(), "post"); !errors.Is(err, sentinel) { + t.Fatalf("error=%v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + api = NewPosts(postTransportFunc(func(ctx context.Context, _ string, _ any) error { return ctx.Err() })) + if _, err := api.ByID(ctx, "post"); !errors.Is(err, context.Canceled) { + t.Fatalf("error=%v", err) + } +} + +func TestPostByIDAcceptsCanonicalReplyRoot(t *testing.T) { + api := NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { + return json.Unmarshal([]byte(`{"id":"reply","channel_id":"channel","user_id":"author","message":"hello","create_at":2,"update_at":2,"delete_at":0,"root_id":"root"}`), out) + })) + post, err := api.ByID(context.Background(), "reply") + if err != nil || post.RootID != "root" { + t.Fatalf("post=%#v error=%v", post, err) + } +} + +func TestPostByIDIsRaceSafe(t *testing.T) { + api := NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { + return json.Unmarshal([]byte(`{"id":"post","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":""}`), out) + })) + var wg sync.WaitGroup + for range 40 { + wg.Add(1) + go func() { defer wg.Done(); _, _ = api.ByID(context.Background(), "post") }() + } + wg.Wait() +} + func TestOrderedPostsPageNormalizesAndSuppressesDeleted(t *testing.T) { var page OrderedPostsPage err := json.Unmarshal([]byte(`{ From 4b1d1862211f27259efa310df060574df91e751c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 00:43:52 +0300 Subject: [PATCH 055/119] feat: add secure attachment binding --- internal/stageinput/doc.go | 14 + internal/stageinput/fifo_other_test.go | 7 + internal/stageinput/fifo_unix_test.go | 7 + internal/stageinput/file_darwin.go | 46 +++ internal/stageinput/file_linux.go | 58 ++++ internal/stageinput/file_other.go | 12 + internal/stageinput/file_unix.go | 79 +++++ internal/stageinput/input.go | 284 +++++++++++++++++ internal/stageinput/input_test.go | 411 +++++++++++++++++++++++++ 9 files changed, 918 insertions(+) create mode 100644 internal/stageinput/doc.go create mode 100644 internal/stageinput/fifo_other_test.go create mode 100644 internal/stageinput/fifo_unix_test.go create mode 100644 internal/stageinput/file_darwin.go create mode 100644 internal/stageinput/file_linux.go create mode 100644 internal/stageinput/file_other.go create mode 100644 internal/stageinput/file_unix.go create mode 100644 internal/stageinput/input.go create mode 100644 internal/stageinput/input_test.go diff --git a/internal/stageinput/doc.go b/internal/stageinput/doc.go new file mode 100644 index 0000000..93bc8b3 --- /dev/null +++ b/internal/stageinput/doc.go @@ -0,0 +1,14 @@ +// Package stageinput binds attachment metadata to a verified file snapshot. +// +// Binding deliberately does not spool content. The apply layer owns a second +// secure reopen, credential scan, digest comparison, and private spool before +// upload. Therefore a path changed after Bind leaves the recorded snapshot +// untouched and must become an apply-time conflict. +// +// Darwin and Linux are supported. Linux uses O_PATH for leaf preflight. Darwin +// has no equivalent, refuses group/other-writable leaf parents, and uses +// descriptor-relative fstatat with no-follow before read-open, followed by +// mandatory descriptor identity comparison. Both reject +// non-local or unknown filesystems: context cancellation is checked between +// reads, but a blocking regular-file read itself cannot be interrupted portably. +package stageinput diff --git a/internal/stageinput/fifo_other_test.go b/internal/stageinput/fifo_other_test.go new file mode 100644 index 0000000..57da278 --- /dev/null +++ b/internal/stageinput/fifo_other_test.go @@ -0,0 +1,7 @@ +//go:build !darwin && !linux + +package stageinput + +import "errors" + +func makeFIFO(string) error { return errors.New("unsupported") } diff --git a/internal/stageinput/fifo_unix_test.go b/internal/stageinput/fifo_unix_test.go new file mode 100644 index 0000000..84d7eab --- /dev/null +++ b/internal/stageinput/fifo_unix_test.go @@ -0,0 +1,7 @@ +//go:build darwin || linux + +package stageinput + +import "golang.org/x/sys/unix" + +func makeFIFO(path string) error { return unix.Mkfifo(path, 0o600) } diff --git a/internal/stageinput/file_darwin.go b/internal/stageinput/file_darwin.go new file mode 100644 index 0000000..278093b --- /dev/null +++ b/internal/stageinput/file_darwin.go @@ -0,0 +1,46 @@ +//go:build darwin + +package stageinput + +import "golang.org/x/sys/unix" + +func platformIdentity(stat *unix.Stat_t) fileIdentity { + return fileIdentity{dev: uint64(stat.Dev), ino: stat.Ino, mode: uint32(stat.Mode), nlink: uint64(stat.Nlink), size: stat.Size, + mtimeNsec: int64(stat.Mtim.Sec)*1e9 + int64(stat.Mtim.Nsec), ctimeNsec: int64(stat.Ctim.Sec)*1e9 + int64(stat.Ctim.Nsec)} +} + +// Darwin has no O_PATH. Refuse attacker-writable parents, then compare the +// no-follow pathname observation with the opened descriptor before reading. +func platformOpenLeaf(parent int, name string) (int, fileIdentity, error) { + var directory unix.Stat_t + if err := unix.Fstat(parent, &directory); err != nil || directory.Uid != uint32(unix.Geteuid()) || uint32(directory.Mode)&0o022 != 0 { + return -1, fileIdentity{}, ErrUnsafeFile + } + var stat unix.Stat_t + if err := unix.Fstatat(parent, name, &stat, unix.AT_SYMLINK_NOFOLLOW); err != nil { + return -1, fileIdentity{}, err + } + preflight, err := identityFromStat(&stat) + if err != nil { + return -1, fileIdentity{}, err + } + fd, err := unix.Openat(parent, name, unix.O_RDONLY|unix.O_CLOEXEC|unix.O_NOFOLLOW|unix.O_NONBLOCK, 0) + if err != nil { + return -1, fileIdentity{}, err + } + if err := unix.Fstat(fd, &stat); err != nil { + unix.Close(fd) + return -1, fileIdentity{}, err + } + opened, err := identityFromStat(&stat) + if err != nil || !preflight.stable(opened) || !platformLocal(fd) { + unix.Close(fd) + return -1, fileIdentity{}, ErrUnsafeFile + } + return fd, opened, nil +} + +func platformLocal(fd int) bool { + var stat unix.Statfs_t + return unix.Fstatfs(fd, &stat) == nil && stat.Flags&unix.MNT_LOCAL != 0 +} diff --git a/internal/stageinput/file_linux.go b/internal/stageinput/file_linux.go new file mode 100644 index 0000000..d2742e7 --- /dev/null +++ b/internal/stageinput/file_linux.go @@ -0,0 +1,58 @@ +//go:build linux + +package stageinput + +import ( + "strconv" + + "golang.org/x/sys/unix" +) + +func platformIdentity(stat *unix.Stat_t) fileIdentity { + return fileIdentity{dev: uint64(stat.Dev), ino: stat.Ino, mode: uint32(stat.Mode), nlink: uint64(stat.Nlink), size: stat.Size, + mtimeNsec: int64(stat.Mtim.Sec)*1e9 + int64(stat.Mtim.Nsec), ctimeNsec: int64(stat.Ctim.Sec)*1e9 + int64(stat.Ctim.Nsec)} +} + +func platformOpenLeaf(parent int, name string) (int, fileIdentity, error) { + pathfd, err := unix.Openat(parent, name, unix.O_PATH|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0) + if err != nil { + return -1, fileIdentity{}, err + } + defer unix.Close(pathfd) + var stat unix.Stat_t + if err := unix.Fstat(pathfd, &stat); err != nil { + return -1, fileIdentity{}, err + } + preflight, err := identityFromStat(&stat) + if err != nil || !platformLocal(pathfd) { + return -1, fileIdentity{}, ErrUnsafeFile + } + // procfs resolves the retained descriptor, never the mutable directory entry. + fd, err := unix.Open("/proc/self/fd/"+strconv.Itoa(pathfd), unix.O_RDONLY|unix.O_CLOEXEC|unix.O_NONBLOCK, 0) + if err != nil { + return -1, fileIdentity{}, err + } + if err := unix.Fstat(fd, &stat); err != nil { + unix.Close(fd) + return -1, fileIdentity{}, err + } + opened, err := identityFromStat(&stat) + if err != nil || !preflight.stable(opened) || !platformLocal(fd) { + unix.Close(fd) + return -1, fileIdentity{}, ErrUnsafeFile + } + return fd, opened, nil +} + +func platformLocal(fd int) bool { + var stat unix.Statfs_t + if unix.Fstatfs(fd, &stat) != nil { + return false + } + switch uint64(stat.Type) { + case 0xEF53, 0x58465342, 0x9123683E, 0x01021994, 0x794C7630, 0x2FC12FC1: + return true + default: + return false + } +} diff --git a/internal/stageinput/file_other.go b/internal/stageinput/file_other.go new file mode 100644 index 0000000..6cbb320 --- /dev/null +++ b/internal/stageinput/file_other.go @@ -0,0 +1,12 @@ +//go:build !darwin && !linux + +package stageinput + +import "os" + +type fileIdentity struct{ size int64 } + +func (a fileIdentity) sameFile(fileIdentity) bool { return false } +func (a fileIdentity) stable(fileIdentity) bool { return false } +func openSecure(string) (*os.File, fileIdentity, error) { return nil, fileIdentity{}, ErrUnsupported } +func fileIdentityOf(*os.File) (fileIdentity, error) { return fileIdentity{}, ErrUnsupported } diff --git a/internal/stageinput/file_unix.go b/internal/stageinput/file_unix.go new file mode 100644 index 0000000..914f7c0 --- /dev/null +++ b/internal/stageinput/file_unix.go @@ -0,0 +1,79 @@ +//go:build darwin || linux + +package stageinput + +import ( + "errors" + "os" + "path/filepath" + "strings" + + "golang.org/x/sys/unix" +) + +type fileIdentity struct { + dev, ino uint64 + mode uint32 + nlink uint64 + size int64 + mtimeNsec int64 + ctimeNsec int64 +} + +func (a fileIdentity) sameFile(b fileIdentity) bool { return a.dev == b.dev && a.ino == b.ino } +func (a fileIdentity) stable(b fileIdentity) bool { + return a.sameFile(b) && a.mode == b.mode && a.nlink == b.nlink && a.size == b.size && a.mtimeNsec == b.mtimeNsec && a.ctimeNsec == b.ctimeNsec +} + +func openSecure(path string) (*os.File, fileIdentity, error) { + if !filepath.IsAbs(path) { + return nil, fileIdentity{}, ErrInvalid + } + parts := strings.Split(strings.TrimPrefix(filepath.Clean(path), string(filepath.Separator)), string(filepath.Separator)) + if len(parts) == 0 || parts[0] == "" { + return nil, fileIdentity{}, ErrInvalid + } + parent, err := unix.Open("/", unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0) + if err != nil { + return nil, fileIdentity{}, ErrUnsafeFile + } + for _, part := range parts[:len(parts)-1] { + next, openErr := unix.Openat(parent, part, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_NOFOLLOW|unix.O_NONBLOCK, 0) + _ = unix.Close(parent) + if openErr != nil { + return nil, fileIdentity{}, ErrUnsafeFile + } + parent = next + } + name := parts[len(parts)-1] + fd, opened, openErr := platformOpenLeaf(parent, name) + _ = unix.Close(parent) + if openErr != nil { + return nil, fileIdentity{}, ErrUnsafeFile + } + if err := unix.SetNonblock(fd, false); err != nil { + _ = unix.Close(fd) + return nil, fileIdentity{}, ErrUnsafeFile + } + file := os.NewFile(uintptr(fd), path) + if file == nil { + _ = unix.Close(fd) + return nil, fileIdentity{}, ErrUnsafeFile + } + return file, opened, nil +} + +func fileIdentityOf(file *os.File) (fileIdentity, error) { + var stat unix.Stat_t + if err := unix.Fstat(int(file.Fd()), &stat); err != nil { + return fileIdentity{}, err + } + return identityFromStat(&stat) +} + +func identityFromStat(stat *unix.Stat_t) (fileIdentity, error) { + if stat.Mode&unix.S_IFMT != unix.S_IFREG || stat.Nlink != 1 { + return fileIdentity{}, errors.New("not regular") + } + return platformIdentity(stat), nil +} diff --git a/internal/stageinput/input.go b/internal/stageinput/input.go new file mode 100644 index 0000000..b27db06 --- /dev/null +++ b/internal/stageinput/input.go @@ -0,0 +1,284 @@ +// Package stageinput validates and binds untrusted attachment inputs before they +// are admitted to the stage store. +package stageinput + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "io" + "mime" + "net/http" + "path/filepath" + "strings" + "unicode" + "unicode/utf8" + + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +const ( + MaxAttachments = 100 + maxPathBytes = 4096 + maxFilenameBytes = 255 + maxMediaTypeBytes = 255 + maxCredentialCount = 64 + maxCredentialBytes = 4096 + maxCredentialsBytes = 64 << 10 +) + +var ( + ErrInvalid = errors.New("stage input: invalid attachment") + ErrCredential = errors.New("stage input: protected credential present") + ErrUnsafeFile = errors.New("stage input: unsafe attachment file") + ErrFileChanged = errors.New("stage input: attachment changed while binding") + ErrUnsupported = errors.New("stage input: secure attachment binding unsupported on this platform") + ErrTooMany = errors.New("stage input: too many attachments") + ErrCredentialSet = errors.New("stage input: invalid protected credential set") +) + +type Attachment struct { + Path string + RemoteFilename string // empty derives the caller path's safe basename + MediaType string // empty detects MIME from the first 512 file bytes +} + +// Bind records a durable-at-rest snapshot only. A later apply must securely +// reopen the canonical path, rescan credentials, rehash, and spool before any +// upload; a changed path must conflict with this recorded identity and digest. +// Bind returns no partial result and never returns contaminated values. +func Bind(ctx context.Context, inputs []Attachment, credentials [][]byte) ([]stagestore.Attachment, error) { + if ctx == nil { + return nil, ErrInvalid + } + if err := ctx.Err(); err != nil { + return nil, err + } + if len(inputs) > MaxAttachments { + return nil, ErrTooMany + } + scanner, err := newScanner(credentials) + if err != nil { + return nil, err + } + prepared := make([]preparedAttachment, len(inputs)) + for i, input := range inputs { + prepared[i], err = prepareMetadata(input) + if err != nil { + return nil, err + } + if scanner.contains([]byte(input.Path)) || scanner.contains([]byte(input.RemoteFilename)) || + scanner.contains([]byte(input.MediaType)) || scanner.contains([]byte(prepared[i].canonical)) || + scanner.contains([]byte(prepared[i].filename)) || scanner.contains([]byte(prepared[i].mediaType)) { + return nil, ErrCredential + } + } + bound := make([]stagestore.Attachment, 0, len(prepared)) + for _, input := range prepared { + if err := ctx.Err(); err != nil { + return nil, err + } + file, before, err := openSecure(input.canonical) + if err != nil { + return nil, err + } + digest, length, prefix, scanErr := scanFile(ctx, file, scanner.stream()) + after, statErr := fileIdentityOf(file) + closeErr := file.Close() + if scanErr != nil { + return nil, scanErr + } + if statErr != nil || closeErr != nil { + return nil, ErrUnsafeFile + } + if !before.stable(after) || length != before.size { + return nil, ErrFileChanged + } + mediaType := input.mediaType + if mediaType == "" { + mediaType = http.DetectContentType(prefix) + if scanner.contains([]byte(mediaType)) { + return nil, ErrCredential + } + } + reopened, reopenedID, err := openSecure(input.canonical) + if err != nil { + return nil, ErrFileChanged + } + reopenCloseErr := reopened.Close() + if reopenCloseErr != nil || !before.stable(reopenedID) { + return nil, ErrFileChanged + } + bound = append(bound, stagestore.Attachment{SuppliedPath: input.supplied, CanonicalPath: input.canonical, + RemoteFilename: input.filename, ByteLength: length, MediaType: mediaType, ContentDigest: digest}) + } + return bound, nil +} + +type preparedAttachment struct{ supplied, canonical, filename, mediaType string } + +func prepareMetadata(input Attachment) (preparedAttachment, error) { + if !validText(input.Path, maxPathBytes) || strings.TrimSpace(input.Path) != input.Path { + return preparedAttachment{}, ErrInvalid + } + absolute, err := filepath.Abs(input.Path) + if err != nil { + return preparedAttachment{}, ErrInvalid + } + canonical := filepath.Clean(absolute) + if !validText(canonical, maxPathBytes) { + return preparedAttachment{}, ErrInvalid + } + filename := input.RemoteFilename + if filename == "" { + filename = filepath.Base(canonical) + } + if !validText(filename, maxFilenameBytes) || strings.TrimSpace(filename) != filename || filename == "." || filename == ".." || strings.ContainsAny(filename, `/\`) || filepath.Base(filename) != filename { + return preparedAttachment{}, ErrInvalid + } + mediaType := input.MediaType + if mediaType != "" { + if !validASCII(mediaType, maxMediaTypeBytes) { + return preparedAttachment{}, ErrInvalid + } + parsed, params, err := mime.ParseMediaType(mediaType) + if err != nil || parsed == "" { + return preparedAttachment{}, ErrInvalid + } + mediaType = mime.FormatMediaType(parsed, params) + if !validASCII(mediaType, maxMediaTypeBytes) { + return preparedAttachment{}, ErrInvalid + } + } + return preparedAttachment{input.Path, canonical, filename, mediaType}, nil +} + +func validText(value string, maximum int) bool { + if value == "" || len(value) > maximum || !utf8.ValidString(value) { + return false + } + for _, r := range value { + if unicode.IsControl(r) { + return false + } + } + return true +} + +func validASCII(value string, maximum int) bool { + if len(value) > maximum { + return false + } + for i := range len(value) { + if value[i] < 0x20 || value[i] > 0x7e { + return false + } + } + return true +} + +func scanFile(ctx context.Context, input io.Reader, scanner *streamScanner) ([32]byte, int64, []byte, error) { + hash := sha256.New() + buf := make([]byte, 32*1024) + prefix := make([]byte, 0, 512) + var length int64 + for { + if err := ctx.Err(); err != nil { + return [32]byte{}, 0, nil, err + } + n, err := input.Read(buf) + if n > 0 { + chunk := buf[:n] + length += int64(n) + _, _ = hash.Write(chunk) + if len(prefix) < cap(prefix) { + prefix = append(prefix, chunk[:min(len(chunk), cap(prefix)-len(prefix))]...) + } + if scanner.write(chunk) { + return [32]byte{}, 0, nil, ErrCredential + } + } + if err == io.EOF { + if contextErr := ctx.Err(); contextErr != nil { + return [32]byte{}, 0, nil, contextErr + } + var digest [32]byte + copy(digest[:], hash.Sum(nil)) + return digest, length, prefix, nil + } + if err != nil || n == 0 { + return [32]byte{}, 0, nil, ErrUnsafeFile + } + } +} + +type tokenPattern struct { + value []byte + failure []int +} +type tokenScanner struct{ patterns []tokenPattern } +type streamScanner struct { + patterns []tokenPattern + states []int +} + +func newScanner(tokens [][]byte) (tokenScanner, error) { + if len(tokens) > maxCredentialCount { + return tokenScanner{}, ErrCredentialSet + } + s := tokenScanner{patterns: make([]tokenPattern, 0, len(tokens))} + total := 0 + for _, token := range tokens { + if len(token) == 0 { + continue + } + total += len(token) + if len(token) > maxCredentialBytes || total > maxCredentialsBytes { + return tokenScanner{}, ErrCredentialSet + } + value := bytes.Clone(token) + failure := make([]int, len(value)) + for i, j := 1, 0; i < len(value); i++ { + for j > 0 && value[i] != value[j] { + j = failure[j-1] + } + if value[i] == value[j] { + j++ + } + failure[i] = j + } + s.patterns = append(s.patterns, tokenPattern{value, failure}) + } + return s, nil +} + +func (s tokenScanner) contains(value []byte) bool { + for _, pattern := range s.patterns { + if bytes.Contains(value, pattern.value) { + return true + } + } + return false +} +func (s tokenScanner) stream() *streamScanner { + return &streamScanner{s.patterns, make([]int, len(s.patterns))} +} +func (s *streamScanner) write(chunk []byte) bool { + for _, b := range chunk { + for i, pattern := range s.patterns { + state := s.states[i] + for state > 0 && b != pattern.value[state] { + state = pattern.failure[state-1] + } + if b == pattern.value[state] { + state++ + } + if state == len(pattern.value) { + return true + } + s.states[i] = state + } + } + return false +} diff --git a/internal/stageinput/input_test.go b/internal/stageinput/input_test.go new file mode 100644 index 0000000..f71bd44 --- /dev/null +++ b/internal/stageinput/input_test.go @@ -0,0 +1,411 @@ +package stageinput + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "io" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +func TestTokenScannerEveryChunkBoundary(t *testing.T) { + token := []byte("active-token") + content := append(append([]byte("prefix:"), token...), []byte(":suffix")...) + for split := 0; split <= len(content); split++ { + scanner := mustScanner(t, [][]byte{token, []byte("another")}).stream() + if found := scanner.write(content[:split]) || scanner.write(content[split:]); !found { + t.Fatalf("split %d was not detected", split) + } + } +} + +func TestScanFileBinaryAndCredential(t *testing.T) { + binary := append([]byte{0, 0xff, 1, 2}, bytes.Repeat([]byte{0x81, 0}, 20_000)...) + digest, length, _, err := scanFile(context.Background(), &oneByteReader{data: binary}, mustScanner(t, [][]byte{[]byte("absent")}).stream()) + if err != nil || length != int64(len(binary)) || digest != sha256.Sum256(binary) { + t.Fatalf("digest=%x length=%d err=%v", digest, length, err) + } + contaminated := append(bytes.Clone(binary), []byte("secret")...) + if _, _, _, err := scanFile(context.Background(), &oneByteReader{data: contaminated}, mustScanner(t, [][]byte{[]byte("secret")}).stream()); !errors.Is(err, ErrCredential) { + t.Fatalf("error=%v", err) + } +} + +func TestBindCapturesMetadataAndRejectsMetadataCredential(t *testing.T) { + path := filepath.Join(localTempDir(t), "payload.bin") + content := []byte{0, 1, 2, 0xff} + if err := os.WriteFile(path, content, 0o600); err != nil { + t.Fatal(err) + } + got, err := Bind(context.Background(), []Attachment{{Path: path, RemoteFilename: "payload.bin", MediaType: "application/octet-stream"}}, [][]byte{[]byte("active-token")}) + if err != nil || len(got) != 1 { + t.Fatalf("attachments=%+v err=%v", got, err) + } + if got[0].SuppliedPath != path || got[0].CanonicalPath != filepath.Clean(path) || got[0].RemoteFilename != "payload.bin" || got[0].MediaType != "application/octet-stream" || got[0].ByteLength != 4 || got[0].ContentDigest != sha256.Sum256(content) { + t.Fatalf("attachment=%+v", got[0]) + } + for _, input := range []Attachment{ + {Path: path, RemoteFilename: "active-token.bin"}, + {Path: path, RemoteFilename: "x", MediaType: "x/active-token"}, + } { + if result, err := Bind(context.Background(), []Attachment{input}, [][]byte{[]byte("active-token")}); !errors.Is(err, ErrCredential) || result != nil { + t.Fatalf("result=%v error=%v", result, err) + } + } +} + +func TestBindDerivesSafeFilenameAndMediaType(t *testing.T) { + path := filepath.Join(localTempDir(t), "note.txt") + if err := os.WriteFile(path, []byte("plain text\n"), 0o600); err != nil { + t.Fatal(err) + } + got, err := Bind(context.Background(), []Attachment{{Path: path}}, nil) + if err != nil { + t.Fatal(err) + } + if got[0].RemoteFilename != "note.txt" || got[0].MediaType != "text/plain; charset=utf-8" { + t.Fatalf("derived metadata=%+v", got[0]) + } +} + +func TestBindValidatesAllMetadataBeforeFilesystemIO(t *testing.T) { + missing := filepath.Join(localTempDir(t), "missing") + for name, input := range map[string]Attachment{ + "path control": {Path: missing + "\n"}, + "path whitespace": {Path: " " + missing}, + "path length": {Path: strings.Repeat("a", maxPathBytes+1)}, + "filename traversal": {Path: missing, RemoteFilename: "../x"}, + "filename control": {Path: missing, RemoteFilename: "x\n"}, + "filename whitespace": {Path: missing, RemoteFilename: " x"}, + "filename length": {Path: missing, RemoteFilename: strings.Repeat("a", maxFilenameBytes+1)}, + "media invalid": {Path: missing, MediaType: "not a mime"}, + "media control": {Path: missing, MediaType: "text/plain\n"}, + } { + t.Run(name, func(t *testing.T) { + if _, err := Bind(context.Background(), []Attachment{input}, nil); !errors.Is(err, ErrInvalid) { + t.Fatalf("error=%v", err) + } + }) + } +} + +func TestBindCanonicalizesMediaType(t *testing.T) { + path := filepath.Join(localTempDir(t), "file") + if err := os.WriteFile(path, []byte("data"), 0o600); err != nil { + t.Fatal(err) + } + got, err := Bind(context.Background(), []Attachment{{Path: path, MediaType: `Text/Plain; Charset="UTF-8"`}}, nil) + if err != nil { + t.Fatal(err) + } + if got[0].MediaType != "text/plain; charset=UTF-8" { + t.Fatalf("media type=%q", got[0].MediaType) + } +} + +func TestBindScansOriginalMetadataBeforeCanonicalization(t *testing.T) { + path := filepath.Join(localTempDir(t), "file") + if err := os.WriteFile(path, []byte("data"), 0o600); err != nil { + t.Fatal(err) + } + input := Attachment{Path: path, RemoteFilename: "Report.TXT", MediaType: "TEXT/PLAIN"} + for _, token := range [][]byte{[]byte("Report.TXT"), []byte("TEXT/PLAIN")} { + if got, err := Bind(context.Background(), []Attachment{input}, [][]byte{token}); !errors.Is(err, ErrCredential) || got != nil { + t.Fatalf("token=%q result=%v err=%v", token, got, err) + } + } +} + +func TestBindRejectsHardlinkedFile(t *testing.T) { + dir := localTempDir(t) + path := filepath.Join(dir, "file") + link := filepath.Join(dir, "hardlink") + if err := os.WriteFile(path, []byte("data"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Link(path, link); err != nil { + t.Fatal(err) + } + if got, err := Bind(context.Background(), []Attachment{{Path: path}}, nil); !errors.Is(err, ErrUnsafeFile) || got != nil { + t.Fatalf("result=%v err=%v", got, err) + } +} + +func TestRecordedSnapshotDoesNotChangeWithLaterPath(t *testing.T) { + dir := localTempDir(t) + path := filepath.Join(dir, "file") + first := []byte("first snapshot") + if err := os.WriteFile(path, first, 0o600); err != nil { + t.Fatal(err) + } + got, err := Bind(context.Background(), []Attachment{{Path: path}}, nil) + if err != nil { + t.Fatal(err) + } + replacement := filepath.Join(dir, "replacement") + if err := os.WriteFile(replacement, []byte("later content"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Rename(replacement, path); err != nil { + t.Fatal(err) + } + if got[0].ContentDigest != sha256.Sum256(first) || got[0].ByteLength != int64(len(first)) { + t.Fatalf("recorded snapshot changed: %+v", got[0]) + } +} + +func TestCredentialSetBounds(t *testing.T) { + path := filepath.Join(localTempDir(t), "file") + if err := os.WriteFile(path, []byte("data"), 0o600); err != nil { + t.Fatal(err) + } + tooMany := make([][]byte, maxCredentialCount+1) + for i := range tooMany { + tooMany[i] = []byte("x") + } + for name, credentials := range map[string][][]byte{ + "count": tooMany, + "token": {bytes.Repeat([]byte("x"), maxCredentialBytes+1)}, + "total": {bytes.Repeat([]byte("x"), maxCredentialsBytes), []byte("y")}, + } { + t.Run(name, func(t *testing.T) { + if got, err := Bind(context.Background(), []Attachment{{Path: path}}, credentials); !errors.Is(err, ErrCredentialSet) || got != nil { + t.Fatalf("result=%v err=%v", got, err) + } + }) + } +} + +func TestStreamScannerDoesNotAllocatePerChunk(t *testing.T) { + scanner := mustScanner(t, [][]byte{[]byte("active-token"), []byte("second-token")}).stream() + chunk := bytes.Repeat([]byte("ordinary binary payload\x00"), 128) + if allocations := testing.AllocsPerRun(100, func() { + if scanner.write(chunk) { + t.Fatal("unexpected match") + } + }); allocations != 0 { + t.Fatalf("streaming allocations=%v", allocations) + } +} + +func TestBindCountBoundsAndNoPartialResult(t *testing.T) { + if got, err := Bind(context.Background(), nil, nil); err != nil || len(got) != 0 { + t.Fatalf("zero=%v err=%v", got, err) + } + dir := localTempDir(t) + path := filepath.Join(dir, "file") + if err := os.WriteFile(path, []byte("ok"), 0o600); err != nil { + t.Fatal(err) + } + inputs := make([]Attachment, MaxAttachments) + for i := range inputs { + inputs[i] = Attachment{Path: path, RemoteFilename: "file"} + } + if got, err := Bind(context.Background(), inputs, nil); err != nil || len(got) != MaxAttachments { + t.Fatalf("hundred=%d err=%v", len(got), err) + } + inputs = append(inputs, Attachment{Path: path, RemoteFilename: "file"}) + if got, err := Bind(context.Background(), inputs, nil); !errors.Is(err, ErrTooMany) || got != nil { + t.Fatalf("101=%v err=%v", got, err) + } + two := []Attachment{{Path: path, RemoteFilename: "ok"}, {Path: path, RemoteFilename: "active-token"}} + if got, err := Bind(context.Background(), two, [][]byte{[]byte("active-token")}); !errors.Is(err, ErrCredential) || got != nil { + t.Fatalf("partial=%v err=%v", got, err) + } +} + +func TestBindRejectsNonRegularAndSymlinks(t *testing.T) { + if runtime.GOOS != "darwin" && runtime.GOOS != "linux" { + t.Skip("secure descriptor walk is supported on darwin and linux") + } + dir := localTempDir(t) + regular := filepath.Join(dir, "regular") + if err := os.WriteFile(regular, []byte("ok"), 0o600); err != nil { + t.Fatal(err) + } + leaf := filepath.Join(dir, "leaf") + if err := os.Symlink(regular, leaf); err != nil { + t.Fatal(err) + } + ancestor := filepath.Join(dir, "ancestor") + if err := os.Symlink(dir, ancestor); err != nil { + t.Fatal(err) + } + paths := []string{dir, leaf, filepath.Join(ancestor, "regular")} + if runtime.GOOS != "windows" { + fifo := filepath.Join(dir, "fifo") + if err := makeFIFO(fifo); err != nil { + t.Fatal(err) + } + paths = append(paths, fifo) + } + for _, path := range paths { + if got, err := Bind(context.Background(), []Attachment{{Path: path, RemoteFilename: "x"}}, nil); !errors.Is(err, ErrUnsafeFile) || got != nil { + t.Errorf("path=%s result=%v err=%v", path, got, err) + } + } +} + +func TestBindRejectsCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if got, err := Bind(ctx, nil, nil); !errors.Is(err, context.Canceled) || got != nil { + t.Fatalf("result=%v err=%v", got, err) + } +} + +func TestScanFileObservesCancellationBetweenReads(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + reader := &cancelingReader{cancel: cancel} + _, _, _, err := scanFile(ctx, reader, mustScanner(t, nil).stream()) + if !errors.Is(err, context.Canceled) || reader.reads != 1 { + t.Fatalf("reads=%d err=%v", reader.reads, err) + } +} + +func TestBindRejectsConcurrentMutation(t *testing.T) { + for name, mutate := range map[string]func(*os.File, int64){ + "in-place": func(file *os.File, _ int64) { + _, _ = file.WriteAt([]byte("b"), 0) + _, _ = file.WriteAt([]byte("a"), 0) + }, + "append": func(file *os.File, size int64) { + _, _ = file.WriteAt([]byte("b"), size) + _ = file.Truncate(size) + }, + "truncate": func(file *os.File, size int64) { + _ = file.Truncate(size - 1) + _ = file.Truncate(size) + }, + } { + t.Run(name, func(t *testing.T) { + path := filepath.Join(localTempDir(t), "large") + const size = 16 << 20 + if err := os.WriteFile(path, bytes.Repeat([]byte("a"), size), 0o600); err != nil { + t.Fatal(err) + } + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + file, err := os.OpenFile(path, os.O_WRONLY, 0) + if err != nil { + return + } + defer file.Close() + for { + select { + case <-stop: + return + default: + mutate(file, size) + } + } + }() + time.Sleep(time.Millisecond) + got, err := Bind(context.Background(), []Attachment{{Path: path, RemoteFilename: "large"}}, nil) + close(stop) + <-done + if (!errors.Is(err, ErrFileChanged) && !errors.Is(err, ErrUnsafeFile)) || got != nil { + t.Fatalf("result=%v err=%v", got, err) + } + }) + } +} + +func TestBindRejectsReplacementDuringScan(t *testing.T) { + dir := localTempDir(t) + path := filepath.Join(dir, "large") + replacement := filepath.Join(dir, "replacement") + content := bytes.Repeat([]byte("a"), 32<<20) + if err := os.WriteFile(path, content, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(replacement, content, 0o600); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { + time.Sleep(time.Millisecond) + done <- os.Rename(replacement, path) + }() + got, err := Bind(context.Background(), []Attachment{{Path: path, RemoteFilename: "large"}}, nil) + if renameErr := <-done; renameErr != nil { + t.Fatal(renameErr) + } + if (!errors.Is(err, ErrFileChanged) && !errors.Is(err, ErrUnsafeFile)) || got != nil { + t.Fatalf("result=%v err=%v", got, err) + } +} + +func FuzzTokenScannerNeverMissesSplitCredential(f *testing.F) { + f.Add([]byte("credential"), []byte("prefix"), []byte("suffix"), uint8(3)) + f.Fuzz(func(t *testing.T, token, prefix, suffix []byte, splitByte uint8) { + if len(token) == 0 || len(token) > 256 || len(prefix)+len(suffix) > 1024 { + t.Skip() + } + value := append(append(bytes.Clone(prefix), token...), suffix...) + split := int(splitByte) % (len(value) + 1) + scanner, err := newScanner([][]byte{token}) + if err != nil { + t.Fatal(err) + } + stream := scanner.stream() + found := stream.write(value[:split]) || stream.write(value[split:]) + if !found { + t.Fatal("missed exact credential") + } + }) +} + +type oneByteReader struct{ data []byte } + +type cancelingReader struct { + cancel context.CancelFunc + reads int +} + +func (r *cancelingReader) Read(p []byte) (int, error) { + r.reads++ + p[0] = 'x' + r.cancel() + return 1, nil +} + +func mustScanner(t *testing.T, tokens [][]byte) tokenScanner { + t.Helper() + scanner, err := newScanner(tokens) + if err != nil { + t.Fatal(err) + } + return scanner +} + +func localTempDir(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp(".", ".stageinput-test-") + if err != nil { + t.Fatal(err) + } + absolute, err := filepath.Abs(dir) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(absolute) }) + return absolute +} + +func (r *oneByteReader) Read(p []byte) (int, error) { + if len(r.data) == 0 { + return 0, io.EOF + } + p[0] = r.data[0] + r.data = r.data[1:] + return 1, nil +} From c5bab453c1305c8acb32ebc829e4b91cffbc94e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 01:18:27 +0300 Subject: [PATCH 056/119] feat: add public stage contracts --- internal/cli/root_test.go | 2 +- internal/schema/stage_test.go | 281 ++++ schemas/v2/examples/stage-cancel-request.json | 1 + schemas/v2/examples/stage-preview.json | 1 + schemas/v2/examples/stage-receipt.json | 1 + schemas/v2/examples/stage-request.json | 1 + schemas/v2/examples/stage-revise-request.json | 1 + schemas/v2/examples/stage.json | 1 + schemas/v2/examples/stages.json | 1 + schemas/v2/stage-cancel-request.schema.json | 100 ++ schemas/v2/stage-preview.schema.json | 1311 +++++++++++++++ schemas/v2/stage-receipt.schema.json | 1163 +++++++++++++ schemas/v2/stage-request.schema.json | 584 +++++++ schemas/v2/stage-revise-request.schema.json | 124 ++ schemas/v2/stage.schema.json | 1484 +++++++++++++++++ schemas/v2/stages.schema.json | 1055 ++++++++++++ 16 files changed, 6110 insertions(+), 1 deletion(-) create mode 100644 internal/schema/stage_test.go create mode 100644 schemas/v2/examples/stage-cancel-request.json create mode 100644 schemas/v2/examples/stage-preview.json create mode 100644 schemas/v2/examples/stage-receipt.json create mode 100644 schemas/v2/examples/stage-request.json create mode 100644 schemas/v2/examples/stage-revise-request.json create mode 100644 schemas/v2/examples/stage.json create mode 100644 schemas/v2/examples/stages.json create mode 100644 schemas/v2/stage-cancel-request.schema.json create mode 100644 schemas/v2/stage-preview.schema.json create mode 100644 schemas/v2/stage-receipt.schema.json create mode 100644 schemas/v2/stage-request.schema.json create mode 100644 schemas/v2/stage-revise-request.schema.json create mode 100644 schemas/v2/stage.schema.json create mode 100644 schemas/v2/stages.schema.json diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index e020655..d27d994 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -57,7 +57,7 @@ func TestSchemaList(t *testing.T) { if code != 0 { t.Fatalf("exit code = %d, want 0; stderr = %q", code, stderr.String()) } - if got, want := stdout.String(), "mm/v2/channel\nmm/v2/channels\nmm/v2/config\nmm/v2/dms\nmm/v2/doctor\nmm/v2/error\nmm/v2/group-dms\nmm/v2/mentions\nmm/v2/search\nmm/v2/store-doctor\nmm/v2/store-migrations\nmm/v2/teams\nmm/v2/thread\nmm/v2/unread\nmm/v2/users\nmm/v2/watch-diagnostic\nmm/v2/watch-event\nmm/v2/whoami\n"; got != want { + if got, want := stdout.String(), "mm/v2/channel\nmm/v2/channels\nmm/v2/config\nmm/v2/dms\nmm/v2/doctor\nmm/v2/error\nmm/v2/group-dms\nmm/v2/mentions\nmm/v2/search\nmm/v2/stage\nmm/v2/stage-cancel-request\nmm/v2/stage-preview\nmm/v2/stage-receipt\nmm/v2/stage-request\nmm/v2/stage-revise-request\nmm/v2/stages\nmm/v2/store-doctor\nmm/v2/store-migrations\nmm/v2/teams\nmm/v2/thread\nmm/v2/unread\nmm/v2/users\nmm/v2/watch-diagnostic\nmm/v2/watch-event\nmm/v2/whoami\n"; got != want { t.Fatalf("stdout = %q, want %q", got, want) } } diff --git a/internal/schema/stage_test.go b/internal/schema/stage_test.go new file mode 100644 index 0000000..a62e172 --- /dev/null +++ b/internal/schema/stage_test.go @@ -0,0 +1,281 @@ +package schema + +import ( + "bytes" + "encoding/json" + "io/fs" + "reflect" + "strconv" + "strings" + "testing" + + publicschemas "github.com/ardasevinc/mattermost-cli/schemas" +) + +// These tests deliberately stop at document-local invariants. The runtime must +// enforce stored-operation applicability for revise, contiguous plan ordinals, +// stageRef/revision agreement, timestamp ordering, and identity-only duplicate +// detection across stage-list rows because JSON Schema cannot observe that state. + +func TestStageMachineSchemasAreRegistered(t *testing.T) { + r, err := Load() + if err != nil { + t.Fatal(err) + } + for _, id := range []string{ + "mm/v2/stage-request", "mm/v2/stage-revise-request", "mm/v2/stage-cancel-request", + "mm/v2/stages", "mm/v2/stage", "mm/v2/stage-receipt", "mm/v2/stage-preview", + } { + if _, err := r.Show(id); err != nil { + t.Fatalf("Show(%q): %v", id, err) + } + } +} + +func TestStageMachineSchemasRejectContradictionsAndLeaks(t *testing.T) { + r, err := Load() + if err != nil { + t.Fatal(err) + } + digest := strings.Repeat("a", 64) + stageID := "stg_0123456789abcdefghijklmnopqrstuv" + stage := `{"stageId":"` + stageID + `","stageRef":"` + stageID + `@1","revision":1,"operation":"create_post","semanticDigest":"` + digest + `","lifecycle":"open","recovery":"none","createdAt":"2026-07-17T10:00:00.000Z","updatedAt":"2026-07-17T10:00:00.000Z","binding":{"serverUrl":"https://mattermost.example/api/v4","serverId":null,"userId":"user-1"},"destination":{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null}}` + cases := map[string][]string{ + "mm/v2/stage-request": { + stageRequest(false, "r1", "create_post", conversationTarget("dm", "username", "arda", "null"), "null", "null", `[]`), + stageRequest(false, "null", "create_post", conversationTarget("dm", "username", "arda", "null"), `"secret"`, "null", `[]`), + stageRequest(false, "null", "create_post", conversationTarget("dm", "username", "arda", "null"), "null", "null", `[{"path":"/tmp/a","remoteFilename":null,"mediaType":null}]`), + stageRequest(true, "r1", "create_post", conversationTarget("dm", "id", "user-id", "null"), `"hello"`, "null", `[]`), + stageRequest(true, "r1", "create_post", conversationTarget("group", "name", "group", "null"), `"hello"`, "null", `[]`), + stageRequest(true, "r1", "create_post", conversationTarget("channel", "name", "town-square", "null"), `"hello"`, "null", `[]`), + stageRequest(true, "r1", "reply", `{"kind":"user","username":"arda"}`, `"hello"`, "null", `[]`), + stageRequest(true, "r1", "delete_post", `{"kind":"post","postId":"p"}`, `"body"`, "null", `[]`), + stageRequest(true, "r1", "react", `{"kind":"post","postId":"p"}`, "null", "null", `[]`), + stageRequest(true, "r1", "resolve_dm", `{"kind":"user","username":"arda"}`, "null", `"wave"`, `[]`), + stageRequest(true, "r1", "resolve_group_dm", `{"kind":"users","usernames":["a","a"]}`, "null", "null", `[]`), + stageRequest(true, "r1", "resolve_group_dm", `{"kind":"users","usernames":["a"]}`, "null", "null", `[]`), + stageRequest(true, "r1", "create_post", conversationTarget("dm", "username", "arda", "null"), `"hello"`, "null", `[{"path":"/tmp/a","remoteFilename":null,"mediaType":null,"contentDigest":"`+digest+`"}]`), + }, + "mm/v2/stages": { + `{"schema":"mm/v2/stages","stages":[` + strings.Replace(stage, `"recovery":"none"`, `"recovery":"forbidden"`, 1) + `],"nextCursor":null}`, + `{"schema":"mm/v2/stages","stages":[` + strings.Replace(stage, `"destination":`, `"body":"secret","destination":`, 1) + `],"nextCursor":null}`, + }, + "mm/v2/stage-receipt": { + `{"schema":"mm/v2/stage-receipt","action":"canceled","revived":false,"replayed":false,"recordedAt":"2026-07-17T10:00:00.000Z","stage":` + stage + `}`, + `{"schema":"mm/v2/stage-receipt","action":"created","revived":true,"replayed":false,"recordedAt":"2026-07-17T10:00:00.000Z","stage":` + stage + `}`, + `{"schema":"mm/v2/stage-receipt","action":"created","revived":false,"replayed":false,"recordedAt":"2026-07-17T10:00:00Z","stage":` + stage + `}`, + `{"schema":"mm/v2/stage-receipt","action":"created","revived":false,"replayed":false,"recordedAt":"2026-07-17T10:00:00.000Z","stage":` + strings.Replace(stage, `"destination":`, `"plan":[],"destination":`, 1) + `}`, + }, + "mm/v2/stage-preview": { + preview("create_post", `{"kind":"reaction","channelId":"c","channelType":"public","teamId":"t","postId":"p","rootPostId":null,"participantIds":[],"emoji":"wave"}`, "create_post", true), + preview("create_post", `{"kind":"conversation","channelId":"c","channelType":"public","teamId":"t","postId":null,"rootPostId":null,"participantIds":[],"emoji":null}`, "delete_post", false), + strings.TrimSuffix(preview("create_post", `{"kind":"conversation","channelId":"c","channelType":"public","teamId":"t","postId":null,"rootPostId":null,"participantIds":[],"emoji":null}`, "create_post", false), "}") + `,"body":"secret"}`, + }, + } + for id, documents := range cases { + for _, document := range documents { + if err := r.Validate(id, strings.NewReader(document)); err == nil { + t.Fatalf("%s accepted contradiction: %s", id, document) + } + } + } +} + +func preview(operation, destination, stepType string, contentValidated bool) string { + return `{"schema":"mm/v2/stage-preview","persist":false,"operation":"` + operation + `","binding":{"serverUrl":"https://mattermost.example/api/v4","serverId":null,"userId":"u"},"destination":` + destination + `,"plan":{"steps":[{"ordinal":1,"type":"` + stepType + `","condition":"always"}]},"contentValidated":` + boolJSON(contentValidated) + `}` +} + +func TestStageRequestAcceptsEveryTargetBranch(t *testing.T) { + r, err := Load() + if err != nil { + t.Fatal(err) + } + valid := []string{ + stageRequest(true, "r1", "create_post", conversationTarget("dm", "username", "arda", "null"), `"hello"`, "null", `[]`), + stageRequest(true, "r2", "create_post", conversationTarget("group", "id", "channel-id", "null"), `"hello"`, "null", `[]`), + stageRequest(true, "r3", "create_post", conversationTarget("channel", "id", "channel-id", "null"), `"hello"`, "null", `[]`), + stageRequest(true, "r4", "create_post", conversationTarget("channel", "name", "town-square", `{"by":"name","value":"vyvo"}`), `"hello"`, "null", `[{"path":"/tmp/a","remoteFilename":null,"mediaType":null}]`), + stageRequest(true, "r5", "reply", `{"kind":"post","postId":"p"}`, `"hello"`, "null", `[]`), + stageRequest(true, "r6", "edit_post", `{"kind":"post","postId":"p"}`, `"hello"`, "null", `[]`), + stageRequest(true, "r7", "delete_post", `{"kind":"post","postId":"p"}`, "null", "null", `[]`), + stageRequest(true, "r8", "react", `{"kind":"post","postId":"p"}`, "null", `"wave"`, `[]`), + stageRequest(true, "r9", "unreact", `{"kind":"post","postId":"p"}`, "null", `"wave"`, `[]`), + stageRequest(true, "r10", "resolve_dm", `{"kind":"user","username":"arda"}`, "null", "null", `[]`), + stageRequest(true, "r11", "resolve_group_dm", `{"kind":"users","usernames":["arda","hakan"]}`, "null", "null", `[]`), + stageRequest(false, "null", "create_post", conversationTarget("channel", "id", "channel-id", "null"), "null", "null", `[]`), + } + for _, document := range valid { + if err := r.Validate("mm/v2/stage-request", strings.NewReader(document)); err != nil { + t.Fatalf("valid stage request rejected: %v\n%s", err, document) + } + } +} + +func TestStageSchemasRejectResidualReviewContradictions(t *testing.T) { + r, err := Load() + if err != nil { + t.Fatal(err) + } + request := example(t, "stage-request.json") + preview := example(t, "stage-preview.json") + show := example(t, "stage.json") + receipt := example(t, "stage-receipt.json") + + for _, whitespace := range []string{"\u00a0", "\u1680", "\u2000", "\u2028", "\u2029", "\u202f", "\u205f", "\u3000", "\ufeff"} { + document := strings.Replace(request, `"body":"hello"`, `"body":`+strconv.Quote(whitespace), 1) + assertInvalid(t, r, "mm/v2/stage-request", document) + } + + validDestination := `{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null}` + destinationContradictions := []string{ + `{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":null,"postId":null,"rootPostId":null,"participantIds":[],"emoji":null}`, + `{"kind":"conversation","channelId":"channel-1","channelType":"private","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":["peer"],"emoji":null}`, + `{"kind":"conversation","channelId":"channel-1","channelType":"dm","teamId":null,"postId":null,"rootPostId":null,"participantIds":[],"emoji":null}`, + `{"kind":"conversation","channelId":"channel-1","channelType":"dm","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":["peer"],"emoji":null}`, + `{"kind":"conversation","channelId":"channel-1","channelType":"group","teamId":null,"postId":null,"rootPostId":null,"participantIds":["claimed-complete"],"emoji":null}`, + `{"kind":"conversation","channelId":null,"channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null}`, + } + for _, destination := range destinationContradictions { + assertInvalid(t, r, "mm/v2/stage-preview", strings.Replace(preview, validDestination, destination, 1)) + } + legacyDirect := `{"kind":"conversation","channelId":"channel-1","channelType":"direct","teamId":null,"postId":null,"rootPostId":null,"participantIds":["peer"],"emoji":null}` + assertInvalid(t, r, "mm/v2/stage-preview", strings.Replace(preview, validDestination, legacyDirect, 1)) + + unresolved := strings.Replace(preview, validDestination, `{"kind":"conversation","channelId":null,"channelType":"dm","teamId":null,"postId":null,"rootPostId":null,"participantIds":["peer"],"emoji":null}`, 1) + assertInvalid(t, r, "mm/v2/stage-preview", unresolved) + compound := strings.Replace(unresolved, `{"ordinal":1,"type":"create_post","condition":"always"}`, `{"ordinal":1,"type":"resolve_conversation","condition":"if_missing"},{"ordinal":2,"type":"upload_attachment","condition":"always"},{"ordinal":3,"type":"create_post","condition":"always"}`, 1) + if err := r.Validate("mm/v2/stage-preview", strings.NewReader(compound)); err != nil { + t.Fatalf("valid unresolved compound preview rejected: %v", err) + } + resolvedWithResolve := strings.Replace(preview, `{"ordinal":1,"type":"create_post","condition":"always"}`, `{"ordinal":1,"type":"resolve_conversation","condition":"if_missing"},{"ordinal":2,"type":"create_post","condition":"always"}`, 1) + assertInvalid(t, r, "mm/v2/stage-preview", resolvedWithResolve) + resolvedWrongCondition := strings.Replace(resolvedWithResolve, `"condition":"if_missing"`, `"condition":"always"`, 1) + assertInvalid(t, r, "mm/v2/stage-preview", resolvedWrongCondition) + + assertInvalid(t, r, "mm/v2/stage", strings.Replace(show, `"state":"present","body":"hello"`, `"state":"pruned","body":null`, 1)) + applying := strings.Replace(show, `"lifecycle":"open"`, `"lifecycle":"applying"`, 1) + if err := r.Validate("mm/v2/stage", strings.NewReader(applying)); err != nil { + t.Fatalf("valid applying stage with present content rejected: %v", err) + } + assertInvalid(t, r, "mm/v2/stage", strings.Replace(applying, `"state":"present","body":"hello"`, `"state":"pruned","body":null`, 1)) + nonContent := strings.NewReplacer( + `"operation":"create_post"`, `"operation":"delete_post"`, + validDestination, `{"kind":"post","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":"post-1","rootPostId":null,"participantIds":[],"emoji":null}`, + `"state":"present","body":"hello"`, `"state":"none","body":null`, + `"type":"create_post"`, `"type":"delete_post"`, + ).Replace(show) + if err := r.Validate("mm/v2/stage", strings.NewReader(nonContent)); err != nil { + t.Fatalf("valid current non-content stage rejected: %v", err) + } + for _, unsafe := range []string{"\x00", "\u061c", "\u200e", "\u200f", "\u202e", "\u2066"} { + attachment := `{"path":` + strconv.Quote("/tmp/a"+unsafe) + `,"canonicalPath":"/tmp/a","remoteFilename":"a.txt","byteLength":1,"mediaType":"text/plain","contentDigest":"` + strings.Repeat("a", 64) + `"}` + document := strings.Replace(show, `"attachmentState":"none","attachments":[]`, `"attachmentState":"retained","attachments":[`+attachment+`]`, 1) + assertInvalid(t, r, "mm/v2/stage", document) + } + for _, padded := range []string{" user-1", "user-1 ", "user id", "user\u00a0id"} { + assertInvalid(t, r, "mm/v2/stage", strings.Replace(show, `"userId":"user-1"`, `"userId":`+strconv.Quote(padded), 1)) + } + + revived := strings.NewReplacer(`"action":"created"`, `"action":"revised"`, `"revived":false`, `"revived":true`, `"stageRef":"stg_0123456789abcdefghijklmnopqrstuv@1"`, `"stageRef":"stg_0123456789abcdefghijklmnopqrstuv@2"`, `"revision":1`, `"revision":2`, `"recovery":"none"`, `"recovery":"resume_partial"`).Replace(receipt) + assertInvalid(t, r, "mm/v2/stage-receipt", revived) + if err := r.Validate("mm/v2/stage-receipt", strings.NewReader(strings.Replace(revived, `"recovery":"resume_partial"`, `"recovery":"none"`, 1))); err != nil { + t.Fatalf("valid revived receipt rejected: %v", err) + } + + list := example(t, "stages.json") + start := strings.Index(list, `[{`) + end := strings.LastIndex(list, `],"nextCursor"`) + row := list[start+1 : end] + oversized := list[:start+1] + strings.Repeat(row+",", 100) + row + list[end:] + assertInvalid(t, r, "mm/v2/stages", oversized) +} + +func TestStageOutputSchemasShareExactDefinitions(t *testing.T) { + var canonical map[string]any + for _, name := range []string{"stage.json", "stage-preview.json", "stage-receipt.json", "stages.json"} { + schemaName := strings.TrimSuffix(name, ".json") + ".schema.json" + data, err := fs.ReadFile(publicschemas.FS, "v2/"+schemaName) + if err != nil { + t.Fatal(err) + } + var document struct { + Definitions map[string]any `json:"$defs"` + } + if err := json.Unmarshal(data, &document); err != nil { + t.Fatal(err) + } + if canonical == nil { + canonical = document.Definitions + continue + } + if !reflect.DeepEqual(canonical, document.Definitions) { + t.Fatalf("%s shared output definitions drifted", schemaName) + } + } +} + +func example(t *testing.T, name string) string { + t.Helper() + data, err := fs.ReadFile(publicschemas.FS, "v2/examples/"+name) + if err != nil { + t.Fatal(err) + } + return string(bytes.TrimSpace(data)) +} + +func assertInvalid(t *testing.T, r *Registry, id, document string) { + t.Helper() + if err := r.Validate(id, strings.NewReader(document)); err == nil { + t.Fatalf("%s accepted invalid document: %s", id, document) + } +} + +func FuzzStageRequestRejectsUnknownTopLevelFields(f *testing.F) { + seeds := []string{ + stageRequest(true, "r1", "create_post", conversationTarget("dm", "username", "arda", "null"), `"hello"`, "null", `[]`), + stageRequest(true, "r2", "create_post", conversationTarget("group", "id", "group-id", "null"), `"hello"`, "null", `[]`), + stageRequest(true, "r3", "create_post", conversationTarget("channel", "id", "channel-id", "null"), `"hello"`, "null", `[]`), + stageRequest(true, "r4", "create_post", conversationTarget("channel", "name", "town-square", `{"by":"id","value":"team-id"}`), `"hello"`, "null", `[]`), + stageRequest(true, "r5", "reply", `{"kind":"post","postId":"p"}`, `"hello"`, "null", `[]`), + stageRequest(true, "r6", "resolve_dm", `{"kind":"user","username":"arda"}`, "null", "null", `[]`), + stageRequest(true, "r7", "resolve_group_dm", `{"kind":"users","usernames":["arda","hakan"]}`, "null", "null", `[]`), + } + for _, seed := range seeds { + f.Add(seed) + } + r, err := Load() + if err != nil { + f.Fatal(err) + } + f.Fuzz(func(t *testing.T, document string) { + if len(document) < 2 || len(document) > 1<<20 || document[len(document)-1] != '}' { + return + } + mutated := document[:len(document)-1] + `,"unknown":true}` + if err := r.Validate("mm/v2/stage-request", strings.NewReader(mutated)); err == nil { + t.Fatalf("stage request accepted unknown field: %s", mutated) + } + }) +} + +func stageRequest(persist bool, requestID, operation, target, body, emoji, attachments string) string { + return `{"schema":"mm/v2/stage-request","persist":` + boolJSON(persist) + `,"requestId":` + quoteUnlessNull(requestID) + `,"operation":"` + operation + `","target":` + target + `,"body":` + body + `,"emoji":` + emoji + `,"attachments":` + attachments + `}` +} + +func conversationTarget(kind, by, value, team string) string { + return `{"kind":"conversation","conversationType":"` + kind + `","selector":{"by":"` + by + `","value":"` + value + `"},"team":` + team + `}` +} + +func quoteUnlessNull(v string) string { + if v == "null" { + return v + } + return `"` + v + `"` +} + +func boolJSON(v bool) string { + if v { + return "true" + } + return "false" +} diff --git a/schemas/v2/examples/stage-cancel-request.json b/schemas/v2/examples/stage-cancel-request.json new file mode 100644 index 0000000..31172a9 --- /dev/null +++ b/schemas/v2/examples/stage-cancel-request.json @@ -0,0 +1 @@ +{"schema":"mm/v2/stage-cancel-request","requestId":"agent-20260717-003","stageId":"stg_0123456789abcdefghijklmnopqrstuv","expectedRevision":2,"expectedDigest":"123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0"} diff --git a/schemas/v2/examples/stage-preview.json b/schemas/v2/examples/stage-preview.json new file mode 100644 index 0000000..2cc4d27 --- /dev/null +++ b/schemas/v2/examples/stage-preview.json @@ -0,0 +1 @@ +{"schema":"mm/v2/stage-preview","persist":false,"operation":"create_post","binding":{"serverUrl":"https://mattermost.example/api/v4","serverId":null,"userId":"user-1"},"destination":{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null},"plan":{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]},"contentValidated":false} diff --git a/schemas/v2/examples/stage-receipt.json b/schemas/v2/examples/stage-receipt.json new file mode 100644 index 0000000..c207db9 --- /dev/null +++ b/schemas/v2/examples/stage-receipt.json @@ -0,0 +1 @@ +{"schema":"mm/v2/stage-receipt","action":"created","revived":false,"replayed":false,"recordedAt":"2026-07-17T10:00:00.000Z","stage":{"stageId":"stg_0123456789abcdefghijklmnopqrstuv","stageRef":"stg_0123456789abcdefghijklmnopqrstuv@1","revision":1,"operation":"create_post","semanticDigest":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","lifecycle":"open","recovery":"none","createdAt":"2026-07-17T10:00:00.000Z","updatedAt":"2026-07-17T10:00:00.000Z","binding":{"serverUrl":"https://mattermost.example/api/v4","serverId":null,"userId":"user-1"},"destination":{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null}}} diff --git a/schemas/v2/examples/stage-request.json b/schemas/v2/examples/stage-request.json new file mode 100644 index 0000000..8460d7e --- /dev/null +++ b/schemas/v2/examples/stage-request.json @@ -0,0 +1 @@ +{"schema":"mm/v2/stage-request","persist":true,"requestId":"agent-20260717-001","operation":"create_post","target":{"kind":"conversation","conversationType":"channel","selector":{"by":"name","value":"town-square"},"team":{"by":"name","value":"vyvo"}},"body":"hello","emoji":null,"attachments":[{"path":"/home/arda/report.pdf","remoteFilename":"report.pdf","mediaType":"application/pdf"}]} diff --git a/schemas/v2/examples/stage-revise-request.json b/schemas/v2/examples/stage-revise-request.json new file mode 100644 index 0000000..790a318 --- /dev/null +++ b/schemas/v2/examples/stage-revise-request.json @@ -0,0 +1 @@ +{"schema":"mm/v2/stage-revise-request","requestId":"agent-20260717-002","stageId":"stg_0123456789abcdefghijklmnopqrstuv","expectedRevision":1,"expectedDigest":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","revive":false,"body":"revised hello","attachments":[]} diff --git a/schemas/v2/examples/stage.json b/schemas/v2/examples/stage.json new file mode 100644 index 0000000..4ee5317 --- /dev/null +++ b/schemas/v2/examples/stage.json @@ -0,0 +1 @@ +{"schema":"mm/v2/stage","stage":{"stageId":"stg_0123456789abcdefghijklmnopqrstuv","stageRef":"stg_0123456789abcdefghijklmnopqrstuv@1","revision":1,"operation":"create_post","semanticDigest":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","lifecycle":"open","recovery":"none","createdAt":"2026-07-17T10:00:00.000Z","updatedAt":"2026-07-17T10:00:00.000Z","binding":{"serverUrl":"https://mattermost.example/api/v4","serverId":null,"userId":"user-1"},"destination":{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null}},"revisionState":"current","content":{"state":"present","body":"hello"},"attachmentState":"none","attachments":[],"plan":{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}} diff --git a/schemas/v2/examples/stages.json b/schemas/v2/examples/stages.json new file mode 100644 index 0000000..7c06f9c --- /dev/null +++ b/schemas/v2/examples/stages.json @@ -0,0 +1 @@ +{"schema":"mm/v2/stages","stages":[{"stageId":"stg_0123456789abcdefghijklmnopqrstuv","stageRef":"stg_0123456789abcdefghijklmnopqrstuv@1","revision":1,"operation":"create_post","semanticDigest":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","lifecycle":"open","recovery":"none","createdAt":"2026-07-17T10:00:00.000Z","updatedAt":"2026-07-17T10:00:00.000Z","binding":{"serverUrl":"https://mattermost.example/api/v4","serverId":null,"userId":"user-1"},"destination":{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null}}],"nextCursor":null} diff --git a/schemas/v2/stage-cancel-request.schema.json b/schemas/v2/stage-cancel-request.schema.json new file mode 100644 index 0000000..ae88ca1 --- /dev/null +++ b/schemas/v2/stage-cancel-request.schema.json @@ -0,0 +1,100 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:stage-cancel-request", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "requestId", + "stageId", + "expectedRevision", + "expectedDigest" + ], + "properties": { + "schema": { + "const": "mm/v2/stage-cancel-request" + }, + "requestId": { + "$ref": "#/$defs/requestId" + }, + "stageId": { + "$ref": "#/$defs/stageId" + }, + "expectedRevision": { + "$ref": "#/$defs/revision" + }, + "expectedDigest": { + "$ref": "#/$defs/digest" + } + }, + "$defs": { + "requestId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._~:-]*$" + }, + "attachment": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "remoteFilename", + "mediaType" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1, + "maxLength": 4096, + "pattern": "^[^\\x{0000}-\\x{001f}\\x{007f}-\\x{009f}\\x{202a}-\\x{202e}\\x{2066}-\\x{2069}]+$" + }, + "remoteFilename": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "maxLength": 255, + "pattern": "^[^/\\\\\\x{0000}-\\x{001f}\\x{007f}-\\x{009f}\\x{202a}-\\x{202e}\\x{2066}-\\x{2069}]+$" + }, + { + "type": "null" + } + ] + }, + "mediaType": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "maxLength": 255, + "pattern": "^[\\x{0021}-\\x{007e}]+$" + }, + { + "type": "null" + } + ] + } + } + }, + "body": { + "type": "string", + "minLength": 1, + "maxLength": 16383, + "pattern": "[^\\s]" + }, + "stageId": { + "type": "string", + "pattern": "^stg_[A-Za-z0-9_-]{32}$" + }, + "revision": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } +} diff --git a/schemas/v2/stage-preview.schema.json b/schemas/v2/stage-preview.schema.json new file mode 100644 index 0000000..c21748a --- /dev/null +++ b/schemas/v2/stage-preview.schema.json @@ -0,0 +1,1311 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:stage-preview", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "persist", + "operation", + "binding", + "destination", + "plan", + "contentValidated" + ], + "properties": { + "schema": { + "const": "mm/v2/stage-preview" + }, + "persist": { + "const": false + }, + "operation": { + "$ref": "#/$defs/operation" + }, + "binding": { + "$ref": "#/$defs/binding" + }, + "destination": { + "$ref": "#/$defs/destination" + }, + "plan": { + "$ref": "#/$defs/plan" + }, + "contentValidated": { + "const": false + } + }, + "allOf": [ + { + "if": { + "properties": { + "operation": { + "const": "create_post" + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/conversationDestination" + }, + "plan": { + "$ref": "#/$defs/createPlan" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "reply" + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/replyDestination" + }, + "plan": { + "$ref": "#/$defs/createPlan" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "enum": [ + "edit_post", + "delete_post" + ] + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/postDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "enum": [ + "react", + "unreact" + ] + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/reactionDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "resolve_dm" + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/dmDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "resolve_group_dm" + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/groupDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "edit_post" + } + } + }, + "then": { + "properties": { + "plan": { + "$ref": "#/$defs/editPlan" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "delete_post" + } + } + }, + "then": { + "properties": { + "plan": { + "$ref": "#/$defs/deletePlan" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "react" + } + } + }, + "then": { + "properties": { + "plan": { + "$ref": "#/$defs/reactPlan" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "unreact" + } + } + }, + "then": { + "properties": { + "plan": { + "$ref": "#/$defs/unreactPlan" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "enum": [ + "resolve_dm", + "resolve_group_dm" + ] + } + } + }, + "then": { + "properties": { + "plan": { + "$ref": "#/$defs/resolvePlan" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "reply" + } + } + }, + "then": { + "properties": { + "plan": { + "$ref": "#/$defs/resolvedCreatePlan" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "create_post" + }, + "destination": { + "properties": { + "channelId": { + "type": "null" + } + } + } + } + }, + "then": { + "properties": { + "plan": { + "$ref": "#/$defs/unresolvedCreatePlan" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "create_post" + }, + "destination": { + "properties": { + "channelId": { + "type": "string" + } + } + } + } + }, + "then": { + "properties": { + "plan": { + "$ref": "#/$defs/resolvedCreatePlan" + } + } + } + } + ], + "$defs": { + "safe": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^\\x{0000}-\\x{001f}\\x{007f}-\\x{009f}\\x{202a}-\\x{202e}\\x{2066}-\\x{2069}]+$" + }, + "nullableSafe": { + "anyOf": [ + { + "$ref": "#/$defs/id" + }, + { + "type": "null" + } + ] + }, + "stageId": { + "type": "string", + "pattern": "^stg_[A-Za-z0-9_-]{32}$" + }, + "revision": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "timestamp": { + "type": "string", + "format": "date-time", + "pattern": "^([0-9]{3}[1-9]|[0-9]{2}[1-9][0-9]|[0-9][1-9][0-9]{2}|[1-9][0-9]{3})-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]{3}Z$" + }, + "serverUrl": { + "type": "string", + "maxLength": 4096, + "format": "uri", + "pattern": "^https?://[^/?#@]+(?::[0-9]+)?(?:/[^?#]*)?$" + }, + "operation": { + "enum": [ + "create_post", + "reply", + "edit_post", + "delete_post", + "react", + "unreact", + "resolve_dm", + "resolve_group_dm" + ] + }, + "binding": { + "type": "object", + "additionalProperties": false, + "required": [ + "serverUrl", + "serverId", + "userId" + ], + "properties": { + "serverUrl": { + "$ref": "#/$defs/serverUrl" + }, + "serverId": { + "$ref": "#/$defs/nullableId" + }, + "userId": { + "$ref": "#/$defs/id" + } + } + }, + "destination": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "channelId", + "channelType", + "teamId", + "postId", + "rootPostId", + "participantIds", + "emoji" + ], + "properties": { + "kind": { + "enum": [ + "conversation", + "post", + "reaction" + ] + }, + "channelId": { + "$ref": "#/$defs/nullableId" + }, + "channelType": { + "enum": [ + "dm", + "group", + "public", + "private", + null + ] + }, + "teamId": { + "$ref": "#/$defs/nullableId" + }, + "postId": { + "$ref": "#/$defs/nullableId" + }, + "rootPostId": { + "$ref": "#/$defs/nullableId" + }, + "participantIds": { + "type": "array", + "maxItems": 100, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/id" + } + }, + "emoji": { + "anyOf": [ + { + "$ref": "#/$defs/emoji" + }, + { + "type": "null" + } + ] + } + } + }, + "conversationDestination": { + "allOf": [ + { + "properties": { + "kind": { + "const": "conversation" + }, + "postId": { + "type": "null" + }, + "rootPostId": { + "type": "null" + }, + "emoji": { + "type": "null" + } + } + }, + { + "oneOf": [ + { + "$ref": "#/$defs/channelDestination" + }, + { + "$ref": "#/$defs/unresolvedDM" + }, + { + "$ref": "#/$defs/unresolvedGroup" + } + ] + } + ] + }, + "replyDestination": { + "allOf": [ + { + "$ref": "#/$defs/channelDestination" + }, + { + "properties": { + "kind": { + "const": "post" + }, + "postId": { + "$ref": "#/$defs/id" + }, + "rootPostId": { + "$ref": "#/$defs/id" + }, + "emoji": { + "type": "null" + } + } + } + ] + }, + "postDestination": { + "allOf": [ + { + "$ref": "#/$defs/channelDestination" + }, + { + "properties": { + "kind": { + "const": "post" + }, + "postId": { + "$ref": "#/$defs/id" + }, + "emoji": { + "type": "null" + } + } + } + ] + }, + "reactionDestination": { + "allOf": [ + { + "$ref": "#/$defs/channelDestination" + }, + { + "properties": { + "kind": { + "const": "reaction" + }, + "postId": { + "$ref": "#/$defs/id" + }, + "emoji": { + "$ref": "#/$defs/emoji" + } + } + } + ] + }, + "dmDestination": { + "allOf": [ + { + "$ref": "#/$defs/unresolvedDM" + }, + { + "properties": { + "kind": { + "const": "conversation" + }, + "postId": { + "type": "null" + }, + "rootPostId": { + "type": "null" + }, + "emoji": { + "type": "null" + } + } + } + ] + }, + "groupDestination": { + "allOf": [ + { + "$ref": "#/$defs/unresolvedGroup" + }, + { + "properties": { + "kind": { + "const": "conversation" + }, + "postId": { + "type": "null" + }, + "rootPostId": { + "type": "null" + }, + "emoji": { + "type": "null" + } + } + } + ] + }, + "content": { + "type": "object", + "additionalProperties": false, + "required": [ + "state", + "body" + ], + "properties": { + "state": { + "enum": [ + "present", + "none", + "pruned" + ] + }, + "body": { + "anyOf": [ + { + "$ref": "#/$defs/body" + }, + { + "type": "null" + } + ] + } + } + }, + "storedAttachment": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "canonicalPath", + "remoteFilename", + "byteLength", + "mediaType", + "contentDigest" + ], + "properties": { + "path": { + "$ref": "#/$defs/safeOutput", + "maxLength": 4096 + }, + "canonicalPath": { + "$ref": "#/$defs/safeOutput", + "maxLength": 4096 + }, + "remoteFilename": { + "$ref": "#/$defs/safeOutput", + "maxLength": 255 + }, + "byteLength": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "mediaType": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "pattern": "^[\\x{0021}-\\x{007e}]+$" + }, + "contentDigest": { + "$ref": "#/$defs/digest" + } + } + }, + "step": { + "type": "object", + "additionalProperties": false, + "required": [ + "ordinal", + "type", + "condition" + ], + "properties": { + "ordinal": { + "type": "integer", + "minimum": 1, + "maximum": 102 + }, + "type": { + "enum": [ + "resolve_conversation", + "upload_attachment", + "create_post", + "edit_post", + "delete_post", + "add_reaction", + "remove_reaction" + ] + }, + "condition": { + "enum": [ + "always", + "if_missing" + ] + } + } + }, + "plan": { + "type": "object", + "additionalProperties": false, + "required": [ + "steps" + ], + "properties": { + "steps": { + "type": "array", + "minItems": 1, + "maxItems": 102, + "items": { + "$ref": "#/$defs/step" + } + } + } + }, + "body": { + "type": "string", + "minLength": 1, + "maxLength": 16383, + "pattern": "[^\\x{0009}\\x{000a}\\x{000b}\\x{000c}\\x{000d}\\x{0020}\\x{00a0}\\x{1680}\\x{2000}-\\x{200a}\\x{2028}\\x{2029}\\x{202f}\\x{205f}\\x{3000}\\x{feff}]" + }, + "emoji": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9_+.-]+$" + }, + "oneStep": { + "type": "object", + "additionalProperties": false, + "required": [ + "steps" + ], + "properties": { + "steps": { + "type": "array", + "minItems": 1, + "maxItems": 1 + } + } + }, + "createPlan": { + "type": "object", + "additionalProperties": false, + "required": [ + "steps" + ], + "properties": { + "steps": { + "type": "array", + "minItems": 1, + "maxItems": 102, + "items": { + "allOf": [ + { + "$ref": "#/$defs/step" + }, + { + "properties": { + "type": { + "enum": [ + "resolve_conversation", + "upload_attachment", + "create_post" + ] + }, + "condition": { + "enum": [ + "always", + "if_missing" + ] + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "resolve_conversation" + } + } + }, + "then": { + "properties": { + "condition": { + "const": "if_missing" + } + } + }, + "else": { + "properties": { + "condition": { + "const": "always" + } + } + } + } + ] + }, + "contains": { + "properties": { + "type": { + "const": "create_post" + } + } + }, + "minContains": 1, + "maxContains": 1 + } + } + }, + "editPlan": { + "$ref": "#/$defs/oneStepWithEdit" + }, + "deletePlan": { + "$ref": "#/$defs/oneStepWithDelete" + }, + "reactPlan": { + "$ref": "#/$defs/oneStepWithReact" + }, + "unreactPlan": { + "$ref": "#/$defs/oneStepWithUnreact" + }, + "resolvePlan": { + "$ref": "#/$defs/oneStepWithResolve" + }, + "oneStepWithEdit": { + "allOf": [ + { + "$ref": "#/$defs/oneStep" + }, + { + "properties": { + "steps": { + "items": { + "properties": { + "type": { + "const": "edit_post" + }, + "condition": { + "const": "always" + } + } + } + } + } + } + ] + }, + "oneStepWithDelete": { + "allOf": [ + { + "$ref": "#/$defs/oneStep" + }, + { + "properties": { + "steps": { + "items": { + "properties": { + "type": { + "const": "delete_post" + }, + "condition": { + "const": "always" + } + } + } + } + } + } + ] + }, + "oneStepWithReact": { + "allOf": [ + { + "$ref": "#/$defs/oneStep" + }, + { + "properties": { + "steps": { + "items": { + "properties": { + "type": { + "const": "add_reaction" + }, + "condition": { + "enum": [ + "always", + "if_missing" + ] + } + } + } + } + } + } + ] + }, + "oneStepWithUnreact": { + "allOf": [ + { + "$ref": "#/$defs/oneStep" + }, + { + "properties": { + "steps": { + "items": { + "properties": { + "type": { + "const": "remove_reaction" + }, + "condition": { + "enum": [ + "always", + "if_missing" + ] + } + } + } + } + } + } + ] + }, + "oneStepWithResolve": { + "allOf": [ + { + "$ref": "#/$defs/oneStep" + }, + { + "properties": { + "steps": { + "items": { + "properties": { + "type": { + "const": "resolve_conversation" + }, + "condition": { + "enum": [ + "always", + "if_missing" + ] + } + } + } + } + } + } + ] + }, + "label": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^\\x{0000}-\\x{001f}\\x{007f}-\\x{009f}\\x{202a}-\\x{202e}\\x{2066}-\\x{2069}]+$" + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^\\x{0000}-\\x{0020}\\x{007f}-\\x{009f}\\x{00a0}\\x{061c}\\x{1680}\\x{2000}-\\x{200f}\\x{2028}-\\x{202f}\\x{205f}\\x{2066}-\\x{2069}\\x{3000}\\x{feff}]+$" + }, + "nullableId": { + "anyOf": [ + { + "$ref": "#/$defs/id" + }, + { + "type": "null" + } + ] + }, + "safeOutput": { + "type": "string", + "minLength": 1, + "maxLength": 4096, + "pattern": "^[^\\x{0000}-\\x{001f}\\x{007f}-\\x{009f}\\x{061c}\\x{200e}-\\x{200f}\\x{202a}-\\x{202e}\\x{2066}-\\x{2069}]+$" + }, + "channelDestination": { + "oneOf": [ + { + "$ref": "#/$defs/publicDestination" + }, + { + "$ref": "#/$defs/privateDestination" + }, + { + "$ref": "#/$defs/existingDM" + }, + { + "$ref": "#/$defs/existingGroup" + } + ] + }, + "publicDestination": { + "properties": { + "channelId": { + "$ref": "#/$defs/id" + }, + "channelType": { + "const": "public" + }, + "teamId": { + "$ref": "#/$defs/id" + }, + "participantIds": { + "maxItems": 0 + } + } + }, + "privateDestination": { + "properties": { + "channelId": { + "$ref": "#/$defs/id" + }, + "channelType": { + "const": "private" + }, + "teamId": { + "$ref": "#/$defs/id" + }, + "participantIds": { + "maxItems": 0 + } + } + }, + "existingDM": { + "properties": { + "channelId": { + "$ref": "#/$defs/id" + }, + "channelType": { + "const": "dm" + }, + "teamId": { + "type": "null" + }, + "participantIds": { + "minItems": 1, + "maxItems": 1 + } + } + }, + "existingGroup": { + "properties": { + "channelId": { + "$ref": "#/$defs/id" + }, + "channelType": { + "const": "group" + }, + "teamId": { + "type": "null" + }, + "participantIds": { + "maxItems": 0 + } + } + }, + "unresolvedDM": { + "properties": { + "channelId": { + "type": "null" + }, + "channelType": { + "const": "dm" + }, + "teamId": { + "type": "null" + }, + "participantIds": { + "minItems": 1, + "maxItems": 1 + } + } + }, + "unresolvedGroup": { + "properties": { + "channelId": { + "type": "null" + }, + "channelType": { + "const": "group" + }, + "teamId": { + "type": "null" + }, + "participantIds": { + "minItems": 2 + } + } + }, + "summary": { + "type": "object", + "additionalProperties": false, + "required": [ + "stageId", + "stageRef", + "revision", + "operation", + "semanticDigest", + "lifecycle", + "recovery", + "createdAt", + "updatedAt", + "binding", + "destination" + ], + "properties": { + "stageId": { + "$ref": "#/$defs/stageId" + }, + "stageRef": { + "type": "string", + "maxLength": 64, + "pattern": "^stg_[A-Za-z0-9_-]{32}@[1-9][0-9]{0,15}$" + }, + "revision": { + "$ref": "#/$defs/revision" + }, + "operation": { + "$ref": "#/$defs/operation" + }, + "semanticDigest": { + "$ref": "#/$defs/digest" + }, + "lifecycle": { + "enum": [ + "open", + "applying", + "completed", + "canceled", + "expired", + "pruned" + ] + }, + "recovery": { + "enum": [ + "none", + "resume_partial", + "force_unknown", + "forbidden" + ] + }, + "createdAt": { + "$ref": "#/$defs/timestamp" + }, + "updatedAt": { + "$ref": "#/$defs/timestamp" + }, + "binding": { + "$ref": "#/$defs/binding" + }, + "destination": { + "$ref": "#/$defs/destination" + } + }, + "allOf": [ + { + "if": { + "properties": { + "lifecycle": { + "enum": [ + "completed", + "canceled", + "expired", + "pruned" + ] + } + } + }, + "then": { + "properties": { + "recovery": { + "const": "forbidden" + } + } + }, + "else": { + "properties": { + "recovery": { + "enum": [ + "none", + "resume_partial", + "force_unknown" + ] + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "create_post" + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/conversationDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "reply" + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/replyDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "enum": [ + "edit_post", + "delete_post" + ] + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/postDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "enum": [ + "react", + "unreact" + ] + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/reactionDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "resolve_dm" + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/dmDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "resolve_group_dm" + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/groupDestination" + } + } + } + } + ] + }, + "resolvedCreatePlan": { + "allOf": [ + { + "$ref": "#/$defs/createPlan" + }, + { + "properties": { + "steps": { + "not": { + "contains": { + "properties": { + "type": { + "const": "resolve_conversation" + } + } + } + } + } + } + } + ] + }, + "unresolvedCreatePlan": { + "allOf": [ + { + "$ref": "#/$defs/createPlan" + }, + { + "properties": { + "steps": { + "contains": { + "properties": { + "type": { + "const": "resolve_conversation" + } + } + }, + "minContains": 1, + "maxContains": 1 + } + } + } + ] + } + } +} diff --git a/schemas/v2/stage-receipt.schema.json b/schemas/v2/stage-receipt.schema.json new file mode 100644 index 0000000..5741178 --- /dev/null +++ b/schemas/v2/stage-receipt.schema.json @@ -0,0 +1,1163 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:stage-receipt", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "action", + "revived", + "replayed", + "recordedAt", + "stage" + ], + "properties": { + "schema": { + "const": "mm/v2/stage-receipt" + }, + "action": { + "enum": [ + "created", + "revised", + "canceled" + ] + }, + "revived": { + "type": "boolean" + }, + "replayed": { + "type": "boolean" + }, + "recordedAt": { + "$ref": "#/$defs/timestamp" + }, + "stage": { + "$ref": "#/$defs/summary" + } + }, + "allOf": [ + { + "if": { + "properties": { + "action": { + "const": "created" + } + } + }, + "then": { + "properties": { + "revived": { + "const": false + }, + "stage": { + "properties": { + "revision": { + "const": 1 + }, + "lifecycle": { + "const": "open" + }, + "recovery": { + "const": "none" + } + } + } + } + } + }, + { + "if": { + "properties": { + "action": { + "const": "revised" + } + } + }, + "then": { + "properties": { + "stage": { + "properties": { + "revision": { + "minimum": 2 + }, + "lifecycle": { + "const": "open" + } + } + } + } + }, + "else": { + "properties": { + "revived": { + "const": false + } + } + } + }, + { + "if": { + "properties": { + "action": { + "const": "canceled" + } + } + }, + "then": { + "properties": { + "stage": { + "properties": { + "lifecycle": { + "const": "canceled" + }, + "recovery": { + "const": "forbidden" + } + } + } + } + } + }, + { + "if": { + "properties": { + "action": { + "const": "revised" + }, + "revived": { + "const": true + } + } + }, + "then": { + "properties": { + "stage": { + "properties": { + "recovery": { + "const": "none" + } + } + } + } + } + } + ], + "$defs": { + "safe": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^\\x{0000}-\\x{001f}\\x{007f}-\\x{009f}\\x{202a}-\\x{202e}\\x{2066}-\\x{2069}]+$" + }, + "nullableSafe": { + "anyOf": [ + { + "$ref": "#/$defs/id" + }, + { + "type": "null" + } + ] + }, + "stageId": { + "type": "string", + "pattern": "^stg_[A-Za-z0-9_-]{32}$" + }, + "revision": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "timestamp": { + "type": "string", + "format": "date-time", + "pattern": "^([0-9]{3}[1-9]|[0-9]{2}[1-9][0-9]|[0-9][1-9][0-9]{2}|[1-9][0-9]{3})-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]{3}Z$" + }, + "serverUrl": { + "type": "string", + "maxLength": 4096, + "format": "uri", + "pattern": "^https?://[^/?#@]+(?::[0-9]+)?(?:/[^?#]*)?$" + }, + "operation": { + "enum": [ + "create_post", + "reply", + "edit_post", + "delete_post", + "react", + "unreact", + "resolve_dm", + "resolve_group_dm" + ] + }, + "binding": { + "type": "object", + "additionalProperties": false, + "required": [ + "serverUrl", + "serverId", + "userId" + ], + "properties": { + "serverUrl": { + "$ref": "#/$defs/serverUrl" + }, + "serverId": { + "$ref": "#/$defs/nullableId" + }, + "userId": { + "$ref": "#/$defs/id" + } + } + }, + "destination": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "channelId", + "channelType", + "teamId", + "postId", + "rootPostId", + "participantIds", + "emoji" + ], + "properties": { + "kind": { + "enum": [ + "conversation", + "post", + "reaction" + ] + }, + "channelId": { + "$ref": "#/$defs/nullableId" + }, + "channelType": { + "enum": [ + "dm", + "group", + "public", + "private", + null + ] + }, + "teamId": { + "$ref": "#/$defs/nullableId" + }, + "postId": { + "$ref": "#/$defs/nullableId" + }, + "rootPostId": { + "$ref": "#/$defs/nullableId" + }, + "participantIds": { + "type": "array", + "maxItems": 100, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/id" + } + }, + "emoji": { + "anyOf": [ + { + "$ref": "#/$defs/emoji" + }, + { + "type": "null" + } + ] + } + } + }, + "conversationDestination": { + "allOf": [ + { + "properties": { + "kind": { + "const": "conversation" + }, + "postId": { + "type": "null" + }, + "rootPostId": { + "type": "null" + }, + "emoji": { + "type": "null" + } + } + }, + { + "oneOf": [ + { + "$ref": "#/$defs/channelDestination" + }, + { + "$ref": "#/$defs/unresolvedDM" + }, + { + "$ref": "#/$defs/unresolvedGroup" + } + ] + } + ] + }, + "replyDestination": { + "allOf": [ + { + "$ref": "#/$defs/channelDestination" + }, + { + "properties": { + "kind": { + "const": "post" + }, + "postId": { + "$ref": "#/$defs/id" + }, + "rootPostId": { + "$ref": "#/$defs/id" + }, + "emoji": { + "type": "null" + } + } + } + ] + }, + "postDestination": { + "allOf": [ + { + "$ref": "#/$defs/channelDestination" + }, + { + "properties": { + "kind": { + "const": "post" + }, + "postId": { + "$ref": "#/$defs/id" + }, + "emoji": { + "type": "null" + } + } + } + ] + }, + "reactionDestination": { + "allOf": [ + { + "$ref": "#/$defs/channelDestination" + }, + { + "properties": { + "kind": { + "const": "reaction" + }, + "postId": { + "$ref": "#/$defs/id" + }, + "emoji": { + "$ref": "#/$defs/emoji" + } + } + } + ] + }, + "dmDestination": { + "allOf": [ + { + "$ref": "#/$defs/unresolvedDM" + }, + { + "properties": { + "kind": { + "const": "conversation" + }, + "postId": { + "type": "null" + }, + "rootPostId": { + "type": "null" + }, + "emoji": { + "type": "null" + } + } + } + ] + }, + "groupDestination": { + "allOf": [ + { + "$ref": "#/$defs/unresolvedGroup" + }, + { + "properties": { + "kind": { + "const": "conversation" + }, + "postId": { + "type": "null" + }, + "rootPostId": { + "type": "null" + }, + "emoji": { + "type": "null" + } + } + } + ] + }, + "content": { + "type": "object", + "additionalProperties": false, + "required": [ + "state", + "body" + ], + "properties": { + "state": { + "enum": [ + "present", + "none", + "pruned" + ] + }, + "body": { + "anyOf": [ + { + "$ref": "#/$defs/body" + }, + { + "type": "null" + } + ] + } + } + }, + "storedAttachment": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "canonicalPath", + "remoteFilename", + "byteLength", + "mediaType", + "contentDigest" + ], + "properties": { + "path": { + "$ref": "#/$defs/safeOutput", + "maxLength": 4096 + }, + "canonicalPath": { + "$ref": "#/$defs/safeOutput", + "maxLength": 4096 + }, + "remoteFilename": { + "$ref": "#/$defs/safeOutput", + "maxLength": 255 + }, + "byteLength": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "mediaType": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "pattern": "^[\\x{0021}-\\x{007e}]+$" + }, + "contentDigest": { + "$ref": "#/$defs/digest" + } + } + }, + "step": { + "type": "object", + "additionalProperties": false, + "required": [ + "ordinal", + "type", + "condition" + ], + "properties": { + "ordinal": { + "type": "integer", + "minimum": 1, + "maximum": 102 + }, + "type": { + "enum": [ + "resolve_conversation", + "upload_attachment", + "create_post", + "edit_post", + "delete_post", + "add_reaction", + "remove_reaction" + ] + }, + "condition": { + "enum": [ + "always", + "if_missing" + ] + } + } + }, + "plan": { + "type": "object", + "additionalProperties": false, + "required": [ + "steps" + ], + "properties": { + "steps": { + "type": "array", + "minItems": 1, + "maxItems": 102, + "items": { + "$ref": "#/$defs/step" + } + } + } + }, + "body": { + "type": "string", + "minLength": 1, + "maxLength": 16383, + "pattern": "[^\\x{0009}\\x{000a}\\x{000b}\\x{000c}\\x{000d}\\x{0020}\\x{00a0}\\x{1680}\\x{2000}-\\x{200a}\\x{2028}\\x{2029}\\x{202f}\\x{205f}\\x{3000}\\x{feff}]" + }, + "emoji": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9_+.-]+$" + }, + "oneStep": { + "type": "object", + "additionalProperties": false, + "required": [ + "steps" + ], + "properties": { + "steps": { + "type": "array", + "minItems": 1, + "maxItems": 1 + } + } + }, + "createPlan": { + "type": "object", + "additionalProperties": false, + "required": [ + "steps" + ], + "properties": { + "steps": { + "type": "array", + "minItems": 1, + "maxItems": 102, + "items": { + "allOf": [ + { + "$ref": "#/$defs/step" + }, + { + "properties": { + "type": { + "enum": [ + "resolve_conversation", + "upload_attachment", + "create_post" + ] + }, + "condition": { + "enum": [ + "always", + "if_missing" + ] + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "resolve_conversation" + } + } + }, + "then": { + "properties": { + "condition": { + "const": "if_missing" + } + } + }, + "else": { + "properties": { + "condition": { + "const": "always" + } + } + } + } + ] + }, + "contains": { + "properties": { + "type": { + "const": "create_post" + } + } + }, + "minContains": 1, + "maxContains": 1 + } + } + }, + "editPlan": { + "$ref": "#/$defs/oneStepWithEdit" + }, + "deletePlan": { + "$ref": "#/$defs/oneStepWithDelete" + }, + "reactPlan": { + "$ref": "#/$defs/oneStepWithReact" + }, + "unreactPlan": { + "$ref": "#/$defs/oneStepWithUnreact" + }, + "resolvePlan": { + "$ref": "#/$defs/oneStepWithResolve" + }, + "oneStepWithEdit": { + "allOf": [ + { + "$ref": "#/$defs/oneStep" + }, + { + "properties": { + "steps": { + "items": { + "properties": { + "type": { + "const": "edit_post" + }, + "condition": { + "const": "always" + } + } + } + } + } + } + ] + }, + "oneStepWithDelete": { + "allOf": [ + { + "$ref": "#/$defs/oneStep" + }, + { + "properties": { + "steps": { + "items": { + "properties": { + "type": { + "const": "delete_post" + }, + "condition": { + "const": "always" + } + } + } + } + } + } + ] + }, + "oneStepWithReact": { + "allOf": [ + { + "$ref": "#/$defs/oneStep" + }, + { + "properties": { + "steps": { + "items": { + "properties": { + "type": { + "const": "add_reaction" + }, + "condition": { + "enum": [ + "always", + "if_missing" + ] + } + } + } + } + } + } + ] + }, + "oneStepWithUnreact": { + "allOf": [ + { + "$ref": "#/$defs/oneStep" + }, + { + "properties": { + "steps": { + "items": { + "properties": { + "type": { + "const": "remove_reaction" + }, + "condition": { + "enum": [ + "always", + "if_missing" + ] + } + } + } + } + } + } + ] + }, + "oneStepWithResolve": { + "allOf": [ + { + "$ref": "#/$defs/oneStep" + }, + { + "properties": { + "steps": { + "items": { + "properties": { + "type": { + "const": "resolve_conversation" + }, + "condition": { + "enum": [ + "always", + "if_missing" + ] + } + } + } + } + } + } + ] + }, + "label": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^\\x{0000}-\\x{001f}\\x{007f}-\\x{009f}\\x{202a}-\\x{202e}\\x{2066}-\\x{2069}]+$" + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^\\x{0000}-\\x{0020}\\x{007f}-\\x{009f}\\x{00a0}\\x{061c}\\x{1680}\\x{2000}-\\x{200f}\\x{2028}-\\x{202f}\\x{205f}\\x{2066}-\\x{2069}\\x{3000}\\x{feff}]+$" + }, + "nullableId": { + "anyOf": [ + { + "$ref": "#/$defs/id" + }, + { + "type": "null" + } + ] + }, + "safeOutput": { + "type": "string", + "minLength": 1, + "maxLength": 4096, + "pattern": "^[^\\x{0000}-\\x{001f}\\x{007f}-\\x{009f}\\x{061c}\\x{200e}-\\x{200f}\\x{202a}-\\x{202e}\\x{2066}-\\x{2069}]+$" + }, + "channelDestination": { + "oneOf": [ + { + "$ref": "#/$defs/publicDestination" + }, + { + "$ref": "#/$defs/privateDestination" + }, + { + "$ref": "#/$defs/existingDM" + }, + { + "$ref": "#/$defs/existingGroup" + } + ] + }, + "publicDestination": { + "properties": { + "channelId": { + "$ref": "#/$defs/id" + }, + "channelType": { + "const": "public" + }, + "teamId": { + "$ref": "#/$defs/id" + }, + "participantIds": { + "maxItems": 0 + } + } + }, + "privateDestination": { + "properties": { + "channelId": { + "$ref": "#/$defs/id" + }, + "channelType": { + "const": "private" + }, + "teamId": { + "$ref": "#/$defs/id" + }, + "participantIds": { + "maxItems": 0 + } + } + }, + "existingDM": { + "properties": { + "channelId": { + "$ref": "#/$defs/id" + }, + "channelType": { + "const": "dm" + }, + "teamId": { + "type": "null" + }, + "participantIds": { + "minItems": 1, + "maxItems": 1 + } + } + }, + "existingGroup": { + "properties": { + "channelId": { + "$ref": "#/$defs/id" + }, + "channelType": { + "const": "group" + }, + "teamId": { + "type": "null" + }, + "participantIds": { + "maxItems": 0 + } + } + }, + "unresolvedDM": { + "properties": { + "channelId": { + "type": "null" + }, + "channelType": { + "const": "dm" + }, + "teamId": { + "type": "null" + }, + "participantIds": { + "minItems": 1, + "maxItems": 1 + } + } + }, + "unresolvedGroup": { + "properties": { + "channelId": { + "type": "null" + }, + "channelType": { + "const": "group" + }, + "teamId": { + "type": "null" + }, + "participantIds": { + "minItems": 2 + } + } + }, + "summary": { + "type": "object", + "additionalProperties": false, + "required": [ + "stageId", + "stageRef", + "revision", + "operation", + "semanticDigest", + "lifecycle", + "recovery", + "createdAt", + "updatedAt", + "binding", + "destination" + ], + "properties": { + "stageId": { + "$ref": "#/$defs/stageId" + }, + "stageRef": { + "type": "string", + "maxLength": 64, + "pattern": "^stg_[A-Za-z0-9_-]{32}@[1-9][0-9]{0,15}$" + }, + "revision": { + "$ref": "#/$defs/revision" + }, + "operation": { + "$ref": "#/$defs/operation" + }, + "semanticDigest": { + "$ref": "#/$defs/digest" + }, + "lifecycle": { + "enum": [ + "open", + "applying", + "completed", + "canceled", + "expired", + "pruned" + ] + }, + "recovery": { + "enum": [ + "none", + "resume_partial", + "force_unknown", + "forbidden" + ] + }, + "createdAt": { + "$ref": "#/$defs/timestamp" + }, + "updatedAt": { + "$ref": "#/$defs/timestamp" + }, + "binding": { + "$ref": "#/$defs/binding" + }, + "destination": { + "$ref": "#/$defs/destination" + } + }, + "allOf": [ + { + "if": { + "properties": { + "lifecycle": { + "enum": [ + "completed", + "canceled", + "expired", + "pruned" + ] + } + } + }, + "then": { + "properties": { + "recovery": { + "const": "forbidden" + } + } + }, + "else": { + "properties": { + "recovery": { + "enum": [ + "none", + "resume_partial", + "force_unknown" + ] + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "create_post" + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/conversationDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "reply" + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/replyDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "enum": [ + "edit_post", + "delete_post" + ] + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/postDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "enum": [ + "react", + "unreact" + ] + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/reactionDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "resolve_dm" + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/dmDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "resolve_group_dm" + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/groupDestination" + } + } + } + } + ] + }, + "resolvedCreatePlan": { + "allOf": [ + { + "$ref": "#/$defs/createPlan" + }, + { + "properties": { + "steps": { + "not": { + "contains": { + "properties": { + "type": { + "const": "resolve_conversation" + } + } + } + } + } + } + } + ] + }, + "unresolvedCreatePlan": { + "allOf": [ + { + "$ref": "#/$defs/createPlan" + }, + { + "properties": { + "steps": { + "contains": { + "properties": { + "type": { + "const": "resolve_conversation" + } + } + }, + "minContains": 1, + "maxContains": 1 + } + } + } + ] + } + } +} diff --git a/schemas/v2/stage-request.schema.json b/schemas/v2/stage-request.schema.json new file mode 100644 index 0000000..145c7d7 --- /dev/null +++ b/schemas/v2/stage-request.schema.json @@ -0,0 +1,584 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:stage-request", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "persist", + "requestId", + "operation", + "target", + "body", + "emoji", + "attachments" + ], + "properties": { + "schema": { + "const": "mm/v2/stage-request" + }, + "persist": { + "type": "boolean" + }, + "requestId": { + "anyOf": [ + { + "$ref": "#/$defs/requestId" + }, + { + "type": "null" + } + ] + }, + "operation": { + "$ref": "#/$defs/operation" + }, + "target": { + "oneOf": [ + { + "$ref": "#/$defs/conversation" + }, + { + "$ref": "#/$defs/post" + }, + { + "$ref": "#/$defs/user" + }, + { + "$ref": "#/$defs/users" + } + ] + }, + "body": { + "anyOf": [ + { + "$ref": "#/$defs/body" + }, + { + "type": "null" + } + ] + }, + "emoji": { + "anyOf": [ + { + "$ref": "#/$defs/emoji" + }, + { + "type": "null" + } + ] + }, + "attachments": { + "type": "array", + "maxItems": 100, + "items": { + "$ref": "#/$defs/attachment" + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "persist": { + "const": false + } + } + }, + "then": { + "properties": { + "requestId": { + "type": "null" + }, + "body": { + "type": "null" + }, + "attachments": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "persist": { + "const": true + } + } + }, + "then": { + "properties": { + "requestId": { + "$ref": "#/$defs/requestId" + } + } + } + }, + { + "if": { + "properties": { + "persist": { + "const": true + }, + "operation": { + "enum": [ + "create_post", + "reply", + "edit_post" + ] + } + } + }, + "then": { + "properties": { + "body": { + "$ref": "#/$defs/body" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "enum": [ + "edit_post", + "delete_post", + "react", + "unreact", + "resolve_dm", + "resolve_group_dm" + ] + } + } + }, + "then": { + "properties": { + "attachments": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "enum": [ + "delete_post", + "react", + "unreact", + "resolve_dm", + "resolve_group_dm" + ] + } + } + }, + "then": { + "properties": { + "body": { + "type": "null" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "enum": [ + "react", + "unreact" + ] + } + } + }, + "then": { + "properties": { + "emoji": { + "$ref": "#/$defs/emoji" + } + } + }, + "else": { + "properties": { + "emoji": { + "type": "null" + } + } + } + } + ], + "oneOf": [ + { + "properties": { + "operation": { + "const": "create_post" + }, + "target": { + "$ref": "#/$defs/conversation" + } + } + }, + { + "properties": { + "operation": { + "enum": [ + "reply", + "edit_post", + "delete_post", + "react", + "unreact" + ] + }, + "target": { + "$ref": "#/$defs/post" + } + } + }, + { + "properties": { + "operation": { + "const": "resolve_dm" + }, + "target": { + "$ref": "#/$defs/user" + } + } + }, + { + "properties": { + "operation": { + "const": "resolve_group_dm" + }, + "target": { + "$ref": "#/$defs/users" + } + } + } + ], + "$defs": { + "operation": { + "enum": [ + "create_post", + "reply", + "edit_post", + "delete_post", + "react", + "unreact", + "resolve_dm", + "resolve_group_dm" + ] + }, + "requestId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._~:-]*$" + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^\\x{0000}-\\x{0020}\\x{007f}-\\x{009f}\\x{202a}-\\x{202e}\\x{2066}-\\x{2069}]+$" + }, + "body": { + "type": "string", + "minLength": 1, + "maxLength": 16383, + "pattern": "[^\\x{0009}\\x{000a}\\x{000b}\\x{000c}\\x{000d}\\x{0020}\\x{00a0}\\x{1680}\\x{2000}-\\x{200a}\\x{2028}\\x{2029}\\x{202f}\\x{205f}\\x{3000}\\x{feff}]" + }, + "emoji": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9_+.-]+$" + }, + "selector": { + "type": "object", + "additionalProperties": false, + "required": [ + "by", + "value" + ], + "properties": { + "by": { + "enum": [ + "id", + "name", + "username" + ] + }, + "value": { + "$ref": "#/$defs/id" + } + } + }, + "team": { + "type": "object", + "additionalProperties": false, + "required": [ + "by", + "value" + ], + "properties": { + "by": { + "enum": [ + "id", + "name" + ] + }, + "value": { + "$ref": "#/$defs/id" + } + } + }, + "conversation": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "conversationType", + "selector", + "team" + ], + "properties": { + "kind": { + "const": "conversation" + }, + "conversationType": { + "const": "dm" + }, + "selector": { + "$ref": "#/$defs/usernameSelector" + }, + "team": { + "type": "null" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "conversationType", + "selector", + "team" + ], + "properties": { + "kind": { + "const": "conversation" + }, + "conversationType": { + "const": "group" + }, + "selector": { + "$ref": "#/$defs/idSelector" + }, + "team": { + "type": "null" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "conversationType", + "selector", + "team" + ], + "properties": { + "kind": { + "const": "conversation" + }, + "conversationType": { + "const": "channel" + }, + "selector": { + "$ref": "#/$defs/idSelector" + }, + "team": { + "type": "null" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "conversationType", + "selector", + "team" + ], + "properties": { + "kind": { + "const": "conversation" + }, + "conversationType": { + "const": "channel" + }, + "selector": { + "$ref": "#/$defs/nameSelector" + }, + "team": { + "$ref": "#/$defs/team" + } + } + } + ] + }, + "idSelector": { + "type": "object", + "additionalProperties": false, + "required": [ + "by", + "value" + ], + "properties": { + "by": { + "const": "id" + }, + "value": { + "$ref": "#/$defs/id" + } + } + }, + "nameSelector": { + "type": "object", + "additionalProperties": false, + "required": [ + "by", + "value" + ], + "properties": { + "by": { + "const": "name" + }, + "value": { + "$ref": "#/$defs/id" + } + } + }, + "usernameSelector": { + "type": "object", + "additionalProperties": false, + "required": [ + "by", + "value" + ], + "properties": { + "by": { + "const": "username" + }, + "value": { + "$ref": "#/$defs/id" + } + } + }, + "post": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "postId" + ], + "properties": { + "kind": { + "const": "post" + }, + "postId": { + "$ref": "#/$defs/id" + } + } + }, + "user": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "username" + ], + "properties": { + "kind": { + "const": "user" + }, + "username": { + "$ref": "#/$defs/id" + } + } + }, + "users": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "usernames" + ], + "properties": { + "kind": { + "const": "users" + }, + "usernames": { + "type": "array", + "minItems": 2, + "maxItems": 100, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/id" + } + } + } + }, + "attachment": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "remoteFilename", + "mediaType" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1, + "maxLength": 4096, + "pattern": "^[^\\x{0000}-\\x{001f}\\x{007f}-\\x{009f}\\x{202a}-\\x{202e}\\x{2066}-\\x{2069}]+$" + }, + "remoteFilename": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "maxLength": 255, + "pattern": "^[^/\\\\\\x{0000}-\\x{001f}\\x{007f}-\\x{009f}\\x{202a}-\\x{202e}\\x{2066}-\\x{2069}]+$" + }, + { + "type": "null" + } + ] + }, + "mediaType": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "maxLength": 255, + "pattern": "^[\\x{0021}-\\x{007e}]+$" + }, + { + "type": "null" + } + ] + } + } + } + } +} diff --git a/schemas/v2/stage-revise-request.schema.json b/schemas/v2/stage-revise-request.schema.json new file mode 100644 index 0000000..9d9837c --- /dev/null +++ b/schemas/v2/stage-revise-request.schema.json @@ -0,0 +1,124 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:stage-revise-request", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "requestId", + "stageId", + "expectedRevision", + "expectedDigest", + "revive", + "body", + "attachments" + ], + "properties": { + "schema": { + "const": "mm/v2/stage-revise-request" + }, + "requestId": { + "$ref": "#/$defs/requestId" + }, + "stageId": { + "$ref": "#/$defs/stageId" + }, + "expectedRevision": { + "$ref": "#/$defs/revision" + }, + "expectedDigest": { + "$ref": "#/$defs/digest" + }, + "revive": { + "type": "boolean" + }, + "body": { + "anyOf": [ + { + "$ref": "#/$defs/body" + }, + { + "type": "null" + } + ] + }, + "attachments": { + "type": "array", + "maxItems": 100, + "items": { + "$ref": "#/$defs/attachment" + } + } + }, + "$comment": "Whether body and attachments apply is determined from the stored immutable operation. Runtime rejects changes for non-content operations and attachments for operations other than create_post or reply.", + "$defs": { + "requestId": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._~:-]*$" + }, + "attachment": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "remoteFilename", + "mediaType" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1, + "maxLength": 4096, + "pattern": "^[^\\x{0000}-\\x{001f}\\x{007f}-\\x{009f}\\x{202a}-\\x{202e}\\x{2066}-\\x{2069}]+$" + }, + "remoteFilename": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "maxLength": 255, + "pattern": "^[^/\\\\\\x{0000}-\\x{001f}\\x{007f}-\\x{009f}\\x{202a}-\\x{202e}\\x{2066}-\\x{2069}]+$" + }, + { + "type": "null" + } + ] + }, + "mediaType": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "maxLength": 255, + "pattern": "^[\\x{0021}-\\x{007e}]+$" + }, + { + "type": "null" + } + ] + } + } + }, + "body": { + "type": "string", + "minLength": 1, + "maxLength": 16383, + "pattern": "[^\\x{0009}\\x{000a}\\x{000b}\\x{000c}\\x{000d}\\x{0020}\\x{00a0}\\x{1680}\\x{2000}-\\x{200a}\\x{2028}\\x{2029}\\x{202f}\\x{205f}\\x{3000}\\x{feff}]" + }, + "stageId": { + "type": "string", + "pattern": "^stg_[A-Za-z0-9_-]{32}$" + }, + "revision": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } +} diff --git a/schemas/v2/stage.schema.json b/schemas/v2/stage.schema.json new file mode 100644 index 0000000..b592179 --- /dev/null +++ b/schemas/v2/stage.schema.json @@ -0,0 +1,1484 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:stage", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "stage", + "revisionState", + "content", + "attachmentState", + "attachments", + "plan" + ], + "properties": { + "schema": { + "const": "mm/v2/stage" + }, + "stage": { + "$ref": "#/$defs/summary" + }, + "revisionState": { + "enum": [ + "current", + "superseded" + ] + }, + "content": { + "$ref": "#/$defs/content" + }, + "attachmentState": { + "enum": [ + "retained", + "none", + "pruned" + ] + }, + "attachments": { + "type": "array", + "maxItems": 100, + "items": { + "$ref": "#/$defs/storedAttachment" + } + }, + "plan": { + "$ref": "#/$defs/plan" + } + }, + "allOf": [ + { + "if": { + "properties": { + "stage": { + "properties": { + "operation": { + "enum": [ + "create_post", + "reply", + "edit_post" + ] + } + } + } + } + }, + "then": { + "properties": { + "content": { + "properties": { + "state": { + "enum": [ + "present", + "pruned" + ] + } + } + } + } + }, + "else": { + "properties": { + "content": { + "properties": { + "state": { + "const": "none" + }, + "body": { + "type": "null" + } + }, + "required": [ + "state", + "body" + ] + }, + "attachmentState": { + "const": "none" + }, + "attachments": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "stage": { + "properties": { + "operation": { + "enum": [ + "edit_post" + ] + } + } + } + } + }, + "then": { + "properties": { + "attachmentState": { + "const": "none" + }, + "attachments": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "stage": { + "properties": { + "lifecycle": { + "enum": [ + "completed", + "pruned" + ] + } + } + } + } + }, + "then": { + "properties": { + "content": { + "properties": { + "state": { + "const": "pruned" + }, + "body": { + "type": "null" + } + } + }, + "attachmentState": { + "enum": [ + "none", + "pruned" + ] + }, + "attachments": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "content": { + "properties": { + "state": { + "const": "present" + } + } + } + } + }, + "then": { + "properties": { + "content": { + "properties": { + "body": { + "$ref": "#/$defs/body" + } + } + } + } + }, + "else": { + "properties": { + "content": { + "properties": { + "body": { + "type": "null" + } + } + } + } + } + }, + { + "if": { + "properties": { + "attachmentState": { + "const": "retained" + } + } + }, + "then": { + "properties": { + "attachments": { + "minItems": 1 + } + } + }, + "else": { + "properties": { + "attachments": { + "maxItems": 0 + } + } + } + }, + { + "if": { + "properties": { + "stage": { + "properties": { + "operation": { + "enum": [ + "create_post", + "reply" + ] + } + } + } + } + }, + "then": { + "properties": { + "plan": { + "$ref": "#/$defs/createPlan" + } + } + } + }, + { + "if": { + "properties": { + "stage": { + "properties": { + "operation": { + "const": "edit_post" + } + } + } + } + }, + "then": { + "properties": { + "plan": { + "$ref": "#/$defs/editPlan" + } + } + } + }, + { + "if": { + "properties": { + "stage": { + "properties": { + "operation": { + "const": "delete_post" + } + } + } + } + }, + "then": { + "properties": { + "plan": { + "$ref": "#/$defs/deletePlan" + } + } + } + }, + { + "if": { + "properties": { + "stage": { + "properties": { + "operation": { + "const": "react" + } + } + } + } + }, + "then": { + "properties": { + "plan": { + "$ref": "#/$defs/reactPlan" + } + } + } + }, + { + "if": { + "properties": { + "stage": { + "properties": { + "operation": { + "const": "unreact" + } + } + } + } + }, + "then": { + "properties": { + "plan": { + "$ref": "#/$defs/unreactPlan" + } + } + } + }, + { + "if": { + "properties": { + "stage": { + "properties": { + "operation": { + "enum": [ + "resolve_dm", + "resolve_group_dm" + ] + } + } + } + } + }, + "then": { + "properties": { + "plan": { + "$ref": "#/$defs/resolvePlan" + } + } + } + }, + { + "if": { + "properties": { + "stage": { + "properties": { + "lifecycle": { + "enum": [ + "open", + "applying" + ] + }, + "operation": { + "enum": [ + "create_post", + "reply", + "edit_post" + ] + } + } + }, + "revisionState": { + "const": "current" + } + } + }, + "then": { + "properties": { + "content": { + "properties": { + "state": { + "const": "present" + } + } + } + } + } + }, + { + "if": { + "properties": { + "stage": { + "properties": { + "operation": { + "const": "reply" + } + } + } + } + }, + "then": { + "properties": { + "plan": { + "$ref": "#/$defs/resolvedCreatePlan" + } + } + } + }, + { + "if": { + "properties": { + "stage": { + "properties": { + "operation": { + "const": "create_post" + }, + "destination": { + "properties": { + "channelId": { + "type": "null" + } + } + } + } + } + } + }, + "then": { + "properties": { + "plan": { + "$ref": "#/$defs/unresolvedCreatePlan" + } + } + } + }, + { + "if": { + "properties": { + "stage": { + "properties": { + "operation": { + "const": "create_post" + }, + "destination": { + "properties": { + "channelId": { + "type": "string" + } + } + } + } + } + } + }, + "then": { + "properties": { + "plan": { + "$ref": "#/$defs/resolvedCreatePlan" + } + } + } + } + ], + "$comment": "Plan ordinal contiguity, resolve-first/upload-before-create ordering, and plan/history equivalence are runtime invariants. stageRef/revision agreement and timestamp ordering are runtime invariants.", + "$defs": { + "safe": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^\\x{0000}-\\x{001f}\\x{007f}-\\x{009f}\\x{202a}-\\x{202e}\\x{2066}-\\x{2069}]+$" + }, + "nullableSafe": { + "anyOf": [ + { + "$ref": "#/$defs/id" + }, + { + "type": "null" + } + ] + }, + "stageId": { + "type": "string", + "pattern": "^stg_[A-Za-z0-9_-]{32}$" + }, + "revision": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "timestamp": { + "type": "string", + "format": "date-time", + "pattern": "^([0-9]{3}[1-9]|[0-9]{2}[1-9][0-9]|[0-9][1-9][0-9]{2}|[1-9][0-9]{3})-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]{3}Z$" + }, + "serverUrl": { + "type": "string", + "maxLength": 4096, + "format": "uri", + "pattern": "^https?://[^/?#@]+(?::[0-9]+)?(?:/[^?#]*)?$" + }, + "operation": { + "enum": [ + "create_post", + "reply", + "edit_post", + "delete_post", + "react", + "unreact", + "resolve_dm", + "resolve_group_dm" + ] + }, + "binding": { + "type": "object", + "additionalProperties": false, + "required": [ + "serverUrl", + "serverId", + "userId" + ], + "properties": { + "serverUrl": { + "$ref": "#/$defs/serverUrl" + }, + "serverId": { + "$ref": "#/$defs/nullableId" + }, + "userId": { + "$ref": "#/$defs/id" + } + } + }, + "destination": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "channelId", + "channelType", + "teamId", + "postId", + "rootPostId", + "participantIds", + "emoji" + ], + "properties": { + "kind": { + "enum": [ + "conversation", + "post", + "reaction" + ] + }, + "channelId": { + "$ref": "#/$defs/nullableId" + }, + "channelType": { + "enum": [ + "dm", + "group", + "public", + "private", + null + ] + }, + "teamId": { + "$ref": "#/$defs/nullableId" + }, + "postId": { + "$ref": "#/$defs/nullableId" + }, + "rootPostId": { + "$ref": "#/$defs/nullableId" + }, + "participantIds": { + "type": "array", + "maxItems": 100, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/id" + } + }, + "emoji": { + "anyOf": [ + { + "$ref": "#/$defs/emoji" + }, + { + "type": "null" + } + ] + } + } + }, + "conversationDestination": { + "allOf": [ + { + "properties": { + "kind": { + "const": "conversation" + }, + "postId": { + "type": "null" + }, + "rootPostId": { + "type": "null" + }, + "emoji": { + "type": "null" + } + } + }, + { + "oneOf": [ + { + "$ref": "#/$defs/channelDestination" + }, + { + "$ref": "#/$defs/unresolvedDM" + }, + { + "$ref": "#/$defs/unresolvedGroup" + } + ] + } + ] + }, + "replyDestination": { + "allOf": [ + { + "$ref": "#/$defs/channelDestination" + }, + { + "properties": { + "kind": { + "const": "post" + }, + "postId": { + "$ref": "#/$defs/id" + }, + "rootPostId": { + "$ref": "#/$defs/id" + }, + "emoji": { + "type": "null" + } + } + } + ] + }, + "postDestination": { + "allOf": [ + { + "$ref": "#/$defs/channelDestination" + }, + { + "properties": { + "kind": { + "const": "post" + }, + "postId": { + "$ref": "#/$defs/id" + }, + "emoji": { + "type": "null" + } + } + } + ] + }, + "reactionDestination": { + "allOf": [ + { + "$ref": "#/$defs/channelDestination" + }, + { + "properties": { + "kind": { + "const": "reaction" + }, + "postId": { + "$ref": "#/$defs/id" + }, + "emoji": { + "$ref": "#/$defs/emoji" + } + } + } + ] + }, + "dmDestination": { + "allOf": [ + { + "$ref": "#/$defs/unresolvedDM" + }, + { + "properties": { + "kind": { + "const": "conversation" + }, + "postId": { + "type": "null" + }, + "rootPostId": { + "type": "null" + }, + "emoji": { + "type": "null" + } + } + } + ] + }, + "groupDestination": { + "allOf": [ + { + "$ref": "#/$defs/unresolvedGroup" + }, + { + "properties": { + "kind": { + "const": "conversation" + }, + "postId": { + "type": "null" + }, + "rootPostId": { + "type": "null" + }, + "emoji": { + "type": "null" + } + } + } + ] + }, + "content": { + "type": "object", + "additionalProperties": false, + "required": [ + "state", + "body" + ], + "properties": { + "state": { + "enum": [ + "present", + "none", + "pruned" + ] + }, + "body": { + "anyOf": [ + { + "$ref": "#/$defs/body" + }, + { + "type": "null" + } + ] + } + } + }, + "storedAttachment": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "canonicalPath", + "remoteFilename", + "byteLength", + "mediaType", + "contentDigest" + ], + "properties": { + "path": { + "$ref": "#/$defs/safeOutput", + "maxLength": 4096 + }, + "canonicalPath": { + "$ref": "#/$defs/safeOutput", + "maxLength": 4096 + }, + "remoteFilename": { + "$ref": "#/$defs/safeOutput", + "maxLength": 255 + }, + "byteLength": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "mediaType": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "pattern": "^[\\x{0021}-\\x{007e}]+$" + }, + "contentDigest": { + "$ref": "#/$defs/digest" + } + } + }, + "step": { + "type": "object", + "additionalProperties": false, + "required": [ + "ordinal", + "type", + "condition" + ], + "properties": { + "ordinal": { + "type": "integer", + "minimum": 1, + "maximum": 102 + }, + "type": { + "enum": [ + "resolve_conversation", + "upload_attachment", + "create_post", + "edit_post", + "delete_post", + "add_reaction", + "remove_reaction" + ] + }, + "condition": { + "enum": [ + "always", + "if_missing" + ] + } + } + }, + "plan": { + "type": "object", + "additionalProperties": false, + "required": [ + "steps" + ], + "properties": { + "steps": { + "type": "array", + "minItems": 1, + "maxItems": 102, + "items": { + "$ref": "#/$defs/step" + } + } + } + }, + "body": { + "type": "string", + "minLength": 1, + "maxLength": 16383, + "pattern": "[^\\x{0009}\\x{000a}\\x{000b}\\x{000c}\\x{000d}\\x{0020}\\x{00a0}\\x{1680}\\x{2000}-\\x{200a}\\x{2028}\\x{2029}\\x{202f}\\x{205f}\\x{3000}\\x{feff}]" + }, + "emoji": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9_+.-]+$" + }, + "oneStep": { + "type": "object", + "additionalProperties": false, + "required": [ + "steps" + ], + "properties": { + "steps": { + "type": "array", + "minItems": 1, + "maxItems": 1 + } + } + }, + "createPlan": { + "type": "object", + "additionalProperties": false, + "required": [ + "steps" + ], + "properties": { + "steps": { + "type": "array", + "minItems": 1, + "maxItems": 102, + "items": { + "allOf": [ + { + "$ref": "#/$defs/step" + }, + { + "properties": { + "type": { + "enum": [ + "resolve_conversation", + "upload_attachment", + "create_post" + ] + }, + "condition": { + "enum": [ + "always", + "if_missing" + ] + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "resolve_conversation" + } + } + }, + "then": { + "properties": { + "condition": { + "const": "if_missing" + } + } + }, + "else": { + "properties": { + "condition": { + "const": "always" + } + } + } + } + ] + }, + "contains": { + "properties": { + "type": { + "const": "create_post" + } + } + }, + "minContains": 1, + "maxContains": 1 + } + } + }, + "editPlan": { + "$ref": "#/$defs/oneStepWithEdit" + }, + "deletePlan": { + "$ref": "#/$defs/oneStepWithDelete" + }, + "reactPlan": { + "$ref": "#/$defs/oneStepWithReact" + }, + "unreactPlan": { + "$ref": "#/$defs/oneStepWithUnreact" + }, + "resolvePlan": { + "$ref": "#/$defs/oneStepWithResolve" + }, + "oneStepWithEdit": { + "allOf": [ + { + "$ref": "#/$defs/oneStep" + }, + { + "properties": { + "steps": { + "items": { + "properties": { + "type": { + "const": "edit_post" + }, + "condition": { + "const": "always" + } + } + } + } + } + } + ] + }, + "oneStepWithDelete": { + "allOf": [ + { + "$ref": "#/$defs/oneStep" + }, + { + "properties": { + "steps": { + "items": { + "properties": { + "type": { + "const": "delete_post" + }, + "condition": { + "const": "always" + } + } + } + } + } + } + ] + }, + "oneStepWithReact": { + "allOf": [ + { + "$ref": "#/$defs/oneStep" + }, + { + "properties": { + "steps": { + "items": { + "properties": { + "type": { + "const": "add_reaction" + }, + "condition": { + "enum": [ + "always", + "if_missing" + ] + } + } + } + } + } + } + ] + }, + "oneStepWithUnreact": { + "allOf": [ + { + "$ref": "#/$defs/oneStep" + }, + { + "properties": { + "steps": { + "items": { + "properties": { + "type": { + "const": "remove_reaction" + }, + "condition": { + "enum": [ + "always", + "if_missing" + ] + } + } + } + } + } + } + ] + }, + "oneStepWithResolve": { + "allOf": [ + { + "$ref": "#/$defs/oneStep" + }, + { + "properties": { + "steps": { + "items": { + "properties": { + "type": { + "const": "resolve_conversation" + }, + "condition": { + "enum": [ + "always", + "if_missing" + ] + } + } + } + } + } + } + ] + }, + "label": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^\\x{0000}-\\x{001f}\\x{007f}-\\x{009f}\\x{202a}-\\x{202e}\\x{2066}-\\x{2069}]+$" + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^\\x{0000}-\\x{0020}\\x{007f}-\\x{009f}\\x{00a0}\\x{061c}\\x{1680}\\x{2000}-\\x{200f}\\x{2028}-\\x{202f}\\x{205f}\\x{2066}-\\x{2069}\\x{3000}\\x{feff}]+$" + }, + "nullableId": { + "anyOf": [ + { + "$ref": "#/$defs/id" + }, + { + "type": "null" + } + ] + }, + "safeOutput": { + "type": "string", + "minLength": 1, + "maxLength": 4096, + "pattern": "^[^\\x{0000}-\\x{001f}\\x{007f}-\\x{009f}\\x{061c}\\x{200e}-\\x{200f}\\x{202a}-\\x{202e}\\x{2066}-\\x{2069}]+$" + }, + "channelDestination": { + "oneOf": [ + { + "$ref": "#/$defs/publicDestination" + }, + { + "$ref": "#/$defs/privateDestination" + }, + { + "$ref": "#/$defs/existingDM" + }, + { + "$ref": "#/$defs/existingGroup" + } + ] + }, + "publicDestination": { + "properties": { + "channelId": { + "$ref": "#/$defs/id" + }, + "channelType": { + "const": "public" + }, + "teamId": { + "$ref": "#/$defs/id" + }, + "participantIds": { + "maxItems": 0 + } + } + }, + "privateDestination": { + "properties": { + "channelId": { + "$ref": "#/$defs/id" + }, + "channelType": { + "const": "private" + }, + "teamId": { + "$ref": "#/$defs/id" + }, + "participantIds": { + "maxItems": 0 + } + } + }, + "existingDM": { + "properties": { + "channelId": { + "$ref": "#/$defs/id" + }, + "channelType": { + "const": "dm" + }, + "teamId": { + "type": "null" + }, + "participantIds": { + "minItems": 1, + "maxItems": 1 + } + } + }, + "existingGroup": { + "properties": { + "channelId": { + "$ref": "#/$defs/id" + }, + "channelType": { + "const": "group" + }, + "teamId": { + "type": "null" + }, + "participantIds": { + "maxItems": 0 + } + } + }, + "unresolvedDM": { + "properties": { + "channelId": { + "type": "null" + }, + "channelType": { + "const": "dm" + }, + "teamId": { + "type": "null" + }, + "participantIds": { + "minItems": 1, + "maxItems": 1 + } + } + }, + "unresolvedGroup": { + "properties": { + "channelId": { + "type": "null" + }, + "channelType": { + "const": "group" + }, + "teamId": { + "type": "null" + }, + "participantIds": { + "minItems": 2 + } + } + }, + "summary": { + "type": "object", + "additionalProperties": false, + "required": [ + "stageId", + "stageRef", + "revision", + "operation", + "semanticDigest", + "lifecycle", + "recovery", + "createdAt", + "updatedAt", + "binding", + "destination" + ], + "properties": { + "stageId": { + "$ref": "#/$defs/stageId" + }, + "stageRef": { + "type": "string", + "maxLength": 64, + "pattern": "^stg_[A-Za-z0-9_-]{32}@[1-9][0-9]{0,15}$" + }, + "revision": { + "$ref": "#/$defs/revision" + }, + "operation": { + "$ref": "#/$defs/operation" + }, + "semanticDigest": { + "$ref": "#/$defs/digest" + }, + "lifecycle": { + "enum": [ + "open", + "applying", + "completed", + "canceled", + "expired", + "pruned" + ] + }, + "recovery": { + "enum": [ + "none", + "resume_partial", + "force_unknown", + "forbidden" + ] + }, + "createdAt": { + "$ref": "#/$defs/timestamp" + }, + "updatedAt": { + "$ref": "#/$defs/timestamp" + }, + "binding": { + "$ref": "#/$defs/binding" + }, + "destination": { + "$ref": "#/$defs/destination" + } + }, + "allOf": [ + { + "if": { + "properties": { + "lifecycle": { + "enum": [ + "completed", + "canceled", + "expired", + "pruned" + ] + } + } + }, + "then": { + "properties": { + "recovery": { + "const": "forbidden" + } + } + }, + "else": { + "properties": { + "recovery": { + "enum": [ + "none", + "resume_partial", + "force_unknown" + ] + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "create_post" + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/conversationDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "reply" + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/replyDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "enum": [ + "edit_post", + "delete_post" + ] + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/postDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "enum": [ + "react", + "unreact" + ] + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/reactionDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "resolve_dm" + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/dmDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "resolve_group_dm" + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/groupDestination" + } + } + } + } + ] + }, + "resolvedCreatePlan": { + "allOf": [ + { + "$ref": "#/$defs/createPlan" + }, + { + "properties": { + "steps": { + "not": { + "contains": { + "properties": { + "type": { + "const": "resolve_conversation" + } + } + } + } + } + } + } + ] + }, + "unresolvedCreatePlan": { + "allOf": [ + { + "$ref": "#/$defs/createPlan" + }, + { + "properties": { + "steps": { + "contains": { + "properties": { + "type": { + "const": "resolve_conversation" + } + } + }, + "minContains": 1, + "maxContains": 1 + } + } + } + ] + } + } +} diff --git a/schemas/v2/stages.schema.json b/schemas/v2/stages.schema.json new file mode 100644 index 0000000..7575248 --- /dev/null +++ b/schemas/v2/stages.schema.json @@ -0,0 +1,1055 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:stages", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "stages", + "nextCursor" + ], + "properties": { + "schema": { + "const": "mm/v2/stages" + }, + "stages": { + "type": "array", + "maxItems": 100, + "items": { + "$ref": "#/$defs/summary" + } + }, + "nextCursor": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "maxLength": 4096, + "pattern": "^[A-Za-z0-9_-]+$" + }, + { + "type": "null" + } + ] + } + }, + "$comment": "Duplicate stage rows and cursor/page consistency are runtime invariants because JSON Schema uniqueItems cannot express identity-only uniqueness without also rejecting distinct revisions.", + "$defs": { + "safe": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^\\x{0000}-\\x{001f}\\x{007f}-\\x{009f}\\x{202a}-\\x{202e}\\x{2066}-\\x{2069}]+$" + }, + "nullableSafe": { + "anyOf": [ + { + "$ref": "#/$defs/id" + }, + { + "type": "null" + } + ] + }, + "stageId": { + "type": "string", + "pattern": "^stg_[A-Za-z0-9_-]{32}$" + }, + "revision": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991 + }, + "digest": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "timestamp": { + "type": "string", + "format": "date-time", + "pattern": "^([0-9]{3}[1-9]|[0-9]{2}[1-9][0-9]|[0-9][1-9][0-9]{2}|[1-9][0-9]{3})-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]{3}Z$" + }, + "serverUrl": { + "type": "string", + "maxLength": 4096, + "format": "uri", + "pattern": "^https?://[^/?#@]+(?::[0-9]+)?(?:/[^?#]*)?$" + }, + "operation": { + "enum": [ + "create_post", + "reply", + "edit_post", + "delete_post", + "react", + "unreact", + "resolve_dm", + "resolve_group_dm" + ] + }, + "binding": { + "type": "object", + "additionalProperties": false, + "required": [ + "serverUrl", + "serverId", + "userId" + ], + "properties": { + "serverUrl": { + "$ref": "#/$defs/serverUrl" + }, + "serverId": { + "$ref": "#/$defs/nullableId" + }, + "userId": { + "$ref": "#/$defs/id" + } + } + }, + "destination": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "channelId", + "channelType", + "teamId", + "postId", + "rootPostId", + "participantIds", + "emoji" + ], + "properties": { + "kind": { + "enum": [ + "conversation", + "post", + "reaction" + ] + }, + "channelId": { + "$ref": "#/$defs/nullableId" + }, + "channelType": { + "enum": [ + "dm", + "group", + "public", + "private", + null + ] + }, + "teamId": { + "$ref": "#/$defs/nullableId" + }, + "postId": { + "$ref": "#/$defs/nullableId" + }, + "rootPostId": { + "$ref": "#/$defs/nullableId" + }, + "participantIds": { + "type": "array", + "maxItems": 100, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/id" + } + }, + "emoji": { + "anyOf": [ + { + "$ref": "#/$defs/emoji" + }, + { + "type": "null" + } + ] + } + } + }, + "conversationDestination": { + "allOf": [ + { + "properties": { + "kind": { + "const": "conversation" + }, + "postId": { + "type": "null" + }, + "rootPostId": { + "type": "null" + }, + "emoji": { + "type": "null" + } + } + }, + { + "oneOf": [ + { + "$ref": "#/$defs/channelDestination" + }, + { + "$ref": "#/$defs/unresolvedDM" + }, + { + "$ref": "#/$defs/unresolvedGroup" + } + ] + } + ] + }, + "replyDestination": { + "allOf": [ + { + "$ref": "#/$defs/channelDestination" + }, + { + "properties": { + "kind": { + "const": "post" + }, + "postId": { + "$ref": "#/$defs/id" + }, + "rootPostId": { + "$ref": "#/$defs/id" + }, + "emoji": { + "type": "null" + } + } + } + ] + }, + "postDestination": { + "allOf": [ + { + "$ref": "#/$defs/channelDestination" + }, + { + "properties": { + "kind": { + "const": "post" + }, + "postId": { + "$ref": "#/$defs/id" + }, + "emoji": { + "type": "null" + } + } + } + ] + }, + "reactionDestination": { + "allOf": [ + { + "$ref": "#/$defs/channelDestination" + }, + { + "properties": { + "kind": { + "const": "reaction" + }, + "postId": { + "$ref": "#/$defs/id" + }, + "emoji": { + "$ref": "#/$defs/emoji" + } + } + } + ] + }, + "dmDestination": { + "allOf": [ + { + "$ref": "#/$defs/unresolvedDM" + }, + { + "properties": { + "kind": { + "const": "conversation" + }, + "postId": { + "type": "null" + }, + "rootPostId": { + "type": "null" + }, + "emoji": { + "type": "null" + } + } + } + ] + }, + "groupDestination": { + "allOf": [ + { + "$ref": "#/$defs/unresolvedGroup" + }, + { + "properties": { + "kind": { + "const": "conversation" + }, + "postId": { + "type": "null" + }, + "rootPostId": { + "type": "null" + }, + "emoji": { + "type": "null" + } + } + } + ] + }, + "content": { + "type": "object", + "additionalProperties": false, + "required": [ + "state", + "body" + ], + "properties": { + "state": { + "enum": [ + "present", + "none", + "pruned" + ] + }, + "body": { + "anyOf": [ + { + "$ref": "#/$defs/body" + }, + { + "type": "null" + } + ] + } + } + }, + "storedAttachment": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "canonicalPath", + "remoteFilename", + "byteLength", + "mediaType", + "contentDigest" + ], + "properties": { + "path": { + "$ref": "#/$defs/safeOutput", + "maxLength": 4096 + }, + "canonicalPath": { + "$ref": "#/$defs/safeOutput", + "maxLength": 4096 + }, + "remoteFilename": { + "$ref": "#/$defs/safeOutput", + "maxLength": 255 + }, + "byteLength": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "mediaType": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "pattern": "^[\\x{0021}-\\x{007e}]+$" + }, + "contentDigest": { + "$ref": "#/$defs/digest" + } + } + }, + "step": { + "type": "object", + "additionalProperties": false, + "required": [ + "ordinal", + "type", + "condition" + ], + "properties": { + "ordinal": { + "type": "integer", + "minimum": 1, + "maximum": 102 + }, + "type": { + "enum": [ + "resolve_conversation", + "upload_attachment", + "create_post", + "edit_post", + "delete_post", + "add_reaction", + "remove_reaction" + ] + }, + "condition": { + "enum": [ + "always", + "if_missing" + ] + } + } + }, + "plan": { + "type": "object", + "additionalProperties": false, + "required": [ + "steps" + ], + "properties": { + "steps": { + "type": "array", + "minItems": 1, + "maxItems": 102, + "items": { + "$ref": "#/$defs/step" + } + } + } + }, + "body": { + "type": "string", + "minLength": 1, + "maxLength": 16383, + "pattern": "[^\\x{0009}\\x{000a}\\x{000b}\\x{000c}\\x{000d}\\x{0020}\\x{00a0}\\x{1680}\\x{2000}-\\x{200a}\\x{2028}\\x{2029}\\x{202f}\\x{205f}\\x{3000}\\x{feff}]" + }, + "emoji": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9_+.-]+$" + }, + "oneStep": { + "type": "object", + "additionalProperties": false, + "required": [ + "steps" + ], + "properties": { + "steps": { + "type": "array", + "minItems": 1, + "maxItems": 1 + } + } + }, + "createPlan": { + "type": "object", + "additionalProperties": false, + "required": [ + "steps" + ], + "properties": { + "steps": { + "type": "array", + "minItems": 1, + "maxItems": 102, + "items": { + "allOf": [ + { + "$ref": "#/$defs/step" + }, + { + "properties": { + "type": { + "enum": [ + "resolve_conversation", + "upload_attachment", + "create_post" + ] + }, + "condition": { + "enum": [ + "always", + "if_missing" + ] + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "resolve_conversation" + } + } + }, + "then": { + "properties": { + "condition": { + "const": "if_missing" + } + } + }, + "else": { + "properties": { + "condition": { + "const": "always" + } + } + } + } + ] + }, + "contains": { + "properties": { + "type": { + "const": "create_post" + } + } + }, + "minContains": 1, + "maxContains": 1 + } + } + }, + "editPlan": { + "$ref": "#/$defs/oneStepWithEdit" + }, + "deletePlan": { + "$ref": "#/$defs/oneStepWithDelete" + }, + "reactPlan": { + "$ref": "#/$defs/oneStepWithReact" + }, + "unreactPlan": { + "$ref": "#/$defs/oneStepWithUnreact" + }, + "resolvePlan": { + "$ref": "#/$defs/oneStepWithResolve" + }, + "oneStepWithEdit": { + "allOf": [ + { + "$ref": "#/$defs/oneStep" + }, + { + "properties": { + "steps": { + "items": { + "properties": { + "type": { + "const": "edit_post" + }, + "condition": { + "const": "always" + } + } + } + } + } + } + ] + }, + "oneStepWithDelete": { + "allOf": [ + { + "$ref": "#/$defs/oneStep" + }, + { + "properties": { + "steps": { + "items": { + "properties": { + "type": { + "const": "delete_post" + }, + "condition": { + "const": "always" + } + } + } + } + } + } + ] + }, + "oneStepWithReact": { + "allOf": [ + { + "$ref": "#/$defs/oneStep" + }, + { + "properties": { + "steps": { + "items": { + "properties": { + "type": { + "const": "add_reaction" + }, + "condition": { + "enum": [ + "always", + "if_missing" + ] + } + } + } + } + } + } + ] + }, + "oneStepWithUnreact": { + "allOf": [ + { + "$ref": "#/$defs/oneStep" + }, + { + "properties": { + "steps": { + "items": { + "properties": { + "type": { + "const": "remove_reaction" + }, + "condition": { + "enum": [ + "always", + "if_missing" + ] + } + } + } + } + } + } + ] + }, + "oneStepWithResolve": { + "allOf": [ + { + "$ref": "#/$defs/oneStep" + }, + { + "properties": { + "steps": { + "items": { + "properties": { + "type": { + "const": "resolve_conversation" + }, + "condition": { + "enum": [ + "always", + "if_missing" + ] + } + } + } + } + } + } + ] + }, + "label": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^\\x{0000}-\\x{001f}\\x{007f}-\\x{009f}\\x{202a}-\\x{202e}\\x{2066}-\\x{2069}]+$" + }, + "id": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^\\x{0000}-\\x{0020}\\x{007f}-\\x{009f}\\x{00a0}\\x{061c}\\x{1680}\\x{2000}-\\x{200f}\\x{2028}-\\x{202f}\\x{205f}\\x{2066}-\\x{2069}\\x{3000}\\x{feff}]+$" + }, + "nullableId": { + "anyOf": [ + { + "$ref": "#/$defs/id" + }, + { + "type": "null" + } + ] + }, + "safeOutput": { + "type": "string", + "minLength": 1, + "maxLength": 4096, + "pattern": "^[^\\x{0000}-\\x{001f}\\x{007f}-\\x{009f}\\x{061c}\\x{200e}-\\x{200f}\\x{202a}-\\x{202e}\\x{2066}-\\x{2069}]+$" + }, + "channelDestination": { + "oneOf": [ + { + "$ref": "#/$defs/publicDestination" + }, + { + "$ref": "#/$defs/privateDestination" + }, + { + "$ref": "#/$defs/existingDM" + }, + { + "$ref": "#/$defs/existingGroup" + } + ] + }, + "publicDestination": { + "properties": { + "channelId": { + "$ref": "#/$defs/id" + }, + "channelType": { + "const": "public" + }, + "teamId": { + "$ref": "#/$defs/id" + }, + "participantIds": { + "maxItems": 0 + } + } + }, + "privateDestination": { + "properties": { + "channelId": { + "$ref": "#/$defs/id" + }, + "channelType": { + "const": "private" + }, + "teamId": { + "$ref": "#/$defs/id" + }, + "participantIds": { + "maxItems": 0 + } + } + }, + "existingDM": { + "properties": { + "channelId": { + "$ref": "#/$defs/id" + }, + "channelType": { + "const": "dm" + }, + "teamId": { + "type": "null" + }, + "participantIds": { + "minItems": 1, + "maxItems": 1 + } + } + }, + "existingGroup": { + "properties": { + "channelId": { + "$ref": "#/$defs/id" + }, + "channelType": { + "const": "group" + }, + "teamId": { + "type": "null" + }, + "participantIds": { + "maxItems": 0 + } + } + }, + "unresolvedDM": { + "properties": { + "channelId": { + "type": "null" + }, + "channelType": { + "const": "dm" + }, + "teamId": { + "type": "null" + }, + "participantIds": { + "minItems": 1, + "maxItems": 1 + } + } + }, + "unresolvedGroup": { + "properties": { + "channelId": { + "type": "null" + }, + "channelType": { + "const": "group" + }, + "teamId": { + "type": "null" + }, + "participantIds": { + "minItems": 2 + } + } + }, + "summary": { + "type": "object", + "additionalProperties": false, + "required": [ + "stageId", + "stageRef", + "revision", + "operation", + "semanticDigest", + "lifecycle", + "recovery", + "createdAt", + "updatedAt", + "binding", + "destination" + ], + "properties": { + "stageId": { + "$ref": "#/$defs/stageId" + }, + "stageRef": { + "type": "string", + "maxLength": 64, + "pattern": "^stg_[A-Za-z0-9_-]{32}@[1-9][0-9]{0,15}$" + }, + "revision": { + "$ref": "#/$defs/revision" + }, + "operation": { + "$ref": "#/$defs/operation" + }, + "semanticDigest": { + "$ref": "#/$defs/digest" + }, + "lifecycle": { + "enum": [ + "open", + "applying", + "completed", + "canceled", + "expired", + "pruned" + ] + }, + "recovery": { + "enum": [ + "none", + "resume_partial", + "force_unknown", + "forbidden" + ] + }, + "createdAt": { + "$ref": "#/$defs/timestamp" + }, + "updatedAt": { + "$ref": "#/$defs/timestamp" + }, + "binding": { + "$ref": "#/$defs/binding" + }, + "destination": { + "$ref": "#/$defs/destination" + } + }, + "allOf": [ + { + "if": { + "properties": { + "lifecycle": { + "enum": [ + "completed", + "canceled", + "expired", + "pruned" + ] + } + } + }, + "then": { + "properties": { + "recovery": { + "const": "forbidden" + } + } + }, + "else": { + "properties": { + "recovery": { + "enum": [ + "none", + "resume_partial", + "force_unknown" + ] + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "create_post" + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/conversationDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "reply" + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/replyDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "enum": [ + "edit_post", + "delete_post" + ] + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/postDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "enum": [ + "react", + "unreact" + ] + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/reactionDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "resolve_dm" + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/dmDestination" + } + } + } + }, + { + "if": { + "properties": { + "operation": { + "const": "resolve_group_dm" + } + } + }, + "then": { + "properties": { + "destination": { + "$ref": "#/$defs/groupDestination" + } + } + } + } + ] + }, + "resolvedCreatePlan": { + "allOf": [ + { + "$ref": "#/$defs/createPlan" + }, + { + "properties": { + "steps": { + "not": { + "contains": { + "properties": { + "type": { + "const": "resolve_conversation" + } + } + } + } + } + } + } + ] + }, + "unresolvedCreatePlan": { + "allOf": [ + { + "$ref": "#/$defs/createPlan" + }, + { + "properties": { + "steps": { + "contains": { + "properties": { + "type": { + "const": "resolve_conversation" + } + } + }, + "minContains": 1, + "maxContains": 1 + } + } + } + ] + } + } +} From 8cc81216f4c3bb44e7616898ff1a786c6e9216be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 01:20:31 +0300 Subject: [PATCH 057/119] feat: add target-bound stage creation --- internal/mattermost/users.go | 18 +- internal/mattermost/users_test.go | 24 ++ internal/staging/service.go | 541 ++++++++++++++++++++++++++++++ internal/staging/service_test.go | 454 +++++++++++++++++++++++++ 4 files changed, 1036 insertions(+), 1 deletion(-) create mode 100644 internal/staging/service.go create mode 100644 internal/staging/service_test.go diff --git a/internal/mattermost/users.go b/internal/mattermost/users.go index 6eaad6b..9a01539 100644 --- a/internal/mattermost/users.go +++ b/internal/mattermost/users.go @@ -141,6 +141,16 @@ func (s *Users) ByUsername(ctx context.Context, username string) (User, error) { if ok && present && strings.EqualFold(user.Username, username) { return user, nil } + return s.ByUsernameFresh(ctx, username) +} + +// ByUsernameFresh always performs an authenticated remote lookup. It is used +// by mutation staging so a username reassigned since an earlier read cannot +// bind the former account from the session cache. +func (s *Users) ByUsernameFresh(ctx context.Context, username string) (User, error) { + if strings.TrimSpace(username) == "" { + return User{}, ErrInvalidUserRequest + } var fetched User if err := s.client.Get(ctx, "/users/username/"+url.PathEscape(username), &fetched); err != nil { return User{}, err @@ -284,8 +294,14 @@ func (s *Users) cache(user User) { delete(s.byName, previousKey) } } + key := strings.ToLower(user.Username) + if previousID, ok := s.byName[key]; ok && previousID != user.ID { + // Keep the previous profile addressable by immutable ID, but remove the + // stale name edge before installing the reassigned owner. + delete(s.byName, key) + } s.byID[user.ID] = user - s.byName[strings.ToLower(user.Username)] = user.ID + s.byName[key] = user.ID } func encodeQueryComponent(value string) string { diff --git a/internal/mattermost/users_test.go b/internal/mattermost/users_test.go index a091616..0f88e97 100644 --- a/internal/mattermost/users_test.go +++ b/internal/mattermost/users_test.go @@ -133,6 +133,30 @@ func TestCacheReplacementCannotResolveAStaleUsername(t *testing.T) { } } +func TestByUsernameFreshObservesReassignmentAndRepairsNameCache(t *testing.T) { + transport := &fakeUsersTransport{responses: []string{ + `{"id":"former","username":"shared"}`, + `{"id":"new-owner","username":"shared"}`, + }} + users := NewUsers(transport) + former, err := users.ByUsername(context.Background(), "shared") + if err != nil || former.ID != "former" { + t.Fatalf("former/error = %#v/%v", former, err) + } + owner, err := users.ByUsernameFresh(context.Background(), "shared") + if err != nil || owner.ID != "new-owner" { + t.Fatalf("owner/error = %#v/%v", owner, err) + } + cached, err := users.ByUsername(context.Background(), "shared") + if err != nil || cached.ID != "new-owner" || len(transport.calls) != 2 { + t.Fatalf("cached/error/calls = %#v/%v/%d", cached, err, len(transport.calls)) + } + byID, err := users.ByID(context.Background(), "former") + if err != nil || byID.ID != "former" || len(transport.calls) != 2 { + t.Fatalf("immutable ID cache = %#v/%v", byID, err) + } +} + func TestByIDsDeduplicatesRequestPreservesOrderAndRequiresCompleteResult(t *testing.T) { transport := &fakeUsersTransport{responses: []string{`[ {"id":"b","username":"bob"},{"id":"a","username":"alice"} diff --git a/internal/staging/service.go b/internal/staging/service.go new file mode 100644 index 0000000..1f4173d --- /dev/null +++ b/internal/staging/service.go @@ -0,0 +1,541 @@ +// Package staging resolves and validates mutation targets before admitting a +// canonical plan to the stage store. +package staging + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "strings" + "unicode" + "unicode/utf8" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/messageinput" + "github.com/ardasevinc/mattermost-cli/internal/serverurl" + "github.com/ardasevinc/mattermost-cli/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +var ( + ErrInvalid = errors.New("staging: invalid request") + ErrTarget = errors.New("staging: target could not be resolved") + ErrCredential = errors.New("staging: protected credential present") + ErrInput = errors.New("staging: message or attachment input rejected") + ErrStore = errors.New("staging: stage could not be persisted") + ErrConflict = errors.New("staging: request conflict") +) + +type ConversationType uint8 + +const ( + Direct ConversationType = iota + 1 + Group + Channel +) + +type SelectorType uint8 + +const ( + ByUsername SelectorType = iota + 1 + ByID + ByName +) + +// Target is deliberately syntactic. Resolved IDs and serialized plans are not +// caller-settable. +type Target struct { + Conversation ConversationType + Selector SelectorType + Value string + Team *TeamSelector +} + +type TeamSelector struct { + By SelectorType // ByID or ByName + Value string +} + +type Attachment = stageinput.Attachment + +type CreatePostInput struct { + RequestID string + Target Target + Body io.Reader + Attachments []Attachment +} + +type DryRunInput struct{ Target Target } + +type Destination struct { + Kind string `json:"kind"` + ChannelID string `json:"channelId"` + ChannelType string `json:"channelType"` + TeamID *string `json:"teamId"` + PostID *string `json:"postId"` + RootPostID *string `json:"rootPostId"` + ParticipantIDs []string `json:"participantIds"` + Emoji *string `json:"emoji"` +} + +type Plan struct { + Steps []PlanStep `json:"steps"` +} +type PlanStep struct { + Ordinal int `json:"ordinal"` + Type string `json:"type"` + Condition string `json:"condition"` +} + +type Preview struct { + ServerURL string + ServerID string + UserID string + Destination Destination + Plan Plan +} + +type CreatePostResult struct { + Preview Preview + Stored stagestore.MutationResult +} + +type Users interface { + Current(context.Context) (mattermost.User, error) + ByUsernameFresh(context.Context, string) (mattermost.User, error) +} +type Channels interface { + ExistingDirect(context.Context, string, string) (mattermost.Channel, bool, error) + ByID(context.Context, string) (mattermost.Channel, error) + ByName(context.Context, string, string) (mattermost.Channel, error) + Member(context.Context, string, string) (mattermost.ChannelMember, error) +} +type Teams interface { + List(context.Context, string) (mattermost.TeamMembership, error) +} +type Store interface { + Create(context.Context, stagestore.CreateInput) (stagestore.MutationResult, error) +} +type AttachmentBinder func(context.Context, []stageinput.Attachment, [][]byte) ([]stagestore.Attachment, error) + +type Service struct { + serverURL, serverID string + users Users + channels Channels + teams Teams + store Store + bind AttachmentBinder + credentials [][]byte +} + +func New(serverBaseURL, serverID string, credentials []string, users Users, channels Channels, teams Teams, store Store) (*Service, error) { + normalized, err := serverurl.Normalize(serverBaseURL) + if err != nil || users == nil || channels == nil || teams == nil || (serverID != "" && !validIdentity(serverID)) { + return nil, ErrInvalid + } + protected := credentialBytes(credentials) + if len(protected) > 64 { + return nil, ErrInvalid + } + total := 0 + for _, credential := range protected { + total += len(credential) + if len(credential) > 4096 || total > 64<<10 { + return nil, ErrInvalid + } + } + if contaminated(protected, normalized+"/api/v4", serverID) { + return nil, ErrCredential + } + return &Service{serverURL: normalized + "/api/v4", serverID: serverID, users: users, channels: channels, teams: teams, store: store, bind: stageinput.Bind, credentials: protected}, nil +} + +// WithAttachmentBinder is intended for narrow tests which must prove dry-run +// and early failures perform no filesystem I/O. +func (s *Service) WithAttachmentBinder(bind AttachmentBinder) *Service { + copy := *s + copy.bind = bind + return © +} + +func (s *Service) DryRunCreatePost(ctx context.Context, in DryRunInput) (Preview, error) { + return s.resolve(ctx, in.Target) +} + +func (s *Service) CreatePost(ctx context.Context, in CreatePostInput) (CreatePostResult, error) { + if s.store == nil || s.bind == nil || in.Body == nil { + return CreatePostResult{}, ErrInvalid + } + if in.RequestID == "" || !validRequestID(in.RequestID) { + return CreatePostResult{}, ErrInvalid + } + callerFields := append([]string{in.RequestID}, targetStrings(in.Target)...) + if contaminated(s.credentials, callerFields...) { + return CreatePostResult{}, ErrCredential + } + preview, err := s.resolve(ctx, in.Target) + if err != nil { + return CreatePostResult{}, err + } + body, err := messageinput.Read(in.Body) + if err != nil { + return CreatePostResult{}, ErrInput + } + if containsCredential(s.credentials, body) { + return CreatePostResult{}, ErrCredential + } + attachments, err := s.bind(ctx, in.Attachments, cloneCredentials(s.credentials)) + if err != nil { + if errors.Is(err, stageinput.ErrCredential) { + return CreatePostResult{}, ErrCredential + } + return CreatePostResult{}, ErrInput + } + if !validBoundAttachments(attachments) { + return CreatePostResult{}, ErrInput + } + preview.Plan = attachmentPlan(len(attachments)) + destination, plan, err := marshalSemantics(preview) + if err != nil { + return CreatePostResult{}, ErrInvalid + } + if contaminated(s.credentials, in.RequestID, preview.ServerURL, preview.ServerID, preview.UserID, string(destination), string(plan)) || containsCredential(s.credentials, body) || attachmentsContaminated(s.credentials, attachments) { + return CreatePostResult{}, ErrCredential + } + stored, err := s.store.Create(ctx, stagestore.CreateInput{RequestID: in.RequestID, Operation: stagestore.CreatePost, + ServerURL: preview.ServerURL, ServerID: preview.ServerID, UserID: preview.UserID, + Content: stagestore.RevisionContent{Body: body, Destination: destination, Plan: plan, Attachments: attachments}}) + if err != nil { + if errors.Is(err, stagestore.ErrConflict) { + return CreatePostResult{}, ErrConflict + } + return CreatePostResult{}, ErrStore + } + return CreatePostResult{Preview: preview, Stored: stored}, nil +} + +func (s *Service) resolve(ctx context.Context, target Target) (Preview, error) { + if ctx == nil || !validTargetSyntax(target) { + return Preview{}, ErrInvalid + } + if contaminated(s.credentials, targetStrings(target)...) { + return Preview{}, ErrCredential + } + current, err := s.users.Current(ctx) + if err != nil || !validResolvedUser(current) { + return Preview{}, ErrTarget + } + if contaminated(s.credentials, current.ID, current.Username) { + return Preview{}, ErrCredential + } + channel, participants, err := s.resolveChannel(ctx, current, target) + if err != nil { + return Preview{}, err + } + var teamID *string + if channel.Type == "O" || channel.Type == "P" { + value := channel.TeamID + teamID = &value + } + preview := Preview{s.serverURL, s.serverID, current.ID, Destination{"conversation", channel.ID, channelType(channel.Type), teamID, nil, nil, participants, nil}, createPostPlan()} + destination, plan, err := marshalSemantics(preview) + if err != nil { + return Preview{}, ErrInvalid + } + fields := append(targetStrings(target), preview.ServerURL, preview.ServerID, preview.UserID, string(destination), string(plan)) + if contaminated(s.credentials, fields...) { + return Preview{}, ErrCredential + } + return preview, nil +} + +func (s *Service) resolveChannel(ctx context.Context, current mattermost.User, target Target) (mattermost.Channel, []string, error) { + switch target.Conversation { + case Direct: + if target.Selector != ByUsername || target.Team != nil { + return mattermost.Channel{}, nil, ErrInvalid + } + peer, err := s.users.ByUsernameFresh(ctx, target.Value) + if err != nil || !validResolvedUser(peer) || !strings.EqualFold(peer.Username, target.Value) || peer.ID == current.ID { + return mattermost.Channel{}, nil, ErrTarget + } + if contaminated(s.credentials, peer.ID, peer.Username) { + return mattermost.Channel{}, nil, ErrCredential + } + channel, found, err := s.channels.ExistingDirect(ctx, current.ID, peer.ID) + if err != nil || !found || !validResolvedChannel(channel) || channel.Type != "D" { + return mattermost.Channel{}, nil, ErrTarget + } + if contaminated(s.credentials, channel.ID, channel.Name) { + return mattermost.Channel{}, nil, ErrCredential + } + return channel, []string{peer.ID}, nil + case Group: + if target.Selector != ByID || target.Team != nil { + return mattermost.Channel{}, nil, ErrInvalid + } + channel, err := s.channels.ByID(ctx, target.Value) + if err != nil || !validResolvedChannel(channel) || channel.Type != "G" { + return mattermost.Channel{}, nil, ErrTarget + } + if contaminated(s.credentials, channel.ID, channel.Name) { + return mattermost.Channel{}, nil, ErrCredential + } + if _, err = s.channels.Member(ctx, channel.ID, current.ID); err != nil { + return mattermost.Channel{}, nil, ErrTarget + } + return channel, []string{}, nil + case Channel: + var channel mattermost.Channel + var err error + if target.Selector == ByID && target.Team == nil { + channel, err = s.channels.ByID(ctx, target.Value) + } else if target.Selector == ByName && target.Team != nil { + team, teamErr := s.resolveTeam(ctx, current.ID, *target.Team) + if teamErr != nil { + return mattermost.Channel{}, nil, teamErr + } + channel, err = s.channels.ByName(ctx, team.ID, target.Value) + } else { + return mattermost.Channel{}, nil, ErrInvalid + } + if err != nil || !validResolvedChannel(channel) || (channel.Type != "O" && channel.Type != "P") { + return mattermost.Channel{}, nil, ErrTarget + } + if contaminated(s.credentials, channel.ID, channel.Name, channel.TeamID) { + return mattermost.Channel{}, nil, ErrCredential + } + if _, err = s.channels.Member(ctx, channel.ID, current.ID); err != nil { + return mattermost.Channel{}, nil, ErrTarget + } + return channel, []string{}, nil + default: + return mattermost.Channel{}, nil, ErrInvalid + } +} + +func (s *Service) resolveTeam(ctx context.Context, userID string, selector TeamSelector) (mattermost.Team, error) { + if (selector.By != ByID && selector.By != ByName) || !validSelectorValue(selector.Value) { + return mattermost.Team{}, ErrInvalid + } + membership, err := s.teams.List(ctx, userID) + if err != nil { + return mattermost.Team{}, ErrTarget + } + var match mattermost.Team + count := 0 + for _, team := range membership.Items() { + if !validResolvedTeam(team) { + return mattermost.Team{}, ErrTarget + } + matched := selector.By == ByID && team.ID == selector.Value + if selector.By == ByName { + matched = team.Name == selector.Value || team.DisplayName == selector.Value + } + if matched { + match, count = team, count+1 + } + } + if count != 1 { + return mattermost.Team{}, ErrTarget + } + if contaminated(s.credentials, match.ID, match.Name, match.DisplayName) { + return mattermost.Team{}, ErrCredential + } + return match, nil +} + +func channelType(value string) string { + return map[string]string{"D": "dm", "G": "group", "O": "public", "P": "private"}[value] +} +func createPostPlan() Plan { + return Plan{[]PlanStep{{1, "create_post", "always"}}} +} +func attachmentPlan(count int) Plan { + steps := make([]PlanStep, 0, count+1) + for i := range count { + steps = append(steps, PlanStep{i + 1, "upload_attachment", "always"}) + } + steps = append(steps, PlanStep{count + 1, "create_post", "always"}) + return Plan{steps} +} +func marshalSemantics(preview Preview) ([]byte, []byte, error) { + destination, err := json.Marshal(preview.Destination) + if err != nil { + return nil, nil, err + } + plan, err := json.Marshal(preview.Plan) + return destination, plan, err +} +func targetStrings(t Target) []string { + v := []string{t.Value} + if t.Team != nil { + v = append(v, t.Team.Value) + } + return v +} +func credentialBytes(values []string) [][]byte { + out := make([][]byte, 0, len(values)) + for _, v := range values { + if v != "" { + out = append(out, []byte(v)) + } + } + return out +} +func cloneCredentials(values [][]byte) [][]byte { + out := make([][]byte, len(values)) + for i := range values { + out[i] = bytes.Clone(values[i]) + } + return out +} +func containsCredential(credentials [][]byte, value []byte) bool { + for _, c := range credentials { + if bytes.Contains(value, c) { + return true + } + } + return false +} +func contaminated(credentials [][]byte, values ...string) bool { + for _, v := range values { + if containsCredential(credentials, []byte(v)) { + return true + } + } + return false +} +func attachmentsContaminated(credentials [][]byte, values []stagestore.Attachment) bool { + for _, v := range values { + if contaminated(credentials, v.SuppliedPath, v.CanonicalPath, v.RemoteFilename, v.MediaType) { + return true + } + } + return false +} + +func validSelectorValue(value string) bool { + if value == "" || len(value) > 256 || !utf8.ValidString(value) || value != strings.TrimSpace(value) { + return false + } + for _, r := range value { + if unsafeIdentityRune(r) || unicode.IsSpace(r) { + return false + } + } + return true +} + +func validIdentity(value string) bool { + return validSelectorValue(value) +} + +func validOptionalText(value string) bool { + return value == "" || validSafeText(value, 256) +} + +func validResolvedUser(user mattermost.User) bool { + return validIdentity(user.ID) && validSelectorValue(user.Username) +} + +func validResolvedChannel(channel mattermost.Channel) bool { + if !validIdentity(channel.ID) || !validSelectorValue(channel.Name) || !validOptionalText(channel.DisplayName) { + return false + } + switch channel.Type { + case "D", "G": + return channel.TeamID == "" + case "O", "P": + return validIdentity(channel.TeamID) + default: + return false + } +} + +func validResolvedTeam(team mattermost.Team) bool { + return validIdentity(team.ID) && validSelectorValue(team.Name) && validOptionalText(team.DisplayName) && (team.Type == "O" || team.Type == "I") +} + +func validBoundAttachments(values []stagestore.Attachment) bool { + if len(values) > 100 { + return false + } + for _, value := range values { + if !validBoundText(value.SuppliedPath, 4096) || !validBoundText(value.CanonicalPath, 4096) || + !validBoundText(value.RemoteFilename, 255) || (value.MediaType != "" && !validBoundText(value.MediaType, 255)) || + value.ByteLength < 0 || value.ContentDigest == ([32]byte{}) { + return false + } + } + return true +} + +func validBoundText(value string, maximum int) bool { + if value == "" || len(value) > maximum || !utf8.ValidString(value) || value != strings.TrimSpace(value) { + return false + } + for _, r := range value { + if unsafeRune(r) { + return false + } + } + return true +} + +func validSafeText(value string, maximum int) bool { + return value != "" && len(value) <= maximum && utf8.ValidString(value) && value == strings.TrimSpace(value) && !strings.ContainsFunc(value, unsafeRune) +} + +func unsafeRune(r rune) bool { + return unicode.IsControl(r) || r == '\u061c' || r == '\u200e' || r == '\u200f' || + r >= '\u202a' && r <= '\u202e' || r >= '\u2066' && r <= '\u2069' +} + +func unsafeIdentityRune(r rune) bool { + return unsafeRune(r) || r >= '\u200b' && r <= '\u200d' || r == '\ufeff' +} + +func validTargetSyntax(target Target) bool { + if !validSelectorValue(target.Value) || target.Selector == ByName && strings.HasPrefix(target.Value, "#") { + return false + } + switch target.Conversation { + case Direct: + return target.Selector == ByUsername && target.Team == nil + case Group: + return target.Selector == ByID && target.Team == nil + case Channel: + if target.Selector == ByID { + return target.Team == nil + } + return target.Selector == ByName && target.Team != nil && + (target.Team.By == ByID || target.Team.By == ByName) && validSelectorValue(target.Team.Value) + default: + return false + } +} + +func validRequestID(value string) bool { + if value == "" { + return true + } + if len(value) > 256 || !requestCharacter(value[0], true) { + return false + } + for i := 1; i < len(value); i++ { + if !requestCharacter(value[i], false) { + return false + } + } + return true +} + +func requestCharacter(value byte, first bool) bool { + if value >= 'A' && value <= 'Z' || value >= 'a' && value <= 'z' || value >= '0' && value <= '9' { + return true + } + return !first && strings.ContainsRune("._~:-", rune(value)) +} diff --git a/internal/staging/service_test.go b/internal/staging/service_test.go new file mode 100644 index 0000000..a289420 --- /dev/null +++ b/internal/staging/service_test.go @@ -0,0 +1,454 @@ +package staging + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +type fakeUsers struct { + current mattermost.User + peer mattermost.User + err error + currentCalls atomic.Int64 +} + +func (f *fakeUsers) Current(context.Context) (mattermost.User, error) { + f.currentCalls.Add(1) + return f.current, f.err +} +func (f *fakeUsers) ByUsernameFresh(context.Context, string) (mattermost.User, error) { + return f.peer, f.err +} + +type fakeChannels struct { + direct mattermost.Channel + found bool + byID mattermost.Channel + byName mattermost.Channel + err error + memberErr error +} + +func (f *fakeChannels) ExistingDirect(context.Context, string, string) (mattermost.Channel, bool, error) { + return f.direct, f.found, f.err +} +func (f *fakeChannels) ByID(context.Context, string) (mattermost.Channel, error) { + return f.byID, f.err +} +func (f *fakeChannels) ByName(context.Context, string, string) (mattermost.Channel, error) { + return f.byName, f.err +} +func (f *fakeChannels) Member(_ context.Context, channelID, userID string) (mattermost.ChannelMember, error) { + return mattermost.ChannelMember{ChannelID: channelID, UserID: userID}, f.memberErr +} + +type emptyTeams struct{} + +func (emptyTeams) List(context.Context, string) (mattermost.TeamMembership, error) { + return mattermost.TeamMembership{}, nil +} + +type teamTransport struct{ payload string } + +func (t teamTransport) Get(_ context.Context, _ string, out any) error { + return json.Unmarshal([]byte(t.payload), out) +} + +type recordingStore struct { + mu sync.Mutex + calls int + in stagestore.CreateInput + err error +} + +func (s *recordingStore) Create(_ context.Context, in stagestore.CreateInput) (stagestore.MutationResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.calls++ + s.in = in + return stagestore.MutationResult{}, s.err +} + +func dmService(t *testing.T, store Store) (*Service, *fakeUsers, *fakeChannels) { + return dmServiceCredentials(t, store, nil) +} + +func dmServiceCredentials(t *testing.T, store Store, credentials []string) (*Service, *fakeUsers, *fakeChannels) { + t.Helper() + u := &fakeUsers{current: mattermost.User{ID: "user-1", Username: "arda"}, peer: mattermost.User{ID: "peer", Username: "hakan"}} + c := &fakeChannels{direct: mattermost.Channel{ID: "dm-1", Type: "D", Name: "user-1__peer"}, found: true} + s, err := New("https://Mattermost.Example/chat/", "", credentials, u, c, emptyTeams{}, store) + if err != nil { + t.Fatal(err) + } + return s, u, c +} + +func dmTarget() Target { return Target{Conversation: Direct, Selector: ByUsername, Value: "hakan"} } + +func TestCreatePostPersistsOneCanonicalExactStage(t *testing.T) { + store := &recordingStore{} + s, _, _ := dmService(t, store) + body := []byte("hello\n") + result, err := s.CreatePost(context.Background(), CreatePostInput{RequestID: "req-1", Target: dmTarget(), Body: bytes.NewReader(body)}) + if err != nil { + t.Fatal(err) + } + if store.calls != 1 { + t.Fatalf("Create calls = %d", store.calls) + } + if !bytes.Equal(store.in.Content.Body, body) { + t.Fatalf("body = %q", store.in.Content.Body) + } + if store.in.ServerURL != "https://mattermost.example/chat/api/v4" || store.in.ServerID != "" || store.in.UserID != "user-1" { + t.Fatalf("binding = %#v", store.in) + } + const exactDestination = `{"kind":"conversation","channelId":"dm-1","channelType":"dm","teamId":null,"postId":null,"rootPostId":null,"participantIds":["peer"],"emoji":null}` + if string(store.in.Content.Destination) != exactDestination { + t.Fatalf("destination = %s", store.in.Content.Destination) + } + if result.Preview.Destination.ChannelID != "dm-1" { + t.Fatalf("preview = %#v", result.Preview) + } +} + +func TestAttachmentPlanIsExplicitAndOrdered(t *testing.T) { + store := &recordingStore{} + s, _, _ := dmService(t, store) + digest := [32]byte{1} + s = s.WithAttachmentBinder(func(context.Context, []Attachment, [][]byte) ([]stagestore.Attachment, error) { + return []stagestore.Attachment{ + {SuppliedPath: "a", CanonicalPath: "/a", RemoteFilename: "a", ByteLength: 1, ContentDigest: digest}, + {SuppliedPath: "b", CanonicalPath: "/b", RemoteFilename: "b", ByteLength: 1, ContentDigest: digest}, + }, nil + }) + result, err := s.CreatePost(context.Background(), CreatePostInput{RequestID: "request-plan", Target: dmTarget(), Body: bytes.NewReader([]byte("hello")), Attachments: []Attachment{{Path: "a"}, {Path: "b"}}}) + if err != nil { + t.Fatal(err) + } + const exactPlan = `{"steps":[{"ordinal":1,"type":"upload_attachment","condition":"always"},{"ordinal":2,"type":"upload_attachment","condition":"always"},{"ordinal":3,"type":"create_post","condition":"always"}]}` + if string(store.in.Content.Plan) != exactPlan { + t.Fatalf("plan = %s", store.in.Content.Plan) + } + got, _ := json.Marshal(result.Preview.Plan) + if string(got) != exactPlan { + t.Fatalf("preview plan = %s", got) + } +} + +func TestCredentialTargetRejectedBeforeRemoteAndSnapshotIsImmutable(t *testing.T) { + credentials := []string{"target-secret"} + store := &recordingStore{} + s, users, _ := dmServiceCredentials(t, store, credentials) + credentials[0] = "changed-after-construction" + _, err := s.DryRunCreatePost(context.Background(), DryRunInput{Target{Conversation: Direct, Selector: ByUsername, Value: "target-secret"}}) + if !errors.Is(err, ErrCredential) || users.currentCalls.Load() != 0 { + t.Fatalf("error/current calls = %v/%d", err, users.currentCalls.Load()) + } + _, err = s.CreatePost(context.Background(), CreatePostInput{RequestID: "request", Target: dmTarget(), Body: bytes.NewReader([]byte("target-secret"))}) + if !errors.Is(err, ErrCredential) || store.calls != 0 { + t.Fatalf("immutable credential error/calls = %v/%d", err, store.calls) + } +} + +func TestMalformedTargetSyntaxIsZeroNetwork(t *testing.T) { + for _, target := range []Target{ + {Conversation: Direct, Selector: ByID, Value: "peer"}, + {Conversation: Group, Selector: ByID, Value: "group", Team: &TeamSelector{By: ByID, Value: "team"}}, + {Conversation: Channel, Selector: ByName, Value: "channel"}, + {Conversation: Channel, Selector: ByName, Value: "channel", Team: &TeamSelector{By: ByUsername, Value: "team"}}, + {Conversation: Channel, Selector: ByID, Value: "bad internal space"}, + {Conversation: Channel, Selector: ByID, Value: "bad\u061c"}, + {Conversation: Channel, Selector: ByID, Value: "bad\u200e"}, + {Conversation: Channel, Selector: ByID, Value: "bad\u200f"}, + {Conversation: Channel, Selector: ByID, Value: "bad\u200b"}, + {Conversation: Channel, Selector: ByID, Value: "bad\u200c"}, + {Conversation: Channel, Selector: ByID, Value: "bad\u200d"}, + {Conversation: Channel, Selector: ByID, Value: "bad\ufeff"}, + {Conversation: Channel, Selector: ByID, Value: "bad\u00a0space"}, + } { + store := &recordingStore{} + s, users, _ := dmService(t, store) + _, err := s.DryRunCreatePost(context.Background(), DryRunInput{Target: target}) + if !errors.Is(err, ErrInvalid) || users.currentCalls.Load() != 0 || store.calls != 0 { + t.Fatalf("target/error/remote/store calls = %#v/%v/%d/%d", target, err, users.currentCalls.Load(), store.calls) + } + } +} + +func TestConstructorAndResolvedIdentityValidation(t *testing.T) { + users := &fakeUsers{current: mattermost.User{ID: "user-1", Username: "arda"}} + channels := &fakeChannels{} + if _, err := New("https://mattermost.example", " bad ", nil, users, channels, emptyTeams{}, nil); !errors.Is(err, ErrInvalid) { + t.Fatalf("server ID error = %v", err) + } + for _, unsafe := range []string{"\u200b", "\u200c", "\u200d", "\ufeff"} { + store := &recordingStore{} + s, resolvedUsers, _ := dmService(t, store) + resolvedUsers.current.ID = "bad" + unsafe + "identity" + reader := &panicReader{} + _, err := s.CreatePost(context.Background(), CreatePostInput{RequestID: "request", Target: dmTarget(), Body: reader}) + if !errors.Is(err, ErrTarget) || reader.read || store.calls != 0 { + t.Fatalf("rune/error/read/calls = %q/%v/%v/%d", unsafe, err, reader.read, store.calls) + } + } +} + +func TestBoundAttachmentRejectsAdditionalBidiControls(t *testing.T) { + for _, unsafe := range []string{"\u061c", "\u200e", "\u200f"} { + store := &recordingStore{} + s, _, _ := dmService(t, store) + digest := [32]byte{1} + s = s.WithAttachmentBinder(func(context.Context, []Attachment, [][]byte) ([]stagestore.Attachment, error) { + return []stagestore.Attachment{{SuppliedPath: "safe", CanonicalPath: "/safe", RemoteFilename: "bad" + unsafe, ByteLength: 1, ContentDigest: digest}}, nil + }) + _, err := s.CreatePost(context.Background(), CreatePostInput{RequestID: "request", Target: dmTarget(), Body: bytes.NewReader([]byte("hello")), Attachments: []Attachment{{Path: "safe"}}}) + if !errors.Is(err, ErrInput) || store.calls != 0 { + t.Fatalf("rune/error/calls = %q/%v/%d", unsafe, err, store.calls) + } + } +} + +func TestCreateRequiresRequestIDAndMapsConflict(t *testing.T) { + store := &recordingStore{} + s, _, _ := dmService(t, store) + _, err := s.CreatePost(context.Background(), CreatePostInput{Target: dmTarget(), Body: bytes.NewReader([]byte("hello"))}) + if !errors.Is(err, ErrInvalid) || store.calls != 0 { + t.Fatalf("empty request error/calls = %v/%d", err, store.calls) + } + store.err = stagestore.ErrConflict + _, err = s.CreatePost(context.Background(), CreatePostInput{RequestID: "request", Target: dmTarget(), Body: bytes.NewReader([]byte("hello"))}) + if !errors.Is(err, ErrConflict) || errors.Is(err, ErrStore) { + t.Fatalf("conflict error = %v", err) + } +} + +func TestRealStoreRequestReplayAndConflict(t *testing.T) { + dir, err := os.MkdirTemp(".", ".staging-store-test-") + if err != nil { + t.Fatal(err) + } + dir, err = filepath.Abs(dir) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + store, err := stagestore.Open(context.Background(), filepath.Join(dir, stagestore.DatabaseFilename)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + s, _, _ := dmService(t, store) + input := func(body string) CreatePostInput { + return CreatePostInput{RequestID: "same-request", Target: dmTarget(), Body: bytes.NewReader([]byte(body))} + } + first, err := s.CreatePost(context.Background(), input("hello")) + if err != nil || first.Stored.Replay { + t.Fatalf("first/error = %#v/%v", first.Stored, err) + } + replay, err := s.CreatePost(context.Background(), input("hello")) + if err != nil || !replay.Stored.Replay || replay.Stored.Stage.ID != first.Stored.Stage.ID { + t.Fatalf("replay/error = %#v/%v", replay.Stored, err) + } + if _, err = s.CreatePost(context.Background(), input("different")); !errors.Is(err, ErrConflict) { + t.Fatalf("conflict = %v", err) + } +} + +type panicReader struct{ read bool } + +func (r *panicReader) Read([]byte) (int, error) { r.read = true; return 0, errors.New("forbidden") } + +func TestDryRunStopsAfterResolution(t *testing.T) { + store := &recordingStore{} + s, _, _ := dmService(t, store) + binderCalls := 0 + s = s.WithAttachmentBinder(func(context.Context, []Attachment, [][]byte) ([]stagestore.Attachment, error) { + binderCalls++ + return nil, nil + }) + preview, err := s.DryRunCreatePost(context.Background(), DryRunInput{Target: dmTarget()}) + if err != nil || preview.Destination.ChannelID != "dm-1" { + t.Fatalf("preview/error = %#v/%v", preview, err) + } + if store.calls != 0 || binderCalls != 0 { + t.Fatalf("store/binder calls = %d/%d", store.calls, binderCalls) + } +} + +func TestDryRunAndPersistResolveIdenticalDestination(t *testing.T) { + store := &recordingStore{} + s, _, _ := dmService(t, store) + dry, err := s.DryRunCreatePost(context.Background(), DryRunInput{Target: dmTarget()}) + if err != nil { + t.Fatal(err) + } + persisted, err := s.CreatePost(context.Background(), CreatePostInput{RequestID: "parity-request", Target: dmTarget(), Body: bytes.NewReader([]byte("hello"))}) + if err != nil { + t.Fatal(err) + } + dryDestination, _ := json.Marshal(dry.Destination) + persistedDestination, _ := json.Marshal(persisted.Preview.Destination) + if !bytes.Equal(dryDestination, persistedDestination) { + t.Fatalf("dry/persist destinations = %s/%s", dryDestination, persistedDestination) + } +} + +func TestDMResolutionFailsClosed(t *testing.T) { + for _, test := range []struct { + name string + alter func(*fakeUsers, *fakeChannels) + }{ + {"none", func(_ *fakeUsers, c *fakeChannels) { c.found = false }}, + {"duplicate", func(_ *fakeUsers, c *fakeChannels) { c.err = mattermost.ErrInvalidChannelsResponse }}, + {"self", func(u *fakeUsers, _ *fakeChannels) { u.peer.ID = u.current.ID }}, + } { + t.Run(test.name, func(t *testing.T) { + store := &recordingStore{} + s, u, c := dmService(t, store) + test.alter(u, c) + _, err := s.DryRunCreatePost(context.Background(), DryRunInput{dmTarget()}) + if !errors.Is(err, ErrTarget) || store.calls != 0 { + t.Fatalf("err/calls = %v/%d", err, store.calls) + } + }) + } +} + +func TestChannelAndGroupRequireTypeAndMembership(t *testing.T) { + store := &recordingStore{} + s, _, channels := dmService(t, store) + channels.byID = mattermost.Channel{ID: "g", Type: "O", TeamID: "t", Name: "x"} + _, err := s.DryRunCreatePost(context.Background(), DryRunInput{Target{Conversation: Group, Selector: ByID, Value: "g"}}) + if !errors.Is(err, ErrTarget) { + t.Fatalf("wrong group type error = %v", err) + } + channels.byID = mattermost.Channel{ID: "c", Type: "P", TeamID: "t", Name: "x"} + channels.memberErr = errors.New("not a member") + _, err = s.DryRunCreatePost(context.Background(), DryRunInput{Target{Conversation: Channel, Selector: ByID, Value: "c"}}) + if !errors.Is(err, ErrTarget) { + t.Fatalf("membership error = %v", err) + } +} + +func TestNameChannelRequiresUniqueExactTeam(t *testing.T) { + store := &recordingStore{} + s, _, channels := dmService(t, store) + channels.byName = mattermost.Channel{ID: "c", Type: "O", TeamID: "t1", Name: "town-square"} + s.teams = mattermost.NewTeams(teamTransport{payload: `[{"id":"t1","name":"alpha","display_name":"Alpha","type":"O"}]`}) + if membership, listErr := s.teams.List(context.Background(), "user-1"); listErr != nil || len(membership.Items()) != 1 { + t.Fatalf("team fixture = %#v/%v", membership.Items(), listErr) + } + preview, err := s.DryRunCreatePost(context.Background(), DryRunInput{Target{Conversation: Channel, Selector: ByName, Value: "town-square", Team: &TeamSelector{By: ByName, Value: "alpha"}}}) + if err != nil || preview.Destination.TeamID == nil || *preview.Destination.TeamID != "t1" { + t.Fatalf("preview/error = %#v/%v", preview, err) + } + s.teams = mattermost.NewTeams(teamTransport{payload: `[{"id":"t1","name":"alpha","display_name":"Same","type":"O"},{"id":"t2","name":"beta","display_name":"Same","type":"O"}]`}) + _, err = s.DryRunCreatePost(context.Background(), DryRunInput{Target{Conversation: Channel, Selector: ByName, Value: "town-square", Team: &TeamSelector{By: ByName, Value: "Same"}}}) + if !errors.Is(err, ErrTarget) { + t.Fatalf("ambiguous team error = %v", err) + } +} + +func TestBodyByteLimitFailsBeforePersistence(t *testing.T) { + store := &recordingStore{} + s, _, _ := dmService(t, store) + _, err := s.CreatePost(context.Background(), CreatePostInput{RequestID: "r", Target: dmTarget(), Body: bytes.NewReader(bytes.Repeat([]byte("x"), 65_536))}) + if !errors.Is(err, ErrInput) || store.calls != 0 { + t.Fatalf("error/calls = %v/%d", err, store.calls) + } +} + +func TestBodyValidationAndCredentialFailuresNeverPersist(t *testing.T) { + const token = "active-secret-credential" + for _, test := range []struct { + name string + request string + body []byte + target Target + }{ + {"empty", "r", []byte(" \n"), dmTarget()}, + {"invalid utf8", "r", []byte{0xff}, dmTarget()}, + {"token body", "r", []byte("hello " + token), dmTarget()}, + {"token request", "r-" + token, []byte("hello"), dmTarget()}, + {"token target", "r", []byte("hello"), Target{Conversation: Direct, Selector: ByUsername, Value: token}}, + } { + t.Run(test.name, func(t *testing.T) { + store := &recordingStore{} + s, _, _ := dmServiceCredentials(t, store, []string{token}) + _, err := s.CreatePost(context.Background(), CreatePostInput{RequestID: test.request, Target: test.target, Body: bytes.NewReader(test.body)}) + if err == nil || store.calls != 0 || bytes.Contains([]byte(err.Error()), []byte(token)) { + t.Fatalf("err/calls = %v/%d", err, store.calls) + } + }) + } +} + +func TestAttachmentCredentialAcrossScanBoundaryNeverPersists(t *testing.T) { + const token = "boundary-credential" + dir, err := os.MkdirTemp(".", ".staging-test-") + if err != nil { + t.Fatal(err) + } + dir, err = filepath.Abs(dir) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + path := filepath.Join(dir, "safe.txt") + data := append(bytes.Repeat([]byte("x"), 32*1024-len(token)/2), []byte(token)...) + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + store := &recordingStore{} + s, _, _ := dmServiceCredentials(t, store, []string{token}) + _, err = s.CreatePost(context.Background(), CreatePostInput{RequestID: "r", Target: dmTarget(), Body: bytes.NewReader([]byte("hello")), Attachments: []Attachment{{Path: path}}}) + if !errors.Is(err, ErrCredential) || store.calls != 0 { + t.Fatalf("err/calls = %v/%d", err, store.calls) + } +} + +func TestStoreCalledExactlyOnceUnderConcurrentRequests(t *testing.T) { + store := &recordingStore{} + s, _, _ := dmService(t, store) + const n = 12 + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + _, _ = s.CreatePost(context.Background(), CreatePostInput{RequestID: fmt.Sprintf("request-%d", i), Target: dmTarget(), Body: bytes.NewReader([]byte("hello"))}) + }(i) + } + wg.Wait() + if store.calls != n { + t.Fatalf("Create calls = %d", store.calls) + } +} + +func FuzzDryRunInvalidTargetsNeverPersist(f *testing.F) { + f.Add("", uint8(0), uint8(0)) + f.Add(" x ", uint8(1), uint8(1)) + f.Fuzz(func(t *testing.T, value string, conversation, selector uint8) { + store := &recordingStore{} + s, _, _ := dmService(t, store) + _, _ = s.DryRunCreatePost(context.Background(), DryRunInput{Target{Conversation: ConversationType(conversation), Selector: SelectorType(selector), Value: value}}) + if store.calls != 0 { + t.Fatalf("dry-run persisted") + } + }) +} + +var _ io.Reader = (*panicReader)(nil) From cff14d5cb61f62dfb364ab3212572476cc79b024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 01:27:03 +0300 Subject: [PATCH 058/119] docs: lock post mutation bindings --- docs/V2_CONTRACT.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/V2_CONTRACT.md b/docs/V2_CONTRACT.md index 239b40e..30f21ac 100644 --- a/docs/V2_CONTRACT.md +++ b/docs/V2_CONTRACT.md @@ -234,6 +234,8 @@ Stage creation is online when remote identity or target resolution is required. Revising a stage creates a new revision and marks older revisions superseded. Apply requires `@` and transactionally compare-and-swaps that exact current revision plus its semantic digest into an applying claim. Structured apply requests also carry the expected digest. Human `stage show` prints the exact apply reference. A concurrent revise produces a local conflict rather than applying unreviewed content. Concurrent processes cannot both ordinarily apply one revision. +Only operations with mutable composition may be revised. Top-level posts and replies may replace body and attachments; post edits may replace body only. Delete, react, unreact, and conversation-resolution stages reject revision instead of creating meaningless no-op revisions. + `stage list` and ordinary receipts omit message bodies and attachment contents. `stage show` is an explicit content-revealing operation and may display the staged body. Human output warns accordingly; machine output includes content only for that explicit command. ## 10. Attachment binding @@ -285,6 +287,14 @@ Stage creation captures enough current remote state to make later drift visible: - group creation binds a deduplicated canonical participant-ID set; - channel sends bind channel ID, type, team identity where applicable, and membership/access. +The closed destination binding carries explicit nullable `postState` and `reactionPresent` fields. `postState` is present only for edit/delete and contains the exact author ID, positive Mattermost `update_at` millisecond value, and a lowercase SHA-256 content digest. `reactionPresent` is present only for react/unreact and records whether the authenticated user already has that exact emoji reaction. Every other operation emits these fields as `null`; absence is not used to mean unknown. + +The post content digest is SHA-256 over canonical UTF-8 JSON with the fixed field order `message`, `fileIds`, `rootId`, `type`. File IDs preserve the server's validated order because attachment order is user-visible. The timestamp remains separately bound so a change that returns to identical visible content is still drift. Arbitrary props, presentation metadata, and reactions are excluded from this digest. + +Reply staging accepts a live accessible root or reply. It binds the selected post ID and the canonical root ID; targeting a reply requires a fresh root read proving a live root with an empty `root_id` in the same channel. Reply and reaction staging may target any otherwise valid live accessible post. Edit and delete additionally require an ordinary user post (`type` empty) authored by the authenticated user. An edit whose desired body already equals the freshly revalidated current body completes as already satisfied without a write. A target deleted after staging is drift, not already satisfied. + +Reaction state comes from the authoritative fresh post-reactions endpoint, not cached or embedded post metadata. The response must be complete, exact-post-bound, duplicate-free, and contain only canonical user/post/emoji identities. Both present and absent states are therefore positive facts. React and unreact use conditional plans and complete without a write when revalidation shows the desired state already holds. + Apply re-fetches relevant state. Changed, deleted, re-authored, moved, inaccessible, or ambiguously resolved targets fail closed. Already-satisfied reaction state succeeds without a write and is reported as such. ## 12. Compound operation semantics From 0896b3a1459976bb2573613c7793813f5bcc135d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 01:30:42 +0300 Subject: [PATCH 059/119] refactor: split staging service boundaries --- internal/staging/conversation.go | 152 +++++++++++ internal/staging/service.go | 417 +------------------------------ internal/staging/types.go | 107 ++++++++ internal/staging/validation.go | 158 ++++++++++++ 4 files changed, 422 insertions(+), 412 deletions(-) create mode 100644 internal/staging/conversation.go create mode 100644 internal/staging/types.go create mode 100644 internal/staging/validation.go diff --git a/internal/staging/conversation.go b/internal/staging/conversation.go new file mode 100644 index 0000000..8c23e45 --- /dev/null +++ b/internal/staging/conversation.go @@ -0,0 +1,152 @@ +package staging + +import ( + "context" + "strings" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" +) + +func validTargetSyntax(target Target) bool { + if !validSelectorValue(target.Value) || target.Selector == ByName && strings.HasPrefix(target.Value, "#") { + return false + } + switch target.Conversation { + case Direct: + return target.Selector == ByUsername && target.Team == nil + case Group: + return target.Selector == ByID && target.Team == nil + case Channel: + if target.Selector == ByID { + return target.Team == nil + } + return target.Selector == ByName && target.Team != nil && + (target.Team.By == ByID || target.Team.By == ByName) && validSelectorValue(target.Team.Value) + default: + return false + } +} + +func (s *Service) resolveConversation(ctx context.Context, target Target) (Preview, error) { + if ctx == nil || !validTargetSyntax(target) { + return Preview{}, ErrInvalid + } + if contaminated(s.credentials, targetStrings(target)...) { + return Preview{}, ErrCredential + } + current, err := s.users.Current(ctx) + if err != nil || !validResolvedUser(current) { + return Preview{}, ErrTarget + } + if contaminated(s.credentials, current.ID, current.Username) { + return Preview{}, ErrCredential + } + channel, participants, err := s.resolveChannel(ctx, current, target) + if err != nil { + return Preview{}, err + } + var teamID *string + if channel.Type == "O" || channel.Type == "P" { + value := channel.TeamID + teamID = &value + } + preview := Preview{s.serverURL, s.serverID, current.ID, Destination{"conversation", channel.ID, channelType(channel.Type), teamID, nil, nil, participants, nil}, attachmentPlan(0)} + destination, plan, err := marshalSemantics(preview) + if err != nil { + return Preview{}, ErrInvalid + } + fields := append(targetStrings(target), preview.ServerURL, preview.ServerID, preview.UserID, string(destination), string(plan)) + if contaminated(s.credentials, fields...) { + return Preview{}, ErrCredential + } + return preview, nil +} + +func (s *Service) resolveChannel(ctx context.Context, current mattermost.User, target Target) (mattermost.Channel, []string, error) { + switch target.Conversation { + case Direct: + peer, err := s.users.ByUsernameFresh(ctx, target.Value) + if err != nil || !validResolvedUser(peer) || !strings.EqualFold(peer.Username, target.Value) || peer.ID == current.ID { + return mattermost.Channel{}, nil, ErrTarget + } + if contaminated(s.credentials, peer.ID, peer.Username) { + return mattermost.Channel{}, nil, ErrCredential + } + channel, found, err := s.channels.ExistingDirect(ctx, current.ID, peer.ID) + if err != nil || !found || !validResolvedChannel(channel) || channel.Type != "D" { + return mattermost.Channel{}, nil, ErrTarget + } + if contaminated(s.credentials, channel.ID, channel.Name) { + return mattermost.Channel{}, nil, ErrCredential + } + return channel, []string{peer.ID}, nil + case Group: + channel, err := s.channels.ByID(ctx, target.Value) + if err != nil || !validResolvedChannel(channel) || channel.Type != "G" { + return mattermost.Channel{}, nil, ErrTarget + } + if contaminated(s.credentials, channel.ID, channel.Name) { + return mattermost.Channel{}, nil, ErrCredential + } + if _, err = s.channels.Member(ctx, channel.ID, current.ID); err != nil { + return mattermost.Channel{}, nil, ErrTarget + } + return channel, []string{}, nil + case Channel: + var channel mattermost.Channel + var err error + if target.Selector == ByID { + channel, err = s.channels.ByID(ctx, target.Value) + } else { + team, teamErr := s.resolveTeam(ctx, current.ID, *target.Team) + if teamErr != nil { + return mattermost.Channel{}, nil, teamErr + } + channel, err = s.channels.ByName(ctx, team.ID, target.Value) + } + if err != nil || !validResolvedChannel(channel) || (channel.Type != "O" && channel.Type != "P") { + return mattermost.Channel{}, nil, ErrTarget + } + if contaminated(s.credentials, channel.ID, channel.Name, channel.TeamID) { + return mattermost.Channel{}, nil, ErrCredential + } + if _, err = s.channels.Member(ctx, channel.ID, current.ID); err != nil { + return mattermost.Channel{}, nil, ErrTarget + } + return channel, []string{}, nil + default: + return mattermost.Channel{}, nil, ErrInvalid + } +} + +func (s *Service) resolveTeam(ctx context.Context, userID string, selector TeamSelector) (mattermost.Team, error) { + membership, err := s.teams.List(ctx, userID) + if err != nil { + return mattermost.Team{}, ErrTarget + } + var match mattermost.Team + count := 0 + for _, team := range membership.Items() { + if !validResolvedTeam(team) { + return mattermost.Team{}, ErrTarget + } + matched := selector.By == ByID && team.ID == selector.Value + if selector.By == ByName { + matched = team.Name == selector.Value || team.DisplayName == selector.Value + } + if matched { + match, count = team, count+1 + } + } + if count != 1 { + return mattermost.Team{}, ErrTarget + } + if contaminated(s.credentials, match.ID, match.Name, match.DisplayName) { + return mattermost.Team{}, ErrCredential + } + return match, nil +} + +func channelType(value string) string { + return map[string]string{"D": "dm", "G": "group", "O": "public", "P": "private"}[value] +} diff --git a/internal/staging/service.go b/internal/staging/service.go index 1f4173d..c8b014a 100644 --- a/internal/staging/service.go +++ b/internal/staging/service.go @@ -3,16 +3,10 @@ package staging import ( - "bytes" "context" "encoding/json" "errors" - "io" - "strings" - "unicode" - "unicode/utf8" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" "github.com/ardasevinc/mattermost-cli/internal/messageinput" "github.com/ardasevinc/mattermost-cli/internal/serverurl" "github.com/ardasevinc/mattermost-cli/internal/stageinput" @@ -28,98 +22,6 @@ var ( ErrConflict = errors.New("staging: request conflict") ) -type ConversationType uint8 - -const ( - Direct ConversationType = iota + 1 - Group - Channel -) - -type SelectorType uint8 - -const ( - ByUsername SelectorType = iota + 1 - ByID - ByName -) - -// Target is deliberately syntactic. Resolved IDs and serialized plans are not -// caller-settable. -type Target struct { - Conversation ConversationType - Selector SelectorType - Value string - Team *TeamSelector -} - -type TeamSelector struct { - By SelectorType // ByID or ByName - Value string -} - -type Attachment = stageinput.Attachment - -type CreatePostInput struct { - RequestID string - Target Target - Body io.Reader - Attachments []Attachment -} - -type DryRunInput struct{ Target Target } - -type Destination struct { - Kind string `json:"kind"` - ChannelID string `json:"channelId"` - ChannelType string `json:"channelType"` - TeamID *string `json:"teamId"` - PostID *string `json:"postId"` - RootPostID *string `json:"rootPostId"` - ParticipantIDs []string `json:"participantIds"` - Emoji *string `json:"emoji"` -} - -type Plan struct { - Steps []PlanStep `json:"steps"` -} -type PlanStep struct { - Ordinal int `json:"ordinal"` - Type string `json:"type"` - Condition string `json:"condition"` -} - -type Preview struct { - ServerURL string - ServerID string - UserID string - Destination Destination - Plan Plan -} - -type CreatePostResult struct { - Preview Preview - Stored stagestore.MutationResult -} - -type Users interface { - Current(context.Context) (mattermost.User, error) - ByUsernameFresh(context.Context, string) (mattermost.User, error) -} -type Channels interface { - ExistingDirect(context.Context, string, string) (mattermost.Channel, bool, error) - ByID(context.Context, string) (mattermost.Channel, error) - ByName(context.Context, string, string) (mattermost.Channel, error) - Member(context.Context, string, string) (mattermost.ChannelMember, error) -} -type Teams interface { - List(context.Context, string) (mattermost.TeamMembership, error) -} -type Store interface { - Create(context.Context, stagestore.CreateInput) (stagestore.MutationResult, error) -} -type AttachmentBinder func(context.Context, []stageinput.Attachment, [][]byte) ([]stagestore.Attachment, error) - type Service struct { serverURL, serverID string users Users @@ -132,7 +34,7 @@ type Service struct { func New(serverBaseURL, serverID string, credentials []string, users Users, channels Channels, teams Teams, store Store) (*Service, error) { normalized, err := serverurl.Normalize(serverBaseURL) - if err != nil || users == nil || channels == nil || teams == nil || (serverID != "" && !validIdentity(serverID)) { + if err != nil || users == nil || channels == nil || teams == nil || (serverID != "" && !validSelectorValue(serverID)) { return nil, ErrInvalid } protected := credentialBytes(credentials) @@ -161,21 +63,18 @@ func (s *Service) WithAttachmentBinder(bind AttachmentBinder) *Service { } func (s *Service) DryRunCreatePost(ctx context.Context, in DryRunInput) (Preview, error) { - return s.resolve(ctx, in.Target) + return s.resolveConversation(ctx, in.Target) } func (s *Service) CreatePost(ctx context.Context, in CreatePostInput) (CreatePostResult, error) { - if s.store == nil || s.bind == nil || in.Body == nil { - return CreatePostResult{}, ErrInvalid - } - if in.RequestID == "" || !validRequestID(in.RequestID) { + if s.store == nil || s.bind == nil || in.Body == nil || !validRequestID(in.RequestID) { return CreatePostResult{}, ErrInvalid } callerFields := append([]string{in.RequestID}, targetStrings(in.Target)...) if contaminated(s.credentials, callerFields...) { return CreatePostResult{}, ErrCredential } - preview, err := s.resolve(ctx, in.Target) + preview, err := s.resolveConversation(ctx, in.Target) if err != nil { return CreatePostResult{}, err } @@ -216,143 +115,6 @@ func (s *Service) CreatePost(ctx context.Context, in CreatePostInput) (CreatePos return CreatePostResult{Preview: preview, Stored: stored}, nil } -func (s *Service) resolve(ctx context.Context, target Target) (Preview, error) { - if ctx == nil || !validTargetSyntax(target) { - return Preview{}, ErrInvalid - } - if contaminated(s.credentials, targetStrings(target)...) { - return Preview{}, ErrCredential - } - current, err := s.users.Current(ctx) - if err != nil || !validResolvedUser(current) { - return Preview{}, ErrTarget - } - if contaminated(s.credentials, current.ID, current.Username) { - return Preview{}, ErrCredential - } - channel, participants, err := s.resolveChannel(ctx, current, target) - if err != nil { - return Preview{}, err - } - var teamID *string - if channel.Type == "O" || channel.Type == "P" { - value := channel.TeamID - teamID = &value - } - preview := Preview{s.serverURL, s.serverID, current.ID, Destination{"conversation", channel.ID, channelType(channel.Type), teamID, nil, nil, participants, nil}, createPostPlan()} - destination, plan, err := marshalSemantics(preview) - if err != nil { - return Preview{}, ErrInvalid - } - fields := append(targetStrings(target), preview.ServerURL, preview.ServerID, preview.UserID, string(destination), string(plan)) - if contaminated(s.credentials, fields...) { - return Preview{}, ErrCredential - } - return preview, nil -} - -func (s *Service) resolveChannel(ctx context.Context, current mattermost.User, target Target) (mattermost.Channel, []string, error) { - switch target.Conversation { - case Direct: - if target.Selector != ByUsername || target.Team != nil { - return mattermost.Channel{}, nil, ErrInvalid - } - peer, err := s.users.ByUsernameFresh(ctx, target.Value) - if err != nil || !validResolvedUser(peer) || !strings.EqualFold(peer.Username, target.Value) || peer.ID == current.ID { - return mattermost.Channel{}, nil, ErrTarget - } - if contaminated(s.credentials, peer.ID, peer.Username) { - return mattermost.Channel{}, nil, ErrCredential - } - channel, found, err := s.channels.ExistingDirect(ctx, current.ID, peer.ID) - if err != nil || !found || !validResolvedChannel(channel) || channel.Type != "D" { - return mattermost.Channel{}, nil, ErrTarget - } - if contaminated(s.credentials, channel.ID, channel.Name) { - return mattermost.Channel{}, nil, ErrCredential - } - return channel, []string{peer.ID}, nil - case Group: - if target.Selector != ByID || target.Team != nil { - return mattermost.Channel{}, nil, ErrInvalid - } - channel, err := s.channels.ByID(ctx, target.Value) - if err != nil || !validResolvedChannel(channel) || channel.Type != "G" { - return mattermost.Channel{}, nil, ErrTarget - } - if contaminated(s.credentials, channel.ID, channel.Name) { - return mattermost.Channel{}, nil, ErrCredential - } - if _, err = s.channels.Member(ctx, channel.ID, current.ID); err != nil { - return mattermost.Channel{}, nil, ErrTarget - } - return channel, []string{}, nil - case Channel: - var channel mattermost.Channel - var err error - if target.Selector == ByID && target.Team == nil { - channel, err = s.channels.ByID(ctx, target.Value) - } else if target.Selector == ByName && target.Team != nil { - team, teamErr := s.resolveTeam(ctx, current.ID, *target.Team) - if teamErr != nil { - return mattermost.Channel{}, nil, teamErr - } - channel, err = s.channels.ByName(ctx, team.ID, target.Value) - } else { - return mattermost.Channel{}, nil, ErrInvalid - } - if err != nil || !validResolvedChannel(channel) || (channel.Type != "O" && channel.Type != "P") { - return mattermost.Channel{}, nil, ErrTarget - } - if contaminated(s.credentials, channel.ID, channel.Name, channel.TeamID) { - return mattermost.Channel{}, nil, ErrCredential - } - if _, err = s.channels.Member(ctx, channel.ID, current.ID); err != nil { - return mattermost.Channel{}, nil, ErrTarget - } - return channel, []string{}, nil - default: - return mattermost.Channel{}, nil, ErrInvalid - } -} - -func (s *Service) resolveTeam(ctx context.Context, userID string, selector TeamSelector) (mattermost.Team, error) { - if (selector.By != ByID && selector.By != ByName) || !validSelectorValue(selector.Value) { - return mattermost.Team{}, ErrInvalid - } - membership, err := s.teams.List(ctx, userID) - if err != nil { - return mattermost.Team{}, ErrTarget - } - var match mattermost.Team - count := 0 - for _, team := range membership.Items() { - if !validResolvedTeam(team) { - return mattermost.Team{}, ErrTarget - } - matched := selector.By == ByID && team.ID == selector.Value - if selector.By == ByName { - matched = team.Name == selector.Value || team.DisplayName == selector.Value - } - if matched { - match, count = team, count+1 - } - } - if count != 1 { - return mattermost.Team{}, ErrTarget - } - if contaminated(s.credentials, match.ID, match.Name, match.DisplayName) { - return mattermost.Team{}, ErrCredential - } - return match, nil -} - -func channelType(value string) string { - return map[string]string{"D": "dm", "G": "group", "O": "public", "P": "private"}[value] -} -func createPostPlan() Plan { - return Plan{[]PlanStep{{1, "create_post", "always"}}} -} func attachmentPlan(count int) Plan { steps := make([]PlanStep, 0, count+1) for i := range count { @@ -361,6 +123,7 @@ func attachmentPlan(count int) Plan { steps = append(steps, PlanStep{count + 1, "create_post", "always"}) return Plan{steps} } + func marshalSemantics(preview Preview) ([]byte, []byte, error) { destination, err := json.Marshal(preview.Destination) if err != nil { @@ -369,173 +132,3 @@ func marshalSemantics(preview Preview) ([]byte, []byte, error) { plan, err := json.Marshal(preview.Plan) return destination, plan, err } -func targetStrings(t Target) []string { - v := []string{t.Value} - if t.Team != nil { - v = append(v, t.Team.Value) - } - return v -} -func credentialBytes(values []string) [][]byte { - out := make([][]byte, 0, len(values)) - for _, v := range values { - if v != "" { - out = append(out, []byte(v)) - } - } - return out -} -func cloneCredentials(values [][]byte) [][]byte { - out := make([][]byte, len(values)) - for i := range values { - out[i] = bytes.Clone(values[i]) - } - return out -} -func containsCredential(credentials [][]byte, value []byte) bool { - for _, c := range credentials { - if bytes.Contains(value, c) { - return true - } - } - return false -} -func contaminated(credentials [][]byte, values ...string) bool { - for _, v := range values { - if containsCredential(credentials, []byte(v)) { - return true - } - } - return false -} -func attachmentsContaminated(credentials [][]byte, values []stagestore.Attachment) bool { - for _, v := range values { - if contaminated(credentials, v.SuppliedPath, v.CanonicalPath, v.RemoteFilename, v.MediaType) { - return true - } - } - return false -} - -func validSelectorValue(value string) bool { - if value == "" || len(value) > 256 || !utf8.ValidString(value) || value != strings.TrimSpace(value) { - return false - } - for _, r := range value { - if unsafeIdentityRune(r) || unicode.IsSpace(r) { - return false - } - } - return true -} - -func validIdentity(value string) bool { - return validSelectorValue(value) -} - -func validOptionalText(value string) bool { - return value == "" || validSafeText(value, 256) -} - -func validResolvedUser(user mattermost.User) bool { - return validIdentity(user.ID) && validSelectorValue(user.Username) -} - -func validResolvedChannel(channel mattermost.Channel) bool { - if !validIdentity(channel.ID) || !validSelectorValue(channel.Name) || !validOptionalText(channel.DisplayName) { - return false - } - switch channel.Type { - case "D", "G": - return channel.TeamID == "" - case "O", "P": - return validIdentity(channel.TeamID) - default: - return false - } -} - -func validResolvedTeam(team mattermost.Team) bool { - return validIdentity(team.ID) && validSelectorValue(team.Name) && validOptionalText(team.DisplayName) && (team.Type == "O" || team.Type == "I") -} - -func validBoundAttachments(values []stagestore.Attachment) bool { - if len(values) > 100 { - return false - } - for _, value := range values { - if !validBoundText(value.SuppliedPath, 4096) || !validBoundText(value.CanonicalPath, 4096) || - !validBoundText(value.RemoteFilename, 255) || (value.MediaType != "" && !validBoundText(value.MediaType, 255)) || - value.ByteLength < 0 || value.ContentDigest == ([32]byte{}) { - return false - } - } - return true -} - -func validBoundText(value string, maximum int) bool { - if value == "" || len(value) > maximum || !utf8.ValidString(value) || value != strings.TrimSpace(value) { - return false - } - for _, r := range value { - if unsafeRune(r) { - return false - } - } - return true -} - -func validSafeText(value string, maximum int) bool { - return value != "" && len(value) <= maximum && utf8.ValidString(value) && value == strings.TrimSpace(value) && !strings.ContainsFunc(value, unsafeRune) -} - -func unsafeRune(r rune) bool { - return unicode.IsControl(r) || r == '\u061c' || r == '\u200e' || r == '\u200f' || - r >= '\u202a' && r <= '\u202e' || r >= '\u2066' && r <= '\u2069' -} - -func unsafeIdentityRune(r rune) bool { - return unsafeRune(r) || r >= '\u200b' && r <= '\u200d' || r == '\ufeff' -} - -func validTargetSyntax(target Target) bool { - if !validSelectorValue(target.Value) || target.Selector == ByName && strings.HasPrefix(target.Value, "#") { - return false - } - switch target.Conversation { - case Direct: - return target.Selector == ByUsername && target.Team == nil - case Group: - return target.Selector == ByID && target.Team == nil - case Channel: - if target.Selector == ByID { - return target.Team == nil - } - return target.Selector == ByName && target.Team != nil && - (target.Team.By == ByID || target.Team.By == ByName) && validSelectorValue(target.Team.Value) - default: - return false - } -} - -func validRequestID(value string) bool { - if value == "" { - return true - } - if len(value) > 256 || !requestCharacter(value[0], true) { - return false - } - for i := 1; i < len(value); i++ { - if !requestCharacter(value[i], false) { - return false - } - } - return true -} - -func requestCharacter(value byte, first bool) bool { - if value >= 'A' && value <= 'Z' || value >= 'a' && value <= 'z' || value >= '0' && value <= '9' { - return true - } - return !first && strings.ContainsRune("._~:-", rune(value)) -} diff --git a/internal/staging/types.go b/internal/staging/types.go new file mode 100644 index 0000000..01a9439 --- /dev/null +++ b/internal/staging/types.go @@ -0,0 +1,107 @@ +package staging + +import ( + "context" + "io" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +type ConversationType uint8 + +const ( + Direct ConversationType = iota + 1 + Group + Channel +) + +type SelectorType uint8 + +const ( + ByUsername SelectorType = iota + 1 + ByID + ByName +) + +// Target is deliberately syntactic. Resolved IDs and serialized plans are not +// caller-settable. +type Target struct { + Conversation ConversationType + Selector SelectorType + Value string + Team *TeamSelector +} + +type TeamSelector struct { + By SelectorType // ByID or ByName + Value string +} + +type Attachment = stageinput.Attachment + +type CreatePostInput struct { + RequestID string + Target Target + Body io.Reader + Attachments []Attachment +} + +type DryRunInput struct{ Target Target } + +type Destination struct { + Kind string `json:"kind"` + ChannelID string `json:"channelId"` + ChannelType string `json:"channelType"` + TeamID *string `json:"teamId"` + PostID *string `json:"postId"` + RootPostID *string `json:"rootPostId"` + ParticipantIDs []string `json:"participantIds"` + Emoji *string `json:"emoji"` +} + +type Plan struct { + Steps []PlanStep `json:"steps"` +} + +type PlanStep struct { + Ordinal int `json:"ordinal"` + Type string `json:"type"` + Condition string `json:"condition"` +} + +type Preview struct { + ServerURL string + ServerID string + UserID string + Destination Destination + Plan Plan +} + +type CreatePostResult struct { + Preview Preview + Stored stagestore.MutationResult +} + +type Users interface { + Current(context.Context) (mattermost.User, error) + ByUsernameFresh(context.Context, string) (mattermost.User, error) +} + +type Channels interface { + ExistingDirect(context.Context, string, string) (mattermost.Channel, bool, error) + ByID(context.Context, string) (mattermost.Channel, error) + ByName(context.Context, string, string) (mattermost.Channel, error) + Member(context.Context, string, string) (mattermost.ChannelMember, error) +} + +type Teams interface { + List(context.Context, string) (mattermost.TeamMembership, error) +} + +type Store interface { + Create(context.Context, stagestore.CreateInput) (stagestore.MutationResult, error) +} + +type AttachmentBinder func(context.Context, []stageinput.Attachment, [][]byte) ([]stagestore.Attachment, error) diff --git a/internal/staging/validation.go b/internal/staging/validation.go new file mode 100644 index 0000000..1404f34 --- /dev/null +++ b/internal/staging/validation.go @@ -0,0 +1,158 @@ +package staging + +import ( + "bytes" + "strings" + "unicode" + "unicode/utf8" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +func targetStrings(t Target) []string { + v := []string{t.Value} + if t.Team != nil { + v = append(v, t.Team.Value) + } + return v +} + +func credentialBytes(values []string) [][]byte { + out := make([][]byte, 0, len(values)) + for _, v := range values { + if v != "" { + out = append(out, []byte(v)) + } + } + return out +} + +func cloneCredentials(values [][]byte) [][]byte { + out := make([][]byte, len(values)) + for i := range values { + out[i] = bytes.Clone(values[i]) + } + return out +} + +func containsCredential(credentials [][]byte, value []byte) bool { + for _, credential := range credentials { + if bytes.Contains(value, credential) { + return true + } + } + return false +} + +func contaminated(credentials [][]byte, values ...string) bool { + for _, value := range values { + if containsCredential(credentials, []byte(value)) { + return true + } + } + return false +} + +func attachmentsContaminated(credentials [][]byte, values []stagestore.Attachment) bool { + for _, value := range values { + if contaminated(credentials, value.SuppliedPath, value.CanonicalPath, value.RemoteFilename, value.MediaType) { + return true + } + } + return false +} + +func validSelectorValue(value string) bool { + if value == "" || len(value) > 256 || !utf8.ValidString(value) || value != strings.TrimSpace(value) { + return false + } + for _, r := range value { + if unsafeIdentityRune(r) || unicode.IsSpace(r) { + return false + } + } + return true +} + +func validResolvedUser(user mattermost.User) bool { + return validSelectorValue(user.ID) && validSelectorValue(user.Username) +} + +func validResolvedChannel(channel mattermost.Channel) bool { + if !validSelectorValue(channel.ID) || !validSelectorValue(channel.Name) || + (channel.DisplayName != "" && !validSafeText(channel.DisplayName, 256)) { + return false + } + switch channel.Type { + case "D", "G": + return channel.TeamID == "" + case "O", "P": + return validSelectorValue(channel.TeamID) + default: + return false + } +} + +func validResolvedTeam(team mattermost.Team) bool { + return validSelectorValue(team.ID) && validSelectorValue(team.Name) && + (team.DisplayName == "" || validSafeText(team.DisplayName, 256)) && (team.Type == "O" || team.Type == "I") +} + +func validBoundAttachments(values []stagestore.Attachment) bool { + if len(values) > 100 { + return false + } + for _, value := range values { + if !validBoundText(value.SuppliedPath, 4096) || !validBoundText(value.CanonicalPath, 4096) || + !validBoundText(value.RemoteFilename, 255) || (value.MediaType != "" && !validBoundText(value.MediaType, 255)) || + value.ByteLength < 0 || value.ContentDigest == ([32]byte{}) { + return false + } + } + return true +} + +func validBoundText(value string, maximum int) bool { + if value == "" || len(value) > maximum || !utf8.ValidString(value) || value != strings.TrimSpace(value) { + return false + } + for _, r := range value { + if unsafeRune(r) { + return false + } + } + return true +} + +func validSafeText(value string, maximum int) bool { + return value != "" && len(value) <= maximum && utf8.ValidString(value) && value == strings.TrimSpace(value) && !strings.ContainsFunc(value, unsafeRune) +} + +func unsafeRune(r rune) bool { + return unicode.IsControl(r) || r == '\u061c' || r == '\u200e' || r == '\u200f' || + r >= '\u202a' && r <= '\u202e' || r >= '\u2066' && r <= '\u2069' +} + +func unsafeIdentityRune(r rune) bool { + return unsafeRune(r) || r >= '\u200b' && r <= '\u200d' || r == '\ufeff' +} + +func validRequestID(value string) bool { + if value == "" || len(value) > 256 || !requestCharacter(value[0], true) { + return false + } + for i := 1; i < len(value); i++ { + if !requestCharacter(value[i], false) { + return false + } + } + return true +} + +func requestCharacter(value byte, first bool) bool { + if value >= 'A' && value <= 'Z' || value >= 'a' && value <= 'z' || value >= '0' && value <= '9' { + return true + } + return !first && strings.ContainsRune("._~:-", rune(value)) +} From 4548bfeab933eed245b0b38810dc81f2284ec67d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 01:39:41 +0300 Subject: [PATCH 060/119] feat: bind post mutation state --- internal/schema/stage_test.go | 108 +++++++++++++++++++++---- internal/staging/conversation.go | 11 ++- internal/staging/service_test.go | 42 +++++++++- internal/staging/types.go | 24 ++++-- schemas/v2/examples/stage-preview.json | 2 +- schemas/v2/examples/stage-receipt.json | 2 +- schemas/v2/examples/stage.json | 2 +- schemas/v2/examples/stages.json | 2 +- schemas/v2/stage-preview.schema.json | 82 ++++++++++++++++++- schemas/v2/stage-receipt.schema.json | 82 ++++++++++++++++++- schemas/v2/stage.schema.json | 82 ++++++++++++++++++- schemas/v2/stages.schema.json | 82 ++++++++++++++++++- 12 files changed, 481 insertions(+), 40 deletions(-) diff --git a/internal/schema/stage_test.go b/internal/schema/stage_test.go index a62e172..aae69e7 100644 --- a/internal/schema/stage_test.go +++ b/internal/schema/stage_test.go @@ -39,7 +39,7 @@ func TestStageMachineSchemasRejectContradictionsAndLeaks(t *testing.T) { } digest := strings.Repeat("a", 64) stageID := "stg_0123456789abcdefghijklmnopqrstuv" - stage := `{"stageId":"` + stageID + `","stageRef":"` + stageID + `@1","revision":1,"operation":"create_post","semanticDigest":"` + digest + `","lifecycle":"open","recovery":"none","createdAt":"2026-07-17T10:00:00.000Z","updatedAt":"2026-07-17T10:00:00.000Z","binding":{"serverUrl":"https://mattermost.example/api/v4","serverId":null,"userId":"user-1"},"destination":{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null}}` + stage := `{"stageId":"` + stageID + `","stageRef":"` + stageID + `@1","revision":1,"operation":"create_post","semanticDigest":"` + digest + `","lifecycle":"open","recovery":"none","createdAt":"2026-07-17T10:00:00.000Z","updatedAt":"2026-07-17T10:00:00.000Z","binding":{"serverUrl":"https://mattermost.example/api/v4","serverId":null,"userId":"user-1"},"destination":{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null,"postState":null,"reactionPresent":null}}` cases := map[string][]string{ "mm/v2/stage-request": { stageRequest(false, "r1", "create_post", conversationTarget("dm", "username", "arda", "null"), "null", "null", `[]`), @@ -67,9 +67,9 @@ func TestStageMachineSchemasRejectContradictionsAndLeaks(t *testing.T) { `{"schema":"mm/v2/stage-receipt","action":"created","revived":false,"replayed":false,"recordedAt":"2026-07-17T10:00:00.000Z","stage":` + strings.Replace(stage, `"destination":`, `"plan":[],"destination":`, 1) + `}`, }, "mm/v2/stage-preview": { - preview("create_post", `{"kind":"reaction","channelId":"c","channelType":"public","teamId":"t","postId":"p","rootPostId":null,"participantIds":[],"emoji":"wave"}`, "create_post", true), - preview("create_post", `{"kind":"conversation","channelId":"c","channelType":"public","teamId":"t","postId":null,"rootPostId":null,"participantIds":[],"emoji":null}`, "delete_post", false), - strings.TrimSuffix(preview("create_post", `{"kind":"conversation","channelId":"c","channelType":"public","teamId":"t","postId":null,"rootPostId":null,"participantIds":[],"emoji":null}`, "create_post", false), "}") + `,"body":"secret"}`, + preview("create_post", `{"kind":"reaction","channelId":"c","channelType":"public","teamId":"t","postId":"p","rootPostId":null,"participantIds":[],"emoji":"wave","postState":null,"reactionPresent":true}`, "create_post", true), + preview("create_post", `{"kind":"conversation","channelId":"c","channelType":"public","teamId":"t","postId":null,"rootPostId":null,"participantIds":[],"emoji":null,"postState":null,"reactionPresent":null}`, "delete_post", false), + strings.TrimSuffix(preview("create_post", `{"kind":"conversation","channelId":"c","channelType":"public","teamId":"t","postId":null,"rootPostId":null,"participantIds":[],"emoji":null,"postState":null,"reactionPresent":null}`, "create_post", false), "}") + `,"body":"secret"}`, }, } for id, documents := range cases { @@ -126,22 +126,22 @@ func TestStageSchemasRejectResidualReviewContradictions(t *testing.T) { assertInvalid(t, r, "mm/v2/stage-request", document) } - validDestination := `{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null}` + validDestination := `{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null,"postState":null,"reactionPresent":null}` destinationContradictions := []string{ - `{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":null,"postId":null,"rootPostId":null,"participantIds":[],"emoji":null}`, - `{"kind":"conversation","channelId":"channel-1","channelType":"private","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":["peer"],"emoji":null}`, - `{"kind":"conversation","channelId":"channel-1","channelType":"dm","teamId":null,"postId":null,"rootPostId":null,"participantIds":[],"emoji":null}`, - `{"kind":"conversation","channelId":"channel-1","channelType":"dm","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":["peer"],"emoji":null}`, - `{"kind":"conversation","channelId":"channel-1","channelType":"group","teamId":null,"postId":null,"rootPostId":null,"participantIds":["claimed-complete"],"emoji":null}`, - `{"kind":"conversation","channelId":null,"channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null}`, + `{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":null,"postId":null,"rootPostId":null,"participantIds":[],"emoji":null,"postState":null,"reactionPresent":null}`, + `{"kind":"conversation","channelId":"channel-1","channelType":"private","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":["peer"],"emoji":null,"postState":null,"reactionPresent":null}`, + `{"kind":"conversation","channelId":"channel-1","channelType":"dm","teamId":null,"postId":null,"rootPostId":null,"participantIds":[],"emoji":null,"postState":null,"reactionPresent":null}`, + `{"kind":"conversation","channelId":"channel-1","channelType":"dm","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":["peer"],"emoji":null,"postState":null,"reactionPresent":null}`, + `{"kind":"conversation","channelId":"channel-1","channelType":"group","teamId":null,"postId":null,"rootPostId":null,"participantIds":["claimed-complete"],"emoji":null,"postState":null,"reactionPresent":null}`, + `{"kind":"conversation","channelId":null,"channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null,"postState":null,"reactionPresent":null}`, } for _, destination := range destinationContradictions { assertInvalid(t, r, "mm/v2/stage-preview", strings.Replace(preview, validDestination, destination, 1)) } - legacyDirect := `{"kind":"conversation","channelId":"channel-1","channelType":"direct","teamId":null,"postId":null,"rootPostId":null,"participantIds":["peer"],"emoji":null}` + legacyDirect := `{"kind":"conversation","channelId":"channel-1","channelType":"direct","teamId":null,"postId":null,"rootPostId":null,"participantIds":["peer"],"emoji":null,"postState":null,"reactionPresent":null}` assertInvalid(t, r, "mm/v2/stage-preview", strings.Replace(preview, validDestination, legacyDirect, 1)) - unresolved := strings.Replace(preview, validDestination, `{"kind":"conversation","channelId":null,"channelType":"dm","teamId":null,"postId":null,"rootPostId":null,"participantIds":["peer"],"emoji":null}`, 1) + unresolved := strings.Replace(preview, validDestination, `{"kind":"conversation","channelId":null,"channelType":"dm","teamId":null,"postId":null,"rootPostId":null,"participantIds":["peer"],"emoji":null,"postState":null,"reactionPresent":null}`, 1) assertInvalid(t, r, "mm/v2/stage-preview", unresolved) compound := strings.Replace(unresolved, `{"ordinal":1,"type":"create_post","condition":"always"}`, `{"ordinal":1,"type":"resolve_conversation","condition":"if_missing"},{"ordinal":2,"type":"upload_attachment","condition":"always"},{"ordinal":3,"type":"create_post","condition":"always"}`, 1) if err := r.Validate("mm/v2/stage-preview", strings.NewReader(compound)); err != nil { @@ -160,7 +160,7 @@ func TestStageSchemasRejectResidualReviewContradictions(t *testing.T) { assertInvalid(t, r, "mm/v2/stage", strings.Replace(applying, `"state":"present","body":"hello"`, `"state":"pruned","body":null`, 1)) nonContent := strings.NewReplacer( `"operation":"create_post"`, `"operation":"delete_post"`, - validDestination, `{"kind":"post","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":"post-1","rootPostId":null,"participantIds":[],"emoji":null}`, + validDestination, `{"kind":"post","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":"post-1","rootPostId":null,"participantIds":[],"emoji":null,"postState":{"authorUserId":"user-1","updateAt":1,"contentDigest":"`+strings.Repeat("a", 64)+`"},"reactionPresent":null}`, `"state":"present","body":"hello"`, `"state":"none","body":null`, `"type":"create_post"`, `"type":"delete_post"`, ).Replace(show) @@ -214,6 +214,86 @@ func TestStageOutputSchemasShareExactDefinitions(t *testing.T) { } } +func TestStageDestinationRemoteStateDiscriminants(t *testing.T) { + r, err := Load() + if err != nil { + t.Fatal(err) + } + digest := strings.Repeat("a", 64) + postState := `{"authorUserId":"author-1","updateAt":1720000000000,"contentDigest":"` + digest + `"}` + conversation := `{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null,"postState":null,"reactionPresent":null}` + reply := `{"kind":"post","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":"post-1","rootPostId":"root-1","participantIds":[],"emoji":null,"postState":null,"reactionPresent":null}` + post := `{"kind":"post","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":"post-1","rootPostId":null,"participantIds":[],"emoji":null,"postState":` + postState + `,"reactionPresent":null}` + reaction := `{"kind":"reaction","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":"post-1","rootPostId":null,"participantIds":[],"emoji":"wave","postState":null,"reactionPresent":true}` + dm := `{"kind":"conversation","channelId":null,"channelType":"dm","teamId":null,"postId":null,"rootPostId":null,"participantIds":["peer-1"],"emoji":null,"postState":null,"reactionPresent":null}` + group := `{"kind":"conversation","channelId":null,"channelType":"group","teamId":null,"postId":null,"rootPostId":null,"participantIds":["peer-1","peer-2"],"emoji":null,"postState":null,"reactionPresent":null}` + + valid := []string{ + preview("create_post", conversation, "create_post", false), + preview("reply", reply, "create_post", false), + preview("edit_post", post, "edit_post", false), + preview("delete_post", post, "delete_post", false), + preview("react", reaction, "add_reaction", false), + preview("unreact", reaction, "remove_reaction", false), + strings.Replace(preview("resolve_dm", dm, "resolve_conversation", false), `"condition":"always"`, `"condition":"if_missing"`, 1), + strings.Replace(preview("resolve_group_dm", group, "resolve_conversation", false), `"condition":"always"`, `"condition":"if_missing"`, 1), + } + for _, document := range valid { + if err := r.Validate("mm/v2/stage-preview", strings.NewReader(document)); err != nil { + t.Fatalf("valid operation destination rejected: %v\n%s", err, document) + } + } + + invalid := []string{ + strings.Replace(valid[0], `,"postState":null`, "", 1), + strings.Replace(valid[0], `,"reactionPresent":null`, "", 1), + strings.Replace(valid[0], `"postState":null`, `"postState":`+postState, 1), + strings.Replace(valid[1], `"reactionPresent":null`, `"reactionPresent":false`, 1), + strings.Replace(valid[2], `"postState":`+postState, `"postState":null`, 1), + strings.Replace(valid[3], `"reactionPresent":null`, `"reactionPresent":false`, 1), + strings.Replace(valid[4], `"reactionPresent":true`, `"reactionPresent":null`, 1), + strings.Replace(valid[5], `"postState":null`, `"postState":`+postState, 1), + strings.Replace(valid[6], `"postState":null`, `"postState":`+postState, 1), + strings.Replace(valid[7], `"reactionPresent":null`, `"reactionPresent":true`, 1), + strings.Replace(valid[2], `"authorUserId":"author-1"`, `"authorUserId":" author-1"`, 1), + strings.Replace(valid[2], `"updateAt":1720000000000`, `"updateAt":0`, 1), + strings.Replace(valid[2], `"updateAt":1720000000000`, `"updateAt":8640000000000001`, 1), + strings.Replace(valid[2], digest, strings.Repeat("A", 64), 1), + strings.Replace(valid[2], digest, strings.Repeat("a", 63), 1), + } + for _, document := range invalid { + assertInvalid(t, r, "mm/v2/stage-preview", document) + } +} + +func TestStageEmojiMatchesMattermostReactionGrammar(t *testing.T) { + r, err := Load() + if err != nil { + t.Fatal(err) + } + valid := []string{strings.Repeat("a", 64), "Wave_ABC-123+OK"} + invalid := []string{strings.Repeat("a", 65), "party.parrot", ":wave:"} + for _, emoji := range valid { + request := stageRequest(true, "reaction-request", "react", `{"kind":"post","postId":"post-1"}`, "null", strconv.Quote(emoji), `[]`) + if err := r.Validate("mm/v2/stage-request", strings.NewReader(request)); err != nil { + t.Fatalf("valid request emoji %q rejected: %v", emoji, err) + } + if err := r.Validate("mm/v2/stage-preview", strings.NewReader(reactionPreview(emoji))); err != nil { + t.Fatalf("valid output emoji %q rejected: %v", emoji, err) + } + } + for _, emoji := range invalid { + request := stageRequest(true, "reaction-request", "react", `{"kind":"post","postId":"post-1"}`, "null", strconv.Quote(emoji), `[]`) + assertInvalid(t, r, "mm/v2/stage-request", request) + assertInvalid(t, r, "mm/v2/stage-preview", reactionPreview(emoji)) + } +} + +func reactionPreview(emoji string) string { + destination := `{"kind":"reaction","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":"post-1","rootPostId":null,"participantIds":[],"emoji":` + strconv.Quote(emoji) + `,"postState":null,"reactionPresent":false}` + return preview("react", destination, "add_reaction", false) +} + func example(t *testing.T, name string) string { t.Helper() data, err := fs.ReadFile(publicschemas.FS, "v2/examples/"+name) diff --git a/internal/staging/conversation.go b/internal/staging/conversation.go index 8c23e45..afa4b3e 100644 --- a/internal/staging/conversation.go +++ b/internal/staging/conversation.go @@ -50,7 +50,16 @@ func (s *Service) resolveConversation(ctx context.Context, target Target) (Previ value := channel.TeamID teamID = &value } - preview := Preview{s.serverURL, s.serverID, current.ID, Destination{"conversation", channel.ID, channelType(channel.Type), teamID, nil, nil, participants, nil}, attachmentPlan(0)} + preview := Preview{ + ServerURL: s.serverURL, + ServerID: s.serverID, + UserID: current.ID, + Destination: Destination{ + Kind: "conversation", ChannelID: channel.ID, ChannelType: channelType(channel.Type), + TeamID: teamID, ParticipantIDs: participants, + }, + Plan: attachmentPlan(0), + } destination, plan, err := marshalSemantics(preview) if err != nil { return Preview{}, ErrInvalid diff --git a/internal/staging/service_test.go b/internal/staging/service_test.go index a289420..19a78a2 100644 --- a/internal/staging/service_test.go +++ b/internal/staging/service_test.go @@ -14,6 +14,7 @@ import ( "testing" "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/schema" "github.com/ardasevinc/mattermost-cli/internal/stagestore" ) @@ -115,7 +116,7 @@ func TestCreatePostPersistsOneCanonicalExactStage(t *testing.T) { if store.in.ServerURL != "https://mattermost.example/chat/api/v4" || store.in.ServerID != "" || store.in.UserID != "user-1" { t.Fatalf("binding = %#v", store.in) } - const exactDestination = `{"kind":"conversation","channelId":"dm-1","channelType":"dm","teamId":null,"postId":null,"rootPostId":null,"participantIds":["peer"],"emoji":null}` + const exactDestination = `{"kind":"conversation","channelId":"dm-1","channelType":"dm","teamId":null,"postId":null,"rootPostId":null,"participantIds":["peer"],"emoji":null,"postState":null,"reactionPresent":null}` if string(store.in.Content.Destination) != exactDestination { t.Fatalf("destination = %s", store.in.Content.Destination) } @@ -306,6 +307,45 @@ func TestDryRunAndPersistResolveIdenticalDestination(t *testing.T) { } } +func TestCreatePostPreviewDestinationAndPlanMatchPublicSchema(t *testing.T) { + store := &recordingStore{} + service, _, _ := dmService(t, store) + result, err := service.CreatePost(context.Background(), CreatePostInput{ + RequestID: "schema-1", Target: dmTarget(), Body: bytes.NewReader([]byte("hello")), + }) + if err != nil { + t.Fatal(err) + } + document := struct { + Schema string `json:"schema"` + Persist bool `json:"persist"` + Operation string `json:"operation"` + Binding any `json:"binding"` + Destination Destination `json:"destination"` + Plan Plan `json:"plan"` + ContentValidated bool `json:"contentValidated"` + }{ + Schema: "mm/v2/stage-preview", Persist: false, Operation: "create_post", + Binding: struct { + ServerURL string `json:"serverUrl"` + ServerID *string `json:"serverId"` + UserID string `json:"userId"` + }{ServerURL: result.Preview.ServerURL, UserID: result.Preview.UserID}, + Destination: result.Preview.Destination, Plan: result.Preview.Plan, + } + encoded, err := json.Marshal(document) + if err != nil { + t.Fatal(err) + } + registry, err := schema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/stage-preview", bytes.NewReader(encoded)); err != nil { + t.Fatalf("producer preview rejected: %v\n%s", err, encoded) + } +} + func TestDMResolutionFailsClosed(t *testing.T) { for _, test := range []struct { name string diff --git a/internal/staging/types.go b/internal/staging/types.go index 01a9439..3081067 100644 --- a/internal/staging/types.go +++ b/internal/staging/types.go @@ -51,14 +51,22 @@ type CreatePostInput struct { type DryRunInput struct{ Target Target } type Destination struct { - Kind string `json:"kind"` - ChannelID string `json:"channelId"` - ChannelType string `json:"channelType"` - TeamID *string `json:"teamId"` - PostID *string `json:"postId"` - RootPostID *string `json:"rootPostId"` - ParticipantIDs []string `json:"participantIds"` - Emoji *string `json:"emoji"` + Kind string `json:"kind"` + ChannelID string `json:"channelId"` + ChannelType string `json:"channelType"` + TeamID *string `json:"teamId"` + PostID *string `json:"postId"` + RootPostID *string `json:"rootPostId"` + ParticipantIDs []string `json:"participantIds"` + Emoji *string `json:"emoji"` + PostState *PostState `json:"postState"` + ReactionPresent *bool `json:"reactionPresent"` +} + +type PostState struct { + AuthorUserID string `json:"authorUserId"` + UpdateAt int64 `json:"updateAt"` + ContentDigest string `json:"contentDigest"` } type Plan struct { diff --git a/schemas/v2/examples/stage-preview.json b/schemas/v2/examples/stage-preview.json index 2cc4d27..9f2bb0b 100644 --- a/schemas/v2/examples/stage-preview.json +++ b/schemas/v2/examples/stage-preview.json @@ -1 +1 @@ -{"schema":"mm/v2/stage-preview","persist":false,"operation":"create_post","binding":{"serverUrl":"https://mattermost.example/api/v4","serverId":null,"userId":"user-1"},"destination":{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null},"plan":{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]},"contentValidated":false} +{"schema":"mm/v2/stage-preview","persist":false,"operation":"create_post","binding":{"serverUrl":"https://mattermost.example/api/v4","serverId":null,"userId":"user-1"},"destination":{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null,"postState":null,"reactionPresent":null},"plan":{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]},"contentValidated":false} diff --git a/schemas/v2/examples/stage-receipt.json b/schemas/v2/examples/stage-receipt.json index c207db9..1cc4f5e 100644 --- a/schemas/v2/examples/stage-receipt.json +++ b/schemas/v2/examples/stage-receipt.json @@ -1 +1 @@ -{"schema":"mm/v2/stage-receipt","action":"created","revived":false,"replayed":false,"recordedAt":"2026-07-17T10:00:00.000Z","stage":{"stageId":"stg_0123456789abcdefghijklmnopqrstuv","stageRef":"stg_0123456789abcdefghijklmnopqrstuv@1","revision":1,"operation":"create_post","semanticDigest":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","lifecycle":"open","recovery":"none","createdAt":"2026-07-17T10:00:00.000Z","updatedAt":"2026-07-17T10:00:00.000Z","binding":{"serverUrl":"https://mattermost.example/api/v4","serverId":null,"userId":"user-1"},"destination":{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null}}} +{"schema":"mm/v2/stage-receipt","action":"created","revived":false,"replayed":false,"recordedAt":"2026-07-17T10:00:00.000Z","stage":{"stageId":"stg_0123456789abcdefghijklmnopqrstuv","stageRef":"stg_0123456789abcdefghijklmnopqrstuv@1","revision":1,"operation":"create_post","semanticDigest":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","lifecycle":"open","recovery":"none","createdAt":"2026-07-17T10:00:00.000Z","updatedAt":"2026-07-17T10:00:00.000Z","binding":{"serverUrl":"https://mattermost.example/api/v4","serverId":null,"userId":"user-1"},"destination":{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null,"postState":null,"reactionPresent":null}}} diff --git a/schemas/v2/examples/stage.json b/schemas/v2/examples/stage.json index 4ee5317..8164992 100644 --- a/schemas/v2/examples/stage.json +++ b/schemas/v2/examples/stage.json @@ -1 +1 @@ -{"schema":"mm/v2/stage","stage":{"stageId":"stg_0123456789abcdefghijklmnopqrstuv","stageRef":"stg_0123456789abcdefghijklmnopqrstuv@1","revision":1,"operation":"create_post","semanticDigest":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","lifecycle":"open","recovery":"none","createdAt":"2026-07-17T10:00:00.000Z","updatedAt":"2026-07-17T10:00:00.000Z","binding":{"serverUrl":"https://mattermost.example/api/v4","serverId":null,"userId":"user-1"},"destination":{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null}},"revisionState":"current","content":{"state":"present","body":"hello"},"attachmentState":"none","attachments":[],"plan":{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}} +{"schema":"mm/v2/stage","stage":{"stageId":"stg_0123456789abcdefghijklmnopqrstuv","stageRef":"stg_0123456789abcdefghijklmnopqrstuv@1","revision":1,"operation":"create_post","semanticDigest":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","lifecycle":"open","recovery":"none","createdAt":"2026-07-17T10:00:00.000Z","updatedAt":"2026-07-17T10:00:00.000Z","binding":{"serverUrl":"https://mattermost.example/api/v4","serverId":null,"userId":"user-1"},"destination":{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null,"postState":null,"reactionPresent":null}},"revisionState":"current","content":{"state":"present","body":"hello"},"attachmentState":"none","attachments":[],"plan":{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}} diff --git a/schemas/v2/examples/stages.json b/schemas/v2/examples/stages.json index 7c06f9c..bf6adaa 100644 --- a/schemas/v2/examples/stages.json +++ b/schemas/v2/examples/stages.json @@ -1 +1 @@ -{"schema":"mm/v2/stages","stages":[{"stageId":"stg_0123456789abcdefghijklmnopqrstuv","stageRef":"stg_0123456789abcdefghijklmnopqrstuv@1","revision":1,"operation":"create_post","semanticDigest":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","lifecycle":"open","recovery":"none","createdAt":"2026-07-17T10:00:00.000Z","updatedAt":"2026-07-17T10:00:00.000Z","binding":{"serverUrl":"https://mattermost.example/api/v4","serverId":null,"userId":"user-1"},"destination":{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null}}],"nextCursor":null} +{"schema":"mm/v2/stages","stages":[{"stageId":"stg_0123456789abcdefghijklmnopqrstuv","stageRef":"stg_0123456789abcdefghijklmnopqrstuv@1","revision":1,"operation":"create_post","semanticDigest":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","lifecycle":"open","recovery":"none","createdAt":"2026-07-17T10:00:00.000Z","updatedAt":"2026-07-17T10:00:00.000Z","binding":{"serverUrl":"https://mattermost.example/api/v4","serverId":null,"userId":"user-1"},"destination":{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null,"postState":null,"reactionPresent":null}}],"nextCursor":null} diff --git a/schemas/v2/stage-preview.schema.json b/schemas/v2/stage-preview.schema.json index c21748a..36e87e6 100644 --- a/schemas/v2/stage-preview.schema.json +++ b/schemas/v2/stage-preview.schema.json @@ -374,7 +374,9 @@ "postId", "rootPostId", "participantIds", - "emoji" + "emoji", + "postState", + "reactionPresent" ], "properties": { "kind": { @@ -422,6 +424,22 @@ "type": "null" } ] + }, + "postState": { + "anyOf": [ + { + "$ref": "#/$defs/postState" + }, + { + "type": "null" + } + ] + }, + "reactionPresent": { + "type": [ + "boolean", + "null" + ] } } }, @@ -440,6 +458,12 @@ }, "emoji": { "type": "null" + }, + "postState": { + "type": "null" + }, + "reactionPresent": { + "type": "null" } } }, @@ -476,6 +500,12 @@ }, "emoji": { "type": "null" + }, + "postState": { + "type": "null" + }, + "reactionPresent": { + "type": "null" } } } @@ -496,6 +526,12 @@ }, "emoji": { "type": "null" + }, + "postState": { + "$ref": "#/$defs/postState" + }, + "reactionPresent": { + "type": "null" } } } @@ -516,6 +552,12 @@ }, "emoji": { "$ref": "#/$defs/emoji" + }, + "postState": { + "type": "null" + }, + "reactionPresent": { + "type": "boolean" } } } @@ -539,6 +581,12 @@ }, "emoji": { "type": "null" + }, + "postState": { + "type": "null" + }, + "reactionPresent": { + "type": "null" } } } @@ -562,6 +610,12 @@ }, "emoji": { "type": "null" + }, + "postState": { + "type": "null" + }, + "reactionPresent": { + "type": "null" } } } @@ -693,8 +747,8 @@ "emoji": { "type": "string", "minLength": 1, - "maxLength": 128, - "pattern": "^[A-Za-z0-9_+.-]+$" + "maxLength": 64, + "pattern": "^[A-Za-z0-9_+-]+$" }, "oneStep": { "type": "object", @@ -1306,6 +1360,28 @@ } } ] + }, + "postState": { + "type": "object", + "additionalProperties": false, + "required": [ + "authorUserId", + "updateAt", + "contentDigest" + ], + "properties": { + "authorUserId": { + "$ref": "#/$defs/id" + }, + "updateAt": { + "type": "integer", + "minimum": 1, + "maximum": 8640000000000000 + }, + "contentDigest": { + "$ref": "#/$defs/digest" + } + } } } } diff --git a/schemas/v2/stage-receipt.schema.json b/schemas/v2/stage-receipt.schema.json index 5741178..66885c3 100644 --- a/schemas/v2/stage-receipt.schema.json +++ b/schemas/v2/stage-receipt.schema.json @@ -226,7 +226,9 @@ "postId", "rootPostId", "participantIds", - "emoji" + "emoji", + "postState", + "reactionPresent" ], "properties": { "kind": { @@ -274,6 +276,22 @@ "type": "null" } ] + }, + "postState": { + "anyOf": [ + { + "$ref": "#/$defs/postState" + }, + { + "type": "null" + } + ] + }, + "reactionPresent": { + "type": [ + "boolean", + "null" + ] } } }, @@ -292,6 +310,12 @@ }, "emoji": { "type": "null" + }, + "postState": { + "type": "null" + }, + "reactionPresent": { + "type": "null" } } }, @@ -328,6 +352,12 @@ }, "emoji": { "type": "null" + }, + "postState": { + "type": "null" + }, + "reactionPresent": { + "type": "null" } } } @@ -348,6 +378,12 @@ }, "emoji": { "type": "null" + }, + "postState": { + "$ref": "#/$defs/postState" + }, + "reactionPresent": { + "type": "null" } } } @@ -368,6 +404,12 @@ }, "emoji": { "$ref": "#/$defs/emoji" + }, + "postState": { + "type": "null" + }, + "reactionPresent": { + "type": "boolean" } } } @@ -391,6 +433,12 @@ }, "emoji": { "type": "null" + }, + "postState": { + "type": "null" + }, + "reactionPresent": { + "type": "null" } } } @@ -414,6 +462,12 @@ }, "emoji": { "type": "null" + }, + "postState": { + "type": "null" + }, + "reactionPresent": { + "type": "null" } } } @@ -545,8 +599,8 @@ "emoji": { "type": "string", "minLength": 1, - "maxLength": 128, - "pattern": "^[A-Za-z0-9_+.-]+$" + "maxLength": 64, + "pattern": "^[A-Za-z0-9_+-]+$" }, "oneStep": { "type": "object", @@ -1158,6 +1212,28 @@ } } ] + }, + "postState": { + "type": "object", + "additionalProperties": false, + "required": [ + "authorUserId", + "updateAt", + "contentDigest" + ], + "properties": { + "authorUserId": { + "$ref": "#/$defs/id" + }, + "updateAt": { + "type": "integer", + "minimum": 1, + "maximum": 8640000000000000 + }, + "contentDigest": { + "$ref": "#/$defs/digest" + } + } } } } diff --git a/schemas/v2/stage.schema.json b/schemas/v2/stage.schema.json index b592179..f2d3f98 100644 --- a/schemas/v2/stage.schema.json +++ b/schemas/v2/stage.schema.json @@ -547,7 +547,9 @@ "postId", "rootPostId", "participantIds", - "emoji" + "emoji", + "postState", + "reactionPresent" ], "properties": { "kind": { @@ -595,6 +597,22 @@ "type": "null" } ] + }, + "postState": { + "anyOf": [ + { + "$ref": "#/$defs/postState" + }, + { + "type": "null" + } + ] + }, + "reactionPresent": { + "type": [ + "boolean", + "null" + ] } } }, @@ -613,6 +631,12 @@ }, "emoji": { "type": "null" + }, + "postState": { + "type": "null" + }, + "reactionPresent": { + "type": "null" } } }, @@ -649,6 +673,12 @@ }, "emoji": { "type": "null" + }, + "postState": { + "type": "null" + }, + "reactionPresent": { + "type": "null" } } } @@ -669,6 +699,12 @@ }, "emoji": { "type": "null" + }, + "postState": { + "$ref": "#/$defs/postState" + }, + "reactionPresent": { + "type": "null" } } } @@ -689,6 +725,12 @@ }, "emoji": { "$ref": "#/$defs/emoji" + }, + "postState": { + "type": "null" + }, + "reactionPresent": { + "type": "boolean" } } } @@ -712,6 +754,12 @@ }, "emoji": { "type": "null" + }, + "postState": { + "type": "null" + }, + "reactionPresent": { + "type": "null" } } } @@ -735,6 +783,12 @@ }, "emoji": { "type": "null" + }, + "postState": { + "type": "null" + }, + "reactionPresent": { + "type": "null" } } } @@ -866,8 +920,8 @@ "emoji": { "type": "string", "minLength": 1, - "maxLength": 128, - "pattern": "^[A-Za-z0-9_+.-]+$" + "maxLength": 64, + "pattern": "^[A-Za-z0-9_+-]+$" }, "oneStep": { "type": "object", @@ -1479,6 +1533,28 @@ } } ] + }, + "postState": { + "type": "object", + "additionalProperties": false, + "required": [ + "authorUserId", + "updateAt", + "contentDigest" + ], + "properties": { + "authorUserId": { + "$ref": "#/$defs/id" + }, + "updateAt": { + "type": "integer", + "minimum": 1, + "maximum": 8640000000000000 + }, + "contentDigest": { + "$ref": "#/$defs/digest" + } + } } } } diff --git a/schemas/v2/stages.schema.json b/schemas/v2/stages.schema.json index 7575248..b625fc1 100644 --- a/schemas/v2/stages.schema.json +++ b/schemas/v2/stages.schema.json @@ -118,7 +118,9 @@ "postId", "rootPostId", "participantIds", - "emoji" + "emoji", + "postState", + "reactionPresent" ], "properties": { "kind": { @@ -166,6 +168,22 @@ "type": "null" } ] + }, + "postState": { + "anyOf": [ + { + "$ref": "#/$defs/postState" + }, + { + "type": "null" + } + ] + }, + "reactionPresent": { + "type": [ + "boolean", + "null" + ] } } }, @@ -184,6 +202,12 @@ }, "emoji": { "type": "null" + }, + "postState": { + "type": "null" + }, + "reactionPresent": { + "type": "null" } } }, @@ -220,6 +244,12 @@ }, "emoji": { "type": "null" + }, + "postState": { + "type": "null" + }, + "reactionPresent": { + "type": "null" } } } @@ -240,6 +270,12 @@ }, "emoji": { "type": "null" + }, + "postState": { + "$ref": "#/$defs/postState" + }, + "reactionPresent": { + "type": "null" } } } @@ -260,6 +296,12 @@ }, "emoji": { "$ref": "#/$defs/emoji" + }, + "postState": { + "type": "null" + }, + "reactionPresent": { + "type": "boolean" } } } @@ -283,6 +325,12 @@ }, "emoji": { "type": "null" + }, + "postState": { + "type": "null" + }, + "reactionPresent": { + "type": "null" } } } @@ -306,6 +354,12 @@ }, "emoji": { "type": "null" + }, + "postState": { + "type": "null" + }, + "reactionPresent": { + "type": "null" } } } @@ -437,8 +491,8 @@ "emoji": { "type": "string", "minLength": 1, - "maxLength": 128, - "pattern": "^[A-Za-z0-9_+.-]+$" + "maxLength": 64, + "pattern": "^[A-Za-z0-9_+-]+$" }, "oneStep": { "type": "object", @@ -1050,6 +1104,28 @@ } } ] + }, + "postState": { + "type": "object", + "additionalProperties": false, + "required": [ + "authorUserId", + "updateAt", + "contentDigest" + ], + "properties": { + "authorUserId": { + "$ref": "#/$defs/id" + }, + "updateAt": { + "type": "integer", + "minimum": 1, + "maximum": 8640000000000000 + }, + "contentDigest": { + "$ref": "#/$defs/digest" + } + } } } } From 103a6d33f969daefec1cb70dfaef522ffc94683d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 01:41:39 +0300 Subject: [PATCH 061/119] fix: align stage emoji input --- schemas/v2/stage-request.schema.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schemas/v2/stage-request.schema.json b/schemas/v2/stage-request.schema.json index 145c7d7..c069277 100644 --- a/schemas/v2/stage-request.schema.json +++ b/schemas/v2/stage-request.schema.json @@ -293,8 +293,8 @@ "emoji": { "type": "string", "minLength": 1, - "maxLength": 128, - "pattern": "^[A-Za-z0-9_+.-]+$" + "maxLength": 64, + "pattern": "^[A-Za-z0-9_+-]+$" }, "selector": { "type": "object", From fbca8c779471268ed7f045c5ec618a18149d1b18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 01:53:19 +0300 Subject: [PATCH 062/119] feat: add strict post mutation reads --- internal/mattermost/posts.go | 206 ++++++++++++++++++++++++++---- internal/mattermost/posts_test.go | 172 ++++++++++++++++++++++++- 2 files changed, 346 insertions(+), 32 deletions(-) diff --git a/internal/mattermost/posts.go b/internal/mattermost/posts.go index 4b6afe5..335c5e8 100644 --- a/internal/mattermost/posts.go +++ b/internal/mattermost/posts.go @@ -5,9 +5,12 @@ import ( "context" "encoding/json" "errors" + "io" "net/url" "strconv" "strings" + "unicode" + "unicode/utf8" ) const MaxPostsPage = 200 @@ -15,13 +18,19 @@ const MaxPostsPage = 200 const ( maxPostIDLength = 128 maxDateMilliseconds = int64(8_640_000_000_000_000) + maxPostTypeLength = 26 + maxPostFileIDs = 100 + maxPostReactions = 10_000 + maxEmojiNameLength = 64 + maxRemoteIDLength = 256 ) var ( - ErrInvalidPostResponse = errors.New("Mattermost returned an invalid post response") - ErrInvalidPostsResponse = errors.New("Mattermost returned an invalid posts response") - ErrInvalidSearchResponse = errors.New("Mattermost returned an invalid search response") - ErrInvalidPostsRequest = errors.New("invalid Mattermost posts request") + ErrInvalidPostResponse = errors.New("Mattermost returned an invalid post response") + ErrInvalidPostsResponse = errors.New("Mattermost returned an invalid posts response") + ErrInvalidSearchResponse = errors.New("Mattermost returned an invalid search response") + ErrInvalidPostsRequest = errors.New("invalid Mattermost posts request") + ErrInvalidReactionsResponse = errors.New("Mattermost returned an invalid reactions response") ) type postTransport interface { @@ -582,40 +591,119 @@ func NewPosts(client postTransport) *Posts { return &Posts{client: client} } type canonicalSinglePost struct{ Post Post } func (p *canonicalSinglePost) UnmarshalJSON(data []byte) error { - var raw struct { - ID json.RawMessage `json:"id"` - ChannelID json.RawMessage `json:"channel_id"` - UserID json.RawMessage `json:"user_id"` - Message json.RawMessage `json:"message"` - CreateAt json.RawMessage `json:"create_at"` - UpdateAt json.RawMessage `json:"update_at"` - DeleteAt json.RawMessage `json:"delete_at"` - RootID json.RawMessage `json:"root_id"` - } - if json.Unmarshal(data, &raw) != nil { + raw, ok := uniqueJSONObject(data) + if !ok { return ErrInvalidPostResponse } - _, idOK := safePostID(raw.ID) - _, channelOK := safePostID(raw.ChannelID) - _, userOK := safePostID(raw.UserID) - _, messageOK := strictString(raw.Message) - createAt, createOK := nonnegativeInteger(raw.CreateAt) - updateAt, updateOK := nonnegativeInteger(raw.UpdateAt) - deleteAt, deleteOK := nonnegativeInteger(raw.DeleteAt) - rootID, rootOK := strictString(raw.RootID) + id, idOK := safePostID(raw["id"]) + channelID, channelOK := safePostID(raw["channel_id"]) + userID, userOK := safePostID(raw["user_id"]) + message, messageOK := strictString(raw["message"]) + createAt, createOK := nonnegativeInteger(raw["create_at"]) + updateAt, updateOK := nonnegativeInteger(raw["update_at"]) + deleteAt, deleteOK := nonnegativeInteger(raw["delete_at"]) + rootID, rootOK := strictString(raw["root_id"]) rootShapeOK := rootOK && (rootID == "" || isSafePostID(rootID)) + postType, typeOK := strictString(raw["type"]) + fileIDs, fileIDsOK := canonicalPostIDs(raw["file_ids"], maxPostFileIDs) if !idOK || !channelOK || !userOK || !messageOK || !createOK || createAt == 0 || createAt > maxDateMilliseconds || - !updateOK || updateAt == 0 || updateAt > maxDateMilliseconds || !deleteOK || deleteAt != 0 || !rootShapeOK { + !updateOK || updateAt == 0 || updateAt > maxDateMilliseconds || !deleteOK || deleteAt != 0 || !rootShapeOK || + !typeOK || !safePresentationString(postType, maxPostTypeLength) || !fileIDsOK { return ErrInvalidPostResponse } - var post Post - if json.Unmarshal(data, &post) != nil { - return ErrInvalidPostResponse + p.Post = Post{ + ID: id, ChannelID: channelID, UserID: userID, Message: message, + CreateAt: createAt, UpdateAt: updateAt, DeleteAt: deleteAt, + RootID: rootID, Type: postType, FileIDs: fileIDs, } - p.Post = post return nil } +func uniqueJSONObject(data []byte) (map[string]json.RawMessage, bool) { + decoder := json.NewDecoder(bytes.NewReader(data)) + token, err := decoder.Token() + if err != nil || token != json.Delim('{') { + return nil, false + } + fields := make(map[string]json.RawMessage) + for decoder.More() { + token, err = decoder.Token() + name, ok := token.(string) + if err != nil || !ok { + return nil, false + } + if _, duplicate := fields[name]; duplicate { + return nil, false + } + var value json.RawMessage + if decoder.Decode(&value) != nil { + return nil, false + } + fields[name] = value + } + if token, err = decoder.Token(); err != nil || token != json.Delim('}') { + return nil, false + } + var trailing any + if err = decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + return nil, false + } + return fields, true +} + +func canonicalPostIDs(raw json.RawMessage, maximum int) ([]string, bool) { + if len(raw) == 0 { + return nil, false + } + if isJSONNull(raw) { + return []string{}, true + } + var values []json.RawMessage + if json.Unmarshal(raw, &values) != nil || len(values) > maximum { + return nil, false + } + result := make([]string, 0, len(values)) + seen := make(map[string]struct{}, len(values)) + for _, candidate := range values { + value, ok := safePostID(candidate) + if !ok { + return nil, false + } + if _, duplicate := seen[value]; duplicate { + return nil, false + } + seen[value] = struct{}{} + result = append(result, value) + } + return result, true +} + +func safePresentationString(value string, maximum int) bool { + if len(value) > maximum || !utf8.ValidString(value) { + return false + } + for _, r := range value { + if unicode.IsControl(r) || r == '\u061c' || r == '\u200e' || r == '\u200f' || + r >= '\u202a' && r <= '\u202e' || r >= '\u2066' && r <= '\u2069' { + return false + } + } + return true +} + +func validEmojiName(value string) bool { + if len(value) == 0 || len(value) > maxEmojiNameLength { + return false + } + for i := range len(value) { + c := value[i] + if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '-' || c == '+') { + return false + } + } + return true +} + // ByID returns one exact, live post. The response must carry the canonical // identity fields needed to bind later operations to the requested post. func (s *Posts) ByID(ctx context.Context, postID string) (Post, error) { @@ -633,6 +721,68 @@ func (s *Posts) ByID(ctx context.Context, postID string) (Post, error) { return post, nil } +// ReactionState authoritatively reads the complete reaction set for one post. +func (s *Posts) ReactionState(ctx context.Context, postID, channelID, userID, emoji string) (bool, error) { + if !isSafePostID(postID) || !isSafePostID(channelID) || !isSafePostID(userID) || !validEmojiName(emoji) { + return false, ErrInvalidPostsRequest + } + var decoded canonicalReactions + decoded.postID = postID + decoded.channelID = channelID + if err := s.client.Get(ctx, "/posts/"+url.PathEscape(postID)+"/reactions", &decoded); err != nil { + return false, err + } + for _, reaction := range decoded.values { + if reaction.UserID == userID && reaction.EmojiName == emoji { + return true, nil + } + } + return false, nil +} + +type canonicalReactions struct { + postID string + channelID string + values []PostReaction +} + +func (r *canonicalReactions) UnmarshalJSON(data []byte) error { + var raw []json.RawMessage + if !isJSONNull(data) && json.Unmarshal(data, &raw) != nil || len(raw) > maxPostReactions { + return ErrInvalidReactionsResponse + } + values := make([]PostReaction, 0, len(raw)) + seen := make(map[string]struct{}, len(raw)) + for _, candidate := range raw { + fields, ok := uniqueJSONObject(candidate) + if !ok { + return ErrInvalidReactionsResponse + } + userID, userOK := safePostID(fields["user_id"]) + postID, postOK := safePostID(fields["post_id"]) + channelID, channelOK := safePostID(fields["channel_id"]) + emoji, emojiOK := strictString(fields["emoji_name"]) + createAt, createOK := nonnegativeInteger(fields["create_at"]) + updateAt, updateOK := nonnegativeInteger(fields["update_at"]) + deleteAt, deleteOK := nonnegativeInteger(fields["delete_at"]) + remoteID, remoteOK := optionalStringValue(fields["remote_id"]) + remoteShapeOK := (remoteOK || isJSONNull(fields["remote_id"])) && safePresentationString(remoteID, maxRemoteIDLength) + if !userOK || !postOK || postID != r.postID || !channelOK || channelID != r.channelID || + !emojiOK || !validEmojiName(emoji) || !createOK || createAt == 0 || createAt > maxDateMilliseconds || + !updateOK || updateAt == 0 || updateAt > maxDateMilliseconds || !deleteOK || deleteAt != 0 || !remoteShapeOK { + return ErrInvalidReactionsResponse + } + key := userID + "\x00" + emoji + if _, duplicate := seen[key]; duplicate { + return ErrInvalidReactionsResponse + } + seen[key] = struct{}{} + values = append(values, PostReaction{UserID: userID, PostID: postID, EmojiName: emoji, CreateAt: createAt}) + } + r.values = values + return nil +} + func (s *Posts) SearchPage(ctx context.Context, teamID string, options SearchPageOptions) (SearchPage, error) { if strings.TrimSpace(teamID) == "" || strings.TrimSpace(options.Terms) == "" || options.Page < 0 || options.PerPage <= 0 || options.PerPage > MaxSearchPage { return SearchPage{}, ErrInvalidPostsRequest diff --git a/internal/mattermost/posts_test.go b/internal/mattermost/posts_test.go index a26178c..1d58438 100644 --- a/internal/mattermost/posts_test.go +++ b/internal/mattermost/posts_test.go @@ -12,6 +12,8 @@ import ( type postTransportFunc func(context.Context, string, any) error +const validReactionRow = `{"user_id":"user","post_id":"post","emoji_name":"Eyes","create_at":1,"update_at":1,"delete_at":0,"remote_id":null,"channel_id":"channel"}` + func (f postTransportFunc) Get(ctx context.Context, path string, out any) error { return f(ctx, path, out) } @@ -33,10 +35,10 @@ func TestPostByIDBuildsExactGETAndRequiresCanonicalLivePost(t *testing.T) { var gotPath string api := NewPosts(postTransportFunc(func(_ context.Context, path string, out any) error { gotPath = path - return json.Unmarshal([]byte(`{"id":"post","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":""}`), out) + return json.Unmarshal([]byte(`{"id":"post","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":"","type":"","file_ids":["file-a","file-b"]}`), out) })) post, err := api.ByID(context.Background(), "post") - if err != nil || post.ID != "post" || post.ChannelID != "channel" || post.UserID != "author" { + if err != nil || post.ID != "post" || post.ChannelID != "channel" || post.UserID != "author" || !reflect.DeepEqual(post.FileIDs, []string{"file-a", "file-b"}) { t.Fatalf("post=%#v error=%v", post, err) } if gotPath != "/posts/post" { @@ -64,6 +66,12 @@ func TestPostByIDRejectsInvalidRequestMismatchMalformedAndDeleted(t *testing.T) "wrong-type root": `{"id":"post","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":7}`, "unsafe root": `{"id":"post","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":"bad/root"}`, "deleted": `{"id":"post","channel_id":"channel","user_id":"author","message":"stale","create_at":1,"update_at":1,"delete_at":2,"root_id":""}`, + "missing type": `{"id":"post","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":"","file_ids":[]}`, + "unsafe type": `{"id":"post","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":"","type":"bad\u001b","file_ids":[]}`, + "oversized type": `{"id":"post","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":"","type":"123456789012345678901234567","file_ids":[]}`, + "missing files": `{"id":"post","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":"","type":""}`, + "duplicate files": `{"id":"post","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":"","type":"","file_ids":["file","file"]}`, + "unsafe file": `{"id":"post","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":"","type":"","file_ids":["bad/file"]}`, } { t.Run(name, func(t *testing.T) { api := NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { @@ -76,6 +84,52 @@ func TestPostByIDRejectsInvalidRequestMismatchMalformedAndDeleted(t *testing.T) } } +func TestPostByIDAcceptsExplicitNullFileIDsAsCanonicalEmpty(t *testing.T) { + api := NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { + return json.Unmarshal([]byte(`{"id":"post","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":"","type":"","file_ids":null}`), out) + })) + post, err := api.ByID(context.Background(), "post") + if err != nil || post.FileIDs == nil || len(post.FileIDs) != 0 { + t.Fatalf("post=%#v error=%v", post, err) + } +} + +func TestPostByIDRejectsDuplicateJSONMembers(t *testing.T) { + base := `"channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":"","type":"","file_ids":[]` + for name, payload := range map[string]string{ + "conflicting": `{"id":"post","id":"other",` + base + `}`, + "identical": `{"id":"post","id":"post",` + base + `}`, + "escaped equivalent": `{"id":"post","\u0069d":"post",` + base + `}`, + } { + t.Run(name, func(t *testing.T) { + api := NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { + return json.Unmarshal([]byte(payload), out) + })) + if _, err := api.ByID(context.Background(), "post"); !errors.Is(err, ErrInvalidPostResponse) { + t.Fatalf("error=%v", err) + } + }) + } +} + +func TestPostByIDUsesOnlyExactCanonicalMemberNames(t *testing.T) { + base := `"channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":"","type":"","file_ids":[]` + api := NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { + return json.Unmarshal([]byte(`{"id":"post","ID":"other",`+base+`}`), out) + })) + post, err := api.ByID(context.Background(), "post") + if err != nil || post.ID != "post" || post.Message != "hello" { + t.Fatalf("post=%#v error=%v", post, err) + } + + api = NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { + return json.Unmarshal([]byte(`{"ID":"post",`+base+`}`), out) + })) + if _, err := api.ByID(context.Background(), "post"); !errors.Is(err, ErrInvalidPostResponse) { + t.Fatalf("missing canonical id error=%v", err) + } +} + func TestPostByIDPropagatesCancellationAndTransportErrors(t *testing.T) { sentinel := errors.New("transport failed") api := NewPosts(postTransportFunc(func(_ context.Context, _ string, _ any) error { return sentinel })) @@ -92,7 +146,7 @@ func TestPostByIDPropagatesCancellationAndTransportErrors(t *testing.T) { func TestPostByIDAcceptsCanonicalReplyRoot(t *testing.T) { api := NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { - return json.Unmarshal([]byte(`{"id":"reply","channel_id":"channel","user_id":"author","message":"hello","create_at":2,"update_at":2,"delete_at":0,"root_id":"root"}`), out) + return json.Unmarshal([]byte(`{"id":"reply","channel_id":"channel","user_id":"author","message":"hello","create_at":2,"update_at":2,"delete_at":0,"root_id":"root","type":"","file_ids":[]}`), out) })) post, err := api.ByID(context.Background(), "reply") if err != nil || post.RootID != "root" { @@ -102,7 +156,7 @@ func TestPostByIDAcceptsCanonicalReplyRoot(t *testing.T) { func TestPostByIDIsRaceSafe(t *testing.T) { api := NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { - return json.Unmarshal([]byte(`{"id":"post","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":""}`), out) + return json.Unmarshal([]byte(`{"id":"post","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":"","type":"","file_ids":[]}`), out) })) var wg sync.WaitGroup for range 40 { @@ -112,6 +166,116 @@ func TestPostByIDIsRaceSafe(t *testing.T) { wg.Wait() } +func TestReactionStateUsesExactFreshGETAndReturnsPresentOrAbsent(t *testing.T) { + for _, tt := range []struct { + name, payload string + want bool + }{ + {"present", `[` + validReactionRow + `]`, true}, + {"absent", `[{"user_id":"other","post_id":"post","emoji_name":"Eyes","create_at":1,"update_at":1,"delete_at":0,"remote_id":"","channel_id":"channel"}]`, false}, + {"empty array", `[]`, false}, + {"real empty null", `null`, false}, + } { + t.Run(tt.name, func(t *testing.T) { + var path string + api := NewPosts(postTransportFunc(func(_ context.Context, got string, out any) error { + path = got + return json.Unmarshal([]byte(tt.payload), out) + })) + got, err := api.ReactionState(context.Background(), "post", "channel", "user", "Eyes") + if err != nil || got != tt.want || path != "/posts/post/reactions" { + t.Fatalf("state=%v path=%q error=%v", got, path, err) + } + }) + } +} + +func TestReactionStateRejectsInvalidInputsBeforeNetwork(t *testing.T) { + for _, args := range [][4]string{{"", "channel", "user", "eyes"}, {"bad/post", "channel", "user", "eyes"}, {"post", "bad/channel", "user", "eyes"}, {"post", "channel", "bad user", "eyes"}, {"post", "channel", "user", "bad:emoji"}} { + called := false + api := NewPosts(postTransportFunc(func(context.Context, string, any) error { called = true; return nil })) + if _, err := api.ReactionState(context.Background(), args[0], args[1], args[2], args[3]); !errors.Is(err, ErrInvalidPostsRequest) || called { + t.Fatalf("args=%q error=%v called=%v", args, err, called) + } + } +} + +func TestReactionStateRejectsIncompleteAmbiguousOrHostileResponses(t *testing.T) { + prefix := strings.TrimSuffix(validReactionRow, "}") + for name, payload := range map[string]string{ + "object": `{}`, + "null item": `[null]`, + "missing field": `[{"user_id":"user","post_id":"post","emoji_name":"Eyes","create_at":1,"update_at":1,"delete_at":0,"channel_id":"channel"}]`, + "wrong type": `[{"user_id":7,"post_id":"post","emoji_name":"Eyes","create_at":1,"update_at":1,"delete_at":0,"remote_id":null,"channel_id":"channel"}]`, + "foreign post": strings.Replace(validReactionRow, `"post_id":"post"`, `"post_id":"other"`, 1), + "foreign channel": strings.Replace(validReactionRow, `"channel_id":"channel"`, `"channel_id":"other"`, 1), + "hostile user": strings.Replace(validReactionRow, `"user_id":"user"`, `"user_id":"bad/user"`, 1), + "bad emoji": strings.Replace(validReactionRow, `"emoji_name":"Eyes"`, `"emoji_name":"bad:emoji"`, 1), + "zero update": strings.Replace(validReactionRow, `"update_at":1`, `"update_at":0`, 1), + "deleted": strings.Replace(validReactionRow, `"delete_at":0`, `"delete_at":2`, 1), + "hostile remote": strings.Replace(validReactionRow, `"remote_id":null`, `"remote_id":"bad\u001b"`, 1), + "duplicate": `[` + prefix + `},` + prefix + `}]`, + "conflicting member": `[` + strings.Replace(validReactionRow, `"user_id":"user"`, `"user_id":"user","user_id":"other"`, 1) + `]`, + "identical member": `[` + strings.Replace(validReactionRow, `"user_id":"user"`, `"user_id":"user","user_id":"user"`, 1) + `]`, + "escaped member": `[` + strings.Replace(validReactionRow, `"user_id":"user"`, `"user_id":"user","user_\u0069d":"user"`, 1) + `]`, + } { + t.Run(name, func(t *testing.T) { + api := NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { return json.Unmarshal([]byte(payload), out) })) + if _, err := api.ReactionState(context.Background(), "post", "channel", "user", "Eyes"); !errors.Is(err, ErrInvalidReactionsResponse) { + t.Fatalf("error=%v", err) + } + }) + } +} + +func TestReactionStateAllowsUnknownAdditiveFields(t *testing.T) { + payload := `[` + strings.TrimSuffix(validReactionRow, "}") + `,"future":{"value":true}}]` + api := NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { return json.Unmarshal([]byte(payload), out) })) + got, err := api.ReactionState(context.Background(), "post", "channel", "user", "Eyes") + if err != nil || !got { + t.Fatalf("state=%v error=%v", got, err) + } +} + +func TestReactionStateBoundsCancellationAndRaceSafety(t *testing.T) { + tooMany := make([]PostReaction, maxPostReactions+1) + payload, err := json.Marshal(tooMany) + if err != nil { + t.Fatal(err) + } + api := NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { return json.Unmarshal(payload, out) })) + if _, err := api.ReactionState(context.Background(), "post", "channel", "user", "eyes"); !errors.Is(err, ErrInvalidReactionsResponse) { + t.Fatalf("bounds error=%v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + api = NewPosts(postTransportFunc(func(ctx context.Context, _ string, _ any) error { return ctx.Err() })) + if _, err := api.ReactionState(ctx, "post", "channel", "user", "eyes"); !errors.Is(err, context.Canceled) { + t.Fatalf("cancel error=%v", err) + } + + api = NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { return json.Unmarshal([]byte(`[]`), out) })) + var wg sync.WaitGroup + for range 40 { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = api.ReactionState(context.Background(), "post", "channel", "user", "eyes") + }() + } + wg.Wait() +} + +func FuzzReactionStateDecoder(f *testing.F) { + f.Add([]byte(`[]`)) + f.Add([]byte(`[` + validReactionRow + `]`)) + f.Fuzz(func(t *testing.T, payload []byte) { + api := NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { return json.Unmarshal(payload, out) })) + _, _ = api.ReactionState(context.Background(), "post", "channel", "user", "Eyes") + }) +} + func TestOrderedPostsPageNormalizesAndSuppressesDeleted(t *testing.T) { var page OrderedPostsPage err := json.Unmarshal([]byte(`{ From befc3289a22fc070ab4cc1b17586a8483af763d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 02:54:05 +0300 Subject: [PATCH 063/119] feat: add replay-safe post staging --- docs/V2_CONTRACT.md | 6 +- internal/cli/store_test.go | 6 +- internal/mattermost/posts.go | 3 +- internal/mattermost/posts_test.go | 22 + internal/schema/registry_test.go | 15 +- internal/schema/stage_test.go | 8 +- internal/stageinput/input.go | 28 ++ internal/stageinput/input_test.go | 12 + internal/stagestore/domain.go | 149 +++++- internal/stagestore/domain_test.go | 179 ++++++- internal/stagestore/schema.go | 4 + internal/staging/conversation.go | 43 +- internal/staging/intent.go | 62 +++ internal/staging/intent_test.go | 31 ++ internal/staging/post.go | 508 +++++++++++++++++++ internal/staging/post_test.go | 580 ++++++++++++++++++++++ internal/staging/service.go | 115 ++++- internal/staging/service_test.go | 151 +++++- internal/staging/types.go | 26 +- internal/staging/validation.go | 17 + schemas/v2/examples/store-doctor.json | 2 +- schemas/v2/examples/store-migrations.json | 2 +- schemas/v2/stage-preview.schema.json | 10 +- schemas/v2/stage-receipt.schema.json | 10 +- schemas/v2/stage.schema.json | 10 +- schemas/v2/stages.schema.json | 10 +- schemas/v2/store-doctor.schema.json | 6 +- schemas/v2/store-migrations.schema.json | 6 +- 28 files changed, 1926 insertions(+), 95 deletions(-) create mode 100644 internal/staging/intent.go create mode 100644 internal/staging/intent_test.go create mode 100644 internal/staging/post.go create mode 100644 internal/staging/post_test.go diff --git a/docs/V2_CONTRACT.md b/docs/V2_CONTRACT.md index 30f21ac..559a9aa 100644 --- a/docs/V2_CONTRACT.md +++ b/docs/V2_CONTRACT.md @@ -201,6 +201,10 @@ mm stage --from-json It consumes one versioned `mm/v2/stage-request` object from stdin. It is the canonical structured agent input, while human subcommands remain first-class rather than wrappers with weaker behavior. Persisted structured requests require a caller-generated `requestId`. Uniqueness is scoped to the normalized server base URL and authenticated user. An identical replay returns the existing stage and receipt; reuse with different semantic content is a local conflict. Human commands may supply the same behavior through `--request-id`. +Stage-create replay equality is caller-intent equality, not equality of the resolved remote snapshot. The digest domain is `mm/v2/stage-request/caller-intent/v1` and binds the operation, unresolved caller target, nullable body and emoji, and ordered normalized attachment metadata. It excludes request ID, server scope, resolved remote facts, file bytes and hashes, sizes, and detected MIME. An identical replay authenticates, loads the original store-authoritative destination and plan, validates content input where applicable, and performs no target, reaction, or file read. Different caller intent conflicts even if it would currently resolve to the same remote target. + +Migration 3 deliberately tombstones pre-caller-intent `mm/v2/stage-request` receipts as `mm/v2/legacy-stage-request-conflict`. Their older digests cannot prove caller-intent equality, so they conflict rather than replay or fall back to remote resolution. Revise and cancel receipts are unchanged. + Structured apply is also first-class: ```text @@ -289,7 +293,7 @@ Stage creation captures enough current remote state to make later drift visible: The closed destination binding carries explicit nullable `postState` and `reactionPresent` fields. `postState` is present only for edit/delete and contains the exact author ID, positive Mattermost `update_at` millisecond value, and a lowercase SHA-256 content digest. `reactionPresent` is present only for react/unreact and records whether the authenticated user already has that exact emoji reaction. Every other operation emits these fields as `null`; absence is not used to mean unknown. -The post content digest is SHA-256 over canonical UTF-8 JSON with the fixed field order `message`, `fileIds`, `rootId`, `type`. File IDs preserve the server's validated order because attachment order is user-visible. The timestamp remains separately bound so a change that returns to identical visible content is still drift. Arbitrary props, presentation metadata, and reactions are excluded from this digest. +The post content digest is SHA-256 over canonical UTF-8 JSON with the fixed field order `message`, `fileIds`, `rootId`, `type`. Normally `message` is the exact string. If it contains an exact active Mattermost credential, `message` is instead an object containing only the ordered non-credential text fragments; neither the credential nor a verifier for its value enters the digest. This exceptional representation deliberately makes credential values indistinguishable while remaining type-distinct from every ordinary message string. The exact positive `update_at` binding remains mandatory, including for credential-elided content, so edit/delete can remediate a leaked credential without accepting later target drift. File IDs preserve the server's validated order because attachment order is user-visible. Arbitrary props, presentation metadata, and reactions are excluded from this digest. Reply staging accepts a live accessible root or reply. It binds the selected post ID and the canonical root ID; targeting a reply requires a fresh root read proving a live root with an empty `root_id` in the same channel. Reply and reaction staging may target any otherwise valid live accessible post. Edit and delete additionally require an ordinary user post (`type` empty) authored by the authenticated user. An edit whose desired body already equals the freshly revalidated current body completes as already satisfied without a write. A target deleted after staging is drift, not already satisfied. diff --git a/internal/cli/store_test.go b/internal/cli/store_test.go index 951feee..34b3d5e 100644 --- a/internal/cli/store_test.go +++ b/internal/cli/store_test.go @@ -36,7 +36,7 @@ func TestStoreDoctorAbsentIsReadOnlyAndSchemaValid(t *testing.T) { if err := registry.Validate("mm/v2/store-doctor", bytes.NewReader(stdout.Bytes())); err != nil { t.Fatalf("schema: %v\n%s", err, stdout.String()) } - for _, fact := range []string{`"filesystemSafe":null`, `"applicationId":null`, `"integrity":null`, `"applied":null`, `"latest":2`, `"valid":null`, `"journalMode":null`} { + for _, fact := range []string{`"filesystemSafe":null`, `"applicationId":null`, `"integrity":null`, `"applied":null`, `"latest":3`, `"valid":null`, `"journalMode":null`} { if !strings.Contains(stdout.String(), fact) { t.Fatalf("absent report omitted %s: %s", fact, stdout.String()) } @@ -110,7 +110,7 @@ func TestStoreMigrationsIsOfflineAndSchemaValid(t *testing.T) { if err := registry.Validate("mm/v2/store-migrations", bytes.NewReader(stdout.Bytes())); err != nil { t.Fatalf("schema: %v\n%s", err, stdout.String()) } - want := "{\"schema\":\"mm/v2/store-migrations\",\"latest\":2,\"migrations\":[{\"version\":1,\"name\":\"core-stage-state\",\"checksum\":\"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f\"},{\"version\":2,\"name\":\"immutable-local-request-receipts\",\"checksum\":\"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c\"}]}\n" + want := "{\"schema\":\"mm/v2/store-migrations\",\"latest\":3,\"migrations\":[{\"version\":1,\"name\":\"core-stage-state\",\"checksum\":\"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f\"},{\"version\":2,\"name\":\"immutable-local-request-receipts\",\"checksum\":\"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c\"},{\"version\":3,\"name\":\"caller-intent-stage-create-replay\",\"checksum\":\"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5\"}]}\n" if stdout.String() != want { t.Fatalf("stdout does not match golden migration contract: %q", stdout.String()) } @@ -310,7 +310,7 @@ func TestStoreHumanOutputStatesBounds(t *testing.T) { t.Fatalf("stdout=%q", stdout.String()) } stdout.Reset() - if code := Execute(context.Background(), []string{"store", "migrations"}, strings.NewReader(""), &stdout, &stderr); code != 0 || !strings.Contains(stdout.String(), "latest: 2\n1 core-stage-state ") { + if code := Execute(context.Background(), []string{"store", "migrations"}, strings.NewReader(""), &stdout, &stderr); code != 0 || !strings.Contains(stdout.String(), "latest: 3\n1 core-stage-state ") { t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } } diff --git a/internal/mattermost/posts.go b/internal/mattermost/posts.go index 335c5e8..a5abf41 100644 --- a/internal/mattermost/posts.go +++ b/internal/mattermost/posts.go @@ -597,7 +597,7 @@ func (p *canonicalSinglePost) UnmarshalJSON(data []byte) error { } id, idOK := safePostID(raw["id"]) channelID, channelOK := safePostID(raw["channel_id"]) - userID, userOK := safePostID(raw["user_id"]) + userID, userOK := strictString(raw["user_id"]) message, messageOK := strictString(raw["message"]) createAt, createOK := nonnegativeInteger(raw["create_at"]) updateAt, updateOK := nonnegativeInteger(raw["update_at"]) @@ -605,6 +605,7 @@ func (p *canonicalSinglePost) UnmarshalJSON(data []byte) error { rootID, rootOK := strictString(raw["root_id"]) rootShapeOK := rootOK && (rootID == "" || isSafePostID(rootID)) postType, typeOK := strictString(raw["type"]) + userOK = userOK && (isSafePostID(userID) || userID == "" && strings.HasPrefix(postType, "system_")) fileIDs, fileIDsOK := canonicalPostIDs(raw["file_ids"], maxPostFileIDs) if !idOK || !channelOK || !userOK || !messageOK || !createOK || createAt == 0 || createAt > maxDateMilliseconds || !updateOK || updateAt == 0 || updateAt > maxDateMilliseconds || !deleteOK || deleteAt != 0 || !rootShapeOK || diff --git a/internal/mattermost/posts_test.go b/internal/mattermost/posts_test.go index 1d58438..588fe4f 100644 --- a/internal/mattermost/posts_test.go +++ b/internal/mattermost/posts_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "reflect" + "strconv" "strings" "sync" "testing" @@ -154,6 +155,27 @@ func TestPostByIDAcceptsCanonicalReplyRoot(t *testing.T) { } } +func TestPostByIDAcceptsOnlyTypedAuthorlessSystemPosts(t *testing.T) { + for _, test := range []struct { + name, userID, postType string + wantErr bool + }{ + {"system", "", "system_join_channel", false}, + {"ordinary authorless", "", "", true}, + {"non-system typed authorless", "", "custom_type", true}, + {"ordinary authored", "author", "", false}, + } { + t.Run(test.name, func(t *testing.T) { + payload := `{"id":"post","channel_id":"channel","user_id":` + strconv.Quote(test.userID) + `,"message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":"","type":` + strconv.Quote(test.postType) + `,"file_ids":[]}` + api := NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { return json.Unmarshal([]byte(payload), out) })) + post, err := api.ByID(context.Background(), "post") + if test.wantErr && !errors.Is(err, ErrInvalidPostResponse) || !test.wantErr && (err != nil || post.UserID != test.userID || post.Type != test.postType) { + t.Fatalf("post/error = %#v/%v", post, err) + } + }) + } +} + func TestPostByIDIsRaceSafe(t *testing.T) { api := NewPosts(postTransportFunc(func(_ context.Context, _ string, out any) error { return json.Unmarshal([]byte(`{"id":"post","channel_id":"channel","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":"","type":"","file_ids":[]}`), out) diff --git a/internal/schema/registry_test.go b/internal/schema/registry_test.go index 14dd6c8..a4d51de 100644 --- a/internal/schema/registry_test.go +++ b/internal/schema/registry_test.go @@ -134,14 +134,15 @@ func TestStoreSchemasRejectSemanticContradictions(t *testing.T) { if err != nil { t.Fatal(err) } - absent := `{"schema":"mm/v2/store-doctor","path":"/tmp/stages.sqlite3","report":{"exists":false,"filesystemSafe":false,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":2,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}}` - issueWithoutRow := `{"schema":"mm/v2/store-doctor","path":"/tmp/stages.sqlite3","report":{"exists":true,"filesystemSafe":true,"applicationId":1296913970,"integrity":["ok"],"integrityTruncated":false,"foreignKeyIssues":1,"foreignKeyRows":[],"foreignKeyTruncated":false,"migrations":{"applied":2,"latest":2,"valid":true},"journalMode":"wal","synchronous":2,"secureDelete":2,"foreignKeys":true,"trustedSchema":false,"queryOnly":true,"walFallback":false,"permissionModelLimitations":[]}}` - contradictoryWAL := `{"schema":"mm/v2/store-doctor","path":"/tmp/stages.sqlite3","report":{"exists":true,"filesystemSafe":true,"applicationId":1296913970,"integrity":["ok"],"integrityTruncated":false,"foreignKeyIssues":0,"foreignKeyRows":[],"foreignKeyTruncated":false,"migrations":{"applied":2,"latest":2,"valid":true},"journalMode":"wal","synchronous":2,"secureDelete":2,"foreignKeys":true,"trustedSchema":false,"queryOnly":true,"walFallback":true,"permissionModelLimitations":[]}}` + absent := `{"schema":"mm/v2/store-doctor","path":"/tmp/stages.sqlite3","report":{"exists":false,"filesystemSafe":false,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":3,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}}` + issueWithoutRow := `{"schema":"mm/v2/store-doctor","path":"/tmp/stages.sqlite3","report":{"exists":true,"filesystemSafe":true,"applicationId":1296913970,"integrity":["ok"],"integrityTruncated":false,"foreignKeyIssues":1,"foreignKeyRows":[],"foreignKeyTruncated":false,"migrations":{"applied":3,"latest":3,"valid":true},"journalMode":"wal","synchronous":2,"secureDelete":2,"foreignKeys":true,"trustedSchema":false,"queryOnly":true,"walFallback":false,"permissionModelLimitations":[]}}` + contradictoryWAL := `{"schema":"mm/v2/store-doctor","path":"/tmp/stages.sqlite3","report":{"exists":true,"filesystemSafe":true,"applicationId":1296913970,"integrity":["ok"],"integrityTruncated":false,"foreignKeyIssues":0,"foreignKeyRows":[],"foreignKeyTruncated":false,"migrations":{"applied":3,"latest":3,"valid":true},"journalMode":"wal","synchronous":2,"secureDelete":2,"foreignKeys":true,"trustedSchema":false,"queryOnly":true,"walFallback":true,"permissionModelLimitations":[]}}` twentyRows := `"row",` + strings.Repeat(`"row",`, 18) + `"row"` - unmarkedTruncation := `{"schema":"mm/v2/store-doctor","path":"/tmp/stages.sqlite3","report":{"exists":true,"filesystemSafe":true,"applicationId":1296913970,"integrity":["ok"],"integrityTruncated":false,"foreignKeyIssues":21,"foreignKeyRows":[` + twentyRows + `],"foreignKeyTruncated":false,"migrations":{"applied":2,"latest":2,"valid":true},"journalMode":"wal","synchronous":2,"secureDelete":2,"foreignKeys":true,"trustedSchema":false,"queryOnly":true,"walFallback":false,"permissionModelLimitations":[]}}` - mixedIntegrity := `{"schema":"mm/v2/store-doctor","path":"/tmp/stages.sqlite3","report":{"exists":true,"filesystemSafe":true,"applicationId":1296913970,"integrity":["ok","damaged"],"integrityTruncated":false,"foreignKeyIssues":0,"foreignKeyRows":[],"foreignKeyTruncated":false,"migrations":{"applied":2,"latest":2,"valid":true},"journalMode":"wal","synchronous":2,"secureDelete":2,"foreignKeys":true,"trustedSchema":false,"queryOnly":true,"walFallback":false,"permissionModelLimitations":[]}}` - wrongLatest := `{"schema":"mm/v2/store-migrations","latest":1,"migrations":[{"version":1,"name":"core-stage-state","checksum":"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"},{"version":2,"name":"immutable-local-request-receipts","checksum":"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"}]}` - wrongOrder := `{"schema":"mm/v2/store-migrations","latest":2,"migrations":[{"version":2,"name":"immutable-local-request-receipts","checksum":"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"},{"version":1,"name":"core-stage-state","checksum":"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"}]}` + unmarkedTruncation := `{"schema":"mm/v2/store-doctor","path":"/tmp/stages.sqlite3","report":{"exists":true,"filesystemSafe":true,"applicationId":1296913970,"integrity":["ok"],"integrityTruncated":false,"foreignKeyIssues":21,"foreignKeyRows":[` + twentyRows + `],"foreignKeyTruncated":false,"migrations":{"applied":3,"latest":3,"valid":true},"journalMode":"wal","synchronous":2,"secureDelete":2,"foreignKeys":true,"trustedSchema":false,"queryOnly":true,"walFallback":false,"permissionModelLimitations":[]}}` + mixedIntegrity := `{"schema":"mm/v2/store-doctor","path":"/tmp/stages.sqlite3","report":{"exists":true,"filesystemSafe":true,"applicationId":1296913970,"integrity":["ok","damaged"],"integrityTruncated":false,"foreignKeyIssues":0,"foreignKeyRows":[],"foreignKeyTruncated":false,"migrations":{"applied":3,"latest":3,"valid":true},"journalMode":"wal","synchronous":2,"secureDelete":2,"foreignKeys":true,"trustedSchema":false,"queryOnly":true,"walFallback":false,"permissionModelLimitations":[]}}` + migration3 := `{"version":3,"name":"caller-intent-stage-create-replay","checksum":"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5"}` + wrongLatest := `{"schema":"mm/v2/store-migrations","latest":1,"migrations":[{"version":1,"name":"core-stage-state","checksum":"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"},{"version":2,"name":"immutable-local-request-receipts","checksum":"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"},` + migration3 + `]}` + wrongOrder := `{"schema":"mm/v2/store-migrations","latest":3,"migrations":[{"version":2,"name":"immutable-local-request-receipts","checksum":"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"},{"version":1,"name":"core-stage-state","checksum":"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"},` + migration3 + `]}` for id, documents := range map[string][]string{"mm/v2/store-doctor": {absent, issueWithoutRow, contradictoryWAL, unmarkedTruncation, mixedIntegrity}, "mm/v2/store-migrations": {wrongLatest, wrongOrder}} { for _, document := range documents { if err := registry.Validate(id, strings.NewReader(document)); err == nil { diff --git a/internal/schema/stage_test.go b/internal/schema/stage_test.go index aae69e7..76b91fe 100644 --- a/internal/schema/stage_test.go +++ b/internal/schema/stage_test.go @@ -233,8 +233,8 @@ func TestStageDestinationRemoteStateDiscriminants(t *testing.T) { preview("reply", reply, "create_post", false), preview("edit_post", post, "edit_post", false), preview("delete_post", post, "delete_post", false), - preview("react", reaction, "add_reaction", false), - preview("unreact", reaction, "remove_reaction", false), + strings.Replace(preview("react", reaction, "add_reaction", false), `"condition":"always"`, `"condition":"if_missing"`, 1), + strings.Replace(preview("unreact", reaction, "remove_reaction", false), `"condition":"always"`, `"condition":"if_missing"`, 1), strings.Replace(preview("resolve_dm", dm, "resolve_conversation", false), `"condition":"always"`, `"condition":"if_missing"`, 1), strings.Replace(preview("resolve_group_dm", group, "resolve_conversation", false), `"condition":"always"`, `"condition":"if_missing"`, 1), } @@ -260,6 +260,8 @@ func TestStageDestinationRemoteStateDiscriminants(t *testing.T) { strings.Replace(valid[2], `"updateAt":1720000000000`, `"updateAt":8640000000000001`, 1), strings.Replace(valid[2], digest, strings.Repeat("A", 64), 1), strings.Replace(valid[2], digest, strings.Repeat("a", 63), 1), + strings.Replace(valid[4], `"condition":"if_missing"`, `"condition":"always"`, 1), + strings.Replace(valid[5], `"condition":"if_missing"`, `"condition":"always"`, 1), } for _, document := range invalid { assertInvalid(t, r, "mm/v2/stage-preview", document) @@ -291,7 +293,7 @@ func TestStageEmojiMatchesMattermostReactionGrammar(t *testing.T) { func reactionPreview(emoji string) string { destination := `{"kind":"reaction","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":"post-1","rootPostId":null,"participantIds":[],"emoji":` + strconv.Quote(emoji) + `,"postState":null,"reactionPresent":false}` - return preview("react", destination, "add_reaction", false) + return strings.Replace(preview("react", destination, "add_reaction", false), `"condition":"always"`, `"condition":"if_missing"`, 1) } func example(t *testing.T, name string) string { diff --git a/internal/stageinput/input.go b/internal/stageinput/input.go index b27db06..e28208b 100644 --- a/internal/stageinput/input.go +++ b/internal/stageinput/input.go @@ -44,6 +44,34 @@ type Attachment struct { MediaType string // empty detects MIME from the first 512 file bytes } +type MetadataIntent struct { + Path string `json:"path"` + RemoteFilename string `json:"remoteFilename"` + MediaType *string `json:"mediaType"` +} + +// Preflight validates and canonicalizes caller-supplied attachment metadata +// without opening or reading the referenced files. +func Preflight(inputs []Attachment) ([]MetadataIntent, error) { + if len(inputs) > MaxAttachments { + return nil, ErrTooMany + } + result := make([]MetadataIntent, 0, len(inputs)) + for _, input := range inputs { + prepared, err := prepareMetadata(input) + if err != nil { + return nil, err + } + var mediaType *string + if input.MediaType != "" { + value := prepared.mediaType + mediaType = &value + } + result = append(result, MetadataIntent{prepared.canonical, prepared.filename, mediaType}) + } + return result, nil +} + // Bind records a durable-at-rest snapshot only. A later apply must securely // reopen the canonical path, rescan credentials, rehash, and spool before any // upload; a changed path must conflict with this recorded identity and digest. diff --git a/internal/stageinput/input_test.go b/internal/stageinput/input_test.go index f71bd44..359108b 100644 --- a/internal/stageinput/input_test.go +++ b/internal/stageinput/input_test.go @@ -109,6 +109,18 @@ func TestBindCanonicalizesMediaType(t *testing.T) { } } +func TestPreflightReturnsNormalizedIntentWithoutOpeningFile(t *testing.T) { + missing := filepath.Join(t.TempDir(), "missing.txt") + intent, err := Preflight([]Attachment{{Path: missing, RemoteFilename: "Report.TXT", MediaType: `Text/Plain; Charset="UTF-8"`}}) + if err != nil || len(intent) != 1 || intent[0].Path != filepath.Clean(missing) || intent[0].RemoteFilename != "Report.TXT" || intent[0].MediaType == nil || *intent[0].MediaType != "text/plain; charset=UTF-8" { + t.Fatalf("intent/error = %#v/%v", intent, err) + } + auto, err := Preflight([]Attachment{{Path: missing}}) + if err != nil || auto[0].RemoteFilename != "missing.txt" || auto[0].MediaType != nil { + t.Fatalf("auto intent/error = %#v/%v", auto, err) + } +} + func TestBindScansOriginalMetadataBeforeCanonicalization(t *testing.T) { path := filepath.Join(localTempDir(t), "file") if err := os.WriteFile(path, []byte("data"), 0o600); err != nil { diff --git a/internal/stagestore/domain.go b/internal/stagestore/domain.go index 718e67d..ecd72a6 100644 --- a/internal/stagestore/domain.go +++ b/internal/stagestore/domain.go @@ -90,10 +90,18 @@ type Composition struct { } type CreateInput struct { RequestID string + RequestDigest [32]byte Operation Operation ServerURL, ServerID, UserID string Content RevisionContent } +type CreateRecord struct { + MutationResult `json:"-"` + Result MutationResult `json:"result"` + RequestDigest [32]byte `json:"requestDigest"` + Destination json.RawMessage `json:"destination"` + Plan json.RawMessage `json:"plan"` +} type ReviseInput struct { StageID, RequestID string ExpectedRevision int64 @@ -137,56 +145,63 @@ type MutationResult struct { } type ListOptions struct{ Limit int } -func (s *Store) Create(ctx context.Context, in CreateInput) (MutationResult, error) { +func (s *Store) Create(ctx context.Context, in CreateInput) (CreateRecord, error) { if err := ctx.Err(); err != nil { - return MutationResult{}, err + return CreateRecord{}, err } content, err := normalizeContent(in.Operation, in.Content) if err != nil || !validOperation(in.Operation) || !canonicalServerURL(in.ServerURL) || !bounded(in.UserID, maxIdentityBytes) || (in.ServerID != "" && !bounded(in.ServerID, maxIdentityBytes)) || !validRequestID(in.RequestID) { - return MutationResult{}, ErrInvalid + return CreateRecord{}, ErrInvalid + } + if in.RequestDigest == ([32]byte{}) { + return CreateRecord{}, ErrInvalid } semantic := semanticDigest(in.Operation, in.ServerURL, in.ServerID, in.UserID, content) - requestDigest := digestValue(struct { - Operation Operation `json:"operation"` - Semantic [32]byte `json:"semanticDigest"` - }{in.Operation, semantic}) tx, err := s.db.BeginTx(ctx, nil) if err != nil { - return MutationResult{}, localError(err) + return CreateRecord{}, localError(err) } defer tx.Rollback() if in.RequestID != "" { - if result, found, e := loadReplay(ctx, tx, in.ServerURL, in.UserID, in.RequestID, "mm/v2/stage-request", requestDigest); e != nil { - return MutationResult{}, e + if result, found, e := findCreate(ctx, tx, in.ServerURL, in.UserID, in.RequestID); e != nil { + return CreateRecord{}, e } else if found { + if result.RequestDigest != in.RequestDigest { + return CreateRecord{}, ErrConflict + } + result.Replay, result.Result.Replay = true, true return result, nil } } id, err := newStageID() if err != nil { - return MutationResult{}, errors.New("stage store: random identity unavailable") + return CreateRecord{}, errors.New("stage store: random identity unavailable") } now := time.Now().UTC() stamp := formatTime(now) if _, err = tx.ExecContext(ctx, `INSERT INTO stages(id,created_at,updated_at,operation,server_url,server_id,user_id,lifecycle,recovery,current_revision) VALUES(?,?,?,?,?,?,?,?,?,1)`, id, stamp, stamp, in.Operation, in.ServerURL, nullable(in.ServerID), in.UserID, LifecycleOpen, RecoveryNone); err != nil { - return MutationResult{}, localError(err) + return CreateRecord{}, localError(err) } if _, err = tx.ExecContext(ctx, `INSERT INTO stage_revisions(stage_id,revision,state,created_at,semantic_digest,body,destination_json,plan_json) VALUES(?,1,'current',?,?,?,?,?)`, id, stamp, semantic[:], nullableBytes(content.Body), string(content.Destination), string(content.Plan)); err != nil { - return MutationResult{}, localError(err) + return CreateRecord{}, localError(err) } if err = insertAttachments(ctx, tx, id, 1, content.Attachments); err != nil { - return MutationResult{}, err + return CreateRecord{}, err } summary := StageSummary{id, in.ServerURL, in.ServerID, in.UserID, in.Operation, LifecycleOpen, RecoveryNone, 1, semantic, now, now} result := MutationResult{"mm/v2/stage-mutation-receipt", "create", summary, false, now, false} - if err = persistReplay(ctx, tx, in.ServerURL, in.UserID, in.RequestID, "mm/v2/stage-request", requestDigest, result, stamp); err != nil { - return MutationResult{}, err + record := CreateRecord{result, result, in.RequestDigest, bytes.Clone(in.Content.Destination), bytes.Clone(in.Content.Plan)} + if record.Destination, record.Plan, err = normalizeCreateProjection(record.Destination, record.Plan); err != nil { + return CreateRecord{}, localError(err) + } + if err = persistCreate(ctx, tx, in.ServerURL, in.UserID, in.RequestID, record, stamp); err != nil { + return CreateRecord{}, err } if err = tx.Commit(); err != nil { - return MutationResult{}, localError(err) + return CreateRecord{}, localError(err) } runCommitHook() - return result, nil + return record, nil } func (s *Store) Revise(ctx context.Context, in ReviseInput) (MutationResult, error) { @@ -638,6 +653,7 @@ func insertAttachments(ctx context.Context, tx *sql.Tx, stage string, revision i type queryer interface { QueryContext(context.Context, string, ...any) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...any) *sql.Row } func readAttachments(ctx context.Context, q queryer, stage string, revision int64) ([]Attachment, error) { @@ -685,6 +701,103 @@ func loadReplay(ctx context.Context, tx *sql.Tx, server, user, id, schema string result.Replay = true return result, true, nil } + +func (s *Store) FindCreate(ctx context.Context, server, user, id string) (CreateRecord, bool, error) { + if ctx == nil || !canonicalServerURL(server) || !bounded(user, maxIdentityBytes) || !validRequestID(id) { + return CreateRecord{}, false, ErrInvalid + } + record, found, err := findCreate(ctx, s.db, server, user, id) + if found { + record.Replay, record.Result.Replay = true, true + } + return record, found, err +} + +func findCreate(ctx context.Context, q queryer, server, user, id string) (CreateRecord, bool, error) { + var schemaName, raw, requestCreated string + var digest []byte + err := q.QueryRowContext(ctx, `SELECT request_schema,request_digest,result_json,created_at FROM local_requests WHERE server_url=? AND user_id=? AND request_id=?`, server, user, id).Scan(&schemaName, &digest, &raw, &requestCreated) + if errors.Is(err, sql.ErrNoRows) { + return CreateRecord{}, false, nil + } + if err != nil { + return CreateRecord{}, false, localError(err) + } + if schemaName == "mm/v2/legacy-stage-request-conflict" || schemaName == "mm/v2/legacy-request-conflict" || schemaName == "mm/v2/stage-revise-request" || schemaName == "mm/v2/stage-cancel-request" { + return CreateRecord{}, false, ErrConflict + } + if schemaName != "mm/v2/stage-request" || len(digest) != 32 { + return CreateRecord{}, false, localError(errors.New("create receipt")) + } + var record CreateRecord + decoder := json.NewDecoder(strings.NewReader(raw)) + decoder.DisallowUnknownFields() + createdAt, createdErr := parseTime(requestCreated) + if decoder.Decode(&record) != nil || decoder.Decode(new(any)) != io.EOF || createdErr != nil || !record.Result.RecordedAt.Equal(createdAt) || !bytes.Equal(digest, record.RequestDigest[:]) || !validReplayResult(record.Result, "mm/v2/stage-request", server, user) { + return CreateRecord{}, false, localError(errors.New("create receipt")) + } + record.MutationResult = record.Result + stage := record.Stage + if stage.Revision != 1 || stage.Lifecycle != LifecycleOpen || stage.Recovery != RecoveryNone || record.Revived || + !stage.CreatedAt.Equal(stage.UpdatedAt) || !stage.CreatedAt.Equal(record.RecordedAt) { + return CreateRecord{}, false, localError(errors.New("create receipt")) + } + var operation Operation + var stageServer, serverID, stageUser, stageCreated, revisionCreated string + var body []byte + var destination, plan string + var semantic []byte + err = q.QueryRowContext(ctx, `SELECT s.operation,s.server_url,coalesce(s.server_id,''),s.user_id,s.created_at,r.created_at,r.body,r.destination_json,r.plan_json,r.semantic_digest FROM stages s JOIN stage_revisions r ON r.stage_id=s.id AND r.revision=? WHERE s.id=?`, stage.Revision, stage.ID).Scan(&operation, &stageServer, &serverID, &stageUser, &stageCreated, &revisionCreated, &body, &destination, &plan, &semantic) + if err != nil { + return CreateRecord{}, false, localError(err) + } + attachments, err := readAttachments(ctx, q, stage.ID, stage.Revision) + if err != nil { + return CreateRecord{}, false, err + } + content, err := normalizeContent(operation, RevisionContent{body, json.RawMessage(destination), json.RawMessage(plan), attachments}) + recordDestination, destinationErr := canonicalObject(record.Destination) + recordPlan, planErr := canonicalObject(record.Plan) + stageCreatedAt, stageCreatedErr := parseTime(stageCreated) + revisionCreatedAt, revisionCreatedErr := parseTime(revisionCreated) + if err != nil || destinationErr != nil || planErr != nil || stageCreatedErr != nil || revisionCreatedErr != nil || !stage.CreatedAt.Equal(stageCreatedAt) || !stage.CreatedAt.Equal(revisionCreatedAt) || len(semantic) != 32 || !bytes.Equal(semantic, stage.SemanticDigest[:]) || semanticDigest(operation, server, serverID, user, content) != stage.SemanticDigest || + !bytes.Equal(content.Destination, recordDestination) || !bytes.Equal(content.Plan, recordPlan) || operation != stage.Operation || stageServer != server || stageUser != user || serverID != stage.ServerID { + return CreateRecord{}, false, localError(errors.New("create projection")) + } + record.Destination, record.Plan = bytes.Clone(record.Destination), bytes.Clone(record.Plan) + return record, true, nil +} + +func normalizeCreateProjection(destination, plan json.RawMessage) (json.RawMessage, json.RawMessage, error) { + projection := struct { + Destination json.RawMessage `json:"destination"` + Plan json.RawMessage `json:"plan"` + }{destination, plan} + raw, err := marshalCanonical(projection) + if err != nil { + return nil, nil, err + } + var normalized struct { + Destination json.RawMessage `json:"destination"` + Plan json.RawMessage `json:"plan"` + } + if err = json.Unmarshal(raw, &normalized); err != nil { + return nil, nil, err + } + return normalized.Destination, normalized.Plan, nil +} + +func persistCreate(ctx context.Context, tx *sql.Tx, server, user, id string, record CreateRecord, stamp string) error { + if id == "" { + return nil + } + raw, err := marshalCanonical(record) + if err != nil { + return localError(err) + } + _, err = tx.ExecContext(ctx, `INSERT INTO local_requests(server_url,user_id,request_id,request_schema,request_digest,result_json,created_at) VALUES(?,?,?,?,?,?,?)`, server, user, id, "mm/v2/stage-request", record.RequestDigest[:], string(raw), stamp) + return localError(err) +} func validReplayResult(result MutationResult, requestSchema, server, user string) bool { action := map[string]string{"mm/v2/stage-request": "create", "mm/v2/stage-revise-request": "revise", "mm/v2/stage-cancel-request": "cancel"}[requestSchema] stage := result.Stage diff --git a/internal/stagestore/domain_test.go b/internal/stagestore/domain_test.go index 9f0169c..8b3dcd9 100644 --- a/internal/stagestore/domain_test.go +++ b/internal/stagestore/domain_test.go @@ -3,6 +3,7 @@ package stagestore import ( + "bytes" "context" "crypto/sha256" "encoding/json" @@ -26,7 +27,183 @@ func attachment(name string) Attachment { return Attachment{"/tmp/" + name, "/private/tmp/" + name, name, 3, "text/plain", sha256.Sum256([]byte(name))} } func createInput(request, body string) CreateInput { - return CreateInput{request, CreatePost, "https://mattermost.example/api/v4", "server-1", "user-1", RevisionContent{[]byte(body), json.RawMessage(`{"kind":"O","channelId":"channel-1"}`), json.RawMessage(`{"steps":[{"kind":"create_post"}]}`), []Attachment{attachment("a.txt"), attachment("b.txt")}}} + return CreateInput{request, sha256.Sum256([]byte(body)), CreatePost, "https://mattermost.example/api/v4", "server-1", "user-1", RevisionContent{[]byte(body), json.RawMessage(`{"kind":"O","channelId":"channel-1"}`), json.RawMessage(`{"steps":[{"kind":"create_post"}]}`), []Attachment{attachment("a.txt"), attachment("b.txt")}}} +} + +func TestFindCreateUsesExactReceiptRevisionAndFailsClosedOnCorruption(t *testing.T) { + s := openDomainStore(t) + in := createInput("exact-revision", "one") + created, err := s.Create(context.Background(), in) + if err != nil { + t.Fatal(err) + } + revised, err := s.Revise(context.Background(), ReviseInput{StageID: created.Stage.ID, RequestID: "revise-exact", ExpectedRevision: 1, ExpectedDigest: created.Stage.SemanticDigest, Composition: Composition{Body: []byte("two"), Attachments: in.Content.Attachments}}) + if err != nil || revised.Stage.Revision != 2 { + t.Fatalf("revise = %#v/%v", revised, err) + } + record, found, err := s.FindCreate(context.Background(), in.ServerURL, in.UserID, in.RequestID) + if err != nil || !found || record.Stage.Revision != 1 || !reflect.DeepEqual(record.Destination, in.Content.Destination) { + t.Fatalf("record = %#v/%v/%v", record, found, err) + } + if _, err = s.db.Exec(`DROP TRIGGER local_requests_immutable_update`); err != nil { + t.Fatal(err) + } + var originalRaw string + if err = s.db.QueryRow(`SELECT result_json FROM local_requests WHERE request_id=?`, in.RequestID).Scan(&originalRaw); err != nil { + t.Fatal(err) + } + var later CreateRecord + if err = json.Unmarshal([]byte(originalRaw), &later); err != nil { + t.Fatal(err) + } + later.Result.Stage.Revision = revised.Stage.Revision + later.Result.Stage.SemanticDigest = revised.Stage.SemanticDigest + laterRaw, err := marshalCanonical(later) + if err != nil { + t.Fatal(err) + } + if _, err = s.db.Exec(`UPDATE local_requests SET result_json=? WHERE request_id=?`, string(laterRaw), in.RequestID); err != nil { + t.Fatal(err) + } + if _, _, err = s.FindCreate(context.Background(), in.ServerURL, in.UserID, in.RequestID); err == nil || errors.Is(err, ErrConflict) { + t.Fatalf("later-revision corruption error = %v", err) + } + if _, err = s.db.Exec(`UPDATE local_requests SET result_json=? WHERE request_id=?`, originalRaw, in.RequestID); err != nil { + t.Fatal(err) + } + other := createInput("other-receipt", "one") + if _, err = s.Create(context.Background(), other); err != nil { + t.Fatal(err) + } + var otherRaw string + if err = s.db.QueryRow(`SELECT result_json FROM local_requests WHERE request_id=?`, other.RequestID).Scan(&otherRaw); err != nil { + t.Fatal(err) + } + if _, err = s.db.Exec(`UPDATE local_requests SET result_json=? WHERE request_id=?`, otherRaw, in.RequestID); err != nil { + t.Fatal(err) + } + if _, _, err = s.FindCreate(context.Background(), in.ServerURL, in.UserID, in.RequestID); err == nil || errors.Is(err, ErrConflict) { + t.Fatalf("cross-stage corruption error = %v", err) + } + if _, err = s.db.Exec(`UPDATE local_requests SET result_json='{}' WHERE request_id=?`, in.RequestID); err != nil { + t.Fatal(err) + } + if _, _, err = s.FindCreate(context.Background(), in.ServerURL, in.UserID, in.RequestID); err == nil || errors.Is(err, ErrConflict) { + t.Fatalf("corruption error = %v", err) + } +} + +func TestConcurrentCreateReturnsOneAuthoritativeProjection(t *testing.T) { + s := openDomainStore(t) + base := createInput("concurrent-create", "body") + const workers = 20 + results := make(chan CreateRecord, workers) + failures := make(chan error, workers) + var wg sync.WaitGroup + for i := range workers { + wg.Add(1) + go func() { + defer wg.Done() + in := base + in.Content.Destination = json.RawMessage(`{"kind":"O","channelId":"channel-` + string(rune('a'+i)) + `"}`) + record, err := s.Create(context.Background(), in) + if err != nil { + failures <- err + return + } + results <- record + }() + } + wg.Wait() + close(results) + close(failures) + for err := range failures { + t.Fatal(err) + } + var id string + var projection []byte + firsts := 0 + count := 0 + for record := range results { + count++ + if !record.Replay { + firsts++ + } + if id == "" { + id, projection = record.Stage.ID, record.Destination + } + if record.Stage.ID != id || !bytes.Equal(record.Destination, projection) { + t.Fatalf("non-authoritative record = %#v", record) + } + } + if count != workers || firsts != 1 { + t.Fatalf("count/firsts = %d/%d", count, firsts) + } +} + +func TestCreateAndReplayReturnCanonicalAuthoritativeProjection(t *testing.T) { + s := openDomainStore(t) + in := createInput("canonical-projection", "body") + in.Content.Destination = json.RawMessage(`{ "kind": "O", "channelId": "channel-1" }`) + in.Content.Plan = json.RawMessage(`{ "steps": [ { "kind": "create_post" } ] }`) + + created, err := s.Create(context.Background(), in) + if err != nil { + t.Fatal(err) + } + wantDestination := []byte(`{"kind":"O","channelId":"channel-1"}`) + wantPlan := []byte(`{"steps":[{"kind":"create_post"}]}`) + if !bytes.Equal(created.Destination, wantDestination) || !bytes.Equal(created.Plan, wantPlan) { + t.Fatalf("created projection = %s / %s", created.Destination, created.Plan) + } + + replayed, err := s.Create(context.Background(), in) + if err != nil || !replayed.Replay { + t.Fatalf("replay = %#v / %v", replayed, err) + } + if !bytes.Equal(replayed.Destination, created.Destination) || !bytes.Equal(replayed.Plan, created.Plan) { + t.Fatalf("replay projection = %s / %s, want %s / %s", replayed.Destination, replayed.Plan, created.Destination, created.Plan) + } +} + +func TestMigrationThreeTombstonesOnlyLegacyStageCreates(t *testing.T) { + path := testPath(t) + original := migrations + migrations = append([]migration(nil), original[:2]...) + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + in := createInput("legacy-create", "body") + if _, err = s.Create(context.Background(), in); err != nil { + t.Fatal(err) + } + if _, err = s.db.Exec(`INSERT INTO local_requests(server_url,user_id,request_id,request_schema,request_digest,result_json,created_at) VALUES(?,?,?,?,?,?,?)`, in.ServerURL, in.UserID, "revise-kept", "mm/v2/stage-revise-request", make([]byte, 32), `{}`, time.Now().UTC().Format(time.RFC3339Nano)); err != nil { + t.Fatal(err) + } + if err = s.Close(); err != nil { + t.Fatal(err) + } + migrations = original + t.Cleanup(func() { migrations = original }) + s, err = Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + var createSchema, reviseSchema string + if err = s.db.QueryRow(`SELECT request_schema FROM local_requests WHERE request_id='legacy-create'`).Scan(&createSchema); err != nil { + t.Fatal(err) + } + if err = s.db.QueryRow(`SELECT request_schema FROM local_requests WHERE request_id='revise-kept'`).Scan(&reviseSchema); err != nil { + t.Fatal(err) + } + if createSchema != "mm/v2/legacy-stage-request-conflict" || reviseSchema != "mm/v2/stage-revise-request" { + t.Fatalf("schemas = %s/%s", createSchema, reviseSchema) + } + if _, _, err = s.FindCreate(context.Background(), in.ServerURL, in.UserID, in.RequestID); !errors.Is(err, ErrConflict) { + t.Fatalf("legacy lookup = %v", err) + } } func reviseInput(stage StageSummary, request, body string) ReviseInput { content := createInput("", body).Content diff --git a/internal/stagestore/schema.go b/internal/stagestore/schema.go index 2aa44c0..af7f9f1 100644 --- a/internal/stagestore/schema.go +++ b/internal/stagestore/schema.go @@ -89,4 +89,8 @@ WHEN NEW.revision > 1 AND EXISTS(SELECT 1 FROM stage_revisions WHERE stage_id=NE AND (NEW.destination_json != (SELECT destination_json FROM stage_revisions WHERE stage_id=NEW.stage_id ORDER BY revision LIMIT 1) OR NEW.plan_json != (SELECT plan_json FROM stage_revisions WHERE stage_id=NEW.stage_id ORDER BY revision LIMIT 1)) BEGIN SELECT RAISE(ABORT, 'stage destination and plan are immutable'); END; +`}, {version: 3, name: "caller-intent-stage-create-replay", sql: ` +DROP TRIGGER local_requests_immutable_update; +UPDATE local_requests SET request_schema='mm/v2/legacy-stage-request-conflict' WHERE request_schema='mm/v2/stage-request'; +CREATE TRIGGER local_requests_immutable_update BEFORE UPDATE ON local_requests BEGIN SELECT RAISE(ABORT, 'local request receipts are immutable'); END; `}} diff --git a/internal/staging/conversation.go b/internal/staging/conversation.go index afa4b3e..b118df2 100644 --- a/internal/staging/conversation.go +++ b/internal/staging/conversation.go @@ -35,12 +35,19 @@ func (s *Service) resolveConversation(ctx context.Context, target Target) (Previ return Preview{}, ErrCredential } current, err := s.users.Current(ctx) - if err != nil || !validResolvedUser(current) { + if err != nil { + return Preview{}, targetReadError(err) + } + if !validResolvedUser(current) { return Preview{}, ErrTarget } if contaminated(s.credentials, current.ID, current.Username) { return Preview{}, ErrCredential } + return s.resolveConversationFor(ctx, target, current) +} + +func (s *Service) resolveConversationFor(ctx context.Context, target Target, current mattermost.User) (Preview, error) { channel, participants, err := s.resolveChannel(ctx, current, target) if err != nil { return Preview{}, err @@ -75,14 +82,20 @@ func (s *Service) resolveChannel(ctx context.Context, current mattermost.User, t switch target.Conversation { case Direct: peer, err := s.users.ByUsernameFresh(ctx, target.Value) - if err != nil || !validResolvedUser(peer) || !strings.EqualFold(peer.Username, target.Value) || peer.ID == current.ID { + if err != nil { + return mattermost.Channel{}, nil, targetReadError(err) + } + if !validResolvedUser(peer) || !strings.EqualFold(peer.Username, target.Value) || peer.ID == current.ID { return mattermost.Channel{}, nil, ErrTarget } if contaminated(s.credentials, peer.ID, peer.Username) { return mattermost.Channel{}, nil, ErrCredential } channel, found, err := s.channels.ExistingDirect(ctx, current.ID, peer.ID) - if err != nil || !found || !validResolvedChannel(channel) || channel.Type != "D" { + if err != nil { + return mattermost.Channel{}, nil, targetReadError(err) + } + if !found || !validResolvedChannel(channel) || channel.Type != "D" { return mattermost.Channel{}, nil, ErrTarget } if contaminated(s.credentials, channel.ID, channel.Name) { @@ -91,13 +104,20 @@ func (s *Service) resolveChannel(ctx context.Context, current mattermost.User, t return channel, []string{peer.ID}, nil case Group: channel, err := s.channels.ByID(ctx, target.Value) - if err != nil || !validResolvedChannel(channel) || channel.Type != "G" { + if err != nil { + return mattermost.Channel{}, nil, targetReadError(err) + } + if !validResolvedChannel(channel) || channel.Type != "G" { return mattermost.Channel{}, nil, ErrTarget } if contaminated(s.credentials, channel.ID, channel.Name) { return mattermost.Channel{}, nil, ErrCredential } - if _, err = s.channels.Member(ctx, channel.ID, current.ID); err != nil { + member, memberErr := s.channels.Member(ctx, channel.ID, current.ID) + if memberErr != nil { + return mattermost.Channel{}, nil, targetReadError(memberErr) + } + if member.ChannelID != channel.ID || member.UserID != current.ID { return mattermost.Channel{}, nil, ErrTarget } return channel, []string{}, nil @@ -113,13 +133,20 @@ func (s *Service) resolveChannel(ctx context.Context, current mattermost.User, t } channel, err = s.channels.ByName(ctx, team.ID, target.Value) } - if err != nil || !validResolvedChannel(channel) || (channel.Type != "O" && channel.Type != "P") { + if err != nil { + return mattermost.Channel{}, nil, targetReadError(err) + } + if !validResolvedChannel(channel) || (channel.Type != "O" && channel.Type != "P") { return mattermost.Channel{}, nil, ErrTarget } if contaminated(s.credentials, channel.ID, channel.Name, channel.TeamID) { return mattermost.Channel{}, nil, ErrCredential } - if _, err = s.channels.Member(ctx, channel.ID, current.ID); err != nil { + member, memberErr := s.channels.Member(ctx, channel.ID, current.ID) + if memberErr != nil { + return mattermost.Channel{}, nil, targetReadError(memberErr) + } + if member.ChannelID != channel.ID || member.UserID != current.ID { return mattermost.Channel{}, nil, ErrTarget } return channel, []string{}, nil @@ -131,7 +158,7 @@ func (s *Service) resolveChannel(ctx context.Context, current mattermost.User, t func (s *Service) resolveTeam(ctx context.Context, userID string, selector TeamSelector) (mattermost.Team, error) { membership, err := s.teams.List(ctx, userID) if err != nil { - return mattermost.Team{}, ErrTarget + return mattermost.Team{}, targetReadError(err) } var match mattermost.Team count := 0 diff --git a/internal/staging/intent.go b/internal/staging/intent.go new file mode 100644 index 0000000..beb2d4f --- /dev/null +++ b/internal/staging/intent.go @@ -0,0 +1,62 @@ +package staging + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + + "github.com/ardasevinc/mattermost-cli/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +type callerIntent struct { + Domain string `json:"domain"` + Operation stagestore.Operation `json:"operation"` + Target any `json:"target"` + Body *string `json:"body"` + Emoji *string `json:"emoji"` + Attachments []stageinput.MetadataIntent `json:"attachments"` +} + +func intentDigest(operation stagestore.Operation, target any, body []byte, emoji string, attachments []stageinput.MetadataIntent) [32]byte { + var bodyValue *string + if body != nil { + value := string(body) + bodyValue = &value + } + var emojiValue *string + if emoji != "" { + emojiValue = &emoji + } + value := callerIntent{"mm/v2/stage-request/caller-intent/v1", operation, target, bodyValue, emojiValue, attachments} + var out bytes.Buffer + encoder := json.NewEncoder(&out) + encoder.SetEscapeHTML(false) + _ = encoder.Encode(value) + return sha256.Sum256(restoreJSONLineSeparators(bytes.TrimSuffix(out.Bytes(), []byte{'\n'}))) +} + +type conversationIntent struct { + Conversation string `json:"conversation"` + Selector string `json:"selector"` + Value string `json:"value"` + Team *teamIntent `json:"team"` +} +type teamIntent struct { + Selector string `json:"selector"` + Value string `json:"value"` +} + +func conversationCallerIntent(target Target) conversationIntent { + conversation := map[ConversationType]string{Direct: "direct", Group: "group", Channel: "channel"}[target.Conversation] + selector := map[SelectorType]string{ByUsername: "username", ByID: "id", ByName: "name"}[target.Selector] + var team *teamIntent + if target.Team != nil { + team = &teamIntent{map[SelectorType]string{ByID: "id", ByName: "name"}[target.Team.By], target.Team.Value} + } + return conversationIntent{conversation, selector, target.Value, team} +} + +type postIntent struct { + PostID string `json:"postId"` +} diff --git a/internal/staging/intent_test.go b/internal/staging/intent_test.go new file mode 100644 index 0000000..b997372 --- /dev/null +++ b/internal/staging/intent_test.go @@ -0,0 +1,31 @@ +package staging + +import ( + "encoding/hex" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +func TestCallerIntentDigestGoldenAndEscapeAwareness(t *testing.T) { + target := conversationIntent{"direct", "username", "hakan", nil} + for _, test := range []struct{ body, want string }{ + {"x\u2028", "fd8d62e0c64b8321387d4abc316bef460ed1a84e8f909e5f0266032b06b89c9b"}, + {`x\u2028`, "23158033b33d3823fa7638c55b00df9383f0747539d231382f40154397dc9b19"}, + } { + digest := intentDigest(stagestore.CreatePost, target, []byte(test.body), "", []stageinput.MetadataIntent{}) + if got := hex.EncodeToString(digest[:]); got != test.want { + t.Fatalf("body %q digest = %s", test.body, got) + } + } +} + +func TestCallerIntentBindsOrderedNormalizedAttachmentMetadata(t *testing.T) { + a := "text/plain" + first := []stageinput.MetadataIntent{{Path: "/a", RemoteFilename: "a", MediaType: &a}, {Path: "/b", RemoteFilename: "b"}} + second := []stageinput.MetadataIntent{first[1], first[0]} + if intentDigest(stagestore.Reply, postIntent{"post"}, []byte("body"), "", first) == intentDigest(stagestore.Reply, postIntent{"post"}, []byte("body"), "", second) { + t.Fatal("attachment order was not bound") + } +} diff --git a/internal/staging/post.go b/internal/staging/post.go new file mode 100644 index 0000000..65dc3e0 --- /dev/null +++ b/internal/staging/post.go @@ -0,0 +1,508 @@ +package staging + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "strings" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/messageinput" + "github.com/ardasevinc/mattermost-cli/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +type postOperation struct { + operation stagestore.Operation + postID string + emoji string +} + +func (s *Service) DryRunReply(ctx context.Context, in PostDryRunInput) (Preview, error) { + return s.resolvePost(ctx, postOperation{operation: stagestore.Reply, postID: in.PostID}) +} + +func (s *Service) Reply(ctx context.Context, in ReplyInput) (CreatePostResult, error) { + if nilDependency(s.store) || s.bind == nil || in.Body == nil || !validRequestID(in.RequestID) { + return CreatePostResult{}, ErrInvalid + } + if contaminated(s.credentials, in.RequestID, in.PostID) || callerAttachmentsContaminated(s.credentials, in.Attachments) { + return CreatePostResult{}, ErrCredential + } + attachmentIntent, err := stageinput.Preflight(in.Attachments) + if err != nil { + return CreatePostResult{}, ErrInput + } + if attachmentIntentContaminated(s.credentials, attachmentIntent) { + return CreatePostResult{}, ErrCredential + } + op := postOperation{operation: stagestore.Reply, postID: in.PostID} + if err := s.validatePostOperation(ctx, op); err != nil { + return CreatePostResult{}, err + } + current, err := s.authenticate(ctx) + if err != nil { + return CreatePostResult{}, err + } + record, found, err := s.findCreate(ctx, current.ID, in.RequestID) + if err != nil { + return CreatePostResult{}, err + } + if found { + if record.Stage.Operation != stagestore.Reply { + return CreatePostResult{}, ErrConflict + } + body, readErr := messageinput.Read(in.Body) + if readErr != nil { + return CreatePostResult{}, ErrInput + } + if containsCredential(s.credentials, body) { + return CreatePostResult{}, ErrCredential + } + return replayResult(record, intentDigest(stagestore.Reply, postIntent{in.PostID}, body, "", attachmentIntent), stagestore.Reply, s.serverURL, current.ID) + } + preview, err := s.resolvePostFor(ctx, op, current) + if err != nil { + return CreatePostResult{}, err + } + body, err := messageinput.Read(in.Body) + if err != nil { + return CreatePostResult{}, ErrInput + } + if containsCredential(s.credentials, body) { + return CreatePostResult{}, ErrCredential + } + attachments, err := s.bind(ctx, in.Attachments, cloneCredentials(s.credentials)) + if err != nil { + if errors.Is(err, stageinput.ErrCredential) { + return CreatePostResult{}, ErrCredential + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return CreatePostResult{}, err + } + return CreatePostResult{}, ErrInput + } + if !validBoundAttachments(attachments) { + return CreatePostResult{}, ErrInput + } + preview.Plan = attachmentPlan(len(attachments)) + return s.persistPost(ctx, in.RequestID, intentDigest(stagestore.Reply, postIntent{in.PostID}, body, "", attachmentIntent), stagestore.Reply, preview, body, attachments) +} + +func (s *Service) DryRunEditPost(ctx context.Context, in PostDryRunInput) (Preview, error) { + return s.resolvePost(ctx, postOperation{operation: stagestore.EditPost, postID: in.PostID}) +} + +func (s *Service) EditPost(ctx context.Context, in EditPostInput) (CreatePostResult, error) { + if nilDependency(s.store) || in.Body == nil || !validRequestID(in.RequestID) { + return CreatePostResult{}, ErrInvalid + } + if contaminated(s.credentials, in.RequestID, in.PostID) { + return CreatePostResult{}, ErrCredential + } + op := postOperation{operation: stagestore.EditPost, postID: in.PostID} + if err := s.validatePostOperation(ctx, op); err != nil { + return CreatePostResult{}, err + } + current, err := s.authenticate(ctx) + if err != nil { + return CreatePostResult{}, err + } + record, found, err := s.findCreate(ctx, current.ID, in.RequestID) + if err != nil { + return CreatePostResult{}, err + } + if found { + if record.Stage.Operation != stagestore.EditPost { + return CreatePostResult{}, ErrConflict + } + body, readErr := messageinput.Read(in.Body) + if readErr != nil { + return CreatePostResult{}, ErrInput + } + if containsCredential(s.credentials, body) { + return CreatePostResult{}, ErrCredential + } + return replayResult(record, intentDigest(stagestore.EditPost, postIntent{in.PostID}, body, "", []stageinput.MetadataIntent{}), stagestore.EditPost, s.serverURL, current.ID) + } + preview, err := s.resolvePostFor(ctx, op, current) + if err != nil { + return CreatePostResult{}, err + } + body, err := messageinput.Read(in.Body) + if err != nil { + return CreatePostResult{}, ErrInput + } + if containsCredential(s.credentials, body) { + return CreatePostResult{}, ErrCredential + } + return s.persistPost(ctx, in.RequestID, intentDigest(stagestore.EditPost, postIntent{in.PostID}, body, "", []stageinput.MetadataIntent{}), stagestore.EditPost, preview, body, nil) +} + +func (s *Service) DryRunDeletePost(ctx context.Context, in PostDryRunInput) (Preview, error) { + return s.resolvePost(ctx, postOperation{operation: stagestore.DeletePost, postID: in.PostID}) +} + +func (s *Service) DeletePost(ctx context.Context, in DeletePostInput) (CreatePostResult, error) { + return s.persistContentless(ctx, in.RequestID, postOperation{operation: stagestore.DeletePost, postID: in.PostID}) +} + +func (s *Service) DryRunReact(ctx context.Context, in ReactionDryRunInput) (Preview, error) { + return s.resolvePost(ctx, postOperation{operation: stagestore.React, postID: in.PostID, emoji: in.Emoji}) +} + +func (s *Service) React(ctx context.Context, in ReactionInput) (CreatePostResult, error) { + return s.persistContentless(ctx, in.RequestID, postOperation{operation: stagestore.React, postID: in.PostID, emoji: in.Emoji}) +} + +func (s *Service) DryRunUnreact(ctx context.Context, in ReactionDryRunInput) (Preview, error) { + return s.resolvePost(ctx, postOperation{operation: stagestore.Unreact, postID: in.PostID, emoji: in.Emoji}) +} + +func (s *Service) Unreact(ctx context.Context, in ReactionInput) (CreatePostResult, error) { + return s.persistContentless(ctx, in.RequestID, postOperation{operation: stagestore.Unreact, postID: in.PostID, emoji: in.Emoji}) +} + +func (s *Service) persistContentless(ctx context.Context, requestID string, op postOperation) (CreatePostResult, error) { + if nilDependency(s.store) || !validRequestID(requestID) { + return CreatePostResult{}, ErrInvalid + } + if contaminated(s.credentials, requestID, op.postID, op.emoji) { + return CreatePostResult{}, ErrCredential + } + if err := s.validatePostOperation(ctx, op); err != nil { + return CreatePostResult{}, err + } + current, err := s.authenticate(ctx) + if err != nil { + return CreatePostResult{}, err + } + digest := intentDigest(op.operation, postIntent{op.postID}, nil, op.emoji, []stageinput.MetadataIntent{}) + record, found, err := s.findCreate(ctx, current.ID, requestID) + if err != nil { + return CreatePostResult{}, err + } + if found { + return replayResult(record, digest, op.operation, s.serverURL, current.ID) + } + preview, err := s.resolvePostFor(ctx, op, current) + if err != nil { + return CreatePostResult{}, err + } + return s.persistPost(ctx, requestID, digest, op.operation, preview, nil, nil) +} + +func (s *Service) resolvePost(ctx context.Context, op postOperation) (Preview, error) { + if err := s.validatePostOperation(ctx, op); err != nil { + return Preview{}, err + } + current, err := s.authenticate(ctx) + if err != nil { + return Preview{}, err + } + return s.resolvePostFor(ctx, op, current) +} + +func (s *Service) validatePostOperation(ctx context.Context, op postOperation) error { + allowed := op.operation == stagestore.Reply || op.operation == stagestore.EditPost || op.operation == stagestore.DeletePost || + op.operation == stagestore.React || op.operation == stagestore.Unreact + if ctx == nil || !validPostID(op.postID) || + !allowed || + ((op.operation == stagestore.React || op.operation == stagestore.Unreact) != (op.emoji != "")) || + (op.emoji != "" && !validEmoji(op.emoji)) || contaminated(s.credentials, op.postID, op.emoji) { + if contaminated(s.credentials, op.postID, op.emoji) { + return ErrCredential + } + return ErrInvalid + } + return nil +} + +func (s *Service) resolvePostFor(ctx context.Context, op postOperation, current mattermost.User) (Preview, error) { + post, err := s.posts.ByID(ctx, op.postID) + if err != nil { + return Preview{}, targetReadError(err) + } + if post.ID != op.postID || !validResolvedPost(post) { + return Preview{}, ErrTarget + } + if contaminated(s.credentials, post.ID, post.ChannelID, post.UserID, post.RootID) { + return Preview{}, ErrCredential + } + if (op.operation == stagestore.EditPost || op.operation == stagestore.DeletePost) && (post.UserID != current.ID || post.Type != "") { + return Preview{}, ErrTarget + } + canonicalRoot := post.RootID + if op.operation == stagestore.Reply && post.RootID == "" { + canonicalRoot = post.ID + } + if op.operation == stagestore.Reply && post.RootID != "" { + root, rootErr := s.posts.ByID(ctx, post.RootID) + if rootErr != nil { + return Preview{}, targetReadError(rootErr) + } + if root.ID != post.RootID || root.RootID != "" || root.ChannelID != post.ChannelID || !validResolvedPost(root) { + return Preview{}, ErrTarget + } + if contaminated(s.credentials, root.ID, root.ChannelID, root.UserID, root.RootID) { + return Preview{}, ErrCredential + } + canonicalRoot = root.ID + } + channel, err := s.channels.ByID(ctx, post.ChannelID) + if err != nil { + return Preview{}, targetReadError(err) + } + if channel.ID != post.ChannelID || !validResolvedChannel(channel) { + return Preview{}, ErrTarget + } + participants, ok := postParticipants(channel, current.ID) + if !ok { + return Preview{}, ErrTarget + } + if contaminated(s.credentials, channel.ID, channel.Name, channel.TeamID) || contaminated(s.credentials, participants...) { + return Preview{}, ErrCredential + } + member, err := s.channels.Member(ctx, channel.ID, current.ID) + if err != nil { + return Preview{}, targetReadError(err) + } + if member.ChannelID != channel.ID || member.UserID != current.ID { + return Preview{}, ErrTarget + } + var teamID *string + if channel.Type == "O" || channel.Type == "P" { + value := channel.TeamID + teamID = &value + } + postID := post.ID + destination := Destination{Kind: "post", ChannelID: channel.ID, ChannelType: channelType(channel.Type), TeamID: teamID, + PostID: &postID, ParticipantIDs: participants} + if canonicalRoot != "" { + destination.RootPostID = &canonicalRoot + } + switch op.operation { + case stagestore.Reply: + case stagestore.EditPost, stagestore.DeletePost: + destination.PostState = &PostState{AuthorUserID: post.UserID, UpdateAt: post.UpdateAt, ContentDigest: digestPost(post, s.credentials)} + case stagestore.React, stagestore.Unreact: + present, reactionErr := s.posts.ReactionState(ctx, post.ID, channel.ID, current.ID, op.emoji) + if reactionErr != nil { + return Preview{}, targetReadError(reactionErr) + } + destination.Kind, destination.Emoji, destination.ReactionPresent = "reaction", &op.emoji, &present + } + preview := Preview{ServerURL: s.serverURL, ServerID: s.serverID, UserID: current.ID, Destination: destination, Plan: postPlan(op.operation)} + destinationJSON, planJSON, err := marshalSemantics(preview) + if err != nil { + return Preview{}, ErrInvalid + } + if contaminated(s.credentials, preview.ServerURL, preview.ServerID, preview.UserID, string(destinationJSON), string(planJSON)) { + return Preview{}, ErrCredential + } + return preview, nil +} + +func (s *Service) persistPost(ctx context.Context, requestID string, requestDigest [32]byte, operation stagestore.Operation, preview Preview, body []byte, attachments []stagestore.Attachment) (CreatePostResult, error) { + destination, plan, err := marshalSemantics(preview) + if err != nil { + return CreatePostResult{}, ErrInvalid + } + if contaminated(s.credentials, requestID, string(destination), string(plan)) || containsCredential(s.credentials, body) || attachmentsContaminated(s.credentials, attachments) { + return CreatePostResult{}, ErrCredential + } + stored, err := s.store.Create(ctx, stagestore.CreateInput{RequestID: requestID, RequestDigest: requestDigest, Operation: operation, ServerURL: preview.ServerURL, + ServerID: preview.ServerID, UserID: preview.UserID, Content: stagestore.RevisionContent{Body: body, Destination: destination, Plan: plan, Attachments: attachments}}) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return CreatePostResult{}, err + } + if errors.Is(err, stagestore.ErrConflict) { + return CreatePostResult{}, ErrConflict + } + return CreatePostResult{}, ErrStore + } + return replayResult(stored, requestDigest, operation, preview.ServerURL, preview.UserID) +} + +func (s *Service) findCreate(ctx context.Context, userID, requestID string) (stagestore.CreateRecord, bool, error) { + record, found, err := s.store.FindCreate(ctx, s.serverURL, userID, requestID) + if err == nil { + return record, found, nil + } + if errors.Is(err, stagestore.ErrConflict) { + return stagestore.CreateRecord{}, false, ErrConflict + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return stagestore.CreateRecord{}, false, err + } + return stagestore.CreateRecord{}, false, ErrStore +} + +func digestPost(post mattermost.Post, credentials [][]byte) string { + canonical := struct { + Message any `json:"message"` + FileIDs []string `json:"fileIds"` + RootID string `json:"rootId"` + Type string `json:"type"` + }{credentialSafePostMessage(post.Message, credentials), post.FileIDs, post.RootID, post.Type} + var buffer bytes.Buffer + encoder := json.NewEncoder(&buffer) + encoder.SetEscapeHTML(false) + _ = encoder.Encode(canonical) + encoded := bytes.TrimSuffix(buffer.Bytes(), []byte{'\n'}) + encoded = restoreJSONLineSeparators(encoded) + digest := sha256.Sum256(encoded) + return hex.EncodeToString(digest[:]) +} + +func credentialSafePostMessage(message string, credentials [][]byte) any { + source := []byte(message) + protected := credentialsByLength(credentials) + fragments := make([]string, 0) + start := 0 + for cursor := 0; cursor < len(source); { + matched := 0 + for _, credential := range protected { + if len(credential) <= len(source)-cursor && bytes.Equal(source[cursor:cursor+len(credential)], credential) { + matched = len(credential) + break + } + } + if matched == 0 { + cursor++ + continue + } + fragments = append(fragments, string(source[start:cursor])) + cursor += matched + start = cursor + } + if len(fragments) == 0 { + return message + } + fragments = append(fragments, string(source[start:])) + return struct { + Fragments []string `json:"credentialElidedFragments"` + }{fragments} +} + +func targetReadError(err error) error { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + return ErrTarget +} + +func postPlan(operation stagestore.Operation) Plan { + typeName, condition := string(operation), "always" + switch operation { + case stagestore.Reply: + typeName = "create_post" + case stagestore.React: + typeName, condition = "add_reaction", "if_missing" + case stagestore.Unreact: + typeName, condition = "remove_reaction", "if_missing" + } + return Plan{Steps: []PlanStep{{Ordinal: 1, Type: typeName, Condition: condition}}} +} + +func postParticipants(channel mattermost.Channel, currentID string) ([]string, bool) { + if channel.Type != "D" { + return []string{}, true + } + parts := strings.Split(channel.Name, "__") + if len(parts) != 2 || !validSelectorValue(parts[0]) || !validSelectorValue(parts[1]) { + return nil, false + } + if parts[0] == currentID && parts[1] == currentID { + return []string{currentID}, true + } + if parts[0] == currentID { + return []string{parts[1]}, true + } + if parts[1] == currentID { + return []string{parts[0]}, true + } + return nil, false +} + +func validResolvedPost(post mattermost.Post) bool { + authorOK := validPostID(post.UserID) || post.UserID == "" && strings.HasPrefix(post.Type, "system_") + return validPostID(post.ID) && validPostID(post.ChannelID) && authorOK && + (post.RootID == "" || validPostID(post.RootID)) && post.UpdateAt > 0 && post.UpdateAt <= 8_640_000_000_000_000 && post.FileIDs != nil +} + +func validPostID(value string) bool { + if len(value) == 0 || len(value) > 128 { + return false + } + for i := range len(value) { + c := value[i] + if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '-') { + return false + } + } + return true +} + +func restoreJSONLineSeparators(encoded []byte) []byte { + result := make([]byte, 0, len(encoded)) + for i := 0; i < len(encoded); { + if i+6 <= len(encoded) && (bytes.Equal(encoded[i:i+6], []byte(`\u2028`)) || bytes.Equal(encoded[i:i+6], []byte(`\u2029`))) { + backslashes := 0 + for j := i - 1; j >= 0 && encoded[j] == '\\'; j-- { + backslashes++ + } + if backslashes%2 == 0 { + if encoded[i+5] == '8' { + result = append(result, "\u2028"...) + } else { + result = append(result, "\u2029"...) + } + i += 6 + continue + } + } + result = append(result, encoded[i]) + i++ + } + return result +} + +func validEmoji(value string) bool { + if len(value) == 0 || len(value) > 64 { + return false + } + for i := range len(value) { + c := value[i] + if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '-' || c == '+') { + return false + } + } + return true +} + +func callerAttachmentsContaminated(credentials [][]byte, values []Attachment) bool { + for _, value := range values { + if contaminated(credentials, value.Path, value.RemoteFilename, value.MediaType) { + return true + } + } + return false +} + +func attachmentIntentContaminated(credentials [][]byte, values []stageinput.MetadataIntent) bool { + for _, value := range values { + mediaType := "" + if value.MediaType != nil { + mediaType = *value.MediaType + } + if contaminated(credentials, value.Path, value.RemoteFilename, mediaType) { + return true + } + } + return false +} diff --git a/internal/staging/post_test.go b/internal/staging/post_test.go new file mode 100644 index 0000000..e99cbdb --- /dev/null +++ b/internal/staging/post_test.go @@ -0,0 +1,580 @@ +package staging + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +type fakePosts struct { + posts map[string]mattermost.Post + present bool + err error + calls *[]string + reaction []string +} + +func (f *fakePosts) ByID(_ context.Context, id string) (mattermost.Post, error) { + *f.calls = append(*f.calls, "post:"+id) + if f.err != nil { + return mattermost.Post{}, f.err + } + return f.posts[id], nil +} +func (f *fakePosts) ReactionState(_ context.Context, postID, channelID, userID, emoji string) (bool, error) { + *f.calls = append(*f.calls, "reaction") + f.reaction = []string{postID, channelID, userID, emoji} + return f.present, f.err +} + +type orderedUsers struct { + user mattermost.User + calls *[]string +} + +func (f orderedUsers) Current(context.Context) (mattermost.User, error) { + *f.calls = append(*f.calls, "current") + return f.user, nil +} +func (orderedUsers) ByUsernameFresh(context.Context, string) (mattermost.User, error) { + return mattermost.User{}, errors.New("unused") +} + +type orderedChannels struct { + channel mattermost.Channel + calls *[]string +} + +func (f orderedChannels) ExistingDirect(context.Context, string, string) (mattermost.Channel, bool, error) { + return mattermost.Channel{}, false, errors.New("unused") +} +func (f orderedChannels) ByID(_ context.Context, id string) (mattermost.Channel, error) { + *f.calls = append(*f.calls, "channel:"+id) + return f.channel, nil +} +func (f orderedChannels) ByName(context.Context, string, string) (mattermost.Channel, error) { + return mattermost.Channel{}, errors.New("unused") +} +func (f orderedChannels) Member(_ context.Context, channelID, userID string) (mattermost.ChannelMember, error) { + *f.calls = append(*f.calls, "member") + return mattermost.ChannelMember{ChannelID: channelID, UserID: userID}, nil +} + +func postService(t *testing.T, post mattermost.Post, channel mattermost.Channel, store Store) (*Service, *fakePosts, *[]string) { + t.Helper() + calls := []string{} + posts := &fakePosts{posts: map[string]mattermost.Post{post.ID: post}, calls: &calls} + service, err := New("https://mattermost.example", "", nil, + orderedUsers{mattermost.User{ID: "user-1", Username: "arda"}, &calls}, orderedChannels{channel, &calls}, emptyTeams{}, posts, store) + if err != nil { + t.Fatal(err) + } + return service, posts, &calls +} + +func ordinaryPost() mattermost.Post { + return mattermost.Post{ID: "post-1", ChannelID: "channel-1", UserID: "user-1", Message: "old", CreateAt: 1, UpdateAt: 2, FileIDs: []string{"file-1"}} +} + +func TestReplyResolvesCanonicalRootBeforeBodyAndPersistsOnce(t *testing.T) { + store := &recordingStore{} + post := ordinaryPost() + post.RootID = "root-1" + service, posts, calls := postService(t, post, mattermost.Channel{ID: "channel-1", TeamID: "team-1", Type: "P", Name: "private"}, store) + posts.posts["root-1"] = mattermost.Post{ID: "root-1", ChannelID: "channel-1", UserID: "other", Message: "root", CreateAt: 1, UpdateAt: 1, FileIDs: []string{}} + reader := &panicReader{} + _, err := service.Reply(context.Background(), ReplyInput{RequestID: "request-1", PostID: "post-1", Body: reader}) + if !errors.Is(err, ErrInput) || !reader.read || store.calls != 0 { + t.Fatalf("error/read/store = %v/%v/%d", err, reader.read, store.calls) + } + wantOrder := []string{"current", "post:post-1", "post:root-1", "channel:channel-1", "member"} + if !reflect.DeepEqual(*calls, wantOrder) { + t.Fatalf("read order = %v", *calls) + } + + result, err := service.Reply(context.Background(), ReplyInput{RequestID: "request-2", PostID: "post-1", Body: bytes.NewBufferString("reply")}) + if err != nil || store.calls != 1 || result.Preview.Destination.RootPostID == nil || *result.Preview.Destination.RootPostID != "root-1" { + t.Fatalf("result/error/store = %#v/%v/%d", result, err, store.calls) + } + if got := string(store.in.Content.Destination); got != `{"kind":"post","channelId":"channel-1","channelType":"private","teamId":"team-1","postId":"post-1","rootPostId":"root-1","participantIds":[],"emoji":null,"postState":null,"reactionPresent":null}` { + t.Fatalf("destination = %s", got) + } +} + +func TestEditBindsCanonicalDigestAndSchemaValidPreview(t *testing.T) { + store := &recordingStore{} + post := ordinaryPost() + service, _, _ := postService(t, post, mattermost.Channel{ID: "channel-1", TeamID: "team-1", Type: "O", Name: "town-square"}, store) + result, err := service.EditPost(context.Background(), EditPostInput{RequestID: "edit-1", PostID: "post-1", Body: bytes.NewBufferString("new")}) + if err != nil { + t.Fatal(err) + } + wire := []byte(`{"message":"old","fileIds":["file-1"],"rootId":"","type":""}`) + want := sha256.Sum256(wire) + if result.Preview.Destination.PostState == nil || result.Preview.Destination.PostState.ContentDigest != hex.EncodeToString(want[:]) || result.Preview.Destination.PostState.UpdateAt != 2 { + t.Fatalf("post state = %#v", result.Preview.Destination.PostState) + } + document := struct { + Schema string `json:"schema"` + Persist bool `json:"persist"` + Operation string `json:"operation"` + Binding any `json:"binding"` + Destination Destination `json:"destination"` + Plan Plan `json:"plan"` + ContentValidated bool `json:"contentValidated"` + }{"mm/v2/stage-preview", false, "edit_post", struct { + ServerURL string `json:"serverUrl"` + ServerID *string `json:"serverId"` + UserID string `json:"userId"` + }{result.Preview.ServerURL, nil, result.Preview.UserID}, result.Preview.Destination, result.Preview.Plan, false} + encoded, _ := json.Marshal(document) + registry, loadErr := schema.Load() + if loadErr != nil { + t.Fatal(loadErr) + } + if err := registry.Validate("mm/v2/stage-preview", bytes.NewReader(encoded)); err != nil { + t.Fatalf("preview rejected: %v\n%s", err, encoded) + } +} + +func TestReactionUsesAuthoritativeStateAndExactBinding(t *testing.T) { + store := &recordingStore{} + post := ordinaryPost() + post.RootID = "root-1" + service, posts, calls := postService(t, post, mattermost.Channel{ID: "channel-1", Type: "D", Name: "user-1__user-1"}, store) + posts.present = true + result, err := service.React(context.Background(), ReactionInput{RequestID: "react-1", PostID: "post-1", Emoji: "Eyes"}) + if err != nil || result.Preview.Destination.ReactionPresent == nil || !*result.Preview.Destination.ReactionPresent { + t.Fatalf("result/error = %#v/%v", result, err) + } + if !reflect.DeepEqual(posts.reaction, []string{"post-1", "channel-1", "user-1", "Eyes"}) || !reflect.DeepEqual(result.Preview.Destination.ParticipantIDs, []string{"user-1"}) { + t.Fatalf("reaction/participants = %v/%v", posts.reaction, result.Preview.Destination.ParticipantIDs) + } + if result.Preview.Destination.RootPostID == nil || *result.Preview.Destination.RootPostID != "root-1" { + t.Fatalf("root binding = %#v", result.Preview.Destination.RootPostID) + } + wantOrder := []string{"current", "post:post-1", "channel:channel-1", "member", "reaction"} + if !reflect.DeepEqual(*calls, wantOrder) || string(store.in.Content.Plan) != `{"steps":[{"ordinal":1,"type":"add_reaction","condition":"if_missing"}]}` { + t.Fatalf("order/plan = %v/%s", *calls, store.in.Content.Plan) + } +} + +func TestPostDigestUsesCanonicalUTF8JSONAndPreservesFileOrder(t *testing.T) { + post := mattermost.Post{Message: "<&>\u2028\u2029", FileIDs: []string{"b", "a"}, RootID: "root"} + if got, want := digestPost(post, nil), "b570850c2b2a6334cb6ad60168d84baf9b0d6198872b58fab3c3646413d3f99e"; got != want { + t.Fatalf("digest = %s, want %s", got, want) + } + post.FileIDs = []string{"a", "b"} + if digestPost(post, nil) == "b570850c2b2a6334cb6ad60168d84baf9b0d6198872b58fab3c3646413d3f99e" { + t.Fatal("file order was not bound") + } + post.Message, post.FileIDs = `<&>\u2028\u2029`, []string{"b", "a"} + if got, want := digestPost(post, nil), "92065e494852d7c2196f8f53227cbbfa7f901ee37beabb6fb84b4fdebe91ada0"; got != want { + t.Fatalf("literal escape digest = %s, want %s", got, want) + } +} + +func TestInvalidPostIDsAreZeroNetwork(t *testing.T) { + service, _, calls := postService(t, ordinaryPost(), mattermost.Channel{ID: "channel-1", Type: "G", Name: "group"}, &recordingStore{}) + for _, id := range []string{"bad/id", "bad\u202e", strings.Repeat("a", 129)} { + *calls = nil + if _, err := service.DryRunDeletePost(context.Background(), PostDryRunInput{PostID: id}); !errors.Is(err, ErrInvalid) || len(*calls) != 0 { + t.Fatalf("id/error/calls = %q/%v/%v", id, err, *calls) + } + } +} + +func TestForeignAndSystemEditDeleteStopBeforeChannelRead(t *testing.T) { + for _, post := range []mattermost.Post{ + {ID: "post-1", ChannelID: "channel-1", UserID: "other", UpdateAt: 1, FileIDs: []string{}}, + {ID: "post-1", ChannelID: "channel-1", UserID: "", Type: "system_join_channel", UpdateAt: 1, FileIDs: []string{}}, + } { + service, _, calls := postService(t, post, mattermost.Channel{ID: "channel-1", Type: "G", Name: "group"}, &recordingStore{}) + if _, err := service.DryRunDeletePost(context.Background(), PostDryRunInput{PostID: "post-1"}); !errors.Is(err, ErrTarget) || !reflect.DeepEqual(*calls, []string{"current", "post:post-1"}) { + t.Fatalf("post/error/calls = %#v/%v/%v", post, err, *calls) + } + } +} + +func TestResolvedPostBoundsAndIdentityFailClosed(t *testing.T) { + for _, alter := range []func(*mattermost.Post){ + func(p *mattermost.Post) { p.UpdateAt = 8_640_000_000_000_001 }, + func(p *mattermost.Post) { p.ChannelID = "bad/channel" }, + func(p *mattermost.Post) { p.RootID = "bad/root" }, + } { + post := ordinaryPost() + alter(&post) + service, _, calls := postService(t, post, mattermost.Channel{ID: "channel-1", Type: "G", Name: "group"}, &recordingStore{}) + if _, err := service.DryRunReact(context.Background(), ReactionDryRunInput{PostID: "post-1", Emoji: "wave"}); !errors.Is(err, ErrTarget) || !reflect.DeepEqual(*calls, []string{"current", "post:post-1"}) { + t.Fatalf("post/error/calls = %#v/%v/%v", post, err, *calls) + } + } +} + +func TestReplyAttachmentMetadataPreflightIsZeroNetworkAndFileIO(t *testing.T) { + for _, attachment := range []Attachment{{Path: "missing", RemoteFilename: "../x"}, {Path: "missing", RemoteFilename: "a/b"}, {Path: "missing", MediaType: "not mime"}} { + service, _, calls := postService(t, ordinaryPost(), mattermost.Channel{ID: "channel-1", Type: "G", Name: "group"}, &recordingStore{}) + reader := &panicReader{} + if _, err := service.Reply(context.Background(), ReplyInput{RequestID: "reply-1", PostID: "post-1", Body: reader, Attachments: []Attachment{attachment}}); !errors.Is(err, ErrInput) || len(*calls) != 0 || reader.read { + t.Fatalf("attachment/error/calls/read = %#v/%v/%v/%v", attachment, err, *calls, reader.read) + } + } +} + +func TestAuthorlessSystemPostAllowsReplyAndReaction(t *testing.T) { + post := mattermost.Post{ID: "post-1", ChannelID: "channel-1", Type: "system_join_channel", UpdateAt: 1, FileIDs: []string{}} + service, posts, _ := postService(t, post, mattermost.Channel{ID: "channel-1", Type: "G", Name: "group"}, &recordingStore{}) + if _, err := service.DryRunReply(context.Background(), PostDryRunInput{PostID: "post-1"}); err != nil { + t.Fatalf("reply = %v", err) + } + posts.present = false + if _, err := service.DryRunReact(context.Background(), ReactionDryRunInput{PostID: "post-1", Emoji: "wave"}); err != nil { + t.Fatalf("react = %v", err) + } +} + +func TestEditCanRemediateCredentialInExistingRemoteMessage(t *testing.T) { + const token = "leaked-active-credential" + post := ordinaryPost() + post.Message = "please remove " + token + store := &recordingStore{} + service, _, _ := postService(t, post, mattermost.Channel{ID: "channel-1", Type: "G", Name: "group"}, store) + service.credentials = [][]byte{[]byte(token)} + if _, err := service.EditPost(context.Background(), EditPostInput{"edit-remediation", "post-1", bytes.NewBufferString("credential removed")}); err != nil { + t.Fatal(err) + } + if bytes.Contains(store.in.Content.Destination, []byte(token)) || bytes.Contains(store.in.Content.Plan, []byte(token)) { + t.Fatal("remote credential leaked into staged semantics") + } + masked := digestPost(post, [][]byte{[]byte(token)}) + if masked == digestPost(post, nil) || !bytes.Contains(store.in.Content.Destination, []byte(masked)) { + t.Fatal("post binding did not mask the active credential before hashing") + } + other := post + other.Message = "please remove another-active-credential" + if digestPost(other, [][]byte{[]byte("another-active-credential")}) != masked { + t.Fatal("post binding remains suitable for offline credential guessing") + } +} + +func TestPostMutationRejectsCallerCredentialBeforeNetwork(t *testing.T) { + post := ordinaryPost() + service, posts, calls := postService(t, post, mattermost.Channel{ID: "channel-1", Type: "G", Name: "group"}, &recordingStore{}) + service.credentials = [][]byte{[]byte("secret")} + _, err := service.React(context.Background(), ReactionInput{RequestID: "request", PostID: "post-1", Emoji: "secret"}) + if !errors.Is(err, ErrCredential) || len(*calls) != 0 || len(posts.reaction) != 0 { + t.Fatalf("error/calls = %v/%v", err, *calls) + } +} + +func TestEditAndDeleteRejectForeignOrSystemPosts(t *testing.T) { + for _, alter := range []func(*mattermost.Post){ + func(p *mattermost.Post) { p.UserID = "other" }, + func(p *mattermost.Post) { p.Type = "system_join_channel" }, + } { + post := ordinaryPost() + alter(&post) + store := &recordingStore{} + service, _, _ := postService(t, post, mattermost.Channel{ID: "channel-1", Type: "G", Name: "group"}, store) + if _, err := service.DeletePost(context.Background(), DeletePostInput{RequestID: "delete-1", PostID: "post-1"}); !errors.Is(err, ErrTarget) || store.calls != 0 { + t.Fatalf("post/error/store = %#v/%v/%d", post, err, store.calls) + } + } +} + +func TestDryRunsNeverReadBodyBindFilesOrPersist(t *testing.T) { + store := &recordingStore{} + service, _, _ := postService(t, ordinaryPost(), mattermost.Channel{ID: "channel-1", Type: "G", Name: "group"}, store) + service = service.WithAttachmentBinder(func(context.Context, []Attachment, [][]byte) ([]stagestore.Attachment, error) { + t.Fatal("binder called") + return nil, nil + }) + if _, err := service.DryRunReply(context.Background(), PostDryRunInput{PostID: "post-1"}); err != nil || store.calls != 0 { + t.Fatalf("error/store = %v/%d", err, store.calls) + } +} + +func TestAllPostMutationDryRunProducersValidate(t *testing.T) { + store := &recordingStore{} + service, posts, _ := postService(t, ordinaryPost(), mattermost.Channel{ID: "channel-1", TeamID: "team-1", Type: "O", Name: "town-square"}, store) + tests := []struct { + operation string + run func() (Preview, error) + }{ + {"reply", func() (Preview, error) { return service.DryRunReply(context.Background(), PostDryRunInput{"post-1"}) }}, + {"edit_post", func() (Preview, error) { + return service.DryRunEditPost(context.Background(), PostDryRunInput{"post-1"}) + }}, + {"delete_post", func() (Preview, error) { + return service.DryRunDeletePost(context.Background(), PostDryRunInput{"post-1"}) + }}, + {"react", func() (Preview, error) { + return service.DryRunReact(context.Background(), ReactionDryRunInput{"post-1", "wave"}) + }}, + {"unreact", func() (Preview, error) { + return service.DryRunUnreact(context.Background(), ReactionDryRunInput{"post-1", "wave"}) + }}, + } + registry, err := schema.Load() + if err != nil { + t.Fatal(err) + } + posts.present = true + for _, test := range tests { + t.Run(test.operation, func(t *testing.T) { + preview, runErr := test.run() + if runErr != nil { + t.Fatal(runErr) + } + document := struct { + Schema, Operation string + Persist bool + Binding any + Destination Destination + Plan Plan + ContentValidated bool + }{"mm/v2/stage-preview", test.operation, false, struct { + ServerURL string `json:"serverUrl"` + ServerID *string `json:"serverId"` + UserID string `json:"userId"` + }{preview.ServerURL, nil, preview.UserID}, preview.Destination, preview.Plan, false} + encoded, marshalErr := json.Marshal(struct { + Schema string `json:"schema"` + Persist bool `json:"persist"` + Operation string `json:"operation"` + Binding any `json:"binding"` + Destination Destination `json:"destination"` + Plan Plan `json:"plan"` + ContentValidated bool `json:"contentValidated"` + }{document.Schema, document.Persist, document.Operation, document.Binding, document.Destination, document.Plan, document.ContentValidated}) + if marshalErr != nil || registry.Validate("mm/v2/stage-preview", bytes.NewReader(encoded)) != nil { + t.Fatalf("producer rejected: %v\n%s", marshalErr, encoded) + } + }) + } + if store.calls != 0 { + t.Fatalf("dry-run store calls = %d", store.calls) + } +} + +func TestDeleteAndReactionPersistenceUseExactOperationsPlansAndState(t *testing.T) { + store := &recordingStore{} + service, posts, _ := postService(t, ordinaryPost(), mattermost.Channel{ID: "channel-1", Type: "G", Name: "group"}, store) + if _, err := service.DeletePost(context.Background(), DeletePostInput{"delete-1", "post-1"}); err != nil || store.in.Operation != stagestore.DeletePost || string(store.in.Content.Plan) != `{"steps":[{"ordinal":1,"type":"delete_post","condition":"always"}]}` || len(store.in.Content.Body) != 0 || len(store.in.Content.Attachments) != 0 { + t.Fatalf("delete/error = %#v/%v", store.in, err) + } + const deleteDestination = `{"kind":"post","channelId":"channel-1","channelType":"group","teamId":null,"postId":"post-1","rootPostId":null,"participantIds":[],"emoji":null,"postState":{"authorUserId":"user-1","updateAt":2,"contentDigest":"f21e180b300ba23e417df2f2dc5f9da53cf8bd538c7863eebe9c186dadf8c35f"},"reactionPresent":null}` + if string(store.in.Content.Destination) != deleteDestination { + t.Fatalf("delete destination = %s", store.in.Content.Destination) + } + for _, operation := range []stagestore.Operation{stagestore.React, stagestore.Unreact} { + for _, present := range []bool{false, true} { + posts.present = present + requestID := string(operation) + map[bool]string{false: "-absent", true: "-present"}[present] + var err error + if operation == stagestore.React { + _, err = service.React(context.Background(), ReactionInput{requestID, "post-1", "wave"}) + } else { + _, err = service.Unreact(context.Background(), ReactionInput{requestID, "post-1", "wave"}) + } + typeName := map[stagestore.Operation]string{stagestore.React: "add_reaction", stagestore.Unreact: "remove_reaction"}[operation] + wantPlan := `{"steps":[{"ordinal":1,"type":"` + typeName + `","condition":"if_missing"}]}` + wantDestination := `{"kind":"reaction","channelId":"channel-1","channelType":"group","teamId":null,"postId":"post-1","rootPostId":null,"participantIds":[],"emoji":"wave","postState":null,"reactionPresent":` + map[bool]string{false: "false", true: "true"}[present] + `}` + if err != nil || store.in.Operation != operation || string(store.in.Content.Plan) != wantPlan || string(store.in.Content.Destination) != wantDestination || len(store.in.Content.Body) != 0 || len(store.in.Content.Attachments) != 0 { + t.Fatalf("operation/present/input/error = %s/%v/%#v/%v", operation, present, store.in, err) + } + } + } +} + +func TestStoreCancellationIsPreserved(t *testing.T) { + for _, sentinel := range []error{context.Canceled, context.DeadlineExceeded} { + store := &recordingStore{err: sentinel} + service, _, _ := postService(t, ordinaryPost(), mattermost.Channel{ID: "channel-1", Type: "G", Name: "group"}, store) + if _, err := service.DeletePost(context.Background(), DeletePostInput{"delete-1", "post-1"}); !errors.Is(err, sentinel) || errors.Is(err, ErrStore) { + t.Fatalf("post cancellation = %v", err) + } + create, _, _ := dmService(t, store) + if _, err := create.CreatePost(context.Background(), CreatePostInput{RequestID: "create-1", Target: dmTarget(), Body: bytes.NewBufferString("hello")}); !errors.Is(err, sentinel) || errors.Is(err, ErrStore) { + t.Fatalf("create cancellation = %v", err) + } + + readService, posts, _ := postService(t, ordinaryPost(), mattermost.Channel{ID: "channel-1", Type: "G", Name: "group"}, &recordingStore{}) + posts.err = sentinel + if _, err := readService.DryRunDeletePost(context.Background(), PostDryRunInput{"post-1"}); !errors.Is(err, sentinel) || errors.Is(err, ErrTarget) { + t.Fatalf("target-read cancellation = %v", err) + } + + binderService, _, _ := postService(t, ordinaryPost(), mattermost.Channel{ID: "channel-1", Type: "G", Name: "group"}, &recordingStore{}) + binderService = binderService.WithAttachmentBinder(func(context.Context, []Attachment, [][]byte) ([]stagestore.Attachment, error) { return nil, sentinel }) + if _, err := binderService.Reply(context.Background(), ReplyInput{RequestID: "reply-1", PostID: "post-1", Body: bytes.NewBufferString("hello"), Attachments: []Attachment{{Path: "safe"}}}); !errors.Is(err, sentinel) || errors.Is(err, ErrInput) { + t.Fatalf("reply binder cancellation = %v", err) + } + + conversation, users, channels := dmService(t, &recordingStore{}) + users.err = sentinel + if _, err := conversation.DryRunCreatePost(context.Background(), DryRunInput{Target: dmTarget()}); !errors.Is(err, sentinel) || errors.Is(err, ErrTarget) { + t.Fatalf("conversation-auth cancellation = %v", err) + } + users.err = nil + channels.err = sentinel + if _, err := conversation.CreatePost(context.Background(), CreatePostInput{RequestID: "conversation-1", Target: dmTarget(), Body: bytes.NewBufferString("hello")}); !errors.Is(err, sentinel) || errors.Is(err, ErrTarget) { + t.Fatalf("conversation-target cancellation = %v", err) + } + } +} + +func realStageStore(t *testing.T) *stagestore.Store { + t.Helper() + dir, err := os.MkdirTemp(".", ".staging-replay-") + if err != nil { + t.Fatal(err) + } + dir, err = filepath.Abs(dir) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + store, err := stagestore.Open(context.Background(), filepath.Join(dir, stagestore.DatabaseFilename)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + return store +} + +func TestRealStorePostMutationReplaysIgnoreRemoteDrift(t *testing.T) { + store := realStageStore(t) + service, posts, calls := postService(t, ordinaryPost(), mattermost.Channel{ID: "channel-1", Type: "G", Name: "group"}, store) + tests := []struct { + name string + first func() (CreatePostResult, error) + replay func() (CreatePostResult, error) + }{ + {"reply", func() (CreatePostResult, error) { + return service.Reply(context.Background(), ReplyInput{RequestID: "replay-reply", PostID: "post-1", Body: bytes.NewBufferString("body")}) + }, func() (CreatePostResult, error) { + return service.Reply(context.Background(), ReplyInput{RequestID: "replay-reply", PostID: "post-1", Body: bytes.NewBufferString("body")}) + }}, + {"edit", func() (CreatePostResult, error) { + return service.EditPost(context.Background(), EditPostInput{"replay-edit", "post-1", bytes.NewBufferString("new")}) + }, func() (CreatePostResult, error) { + return service.EditPost(context.Background(), EditPostInput{"replay-edit", "post-1", bytes.NewBufferString("new")}) + }}, + {"delete", func() (CreatePostResult, error) { + return service.DeletePost(context.Background(), DeletePostInput{"replay-delete", "post-1"}) + }, func() (CreatePostResult, error) { + return service.DeletePost(context.Background(), DeletePostInput{"replay-delete", "post-1"}) + }}, + {"react", func() (CreatePostResult, error) { + posts.present = false + return service.React(context.Background(), ReactionInput{"replay-react", "post-1", "wave"}) + }, func() (CreatePostResult, error) { + return service.React(context.Background(), ReactionInput{"replay-react", "post-1", "wave"}) + }}, + {"unreact", func() (CreatePostResult, error) { + posts.present = true + return service.Unreact(context.Background(), ReactionInput{"replay-unreact", "post-1", "wave"}) + }, func() (CreatePostResult, error) { + return service.Unreact(context.Background(), ReactionInput{"replay-unreact", "post-1", "wave"}) + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + posts.err = nil + first, err := test.first() + if err != nil { + t.Fatal(err) + } + firstDestination, _ := json.Marshal(first.Preview.Destination) + before := len(*calls) + posts.err = errors.New("remote deleted") + posts.present = !posts.present + replay, err := test.replay() + if err != nil || !replay.Stored.Replay || replay.Stored.Stage.ID != first.Stored.Stage.ID { + t.Fatalf("replay/error = %#v/%v", replay, err) + } + replayDestination, _ := json.Marshal(replay.Preview.Destination) + if !bytes.Equal(firstDestination, replayDestination) || len(*calls) != before+1 || (*calls)[before] != "current" { + t.Fatalf("destination/calls drift = %s/%s/%v", firstDestination, replayDestination, (*calls)[before:]) + } + }) + } +} + +func TestRealStoreReplyReplayDoesNotOpenMissingAttachment(t *testing.T) { + store := realStageStore(t) + service, _, _ := postService(t, ordinaryPost(), mattermost.Channel{ID: "channel-1", Type: "G", Name: "group"}, store) + dir, err := os.MkdirTemp(".", ".attachment-replay-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + path := filepath.Join(dir, "attachment.txt") + if err := os.WriteFile(path, []byte("file"), 0o600); err != nil { + t.Fatal(err) + } + in := func() ReplyInput { + return ReplyInput{RequestID: "attachment-replay", PostID: "post-1", Body: bytes.NewBufferString("body"), Attachments: []Attachment{{Path: path}}} + } + first, err := service.Reply(context.Background(), in()) + if err != nil { + t.Fatal(err) + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + service = service.WithAttachmentBinder(func(context.Context, []Attachment, [][]byte) ([]stagestore.Attachment, error) { + t.Fatal("binder called on replay") + return nil, nil + }) + replay, err := service.Reply(context.Background(), in()) + if err != nil || !replay.Stored.Replay || replay.Stored.Stage.ID != first.Stored.Stage.ID { + t.Fatalf("replay/error = %#v/%v", replay, err) + } +} + +type foundCreateStore struct { + record stagestore.CreateRecord + finds int +} + +func (s *foundCreateStore) FindCreate(context.Context, string, string, string) (stagestore.CreateRecord, bool, error) { + s.finds++ + return s.record, true, nil +} +func (*foundCreateStore) Create(context.Context, stagestore.CreateInput) (stagestore.CreateRecord, error) { + panic("Create called") +} + +func TestReplayOperationConflictPrecedesBodyAndAuthPrecedesFind(t *testing.T) { + store := &foundCreateStore{record: stagestore.CreateRecord{MutationResult: stagestore.MutationResult{Stage: stagestore.StageSummary{Operation: stagestore.Reply}}}} + service, _, _ := postService(t, ordinaryPost(), mattermost.Channel{ID: "channel-1", Type: "G", Name: "group"}, store) + reader := &panicReader{} + if _, err := service.EditPost(context.Background(), EditPostInput{"same-request", "post-1", reader}); !errors.Is(err, ErrConflict) || reader.read || store.finds != 1 { + t.Fatalf("error/read/finds = %v/%v/%d", err, reader.read, store.finds) + } + + calls := []string{} + badUsers := orderedUsers{mattermost.User{ID: "bad\u200b", Username: "arda"}, &calls} + service, err := New("https://mattermost.example", "", nil, badUsers, orderedChannels{mattermost.Channel{}, &calls}, emptyTeams{}, &fakePosts{calls: &calls}, store) + if err != nil { + t.Fatal(err) + } + store.finds = 0 + if _, err = service.DeletePost(context.Background(), DeletePostInput{"request", "post-1"}); !errors.Is(err, ErrTarget) || store.finds != 0 { + t.Fatalf("error/finds = %v/%d", err, store.finds) + } +} diff --git a/internal/staging/service.go b/internal/staging/service.go index c8b014a..a8cbac3 100644 --- a/internal/staging/service.go +++ b/internal/staging/service.go @@ -3,10 +3,14 @@ package staging import ( + "bytes" "context" "encoding/json" "errors" + "io" + "reflect" + "github.com/ardasevinc/mattermost-cli/internal/mattermost" "github.com/ardasevinc/mattermost-cli/internal/messageinput" "github.com/ardasevinc/mattermost-cli/internal/serverurl" "github.com/ardasevinc/mattermost-cli/internal/stageinput" @@ -27,14 +31,15 @@ type Service struct { users Users channels Channels teams Teams + posts Posts store Store bind AttachmentBinder credentials [][]byte } -func New(serverBaseURL, serverID string, credentials []string, users Users, channels Channels, teams Teams, store Store) (*Service, error) { +func New(serverBaseURL, serverID string, credentials []string, users Users, channels Channels, teams Teams, posts Posts, store Store) (*Service, error) { normalized, err := serverurl.Normalize(serverBaseURL) - if err != nil || users == nil || channels == nil || teams == nil || (serverID != "" && !validSelectorValue(serverID)) { + if err != nil || nilDependency(users) || nilDependency(channels) || nilDependency(teams) || nilDependency(posts) || (serverID != "" && !validSelectorValue(serverID)) { return nil, ErrInvalid } protected := credentialBytes(credentials) @@ -51,7 +56,7 @@ func New(serverBaseURL, serverID string, credentials []string, users Users, chan if contaminated(protected, normalized+"/api/v4", serverID) { return nil, ErrCredential } - return &Service{serverURL: normalized + "/api/v4", serverID: serverID, users: users, channels: channels, teams: teams, store: store, bind: stageinput.Bind, credentials: protected}, nil + return &Service{serverURL: normalized + "/api/v4", serverID: serverID, users: users, channels: channels, teams: teams, posts: posts, store: store, bind: stageinput.Bind, credentials: protected}, nil } // WithAttachmentBinder is intended for narrow tests which must prove dry-run @@ -67,14 +72,52 @@ func (s *Service) DryRunCreatePost(ctx context.Context, in DryRunInput) (Preview } func (s *Service) CreatePost(ctx context.Context, in CreatePostInput) (CreatePostResult, error) { - if s.store == nil || s.bind == nil || in.Body == nil || !validRequestID(in.RequestID) { + if nilDependency(s.store) || s.bind == nil || in.Body == nil || !validRequestID(in.RequestID) { + return CreatePostResult{}, ErrInvalid + } + if !validTargetSyntax(in.Target) { return CreatePostResult{}, ErrInvalid } callerFields := append([]string{in.RequestID}, targetStrings(in.Target)...) - if contaminated(s.credentials, callerFields...) { + if contaminated(s.credentials, callerFields...) || callerAttachmentsContaminated(s.credentials, in.Attachments) { return CreatePostResult{}, ErrCredential } - preview, err := s.resolveConversation(ctx, in.Target) + attachmentIntent, err := stageinput.Preflight(in.Attachments) + if err != nil { + return CreatePostResult{}, ErrInput + } + if attachmentIntentContaminated(s.credentials, attachmentIntent) { + return CreatePostResult{}, ErrCredential + } + current, err := s.authenticate(ctx) + if err != nil { + return CreatePostResult{}, err + } + record, found, err := s.store.FindCreate(ctx, s.serverURL, current.ID, in.RequestID) + if err != nil { + if errors.Is(err, stagestore.ErrConflict) { + return CreatePostResult{}, ErrConflict + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return CreatePostResult{}, err + } + return CreatePostResult{}, ErrStore + } + if found { + if record.Stage.Operation != stagestore.CreatePost { + return CreatePostResult{}, ErrConflict + } + body, readErr := messageinput.Read(in.Body) + if readErr != nil { + return CreatePostResult{}, ErrInput + } + if containsCredential(s.credentials, body) { + return CreatePostResult{}, ErrCredential + } + digest := intentDigest(stagestore.CreatePost, conversationCallerIntent(in.Target), body, "", attachmentIntent) + return replayResult(record, digest, stagestore.CreatePost, s.serverURL, current.ID) + } + preview, err := s.resolveConversationFor(ctx, in.Target, current) if err != nil { return CreatePostResult{}, err } @@ -90,6 +133,9 @@ func (s *Service) CreatePost(ctx context.Context, in CreatePostInput) (CreatePos if errors.Is(err, stageinput.ErrCredential) { return CreatePostResult{}, ErrCredential } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return CreatePostResult{}, err + } return CreatePostResult{}, ErrInput } if !validBoundAttachments(attachments) { @@ -104,15 +150,68 @@ func (s *Service) CreatePost(ctx context.Context, in CreatePostInput) (CreatePos return CreatePostResult{}, ErrCredential } stored, err := s.store.Create(ctx, stagestore.CreateInput{RequestID: in.RequestID, Operation: stagestore.CreatePost, - ServerURL: preview.ServerURL, ServerID: preview.ServerID, UserID: preview.UserID, + RequestDigest: intentDigest(stagestore.CreatePost, conversationCallerIntent(in.Target), body, "", attachmentIntent), + ServerURL: preview.ServerURL, ServerID: preview.ServerID, UserID: preview.UserID, Content: stagestore.RevisionContent{Body: body, Destination: destination, Plan: plan, Attachments: attachments}}) if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return CreatePostResult{}, err + } if errors.Is(err, stagestore.ErrConflict) { return CreatePostResult{}, ErrConflict } return CreatePostResult{}, ErrStore } - return CreatePostResult{Preview: preview, Stored: stored}, nil + return replayResult(stored, stored.RequestDigest, stagestore.CreatePost, preview.ServerURL, preview.UserID) +} + +func (s *Service) authenticate(ctx context.Context) (mattermost.User, error) { + if ctx == nil { + return mattermost.User{}, ErrInvalid + } + current, err := s.users.Current(ctx) + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return mattermost.User{}, err + } + if err != nil || !validResolvedUser(current) { + return mattermost.User{}, ErrTarget + } + if contaminated(s.credentials, current.ID, current.Username) { + return mattermost.User{}, ErrCredential + } + return current, nil +} + +func replayResult(record stagestore.CreateRecord, digest [32]byte, operation stagestore.Operation, serverURL, userID string) (CreatePostResult, error) { + if record.RequestDigest != digest || record.Stage.Operation != operation { + return CreatePostResult{}, ErrConflict + } + if record.Stage.ServerURL != serverURL || record.Stage.UserID != userID { + return CreatePostResult{}, ErrStore + } + var destination Destination + var plan Plan + dd := json.NewDecoder(bytes.NewReader(record.Destination)) + dd.DisallowUnknownFields() + pd := json.NewDecoder(bytes.NewReader(record.Plan)) + pd.DisallowUnknownFields() + if dd.Decode(&destination) != nil || dd.Decode(new(any)) != io.EOF || pd.Decode(&plan) != nil || pd.Decode(new(any)) != io.EOF { + return CreatePostResult{}, ErrStore + } + preview := Preview{ServerURL: record.Stage.ServerURL, ServerID: record.Stage.ServerID, UserID: record.Stage.UserID, Destination: destination, Plan: plan} + destinationRaw, planRaw, err := marshalSemantics(preview) + if err != nil || !bytes.Equal(destinationRaw, record.Destination) || !bytes.Equal(planRaw, record.Plan) { + return CreatePostResult{}, ErrStore + } + return CreatePostResult{preview, record.MutationResult}, nil +} + +func nilDependency(value any) bool { + if value == nil { + return true + } + v := reflect.ValueOf(value) + return (v.Kind() == reflect.Chan || v.Kind() == reflect.Func || v.Kind() == reflect.Interface || v.Kind() == reflect.Map || v.Kind() == reflect.Pointer || v.Kind() == reflect.Slice) && v.IsNil() } func attachmentPlan(count int) Plan { diff --git a/internal/staging/service_test.go b/internal/staging/service_test.go index 19a78a2..dcbe36e 100644 --- a/internal/staging/service_test.go +++ b/internal/staging/service_test.go @@ -3,6 +3,7 @@ package staging import ( "bytes" "context" + "crypto/sha256" "encoding/json" "errors" "fmt" @@ -61,6 +62,15 @@ func (emptyTeams) List(context.Context, string) (mattermost.TeamMembership, erro return mattermost.TeamMembership{}, nil } +type emptyPosts struct{} + +func (emptyPosts) ByID(context.Context, string) (mattermost.Post, error) { + return mattermost.Post{}, errors.New("unused") +} +func (emptyPosts) ReactionState(context.Context, string, string, string, string) (bool, error) { + return false, errors.New("unused") +} + type teamTransport struct{ payload string } func (t teamTransport) Get(_ context.Context, _ string, out any) error { @@ -74,12 +84,16 @@ type recordingStore struct { err error } -func (s *recordingStore) Create(_ context.Context, in stagestore.CreateInput) (stagestore.MutationResult, error) { +func (s *recordingStore) FindCreate(context.Context, string, string, string) (stagestore.CreateRecord, bool, error) { + return stagestore.CreateRecord{}, false, nil +} +func (s *recordingStore) Create(_ context.Context, in stagestore.CreateInput) (stagestore.CreateRecord, error) { s.mu.Lock() defer s.mu.Unlock() s.calls++ s.in = in - return stagestore.MutationResult{}, s.err + result := stagestore.MutationResult{Stage: stagestore.StageSummary{Operation: in.Operation, ServerURL: in.ServerURL, ServerID: in.ServerID, UserID: in.UserID}} + return stagestore.CreateRecord{MutationResult: result, Result: result, RequestDigest: in.RequestDigest, Destination: in.Content.Destination, Plan: in.Content.Plan}, s.err } func dmService(t *testing.T, store Store) (*Service, *fakeUsers, *fakeChannels) { @@ -90,7 +104,7 @@ func dmServiceCredentials(t *testing.T, store Store, credentials []string) (*Ser t.Helper() u := &fakeUsers{current: mattermost.User{ID: "user-1", Username: "arda"}, peer: mattermost.User{ID: "peer", Username: "hakan"}} c := &fakeChannels{direct: mattermost.Channel{ID: "dm-1", Type: "D", Name: "user-1__peer"}, found: true} - s, err := New("https://Mattermost.Example/chat/", "", credentials, u, c, emptyTeams{}, store) + s, err := New("https://Mattermost.Example/chat/", "", credentials, u, c, emptyTeams{}, emptyPosts{}, store) if err != nil { t.Fatal(err) } @@ -192,7 +206,7 @@ func TestMalformedTargetSyntaxIsZeroNetwork(t *testing.T) { func TestConstructorAndResolvedIdentityValidation(t *testing.T) { users := &fakeUsers{current: mattermost.User{ID: "user-1", Username: "arda"}} channels := &fakeChannels{} - if _, err := New("https://mattermost.example", " bad ", nil, users, channels, emptyTeams{}, nil); !errors.Is(err, ErrInvalid) { + if _, err := New("https://mattermost.example", " bad ", nil, users, channels, emptyTeams{}, emptyPosts{}, nil); !errors.Is(err, ErrInvalid) { t.Fatalf("server ID error = %v", err) } for _, unsafe := range []string{"\u200b", "\u200c", "\u200d", "\ufeff"} { @@ -207,6 +221,25 @@ func TestConstructorAndResolvedIdentityValidation(t *testing.T) { } } +func TestConstructorRejectsTypedNilDependencies(t *testing.T) { + u := &fakeUsers{} + c := &fakeChannels{} + p := &fakePosts{calls: &[]string{}} + for name, dependencies := range map[string][]any{ + "users": {(*fakeUsers)(nil), c, emptyTeams{}, p}, + "channels": {u, (*fakeChannels)(nil), emptyTeams{}, p}, + "teams": {u, c, (*emptyTeams)(nil), p}, + "posts": {u, c, emptyTeams{}, (*fakePosts)(nil)}, + } { + t.Run(name, func(t *testing.T) { + _, err := New("https://mattermost.example", "", nil, dependencies[0].(Users), dependencies[1].(Channels), dependencies[2].(Teams), dependencies[3].(Posts), nil) + if !errors.Is(err, ErrInvalid) { + t.Fatalf("error = %v", err) + } + }) + } +} + func TestBoundAttachmentRejectsAdditionalBidiControls(t *testing.T) { for _, unsafe := range []string{"\u061c", "\u200e", "\u200f"} { store := &recordingStore{} @@ -251,7 +284,7 @@ func TestRealStoreRequestReplayAndConflict(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = store.Close() }) - s, _, _ := dmService(t, store) + s, _, channels := dmService(t, store) input := func(body string) CreatePostInput { return CreatePostInput{RequestID: "same-request", Target: dmTarget(), Body: bytes.NewReader([]byte(body))} } @@ -259,6 +292,7 @@ func TestRealStoreRequestReplayAndConflict(t *testing.T) { if err != nil || first.Stored.Replay { t.Fatalf("first/error = %#v/%v", first.Stored, err) } + channels.err = errors.New("remote target disappeared") replay, err := s.CreatePost(context.Background(), input("hello")) if err != nil || !replay.Stored.Replay || replay.Stored.Stage.ID != first.Stored.Stage.ID { t.Fatalf("replay/error = %#v/%v", replay.Stored, err) @@ -268,6 +302,113 @@ func TestRealStoreRequestReplayAndConflict(t *testing.T) { } } +func TestCreateReplayRejectsCredentialInRawAttachmentMetadataBeforeOpen(t *testing.T) { + const token = "active-attachment-token" + store := realStageStore(t) + s, users, _ := dmServiceCredentials(t, store, []string{token}) + binderCalls := 0 + s = s.WithAttachmentBinder(func(context.Context, []Attachment, [][]byte) ([]stagestore.Attachment, error) { + binderCalls++ + return []stagestore.Attachment{{ + SuppliedPath: "safe.txt", + CanonicalPath: "/tmp/safe.txt", + RemoteFilename: "safe.txt", + ByteLength: 1, + MediaType: "text/plain", + ContentDigest: sha256.Sum256([]byte("x")), + }}, nil + }) + first := CreatePostInput{ + RequestID: "raw-attachment-credential", + Target: dmTarget(), + Body: bytes.NewBufferString("hello"), + Attachments: []Attachment{{Path: "safe.txt"}}, + } + if _, err := s.CreatePost(context.Background(), first); err != nil { + t.Fatal(err) + } + currentCalls := users.currentCalls.Load() + replay := first + replay.Body = bytes.NewBufferString("hello") + replay.Attachments = []Attachment{{Path: token + "/../safe.txt"}} + if _, err := s.CreatePost(context.Background(), replay); !errors.Is(err, ErrCredential) { + t.Fatalf("replay error = %v", err) + } + if binderCalls != 1 || users.currentCalls.Load() != currentCalls { + t.Fatalf("binder/current calls = %d/%d, want 1/%d", binderCalls, users.currentCalls.Load(), currentCalls) + } +} + +type racingChannels struct{ sequence atomic.Int64 } + +func (*racingChannels) ExistingDirect(context.Context, string, string) (mattermost.Channel, bool, error) { + return mattermost.Channel{}, false, errors.New("unused") +} +func (*racingChannels) ByID(context.Context, string) (mattermost.Channel, error) { + return mattermost.Channel{}, errors.New("unused") +} +func (r *racingChannels) ByName(context.Context, string, string) (mattermost.Channel, error) { + suffix := "a" + if r.sequence.Add(1)%2 == 0 { + suffix = "b" + } + return mattermost.Channel{ID: "channel-" + suffix, TeamID: "team-1", Type: "O", Name: "town"}, nil +} +func (*racingChannels) Member(_ context.Context, channelID, userID string) (mattermost.ChannelMember, error) { + return mattermost.ChannelMember{ChannelID: channelID, UserID: userID}, nil +} + +func TestConcurrentIdenticalServiceRequestsReturnWinnerProjection(t *testing.T) { + store := realStageStore(t) + c := &racingChannels{} + u := &fakeUsers{current: mattermost.User{ID: "user-1", Username: "arda"}} + teams := mattermost.NewTeams(teamTransport{payload: `[{"id":"team-1","name":"team","display_name":"Team","type":"O"}]`}) + service, err := New("https://mattermost.example", "", nil, u, c, teams, emptyPosts{}, store) + if err != nil { + t.Fatal(err) + } + target := Target{Conversation: Channel, Selector: ByName, Value: "town", Team: &TeamSelector{By: ByID, Value: "team-1"}} + const workers = 20 + results := make(chan CreatePostResult, workers) + failures := make(chan error, workers) + var wg sync.WaitGroup + for range workers { + wg.Add(1) + go func() { + defer wg.Done() + result, err := service.CreatePost(context.Background(), CreatePostInput{RequestID: "racing-request", Target: target, Body: bytes.NewBufferString("body")}) + if err != nil { + failures <- err + return + } + results <- result + }() + } + wg.Wait() + close(results) + close(failures) + for err := range failures { + t.Fatal(err) + } + var id, channel string + firsts, count := 0, 0 + for result := range results { + count++ + if !result.Stored.Replay { + firsts++ + } + if id == "" { + id, channel = result.Stored.Stage.ID, result.Preview.Destination.ChannelID + } + if result.Stored.Stage.ID != id || result.Preview.Destination.ChannelID != channel { + t.Fatalf("result diverged = %#v", result) + } + } + if count != workers || firsts != 1 { + t.Fatalf("count/firsts = %d/%d", count, firsts) + } +} + type panicReader struct{ read bool } func (r *panicReader) Read([]byte) (int, error) { r.read = true; return 0, errors.New("forbidden") } diff --git a/internal/staging/types.go b/internal/staging/types.go index 3081067..d67604e 100644 --- a/internal/staging/types.go +++ b/internal/staging/types.go @@ -48,6 +48,24 @@ type CreatePostInput struct { Attachments []Attachment } +type ReplyInput struct { + RequestID string + PostID string + Body io.Reader + Attachments []Attachment +} + +type EditPostInput struct { + RequestID string + PostID string + Body io.Reader +} + +type DeletePostInput struct{ RequestID, PostID string } +type ReactionInput struct{ RequestID, PostID, Emoji string } +type PostDryRunInput struct{ PostID string } +type ReactionDryRunInput struct{ PostID, Emoji string } + type DryRunInput struct{ Target Target } type Destination struct { @@ -108,8 +126,14 @@ type Teams interface { List(context.Context, string) (mattermost.TeamMembership, error) } +type Posts interface { + ByID(context.Context, string) (mattermost.Post, error) + ReactionState(context.Context, string, string, string, string) (bool, error) +} + type Store interface { - Create(context.Context, stagestore.CreateInput) (stagestore.MutationResult, error) + FindCreate(context.Context, string, string, string) (stagestore.CreateRecord, bool, error) + Create(context.Context, stagestore.CreateInput) (stagestore.CreateRecord, error) } type AttachmentBinder func(context.Context, []stageinput.Attachment, [][]byte) ([]stagestore.Attachment, error) diff --git a/internal/staging/validation.go b/internal/staging/validation.go index 1404f34..2722edc 100644 --- a/internal/staging/validation.go +++ b/internal/staging/validation.go @@ -2,6 +2,7 @@ package staging import ( "bytes" + "sort" "strings" "unicode" "unicode/utf8" @@ -36,6 +37,22 @@ func cloneCredentials(values [][]byte) [][]byte { return out } +func credentialsByLength(values [][]byte) [][]byte { + out := make([][]byte, 0, len(values)) + for _, value := range values { + if len(value) > 0 { + out = append(out, bytes.Clone(value)) + } + } + sort.Slice(out, func(i, j int) bool { + if len(out[i]) != len(out[j]) { + return len(out[i]) > len(out[j]) + } + return bytes.Compare(out[i], out[j]) < 0 + }) + return out +} + func containsCredential(credentials [][]byte, value []byte) bool { for _, credential := range credentials { if bytes.Contains(value, credential) { diff --git a/schemas/v2/examples/store-doctor.json b/schemas/v2/examples/store-doctor.json index fc442cb..13563a9 100644 --- a/schemas/v2/examples/store-doctor.json +++ b/schemas/v2/examples/store-doctor.json @@ -1 +1 @@ -{"schema":"mm/v2/store-doctor","path":"/home/arda/.local/state/mattermost-cli/stages.sqlite3","report":{"exists":false,"filesystemSafe":null,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":2,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}} +{"schema":"mm/v2/store-doctor","path":"/home/arda/.local/state/mattermost-cli/stages.sqlite3","report":{"exists":false,"filesystemSafe":null,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":3,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}} diff --git a/schemas/v2/examples/store-migrations.json b/schemas/v2/examples/store-migrations.json index 9cf9d2a..4ef9af2 100644 --- a/schemas/v2/examples/store-migrations.json +++ b/schemas/v2/examples/store-migrations.json @@ -1 +1 @@ -{"schema":"mm/v2/store-migrations","latest":2,"migrations":[{"version":1,"name":"core-stage-state","checksum":"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"},{"version":2,"name":"immutable-local-request-receipts","checksum":"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"}]} +{"schema":"mm/v2/store-migrations","latest":3,"migrations":[{"version":1,"name":"core-stage-state","checksum":"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"},{"version":2,"name":"immutable-local-request-receipts","checksum":"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"},{"version":3,"name":"caller-intent-stage-create-replay","checksum":"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5"}]} diff --git a/schemas/v2/stage-preview.schema.json b/schemas/v2/stage-preview.schema.json index 36e87e6..e3a2d54 100644 --- a/schemas/v2/stage-preview.schema.json +++ b/schemas/v2/stage-preview.schema.json @@ -909,10 +909,7 @@ "const": "add_reaction" }, "condition": { - "enum": [ - "always", - "if_missing" - ] + "const": "if_missing" } } } @@ -935,10 +932,7 @@ "const": "remove_reaction" }, "condition": { - "enum": [ - "always", - "if_missing" - ] + "const": "if_missing" } } } diff --git a/schemas/v2/stage-receipt.schema.json b/schemas/v2/stage-receipt.schema.json index 66885c3..08761db 100644 --- a/schemas/v2/stage-receipt.schema.json +++ b/schemas/v2/stage-receipt.schema.json @@ -761,10 +761,7 @@ "const": "add_reaction" }, "condition": { - "enum": [ - "always", - "if_missing" - ] + "const": "if_missing" } } } @@ -787,10 +784,7 @@ "const": "remove_reaction" }, "condition": { - "enum": [ - "always", - "if_missing" - ] + "const": "if_missing" } } } diff --git a/schemas/v2/stage.schema.json b/schemas/v2/stage.schema.json index f2d3f98..214435a 100644 --- a/schemas/v2/stage.schema.json +++ b/schemas/v2/stage.schema.json @@ -1082,10 +1082,7 @@ "const": "add_reaction" }, "condition": { - "enum": [ - "always", - "if_missing" - ] + "const": "if_missing" } } } @@ -1108,10 +1105,7 @@ "const": "remove_reaction" }, "condition": { - "enum": [ - "always", - "if_missing" - ] + "const": "if_missing" } } } diff --git a/schemas/v2/stages.schema.json b/schemas/v2/stages.schema.json index b625fc1..6cd8fa2 100644 --- a/schemas/v2/stages.schema.json +++ b/schemas/v2/stages.schema.json @@ -653,10 +653,7 @@ "const": "add_reaction" }, "condition": { - "enum": [ - "always", - "if_missing" - ] + "const": "if_missing" } } } @@ -679,10 +676,7 @@ "const": "remove_reaction" }, "condition": { - "enum": [ - "always", - "if_missing" - ] + "const": "if_missing" } } } diff --git a/schemas/v2/store-doctor.schema.json b/schemas/v2/store-doctor.schema.json index 2b310ac..5a474bb 100644 --- a/schemas/v2/store-doctor.schema.json +++ b/schemas/v2/store-doctor.schema.json @@ -15,7 +15,7 @@ "exists": { "type": "boolean" }, "filesystemSafe": {}, "applicationId": {}, "integrity": {}, "integrityTruncated": {}, "foreignKeyIssues": {}, "foreignKeyRows": {}, "foreignKeyTruncated": {}, "journalMode": {}, "synchronous": {}, "secureDelete": {}, "foreignKeys": {}, "trustedSchema": {}, "queryOnly": {}, "walFallback": {}, - "migrations": { "type": "object", "additionalProperties": false, "required": ["applied", "latest", "valid"], "properties": { "applied": {}, "latest": { "const": 2 }, "valid": {} } }, + "migrations": { "type": "object", "additionalProperties": false, "required": ["applied", "latest", "valid"], "properties": { "applied": {}, "latest": { "const": 3 }, "valid": {} } }, "permissionModelLimitations": { "type": "array", "items": { "$ref": "#/$defs/safeString" } } }, "allOf": [ @@ -23,14 +23,14 @@ "filesystemSafe": { "const": null }, "applicationId": { "const": null }, "integrity": { "const": null }, "integrityTruncated": { "const": null }, "foreignKeyIssues": { "const": null }, "foreignKeyRows": { "const": null }, "foreignKeyTruncated": { "const": null }, "journalMode": { "const": null }, "synchronous": { "const": null }, "secureDelete": { "const": null }, "foreignKeys": { "const": null }, "trustedSchema": { "const": null }, "queryOnly": { "const": null }, "walFallback": { "const": null }, - "migrations": { "properties": { "applied": { "const": null }, "latest": { "const": 2 }, "valid": { "const": null } } } + "migrations": { "properties": { "applied": { "const": null }, "latest": { "const": 3 }, "valid": { "const": null } } } } } }, { "if": { "properties": { "exists": { "const": true } }, "required": ["exists"] }, "then": { "properties": { "filesystemSafe": { "const": true }, "applicationId": { "const": 1296913970 }, "integrity": { "$ref": "#/$defs/nonemptyBoundedStrings" }, "integrityTruncated": { "type": "boolean" }, "foreignKeyIssues": { "type": "integer", "minimum": 0, "maximum": 21 }, "foreignKeyRows": { "$ref": "#/$defs/boundedStrings" }, "foreignKeyTruncated": { "type": "boolean" }, "journalMode": { "enum": ["wal", "delete", "truncate", "persist"] }, "synchronous": { "type": "integer" }, "secureDelete": { "type": "integer" }, "foreignKeys": { "const": true }, "trustedSchema": { "const": false }, "queryOnly": { "const": true }, "walFallback": { "type": "boolean" }, - "migrations": { "properties": { "applied": { "const": 2 }, "latest": { "const": 2 }, "valid": { "const": true } } } + "migrations": { "properties": { "applied": { "const": 3 }, "latest": { "const": 3 }, "valid": { "const": true } } } }, "allOf": [ { "oneOf": [ { "properties": { "integrity": { "const": ["ok"] }, "integrityTruncated": { "const": false } } }, diff --git a/schemas/v2/store-migrations.schema.json b/schemas/v2/store-migrations.schema.json index 8c33e6f..d11df4f 100644 --- a/schemas/v2/store-migrations.schema.json +++ b/schemas/v2/store-migrations.schema.json @@ -6,13 +6,15 @@ "required": ["schema", "latest", "migrations"], "properties": { "schema": { "const": "mm/v2/store-migrations" }, - "latest": { "const": 2 }, + "latest": { "const": 3 }, "migrations": { - "type": "array", "minItems": 2, "maxItems": 2, + "type": "array", "minItems": 3, "maxItems": 3, "prefixItems": [{ "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { "version": { "const": 1 }, "name": { "const": "core-stage-state" }, "checksum": { "const": "e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f" } } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { "version": { "const": 2 }, "name": { "const": "immutable-local-request-receipts" }, "checksum": { "const": "ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c" } + } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { + "version": { "const": 3 }, "name": { "const": "caller-intent-stage-create-replay" }, "checksum": { "const": "237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5" } } }], "items": false } From 441a84c94f05fdb5dd5889aa6754427bbd23e630 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 03:33:28 +0300 Subject: [PATCH 064/119] feat: harden stage lifecycle storage --- docs/V2_CONTRACT.md | 4 +- internal/cli/store_test.go | 6 +- internal/stagecursor/cursor.go | 139 ++++++++++++++ internal/stagecursor/cursor_test.go | 107 +++++++++++ internal/stagestore/domain.go | 198 +++++++++++++++++--- internal/stagestore/domain_test.go | 163 +++++++++++++++- internal/stagestore/schema.go | 4 + internal/staging/intent.go | 17 +- internal/staging/post.go | 3 + internal/staging/revision.go | 195 +++++++++++++++++++ internal/staging/revision_test.go | 173 +++++++++++++++++ internal/staging/service.go | 21 ++- internal/staging/service_test.go | 4 +- internal/staging/types.go | 28 +++ internal/staging/validation.go | 5 +- schemas/v2/examples/store-doctor.json | 2 +- schemas/v2/examples/store-migrations.json | 2 +- schemas/v2/stage-revise-request.schema.json | 19 +- schemas/v2/store-doctor.schema.json | 6 +- schemas/v2/store-migrations.schema.json | 6 +- 20 files changed, 1044 insertions(+), 58 deletions(-) create mode 100644 internal/stagecursor/cursor.go create mode 100644 internal/stagecursor/cursor_test.go create mode 100644 internal/staging/revision.go create mode 100644 internal/staging/revision_test.go diff --git a/docs/V2_CONTRACT.md b/docs/V2_CONTRACT.md index 559a9aa..29df5bb 100644 --- a/docs/V2_CONTRACT.md +++ b/docs/V2_CONTRACT.md @@ -203,7 +203,9 @@ It consumes one versioned `mm/v2/stage-request` object from stdin. It is the can Stage-create replay equality is caller-intent equality, not equality of the resolved remote snapshot. The digest domain is `mm/v2/stage-request/caller-intent/v1` and binds the operation, unresolved caller target, nullable body and emoji, and ordered normalized attachment metadata. It excludes request ID, server scope, resolved remote facts, file bytes and hashes, sizes, and detected MIME. An identical replay authenticates, loads the original store-authoritative destination and plan, validates content input where applicable, and performs no target, reaction, or file read. Different caller intent conflicts even if it would currently resolve to the same remote target. -Migration 3 deliberately tombstones pre-caller-intent `mm/v2/stage-request` receipts as `mm/v2/legacy-stage-request-conflict`. Their older digests cannot prove caller-intent equality, so they conflict rather than replay or fall back to remote resolution. Revise and cancel receipts are unchanged. +Revision replay uses the same fail-closed model with a distinct `mm/v2/stage-revise-request/caller-intent/v1` digest domain. It binds the immutable operation, stage ID, expected revision and semantic digest, revive intent, nullable replacement body, and nullable ordered attachment metadata. Null body or attachment metadata preserves the stored value; an empty attachment list clears it. The digest excludes attachment file bytes and derived file facts, allowing an identical retry to return its durable receipt before reopening a missing or changed source file. + +Migration 3 deliberately tombstones pre-caller-intent `mm/v2/stage-request` receipts as `mm/v2/legacy-stage-request-conflict`. Migration 4 likewise tombstones pre-caller-intent `mm/v2/stage-revise-request` receipts as `mm/v2/legacy-stage-revise-conflict`. Those older digests cannot prove caller-intent equality, so they conflict rather than replay, resolve remote state, or reopen source files. Cancel receipts are unchanged. This v2 database format remains development-only and has not shipped; the explicit tombstones preserve deterministic local upgrade behavior without claiming false replay compatibility. Structured apply is also first-class: diff --git a/internal/cli/store_test.go b/internal/cli/store_test.go index 34b3d5e..9360ee8 100644 --- a/internal/cli/store_test.go +++ b/internal/cli/store_test.go @@ -36,7 +36,7 @@ func TestStoreDoctorAbsentIsReadOnlyAndSchemaValid(t *testing.T) { if err := registry.Validate("mm/v2/store-doctor", bytes.NewReader(stdout.Bytes())); err != nil { t.Fatalf("schema: %v\n%s", err, stdout.String()) } - for _, fact := range []string{`"filesystemSafe":null`, `"applicationId":null`, `"integrity":null`, `"applied":null`, `"latest":3`, `"valid":null`, `"journalMode":null`} { + for _, fact := range []string{`"filesystemSafe":null`, `"applicationId":null`, `"integrity":null`, `"applied":null`, `"latest":4`, `"valid":null`, `"journalMode":null`} { if !strings.Contains(stdout.String(), fact) { t.Fatalf("absent report omitted %s: %s", fact, stdout.String()) } @@ -110,7 +110,7 @@ func TestStoreMigrationsIsOfflineAndSchemaValid(t *testing.T) { if err := registry.Validate("mm/v2/store-migrations", bytes.NewReader(stdout.Bytes())); err != nil { t.Fatalf("schema: %v\n%s", err, stdout.String()) } - want := "{\"schema\":\"mm/v2/store-migrations\",\"latest\":3,\"migrations\":[{\"version\":1,\"name\":\"core-stage-state\",\"checksum\":\"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f\"},{\"version\":2,\"name\":\"immutable-local-request-receipts\",\"checksum\":\"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c\"},{\"version\":3,\"name\":\"caller-intent-stage-create-replay\",\"checksum\":\"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5\"}]}\n" + want := "{\"schema\":\"mm/v2/store-migrations\",\"latest\":4,\"migrations\":[{\"version\":1,\"name\":\"core-stage-state\",\"checksum\":\"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f\"},{\"version\":2,\"name\":\"immutable-local-request-receipts\",\"checksum\":\"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c\"},{\"version\":3,\"name\":\"caller-intent-stage-create-replay\",\"checksum\":\"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5\"},{\"version\":4,\"name\":\"caller-intent-stage-revise-replay\",\"checksum\":\"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4\"}]}\n" if stdout.String() != want { t.Fatalf("stdout does not match golden migration contract: %q", stdout.String()) } @@ -310,7 +310,7 @@ func TestStoreHumanOutputStatesBounds(t *testing.T) { t.Fatalf("stdout=%q", stdout.String()) } stdout.Reset() - if code := Execute(context.Background(), []string{"store", "migrations"}, strings.NewReader(""), &stdout, &stderr); code != 0 || !strings.Contains(stdout.String(), "latest: 3\n1 core-stage-state ") { + if code := Execute(context.Background(), []string{"store", "migrations"}, strings.NewReader(""), &stdout, &stderr); code != 0 || !strings.Contains(stdout.String(), "latest: 4\n1 core-stage-state ") { t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } } diff --git a/internal/stagecursor/cursor.go b/internal/stagecursor/cursor.go new file mode 100644 index 0000000..2a0d8ba --- /dev/null +++ b/internal/stagecursor/cursor.go @@ -0,0 +1,139 @@ +// Package stagecursor encodes and decodes opaque stage-list cursors. +package stagecursor + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "io" + "regexp" + "time" + "unicode/utf8" +) + +const ( + maxEncodedLength = 1024 + maxDecodedLength = 768 + maxStageIDLength = 128 +) + +var ( + // ErrInvalidCursor is returned for every invalid cursor without reflecting + // attacker-controlled input. + ErrInvalidCursor = errors.New("invalid stage cursor") + safeStageID = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) +) + +// Boundary is the final stage in a deterministic stage-list page. +type Boundary struct { + UpdatedAt time.Time + StageID string +} + +type wireCursor struct { + Version int `json:"v"` + Scope string `json:"scope"` + Boundary wireBoundary `json:"boundary"` +} + +type wireBoundary struct { + UpdatedAt string `json:"updatedAt"` + StageID string `json:"stageId"` +} + +// Encode returns canonical, unpadded base64url JSON for boundary. +func Encode(boundary Boundary) (string, error) { + if !validBoundary(boundary) { + return "", ErrInvalidCursor + } + wire := wireCursor{1, "stages", wireBoundary{ + UpdatedAt: boundary.UpdatedAt.Format(time.RFC3339Nano), + StageID: boundary.StageID, + }} + data, err := json.Marshal(wire) + if err != nil || len(data) == 0 || len(data) > maxDecodedLength { + return "", ErrInvalidCursor + } + encoded := base64.RawURLEncoding.EncodeToString(data) + if len(encoded) > maxEncodedLength { + return "", ErrInvalidCursor + } + return encoded, nil +} + +// Decode validates an opaque cursor and returns its stage-list boundary. +func Decode(encoded string) (Boundary, error) { + if len(encoded) == 0 || len(encoded) > maxEncodedLength || !safeStageID.MatchString(encoded) { + return Boundary{}, ErrInvalidCursor + } + data, err := base64.RawURLEncoding.DecodeString(encoded) + if err != nil || len(data) == 0 || len(data) > maxDecodedLength || !utf8.Valid(data) || + base64.RawURLEncoding.EncodeToString(data) != encoded { + return Boundary{}, ErrInvalidCursor + } + + wire, ok := decodeWire(data) + if !ok || wire.Version != 1 || wire.Scope != "stages" || !validStageID(wire.Boundary.StageID) { + return Boundary{}, ErrInvalidCursor + } + updatedAt, err := time.Parse(time.RFC3339Nano, wire.Boundary.UpdatedAt) + if err != nil || updatedAt.IsZero() || wire.Boundary.UpdatedAt != updatedAt.UTC().Format(time.RFC3339Nano) { + return Boundary{}, ErrInvalidCursor + } + boundary := Boundary{UpdatedAt: updatedAt, StageID: wire.Boundary.StageID} + canonical, err := Encode(boundary) + if err != nil || canonical != encoded { + return Boundary{}, ErrInvalidCursor + } + return boundary, nil +} + +func validBoundary(boundary Boundary) bool { + if boundary.UpdatedAt.IsZero() || !validStageID(boundary.StageID) { + return false + } + _, offset := boundary.UpdatedAt.Zone() + return offset == 0 && boundary.UpdatedAt.Format(time.RFC3339Nano) == boundary.UpdatedAt.UTC().Format(time.RFC3339Nano) +} + +func validStageID(id string) bool { + return len(id) >= 1 && len(id) <= maxStageIDLength && safeStageID.MatchString(id) +} + +func decodeWire(data []byte) (wireCursor, bool) { + var outer map[string]json.RawMessage + decoder := json.NewDecoder(bytes.NewReader(data)) + if decoder.Decode(&outer) != nil || !onlyKeys(outer, "v", "scope", "boundary") { + return wireCursor{}, false + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return wireCursor{}, false + } + var inner map[string]json.RawMessage + if json.Unmarshal(outer["boundary"], &inner) != nil || !onlyKeys(inner, "updatedAt", "stageId") { + return wireCursor{}, false + } + var wire wireCursor + typed := json.NewDecoder(bytes.NewReader(data)) + typed.DisallowUnknownFields() + if typed.Decode(&wire) != nil { + return wireCursor{}, false + } + if err := typed.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return wireCursor{}, false + } + return wire, true +} + +func onlyKeys(value map[string]json.RawMessage, keys ...string) bool { + if value == nil || len(value) != len(keys) { + return false + } + for _, key := range keys { + if _, exists := value[key]; !exists { + return false + } + } + return true +} diff --git a/internal/stagecursor/cursor_test.go b/internal/stagecursor/cursor_test.go new file mode 100644 index 0000000..f937f54 --- /dev/null +++ b/internal/stagecursor/cursor_test.go @@ -0,0 +1,107 @@ +package stagecursor + +import ( + "encoding/base64" + "errors" + "reflect" + "strings" + "testing" + "time" +) + +func testBoundary() Boundary { + return Boundary{UpdatedAt: time.Date(2026, 7, 17, 12, 34, 56, 123456789, time.UTC), StageID: "stg_0123456789abcdefghijklmnopqrstuv"} +} + +func TestRoundTripIsCanonical(t *testing.T) { + want := testBoundary() + encoded, err := Encode(want) + if err != nil { + t.Fatal(err) + } + const canonical = "eyJ2IjoxLCJzY29wZSI6InN0YWdlcyIsImJvdW5kYXJ5Ijp7InVwZGF0ZWRBdCI6IjIwMjYtMDctMTdUMTI6MzQ6NTYuMTIzNDU2Nzg5WiIsInN0YWdlSWQiOiJzdGdfMDEyMzQ1Njc4OWFiY2RlZmdoaWprbG1ub3BxcnN0dXYifX0" + if encoded != canonical { + t.Fatalf("Encode() = %q, want %q", encoded, canonical) + } + got, err := Decode(encoded) + if err != nil || !reflect.DeepEqual(got, want) { + t.Fatalf("Decode() = %#v, %v; want %#v", got, err, want) + } +} + +func TestDecodeRejectsNonCanonicalAndMalformedCursors(t *testing.T) { + encode := func(value string) string { return base64.RawURLEncoding.EncodeToString([]byte(value)) } + valid := `{"v":1,"scope":"stages","boundary":{"updatedAt":"2026-07-17T12:34:56.123456789Z","stageId":"stage_1"}}` + tests := map[string]string{ + "empty": "", + "invalid base64": "not+base64", + "padding": encode(valid) + "=", + "encoded too long": strings.Repeat("a", maxEncodedLength+1), + "decoded too long": base64.RawURLEncoding.EncodeToString([]byte(strings.Repeat(" ", maxDecodedLength+1))), + "invalid UTF-8": base64.RawURLEncoding.EncodeToString([]byte{0xff}), + "wrong version": encode(strings.Replace(valid, `"v":1`, `"v":2`, 1)), + "wrong scope": encode(strings.Replace(valid, `"stages"`, `"stage"`, 1)), + "outer extra": encode(strings.TrimSuffix(valid, "}") + `,"extra":true}`), + "boundary extra": encode(strings.Replace(valid, `"stageId":"stage_1"`, `"stageId":"stage_1","extra":true`, 1)), + "missing field": encode(strings.Replace(valid, `,"stageId":"stage_1"`, "", 1)), + "duplicate key": encode(strings.Replace(valid, `"v":1`, `"v":1,"v":1`, 1)), + "trailing JSON": encode(valid + `{}`), + "whitespace": encode(" " + valid), + "reordered": encode(`{"scope":"stages","v":1,"boundary":{"updatedAt":"2026-07-17T12:34:56.123456789Z","stageId":"stage_1"}}`), + "offset timestamp": encode(strings.Replace(valid, "2026-07-17T12:34:56.123456789Z", "2026-07-17T15:34:56.123456789+03:00", 1)), + "noncanonical time": encode(strings.Replace(valid, "2026-07-17T12:34:56.123456789Z", "2026-07-17T12:34:56.1234567890Z", 1)), + "zero timestamp": encode(strings.Replace(valid, "2026-07-17T12:34:56.123456789Z", "0001-01-01T00:00:00Z", 1)), + "unsafe stage ID": encode(strings.Replace(valid, "stage_1", "stage 1", 1)), + "long stage ID": encode(strings.Replace(valid, "stage_1", strings.Repeat("a", maxStageIDLength+1), 1)), + } + for name, encoded := range tests { + t.Run(name, func(t *testing.T) { + _, err := Decode(encoded) + if !errors.Is(err, ErrInvalidCursor) || err.Error() != "invalid stage cursor" { + t.Fatalf("error = %v, want generic ErrInvalidCursor", err) + } + }) + } +} + +func TestEncodeRejectsInvalidBoundaries(t *testing.T) { + valid := testBoundary() + invalid := []Boundary{ + {}, + {UpdatedAt: valid.UpdatedAt}, + {UpdatedAt: time.Time{}, StageID: valid.StageID}, + {UpdatedAt: valid.UpdatedAt, StageID: "unsafe id"}, + {UpdatedAt: valid.UpdatedAt, StageID: strings.Repeat("a", maxStageIDLength+1)}, + {UpdatedAt: valid.UpdatedAt.In(time.FixedZone("EEST", 3*60*60)), StageID: valid.StageID}, + } + for i, boundary := range invalid { + if _, err := Encode(boundary); !errors.Is(err, ErrInvalidCursor) { + t.Errorf("case %d: error = %v, want ErrInvalidCursor", i, err) + } + } + max := valid + max.StageID = strings.Repeat("a", maxStageIDLength) + if _, err := Encode(max); err != nil { + t.Fatalf("maximum safe stage ID rejected: %v", err) + } +} + +func FuzzDecode(f *testing.F) { + encoded, _ := Encode(testBoundary()) + for _, seed := range []string{encoded, "", "e30=", "not_json"} { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, input string) { + boundary, err := Decode(input) + if err != nil { + if !errors.Is(err, ErrInvalidCursor) { + t.Fatalf("unexpected error: %v", err) + } + return + } + roundTrip, err := Encode(boundary) + if err != nil || roundTrip != input { + t.Fatalf("canonical round trip = %q, %v; want %q", roundTrip, err, input) + } + }) +} diff --git a/internal/stagestore/domain.go b/internal/stagestore/domain.go index ecd72a6..1c5b19b 100644 --- a/internal/stagestore/domain.go +++ b/internal/stagestore/domain.go @@ -18,6 +18,7 @@ import ( "github.com/ardasevinc/mattermost-cli/internal/messageinput" "github.com/ardasevinc/mattermost-cli/internal/serverurl" + "github.com/ardasevinc/mattermost-cli/internal/stagecursor" ) const ( @@ -106,6 +107,7 @@ type ReviseInput struct { StageID, RequestID string ExpectedRevision int64 ExpectedDigest [32]byte + RequestDigest [32]byte Revive bool Composition Composition } @@ -143,7 +145,23 @@ type MutationResult struct { RecordedAt time.Time `json:"recordedAt"` Replay bool `json:"-"` } -type ListOptions struct{ Limit int } +type ListOptions struct { + Limit int + After *stagecursor.Boundary +} + +// ListRecord is the non-content projection used by public stage listings. +// Destination is included in the listing query so callers need not issue one +// Show query per stage. +type ListRecord struct { + StageSummary + Destination json.RawMessage +} + +type ListPage struct { + Records []ListRecord + NextCursor *string +} func (s *Store) Create(ctx context.Context, in CreateInput) (CreateRecord, error) { if err := ctx.Err(); err != nil { @@ -220,20 +238,23 @@ func (s *Store) Revise(ctx context.Context, in ReviseInput) (MutationResult, err if err != nil { return MutationResult{}, err } + if !validOperation(base.Operation) { + return MutationResult{}, localError(errors.New("stored operation")) + } + if base.Operation != CreatePost && base.Operation != Reply && base.Operation != EditPost { + return MutationResult{}, ErrNotEligible + } composition, err := normalizeComposition(base.Operation, in.Composition) if err != nil { return MutationResult{}, ErrInvalid } content := RevisionContent{composition.Body, bytes.Clone(base.Destination), bytes.Clone(base.Plan), composition.Attachments} - requestDigest := digestValue(struct { - Action, StageID string - ExpectedRevision int64 - ExpectedDigest [32]byte - Revive bool - Composition Composition - }{"revise", in.StageID, in.ExpectedRevision, in.ExpectedDigest, in.Revive, composition}) + requestDigest := in.RequestDigest + if in.RequestID != "" && requestDigest == ([32]byte{}) { + return MutationResult{}, ErrInvalid + } if in.RequestID != "" { - if result, found, e := loadReplay(ctx, tx, base.ServerURL, base.UserID, in.RequestID, "mm/v2/stage-revise-request", requestDigest); e != nil { + if result, found, e := loadReviseReplay(ctx, tx, base.ServerURL, base.UserID, in.RequestID, requestDigest); e != nil { return MutationResult{}, e } else if found { return result, nil @@ -312,6 +333,9 @@ func (s *Store) Cancel(ctx context.Context, in CancelInput) (MutationResult, err if result, found, e := loadReplay(ctx, tx, base.ServerURL, base.UserID, in.RequestID, "mm/v2/stage-cancel-request", digest); e != nil { return MutationResult{}, e } else if found { + if result.Stage != base.StageSummary || result.Action != "cancel" || result.Stage.Lifecycle != LifecycleCanceled || result.Stage.Recovery != RecoveryForbidden { + return MutationResult{}, localError(errors.New("cancel receipt projection")) + } return result, nil } } @@ -354,33 +378,86 @@ func (s *Store) Show(ctx context.Context, id string) (StageDetail, error) { return detail, err } detail.Attachments, err = readAttachments(ctx, s.db, id, detail.Revision) - return detail, err + if err != nil { + return detail, err + } + if !VerifyDetail(detail) { + return StageDetail{}, localError(errors.New("stored stage detail")) + } + return detail, nil } func (s *Store) List(ctx context.Context, o ListOptions) ([]StageSummary, error) { + page, err := s.ListRecords(ctx, o) + if err != nil { + return nil, err + } + out := make([]StageSummary, len(page.Records)) + for i := range page.Records { + out[i] = page.Records[i].StageSummary + } + return out, nil +} + +func (s *Store) ListRecords(ctx context.Context, o ListOptions) (ListPage, error) { limit := o.Limit if limit == 0 { limit = 50 } if limit < 1 || limit > maxListLimit { - return nil, ErrInvalid + return ListPage{}, ErrInvalid + } + var boundaryTime, boundaryID string + if o.After != nil { + encoded, encodeErr := stagecursor.Encode(*o.After) + if encodeErr != nil || encoded == "" { + return ListPage{}, ErrInvalid + } + boundaryTime, boundaryID = formatListTime(o.After.UpdatedAt), o.After.StageID } - rows, err := s.db.QueryContext(ctx, `SELECT s.id,s.server_url,coalesce(s.server_id,''),s.user_id,s.operation,s.lifecycle,s.recovery,r.revision,r.semantic_digest,s.created_at,s.updated_at FROM stages s JOIN stage_revisions r ON r.stage_id=s.id AND r.revision=s.current_revision ORDER BY s.updated_at DESC,s.id ASC LIMIT ?`, limit) + rows, err := s.db.QueryContext(ctx, `SELECT s.id,s.server_url,coalesce(s.server_id,''),s.user_id,s.operation,s.lifecycle,s.recovery,r.revision,r.semantic_digest,s.created_at,s.updated_at,r.destination_json + FROM stages s LEFT JOIN stage_revisions r ON r.stage_id=s.id AND r.revision=s.current_revision + WHERE ?='' OR substr(s.updated_at,1,23) < ? OR (substr(s.updated_at,1,23) = ? AND s.id > ?) + ORDER BY substr(s.updated_at,1,23) DESC,s.id ASC LIMIT ?`, boundaryTime, boundaryTime, boundaryTime, boundaryID, limit+1) if err != nil { - return nil, localError(err) + return ListPage{}, localError(err) } defer rows.Close() - out := make([]StageSummary, 0) + out := make([]ListRecord, 0, limit+1) for rows.Next() { - v, e := scanSummary(rows) + var v ListRecord + var digest []byte + var created, updated, destination string + e := rows.Scan(&v.ID, &v.ServerURL, &v.ServerID, &v.UserID, &v.Operation, &v.Lifecycle, &v.Recovery, &v.Revision, &digest, &created, &updated, &destination) + if e == nil && len(digest) != 32 { + e = errors.New("digest") + } + if e == nil { + copy(v.SemanticDigest[:], digest) + v.CreatedAt, e = parseTime(created) + } + if e == nil { + v.UpdatedAt, e = parseTime(updated) + } if e != nil { - return nil, e + return ListPage{}, localError(e) } + v.Destination = json.RawMessage(destination) out = append(out, v) } if err = rows.Err(); err != nil { - return nil, localError(err) + return ListPage{}, localError(err) + } + page := ListPage{Records: out} + if len(out) > limit { + page.Records = out[:limit] + last := page.Records[len(page.Records)-1] + cursor, encodeErr := stagecursor.Encode(stagecursor.Boundary{UpdatedAt: last.UpdatedAt, StageID: last.ID}) + if encodeErr != nil { + return ListPage{}, localError(encodeErr) + } + page.NextCursor = &cursor } - return out, nil + return page, nil } const currentDetailSQL = `SELECT s.id,s.server_url,coalesce(s.server_id,''),s.user_id,s.operation,s.lifecycle,s.recovery,r.revision,r.semantic_digest,s.created_at,s.updated_at,r.created_at,r.body,r.destination_json,r.plan_json FROM stages s JOIN stage_revisions r ON r.stage_id=s.id AND r.revision=s.current_revision WHERE s.id=?` @@ -462,6 +539,37 @@ func semanticDigest(op Operation, server, serverID, user string, c RevisionConte Content semanticContent `json:"content"` }{op, server, serverID, user, c.semantic()}) } + +// ComputeSemanticDigest normalizes a complete retained revision and returns +// the digest used to bind review and apply to its exact semantics. +func ComputeSemanticDigest(op Operation, server, serverID, user string, content RevisionContent) ([32]byte, error) { + if !validOperation(op) || !canonicalServerURL(server) || !bounded(user, maxIdentityBytes) || serverID != "" && !bounded(serverID, maxIdentityBytes) { + return [32]byte{}, ErrInvalid + } + normalized, err := normalizeContent(op, content) + if err != nil { + return [32]byte{}, ErrInvalid + } + return semanticDigest(op, server, serverID, user, normalized), nil +} + +// VerifyDetail checks the current stored projection before content is exposed. +func VerifyDetail(detail StageDetail) bool { + if !validStoredSummary(detail.StageSummary) || detail.RevisionCreatedAt.IsZero() || detail.RevisionCreatedAt.Before(detail.CreatedAt) || detail.RevisionCreatedAt.After(detail.UpdatedAt) { + return false + } + if detail.Lifecycle == LifecycleCompleted || detail.Lifecycle == LifecyclePruned { + if detail.Body != nil || len(detail.Attachments) != 0 { + return false + } + _, destinationErr := canonicalObject(detail.Destination) + _, planErr := canonicalObject(detail.Plan) + return destinationErr == nil && planErr == nil + } + digest, err := ComputeSemanticDigest(detail.Operation, detail.ServerURL, detail.ServerID, detail.UserID, + RevisionContent{detail.Body, detail.Destination, detail.Plan, detail.Attachments}) + return err == nil && digest == detail.SemanticDigest +} func normalizeContent(op Operation, v RevisionContent) (RevisionContent, error) { destination, err := canonicalObject(v.Destination) if err != nil { @@ -678,10 +786,10 @@ func readAttachments(ctx context.Context, q queryer, stage string, revision int6 return out, localError(rows.Err()) } -func loadReplay(ctx context.Context, tx *sql.Tx, server, user, id, schema string, digest [32]byte) (MutationResult, bool, error) { +func loadReplay(ctx context.Context, q queryer, server, user, id, schema string, digest [32]byte) (MutationResult, bool, error) { var storedSchema, raw, created string var stored []byte - err := tx.QueryRowContext(ctx, `SELECT request_schema,request_digest,result_json,created_at FROM local_requests WHERE server_url=? AND user_id=? AND request_id=?`, server, user, id).Scan(&storedSchema, &stored, &raw, &created) + err := q.QueryRowContext(ctx, `SELECT request_schema,request_digest,result_json,created_at FROM local_requests WHERE server_url=? AND user_id=? AND request_id=?`, server, user, id).Scan(&storedSchema, &stored, &raw, &created) if errors.Is(err, sql.ErrNoRows) { return MutationResult{}, false, nil } @@ -702,6 +810,43 @@ func loadReplay(ctx context.Context, tx *sql.Tx, server, user, id, schema string return result, true, nil } +func (s *Store) FindRevise(ctx context.Context, server, user, id string, digest [32]byte) (MutationResult, bool, error) { + if ctx == nil || !canonicalServerURL(server) || !bounded(user, maxIdentityBytes) || !validRequestID(id) || id == "" || digest == ([32]byte{}) { + return MutationResult{}, false, ErrInvalid + } + return loadReviseReplay(ctx, s.db, server, user, id, digest) +} + +func loadReviseReplay(ctx context.Context, q queryer, server, user, id string, digest [32]byte) (MutationResult, bool, error) { + result, found, err := loadReplay(ctx, q, server, user, id, "mm/v2/stage-revise-request", digest) + if err != nil || !found { + return result, found, err + } + var operation Operation + var storedServer, serverID, storedUser, stageCreated, revisionCreated, destination, plan string + var semantic, body []byte + err = q.QueryRowContext(ctx, `SELECT s.operation,s.server_url,coalesce(s.server_id,''),s.user_id,s.created_at,r.created_at,r.semantic_digest,r.body,r.destination_json,r.plan_json + FROM stages s JOIN stage_revisions r ON r.stage_id=s.id WHERE s.id=? AND r.revision=?`, result.Stage.ID, result.Stage.Revision). + Scan(&operation, &storedServer, &serverID, &storedUser, &stageCreated, &revisionCreated, &semantic, &body, &destination, &plan) + if err != nil { + return MutationResult{}, false, localError(err) + } + attachments, err := readAttachments(ctx, q, result.Stage.ID, result.Stage.Revision) + if err != nil { + return MutationResult{}, false, err + } + stageCreatedAt, stageTimeErr := parseTime(stageCreated) + revisionCreatedAt, revisionTimeErr := parseTime(revisionCreated) + content, contentErr := normalizeContent(operation, RevisionContent{body, json.RawMessage(destination), json.RawMessage(plan), attachments}) + if stageTimeErr != nil || revisionTimeErr != nil || contentErr != nil || len(semantic) != 32 || operation != result.Stage.Operation || + storedServer != server || serverID != result.Stage.ServerID || storedUser != user || !stageCreatedAt.Equal(result.Stage.CreatedAt) || + !revisionCreatedAt.Equal(result.Stage.UpdatedAt) || !revisionCreatedAt.Equal(result.RecordedAt) || + !bytes.Equal(semantic, result.Stage.SemanticDigest[:]) || semanticDigest(operation, storedServer, serverID, storedUser, content) != result.Stage.SemanticDigest { + return MutationResult{}, false, localError(errors.New("revise receipt projection")) + } + return result, true, nil +} + func (s *Store) FindCreate(ctx context.Context, server, user, id string) (CreateRecord, bool, error) { if ctx == nil || !canonicalServerURL(server) || !bounded(user, maxIdentityBytes) || !validRequestID(id) { return CreateRecord{}, false, ErrInvalid @@ -723,7 +868,7 @@ func findCreate(ctx context.Context, q queryer, server, user, id string) (Create if err != nil { return CreateRecord{}, false, localError(err) } - if schemaName == "mm/v2/legacy-stage-request-conflict" || schemaName == "mm/v2/legacy-request-conflict" || schemaName == "mm/v2/stage-revise-request" || schemaName == "mm/v2/stage-cancel-request" { + if schemaName == "mm/v2/legacy-stage-request-conflict" || schemaName == "mm/v2/legacy-stage-revise-conflict" || schemaName == "mm/v2/legacy-request-conflict" || schemaName == "mm/v2/stage-revise-request" || schemaName == "mm/v2/stage-cancel-request" { return CreateRecord{}, false, ErrConflict } if schemaName != "mm/v2/stage-request" || len(digest) != 32 { @@ -819,6 +964,12 @@ func validRecovery(v Recovery) bool { } return false } + +func validStoredSummary(stage StageSummary) bool { + return bounded(stage.ID, maxIdentityBytes) && canonicalServerURL(stage.ServerURL) && bounded(stage.UserID, maxIdentityBytes) && validOperation(stage.Operation) && + stage.Revision > 0 && stage.SemanticDigest != ([32]byte{}) && validLifecycle(stage.Lifecycle) && validRecovery(stage.Recovery) && + !stage.CreatedAt.IsZero() && !stage.UpdatedAt.IsZero() && !stage.UpdatedAt.Before(stage.CreatedAt) +} func persistReplay(ctx context.Context, tx *sql.Tx, server, user, id, schema string, digest [32]byte, result MutationResult, stamp string) error { if id == "" { return nil @@ -938,8 +1089,9 @@ func parseTime(v string) (time.Time, error) { } return t, nil } -func formatTime(v time.Time) string { return v.UTC().Format("2006-01-02T15:04:05.000000000Z") } -func oneRow(v sql.Result) bool { n, err := v.RowsAffected(); return err == nil && n == 1 } +func formatTime(v time.Time) string { return v.UTC().Format("2006-01-02T15:04:05.000000000Z") } +func formatListTime(v time.Time) string { return v.UTC().Format("2006-01-02T15:04:05.000") } +func oneRow(v sql.Result) bool { n, err := v.RowsAffected(); return err == nil && n == 1 } func runCommitHook() { commitHook.RLock() fn := commitHook.fn diff --git a/internal/stagestore/domain_test.go b/internal/stagestore/domain_test.go index 8b3dcd9..edde41c 100644 --- a/internal/stagestore/domain_test.go +++ b/internal/stagestore/domain_test.go @@ -12,6 +12,8 @@ import ( "sync" "testing" "time" + + "github.com/ardasevinc/mattermost-cli/internal/stagecursor" ) func openDomainStore(t *testing.T) *Store { @@ -37,7 +39,8 @@ func TestFindCreateUsesExactReceiptRevisionAndFailsClosedOnCorruption(t *testing if err != nil { t.Fatal(err) } - revised, err := s.Revise(context.Background(), ReviseInput{StageID: created.Stage.ID, RequestID: "revise-exact", ExpectedRevision: 1, ExpectedDigest: created.Stage.SemanticDigest, Composition: Composition{Body: []byte("two"), Attachments: in.Content.Attachments}}) + revised, err := s.Revise(context.Background(), ReviseInput{StageID: created.Stage.ID, RequestID: "revise-exact", ExpectedRevision: 1, ExpectedDigest: created.Stage.SemanticDigest, + RequestDigest: sha256.Sum256([]byte("revise-exact")), Composition: Composition{Body: []byte("two"), Attachments: in.Content.Attachments}}) if err != nil || revised.Stage.Revision != 2 { t.Fatalf("revise = %#v/%v", revised, err) } @@ -166,7 +169,7 @@ func TestCreateAndReplayReturnCanonicalAuthoritativeProjection(t *testing.T) { } } -func TestMigrationThreeTombstonesOnlyLegacyStageCreates(t *testing.T) { +func TestCallerIntentMigrationsTombstoneLegacyCreateAndReviseReceipts(t *testing.T) { path := testPath(t) original := migrations migrations = append([]migration(nil), original[:2]...) @@ -198,16 +201,20 @@ func TestMigrationThreeTombstonesOnlyLegacyStageCreates(t *testing.T) { if err = s.db.QueryRow(`SELECT request_schema FROM local_requests WHERE request_id='revise-kept'`).Scan(&reviseSchema); err != nil { t.Fatal(err) } - if createSchema != "mm/v2/legacy-stage-request-conflict" || reviseSchema != "mm/v2/stage-revise-request" { + if createSchema != "mm/v2/legacy-stage-request-conflict" || reviseSchema != "mm/v2/legacy-stage-revise-conflict" { t.Fatalf("schemas = %s/%s", createSchema, reviseSchema) } if _, _, err = s.FindCreate(context.Background(), in.ServerURL, in.UserID, in.RequestID); !errors.Is(err, ErrConflict) { t.Fatalf("legacy lookup = %v", err) } + if _, _, err = s.FindCreate(context.Background(), in.ServerURL, in.UserID, "revise-kept"); !errors.Is(err, ErrConflict) { + t.Fatalf("legacy revise ID reuse = %v", err) + } } func reviseInput(stage StageSummary, request, body string) ReviseInput { content := createInput("", body).Content - return ReviseInput{stage.ID, request, stage.Revision, stage.SemanticDigest, false, Composition{content.Body, content.Attachments}} + return ReviseInput{StageID: stage.ID, RequestID: request, ExpectedRevision: stage.Revision, ExpectedDigest: stage.SemanticDigest, + RequestDigest: sha256.Sum256([]byte(request + "\x00" + body)), Composition: Composition{Body: content.Body, Attachments: content.Attachments}} } func TestRevisePreservesImmutableDestinationAndPlan(t *testing.T) { @@ -236,6 +243,58 @@ func TestRevisePreservesImmutableDestinationAndPlan(t *testing.T) { } } +func TestReviseCallerIntentReplayIgnoresBoundFileDrift(t *testing.T) { + s := openDomainStore(t) + created, err := s.Create(context.Background(), createInput("", "one")) + if err != nil { + t.Fatal(err) + } + requestDigest := sha256.Sum256([]byte("caller body and attachment metadata")) + first := reviseInput(created.Stage, "revise-replay", "two") + first.RequestDigest = requestDigest + revised, err := s.Revise(context.Background(), first) + if err != nil { + t.Fatal(err) + } + retry := first + retry.Composition = Composition{Body: []byte("different bound bytes"), Attachments: []Attachment{attachment("changed.txt")}} + replayed, err := s.Revise(context.Background(), retry) + if err != nil || !replayed.Replay || replayed.Stage != revised.Stage { + t.Fatalf("replayed=%+v err=%v want=%+v", replayed, err, revised) + } + lookup, found, err := s.FindRevise(context.Background(), revised.Stage.ServerURL, revised.Stage.UserID, first.RequestID, requestDigest) + if err != nil || !found || !lookup.Replay || lookup.Stage != revised.Stage { + t.Fatalf("lookup=%+v found=%v err=%v", lookup, found, err) + } + wrong := requestDigest + wrong[0] ^= 0xff + if _, _, err = s.FindRevise(context.Background(), revised.Stage.ServerURL, revised.Stage.UserID, first.RequestID, wrong); !errors.Is(err, ErrConflict) { + t.Fatalf("wrong caller intent = %v", err) + } + if _, err = s.db.Exec(`DROP TRIGGER local_requests_immutable_update`); err != nil { + t.Fatal(err) + } + var raw string + if err = s.db.QueryRow(`SELECT result_json FROM local_requests WHERE request_id=?`, first.RequestID).Scan(&raw); err != nil { + t.Fatal(err) + } + var corrupted MutationResult + if err = json.Unmarshal([]byte(raw), &corrupted); err != nil { + t.Fatal(err) + } + corrupted.Stage.ID = "stg_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + encoded, err := marshalCanonical(corrupted) + if err != nil { + t.Fatal(err) + } + if _, err = s.db.Exec(`UPDATE local_requests SET result_json=? WHERE request_id=?`, string(encoded), first.RequestID); err != nil { + t.Fatal(err) + } + if _, _, err = s.FindRevise(context.Background(), revised.Stage.ServerURL, revised.Stage.UserID, first.RequestID, requestDigest); err == nil || errors.Is(err, ErrConflict) { + t.Fatalf("corrupt revise projection = %v", err) + } +} + func TestCreateShowListPrivacyAttachmentsAndCanonicalDigest(t *testing.T) { s := openDomainStore(t) created, err := s.Create(context.Background(), createInput("create-1", "private body\n")) @@ -291,6 +350,76 @@ func TestCreateShowListPrivacyAttachmentsAndCanonicalDigest(t *testing.T) { } } +func TestListRecordsUsesHonestKeysetPagination(t *testing.T) { + s := openDomainStore(t) + for i := range 5 { + created, err := s.Create(context.Background(), createInput("", string(rune('a'+i)))) + if err != nil { + t.Fatal(err) + } + stamp := time.Date(2026, 1, 1, 0, 0, 0, 123456780+i, time.UTC) + if _, err = s.db.Exec(`UPDATE stages SET updated_at=? WHERE id=?`, formatTime(stamp), created.Stage.ID); err != nil { + t.Fatal(err) + } + } + var after *stagecursor.Boundary + seen := make(map[string]struct{}) + for pageNumber := 0; ; pageNumber++ { + page, err := s.ListRecords(context.Background(), ListOptions{Limit: 2, After: after}) + if err != nil || len(page.Records) == 0 || len(page.Records) > 2 { + t.Fatalf("page %d = %+v err=%v", pageNumber, page, err) + } + for _, record := range page.Records { + if _, exists := seen[record.ID]; exists { + t.Fatalf("duplicate stage %s", record.ID) + } + seen[record.ID] = struct{}{} + } + if page.NextCursor == nil { + break + } + boundary, err := stagecursor.Decode(*page.NextCursor) + if err != nil { + t.Fatal(err) + } + after = &boundary + } + if len(seen) != 5 { + t.Fatalf("listed %d stages", len(seen)) + } +} + +func TestListRecordsFailsClosedWhenCurrentRevisionIsMissing(t *testing.T) { + s := openDomainStore(t) + created, err := s.Create(context.Background(), createInput("", "body")) + if err != nil { + t.Fatal(err) + } + if _, err = s.db.Exec(`PRAGMA foreign_keys=OFF`); err != nil { + t.Fatal(err) + } + if _, err = s.db.Exec(`UPDATE stages SET current_revision=999 WHERE id=?`, created.Stage.ID); err != nil { + t.Fatal(err) + } + if _, err = s.ListRecords(context.Background(), ListOptions{}); err == nil || errors.Is(err, ErrNotFound) { + t.Fatalf("corrupt current revision = %v", err) + } +} + +func TestShowFailsClosedWhenRetainedContentBreaksSemanticDigest(t *testing.T) { + s := openDomainStore(t) + created, err := s.Create(context.Background(), createInput("", "original")) + if err != nil { + t.Fatal(err) + } + if _, err = s.db.Exec(`UPDATE stage_revisions SET body=? WHERE stage_id=? AND revision=1`, []byte("modified"), created.Stage.ID); err != nil { + t.Fatal(err) + } + if _, err = s.Show(context.Background(), created.Stage.ID); err == nil || errors.Is(err, ErrInvalid) || errors.Is(err, ErrNotFound) { + t.Fatalf("semantic corruption = %v", err) + } +} + func TestImmutableReplaySnapshots(t *testing.T) { s := openDomainStore(t) first, err := s.Create(context.Background(), createInput("same.request:1", "one")) @@ -470,9 +599,31 @@ func TestOperationContentApplicability(t *testing.T) { validDelete.Operation = DeletePost validDelete.Content.Body = nil validDelete.Content.Attachments = nil - if _, err := s.Create(context.Background(), validDelete); err != nil { + createdDelete, err := s.Create(context.Background(), validDelete) + if err != nil { + t.Fatal(err) + } + if _, err = s.Revise(context.Background(), ReviseInput{StageID: createdDelete.Stage.ID, ExpectedRevision: 1, ExpectedDigest: createdDelete.Stage.SemanticDigest}); !errors.Is(err, ErrNotEligible) { + t.Fatalf("delete revision error = %v", err) + } +} + +func TestReviseDistinguishesCorruptOperationFromImmutableOperation(t *testing.T) { + s := openDomainStore(t) + created, err := s.Create(context.Background(), createInput("operation-corruption", "one")) + if err != nil { t.Fatal(err) } + if _, err = s.db.Exec(`PRAGMA ignore_check_constraints=ON`); err != nil { + t.Fatal(err) + } + if _, err = s.db.Exec(`UPDATE stages SET operation='corrupt' WHERE id=?`, created.Stage.ID); err != nil { + t.Fatal(err) + } + input := reviseInput(created.Stage, "operation-corruption-revise", "two") + if _, err = s.Revise(context.Background(), input); err == nil || errors.Is(err, ErrInvalid) || errors.Is(err, ErrNotEligible) { + t.Fatalf("corrupt operation error = %v", err) + } } func TestStrictCanonicalObjects(t *testing.T) { @@ -529,7 +680,7 @@ func TestBoundsRequestIDOrderingAndCommitCancellation(t *testing.T) { } }) } - if _, err := s.List(context.Background(), ListOptions{maxListLimit + 1}); !errors.Is(err, ErrInvalid) { + if _, err := s.List(context.Background(), ListOptions{Limit: maxListLimit + 1}); !errors.Is(err, ErrInvalid) { t.Fatalf("list=%v", err) } preCanceled, stop := context.WithCancel(context.Background()) diff --git a/internal/stagestore/schema.go b/internal/stagestore/schema.go index af7f9f1..a2a3ba3 100644 --- a/internal/stagestore/schema.go +++ b/internal/stagestore/schema.go @@ -93,4 +93,8 @@ BEGIN SELECT RAISE(ABORT, 'stage destination and plan are immutable'); END; DROP TRIGGER local_requests_immutable_update; UPDATE local_requests SET request_schema='mm/v2/legacy-stage-request-conflict' WHERE request_schema='mm/v2/stage-request'; CREATE TRIGGER local_requests_immutable_update BEFORE UPDATE ON local_requests BEGIN SELECT RAISE(ABORT, 'local request receipts are immutable'); END; +`}, {version: 4, name: "caller-intent-stage-revise-replay", sql: ` +DROP TRIGGER local_requests_immutable_update; +UPDATE local_requests SET request_schema='mm/v2/legacy-stage-revise-conflict' WHERE request_schema='mm/v2/stage-revise-request'; +CREATE TRIGGER local_requests_immutable_update BEFORE UPDATE ON local_requests BEGIN SELECT RAISE(ABORT, 'local request receipts are immutable'); END; `}} diff --git a/internal/staging/intent.go b/internal/staging/intent.go index beb2d4f..a44d7f0 100644 --- a/internal/staging/intent.go +++ b/internal/staging/intent.go @@ -19,6 +19,14 @@ type callerIntent struct { } func intentDigest(operation stagestore.Operation, target any, body []byte, emoji string, attachments []stageinput.MetadataIntent) [32]byte { + return callerIntentDigest("mm/v2/stage-request/caller-intent/v1", operation, target, body, emoji, attachments) +} + +func revisionRequestDigest(operation stagestore.Operation, target revisionIntent, body []byte, attachments []stageinput.MetadataIntent) [32]byte { + return callerIntentDigest("mm/v2/stage-revise-request/caller-intent/v1", operation, target, body, "", attachments) +} + +func callerIntentDigest(domain string, operation stagestore.Operation, target any, body []byte, emoji string, attachments []stageinput.MetadataIntent) [32]byte { var bodyValue *string if body != nil { value := string(body) @@ -28,7 +36,7 @@ func intentDigest(operation stagestore.Operation, target any, body []byte, emoji if emoji != "" { emojiValue = &emoji } - value := callerIntent{"mm/v2/stage-request/caller-intent/v1", operation, target, bodyValue, emojiValue, attachments} + value := callerIntent{domain, operation, target, bodyValue, emojiValue, attachments} var out bytes.Buffer encoder := json.NewEncoder(&out) encoder.SetEscapeHTML(false) @@ -60,3 +68,10 @@ func conversationCallerIntent(target Target) conversationIntent { type postIntent struct { PostID string `json:"postId"` } + +type revisionIntent struct { + StageID string `json:"stageId"` + ExpectedRevision int64 `json:"expectedRevision"` + ExpectedDigest [32]byte `json:"expectedDigest"` + Revive bool `json:"revive"` +} diff --git a/internal/staging/post.go b/internal/staging/post.go index 65dc3e0..8911030 100644 --- a/internal/staging/post.go +++ b/internal/staging/post.go @@ -329,6 +329,9 @@ func (s *Service) persistPost(ctx context.Context, requestID string, requestDige } func (s *Service) findCreate(ctx context.Context, userID, requestID string) (stagestore.CreateRecord, bool, error) { + if requestID == "" { + return stagestore.CreateRecord{}, false, nil + } record, found, err := s.store.FindCreate(ctx, s.serverURL, userID, requestID) if err == nil { return record, found, nil diff --git a/internal/staging/revision.go b/internal/staging/revision.go new file mode 100644 index 0000000..67b4a7d --- /dev/null +++ b/internal/staging/revision.go @@ -0,0 +1,195 @@ +package staging + +import ( + "bytes" + "context" + "encoding/hex" + "errors" + + "github.com/ardasevinc/mattermost-cli/internal/messageinput" + "github.com/ardasevinc/mattermost-cli/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +var ErrNotEligible = errors.New("staging: lifecycle transition not allowed") + +// Reviser admits offline revision and cancellation requests without resolving +// or mutating their immutable destination. +type Reviser struct { + store RevisionStore + bind AttachmentBinder + credentials [][]byte +} + +func NewReviser(credentials []string, store RevisionStore, bind AttachmentBinder) (*Reviser, error) { + protected := credentialBytes(credentials) + if nilDependency(store) || bind == nil || len(protected) > 64 { + return nil, ErrInvalid + } + total := 0 + for _, credential := range protected { + total += len(credential) + if len(credential) > 4096 || total > 64<<10 { + return nil, ErrInvalid + } + } + return &Reviser{store: store, bind: bind, credentials: protected}, nil +} + +func (r *Reviser) Revise(ctx context.Context, in ReviseInput) (RevisionResult, error) { + if ctx == nil || !validStageMutation(in.StageID, in.RequestID, in.ExpectedRevision) { + return RevisionResult{}, ErrInvalid + } + if err := ctx.Err(); err != nil { + return RevisionResult{}, err + } + if contaminated(r.credentials, in.StageID, in.RequestID, hex.EncodeToString(in.ExpectedDigest[:])) || callerAttachmentsContaminated(r.credentials, in.Attachments) { + return RevisionResult{}, ErrCredential + } + var attachmentIntent []stageinput.MetadataIntent + var err error + if in.Attachments != nil { + attachmentIntent, err = stageinput.Preflight(in.Attachments) + if err != nil { + return RevisionResult{}, ErrInput + } + if attachmentIntentContaminated(r.credentials, attachmentIntent) { + return RevisionResult{}, ErrCredential + } + } + detail, err := r.show(ctx, in.StageID) + if err != nil { + return RevisionResult{}, err + } + if detail.ID != in.StageID { + return RevisionResult{}, ErrStore + } + if detail.Operation != stagestore.CreatePost && detail.Operation != stagestore.Reply && detail.Operation != stagestore.EditPost { + return RevisionResult{}, ErrNotEligible + } + if attachmentsContaminated(r.credentials, detail.Attachments) { + return RevisionResult{}, ErrCredential + } + if detail.Operation == stagestore.EditPost && in.Attachments != nil { + return RevisionResult{}, ErrNotEligible + } + var body []byte + if in.Body == nil { + body = bytes.Clone(detail.Body) + } else { + body, err = messageinput.Read(in.Body) + if err != nil { + return RevisionResult{}, ErrInput + } + } + if containsCredential(r.credentials, body) { + return RevisionResult{}, ErrCredential + } + var suppliedBody []byte + if in.Body != nil { + suppliedBody = body + } + requestDigest := revisionRequestDigest(detail.Operation, revisionIntent{in.StageID, in.ExpectedRevision, in.ExpectedDigest, in.Revive}, suppliedBody, attachmentIntent) + if in.RequestID != "" { + replayed, found, findErr := r.store.FindRevise(ctx, detail.ServerURL, detail.UserID, in.RequestID, requestDigest) + if findErr != nil { + return RevisionResult{}, mapRevisionStoreError(findErr) + } + if found { + if replayed.Stage.ID != in.StageID || replayed.Stage.Operation != detail.Operation || replayed.Stage.ServerURL != detail.ServerURL || replayed.Stage.UserID != detail.UserID { + return RevisionResult{}, ErrStore + } + return RevisionResult{Stored: replayed, Destination: bytes.Clone(detail.Destination)}, nil + } + } + attachments := append([]stagestore.Attachment(nil), detail.Attachments...) + if in.Attachments != nil { + attachments, err = r.bind(ctx, in.Attachments, cloneCredentials(r.credentials)) + if err != nil { + return RevisionResult{}, mapBinderError(err) + } + if len(attachments) != len(in.Attachments) || !validBoundAttachments(attachments) { + return RevisionResult{}, ErrInput + } + if attachmentsContaminated(r.credentials, attachments) { + return RevisionResult{}, ErrCredential + } + } + if err := ctx.Err(); err != nil { + return RevisionResult{}, err + } + stored, err := r.store.Revise(ctx, stagestore.ReviseInput{StageID: in.StageID, RequestID: in.RequestID, ExpectedRevision: in.ExpectedRevision, + ExpectedDigest: in.ExpectedDigest, RequestDigest: requestDigest, Revive: in.Revive, Composition: stagestore.Composition{Body: bytes.Clone(body), Attachments: attachments}}) + if err != nil { + return RevisionResult{}, mapRevisionStoreError(err) + } + return RevisionResult{Stored: stored, Destination: bytes.Clone(detail.Destination)}, nil +} + +func (r *Reviser) Cancel(ctx context.Context, in CancelInput) (RevisionResult, error) { + if ctx == nil || !validStageMutation(in.StageID, in.RequestID, in.ExpectedRevision) { + return RevisionResult{}, ErrInvalid + } + if err := ctx.Err(); err != nil { + return RevisionResult{}, err + } + if contaminated(r.credentials, in.StageID, in.RequestID, hex.EncodeToString(in.ExpectedDigest[:])) { + return RevisionResult{}, ErrCredential + } + detail, err := r.show(ctx, in.StageID) + if err != nil { + return RevisionResult{}, err + } + if detail.ID != in.StageID { + return RevisionResult{}, ErrStore + } + if err := ctx.Err(); err != nil { + return RevisionResult{}, err + } + stored, err := r.store.Cancel(ctx, stagestore.CancelInput{StageID: in.StageID, RequestID: in.RequestID, ExpectedRevision: in.ExpectedRevision, ExpectedDigest: in.ExpectedDigest}) + if err != nil { + return RevisionResult{}, mapRevisionStoreError(err) + } + if stored.Stage.ID != in.StageID || stored.Stage.Operation != detail.Operation || stored.Stage.ServerURL != detail.ServerURL || stored.Stage.UserID != detail.UserID { + return RevisionResult{}, ErrStore + } + return RevisionResult{Stored: stored, Destination: bytes.Clone(detail.Destination)}, nil +} + +func (r *Reviser) show(ctx context.Context, stageID string) (stagestore.StageDetail, error) { + detail, err := r.store.Show(ctx, stageID) + if err != nil { + return stagestore.StageDetail{}, mapRevisionStoreError(err) + } + return detail, nil +} + +func validStageMutation(stageID, requestID string, revision int64) bool { + return validSelectorValue(stageID) && validRequestID(requestID) && revision > 0 && revision <= 9_007_199_254_740_991 +} + +func mapBinderError(err error) error { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + if errors.Is(err, stageinput.ErrCredential) { + return ErrCredential + } + return ErrInput +} + +func mapRevisionStoreError(err error) error { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + if errors.Is(err, stagestore.ErrConflict) { + return ErrConflict + } + if errors.Is(err, stagestore.ErrNotFound) { + return ErrNotFound + } + if errors.Is(err, stagestore.ErrNotEligible) { + return ErrNotEligible + } + return ErrStore +} diff --git a/internal/staging/revision_test.go b/internal/staging/revision_test.go new file mode 100644 index 0000000..b93c2d2 --- /dev/null +++ b/internal/staging/revision_test.go @@ -0,0 +1,173 @@ +package staging + +import ( + "bytes" + "context" + "encoding/hex" + "errors" + "strings" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +type revisionStoreStub struct { + detail stagestore.StageDetail + showErr, reviseErr, cancelErr error + findErr error + findResult stagestore.MutationResult + findFound bool + findCalls, reviseCalls, cancelCalls int + findDigest [32]byte + reviseIn stagestore.ReviseInput +} + +func (s *revisionStoreStub) Show(context.Context, string) (stagestore.StageDetail, error) { + return s.detail, s.showErr +} +func (s *revisionStoreStub) Revise(_ context.Context, in stagestore.ReviseInput) (stagestore.MutationResult, error) { + s.reviseCalls++ + s.reviseIn = in + return stagestore.MutationResult{Action: "revise"}, s.reviseErr +} +func (s *revisionStoreStub) FindRevise(_ context.Context, _, _, _ string, digest [32]byte) (stagestore.MutationResult, bool, error) { + s.findCalls++ + s.findDigest = digest + return s.findResult, s.findFound, s.findErr +} +func (s *revisionStoreStub) Cancel(context.Context, stagestore.CancelInput) (stagestore.MutationResult, error) { + s.cancelCalls++ + return stagestore.MutationResult{Action: "cancel", Stage: s.detail.StageSummary}, s.cancelErr +} + +func revisionFixture(operation stagestore.Operation) (*revisionStoreStub, [32]byte) { + digest := [32]byte{1} + return &revisionStoreStub{detail: stagestore.StageDetail{StageSummary: stagestore.StageSummary{ID: "stage-1", Operation: operation, Revision: 1, SemanticDigest: digest}, Destination: []byte(`{"kind":"conversation"}`)}}, digest +} + +func TestReviserPreservesBodyAttachmentOrderAndDestinationSnapshot(t *testing.T) { + store, digest := revisionFixture(stagestore.Reply) + binder := func(_ context.Context, in []stageinput.Attachment, credentials [][]byte) ([]stagestore.Attachment, error) { + if len(in) != 2 || string(credentials[0]) != "active-token" { + t.Fatal("binder did not receive ordered inputs and protected credential") + } + return []stagestore.Attachment{ + {SuppliedPath: "/a", CanonicalPath: "/a", RemoteFilename: "a", ByteLength: 1, ContentDigest: [32]byte{1}}, + {SuppliedPath: "/b", CanonicalPath: "/b", RemoteFilename: "b", ByteLength: 1, ContentDigest: [32]byte{2}}, + }, nil + } + r, err := NewReviser([]string{"active-token"}, store, binder) + if err != nil { + t.Fatal(err) + } + result, err := r.Revise(context.Background(), ReviseInput{StageID: "stage-1", RequestID: "request-1", ExpectedRevision: 1, ExpectedDigest: digest, + Body: strings.NewReader(" exact body\n"), Attachments: []Attachment{{Path: "/a"}, {Path: "/b"}}}) + if err != nil || store.reviseCalls != 1 || string(store.reviseIn.Composition.Body) != " exact body\n" || store.reviseIn.Composition.Attachments[1].RemoteFilename != "b" { + t.Fatalf("unexpected revision: result=%+v err=%v calls=%d input=%+v", result, err, store.reviseCalls, store.reviseIn) + } + store.detail.Destination[0] = 'x' + if string(result.Destination) != `{"kind":"conversation"}` { + t.Fatal("result destination aliases store detail") + } +} + +func TestReviserRejectsBeforeStoreMutation(t *testing.T) { + for _, tc := range []struct { + name, body string + op stagestore.Operation + attachments []Attachment + want error + }{ + {"credential body", "before active-token after", stagestore.Reply, nil, ErrCredential}, + {"oversized body", strings.Repeat("x", 65536), stagestore.Reply, nil, ErrInput}, + {"inapplicable operation", "body", stagestore.DeletePost, nil, ErrNotEligible}, + {"edit attachments", "body", stagestore.EditPost, []Attachment{{Path: "/a"}}, ErrNotEligible}, + } { + t.Run(tc.name, func(t *testing.T) { + store, digest := revisionFixture(tc.op) + r, _ := NewReviser([]string{"active-token"}, store, func(context.Context, []stageinput.Attachment, [][]byte) ([]stagestore.Attachment, error) { + t.Fatal("unexpected bind") + return nil, nil + }) + _, err := r.Revise(context.Background(), ReviseInput{StageID: "stage-1", RequestID: "request-1", ExpectedRevision: 1, ExpectedDigest: digest, Body: strings.NewReader(tc.body), Attachments: tc.attachments}) + if !errors.Is(err, tc.want) || store.reviseCalls != 0 { + t.Fatalf("err=%v calls=%d", err, store.reviseCalls) + } + }) + } +} + +func TestReviserMapsMutationErrorsAndPreservesCancellation(t *testing.T) { + for _, tc := range []struct{ source, want error }{{stagestore.ErrConflict, ErrConflict}, {stagestore.ErrNotFound, ErrNotFound}, {stagestore.ErrNotEligible, ErrNotEligible}, {errors.New("secret backend detail"), ErrStore}, {context.Canceled, context.Canceled}} { + store, digest := revisionFixture(stagestore.EditPost) + store.reviseErr = tc.source + r, _ := NewReviser(nil, store, stageinput.Bind) + _, err := r.Revise(context.Background(), ReviseInput{StageID: "stage-1", RequestID: "request-1", ExpectedRevision: 1, ExpectedDigest: digest, Body: bytes.NewBufferString("body")}) + if !errors.Is(err, tc.want) || store.reviseCalls != 1 { + t.Fatalf("source=%v err=%v calls=%d", tc.source, err, store.reviseCalls) + } + } +} + +func TestReviserCancelCallsStoreOnceAndReturnsPreloadedDestination(t *testing.T) { + store, digest := revisionFixture(stagestore.CreatePost) + r, _ := NewReviser(nil, store, stageinput.Bind) + result, err := r.Cancel(context.Background(), CancelInput{StageID: "stage-1", RequestID: "cancel-1", ExpectedRevision: 1, ExpectedDigest: digest}) + if err != nil || store.cancelCalls != 1 || result.Stored.Action != "cancel" || string(result.Destination) != `{"kind":"conversation"}` { + t.Fatalf("result=%+v err=%v calls=%d", result, err, store.cancelCalls) + } +} + +func TestReviserReplayDoesNotOpenAttachment(t *testing.T) { + store, digest := revisionFixture(stagestore.Reply) + store.detail.ServerURL, store.detail.UserID = "https://mattermost.example/api/v4", "user-1" + store.findFound = true + store.findResult = stagestore.MutationResult{Action: "revise", Replay: true, Stage: store.detail.StageSummary} + r, _ := NewReviser(nil, store, func(context.Context, []stageinput.Attachment, [][]byte) ([]stagestore.Attachment, error) { + t.Fatal("replay attempted attachment binding") + return nil, nil + }) + result, err := r.Revise(context.Background(), ReviseInput{StageID: "stage-1", RequestID: "replay-1", ExpectedRevision: 1, ExpectedDigest: digest, + Body: strings.NewReader("body"), Attachments: []Attachment{{Path: "/missing-after-first-attempt"}}}) + if err != nil || !result.Stored.Replay || store.findCalls != 1 || store.reviseCalls != 0 || store.findDigest == ([32]byte{}) { + t.Fatalf("result=%+v err=%v find=%d revise=%d", result, err, store.findCalls, store.reviseCalls) + } +} + +func TestReviserNilBodyAndAttachmentsPreserveCurrentComposition(t *testing.T) { + store, digest := revisionFixture(stagestore.Reply) + store.detail.Body = []byte("existing body") + store.detail.Attachments = []stagestore.Attachment{{SuppliedPath: "/a", CanonicalPath: "/a", RemoteFilename: "a", ByteLength: 1, ContentDigest: [32]byte{1}}} + r, _ := NewReviser(nil, store, func(context.Context, []stageinput.Attachment, [][]byte) ([]stagestore.Attachment, error) { + t.Fatal("preserve attempted attachment binding") + return nil, nil + }) + _, err := r.Revise(context.Background(), ReviseInput{StageID: "stage-1", ExpectedRevision: 1, ExpectedDigest: digest}) + if err != nil || string(store.reviseIn.Composition.Body) != "existing body" || len(store.reviseIn.Composition.Attachments) != 1 { + t.Fatalf("err=%v input=%+v", err, store.reviseIn) + } + store.detail.Body[0] = 'X' + store.detail.Attachments[0].SuppliedPath = "/changed" + if string(store.reviseIn.Composition.Body) != "existing body" || store.reviseIn.Composition.Attachments[0].SuppliedPath != "/a" { + t.Fatal("preserved composition aliases store detail") + } +} + +func TestReviserRejectsCredentialInExpectedDigest(t *testing.T) { + store, digest := revisionFixture(stagestore.Reply) + credential := hex.EncodeToString(digest[:]) + r, err := NewReviser([]string{credential}, store, stageinput.Bind) + if err != nil { + t.Fatal(err) + } + if _, err = r.Revise(context.Background(), ReviseInput{StageID: "stage-1", RequestID: "revise-1", ExpectedRevision: 1, ExpectedDigest: digest, Body: strings.NewReader("body")}); !errors.Is(err, ErrCredential) { + t.Fatalf("revise digest credential = %v", err) + } + if _, err = r.Cancel(context.Background(), CancelInput{StageID: "stage-1", RequestID: "cancel-1", ExpectedRevision: 1, ExpectedDigest: digest}); !errors.Is(err, ErrCredential) { + t.Fatalf("cancel digest credential = %v", err) + } + if store.reviseCalls != 0 || store.cancelCalls != 0 { + t.Fatalf("store mutated: revise=%d cancel=%d", store.reviseCalls, store.cancelCalls) + } +} diff --git a/internal/staging/service.go b/internal/staging/service.go index a8cbac3..272f04c 100644 --- a/internal/staging/service.go +++ b/internal/staging/service.go @@ -24,6 +24,7 @@ var ( ErrInput = errors.New("staging: message or attachment input rejected") ErrStore = errors.New("staging: stage could not be persisted") ErrConflict = errors.New("staging: request conflict") + ErrNotFound = errors.New("staging: stage not found") ) type Service struct { @@ -93,15 +94,19 @@ func (s *Service) CreatePost(ctx context.Context, in CreatePostInput) (CreatePos if err != nil { return CreatePostResult{}, err } - record, found, err := s.store.FindCreate(ctx, s.serverURL, current.ID, in.RequestID) - if err != nil { - if errors.Is(err, stagestore.ErrConflict) { - return CreatePostResult{}, ErrConflict + var record stagestore.CreateRecord + var found bool + if in.RequestID != "" { + record, found, err = s.store.FindCreate(ctx, s.serverURL, current.ID, in.RequestID) + if err != nil { + if errors.Is(err, stagestore.ErrConflict) { + return CreatePostResult{}, ErrConflict + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return CreatePostResult{}, err + } + return CreatePostResult{}, ErrStore } - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return CreatePostResult{}, err - } - return CreatePostResult{}, ErrStore } if found { if record.Stage.Operation != stagestore.CreatePost { diff --git a/internal/staging/service_test.go b/internal/staging/service_test.go index dcbe36e..9faea80 100644 --- a/internal/staging/service_test.go +++ b/internal/staging/service_test.go @@ -255,11 +255,11 @@ func TestBoundAttachmentRejectsAdditionalBidiControls(t *testing.T) { } } -func TestCreateRequiresRequestIDAndMapsConflict(t *testing.T) { +func TestCreateAllowsHumanRequestWithoutReplayIDAndMapsConflict(t *testing.T) { store := &recordingStore{} s, _, _ := dmService(t, store) _, err := s.CreatePost(context.Background(), CreatePostInput{Target: dmTarget(), Body: bytes.NewReader([]byte("hello"))}) - if !errors.Is(err, ErrInvalid) || store.calls != 0 { + if err != nil || store.calls != 1 || store.in.RequestID != "" { t.Fatalf("empty request error/calls = %v/%d", err, store.calls) } store.err = stagestore.ErrConflict diff --git a/internal/staging/types.go b/internal/staging/types.go index d67604e..5b3ebf5 100644 --- a/internal/staging/types.go +++ b/internal/staging/types.go @@ -137,3 +137,31 @@ type Store interface { } type AttachmentBinder func(context.Context, []stageinput.Attachment, [][]byte) ([]stagestore.Attachment, error) + +// RevisionStore is the narrow offline lifecycle surface used by Reviser. +type RevisionStore interface { + Show(context.Context, string) (stagestore.StageDetail, error) + FindRevise(context.Context, string, string, string, [32]byte) (stagestore.MutationResult, bool, error) + Revise(context.Context, stagestore.ReviseInput) (stagestore.MutationResult, error) + Cancel(context.Context, stagestore.CancelInput) (stagestore.MutationResult, error) +} + +type ReviseInput struct { + StageID, RequestID string + ExpectedRevision int64 + ExpectedDigest [32]byte + Revive bool + Body io.Reader + Attachments []Attachment +} + +type CancelInput struct { + StageID, RequestID string + ExpectedRevision int64 + ExpectedDigest [32]byte +} + +type RevisionResult struct { + Stored stagestore.MutationResult + Destination []byte +} diff --git a/internal/staging/validation.go b/internal/staging/validation.go index 2722edc..1205b45 100644 --- a/internal/staging/validation.go +++ b/internal/staging/validation.go @@ -156,7 +156,10 @@ func unsafeIdentityRune(r rune) bool { } func validRequestID(value string) bool { - if value == "" || len(value) > 256 || !requestCharacter(value[0], true) { + if value == "" { + return true + } + if len(value) > 256 || !requestCharacter(value[0], true) { return false } for i := 1; i < len(value); i++ { diff --git a/schemas/v2/examples/store-doctor.json b/schemas/v2/examples/store-doctor.json index 13563a9..539d428 100644 --- a/schemas/v2/examples/store-doctor.json +++ b/schemas/v2/examples/store-doctor.json @@ -1 +1 @@ -{"schema":"mm/v2/store-doctor","path":"/home/arda/.local/state/mattermost-cli/stages.sqlite3","report":{"exists":false,"filesystemSafe":null,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":3,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}} +{"schema":"mm/v2/store-doctor","path":"/home/arda/.local/state/mattermost-cli/stages.sqlite3","report":{"exists":false,"filesystemSafe":null,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":4,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}} diff --git a/schemas/v2/examples/store-migrations.json b/schemas/v2/examples/store-migrations.json index 4ef9af2..6201317 100644 --- a/schemas/v2/examples/store-migrations.json +++ b/schemas/v2/examples/store-migrations.json @@ -1 +1 @@ -{"schema":"mm/v2/store-migrations","latest":3,"migrations":[{"version":1,"name":"core-stage-state","checksum":"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"},{"version":2,"name":"immutable-local-request-receipts","checksum":"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"},{"version":3,"name":"caller-intent-stage-create-replay","checksum":"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5"}]} +{"schema":"mm/v2/store-migrations","latest":4,"migrations":[{"version":1,"name":"core-stage-state","checksum":"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"},{"version":2,"name":"immutable-local-request-receipts","checksum":"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"},{"version":3,"name":"caller-intent-stage-create-replay","checksum":"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5"},{"version":4,"name":"caller-intent-stage-revise-replay","checksum":"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4"}]} diff --git a/schemas/v2/stage-revise-request.schema.json b/schemas/v2/stage-revise-request.schema.json index 9d9837c..00a4020 100644 --- a/schemas/v2/stage-revise-request.schema.json +++ b/schemas/v2/stage-revise-request.schema.json @@ -43,14 +43,21 @@ ] }, "attachments": { - "type": "array", - "maxItems": 100, - "items": { - "$ref": "#/$defs/attachment" - } + "anyOf": [ + { + "type": "array", + "maxItems": 100, + "items": { + "$ref": "#/$defs/attachment" + } + }, + { + "type": "null" + } + ] } }, - "$comment": "Whether body and attachments apply is determined from the stored immutable operation. Runtime rejects changes for non-content operations and attachments for operations other than create_post or reply.", + "$comment": "A null body or attachments value preserves that part of the current composition; an empty attachment array clears attachments. Whether changes apply is determined from the stored immutable operation. Runtime rejects changes for non-content operations and attachment changes for operations other than create_post or reply.", "$defs": { "requestId": { "type": "string", diff --git a/schemas/v2/store-doctor.schema.json b/schemas/v2/store-doctor.schema.json index 5a474bb..b631590 100644 --- a/schemas/v2/store-doctor.schema.json +++ b/schemas/v2/store-doctor.schema.json @@ -15,7 +15,7 @@ "exists": { "type": "boolean" }, "filesystemSafe": {}, "applicationId": {}, "integrity": {}, "integrityTruncated": {}, "foreignKeyIssues": {}, "foreignKeyRows": {}, "foreignKeyTruncated": {}, "journalMode": {}, "synchronous": {}, "secureDelete": {}, "foreignKeys": {}, "trustedSchema": {}, "queryOnly": {}, "walFallback": {}, - "migrations": { "type": "object", "additionalProperties": false, "required": ["applied", "latest", "valid"], "properties": { "applied": {}, "latest": { "const": 3 }, "valid": {} } }, + "migrations": { "type": "object", "additionalProperties": false, "required": ["applied", "latest", "valid"], "properties": { "applied": {}, "latest": { "const": 4 }, "valid": {} } }, "permissionModelLimitations": { "type": "array", "items": { "$ref": "#/$defs/safeString" } } }, "allOf": [ @@ -23,14 +23,14 @@ "filesystemSafe": { "const": null }, "applicationId": { "const": null }, "integrity": { "const": null }, "integrityTruncated": { "const": null }, "foreignKeyIssues": { "const": null }, "foreignKeyRows": { "const": null }, "foreignKeyTruncated": { "const": null }, "journalMode": { "const": null }, "synchronous": { "const": null }, "secureDelete": { "const": null }, "foreignKeys": { "const": null }, "trustedSchema": { "const": null }, "queryOnly": { "const": null }, "walFallback": { "const": null }, - "migrations": { "properties": { "applied": { "const": null }, "latest": { "const": 3 }, "valid": { "const": null } } } + "migrations": { "properties": { "applied": { "const": null }, "latest": { "const": 4 }, "valid": { "const": null } } } } } }, { "if": { "properties": { "exists": { "const": true } }, "required": ["exists"] }, "then": { "properties": { "filesystemSafe": { "const": true }, "applicationId": { "const": 1296913970 }, "integrity": { "$ref": "#/$defs/nonemptyBoundedStrings" }, "integrityTruncated": { "type": "boolean" }, "foreignKeyIssues": { "type": "integer", "minimum": 0, "maximum": 21 }, "foreignKeyRows": { "$ref": "#/$defs/boundedStrings" }, "foreignKeyTruncated": { "type": "boolean" }, "journalMode": { "enum": ["wal", "delete", "truncate", "persist"] }, "synchronous": { "type": "integer" }, "secureDelete": { "type": "integer" }, "foreignKeys": { "const": true }, "trustedSchema": { "const": false }, "queryOnly": { "const": true }, "walFallback": { "type": "boolean" }, - "migrations": { "properties": { "applied": { "const": 3 }, "latest": { "const": 3 }, "valid": { "const": true } } } + "migrations": { "properties": { "applied": { "const": 4 }, "latest": { "const": 4 }, "valid": { "const": true } } } }, "allOf": [ { "oneOf": [ { "properties": { "integrity": { "const": ["ok"] }, "integrityTruncated": { "const": false } } }, diff --git a/schemas/v2/store-migrations.schema.json b/schemas/v2/store-migrations.schema.json index d11df4f..abef0d0 100644 --- a/schemas/v2/store-migrations.schema.json +++ b/schemas/v2/store-migrations.schema.json @@ -6,15 +6,17 @@ "required": ["schema", "latest", "migrations"], "properties": { "schema": { "const": "mm/v2/store-migrations" }, - "latest": { "const": 3 }, + "latest": { "const": 4 }, "migrations": { - "type": "array", "minItems": 3, "maxItems": 3, + "type": "array", "minItems": 4, "maxItems": 4, "prefixItems": [{ "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { "version": { "const": 1 }, "name": { "const": "core-stage-state" }, "checksum": { "const": "e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f" } } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { "version": { "const": 2 }, "name": { "const": "immutable-local-request-receipts" }, "checksum": { "const": "ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c" } } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { "version": { "const": 3 }, "name": { "const": "caller-intent-stage-create-replay" }, "checksum": { "const": "237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5" } + } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { + "version": { "const": 4 }, "name": { "const": "caller-intent-stage-revise-replay" }, "checksum": { "const": "c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4" } } }], "items": false } From 18f3bc0935b1bf5ae104dbfdd86a1f6a32f1c4e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 03:33:35 +0300 Subject: [PATCH 065/119] feat: add strict stage request decoding --- internal/schema/registry.go | 89 ++++- internal/schema/registry_test.go | 42 +++ internal/stagerequest/request.go | 455 ++++++++++++++++++++++++++ internal/stagerequest/request_test.go | 236 +++++++++++++ 4 files changed, 815 insertions(+), 7 deletions(-) create mode 100644 internal/stagerequest/request.go create mode 100644 internal/stagerequest/request_test.go diff --git a/internal/schema/registry.go b/internal/schema/registry.go index ab23a2d..2f07d86 100644 --- a/internal/schema/registry.go +++ b/internal/schema/registry.go @@ -9,6 +9,7 @@ import ( "io/fs" "slices" "strings" + "unicode/utf8" jsonschema "github.com/santhosh-tekuri/jsonschema/v6" @@ -100,37 +101,111 @@ func (r *Registry) Show(id string) ([]byte, error) { } func (r *Registry) Validate(id string, input io.Reader) error { + _, err := r.ReadAndValidate(id, input) + return err +} + +// ReadAndValidate consumes one bounded JSON document, validates it, and returns +// the exact bytes consumed so callers can decode the already-validated input +// without reading a potentially non-repeatable source twice. +func (r *Registry) ReadAndValidate(id string, input io.Reader) ([]byte, error) { compiled, ok := r.compiled[id] if !ok { - return fmt.Errorf("unknown schema") + return nil, fmt.Errorf("unknown schema") } limited := io.LimitReader(input, maxDocumentBytes+1) data, err := io.ReadAll(limited) if err != nil { - return &InputReadError{err: err} + return nil, &InputReadError{err: err} } if len(data) > maxDocumentBytes { - return fmt.Errorf("JSON document exceeds %d bytes", maxDocumentBytes) + return nil, fmt.Errorf("JSON document exceeds %d bytes", maxDocumentBytes) + } + if !utf8.Valid(data) { + return nil, fmt.Errorf("decode JSON document: invalid UTF-8") + } + if err := rejectUnpairedSurrogateEscapes(data); err != nil { + return nil, fmt.Errorf("decode JSON document: %w", err) } decoder := json.NewDecoder(bytes.NewReader(data)) decoder.UseNumber() var document any if err := decoder.Decode(&document); err != nil { - return fmt.Errorf("decode JSON document: %w", err) + return nil, fmt.Errorf("decode JSON document: %w", err) } var trailing any if err := decoder.Decode(&trailing); err != io.EOF { if err == nil { - return fmt.Errorf("decode JSON document: trailing JSON value") + return nil, fmt.Errorf("decode JSON document: trailing JSON value") } - return fmt.Errorf("decode JSON document trailer: %w", err) + return nil, fmt.Errorf("decode JSON document trailer: %w", err) } if err := compiled.Validate(document); err != nil { - return fmt.Errorf("document does not match %s", id) + return nil, fmt.Errorf("document does not match %s", id) + } + return bytes.Clone(data), nil +} + +// encoding/json replaces unpaired UTF-16 surrogate escapes with U+FFFD. Reject +// them first so validation never changes the meaning of caller-controlled text. +func rejectUnpairedSurrogateEscapes(data []byte) error { + for i := 0; i < len(data); i++ { + if data[i] != '"' { + continue + } + for i++; i < len(data) && data[i] != '"'; i++ { + if data[i] != '\\' { + continue + } + i++ + if i >= len(data) { + return nil // the JSON decoder reports the syntax error + } + if data[i] != 'u' || i+4 >= len(data) { + continue + } + value, ok := hexQuad(data[i+1 : i+5]) + if !ok { + continue + } + i += 4 + if value >= 0xdc00 && value <= 0xdfff { + return fmt.Errorf("unpaired UTF-16 surrogate escape") + } + if value < 0xd800 || value > 0xdbff { + continue + } + if i+6 >= len(data) || data[i+1] != '\\' || data[i+2] != 'u' { + return fmt.Errorf("unpaired UTF-16 surrogate escape") + } + low, ok := hexQuad(data[i+3 : i+7]) + if !ok || low < 0xdc00 || low > 0xdfff { + return fmt.Errorf("unpaired UTF-16 surrogate escape") + } + i += 6 + } } return nil } +func hexQuad(value []byte) (uint16, bool) { + var out uint16 + for _, c := range value { + out <<= 4 + switch { + case c >= '0' && c <= '9': + out |= uint16(c - '0') + case c >= 'a' && c <= 'f': + out |= uint16(c-'a') + 10 + case c >= 'A' && c <= 'F': + out |= uint16(c-'A') + 10 + default: + return 0, false + } + } + return out, true +} + func logicalIdentifier(document map[string]any) (string, bool) { properties, ok := document["properties"].(map[string]any) if !ok { diff --git a/internal/schema/registry_test.go b/internal/schema/registry_test.go index a4d51de..e4a7e11 100644 --- a/internal/schema/registry_test.go +++ b/internal/schema/registry_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "errors" + "fmt" "io/fs" "strings" "testing" @@ -11,6 +12,47 @@ import ( publicschemas "github.com/ardasevinc/mattermost-cli/schemas" ) +func TestReadAndValidateReturnsExactDocument(t *testing.T) { + r, err := Load() + if err != nil { + t.Fatal(err) + } + document := []byte(" \n" + `{"schema":"mm/v2/stage-cancel-request","requestId":"r","stageId":"stg_abcdefghijklmnopqrstuvwxyzABCDEF","expectedRevision":1,"expectedDigest":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}` + "\n") + got, err := r.ReadAndValidate("mm/v2/stage-cancel-request", bytes.NewReader(document)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, document) { + t.Fatalf("bytes changed: %q", got) + } + got[0] = 'x' + second, err := r.ReadAndValidate("mm/v2/stage-cancel-request", bytes.NewReader(document)) + if err != nil || !bytes.Equal(second, document) { + t.Fatalf("returned bytes were not independent: %q, %v", second, err) + } +} + +func TestValidateRejectsInvalidUnicodeWithoutReplacement(t *testing.T) { + r, err := Load() + if err != nil { + t.Fatal(err) + } + base := `{"schema":"mm/v2/stage-request","persist":true,"requestId":"r","operation":"edit_post","target":{"kind":"post","postId":"p"},"body":"%s","emoji":null,"attachments":[]}` + invalidUTF8 := []byte(fmt.Sprintf(base, "x")) + invalidUTF8[bytes.Index(invalidUTF8, []byte(`"x"`))+1] = 0xff + for name, document := range map[string][]byte{ + "raw invalid UTF-8": invalidUTF8, + "lone high surrogate": []byte(fmt.Sprintf(base, `\ud800`)), + "lone low surrogate": []byte(fmt.Sprintf(base, `\udc00`)), + } { + t.Run(name, func(t *testing.T) { + if err := r.Validate("mm/v2/stage-request", bytes.NewReader(document)); err == nil { + t.Fatal("invalid Unicode accepted") + } + }) + } +} + func TestEmbeddedExamplesValidate(t *testing.T) { registry, err := Load() if err != nil { diff --git a/internal/stagerequest/request.go b/internal/stagerequest/request.go new file mode 100644 index 0000000..37ba6d0 --- /dev/null +++ b/internal/stagerequest/request.go @@ -0,0 +1,455 @@ +// Package stagerequest decodes the public stage mutation request contracts and +// converts their syntactic values into staging-domain inputs. +package stagerequest + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "math/big" + "reflect" + "strings" + "sync" + "unicode/utf8" + + "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/internal/staging" +) + +const ( + StageSchema = "mm/v2/stage-request" + ReviseSchema = "mm/v2/stage-revise-request" + CancelSchema = "mm/v2/stage-cancel-request" +) + +var ErrInvalid = errors.New("invalid stage request") + +type Operation string + +const ( + CreatePost Operation = "create_post" + Reply Operation = "reply" + EditPost Operation = "edit_post" + DeletePost Operation = "delete_post" + React Operation = "react" + Unreact Operation = "unreact" + ResolveDM Operation = "resolve_dm" + ResolveGroupDM Operation = "resolve_group_dm" +) + +type Selector struct { + By string `json:"by"` + Value string `json:"value"` +} + +type TeamSelector struct { + By string `json:"by"` + Value string `json:"value"` +} + +// Target is the tagged union used by the public schema. Fields not belonging +// to its Kind remain empty; Team is nil for an explicit JSON null. +type Target struct { + Kind string `json:"kind"` + ConversationType string `json:"conversationType,omitempty"` + Selector Selector `json:"selector,omitempty"` + Team *TeamSelector `json:"team,omitempty"` + PostID string `json:"postId,omitempty"` + Username string `json:"username,omitempty"` + Usernames []string `json:"usernames,omitempty"` +} + +func (t Target) MarshalJSON() ([]byte, error) { + switch t.Kind { + case "conversation": + return json.Marshal(struct { + Kind string `json:"kind"` + ConversationType string `json:"conversationType"` + Selector Selector `json:"selector"` + Team *TeamSelector `json:"team"` + }{t.Kind, t.ConversationType, t.Selector, t.Team}) + case "post": + return json.Marshal(struct { + Kind string `json:"kind"` + PostID string `json:"postId"` + }{t.Kind, t.PostID}) + case "user": + return json.Marshal(struct { + Kind string `json:"kind"` + Username string `json:"username"` + }{t.Kind, t.Username}) + case "users": + return json.Marshal(struct { + Kind string `json:"kind"` + Usernames []string `json:"usernames"` + }{t.Kind, t.Usernames}) + default: + return json.Marshal(struct { + Kind string `json:"kind"` + }{t.Kind}) + } +} + +// Attachment preserves explicit JSON null as nil. Conversion maps nil to the +// staging layer's empty value, meaning derive filename or detect media type. +type Attachment struct { + Path string `json:"path"` + RemoteFilename *string `json:"remoteFilename"` + MediaType *string `json:"mediaType"` +} + +type StageRequest struct { + Schema string `json:"schema"` + Persist bool `json:"persist"` + RequestID *string `json:"requestId"` + Operation Operation `json:"operation"` + Target Target `json:"target"` + Body *string `json:"body"` + Emoji *string `json:"emoji"` + Attachments []Attachment `json:"attachments"` +} + +type ReviseRequest struct { + Schema string `json:"schema"` + RequestID string `json:"requestId"` + StageID string `json:"stageId"` + ExpectedRevision ExactInt64 `json:"expectedRevision"` + ExpectedDigest string `json:"expectedDigest"` + Revive bool `json:"revive"` + Body *string `json:"body"` + Attachments []Attachment `json:"attachments"` +} + +type CancelRequest struct { + Schema string `json:"schema"` + RequestID string `json:"requestId"` + StageID string `json:"stageId"` + ExpectedRevision ExactInt64 `json:"expectedRevision"` + ExpectedDigest string `json:"expectedDigest"` +} + +type ExactInt64 int64 + +func (n *ExactInt64) UnmarshalJSON(data []byte) error { + var number json.Number + if err := json.Unmarshal(data, &number); err != nil { + return err + } + rational, ok := new(big.Rat).SetString(number.String()) + if !ok || !rational.IsInt() || !rational.Num().IsInt64() { + return fmt.Errorf("not an exact int64") + } + *n = ExactInt64(rational.Num().Int64()) + return nil +} + +func (n ExactInt64) MarshalJSON() ([]byte, error) { return []byte(fmt.Sprint(int64(n))), nil } + +type Decoder struct{ registry *schema.Registry } + +var conversionRegistry struct { + sync.Once + value *schema.Registry + err error +} + +func NewDecoder() (*Decoder, error) { + r, err := schema.Load() + if err != nil { + return nil, err + } + return &Decoder{registry: r}, nil +} + +func (d *Decoder) DecodeStage(input io.Reader) (StageRequest, error) { + return decode[StageRequest](d, StageSchema, input) +} + +func (d *Decoder) DecodeRevise(input io.Reader) (ReviseRequest, error) { + return decode[ReviseRequest](d, ReviseSchema, input) +} + +func (d *Decoder) DecodeCancel(input io.Reader) (CancelRequest, error) { + return decode[CancelRequest](d, CancelSchema, input) +} + +func decode[T any](d *Decoder, id string, input io.Reader) (T, error) { + var zero T + if d == nil || d.registry == nil || input == nil { + return zero, ErrInvalid + } + raw, err := d.registry.ReadAndValidate(id, input) + if err != nil { + if schema.IsInputReadError(err) { + return zero, err + } + return zero, fmt.Errorf("%w: %v", ErrInvalid, err) + } + if err := rejectDuplicateNames(raw); err != nil { + return zero, fmt.Errorf("%w: duplicate object member", ErrInvalid) + } + var value T + if err := json.Unmarshal(raw, &value); err != nil { + return zero, fmt.Errorf("%w: typed decode", ErrInvalid) + } + return value, nil +} + +func validateConversion(id string, value any) error { + if !validUTF8Strings(reflect.ValueOf(value)) { + return ErrInvalid + } + conversionRegistry.Do(func() { conversionRegistry.value, conversionRegistry.err = schema.Load() }) + if conversionRegistry.err != nil { + return conversionRegistry.err + } + raw, err := json.Marshal(value) + if err != nil || conversionRegistry.value.Validate(id, bytes.NewReader(raw)) != nil { + return ErrInvalid + } + return nil +} + +func validUTF8Strings(value reflect.Value) bool { + if !value.IsValid() { + return true + } + if value.Kind() == reflect.Pointer { + return value.IsNil() || validUTF8Strings(value.Elem()) + } + switch value.Kind() { + case reflect.String: + return utf8.ValidString(value.String()) + case reflect.Struct: + for i := 0; i < value.NumField(); i++ { + if !validUTF8Strings(value.Field(i)) { + return false + } + } + case reflect.Slice, reflect.Array: + for i := 0; i < value.Len(); i++ { + if !validUTF8Strings(value.Index(i)) { + return false + } + } + } + return true +} + +func rejectDuplicateNames(raw []byte) error { + decoder := json.NewDecoder(bytes.NewReader(raw)) + var walk func() error + walk = func() error { + token, err := decoder.Token() + if err != nil { + return err + } + delim, ok := token.(json.Delim) + if !ok { + return nil + } + switch delim { + case '{': + seen := make(map[string]struct{}) + for decoder.More() { + nameToken, err := decoder.Token() + if err != nil { + return err + } + name, ok := nameToken.(string) + if !ok { + return ErrInvalid + } + if _, exists := seen[name]; exists { + return ErrInvalid + } + seen[name] = struct{}{} + if err := walk(); err != nil { + return err + } + } + _, err = decoder.Token() + return err + case '[': + for decoder.More() { + if err := walk(); err != nil { + return err + } + } + _, err = decoder.Token() + return err + default: + return ErrInvalid + } + } + return walk() +} + +func (t Target) StagingTarget() (staging.Target, error) { + if t.Kind != "conversation" { + return staging.Target{}, ErrInvalid + } + conversation := map[string]staging.ConversationType{"dm": staging.Direct, "group": staging.Group, "channel": staging.Channel}[t.ConversationType] + selector := map[string]staging.SelectorType{"username": staging.ByUsername, "id": staging.ByID, "name": staging.ByName}[t.Selector.By] + if conversation == 0 || selector == 0 || t.Selector.Value == "" { + return staging.Target{}, ErrInvalid + } + out := staging.Target{Conversation: conversation, Selector: selector, Value: t.Selector.Value} + if t.Team != nil { + by := map[string]staging.SelectorType{"id": staging.ByID, "name": staging.ByName}[t.Team.By] + if by == 0 || t.Team.Value == "" { + return staging.Target{}, ErrInvalid + } + out.Team = &staging.TeamSelector{By: by, Value: t.Team.Value} + } + if conversation == staging.Direct && (selector != staging.ByUsername || out.Team != nil) || + conversation == staging.Group && (selector != staging.ByID || out.Team != nil) || + conversation == staging.Channel && selector == staging.ByID && out.Team != nil || + conversation == staging.Channel && selector == staging.ByName && out.Team == nil { + return staging.Target{}, ErrInvalid + } + return out, nil +} + +func (a Attachment) StagingAttachment() stageinput.Attachment { + out := stageinput.Attachment{Path: a.Path} + if a.RemoteFilename != nil { + out.RemoteFilename = *a.RemoteFilename + } + if a.MediaType != nil { + out.MediaType = *a.MediaType + } + return out +} + +func StagingAttachments(values []Attachment) []stageinput.Attachment { + if values == nil { + return nil + } + out := make([]stageinput.Attachment, len(values)) + for i := range values { + out[i] = values[i].StagingAttachment() + } + return out +} + +func (r StageRequest) CreatePostInput() (staging.CreatePostInput, error) { + if validateConversion(StageSchema, r) != nil || !r.Persist || r.Operation != CreatePost || r.RequestID == nil || r.Body == nil || r.Emoji != nil { + return staging.CreatePostInput{}, ErrInvalid + } + target, err := r.Target.StagingTarget() + if err != nil { + return staging.CreatePostInput{}, err + } + return staging.CreatePostInput{RequestID: *r.RequestID, Target: target, Body: strings.NewReader(*r.Body), Attachments: StagingAttachments(r.Attachments)}, nil +} + +func (r StageRequest) DryRunCreatePostInput() (staging.DryRunInput, error) { + if validateConversion(StageSchema, r) != nil || r.Persist || r.Operation != CreatePost || r.RequestID != nil || r.Body != nil || r.Emoji != nil || len(r.Attachments) != 0 { + return staging.DryRunInput{}, ErrInvalid + } + target, err := r.Target.StagingTarget() + return staging.DryRunInput{Target: target}, err +} + +func (r StageRequest) ReplyInput() (staging.ReplyInput, error) { + if validateConversion(StageSchema, r) != nil || !r.Persist || r.Operation != Reply || r.RequestID == nil || r.Body == nil || r.Emoji != nil || r.Target.Kind != "post" || r.Target.PostID == "" { + return staging.ReplyInput{}, ErrInvalid + } + return staging.ReplyInput{RequestID: *r.RequestID, PostID: r.Target.PostID, Body: strings.NewReader(*r.Body), Attachments: StagingAttachments(r.Attachments)}, nil +} + +func (r StageRequest) EditPostInput() (staging.EditPostInput, error) { + if validateConversion(StageSchema, r) != nil || !r.Persist || r.Operation != EditPost || r.RequestID == nil || r.Body == nil || r.Emoji != nil || r.Target.Kind != "post" || r.Target.PostID == "" || len(r.Attachments) != 0 { + return staging.EditPostInput{}, ErrInvalid + } + return staging.EditPostInput{RequestID: *r.RequestID, PostID: r.Target.PostID, Body: strings.NewReader(*r.Body)}, nil +} + +func (r StageRequest) DeletePostInput() (staging.DeletePostInput, error) { + if validateConversion(StageSchema, r) != nil || !r.Persist || r.Operation != DeletePost || r.RequestID == nil || r.Emoji != nil || !r.contentlessPost() { + return staging.DeletePostInput{}, ErrInvalid + } + return staging.DeletePostInput{RequestID: *r.RequestID, PostID: r.Target.PostID}, nil +} + +func (r StageRequest) ReactionInput() (staging.ReactionInput, error) { + if validateConversion(StageSchema, r) != nil || !r.Persist || (r.Operation != React && r.Operation != Unreact) || r.RequestID == nil || !r.contentlessPost() || r.Emoji == nil { + return staging.ReactionInput{}, ErrInvalid + } + return staging.ReactionInput{RequestID: *r.RequestID, PostID: r.Target.PostID, Emoji: *r.Emoji}, nil +} + +func (r StageRequest) PostDryRunInput() (staging.PostDryRunInput, error) { + if validateConversion(StageSchema, r) != nil || r.Persist || (r.Operation != Reply && r.Operation != EditPost && r.Operation != DeletePost) || r.RequestID != nil || r.Emoji != nil || !r.contentlessPost() { + return staging.PostDryRunInput{}, ErrInvalid + } + return staging.PostDryRunInput{PostID: r.Target.PostID}, nil +} + +func (r StageRequest) ReactionDryRunInput() (staging.ReactionDryRunInput, error) { + if validateConversion(StageSchema, r) != nil || r.Persist || (r.Operation != React && r.Operation != Unreact) || r.RequestID != nil || !r.contentlessPost() || r.Emoji == nil { + return staging.ReactionDryRunInput{}, ErrInvalid + } + return staging.ReactionDryRunInput{PostID: r.Target.PostID, Emoji: *r.Emoji}, nil +} + +func (r StageRequest) contentlessPost() bool { + return r.Target.Kind == "post" && r.Target.PostID != "" && r.Body == nil && len(r.Attachments) == 0 +} + +func (r StageRequest) ResolveDMTarget() (staging.Target, error) { + if validateConversion(StageSchema, r) != nil || r.Operation != ResolveDM || r.Target.Kind != "user" || r.Target.Username == "" || r.Body != nil || r.Emoji != nil || len(r.Attachments) != 0 || r.Persist != (r.RequestID != nil) { + return staging.Target{}, ErrInvalid + } + return staging.Target{Conversation: staging.Direct, Selector: staging.ByUsername, Value: r.Target.Username}, nil +} + +func (r StageRequest) ResolveGroupUsernames() ([]string, error) { + if validateConversion(StageSchema, r) != nil || r.Operation != ResolveGroupDM || r.Target.Kind != "users" || len(r.Target.Usernames) < 2 || r.Body != nil || r.Emoji != nil || len(r.Attachments) != 0 || r.Persist != (r.RequestID != nil) { + return nil, ErrInvalid + } + return append([]string(nil), r.Target.Usernames...), nil +} + +func decodeDigest(value string) ([32]byte, error) { + var out [32]byte + if len(value) != 64 { + return out, ErrInvalid + } + for _, c := range value { + if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) { + return out, ErrInvalid + } + } + decoded, err := hex.DecodeString(value) + if err != nil || len(decoded) != len(out) { + return out, ErrInvalid + } + copy(out[:], decoded) + return out, nil +} + +func (r ReviseRequest) ReviseInput() (staging.ReviseInput, error) { + digest, err := decodeDigest(r.ExpectedDigest) + if err != nil || validateConversion(ReviseSchema, r) != nil { + return staging.ReviseInput{}, ErrInvalid + } + var body io.Reader + if r.Body != nil { + body = strings.NewReader(*r.Body) + } + return staging.ReviseInput{StageID: r.StageID, RequestID: r.RequestID, ExpectedRevision: int64(r.ExpectedRevision), ExpectedDigest: digest, Revive: r.Revive, Body: body, Attachments: StagingAttachments(r.Attachments)}, nil +} + +func (r CancelRequest) CancelInput() (staging.CancelInput, error) { + digest, err := decodeDigest(r.ExpectedDigest) + if err != nil || validateConversion(CancelSchema, r) != nil { + return staging.CancelInput{}, ErrInvalid + } + return staging.CancelInput{StageID: r.StageID, RequestID: r.RequestID, ExpectedRevision: int64(r.ExpectedRevision), ExpectedDigest: digest}, nil +} diff --git a/internal/stagerequest/request_test.go b/internal/stagerequest/request_test.go new file mode 100644 index 0000000..c442ac3 --- /dev/null +++ b/internal/stagerequest/request_test.go @@ -0,0 +1,236 @@ +package stagerequest + +import ( + "errors" + "io" + "reflect" + "strings" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/internal/staging" +) + +func decoder(t *testing.T) *Decoder { + t.Helper() + d, err := NewDecoder() + if err != nil { + t.Fatal(err) + } + return d +} + +func stageJSON(persist bool, operation, target, body, emoji, attachments string) string { + requestID := "null" + if persist { + requestID = `"req-1"` + } + return `{"schema":"mm/v2/stage-request","persist":` + map[bool]string{true: "true", false: "false"}[persist] + + `,"requestId":` + requestID + `,"operation":"` + operation + `","target":` + target + + `,"body":` + body + `,"emoji":` + emoji + `,"attachments":` + attachments + `}` +} + +func TestDecodeStageEveryOperationPersistAndDryRun(t *testing.T) { + post := `{"kind":"post","postId":"post-1"}` + conversation := `{"kind":"conversation","conversationType":"channel","selector":{"by":"name","value":"town-square"},"team":{"by":"id","value":"team-1"}}` + user := `{"kind":"user","username":"alice"}` + users := `{"kind":"users","usernames":["alice","bob"]}` + cases := []struct{ operation, target, body, emoji, attachments string }{ + {"create_post", conversation, `"hello"`, "null", `[{"path":"/tmp/a","remoteFilename":null,"mediaType":null}]`}, + {"reply", post, `"hello"`, "null", `[{"path":"/tmp/a","remoteFilename":"a.txt","mediaType":"text/plain"}]`}, + {"edit_post", post, `"hello"`, "null", `[]`}, + {"delete_post", post, "null", "null", `[]`}, + {"react", post, "null", `"wave"`, `[]`}, + {"unreact", post, "null", `"wave"`, `[]`}, + {"resolve_dm", user, "null", "null", `[]`}, + {"resolve_group_dm", users, "null", "null", `[]`}, + } + for _, tc := range cases { + t.Run(tc.operation+"/persist", func(t *testing.T) { + got, err := decoder(t).DecodeStage(strings.NewReader(stageJSON(true, tc.operation, tc.target, tc.body, tc.emoji, tc.attachments))) + if err != nil || !got.Persist || got.RequestID == nil || got.Operation != Operation(tc.operation) { + t.Fatalf("got %#v, %v", got, err) + } + }) + t.Run(tc.operation+"/dry", func(t *testing.T) { + got, err := decoder(t).DecodeStage(strings.NewReader(stageJSON(false, tc.operation, tc.target, "null", tc.emoji, `[]`))) + if err != nil || got.Persist || got.RequestID != nil || got.Body != nil || got.Attachments == nil { + t.Fatalf("got %#v, %v", got, err) + } + }) + } +} + +func TestDecodeStageMarkdownBoundsAndExactReader(t *testing.T) { + for _, size := range []int{1, 16383} { + body := strings.Repeat("x", size) + r, err := decoder(t).DecodeStage(strings.NewReader(stageJSON(true, "edit_post", `{"kind":"post","postId":"p"}`, `"`+body+`"`, "null", `[]`))) + if err != nil { + t.Fatalf("size %d: %v", size, err) + } + in, err := r.EditPostInput() + if err != nil { + t.Fatal(err) + } + got, _ := io.ReadAll(in.Body) + if string(got) != body { + t.Fatalf("body changed at size %d", size) + } + } + body := strings.Repeat("x", 16384) + if _, err := decoder(t).DecodeStage(strings.NewReader(stageJSON(true, "edit_post", `{"kind":"post","postId":"p"}`, `"`+body+`"`, "null", `[]`))); !errors.Is(err, ErrInvalid) { + t.Fatalf("long body: %v", err) + } +} + +func TestDecodeRejectsMalformedTrailingDuplicateAndUnknown(t *testing.T) { + valid := stageJSON(false, "delete_post", `{"kind":"post","postId":"p"}`, "null", "null", `[]`) + cases := []string{ + `{`, valid + `{}`, strings.Replace(valid, `"persist":false`, `"persist":false,"persist":false`, 1), + strings.Replace(valid, `"attachments":[]`, `"attachments":[],"unknown":true`, 1), + } + for i, input := range cases { + if _, err := decoder(t).DecodeStage(strings.NewReader(input)); !errors.Is(err, ErrInvalid) { + t.Fatalf("case %d: %v", i, err) + } + } +} + +type failingReader struct{ err error } + +func (r failingReader) Read([]byte) (int, error) { return 0, r.err } + +func TestDecodePreservesPhysicalReadFailure(t *testing.T) { + sentinel := errors.New("disk vanished") + _, err := decoder(t).DecodeStage(failingReader{sentinel}) + if !schema.IsInputReadError(err) || !errors.Is(err, sentinel) || errors.Is(err, ErrInvalid) { + t.Fatalf("wrong error identity: %v", err) + } +} + +func TestStageConversionsAndNullAttachmentSemantics(t *testing.T) { + r, err := decoder(t).DecodeStage(strings.NewReader(stageJSON(true, "create_post", + `{"kind":"conversation","conversationType":"channel","selector":{"by":"name","value":"town-square"},"team":{"by":"name","value":"main"}}`, + `"line 1\nline 2"`, "null", `[{"path":"relative.txt","remoteFilename":null,"mediaType":null}]`))) + if err != nil { + t.Fatal(err) + } + in, err := r.CreatePostInput() + if err != nil { + t.Fatal(err) + } + if in.Target.Conversation != staging.Channel || in.Target.Selector != staging.ByName || in.Target.Team == nil || in.Target.Team.By != staging.ByName || in.Attachments[0].RemoteFilename != "" || in.Attachments[0].MediaType != "" { + t.Fatalf("bad conversion: %#v", in) + } + body, _ := io.ReadAll(in.Body) + if string(body) != "line 1\nline 2" { + t.Fatalf("body changed: %q", body) + } + + mutated := r + mutated.Operation = Reply + if _, err := mutated.CreatePostInput(); !errors.Is(err, ErrInvalid) { + t.Fatalf("mutated DTO accepted: %v", err) + } +} + +func TestReviseAndCancelDecodeAndStoreConversions(t *testing.T) { + digestText := strings.Repeat("ab", 32) + reviseJSON := `{"schema":"mm/v2/stage-revise-request","requestId":"r","stageId":"stg_abcdefghijklmnopqrstuvwxyzABCDEF","expectedRevision":2,"expectedDigest":"` + digestText + `","revive":true,"body":null,"attachments":[]}` + revise, err := decoder(t).DecodeRevise(strings.NewReader(reviseJSON)) + if err != nil || revise.Body != nil || revise.Attachments == nil { + t.Fatalf("decode revise: %#v %v", revise, err) + } + reviseInput, err := revise.ReviseInput() + if err != nil || reviseInput.ExpectedDigest[0] != 0xab || reviseInput.ExpectedRevision != 2 || !reviseInput.Revive || reviseInput.Body != nil { + t.Fatalf("revise conversion: %#v %v", reviseInput, err) + } + preserve, err := decoder(t).DecodeRevise(strings.NewReader(strings.Replace(reviseJSON, `"attachments":[]`, `"attachments":null`, 1))) + if err != nil || preserve.Attachments != nil { + t.Fatalf("decode attachment preservation: %#v %v", preserve, err) + } + preserveInput, err := preserve.ReviseInput() + if err != nil || preserveInput.Attachments != nil { + t.Fatalf("convert attachment preservation: %#v %v", preserveInput, err) + } + + cancelJSON := `{"schema":"mm/v2/stage-cancel-request","requestId":"r","stageId":"stg_abcdefghijklmnopqrstuvwxyzABCDEF","expectedRevision":3,"expectedDigest":"` + digestText + `"}` + cancel, err := decoder(t).DecodeCancel(strings.NewReader(cancelJSON)) + if err != nil { + t.Fatal(err) + } + cancelInput, err := cancel.CancelInput() + if err != nil || cancelInput.ExpectedDigest != reviseInput.ExpectedDigest || cancelInput.ExpectedRevision != 3 { + t.Fatalf("cancel conversion: %#v %v", cancelInput, err) + } + + cancel.ExpectedDigest = strings.ToUpper(digestText) + if _, err := cancel.CancelInput(); !errors.Is(err, ErrInvalid) { + t.Fatalf("uppercase digest accepted: %v", err) + } + revise.Attachments = []Attachment{{Path: "/tmp/a"}} + if _, err := revise.ReviseInput(); err != nil { + t.Fatalf("raw attachment rejected: %v", err) + } +} + +func TestResolveConversionsCloneUsernames(t *testing.T) { + dm := StageRequest{Schema: StageSchema, Operation: ResolveDM, Target: Target{Kind: "user", Username: "alice"}, Attachments: []Attachment{}} + target, err := dm.ResolveDMTarget() + if err != nil || target.Conversation != staging.Direct || target.Selector != staging.ByUsername || target.Value != "alice" { + t.Fatalf("dm: %#v %v", target, err) + } + group := StageRequest{Schema: StageSchema, Operation: ResolveGroupDM, Target: Target{Kind: "users", Usernames: []string{"a", "b"}}, Attachments: []Attachment{}} + names, err := group.ResolveGroupUsernames() + if err != nil || !reflect.DeepEqual(names, group.Target.Usernames) { + t.Fatalf("group: %#v %v", names, err) + } + names[0] = "changed" + if group.Target.Usernames[0] != "a" { + t.Fatal("conversion leaked mutable slice") + } +} + +func TestReviseInputClonesRawAttachments(t *testing.T) { + r := ReviseRequest{Schema: ReviseSchema, RequestID: "r", StageID: "stg_abcdefghijklmnopqrstuvwxyzABCDEF", ExpectedRevision: 1, ExpectedDigest: strings.Repeat("01", 32), Attachments: []Attachment{{Path: "/a"}}} + in, err := r.ReviseInput() + if err != nil { + t.Fatal(err) + } + r.Attachments[0].Path = "/changed" + if in.Attachments[0].Path != "/a" { + t.Fatal("conversion leaked attachment slice") + } +} + +func TestDecodeIntegralRevisionLexicalFormsExactly(t *testing.T) { + for _, lexical := range []string{"1.0", "1e0", "9007199254740991.0", "9007199254740991e0"} { + input := `{"schema":"mm/v2/stage-cancel-request","requestId":"r","stageId":"stg_abcdefghijklmnopqrstuvwxyzABCDEF","expectedRevision":` + lexical + `,"expectedDigest":"` + strings.Repeat("0", 64) + `"}` + request, err := decoder(t).DecodeCancel(strings.NewReader(input)) + if err != nil { + t.Fatalf("%s: %v", lexical, err) + } + converted, err := request.CancelInput() + if err != nil || converted.ExpectedRevision < 1 || converted.ExpectedRevision > 9007199254740991 { + t.Fatalf("%s: %#v, %v", lexical, converted, err) + } + } +} + +func TestConversionsRejectMutatedContractFields(t *testing.T) { + base := CancelRequest{Schema: CancelSchema, RequestID: "r", StageID: "stg_abcdefghijklmnopqrstuvwxyzABCDEF", ExpectedRevision: 1, ExpectedDigest: strings.Repeat("0", 64)} + for name, mutate := range map[string]func(*CancelRequest){ + "request pattern": func(r *CancelRequest) { r.RequestID = "bad id" }, + "stage id": func(r *CancelRequest) { r.StageID = "stg_bad" }, + "revision max": func(r *CancelRequest) { r.ExpectedRevision = 9007199254740992 }, + "invalid UTF-8": func(r *CancelRequest) { r.RequestID = string([]byte{0xff}) }, + } { + t.Run(name, func(t *testing.T) { + request := base + mutate(&request) + if _, err := request.CancelInput(); !errors.Is(err, ErrInvalid) { + t.Fatalf("mutated DTO accepted: %v", err) + } + }) + } +} From 6941607086081512a2b6586f11d06b0dee5d040f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 03:33:40 +0300 Subject: [PATCH 066/119] feat: add public stage documents --- internal/stageoutput/output.go | 394 ++++++++++++++++++++++++++++ internal/stageoutput/output_test.go | 179 +++++++++++++ 2 files changed, 573 insertions(+) create mode 100644 internal/stageoutput/output.go create mode 100644 internal/stageoutput/output_test.go diff --git a/internal/stageoutput/output.go b/internal/stageoutput/output.go new file mode 100644 index 0000000..b232ab3 --- /dev/null +++ b/internal/stageoutput/output.go @@ -0,0 +1,394 @@ +// Package stageoutput converts trusted staging domain values into strict public +// stage documents. Every constructor validates the finished document against +// its embedded schema and rejects active credentials anywhere in emitted text. +package stageoutput + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "reflect" + "strconv" + "strings" + "sync" + "time" + + "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/internal/stagecursor" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/internal/staging" +) + +var ErrInvalid = errors.New("stage output: invalid or unsafe state") + +type Binding struct { + ServerURL string `json:"serverUrl"` + ServerID *string `json:"serverId"` + UserID string `json:"userId"` +} +type Summary struct { + StageID string `json:"stageId"` + StageRef string `json:"stageRef"` + Revision int64 `json:"revision"` + Operation string `json:"operation"` + SemanticDigest string `json:"semanticDigest"` + Lifecycle string `json:"lifecycle"` + Recovery string `json:"recovery"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` + Binding Binding `json:"binding"` + Destination staging.Destination `json:"destination"` +} +type Preview struct { + Schema string `json:"schema"` + Persist bool `json:"persist"` + Operation string `json:"operation"` + Binding Binding `json:"binding"` + Destination staging.Destination `json:"destination"` + Plan staging.Plan `json:"plan"` + ContentValidated bool `json:"contentValidated"` +} +type Receipt struct { + Schema string `json:"schema"` + Action string `json:"action"` + Revived bool `json:"revived"` + Replayed bool `json:"replayed"` + RecordedAt string `json:"recordedAt"` + Stage Summary `json:"stage"` +} +type Content struct { + State string `json:"state"` + Body *string `json:"body"` +} +type Attachment struct { + Path string `json:"path"` + CanonicalPath string `json:"canonicalPath"` + RemoteFilename string `json:"remoteFilename"` + ByteLength int64 `json:"byteLength"` + MediaType string `json:"mediaType"` + ContentDigest string `json:"contentDigest"` +} +type Stage struct { + Schema string `json:"schema"` + Stage Summary `json:"stage"` + RevisionState string `json:"revisionState"` + Content Content `json:"content"` + AttachmentState string `json:"attachmentState"` + Attachments []Attachment `json:"attachments"` + Plan staging.Plan `json:"plan"` +} +type Stages struct { + Schema string `json:"schema"` + Stages []Summary `json:"stages"` + NextCursor *string `json:"nextCursor"` +} + +func NewPreview(operation stagestore.Operation, in staging.Preview, credentials []string) (Preview, error) { + out := Preview{"mm/v2/stage-preview", false, string(operation), binding(in.ServerURL, in.ServerID, in.UserID), cloneDestination(in.Destination), clonePlan(in.Plan), false} + if !validPlan(operation, out.Plan, -1) { + return Preview{}, ErrInvalid + } + return out, validate("mm/v2/stage-preview", out, credentials) +} + +func NewCreateReceipt(in staging.CreatePostResult, credentials []string) (Receipt, error) { + d, err := decodeDestination(in.Stored.Stage.Operation, mustJSON(in.Preview.Destination)) + if err != nil { + return Receipt{}, ErrInvalid + } + return newReceipt(in.Stored, d, credentials) +} + +func NewReceipt(in stagestore.MutationResult, destination json.RawMessage, credentials []string) (Receipt, error) { + d, err := decodeDestination(in.Stage.Operation, destination) + if err != nil { + return Receipt{}, ErrInvalid + } + return newReceipt(in, d, credentials) +} + +func newReceipt(in stagestore.MutationResult, d staging.Destination, credentials []string) (Receipt, error) { + actions := map[string]string{"create": "created", "revise": "revised", "cancel": "canceled"} + if !validSummaryTimes(in.Stage) || in.RecordedAt.IsZero() || emittedTime(in.RecordedAt).Before(emittedTime(in.Stage.UpdatedAt)) { + return Receipt{}, ErrInvalid + } + out := Receipt{"mm/v2/stage-receipt", actions[in.Action], in.Revived, in.Replay, stamp(in.RecordedAt), summary(in.Stage, d)} + return out, validate("mm/v2/stage-receipt", out, credentials) +} + +func NewStage(in stagestore.StageDetail, credentials []string) (Stage, error) { + if !stagestore.VerifyDetail(in) { + return Stage{}, ErrInvalid + } + d, err := decodeDestination(in.Operation, in.Destination) + if err != nil { + return Stage{}, ErrInvalid + } + var plan staging.Plan + if err := strictDecode(in.Plan, &plan); err != nil { + return Stage{}, ErrInvalid + } + plan = clonePlan(plan) + contentful := in.Operation == stagestore.CreatePost || in.Operation == stagestore.Reply || in.Operation == stagestore.EditPost + pruned := in.Lifecycle == stagestore.LifecycleCompleted || in.Lifecycle == stagestore.LifecyclePruned + attachmentCount := len(in.Attachments) + if pruned { + attachmentCount = -1 + } + if !validSummaryTimes(in.StageSummary) || in.RevisionCreatedAt.IsZero() || + emittedTime(in.RevisionCreatedAt).Before(emittedTime(in.CreatedAt)) || emittedTime(in.RevisionCreatedAt).After(emittedTime(in.UpdatedAt)) || + !validPlan(in.Operation, plan, attachmentCount) { + return Stage{}, ErrInvalid + } + out := Stage{Schema: "mm/v2/stage", Stage: summary(in.StageSummary, d), RevisionState: "current", Attachments: make([]Attachment, 0), Plan: plan} + if (!contentful && (in.Body != nil || len(in.Attachments) != 0)) || + (in.Operation == stagestore.EditPost && len(in.Attachments) != 0) || + (contentful && !pruned && in.Body == nil) || + (pruned && (in.Body != nil || len(in.Attachments) != 0)) { + return Stage{}, ErrInvalid + } + if !contentful { + out.Content = Content{"none", nil} + } else if pruned || in.Body == nil { + out.Content = Content{"pruned", nil} + } else { + body := string(bytes.Clone(in.Body)) + out.Content = Content{"present", &body} + } + if !contentful || in.Operation == stagestore.EditPost { + out.AttachmentState = "none" + } else if pruned { + out.AttachmentState = "none" + for _, step := range plan.Steps { + if step.Type == "upload_attachment" { + out.AttachmentState = "pruned" + break + } + } + } else if len(in.Attachments) == 0 { + out.AttachmentState = "none" + } else { + out.AttachmentState = "retained" + } + if out.AttachmentState == "retained" { + for _, a := range in.Attachments { + out.Attachments = append(out.Attachments, Attachment{a.SuppliedPath, a.CanonicalPath, a.RemoteFilename, a.ByteLength, a.MediaType, hex.EncodeToString(a.ContentDigest[:])}) + } + } + return out, validate("mm/v2/stage", out, credentials) +} + +func NewStages(page stagestore.ListPage, credentials []string) (Stages, error) { + in := page.Records + if len(in) > 100 { + return Stages{}, ErrInvalid + } + out := Stages{Schema: "mm/v2/stages", Stages: make([]Summary, 0, len(in))} + if page.NextCursor != nil { + if len(in) == 0 { + return Stages{}, ErrInvalid + } + boundary, err := stagecursor.Decode(*page.NextCursor) + last := in[len(in)-1] + if err != nil || !boundary.UpdatedAt.Equal(last.UpdatedAt) || boundary.StageID != last.ID { + return Stages{}, ErrInvalid + } + v := *page.NextCursor + out.NextCursor = &v + } + seen := make(map[string]struct{}, len(in)) + for i, record := range in { + updated := emittedTime(record.UpdatedAt) + if _, exists := seen[record.ID]; exists || !validSummaryTimes(record.StageSummary) || + i > 0 && (updated.After(emittedTime(in[i-1].UpdatedAt)) || (updated.Equal(emittedTime(in[i-1].UpdatedAt)) && record.ID < in[i-1].ID)) { + return Stages{}, ErrInvalid + } + seen[record.ID] = struct{}{} + d, err := decodeDestination(record.Operation, record.Destination) + if err != nil { + return Stages{}, ErrInvalid + } + out.Stages = append(out.Stages, summary(record.StageSummary, d)) + } + return out, validate("mm/v2/stages", out, credentials) +} + +func summary(s stagestore.StageSummary, d staging.Destination) Summary { + return Summary{s.ID, s.ID + "@" + strconv.FormatInt(s.Revision, 10), s.Revision, string(s.Operation), hex.EncodeToString(s.SemanticDigest[:]), string(s.Lifecycle), string(s.Recovery), stamp(s.CreatedAt), stamp(s.UpdatedAt), binding(s.ServerURL, s.ServerID, s.UserID), cloneDestination(d)} +} +func binding(url, id, user string) Binding { + var p *string + if id != "" { + v := id + p = &v + } + return Binding{url, p, user} +} +func stamp(t time.Time) string { return t.UTC().Format("2006-01-02T15:04:05.000Z") } +func emittedTime(t time.Time) time.Time { return t.UTC().Truncate(time.Millisecond) } +func validSummaryTimes(s stagestore.StageSummary) bool { + return s.SemanticDigest != ([32]byte{}) && !s.CreatedAt.IsZero() && !s.UpdatedAt.IsZero() && !emittedTime(s.UpdatedAt).Before(emittedTime(s.CreatedAt)) +} + +func validPlan(operation stagestore.Operation, plan staging.Plan, attachmentCount int) bool { + if len(plan.Steps) == 0 { + return false + } + for i, step := range plan.Steps { + if step.Ordinal != i+1 { + return false + } + } + terminalType, terminalCondition := "", "always" + switch operation { + case stagestore.CreatePost, stagestore.Reply: + terminalType = "create_post" + start := 0 + if first := plan.Steps[0]; first.Type == "resolve_conversation" && first.Condition == "if_missing" { + start = 1 + } + uploads := len(plan.Steps) - start - 1 + if uploads < 0 { + return false + } + if attachmentCount >= 0 && uploads != attachmentCount { + return false + } + for _, step := range plan.Steps[start : start+uploads] { + if step.Type != "upload_attachment" || step.Condition != "always" { + return false + } + } + case stagestore.EditPost: + terminalType = "edit_post" + case stagestore.DeletePost: + terminalType = "delete_post" + case stagestore.React: + terminalType, terminalCondition = "add_reaction", "if_missing" + case stagestore.Unreact: + terminalType, terminalCondition = "remove_reaction", "if_missing" + case stagestore.ResolveDM, stagestore.ResolveGroupDM: + terminalType, terminalCondition = "resolve_conversation", "if_missing" + default: + return false + } + last := plan.Steps[len(plan.Steps)-1] + return last.Type == terminalType && last.Condition == terminalCondition && + (operation == stagestore.CreatePost || operation == stagestore.Reply || len(plan.Steps) == 1) +} +func cloneDestination(d staging.Destination) staging.Destination { + out := d + out.ParticipantIDs = append([]string{}, d.ParticipantIDs...) + if out.ParticipantIDs == nil { + out.ParticipantIDs = []string{} + } + if d.TeamID != nil { + v := *d.TeamID + out.TeamID = &v + } + if d.PostID != nil { + v := *d.PostID + out.PostID = &v + } + if d.RootPostID != nil { + v := *d.RootPostID + out.RootPostID = &v + } + if d.Emoji != nil { + v := *d.Emoji + out.Emoji = &v + } + if d.PostState != nil { + v := *d.PostState + out.PostState = &v + } + if d.ReactionPresent != nil { + v := *d.ReactionPresent + out.ReactionPresent = &v + } + return out +} +func clonePlan(p staging.Plan) staging.Plan { + return staging.Plan{Steps: append([]staging.PlanStep{}, p.Steps...)} +} +func mustJSON(v any) []byte { b, _ := json.Marshal(v); return b } +func decodeDestination(_ stagestore.Operation, raw []byte) (staging.Destination, error) { + var d staging.Destination + if err := strictDecode(raw, &d); err != nil { + return d, err + } + return cloneDestination(d), nil +} +func strictDecode(raw []byte, out any) error { + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(out); err != nil { + return err + } + if err := dec.Decode(new(any)); err != io.EOF { + return errors.New("invalid JSON trailer") + } + return nil +} + +var registryOnce sync.Once +var registry *schema.Registry +var registryErr error + +func validate(id string, value any, credentials []string) error { + b, err := json.Marshal(value) + if err != nil { + return ErrInvalid + } + for _, credential := range credentials { + if credential != "" && (strings.Contains(string(b), credential) || containsString(reflect.ValueOf(value), credential)) { + return ErrInvalid + } + } + registryOnce.Do(func() { registry, registryErr = schema.Load() }) + if registryErr != nil { + return fmt.Errorf("%w: schemas unavailable", ErrInvalid) + } + if err = registry.Validate(id, bytes.NewReader(b)); err != nil { + return ErrInvalid + } + return nil +} + +func containsString(v reflect.Value, needle string) bool { + if !v.IsValid() { + return false + } + if v.Kind() == reflect.Pointer || v.Kind() == reflect.Interface { + if v.IsNil() { + return false + } + return containsString(v.Elem(), needle) + } + switch v.Kind() { + case reflect.String: + return strings.Contains(v.String(), needle) + case reflect.Struct: + for i := 0; i < v.NumField(); i++ { + if containsString(v.Field(i), needle) { + return true + } + } + case reflect.Slice, reflect.Array: + for i := 0; i < v.Len(); i++ { + if containsString(v.Index(i), needle) { + return true + } + } + case reflect.Map: + for _, key := range v.MapKeys() { + if containsString(key, needle) || containsString(v.MapIndex(key), needle) { + return true + } + } + } + return false +} diff --git a/internal/stageoutput/output_test.go b/internal/stageoutput/output_test.go new file mode 100644 index 0000000..b972b88 --- /dev/null +++ b/internal/stageoutput/output_test.go @@ -0,0 +1,179 @@ +package stageoutput + +import ( + "encoding/json" + "errors" + "testing" + "time" + + "github.com/ardasevinc/mattermost-cli/internal/stagecursor" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/internal/staging" +) + +func TestConstructorsProduceStrictDocumentsAndSealInputs(t *testing.T) { + destination := conversationDestination() + plan := staging.Plan{Steps: []staging.PlanStep{{Ordinal: 1, Type: "create_post", Condition: "always"}}} + preview, err := NewPreview(stagestore.CreatePost, staging.Preview{ServerURL: "https://mattermost.example/api/v4", UserID: "user-1", Destination: destination, Plan: plan}, nil) + if err != nil { + t.Fatal(err) + } + destination.ParticipantIDs = append(destination.ParticipantIDs, "mutated") + plan.Steps[0].Type = "delete_post" + if len(preview.Destination.ParticipantIDs) != 0 || preview.Plan.Steps[0].Type != "create_post" { + t.Fatal("preview aliases mutable input") + } + detail := validDetail() + stage, err := NewStage(detail, nil) + if err != nil { + t.Fatal(err) + } + detail.Body[0] = 'X' + detail.Attachments[0].SuppliedPath = "changed" + if *stage.Content.Body != "hello" || stage.Attachments[0].Path != "/tmp/a.txt" { + t.Fatal("stage aliases mutable input") + } + encoded, _ := json.Marshal(stage) + if string(encoded) == "" || stage.Stage.CreatedAt != "2026-07-17T08:00:00.123Z" { + t.Fatalf("bad stage projection: %s", encoded) + } + + mutation := stagestore.MutationResult{Action: "create", Stage: detail.StageSummary, RecordedAt: detail.CreatedAt, Replay: true} + receipt, err := NewReceipt(mutation, detail.Destination, nil) + if err != nil || receipt.Action != "created" || !receipt.Replayed { + t.Fatalf("receipt: %#v, %v", receipt, err) + } + list, err := NewStages(stagestore.ListPage{Records: []stagestore.ListRecord{{StageSummary: detail.StageSummary, Destination: detail.Destination}}}, nil) + if err != nil || len(list.Stages) != 1 || list.NextCursor != nil { + t.Fatalf("list: %#v, %v", list, err) + } +} + +func TestConstructorsFailClosedOnCorruptionAndCredentials(t *testing.T) { + detail := validDetail() + for name, mutate := range map[string]func(*stagestore.StageDetail){ + "operation": func(v *stagestore.StageDetail) { v.Operation = "bogus" }, + "destination": func(v *stagestore.StageDetail) { v.Destination = json.RawMessage(`{"kind":"conversation"}`) }, + "plan": func(v *stagestore.StageDetail) { v.Plan = json.RawMessage(`{"steps":[]}`) }, + "lifecycle": func(v *stagestore.StageDetail) { v.Lifecycle = stagestore.LifecycleCompleted }, + } { + t.Run(name, func(t *testing.T) { + v := detail + mutate(&v) + if _, err := NewStage(v, nil); !errors.Is(err, ErrInvalid) { + t.Fatalf("err=%v", err) + } + }) + } + for name, mutate := range map[string]func(*stagestore.StageDetail){ + "body": func(v *stagestore.StageDetail) { v.Body = []byte("hello active-secret") }, + "path": func(v *stagestore.StageDetail) { v.Attachments[0].SuppliedPath = "/active-secret/file" }, + "binding": func(v *stagestore.StageDetail) { v.UserID = "active-secret" }, + "destination": func(v *stagestore.StageDetail) { + v.Destination = json.RawMessage(`{"kind":"conversation","channelId":"active-secret","channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null,"postState":null,"reactionPresent":null}`) + }, + } { + t.Run("credential_"+name, func(t *testing.T) { + v := detail + mutate(&v) + if _, err := NewStage(v, []string{"active-secret"}); !errors.Is(err, ErrInvalid) { + t.Fatalf("err=%v", err) + } + }) + } + if _, err := NewStages(stagestore.ListPage{Records: make([]stagestore.ListRecord, 101)}, nil); !errors.Is(err, ErrInvalid) { + t.Fatalf("oversize err=%v", err) + } +} + +func TestStageRejectsRetentionPlanAndTimestampContradictions(t *testing.T) { + base := validDetail() + pruned := base + pruned.Lifecycle, pruned.Recovery, pruned.Body, pruned.Attachments = stagestore.LifecyclePruned, stagestore.RecoveryForbidden, nil, nil + projected, err := NewStage(pruned, nil) + if err != nil || projected.AttachmentState != "pruned" { + t.Fatalf("valid pruned projection: %v", err) + } + pruned.Plan = json.RawMessage(`{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) + projected, err = NewStage(pruned, nil) + if err != nil || projected.AttachmentState != "none" { + t.Fatalf("pruned no-attachment projection: %#v %v", projected, err) + } + cases := map[string]func(*stagestore.StageDetail){ + "pruned body retained": func(v *stagestore.StageDetail) { + v.Lifecycle, v.Recovery = stagestore.LifecyclePruned, stagestore.RecoveryForbidden + }, + "pruned attachment retained": func(v *stagestore.StageDetail) { + v.Lifecycle, v.Recovery, v.Body = stagestore.LifecyclePruned, stagestore.RecoveryForbidden, nil + }, + "attachment count": func(v *stagestore.StageDetail) { v.Attachments = nil }, + "ordinal gap": func(v *stagestore.StageDetail) { + v.Plan = json.RawMessage(`{"steps":[{"ordinal":2,"type":"upload_attachment","condition":"always"},{"ordinal":3,"type":"create_post","condition":"always"}]}`) + }, + "step order": func(v *stagestore.StageDetail) { + v.Plan = json.RawMessage(`{"steps":[{"ordinal":1,"type":"create_post","condition":"always"},{"ordinal":2,"type":"upload_attachment","condition":"always"}]}`) + }, + "zero revision time": func(v *stagestore.StageDetail) { v.RevisionCreatedAt = time.Time{} }, + "revision before create": func(v *stagestore.StageDetail) { v.RevisionCreatedAt = v.CreatedAt.Add(-time.Millisecond) }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + v := base + mutate(&v) + if _, err := NewStage(v, nil); !errors.Is(err, ErrInvalid) { + t.Fatalf("err=%v", err) + } + }) + } +} + +func TestReceiptsAndListsEnforceEmittedTimestampOrderAndUniqueIDs(t *testing.T) { + detail := validDetail() + detail.CreatedAt = time.Time{} + if _, err := NewStages(stagestore.ListPage{Records: []stagestore.ListRecord{{StageSummary: detail.StageSummary, Destination: detail.Destination}}}, nil); !errors.Is(err, ErrInvalid) { + t.Fatalf("zero timestamp err=%v", err) + } + + detail = validDetail() + mutation := stagestore.MutationResult{Action: "create", Stage: detail.StageSummary, RecordedAt: detail.UpdatedAt.Add(-time.Millisecond)} + if _, err := NewReceipt(mutation, detail.Destination, nil); !errors.Is(err, ErrInvalid) { + t.Fatalf("receipt timestamp err=%v", err) + } + + first := stagestore.ListRecord{StageSummary: detail.StageSummary, Destination: detail.Destination} + second := first + second.UpdatedAt = first.UpdatedAt.Add(-time.Nanosecond) + if _, err := NewStages(stagestore.ListPage{Records: []stagestore.ListRecord{first, second}}, nil); !errors.Is(err, ErrInvalid) { + t.Fatalf("duplicate stage err=%v", err) + } + first.ID, second.ID = "stg_zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", "stg_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + if _, err := NewStages(stagestore.ListPage{Records: []stagestore.ListRecord{first, second}}, nil); !errors.Is(err, ErrInvalid) { + t.Fatalf("millisecond ordering err=%v", err) + } + cursor, err := stagecursor.Encode(stagecursor.Boundary{UpdatedAt: first.UpdatedAt.UTC(), StageID: first.ID}) + if err != nil { + t.Fatal(err) + } + if _, err = NewStages(stagestore.ListPage{Records: []stagestore.ListRecord{first}, NextCursor: &cursor}, nil); err != nil { + t.Fatalf("valid cursor err=%v", err) + } + wrong, _ := stagecursor.Encode(stagecursor.Boundary{UpdatedAt: first.UpdatedAt.UTC(), StageID: second.ID}) + if _, err = NewStages(stagestore.ListPage{Records: []stagestore.ListRecord{first}, NextCursor: &wrong}, nil); !errors.Is(err, ErrInvalid) { + t.Fatalf("unbound cursor err=%v", err) + } +} + +func conversationDestination() staging.Destination { + team := "team-1" + return staging.Destination{Kind: "conversation", ChannelID: "channel-1", ChannelType: "public", TeamID: &team, ParticipantIDs: []string{}} +} +func validDetail() stagestore.StageDetail { + tm := time.Date(2026, 7, 17, 10, 0, 0, 123456789, time.FixedZone("offset", 7200)) + attachmentDigest := [32]byte{2} + d, _ := json.Marshal(conversationDestination()) + p, _ := json.Marshal(staging.Plan{Steps: []staging.PlanStep{{Ordinal: 1, Type: "upload_attachment", Condition: "always"}, {Ordinal: 2, Type: "create_post", Condition: "always"}}}) + detail := stagestore.StageDetail{StageSummary: stagestore.StageSummary{ID: "stg_0123456789abcdefghijklmnopqrstuv", ServerURL: "https://mattermost.example/api/v4", UserID: "user-1", Operation: stagestore.CreatePost, Lifecycle: stagestore.LifecycleOpen, Recovery: stagestore.RecoveryNone, Revision: 1, CreatedAt: tm, UpdatedAt: tm}, RevisionCreatedAt: tm, Body: []byte("hello"), Destination: d, Plan: p, Attachments: []stagestore.Attachment{{SuppliedPath: "/tmp/a.txt", CanonicalPath: "/private/tmp/a.txt", RemoteFilename: "a.txt", ByteLength: 5, MediaType: "text/plain", ContentDigest: attachmentDigest}}} + detail.SemanticDigest, _ = stagestore.ComputeSemanticDigest(detail.Operation, detail.ServerURL, detail.ServerID, detail.UserID, + stagestore.RevisionContent{Body: detail.Body, Destination: detail.Destination, Plan: detail.Plan, Attachments: detail.Attachments}) + return detail +} From cb22df011a1a193cb3d6931432314e55eb880aba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 03:39:35 +0300 Subject: [PATCH 067/119] feat: add secure stage content acquisition --- internal/stagecontent/content.go | 212 ++++++++++++++++++++++++ internal/stagecontent/content_test.go | 223 ++++++++++++++++++++++++++ 2 files changed, 435 insertions(+) create mode 100644 internal/stagecontent/content.go create mode 100644 internal/stagecontent/content_test.go diff --git a/internal/stagecontent/content.go b/internal/stagecontent/content.go new file mode 100644 index 0000000..f2eea3a --- /dev/null +++ b/internal/stagecontent/content.go @@ -0,0 +1,212 @@ +// Package stagecontent acquires human stage message content without depending +// on CLI flag wiring. Dry-run paths can therefore skip acquisition entirely. +package stagecontent + +import ( + "context" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/ardasevinc/mattermost-cli/internal/messageinput" +) + +var ( + ErrConflictingSources = errors.New("stage content: multiple content sources") + ErrContentRequired = errors.New("stage content: content source required") + ErrEditorNotConfigured = errors.New("stage content: editor not configured") + ErrInvalidEditor = errors.New("stage content: invalid editor command") + ErrEditorFailed = errors.New("stage content: editor failed") + ErrEditorOutput = errors.New("stage content: could not read editor output") +) + +// Request describes the content-related facts already resolved by a caller. +// MessageSet distinguishes an explicit empty --message value from an absent +// flag. Machine mode may use an explicit message or piped stdin, but never an +// editor. +type Request struct { + Stdin io.Reader + Message string + MessageSet bool + Machine bool +} + +// EditorInvocation is an argv-safe editor execution request. Command and Args +// are already parsed; implementations must execute them directly, never via a +// shell. Path names the private 0600 file to edit. +type EditorInvocation struct { + Command string + Args []string + Path string +} + +// Runtime holds injectable host behavior for deterministic tests. Nil +// functions select the production implementations. +type Runtime struct { + IsTTY func(io.Reader) bool + LookupEnv func(string) (string, bool) + RunEditor func(context.Context, EditorInvocation) error +} + +// Acquire selects exactly one source. Non-TTY stdin is an explicit source and +// conflicts with --message. With TTY stdin, only human mode may fall back to +// VISUAL and then EDITOR. Returned bytes preserve the selected source exactly. +func Acquire(ctx context.Context, request Request, runtime Runtime) ([]byte, error) { + if ctx == nil { + return nil, ErrContentRequired + } + isTTY := runtime.IsTTY + if isTTY == nil { + isTTY = defaultIsTTY + } + piped := request.Stdin != nil && !isTTY(request.Stdin) + if request.MessageSet && piped { + return nil, ErrConflictingSources + } + if request.MessageSet { + return messageinput.Read(strings.NewReader(request.Message)) + } + if piped { + return messageinput.Read(request.Stdin) + } + if request.Machine { + return nil, ErrContentRequired + } + return acquireEditor(ctx, runtime) +} + +func acquireEditor(ctx context.Context, runtime Runtime) ([]byte, error) { + lookup := runtime.LookupEnv + if lookup == nil { + lookup = os.LookupEnv + } + command := "" + for _, name := range []string{"VISUAL", "EDITOR"} { + if value, ok := lookup(name); ok && strings.TrimSpace(value) != "" { + command = value + break + } + } + if command == "" { + return nil, ErrEditorNotConfigured + } + argv, err := splitCommand(command) + if err != nil || len(argv) == 0 || argv[0] == "" { + return nil, ErrInvalidEditor + } + + directory, err := os.MkdirTemp("", "mm-stage-content-") + if err != nil { + return nil, ErrEditorOutput + } + defer os.RemoveAll(directory) + if err := os.Chmod(directory, 0o700); err != nil { + return nil, ErrEditorOutput + } + path := filepath.Join(directory, "message") + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return nil, ErrEditorOutput + } + if err := file.Close(); err != nil { + return nil, ErrEditorOutput + } + + run := runtime.RunEditor + if run == nil { + run = defaultRunEditor + } + invocation := EditorInvocation{Command: argv[0], Args: append([]string(nil), argv[1:]...), Path: path} + if err := run(ctx, invocation); err != nil { + return nil, ErrEditorFailed + } + file, err = os.Open(path) + if err != nil { + return nil, ErrEditorOutput + } + defer file.Close() + info, err := file.Stat() + if err != nil || !info.Mode().IsRegular() { + return nil, ErrEditorOutput + } + return messageinput.Read(file) +} + +func defaultIsTTY(input io.Reader) bool { + file, ok := input.(*os.File) + if !ok { + return false + } + info, err := file.Stat() + return err == nil && info.Mode()&os.ModeCharDevice != 0 +} + +func defaultRunEditor(ctx context.Context, invocation EditorInvocation) error { + args := append(append([]string(nil), invocation.Args...), invocation.Path) + command := exec.CommandContext(ctx, invocation.Command, args...) + command.Stdin = os.Stdin + command.Stdout = os.Stdout + command.Stderr = os.Stderr + return command.Run() +} + +// splitCommand accepts the quoting needed by common editor settings while +// deliberately omitting shell expansion, substitution, operators, and globbing. +func splitCommand(value string) ([]string, error) { + var result []string + var current strings.Builder + inSingle, inDouble, escaped, started := false, false, false, false + flush := func() { + if started { + result = append(result, current.String()) + current.Reset() + started = false + } + } + for _, r := range value { + if r == 0 || r == '\n' || r == '\r' { + return nil, ErrInvalidEditor + } + if escaped { + current.WriteRune(r) + started, escaped = true, false + continue + } + switch { + case inSingle: + if r == '\'' { + inSingle = false + } else { + current.WriteRune(r) + } + case inDouble: + switch r { + case '"': + inDouble = false + case '\\': + escaped = true + default: + current.WriteRune(r) + } + case r == '\\': + escaped, started = true, true + case r == '\'': + inSingle, started = true, true + case r == '"': + inDouble, started = true, true + case r == ' ' || r == '\t': + flush() + default: + current.WriteRune(r) + started = true + } + } + if escaped || inSingle || inDouble { + return nil, ErrInvalidEditor + } + flush() + return result, nil +} diff --git a/internal/stagecontent/content_test.go b/internal/stagecontent/content_test.go new file mode 100644 index 0000000..890da3a --- /dev/null +++ b/internal/stagecontent/content_test.go @@ -0,0 +1,223 @@ +package stagecontent + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/messageinput" +) + +func TestAcquireExplicitMessagePreservesBytes(t *testing.T) { + want := "first\nsecond\n" + run := false + got, err := Acquire(context.Background(), Request{Stdin: strings.NewReader("ignored tty"), Message: want, MessageSet: true}, Runtime{ + IsTTY: func(io.Reader) bool { return true }, + RunEditor: func(context.Context, EditorInvocation) error { run = true; return nil }, + }) + if err != nil || string(got) != want || run { + t.Fatalf("content=%q editorRun=%v err=%v", got, run, err) + } +} + +func TestAcquirePipedStdinPreservesFinalNewline(t *testing.T) { + want := []byte("markdown **exactly**\n") + got, err := Acquire(context.Background(), Request{Stdin: bytes.NewReader(want)}, Runtime{ + IsTTY: func(io.Reader) bool { return false }, + }) + if err != nil || !bytes.Equal(got, want) { + t.Fatalf("content=%q err=%v", got, err) + } +} + +func TestAcquireRejectsConflictingMessageAndPipe(t *testing.T) { + got, err := Acquire(context.Background(), Request{Stdin: strings.NewReader("pipe"), Message: "flag", MessageSet: true}, Runtime{ + IsTTY: func(io.Reader) bool { return false }, + }) + if got != nil || !errors.Is(err, ErrConflictingSources) { + t.Fatalf("content=%q err=%v", got, err) + } +} + +func TestAcquireMachineTTYRequiresExplicitContent(t *testing.T) { + lookedUp, ran := false, false + got, err := Acquire(context.Background(), Request{Stdin: strings.NewReader("tty"), Machine: true}, Runtime{ + IsTTY: func(io.Reader) bool { return true }, + LookupEnv: func(string) (string, bool) { lookedUp = true; return "editor", true }, + RunEditor: func(context.Context, EditorInvocation) error { ran = true; return nil }, + }) + if got != nil || !errors.Is(err, ErrContentRequired) || lookedUp || ran { + t.Fatalf("content=%q lookup=%v run=%v err=%v", got, lookedUp, ran, err) + } +} + +func TestAcquireMachineAcceptsPipeAndMessage(t *testing.T) { + for name, request := range map[string]Request{ + "pipe": {Stdin: strings.NewReader("pipe"), Machine: true}, + "message": {Stdin: strings.NewReader("tty"), Message: "flag", MessageSet: true, Machine: true}, + } { + t.Run(name, func(t *testing.T) { + got, err := Acquire(context.Background(), request, Runtime{IsTTY: func(io.Reader) bool { return name == "message" }}) + if err != nil || string(got) != nameValue(name) { + t.Fatalf("content=%q err=%v", got, err) + } + }) + } +} + +func nameValue(name string) string { + if name == "message" { + return "flag" + } + return "pipe" +} + +func TestAcquireEditorVisualPrecedenceAndSafeArgv(t *testing.T) { + environment := map[string]string{"VISUAL": `code --wait --reuse-window "two words"`, "EDITOR": "fallback"} + var invocation EditorInvocation + got, err := Acquire(context.Background(), Request{Stdin: strings.NewReader("tty")}, Runtime{ + IsTTY: func(io.Reader) bool { return true }, + LookupEnv: func(name string) (string, bool) { value, ok := environment[name]; return value, ok }, + RunEditor: func(_ context.Context, value EditorInvocation) error { + invocation = value + return os.WriteFile(value.Path, []byte("edited\n"), 0o600) + }, + }) + if err != nil || string(got) != "edited\n" { + t.Fatalf("content=%q err=%v", got, err) + } + if invocation.Command != "code" || !reflect.DeepEqual(invocation.Args, []string{"--wait", "--reuse-window", "two words"}) { + t.Fatalf("invocation=%+v", invocation) + } +} + +func TestAcquireEditorFallsBackToEditor(t *testing.T) { + var command string + _, err := Acquire(context.Background(), Request{Stdin: strings.NewReader("tty")}, Runtime{ + IsTTY: func(io.Reader) bool { return true }, + LookupEnv: func(name string) (string, bool) { + if name == "VISUAL" { + return " ", true + } + return `vim -f`, true + }, + RunEditor: func(_ context.Context, value EditorInvocation) error { + command = value.Command + return os.WriteFile(value.Path, []byte("ok"), 0o600) + }, + }) + if err != nil || command != "vim" { + t.Fatalf("command=%q err=%v", command, err) + } +} + +func TestAcquireEditorPrivatePermissionsAndCleanup(t *testing.T) { + var path, directory string + _, err := Acquire(context.Background(), Request{Stdin: strings.NewReader("tty")}, Runtime{ + IsTTY: func(io.Reader) bool { return true }, + LookupEnv: func(string) (string, bool) { return "editor", true }, + RunEditor: func(_ context.Context, value EditorInvocation) error { + path, directory = value.Path, filepath.Dir(value.Path) + dirInfo, statErr := os.Stat(directory) + if statErr != nil { + return statErr + } + fileInfo, statErr := os.Stat(path) + if statErr != nil { + return statErr + } + if dirInfo.Mode().Perm() != 0o700 || fileInfo.Mode().Perm() != 0o600 { + return errors.New("insecure permissions") + } + return os.WriteFile(path, []byte("ok"), 0o600) + }, + }) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("temporary file remains: %v", err) + } + if _, err := os.Stat(directory); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("temporary directory remains: %v", err) + } +} + +func TestAcquireEditorFailureIsNarrowAndCleansUp(t *testing.T) { + physical := errors.New("secret command and path detail") + var path string + got, err := Acquire(context.Background(), Request{Stdin: strings.NewReader("tty")}, Runtime{ + IsTTY: func(io.Reader) bool { return true }, + LookupEnv: func(string) (string, bool) { return "editor", true }, + RunEditor: func(_ context.Context, value EditorInvocation) error { path = value.Path; return physical }, + }) + if got != nil || !errors.Is(err, ErrEditorFailed) || errors.Is(err, physical) || strings.Contains(err.Error(), physical.Error()) || strings.Contains(err.Error(), path) { + t.Fatalf("content=%q err=%v", got, err) + } + if _, statErr := os.Stat(filepath.Dir(path)); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("temporary directory remains: %v", statErr) + } +} + +func TestAcquireAllSourcesUseMessageValidation(t *testing.T) { + for name, run := range map[string]func() ([]byte, error){ + "empty message": func() ([]byte, error) { + return Acquire(context.Background(), Request{MessageSet: true}, Runtime{}) + }, + "invalid stdin": func() ([]byte, error) { + return Acquire(context.Background(), Request{Stdin: bytes.NewReader([]byte{0xc3, 0x28})}, Runtime{IsTTY: func(io.Reader) bool { return false }}) + }, + "empty editor": func() ([]byte, error) { + return editorResult(nil) + }, + "invalid editor": func() ([]byte, error) { + return editorResult([]byte{0xc3, 0x28}) + }, + } { + t.Run(name, func(t *testing.T) { + got, err := run() + want := messageinput.ErrEmpty + if strings.Contains(name, "invalid") { + want = messageinput.ErrInvalidUTF8 + } + if got != nil || !errors.Is(err, want) { + t.Fatalf("content=%q err=%v want=%v", got, err, want) + } + }) + } +} + +func editorResult(content []byte) ([]byte, error) { + return Acquire(context.Background(), Request{Stdin: strings.NewReader("tty")}, Runtime{ + IsTTY: func(io.Reader) bool { return true }, + LookupEnv: func(string) (string, bool) { return "editor", true }, + RunEditor: func(_ context.Context, value EditorInvocation) error { return os.WriteFile(value.Path, content, 0o600) }, + }) +} + +func TestAcquireEditorConfigurationErrors(t *testing.T) { + base := Request{Stdin: strings.NewReader("tty")} + tty := func(io.Reader) bool { return true } + if _, err := Acquire(context.Background(), base, Runtime{IsTTY: tty, LookupEnv: func(string) (string, bool) { return "", false }}); !errors.Is(err, ErrEditorNotConfigured) { + t.Fatalf("missing editor error=%v", err) + } + for _, value := range []string{`editor "unterminated`, "editor\nother", `editor trailing\`} { + if _, err := Acquire(context.Background(), base, Runtime{IsTTY: tty, LookupEnv: func(string) (string, bool) { return value, true }}); !errors.Is(err, ErrInvalidEditor) { + t.Fatalf("command=%q error=%v", value, err) + } + } +} + +func TestSplitCommandDoesNotInterpretShellSyntax(t *testing.T) { + got, err := splitCommand(`editor ';' '$(touch nope)' "" escaped\ value`) + want := []string{"editor", ";", "$(touch nope)", "", "escaped value"} + if err != nil || !reflect.DeepEqual(got, want) { + t.Fatalf("argv=%q err=%v", got, err) + } +} From 20820f1160dafe1edcb5b0f4d301e82c51c5b61c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 03:42:35 +0300 Subject: [PATCH 068/119] feat: add stage inspection commands --- internal/cli/root.go | 5 + internal/cli/root_test.go | 1 + internal/cli/runtime.go | 4 + internal/cli/stage_inspect.go | 224 +++++++++++++++++++++++++++ internal/cli/stage_inspect_test.go | 237 +++++++++++++++++++++++++++++ 5 files changed, 471 insertions(+) create mode 100644 internal/cli/stage_inspect.go create mode 100644 internal/cli/stage_inspect_test.go diff --git a/internal/cli/root.go b/internal/cli/root.go index ece11f9..c8866af 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -87,6 +87,10 @@ func machineErrorCode(err error) string { if errors.As(err, &operation) { return operation.code } + var local localStateFailure + if errors.As(err, &local) { + return "local_state" + } return "invalid_invocation" } @@ -125,6 +129,7 @@ func newRootWithState(state *rootState) *cobra.Command { cmd.PersistentFlags().BoolVar(&state.flags.noThreads, "no-threads", false, "return selected seed posts only") cmd.AddCommand(newSchemaCommand(state)) cmd.AddCommand(newStoreCommand(state)) + cmd.AddCommand(newStageCommand(state)) cmd.AddCommand(newConfigCommand(state)) cmd.AddCommand(newDoctorCommand(state)) cmd.AddCommand(newWhoAmICommand(state)) diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index d27d994..efcdd01 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -174,6 +174,7 @@ func TestMachineErrorCodePreservesSemantics(t *testing.T) { {readFailure(&api.APIError{Status: 401}), "authentication"}, {readFailure(&api.APIError{Status: 403}), "authorization"}, {outputError{err: errors.New("bad")}, "internal"}, + {localStateFailure{err: errors.New("bad")}, "local_state"}, } for _, test := range tests { if got := machineErrorCode(test.err); got != test.want { diff --git a/internal/cli/runtime.go b/internal/cli/runtime.go index aeb972a..9a79004 100644 --- a/internal/cli/runtime.go +++ b/internal/cli/runtime.go @@ -313,6 +313,10 @@ func exitCode(err error) int { if errors.As(err, &outputFailure) { return 3 } + var local localStateFailure + if errors.As(err, &local) { + return 6 + } var classified classifiedError if errors.As(err, &classified) && classified.class == classRead { return 3 diff --git a/internal/cli/stage_inspect.go b/internal/cli/stage_inspect.go new file mode 100644 index 0000000..58604dd --- /dev/null +++ b/internal/cli/stage_inspect.go @@ -0,0 +1,224 @@ +package cli + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "strconv" + "strings" + + "github.com/spf13/cobra" + + "github.com/ardasevinc/mattermost-cli/internal/output" + "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/internal/stagecursor" + "github.com/ardasevinc/mattermost-cli/internal/stageoutput" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +type localStateFailure struct{ err error } + +func (e localStateFailure) Error() string { return e.err.Error() } +func (e localStateFailure) Unwrap() error { return e.err } + +func newStageCommand(state *rootState) *cobra.Command { + command := &cobra.Command{ + Use: "stage", + Short: "Create and inspect staged Mattermost changes", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + if state.flags.json { + return invalidFailure("--json requires a stage subcommand") + } + return cmd.Help() + }, + PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { + return resolveStoreRedaction(state, cmd) + }, + } + command.AddCommand(newStageListCommand(state), newStageShowCommand(state)) + return command +} + +func newStageListCommand(state *rootState) *cobra.Command { + var limit int + var cursor string + command := &cobra.Command{ + Use: "list", + Short: "List staged changes without revealing content", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + if limit < 1 || limit > 100 { + return invalidFailure("--limit must be between 1 and 100") + } + var after *stagecursor.Boundary + if flagChanged(cmd, "cursor") { + if strings.TrimSpace(cursor) == "" { + return invalidFailure("--cursor cannot be empty") + } + decoded, err := stagecursor.Decode(cursor) + if err != nil { + return invalidFailure("invalid stage cursor") + } + after = &decoded + } + store, absent, err := openStageStoreReadOnly(cmd, state) + if err != nil { + return err + } + if absent { + return writeStages(state, stageoutput.Stages{Schema: "mm/v2/stages", Stages: []stageoutput.Summary{}}) + } + defer store.Close() + page, err := store.ListRecords(cmd.Context(), stagestore.ListOptions{Limit: limit, After: after}) + if err != nil { + return localStateFailure{fmt.Errorf("could not list stages")} + } + document, err := stageoutput.NewStages(page, state.credentials) + if err != nil { + return localStateFailure{fmt.Errorf("stored stage data is invalid")} + } + return writeStages(state, document) + }, + } + command.Flags().IntVar(&limit, "limit", 50, "maximum stages to return (1-100)") + command.Flags().StringVar(&cursor, "cursor", "", "resume deterministic stage history") + return command +} + +func newStageShowCommand(state *rootState) *cobra.Command { + return &cobra.Command{ + Use: "show ", + Short: "Show a stage, including retained content and attachment paths", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + store, absent, err := openStageStoreReadOnly(cmd, state) + if err != nil { + return err + } + if absent { + return localStateFailure{fmt.Errorf("stage not found")} + } + defer store.Close() + detail, err := store.Show(cmd.Context(), args[0]) + if errors.Is(err, stagestore.ErrInvalid) { + return invalidFailure("invalid stage id") + } + if errors.Is(err, stagestore.ErrNotFound) { + return localStateFailure{fmt.Errorf("stage not found")} + } + if err != nil { + return localStateFailure{fmt.Errorf("could not read stage")} + } + document, err := stageoutput.NewStage(detail, state.credentials) + if err != nil { + return localStateFailure{fmt.Errorf("stored stage data is invalid")} + } + if state.flags.json { + return writeStageJSON(state, document) + } + if err := writeAll(state.streams.err, []byte("warning: stage show reveals retained message content and attachment paths\n")); err != nil { + return err + } + return writeStageHuman(state, document) + }, + } +} + +func openStageStoreReadOnly(cmd *cobra.Command, state *rootState) (*stagestore.Store, bool, error) { + paths, err := storePaths(state) + if err != nil { + return nil, false, err + } + if _, err = os.Stat(paths.DBPath); errors.Is(err, fs.ErrNotExist) { + return nil, true, nil + } else if err != nil { + return nil, false, localStateFailure{fmt.Errorf("could not inspect stage store")} + } + store, err := stagestore.OpenReadOnly(cmd.Context(), paths.DBPath) + if errors.Is(err, fs.ErrNotExist) { + return nil, true, nil + } + if err != nil { + return nil, false, localStateFailure{fmt.Errorf("could not safely open stage store")} + } + return store, false, nil +} + +func writeStageJSON(state *rootState, document any) error { + if _, err := output.WriteBoundedJSON(state.streams.out, document); err != nil { + return outputError{err: err} + } + return nil +} + +func writeStages(state *rootState, document stageoutput.Stages) error { + if state.flags.json { + return writeStageJSON(state, document) + } + var lines []string + for _, stage := range document.Stages { + lines = append(lines, strings.Join([]string{ + safeStoreValue(state, stage.StageRef), + safeStoreValue(state, stage.Operation), + safeStoreValue(state, stage.Lifecycle), + "recovery=" + safeStoreValue(state, stage.Recovery), + "updated=" + safeStoreValue(state, stage.UpdatedAt), + }, "\t")) + } + if len(lines) == 0 { + lines = append(lines, "no stages") + } + if document.NextCursor != nil { + lines = append(lines, "next cursor: "+safeStoreValue(state, *document.NextCursor)) + } + return writeAll(state.streams.out, []byte(strings.Join(lines, "\n")+"\n")) +} + +func writeStageHuman(state *rootState, document stageoutput.Stage) error { + destination, err := json.Marshal(document.Stage.Destination) + if err != nil { + return internalFailure(err) + } + lines := []string{ + "stage: " + safeStoreValue(state, document.Stage.StageRef), + "operation: " + safeStoreValue(state, document.Stage.Operation), + "lifecycle: " + safeStoreValue(state, document.Stage.Lifecycle), + "recovery: " + safeStoreValue(state, document.Stage.Recovery), + "destination: " + safeStoreValue(state, string(destination)), + "content state: " + safeStoreValue(state, document.Content.State), + } + if document.Content.Body != nil { + lines = append(lines, "body:\n"+safeStageContent(state, *document.Content.Body)) + } + lines = append(lines, "attachment state: "+safeStoreValue(state, document.AttachmentState)) + for index, attachment := range document.Attachments { + prefix := "attachment " + strconv.Itoa(index+1) + ": " + lines = append(lines, prefix+safeStoreValue(state, attachment.Path), " canonical: "+safeStoreValue(state, attachment.CanonicalPath), " remote filename: "+safeStoreValue(state, attachment.RemoteFilename)) + } + lines = append(lines, "plan:") + for _, step := range document.Plan.Steps { + lines = append(lines, fmt.Sprintf(" %d. %s (%s)", step.Ordinal, safeStoreValue(state, step.Type), safeStoreValue(state, step.Condition))) + } + apply := "unavailable" + if document.Stage.Lifecycle == string(stagestore.LifecycleOpen) { + switch document.Stage.Recovery { + case string(stagestore.RecoveryNone): + apply = "mm apply " + safeStoreValue(state, document.Stage.StageRef) + case string(stagestore.RecoveryPartial): + apply = "mm apply " + safeStoreValue(state, document.Stage.StageRef) + " --resume-partial" + case string(stagestore.RecoveryUnknown): + apply = "mm apply " + safeStoreValue(state, document.Stage.StageRef) + " --force-unknown" + } + } + lines = append(lines, "apply: "+apply) + return writeAll(state.streams.out, []byte(strings.Join(lines, "\n")+"\n")) +} + +func safeStageContent(state *rootState, value string) string { + return presentation.PreprocessWithOptions(value, presentation.Options{ + Credentials: state.credentials, DisableHeuristics: state.disableHeuristics, + }).Text +} diff --git a/internal/cli/stage_inspect_test.go b/internal/cli/stage_inspect_test.go new file mode 100644 index 0000000..786cf5c --- /dev/null +++ b/internal/cli/stage_inspect_test.go @@ -0,0 +1,237 @@ +//go:build darwin || linux + +package cli + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +func stageInspectCommand(t *testing.T, home, stateRoot string, jsonOutput bool, stdout, stderr *bytes.Buffer) (*rootState, *cobraCommandShim) { + t.Helper() + state := &rootState{streams: streams{in: strings.NewReader(""), out: stdout, err: stderr}, deps: defaultDependencies(stdout)} + state.deps.homeDir = func() (string, error) { return home, nil } + state.deps.lookupEnv = func(key string) (string, bool) { + if key == "XDG_STATE_HOME" { + return stateRoot, true + } + return "", false + } + state.flags.json = jsonOutput + command := newStageCommand(state) + command.SilenceUsage = true + command.SilenceErrors = true + command.SetOut(stdout) + command.SetErr(stderr) + return state, &cobraCommandShim{command} +} + +// The shim keeps test call sites small without changing production command APIs. +type cobraCommandShim struct { + command interface { + SetArgs([]string) + ExecuteContext(context.Context) error + } +} + +func (s *cobraCommandShim) execute(ctx context.Context, args ...string) error { + s.command.SetArgs(args) + return s.command.ExecuteContext(ctx) +} + +func TestStageListAbsentIsOfflineReadOnlyAndSchemaValid(t *testing.T) { + home, stateRoot := t.TempDir(), filepath.Join(t.TempDir(), "state") + var stdout, stderr bytes.Buffer + _, command := stageInspectCommand(t, home, stateRoot, true, &stdout, &stderr) + if err := command.execute(t.Context(), "list"); err != nil { + t.Fatal(err) + } + if stderr.Len() != 0 { + t.Fatalf("stderr=%q", stderr.String()) + } + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/stages", bytes.NewReader(stdout.Bytes())); err != nil { + t.Fatalf("schema: %v\n%s", err, stdout.String()) + } + if stdout.String() != "{\"schema\":\"mm/v2/stages\",\"stages\":[],\"nextCursor\":null}\n" { + t.Fatalf("stdout=%q", stdout.String()) + } + if _, err := os.Stat(stateRoot); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("stage list created state: %v", err) + } +} + +func TestStageListValidatesBoundsAndCursorBeforeStoreAccess(t *testing.T) { + for _, args := range [][]string{{"list", "--limit", "0"}, {"list", "--limit", "101"}, {"list", "--cursor="}, {"list", "--cursor", "not-a-cursor"}} { + t.Run(strings.Join(args, "_"), func(t *testing.T) { + stateRoot := filepath.Join(t.TempDir(), "absent") + var stdout, stderr bytes.Buffer + _, command := stageInspectCommand(t, t.TempDir(), stateRoot, false, &stdout, &stderr) + err := command.execute(t.Context(), args...) + if err == nil || exitCode(err) != 2 || stdout.Len() != 0 { + t.Fatalf("err=%v stdout=%q", err, stdout.String()) + } + if _, statErr := os.Stat(stateRoot); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("validation touched store: %v", statErr) + } + }) + } +} + +func createInspectionStage(t *testing.T, home, stateRoot, body string) string { + t.Helper() + paths, err := stagestore.ResolvePaths(home, func(key string) (string, bool) { return stateRoot, key == "XDG_STATE_HOME" }) + if err != nil { + t.Fatal(err) + } + store, err := stagestore.Open(t.Context(), paths.DBPath) + if err != nil { + t.Fatal(err) + } + destination := json.RawMessage(`{"kind":"conversation","channelId":"channel-1","channelType":"public","teamId":"team-1","postId":null,"rootPostId":null,"participantIds":[],"emoji":null,"postState":null,"reactionPresent":null}`) + plan := json.RawMessage(`{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) + result, err := store.Create(t.Context(), stagestore.CreateInput{ + RequestID: "inspection-stage", RequestDigest: sha256.Sum256([]byte("request")), Operation: stagestore.CreatePost, + ServerURL: "https://mattermost.example/api/v4", UserID: "user-1", + Content: stagestore.RevisionContent{Body: []byte(body), Destination: destination, Plan: plan}, + }) + if closeErr := store.Close(); err == nil { + err = closeErr + } + if err != nil { + t.Fatal(err) + } + return result.Stage.ID +} + +func TestStageListOmitsContentAndShowExplicitlyRevealsProjectedContent(t *testing.T) { + home, stateRoot := t.TempDir(), filepath.Join(t.TempDir(), "state") + stageID := createInspectionStage(t, home, stateRoot, "review me") + + var listOut, listErr bytes.Buffer + _, list := stageInspectCommand(t, home, stateRoot, true, &listOut, &listErr) + if err := list.execute(t.Context(), "list", "--limit", "1"); err != nil { + t.Fatal(err) + } + if strings.Contains(listOut.String(), "review me") || strings.Contains(listOut.String(), `"body"`) || strings.Contains(listOut.String(), `"path"`) { + t.Fatalf("list leaked content: %s", listOut.String()) + } + registry, _ := mmSchema.Load() + if err := registry.Validate("mm/v2/stages", bytes.NewReader(listOut.Bytes())); err != nil { + t.Fatalf("list schema: %v", err) + } + + var showOut, showErr bytes.Buffer + _, show := stageInspectCommand(t, home, stateRoot, true, &showOut, &showErr) + if err := show.execute(t.Context(), "show", stageID); err != nil { + t.Fatal(err) + } + if showErr.Len() != 0 || !strings.Contains(showOut.String(), `"body":"review me"`) { + t.Fatalf("stdout=%q stderr=%q", showOut.String(), showErr.String()) + } + if err := registry.Validate("mm/v2/stage", bytes.NewReader(showOut.Bytes())); err != nil { + t.Fatalf("show schema: %v\n%s", err, showOut.String()) + } + + showOut.Reset() + showErr.Reset() + _, human := stageInspectCommand(t, home, stateRoot, false, &showOut, &showErr) + if err := human.execute(t.Context(), "show", stageID); err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(showErr.String(), "warning:") || !strings.Contains(showOut.String(), "body:\nreview me") || !strings.Contains(showOut.String(), "apply: mm apply "+stageID+"@1") { + t.Fatalf("stdout=%q stderr=%q", showOut.String(), showErr.String()) + } +} + +func TestStageShowHumanPreservesMarkdownLinesAndShowsOrderedPlan(t *testing.T) { + home, stateRoot := t.TempDir(), filepath.Join(t.TempDir(), "state") + stageID := createInspectionStage(t, home, stateRoot, "# heading\n\n- one\n- two") + var stdout, stderr bytes.Buffer + _, command := stageInspectCommand(t, home, stateRoot, false, &stdout, &stderr) + if err := command.execute(t.Context(), "show", stageID); err != nil { + t.Fatal(err) + } + want := "body:\n# heading\n\n- one\n- two\n" + if !strings.Contains(stdout.String(), want) || !strings.Contains(stdout.String(), "plan:\n 1. create_post (always)\n") { + t.Fatalf("stdout=%q", stdout.String()) + } + if !strings.Contains(stdout.String(), "apply: mm apply "+stageID+"@1") || !strings.HasPrefix(stderr.String(), "warning:") { + t.Fatalf("stdout=%q stderr=%q", stdout.String(), stderr.String()) + } +} + +func TestStageShowAbsentIsLocalStateFailure(t *testing.T) { + var stdout, stderr bytes.Buffer + _, command := stageInspectCommand(t, t.TempDir(), filepath.Join(t.TempDir(), "absent"), true, &stdout, &stderr) + err := command.execute(t.Context(), "show", "stg_0123456789abcdefghijklmnopqrstuv") + var local localStateFailure + if !errors.As(err, &local) || stdout.Len() != 0 || stderr.Len() != 0 { + t.Fatalf("err=%v stdout=%q stderr=%q", err, stdout.String(), stderr.String()) + } +} + +func TestStageInspectionIsWiredThroughRootWithStableMachineErrors(t *testing.T) { + t.Setenv("XDG_STATE_HOME", filepath.Join(t.TempDir(), "state")) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(t.TempDir(), "config")) + t.Setenv("MM_URL", "") + t.Setenv("MM_TOKEN", "") + + var stdout, stderr bytes.Buffer + if code := Execute(t.Context(), []string{"--json", "stage", "list"}, strings.NewReader(""), &stdout, &stderr); code != 0 { + t.Fatalf("list exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + if stdout.String() != "{\"schema\":\"mm/v2/stages\",\"stages\":[],\"nextCursor\":null}\n" || stderr.Len() != 0 { + t.Fatalf("list stdout=%q stderr=%q", stdout.String(), stderr.String()) + } + + stdout.Reset() + stderr.Reset() + stageID := "stg_0123456789abcdefghijklmnopqrstuv" + if code := Execute(t.Context(), []string{"--json", "stage", "show", stageID}, strings.NewReader(""), &stdout, &stderr); code != 6 { + t.Fatalf("show exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + if stdout.Len() != 0 || !strings.Contains(stderr.String(), `"code":"local_state"`) || !strings.Contains(stderr.String(), `"exitCode":6`) { + t.Fatalf("show stdout=%q stderr=%q", stdout.String(), stderr.String()) + } +} + +func TestConcurrentStageListReadOnly(t *testing.T) { + home, stateRoot := t.TempDir(), filepath.Join(t.TempDir(), "state") + createInspectionStage(t, home, stateRoot, "concurrent") + const workers = 8 + var wait sync.WaitGroup + errs := make(chan error, workers) + for range workers { + wait.Add(1) + go func() { + defer wait.Done() + var stdout, stderr bytes.Buffer + _, command := stageInspectCommand(t, home, stateRoot, true, &stdout, &stderr) + if err := command.execute(context.Background(), "list"); err != nil { + errs <- err + } else if !strings.Contains(stdout.String(), `"schema":"mm/v2/stages"`) || stderr.Len() != 0 { + errs <- errors.New("invalid concurrent output") + } + }() + } + wait.Wait() + close(errs) + for err := range errs { + t.Error(err) + } +} From e4fea851aca0f172b26b6ce8dc1718b21650b41b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 03:51:30 +0300 Subject: [PATCH 069/119] feat: add public post staging commands --- internal/cli/stage_create.go | 499 ++++++++++++++++++++++++++++++ internal/cli/stage_create_test.go | 231 ++++++++++++++ internal/cli/stage_inspect.go | 28 +- 3 files changed, 750 insertions(+), 8 deletions(-) create mode 100644 internal/cli/stage_create.go create mode 100644 internal/cli/stage_create_test.go diff --git a/internal/cli/stage_create.go b/internal/cli/stage_create.go new file mode 100644 index 0000000..edb5ed3 --- /dev/null +++ b/internal/cli/stage_create.go @@ -0,0 +1,499 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/spf13/cobra" + + "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/internal/stagecontent" + "github.com/ardasevinc/mattermost-cli/internal/stageoutput" + "github.com/ardasevinc/mattermost-cli/internal/stagerequest" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/internal/staging" +) + +type stageCompositionFlags struct { + dryRun bool + requestID string + message string + attachments []string +} + +func newStageCreationCommands(state *rootState) []*cobra.Command { + return []*cobra.Command{ + newStageSendCommand(state), + newStageContentPostCommand(state, stagestore.Reply), + newStageContentPostCommand(state, stagestore.EditPost), + newStageContentlessPostCommand(state, stagestore.DeletePost), + newStageReactionCommand(state, stagestore.React), + newStageReactionCommand(state, stagestore.Unreact), + } +} + +func newStageSendCommand(state *rootState) *cobra.Command { + command := &cobra.Command{Use: "send", Short: "Stage a new message", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { return cmd.Help() }} + command.AddCommand(newStageSendTargetCommand(state, "dm"), newStageSendTargetCommand(state, "group"), newStageSendTargetCommand(state, "channel")) + return command +} + +func newStageSendTargetCommand(state *rootState, kind string) *cobra.Command { + var flags stageCompositionFlags + var teamName, teamID string + use, short := kind+" ", "Stage a message to an existing "+kind+" conversation" + command := &cobra.Command{Use: use, Short: short, Args: cobra.ExactArgs(1)} + addStageCompositionFlags(command, &flags, true) + if kind == "channel" { + command.Flags().StringVar(&teamName, "team", "", "resolve a channel name in this team name") + command.Flags().StringVar(&teamID, "team-id", "", "resolve a channel name in this exact team ID") + } + command.RunE = func(cmd *cobra.Command, args []string) error { + if teamName != "" && teamID != "" { + return invalidFailure("--team and --team-id cannot be combined") + } + target := staging.Target{Value: args[0]} + switch kind { + case "dm": + target.Conversation, target.Selector = staging.Direct, staging.ByUsername + case "group": + target.Conversation, target.Selector = staging.Group, staging.ByID + case "channel": + target.Conversation, target.Selector = staging.Channel, staging.ByID + if teamName != "" || teamID != "" { + target.Selector = staging.ByName + target.Team = &staging.TeamSelector{By: staging.ByName, Value: teamName} + if teamID != "" { + target.Team.By, target.Team.Value = staging.ByID, teamID + } + } + } + return runHumanCreatePost(cmd, state, flags, target) + } + return command +} + +func newStageContentPostCommand(state *rootState, operation stagestore.Operation) *cobra.Command { + var flags stageCompositionFlags + use, short := "reply ", "Stage a reply" + attachments := true + if operation == stagestore.EditPost { + use, short, attachments = "post-edit ", "Stage an edit to your post", false + } + command := &cobra.Command{Use: use, Short: short, Args: cobra.ExactArgs(1)} + addStageCompositionFlags(command, &flags, attachments) + command.RunE = func(cmd *cobra.Command, args []string) error { + if err := validateHumanStageFlags(cmd, flags, attachments); err != nil { + return err + } + if flags.dryRun { + return runPostDryRun(cmd, state, operation, staging.PostDryRunInput{PostID: args[0]}) + } + body, err := acquireStageBody(cmd, state, flags.message) + if err != nil { + return err + } + service, closeStore, err := openStagingService(cmd, state, true) + if err != nil { + return err + } + var result staging.CreatePostResult + if operation == stagestore.Reply { + result, err = service.Reply(cmd.Context(), staging.ReplyInput{RequestID: flags.requestID, PostID: args[0], Body: bytes.NewReader(body), Attachments: stageAttachments(flags.attachments)}) + } else { + result, err = service.EditPost(cmd.Context(), staging.EditPostInput{RequestID: flags.requestID, PostID: args[0], Body: bytes.NewReader(body)}) + } + return finishStageCreate(state, result, err, closeStore) + } + return command +} + +func newStageContentlessPostCommand(state *rootState, operation stagestore.Operation) *cobra.Command { + var dryRun bool + var requestID string + command := &cobra.Command{Use: "post-delete ", Short: "Stage deletion of your post", Args: cobra.ExactArgs(1)} + command.Flags().BoolVar(&dryRun, "dry-run", false, "preview without persisting a stage") + command.Flags().StringVar(&requestID, "request-id", "", "caller-generated replay key") + command.RunE = func(cmd *cobra.Command, args []string) error { + if dryRun { + if requestID != "" { + return invalidFailure("--request-id cannot be used with --dry-run") + } + return runPostDryRun(cmd, state, operation, staging.PostDryRunInput{PostID: args[0]}) + } + service, closeStore, err := openStagingService(cmd, state, true) + if err != nil { + return err + } + result, err := service.DeletePost(cmd.Context(), staging.DeletePostInput{RequestID: requestID, PostID: args[0]}) + return finishStageCreate(state, result, err, closeStore) + } + return command +} + +func newStageReactionCommand(state *rootState, operation stagestore.Operation) *cobra.Command { + var dryRun bool + var requestID string + name := string(operation) + command := &cobra.Command{Use: name + " ", Short: "Stage a post reaction change", Args: cobra.ExactArgs(2)} + command.Flags().BoolVar(&dryRun, "dry-run", false, "preview without persisting a stage") + command.Flags().StringVar(&requestID, "request-id", "", "caller-generated replay key") + command.RunE = func(cmd *cobra.Command, args []string) error { + input := staging.ReactionDryRunInput{PostID: args[0], Emoji: args[1]} + if dryRun { + if requestID != "" { + return invalidFailure("--request-id cannot be used with --dry-run") + } + return runReactionDryRun(cmd, state, operation, input) + } + service, closeStore, err := openStagingService(cmd, state, true) + if err != nil { + return err + } + mutation := staging.ReactionInput{RequestID: requestID, PostID: args[0], Emoji: args[1]} + var result staging.CreatePostResult + if operation == stagestore.React { + result, err = service.React(cmd.Context(), mutation) + } else { + result, err = service.Unreact(cmd.Context(), mutation) + } + return finishStageCreate(state, result, err, closeStore) + } + return command +} + +func addStageCompositionFlags(command *cobra.Command, flags *stageCompositionFlags, attachments bool) { + command.Flags().BoolVar(&flags.dryRun, "dry-run", false, "preview without reading content or persisting a stage") + command.Flags().StringVar(&flags.requestID, "request-id", "", "caller-generated replay key") + command.Flags().StringVar(&flags.message, "message", "", "message text (visible in shell history and process inspection)") + if attachments { + command.Flags().StringArrayVar(&flags.attachments, "attachment", nil, "attachment path (repeatable)") + } +} + +func validateHumanStageFlags(cmd *cobra.Command, flags stageCompositionFlags, attachments bool) error { + if flags.dryRun && (cmd.Flags().Changed("message") || len(flags.attachments) != 0 || flags.requestID != "") { + return invalidFailure("--dry-run cannot be combined with content, attachments, or --request-id") + } + if !attachments && len(flags.attachments) != 0 { + return invalidFailure("attachments are not supported for this operation") + } + return nil +} + +func runHumanCreatePost(cmd *cobra.Command, state *rootState, flags stageCompositionFlags, target staging.Target) error { + if err := validateHumanStageFlags(cmd, flags, true); err != nil { + return err + } + if flags.dryRun { + service, closeStore, err := openStagingService(cmd, state, false) + if err != nil { + return err + } + defer closeStore() + preview, err := service.DryRunCreatePost(cmd.Context(), staging.DryRunInput{Target: target}) + return writeStagePreview(state, stagestore.CreatePost, preview, err) + } + body, err := acquireStageBody(cmd, state, flags.message) + if err != nil { + return err + } + service, closeStore, err := openStagingService(cmd, state, true) + if err != nil { + return err + } + result, err := service.CreatePost(cmd.Context(), staging.CreatePostInput{RequestID: flags.requestID, Target: target, Body: bytes.NewReader(body), Attachments: stageAttachments(flags.attachments)}) + return finishStageCreate(state, result, err, closeStore) +} + +func acquireStageBody(cmd *cobra.Command, state *rootState, message string) ([]byte, error) { + body, err := stagecontent.Acquire(cmd.Context(), stagecontent.Request{Stdin: state.streams.in, Message: message, MessageSet: cmd.Flags().Changed("message"), Machine: state.flags.json}, stagecontent.Runtime{}) + if err == nil { + return body, nil + } + switch { + case errors.Is(err, stagecontent.ErrConflictingSources): + return nil, invalidFailure("choose exactly one message source") + case errors.Is(err, stagecontent.ErrContentRequired): + return nil, invalidFailure("message content is required") + case errors.Is(err, stagecontent.ErrEditorNotConfigured): + return nil, invalidFailure("set VISUAL or EDITOR, pipe content, or use --message") + case errors.Is(err, stagecontent.ErrEditorFailed): + return nil, invalidFailure("editor exited without producing an accepted message") + default: + return nil, invalidFailure("message content could not be accepted") + } +} + +func stageAttachments(paths []string) []staging.Attachment { + result := make([]staging.Attachment, len(paths)) + for index, path := range paths { + result[index] = staging.Attachment{Path: path} + } + return result +} + +func openStagingService(cmd *cobra.Command, state *rootState, persist bool) (*staging.Service, func() error, error) { + runtime, err := state.runtimeFor(cmd) + if err != nil { + return nil, func() error { return nil }, err + } + var store *stagestore.Store + var storeDependency staging.Store + if persist { + paths, pathErr := storePaths(state) + if pathErr != nil { + return nil, func() error { return nil }, pathErr + } + store, err = stagestore.Open(cmd.Context(), paths.DBPath) + if err != nil { + return nil, func() error { return nil }, localStateFailure{fmt.Errorf("could not safely open stage store")} + } + storeDependency = store + } + closeStore := func() error { + if store != nil { + return store.Close() + } + return nil + } + service, err := staging.New(runtime.Config.URL, "", state.credentials, runtime.Users, runtime.Channels, runtime.Teams, runtime.Posts, storeDependency) + if err != nil { + _ = closeStore() + return nil, func() error { return nil }, classifyStageError(err) + } + return service, closeStore, nil +} + +func runPostDryRun(cmd *cobra.Command, state *rootState, operation stagestore.Operation, input staging.PostDryRunInput) error { + service, closeStore, err := openStagingService(cmd, state, false) + if err != nil { + return err + } + defer func() { _ = closeStore() }() + var preview staging.Preview + switch operation { + case stagestore.Reply: + preview, err = service.DryRunReply(cmd.Context(), input) + case stagestore.EditPost: + preview, err = service.DryRunEditPost(cmd.Context(), input) + case stagestore.DeletePost: + preview, err = service.DryRunDeletePost(cmd.Context(), input) + default: + return internalFailure(errors.New("unsupported stage operation")) + } + return writeStagePreview(state, operation, preview, err) +} + +func runReactionDryRun(cmd *cobra.Command, state *rootState, operation stagestore.Operation, input staging.ReactionDryRunInput) error { + service, closeStore, err := openStagingService(cmd, state, false) + if err != nil { + return err + } + defer func() { _ = closeStore() }() + var preview staging.Preview + if operation == stagestore.React { + preview, err = service.DryRunReact(cmd.Context(), input) + } else { + preview, err = service.DryRunUnreact(cmd.Context(), input) + } + return writeStagePreview(state, operation, preview, err) +} + +func runStructuredStage(cmd *cobra.Command, state *rootState) error { + decoder, err := stagerequest.NewDecoder() + if err != nil { + return internalFailure(err) + } + request, err := decoder.DecodeStage(state.streams.in) + if err != nil { + if schema.IsInputReadError(err) { + return readFailure(errors.New("could not read stage request")) + } + return invalidFailure("invalid stage request") + } + return dispatchStructuredStage(cmd, state, request) +} + +func dispatchStructuredStage(cmd *cobra.Command, state *rootState, request stagerequest.StageRequest) error { + if request.Operation == stagerequest.ResolveDM || request.Operation == stagerequest.ResolveGroupDM { + return invalidFailure("unsupported stage operation") + } + service, closeStore, err := openStagingService(cmd, state, request.Persist) + if err != nil { + return err + } + if !request.Persist { + defer func() { _ = closeStore() }() + return dispatchStructuredPreview(cmd.Context(), state, service, request) + } + var result staging.CreatePostResult + switch request.Operation { + case stagerequest.CreatePost: + input, conversionErr := request.CreatePostInput() + if conversionErr != nil { + return invalidFailure("invalid stage request") + } + result, err = service.CreatePost(cmd.Context(), input) + case stagerequest.Reply: + input, conversionErr := request.ReplyInput() + if conversionErr != nil { + return invalidFailure("invalid stage request") + } + result, err = service.Reply(cmd.Context(), input) + case stagerequest.EditPost: + input, conversionErr := request.EditPostInput() + if conversionErr != nil { + return invalidFailure("invalid stage request") + } + result, err = service.EditPost(cmd.Context(), input) + case stagerequest.DeletePost: + input, conversionErr := request.DeletePostInput() + if conversionErr != nil { + return invalidFailure("invalid stage request") + } + result, err = service.DeletePost(cmd.Context(), input) + case stagerequest.React, stagerequest.Unreact: + input, conversionErr := request.ReactionInput() + if conversionErr != nil { + return invalidFailure("invalid stage request") + } + if request.Operation == stagerequest.React { + result, err = service.React(cmd.Context(), input) + } else { + result, err = service.Unreact(cmd.Context(), input) + } + default: + return invalidFailure("unsupported stage operation") + } + return finishStageCreate(state, result, err, closeStore) +} + +func dispatchStructuredPreview(ctx context.Context, state *rootState, service *staging.Service, request stagerequest.StageRequest) error { + var preview staging.Preview + var operation stagestore.Operation + var err error + switch request.Operation { + case stagerequest.CreatePost: + operation = stagestore.CreatePost + input, conversionErr := request.DryRunCreatePostInput() + if conversionErr != nil { + return invalidFailure("invalid stage request") + } + preview, err = service.DryRunCreatePost(ctx, input) + case stagerequest.Reply, stagerequest.EditPost, stagerequest.DeletePost: + operation = mapStageOperation(request.Operation) + input, conversionErr := request.PostDryRunInput() + if conversionErr != nil { + return invalidFailure("invalid stage request") + } + switch request.Operation { + case stagerequest.Reply: + preview, err = service.DryRunReply(ctx, input) + case stagerequest.EditPost: + preview, err = service.DryRunEditPost(ctx, input) + case stagerequest.DeletePost: + preview, err = service.DryRunDeletePost(ctx, input) + } + case stagerequest.React, stagerequest.Unreact: + operation = mapStageOperation(request.Operation) + input, conversionErr := request.ReactionDryRunInput() + if conversionErr != nil { + return invalidFailure("invalid stage request") + } + if request.Operation == stagerequest.React { + preview, err = service.DryRunReact(ctx, input) + } else { + preview, err = service.DryRunUnreact(ctx, input) + } + default: + return invalidFailure("unsupported stage operation") + } + return writeStagePreview(state, operation, preview, err) +} + +func mapStageOperation(operation stagerequest.Operation) stagestore.Operation { + return map[stagerequest.Operation]stagestore.Operation{ + stagerequest.CreatePost: stagestore.CreatePost, stagerequest.Reply: stagestore.Reply, + stagerequest.EditPost: stagestore.EditPost, stagerequest.DeletePost: stagestore.DeletePost, + stagerequest.React: stagestore.React, stagerequest.Unreact: stagestore.Unreact, + stagerequest.ResolveDM: stagestore.ResolveDM, stagerequest.ResolveGroupDM: stagestore.ResolveGroupDM, + }[operation] +} + +func writeStagePreview(state *rootState, operation stagestore.Operation, preview staging.Preview, err error) error { + if err != nil { + return classifyStageError(err) + } + document, err := stageoutput.NewPreview(operation, preview, state.credentials) + if err != nil { + return localStateFailure{errors.New("stage preview is invalid")} + } + if state.flags.json { + return writeStageJSON(state, document) + } + destination, _ := json.Marshal(document.Destination) + lines := []string{"dry run: no stage persisted", "operation: " + safeStoreValue(state, document.Operation), "destination: " + safeStoreValue(state, string(destination)), "content validated: no", "plan:"} + for _, step := range document.Plan.Steps { + lines = append(lines, fmt.Sprintf(" %d. %s (%s)", step.Ordinal, safeStoreValue(state, step.Type), safeStoreValue(state, step.Condition))) + } + return writeAll(state.streams.out, []byte(strings.Join(lines, "\n")+"\n")) +} + +func writeStageCreateResult(state *rootState, result staging.CreatePostResult, err error) error { + if err != nil { + return classifyStageError(err) + } + document, err := stageoutput.NewCreateReceipt(result, state.credentials) + if err != nil { + return localStateFailure{errors.New("stored stage receipt is invalid")} + } + if state.flags.json { + return writeStageJSON(state, document) + } + destination, _ := json.Marshal(document.Stage.Destination) + lines := []string{ + "staged: " + safeStoreValue(state, document.Stage.StageRef), + "operation: " + safeStoreValue(state, document.Stage.Operation), + "destination: " + safeStoreValue(state, string(destination)), + "replayed: " + fmt.Sprint(document.Replayed), + "apply: mm apply " + safeStoreValue(state, document.Stage.StageRef), + } + return writeAll(state.streams.out, []byte(strings.Join(lines, "\n")+"\n")) +} + +func finishStageCreate(state *rootState, result staging.CreatePostResult, operationErr error, closeStore func() error) error { + if err := closeStore(); err != nil { + return localStateFailure{errors.New("could not close stage store safely")} + } + return writeStageCreateResult(state, result, operationErr) +} + +func classifyStageError(err error) error { + if err == nil { + return nil + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return readFailure(errors.New("stage operation canceled before persistence")) + } + switch { + case errors.Is(err, staging.ErrInvalid): + return invalidFailure("invalid stage request") + case errors.Is(err, staging.ErrInput): + return invalidFailure("message or attachment input was rejected") + case errors.Is(err, staging.ErrCredential): + return invalidFailure("protected Mattermost credential present in staged input") + case errors.Is(err, staging.ErrTarget): + return readFailure(errors.New("could not resolve the exact Mattermost target")) + case errors.Is(err, staging.ErrConflict): + return localStateFailure{errors.New("stage request conflicts with durable local state")} + case errors.Is(err, staging.ErrStore): + return localStateFailure{errors.New("could not persist stage state")} + default: + return internalFailure(errors.New("stage operation failed")) + } +} diff --git a/internal/cli/stage_create_test.go b/internal/cli/stage_create_test.go new file mode 100644 index 0000000..166321e --- /dev/null +++ b/internal/cli/stage_create_test.go @@ -0,0 +1,231 @@ +//go:build darwin || linux + +package cli + +import ( + "bytes" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +type failOnRead struct{ reads atomic.Int32 } + +func (r *failOnRead) Read([]byte) (int, error) { + r.reads.Add(1) + return 0, errors.New("stdin must not be read") +} + +func stageTargetServer(t *testing.T, methods *[]string) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + *methods = append(*methods, request.Method+" "+request.URL.Path) + if request.Header.Get("Authorization") != "Bearer test-token" { + t.Fatalf("authorization=%q", request.Header.Get("Authorization")) + } + switch request.URL.Path { + case "/api/v4/users/me": + writeJSON(t, writer, `{"id":"user1","username":"arda"}`) + case "/api/v4/channels/channel1": + writeJSON(t, writer, `{"id":"channel1","team_id":"team1","type":"O","name":"town-square","display_name":"Town Square"}`) + case "/api/v4/channels/channel1/members/user1": + writeJSON(t, writer, `{"channel_id":"channel1","user_id":"user1"}`) + default: + t.Fatalf("unexpected request: %s %s", request.Method, request.URL.String()) + } + })) +} + +func setStageEnvironment(t *testing.T, serverURL, stateRoot string) { + t.Helper() + t.Setenv("HOME", t.TempDir()) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(t.TempDir(), "config")) + t.Setenv("XDG_STATE_HOME", stateRoot) + t.Setenv("MM_URL", serverURL) + t.Setenv("MM_TOKEN", "test-token") +} + +func TestStageSendDryRunSkipsContentAndPersistence(t *testing.T) { + var methods []string + server := stageTargetServer(t, &methods) + defer server.Close() + stateRoot := filepath.Join(t.TempDir(), "state") + setStageEnvironment(t, server.URL, stateRoot) + stdin := new(failOnRead) + var stdout, stderr bytes.Buffer + code := Execute(t.Context(), []string{"--json", "stage", "send", "channel", "channel1", "--dry-run"}, stdin, &stdout, &stderr) + if code != 0 || stderr.Len() != 0 || stdin.reads.Load() != 0 { + t.Fatalf("exit=%d reads=%d stdout=%q stderr=%q", code, stdin.reads.Load(), stdout.String(), stderr.String()) + } + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/stage-preview", bytes.NewReader(stdout.Bytes())); err != nil { + t.Fatalf("schema: %v\n%s", err, stdout.String()) + } + if strings.Contains(stdout.String(), "test-token") || !strings.Contains(stdout.String(), `"persist":false`) || !strings.Contains(stdout.String(), `"contentValidated":false`) { + t.Fatalf("stdout=%s", stdout.String()) + } + for _, method := range methods { + if !strings.HasPrefix(method, http.MethodGet+" ") { + t.Fatalf("dry-run mutation request: %s", method) + } + } + if _, err := os.Stat(stateRoot); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("dry-run created local state: %v", err) + } +} + +func TestStructuredStagePreservesShortAndLongMarkdownWithoutRemoteMutation(t *testing.T) { + bodies := map[string]string{ + "short": "# hello\n\n- **bold**\n- [link](https://example.com)", + "long": "# long\n\n" + strings.Repeat("- **item**\n", 1488), + } + for name, body := range bodies { + t.Run(name, func(t *testing.T) { + var methods []string + server := stageTargetServer(t, &methods) + defer server.Close() + stateRoot := filepath.Join(t.TempDir(), "state") + setStageEnvironment(t, server.URL, stateRoot) + request := map[string]any{ + "schema": "mm/v2/stage-request", "persist": true, "requestId": "test-" + name, + "operation": "create_post", + "target": map[string]any{"kind": "conversation", "conversationType": "channel", "selector": map[string]any{"by": "id", "value": "channel1"}, "team": nil}, + "body": body, "emoji": nil, "attachments": []any{}, + } + encoded, err := json.Marshal(request) + if err != nil { + t.Fatal(err) + } + var stdout, stderr bytes.Buffer + code := Execute(t.Context(), []string{"stage", "--from-json"}, bytes.NewReader(encoded), &stdout, &stderr) + if code != 0 || stderr.Len() != 0 { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/stage-receipt", bytes.NewReader(stdout.Bytes())); err != nil { + t.Fatalf("schema: %v\n%s", err, stdout.String()) + } + if strings.Contains(stdout.String(), body) || strings.Contains(stdout.String(), "test-token") { + t.Fatalf("receipt leaked content or credential") + } + var receipt stageoutputReceipt + if err := json.Unmarshal(stdout.Bytes(), &receipt); err != nil { + t.Fatal(err) + } + paths, err := stagestore.ResolvePaths(t.TempDir(), func(key string) (string, bool) { return stateRoot, key == "XDG_STATE_HOME" }) + if err != nil { + t.Fatal(err) + } + store, err := stagestore.OpenReadOnly(t.Context(), paths.DBPath) + if err != nil { + t.Fatal(err) + } + detail, err := store.Show(t.Context(), receipt.Stage.StageID) + closeErr := store.Close() + if err != nil || closeErr != nil { + t.Fatalf("show=%v close=%v", err, closeErr) + } + if string(detail.Body) != body { + t.Fatalf("body changed: got bytes=%d want=%d", len(detail.Body), len(body)) + } + for _, method := range methods { + if !strings.HasPrefix(method, http.MethodGet+" ") { + t.Fatalf("staging dispatched mutation: %s", method) + } + } + }) + } +} + +type stageoutputReceipt struct { + Stage struct { + StageID string `json:"stageId"` + } `json:"stage"` +} + +func TestStructuredStageRejectsActiveCredentialBeforePersistence(t *testing.T) { + var methods []string + server := stageTargetServer(t, &methods) + defer server.Close() + stateRoot := filepath.Join(t.TempDir(), "state") + setStageEnvironment(t, server.URL, stateRoot) + request := `{"schema":"mm/v2/stage-request","persist":true,"requestId":"credential-test","operation":"create_post","target":{"kind":"conversation","conversationType":"channel","selector":{"by":"id","value":"channel1"},"team":null},"body":"do not send test-token ever","emoji":null,"attachments":[]}` + var stdout, stderr bytes.Buffer + code := Execute(t.Context(), []string{"stage", "--from-json"}, strings.NewReader(request), &stdout, &stderr) + if code != 2 || stdout.Len() != 0 || strings.Contains(stderr.String(), "test-token") || !strings.Contains(stderr.String(), `"code":"invalid_input"`) { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + paths, err := stagestore.ResolvePaths(t.TempDir(), func(key string) (string, bool) { return stateRoot, key == "XDG_STATE_HOME" }) + if err != nil { + t.Fatal(err) + } + store, err := stagestore.OpenReadOnly(t.Context(), paths.DBPath) + if err != nil { + t.Fatal(err) + } + page, err := store.ListRecords(t.Context(), stagestore.ListOptions{Limit: 10}) + closeErr := store.Close() + if err != nil || closeErr != nil || len(page.Records) != 0 { + t.Fatalf("list=%v close=%v records=%d", err, closeErr, len(page.Records)) + } +} + +func TestStructuredStageReplayUsesStoredTargetAndConflictingReuseFailsClosed(t *testing.T) { + var methods []string + server := stageTargetServer(t, &methods) + defer server.Close() + stateRoot := filepath.Join(t.TempDir(), "state") + setStageEnvironment(t, server.URL, stateRoot) + request := func(body string) string { + encoded, err := json.Marshal(map[string]any{ + "schema": "mm/v2/stage-request", "persist": true, "requestId": "stable-replay", + "operation": "create_post", + "target": map[string]any{"kind": "conversation", "conversationType": "channel", "selector": map[string]any{"by": "id", "value": "channel1"}, "team": nil}, + "body": body, "emoji": nil, "attachments": []any{}, + }) + if err != nil { + t.Fatal(err) + } + return string(encoded) + } + run := func(body string) (int, string, string) { + var stdout, stderr bytes.Buffer + code := Execute(t.Context(), []string{"stage", "--from-json"}, strings.NewReader(request(body)), &stdout, &stderr) + return code, stdout.String(), stderr.String() + } + code, firstOut, firstErr := run("same **markdown**") + if code != 0 || firstErr != "" || !strings.Contains(firstOut, `"replayed":false`) { + t.Fatalf("first exit=%d stdout=%q stderr=%q", code, firstOut, firstErr) + } + beforeReplay := len(methods) + code, replayOut, replayErr := run("same **markdown**") + if code != 0 || replayErr != "" || !strings.Contains(replayOut, `"replayed":true`) { + t.Fatalf("replay exit=%d stdout=%q stderr=%q", code, replayOut, replayErr) + } + if got := methods[beforeReplay:]; len(got) != 1 || got[0] != "GET /api/v4/users/me" { + t.Fatalf("replay re-resolved target: %v", got) + } + beforeConflict := len(methods) + code, conflictOut, conflictErr := run("changed **markdown**") + if code != 6 || conflictOut != "" || !strings.Contains(conflictErr, `"code":"local_state"`) { + t.Fatalf("conflict exit=%d stdout=%q stderr=%q", code, conflictOut, conflictErr) + } + if got := methods[beforeConflict:]; len(got) != 1 || got[0] != "GET /api/v4/users/me" { + t.Fatalf("conflict re-resolved target: %v", got) + } +} diff --git a/internal/cli/stage_inspect.go b/internal/cli/stage_inspect.go index 58604dd..4aa1549 100644 --- a/internal/cli/stage_inspect.go +++ b/internal/cli/stage_inspect.go @@ -24,21 +24,33 @@ func (e localStateFailure) Error() string { return e.err.Error() } func (e localStateFailure) Unwrap() error { return e.err } func newStageCommand(state *rootState) *cobra.Command { + var fromJSON bool command := &cobra.Command{ Use: "stage", Short: "Create and inspect staged Mattermost changes", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, _ []string) error { - if state.flags.json { - return invalidFailure("--json requires a stage subcommand") + } + command.RunE = func(cmd *cobra.Command, _ []string) error { + if fromJSON { + return runStructuredStage(cmd, state) + } + if state.flags.json { + return invalidFailure("--json requires a stage subcommand") + } + return cmd.Help() + } + command.PersistentPreRunE = func(cmd *cobra.Command, _ []string) error { + if fromJSON { + state.flags.json = true + if cmd != command { + return invalidFailure("--from-json cannot be combined with a stage subcommand") } - return cmd.Help() - }, - PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { - return resolveStoreRedaction(state, cmd) - }, + } + return resolveStoreRedaction(state, cmd) } + command.PersistentFlags().BoolVar(&fromJSON, "from-json", false, "read one versioned stage request from stdin") command.AddCommand(newStageListCommand(state), newStageShowCommand(state)) + command.AddCommand(newStageCreationCommands(state)...) return command } From dea4079db987d7fb2401d81aa3bf8761bc2a79fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 03:56:11 +0300 Subject: [PATCH 070/119] feat: add stage revision management --- internal/cli/stage_create.go | 4 + internal/cli/stage_inspect.go | 3 +- internal/cli/stage_manage.go | 261 ++++++++++++++++++++++++++ internal/cli/stage_manage_test.go | 137 ++++++++++++++ internal/stagecontent/content.go | 17 +- internal/stagecontent/content_test.go | 21 +++ 6 files changed, 440 insertions(+), 3 deletions(-) create mode 100644 internal/cli/stage_manage.go create mode 100644 internal/cli/stage_manage_test.go diff --git a/internal/cli/stage_create.go b/internal/cli/stage_create.go index edb5ed3..b6e87ce 100644 --- a/internal/cli/stage_create.go +++ b/internal/cli/stage_create.go @@ -491,6 +491,10 @@ func classifyStageError(err error) error { return readFailure(errors.New("could not resolve the exact Mattermost target")) case errors.Is(err, staging.ErrConflict): return localStateFailure{errors.New("stage request conflicts with durable local state")} + case errors.Is(err, staging.ErrNotFound): + return localStateFailure{errors.New("stage not found")} + case errors.Is(err, staging.ErrNotEligible): + return localStateFailure{errors.New("stage lifecycle transition is not allowed")} case errors.Is(err, staging.ErrStore): return localStateFailure{errors.New("could not persist stage state")} default: diff --git a/internal/cli/stage_inspect.go b/internal/cli/stage_inspect.go index 4aa1549..ebb0462 100644 --- a/internal/cli/stage_inspect.go +++ b/internal/cli/stage_inspect.go @@ -42,7 +42,7 @@ func newStageCommand(state *rootState) *cobra.Command { command.PersistentPreRunE = func(cmd *cobra.Command, _ []string) error { if fromJSON { state.flags.json = true - if cmd != command { + if cmd != command && cmd.Name() != "revise" && cmd.Name() != "cancel" { return invalidFailure("--from-json cannot be combined with a stage subcommand") } } @@ -51,6 +51,7 @@ func newStageCommand(state *rootState) *cobra.Command { command.PersistentFlags().BoolVar(&fromJSON, "from-json", false, "read one versioned stage request from stdin") command.AddCommand(newStageListCommand(state), newStageShowCommand(state)) command.AddCommand(newStageCreationCommands(state)...) + command.AddCommand(newStageManagementCommands(state, &fromJSON)...) return command } diff --git a/internal/cli/stage_manage.go b/internal/cli/stage_manage.go new file mode 100644 index 0000000..e32496e --- /dev/null +++ b/internal/cli/stage_manage.go @@ -0,0 +1,261 @@ +package cli + +import ( + "bytes" + "errors" + "fmt" + "io/fs" + "os" + + "github.com/spf13/cobra" + + "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/internal/stagecontent" + "github.com/ardasevinc/mattermost-cli/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/internal/stageoutput" + "github.com/ardasevinc/mattermost-cli/internal/stagerequest" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/internal/staging" +) + +func newStageManagementCommands(state *rootState, fromJSON *bool) []*cobra.Command { + return []*cobra.Command{newStageReviseCommand(state, fromJSON), newStageCancelCommand(state, fromJSON)} +} + +func newStageReviseCommand(state *rootState, fromJSON *bool) *cobra.Command { + var requestID, message string + var attachments []string + var clearAttachments, revive bool + command := &cobra.Command{Use: "revise ", Short: "Create a new revision of staged content"} + command.Args = func(cmd *cobra.Command, args []string) error { + if *fromJSON { + return cobra.NoArgs(cmd, args) + } + return cobra.ExactArgs(1)(cmd, args) + } + command.Flags().StringVar(&requestID, "request-id", "", "caller-generated replay key") + command.Flags().StringVar(&message, "message", "", "replacement text (visible in shell history and process inspection)") + command.Flags().StringArrayVar(&attachments, "attachment", nil, "replacement attachment path (repeatable)") + command.Flags().BoolVar(&clearAttachments, "clear-attachments", false, "replace retained attachments with an empty set") + command.Flags().BoolVar(&revive, "revive", false, "revive an expired stage while revising it") + command.RunE = func(cmd *cobra.Command, args []string) error { + if *fromJSON { + if anyFlagChanged(cmd, "request-id", "message", "attachment", "clear-attachments", "revive") { + return invalidFailure("--from-json cannot be combined with human revision flags") + } + return runStructuredRevise(cmd, state) + } + if clearAttachments && len(attachments) != 0 { + return invalidFailure("--clear-attachments and --attachment cannot be combined") + } + detail, err := readStageDetail(cmd, state, args[0]) + if err != nil { + return err + } + if detail.Operation != stagestore.CreatePost && detail.Operation != stagestore.Reply && detail.Operation != stagestore.EditPost { + return localStateFailure{errors.New("this stage operation cannot be revised")} + } + if detail.Operation == stagestore.EditPost && (clearAttachments || len(attachments) != 0) { + return invalidFailure("post-edit stages do not support attachments") + } + body, err := stagecontent.Acquire(cmd.Context(), stagecontent.Request{ + Stdin: state.streams.in, Message: message, MessageSet: cmd.Flags().Changed("message"), Initial: detail.Body, + }, stagecontent.Runtime{}) + if err != nil { + return mapStageContentError(err) + } + var replacementAttachments []staging.Attachment + if clearAttachments { + replacementAttachments = []staging.Attachment{} + } else if cmd.Flags().Changed("attachment") { + replacementAttachments = stageAttachments(attachments) + } + input := staging.ReviseInput{StageID: detail.ID, RequestID: requestID, ExpectedRevision: detail.Revision, + ExpectedDigest: detail.SemanticDigest, Revive: revive, Body: bytes.NewReader(body), Attachments: replacementAttachments} + return executeStageRevise(cmd, state, input) + } + return command +} + +func newStageCancelCommand(state *rootState, fromJSON *bool) *cobra.Command { + var requestID string + command := &cobra.Command{Use: "cancel ", Short: "Cancel an open stage"} + command.Args = func(cmd *cobra.Command, args []string) error { + if *fromJSON { + return cobra.NoArgs(cmd, args) + } + return cobra.ExactArgs(1)(cmd, args) + } + command.Flags().StringVar(&requestID, "request-id", "", "caller-generated replay key") + command.RunE = func(cmd *cobra.Command, args []string) error { + if *fromJSON { + if cmd.Flags().Changed("request-id") { + return invalidFailure("--from-json cannot be combined with --request-id") + } + return runStructuredCancel(cmd, state) + } + detail, err := readStageDetail(cmd, state, args[0]) + if err != nil { + return err + } + return executeStageCancel(cmd, state, staging.CancelInput{StageID: detail.ID, RequestID: requestID, ExpectedRevision: detail.Revision, ExpectedDigest: detail.SemanticDigest}) + } + return command +} + +func anyFlagChanged(command *cobra.Command, names ...string) bool { + for _, name := range names { + if command.Flags().Changed(name) { + return true + } + } + return false +} + +func readStageDetail(cmd *cobra.Command, state *rootState, stageID string) (stagestore.StageDetail, error) { + store, absent, err := openStageStoreReadOnly(cmd, state) + if err != nil { + return stagestore.StageDetail{}, err + } + if absent { + return stagestore.StageDetail{}, localStateFailure{errors.New("stage not found")} + } + detail, showErr := store.Show(cmd.Context(), stageID) + closeErr := store.Close() + if closeErr != nil { + return stagestore.StageDetail{}, localStateFailure{errors.New("could not close stage store safely")} + } + if errors.Is(showErr, stagestore.ErrInvalid) { + return stagestore.StageDetail{}, invalidFailure("invalid stage id") + } + if errors.Is(showErr, stagestore.ErrNotFound) { + return stagestore.StageDetail{}, localStateFailure{errors.New("stage not found")} + } + if showErr != nil { + return stagestore.StageDetail{}, localStateFailure{errors.New("could not read stage")} + } + return detail, nil +} + +func openExistingStageStore(cmd *cobra.Command, state *rootState) (*stagestore.Store, error) { + paths, err := storePaths(state) + if err != nil { + return nil, err + } + if _, err = os.Stat(paths.DBPath); errors.Is(err, fs.ErrNotExist) { + return nil, localStateFailure{errors.New("stage store does not exist")} + } else if err != nil { + return nil, localStateFailure{errors.New("could not inspect stage store")} + } + store, err := stagestore.Open(cmd.Context(), paths.DBPath) + if err != nil { + return nil, localStateFailure{errors.New("could not safely open stage store")} + } + return store, nil +} + +func executeStageRevise(cmd *cobra.Command, state *rootState, input staging.ReviseInput) error { + store, err := openExistingStageStore(cmd, state) + if err != nil { + return err + } + reviser, err := staging.NewReviser(state.credentials, store, stageinput.Bind) + if err != nil { + _ = store.Close() + return classifyStageError(err) + } + result, operationErr := reviser.Revise(cmd.Context(), input) + if closeErr := store.Close(); closeErr != nil { + return localStateFailure{errors.New("could not close stage store safely")} + } + return writeStageRevisionResult(state, result, operationErr) +} + +func executeStageCancel(cmd *cobra.Command, state *rootState, input staging.CancelInput) error { + store, err := openExistingStageStore(cmd, state) + if err != nil { + return err + } + reviser, err := staging.NewReviser(state.credentials, store, stageinput.Bind) + if err != nil { + _ = store.Close() + return classifyStageError(err) + } + result, operationErr := reviser.Cancel(cmd.Context(), input) + if closeErr := store.Close(); closeErr != nil { + return localStateFailure{errors.New("could not close stage store safely")} + } + return writeStageRevisionResult(state, result, operationErr) +} + +func runStructuredRevise(cmd *cobra.Command, state *rootState) error { + decoder, err := stagerequest.NewDecoder() + if err != nil { + return internalFailure(err) + } + request, err := decoder.DecodeRevise(state.streams.in) + if err != nil { + return classifyStageRequestDecode(err, "revision") + } + input, err := request.ReviseInput() + if err != nil { + return invalidFailure("invalid stage revision request") + } + return executeStageRevise(cmd, state, input) +} + +func runStructuredCancel(cmd *cobra.Command, state *rootState) error { + decoder, err := stagerequest.NewDecoder() + if err != nil { + return internalFailure(err) + } + request, err := decoder.DecodeCancel(state.streams.in) + if err != nil { + return classifyStageRequestDecode(err, "cancel") + } + input, err := request.CancelInput() + if err != nil { + return invalidFailure("invalid stage cancel request") + } + return executeStageCancel(cmd, state, input) +} + +func classifyStageRequestDecode(err error, action string) error { + if schema.IsInputReadError(err) { + return readFailure(fmt.Errorf("could not read stage %s request", action)) + } + return invalidFailure("invalid stage " + action + " request") +} + +func writeStageRevisionResult(state *rootState, result staging.RevisionResult, err error) error { + if err != nil { + return classifyStageError(err) + } + document, err := stageoutput.NewReceipt(result.Stored, result.Destination, state.credentials) + if err != nil { + return localStateFailure{errors.New("stored stage receipt is invalid")} + } + if state.flags.json { + return writeStageJSON(state, document) + } + line := document.Action + ": " + safeStoreValue(state, document.Stage.StageRef) + "\n" + if document.Action == "revised" { + line += "apply: mm apply " + safeStoreValue(state, document.Stage.StageRef) + "\n" + } + return writeAll(state.streams.out, []byte(line)) +} + +func mapStageContentError(err error) error { + switch { + case errors.Is(err, stagecontent.ErrConflictingSources): + return invalidFailure("choose exactly one message source") + case errors.Is(err, stagecontent.ErrContentRequired): + return invalidFailure("message content is required") + case errors.Is(err, stagecontent.ErrEditorNotConfigured): + return invalidFailure("set VISUAL or EDITOR, pipe content, or use --message") + case errors.Is(err, stagecontent.ErrEditorFailed): + return invalidFailure("editor exited without producing an accepted message") + default: + return invalidFailure("message content could not be accepted") + } +} diff --git a/internal/cli/stage_manage_test.go b/internal/cli/stage_manage_test.go new file mode 100644 index 0000000..07fd816 --- /dev/null +++ b/internal/cli/stage_manage_test.go @@ -0,0 +1,137 @@ +//go:build darwin || linux + +package cli + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "path/filepath" + "strings" + "testing" + + mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +func setOfflineStageEnvironment(t *testing.T, stateRoot string) { + t.Helper() + t.Setenv("HOME", t.TempDir()) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(t.TempDir(), "config")) + t.Setenv("XDG_STATE_HOME", stateRoot) + t.Setenv("MM_URL", "") + t.Setenv("MM_TOKEN", "") +} + +func loadStoredStage(t *testing.T, stateRoot, stageID string) stagestore.StageDetail { + t.Helper() + paths, err := stagestore.ResolvePaths(t.TempDir(), func(key string) (string, bool) { return stateRoot, key == "XDG_STATE_HOME" }) + if err != nil { + t.Fatal(err) + } + store, err := stagestore.OpenReadOnly(t.Context(), paths.DBPath) + if err != nil { + t.Fatal(err) + } + detail, err := store.Show(t.Context(), stageID) + closeErr := store.Close() + if err != nil || closeErr != nil { + t.Fatalf("show=%v close=%v", err, closeErr) + } + return detail +} + +func TestStructuredReviseAndCancelAreOfflineCASBoundAndReplayable(t *testing.T) { + home, stateRoot := t.TempDir(), filepath.Join(t.TempDir(), "state") + stageID := createInspectionStage(t, home, stateRoot, "# original\n") + setOfflineStageEnvironment(t, stateRoot) + original := loadStoredStage(t, stateRoot, stageID) + replacement := "# revised\n\n" + strings.Repeat("- durable\n", 100) + reviseRequest := map[string]any{ + "schema": "mm/v2/stage-revise-request", "requestId": "revision-1", "stageId": stageID, + "expectedRevision": original.Revision, "expectedDigest": hex.EncodeToString(original.SemanticDigest[:]), + "revive": false, "body": replacement, "attachments": nil, + } + reviseJSON, err := json.Marshal(reviseRequest) + if err != nil { + t.Fatal(err) + } + var stdout, stderr bytes.Buffer + code := Execute(t.Context(), []string{"stage", "revise", "--from-json"}, bytes.NewReader(reviseJSON), &stdout, &stderr) + if code != 0 || stderr.Len() != 0 { + t.Fatalf("revise exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/stage-receipt", bytes.NewReader(stdout.Bytes())); err != nil { + t.Fatalf("revision receipt schema: %v\n%s", err, stdout.String()) + } + if strings.Contains(stdout.String(), replacement) || !strings.Contains(stdout.String(), `"action":"revised"`) || !strings.Contains(stdout.String(), `"revision":2`) { + t.Fatalf("revision receipt=%s", stdout.String()) + } + revised := loadStoredStage(t, stateRoot, stageID) + if revised.Revision != 2 || string(revised.Body) != replacement { + t.Fatalf("revision=%d body=%q", revised.Revision, revised.Body) + } + + cancelRequest := map[string]any{ + "schema": "mm/v2/stage-cancel-request", "requestId": "cancel-1", "stageId": stageID, + "expectedRevision": revised.Revision, "expectedDigest": hex.EncodeToString(revised.SemanticDigest[:]), + } + cancelJSON, err := json.Marshal(cancelRequest) + if err != nil { + t.Fatal(err) + } + stdout.Reset() + stderr.Reset() + code = Execute(t.Context(), []string{"stage", "cancel", "--from-json"}, bytes.NewReader(cancelJSON), &stdout, &stderr) + if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), `"action":"canceled"`) || !strings.Contains(stdout.String(), `"replayed":false`) { + t.Fatalf("cancel exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + stdout.Reset() + stderr.Reset() + code = Execute(t.Context(), []string{"stage", "cancel", "--from-json"}, bytes.NewReader(cancelJSON), &stdout, &stderr) + if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), `"replayed":true`) { + t.Fatalf("cancel replay exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + canceled := loadStoredStage(t, stateRoot, stageID) + if canceled.Lifecycle != stagestore.LifecycleCanceled || canceled.Recovery != stagestore.RecoveryForbidden || string(canceled.Body) != replacement { + t.Fatalf("canceled=%+v body=%q", canceled.StageSummary, canceled.Body) + } +} + +func TestHumanReviseUsesCurrentCASAndHumanCancelClosesStage(t *testing.T) { + home, stateRoot := t.TempDir(), filepath.Join(t.TempDir(), "state") + stageID := createInspectionStage(t, home, stateRoot, "old") + setOfflineStageEnvironment(t, stateRoot) + var stdout, stderr bytes.Buffer + code := Execute(t.Context(), []string{"stage", "revise", stageID, "--message", "new **markdown**"}, nil, &stdout, &stderr) + if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), "revised: "+stageID+"@2") { + t.Fatalf("revise exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + if got := loadStoredStage(t, stateRoot, stageID); string(got.Body) != "new **markdown**" || got.Revision != 2 { + t.Fatalf("detail revision=%d body=%q", got.Revision, got.Body) + } + stdout.Reset() + stderr.Reset() + code = Execute(t.Context(), []string{"stage", "cancel", stageID}, nil, &stdout, &stderr) + if code != 0 || stderr.Len() != 0 || stdout.String() != "canceled: "+stageID+"@2\n" { + t.Fatalf("cancel exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } +} + +func TestStageMutationFromJSONRejectsHumanFlagMixingAsMachineError(t *testing.T) { + setOfflineStageEnvironment(t, filepath.Join(t.TempDir(), "state")) + for _, args := range [][]string{ + {"stage", "revise", "--from-json", "--message", "nope"}, + {"stage", "cancel", "--from-json", "--request-id", "nope"}, + } { + var stdout, stderr bytes.Buffer + code := Execute(t.Context(), args, strings.NewReader(`{}`), &stdout, &stderr) + if code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), `"schema":"mm/v2/error"`) || !strings.Contains(stderr.String(), `"code":"invalid_input"`) { + t.Fatalf("args=%v exit=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String()) + } + } +} diff --git a/internal/stagecontent/content.go b/internal/stagecontent/content.go index f2eea3a..735a2c4 100644 --- a/internal/stagecontent/content.go +++ b/internal/stagecontent/content.go @@ -3,6 +3,7 @@ package stagecontent import ( + "bytes" "context" "errors" "io" @@ -32,6 +33,7 @@ type Request struct { Message string MessageSet bool Machine bool + Initial []byte } // EditorInvocation is an argv-safe editor execution request. Command and Args @@ -75,10 +77,10 @@ func Acquire(ctx context.Context, request Request, runtime Runtime) ([]byte, err if request.Machine { return nil, ErrContentRequired } - return acquireEditor(ctx, runtime) + return acquireEditor(ctx, runtime, request.Initial) } -func acquireEditor(ctx context.Context, runtime Runtime) ([]byte, error) { +func acquireEditor(ctx context.Context, runtime Runtime, initial []byte) ([]byte, error) { lookup := runtime.LookupEnv if lookup == nil { lookup = os.LookupEnv @@ -111,6 +113,17 @@ func acquireEditor(ctx context.Context, runtime Runtime) ([]byte, error) { if err != nil { return nil, ErrEditorOutput } + if len(initial) != 0 { + validated, validateErr := messageinput.Read(bytes.NewReader(initial)) + if validateErr != nil || !bytes.Equal(validated, initial) { + _ = file.Close() + return nil, ErrEditorOutput + } + if _, err = file.Write(validated); err != nil { + _ = file.Close() + return nil, ErrEditorOutput + } + } if err := file.Close(); err != nil { return nil, ErrEditorOutput } diff --git a/internal/stagecontent/content_test.go b/internal/stagecontent/content_test.go index 890da3a..f7dbe81 100644 --- a/internal/stagecontent/content_test.go +++ b/internal/stagecontent/content_test.go @@ -97,6 +97,27 @@ func TestAcquireEditorVisualPrecedenceAndSafeArgv(t *testing.T) { } } +func TestAcquireEditorStartsFromExactValidatedInitialContent(t *testing.T) { + want := []byte("# existing\n\n- item\n") + got, err := Acquire(context.Background(), Request{Stdin: strings.NewReader("tty"), Initial: want}, Runtime{ + IsTTY: func(io.Reader) bool { return true }, + LookupEnv: func(string) (string, bool) { return "editor", true }, + RunEditor: func(_ context.Context, value EditorInvocation) error { + seeded, readErr := os.ReadFile(value.Path) + if readErr != nil { + return readErr + } + if !bytes.Equal(seeded, want) { + return errors.New("editor seed changed") + } + return os.WriteFile(value.Path, append(seeded, []byte("edited\n")...), 0o600) + }, + }) + if err != nil || string(got) != string(want)+"edited\n" { + t.Fatalf("content=%q err=%v", got, err) + } +} + func TestAcquireEditorFallsBackToEditor(t *testing.T) { var command string _, err := Acquire(context.Background(), Request{Stdin: strings.NewReader("tty")}, Runtime{ From ad43b23283cd2318b12e1f1a2e6f0ae36f380e23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 04:02:23 +0300 Subject: [PATCH 071/119] feat: add conversation creation staging --- internal/cli/stage_create.go | 77 ++++++++- internal/cli/stage_create_test.go | 64 ++++++++ internal/staging/conversation_stage.go | 173 ++++++++++++++++++++ internal/staging/conversation_stage_test.go | 119 ++++++++++++++ internal/staging/types.go | 84 ++++++++++ 5 files changed, 514 insertions(+), 3 deletions(-) create mode 100644 internal/staging/conversation_stage.go create mode 100644 internal/staging/conversation_stage_test.go diff --git a/internal/cli/stage_create.go b/internal/cli/stage_create.go index b6e87ce..761875c 100644 --- a/internal/cli/stage_create.go +++ b/internal/cli/stage_create.go @@ -33,9 +33,57 @@ func newStageCreationCommands(state *rootState) []*cobra.Command { newStageContentlessPostCommand(state, stagestore.DeletePost), newStageReactionCommand(state, stagestore.React), newStageReactionCommand(state, stagestore.Unreact), + newStageConversationCreateCommand(state, stagestore.ResolveDM), + newStageConversationCreateCommand(state, stagestore.ResolveGroupDM), } } +func newStageConversationCreateCommand(state *rootState, operation stagestore.Operation) *cobra.Command { + var dryRun bool + var requestID string + use, short := "dm-create ", "Stage creation or resolution of an exact direct conversation" + args := cobra.ExactArgs(1) + if operation == stagestore.ResolveGroupDM { + use, short = "group-create ...", "Stage creation or resolution of an exact group conversation" + args = func(_ *cobra.Command, values []string) error { + if len(values) < 2 || len(values) > 100 { + return invalidFailure("group-create requires between 2 and 100 usernames") + } + return nil + } + } + command := &cobra.Command{Use: use, Short: short, Args: args} + command.Flags().BoolVar(&dryRun, "dry-run", false, "preview without persisting a stage") + command.Flags().StringVar(&requestID, "request-id", "", "caller-generated replay key") + command.RunE = func(cmd *cobra.Command, values []string) error { + if dryRun && requestID != "" { + return invalidFailure("--request-id cannot be used with --dry-run") + } + service, closeStore, err := openStagingService(cmd, state, !dryRun) + if err != nil { + return err + } + if dryRun { + defer func() { _ = closeStore() }() + var preview staging.Preview + if operation == stagestore.ResolveDM { + preview, err = service.DryRunResolveDM(cmd.Context(), staging.Target{Conversation: staging.Direct, Selector: staging.ByUsername, Value: values[0]}) + } else { + preview, err = service.DryRunResolveGroup(cmd.Context(), values) + } + return writeStagePreview(state, operation, preview, err) + } + var result staging.CreatePostResult + if operation == stagestore.ResolveDM { + result, err = service.ResolveDM(cmd.Context(), staging.ResolveDMInput{RequestID: requestID, Target: staging.Target{Conversation: staging.Direct, Selector: staging.ByUsername, Value: values[0]}}) + } else { + result, err = service.ResolveGroup(cmd.Context(), staging.ResolveGroupInput{RequestID: requestID, Usernames: append([]string(nil), values...)}) + } + return finishStageCreate(state, result, err, closeStore) + } + return command +} + func newStageSendCommand(state *rootState) *cobra.Command { command := &cobra.Command{Use: "send", Short: "Stage a new message", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { return cmd.Help() }} command.AddCommand(newStageSendTargetCommand(state, "dm"), newStageSendTargetCommand(state, "group"), newStageSendTargetCommand(state, "channel")) @@ -320,9 +368,6 @@ func runStructuredStage(cmd *cobra.Command, state *rootState) error { } func dispatchStructuredStage(cmd *cobra.Command, state *rootState, request stagerequest.StageRequest) error { - if request.Operation == stagerequest.ResolveDM || request.Operation == stagerequest.ResolveGroupDM { - return invalidFailure("unsupported stage operation") - } service, closeStore, err := openStagingService(cmd, state, request.Persist) if err != nil { return err @@ -367,6 +412,18 @@ func dispatchStructuredStage(cmd *cobra.Command, state *rootState, request stage } else { result, err = service.Unreact(cmd.Context(), input) } + case stagerequest.ResolveDM: + target, conversionErr := request.ResolveDMTarget() + if conversionErr != nil { + return invalidFailure("invalid stage request") + } + result, err = service.ResolveDM(cmd.Context(), staging.ResolveDMInput{RequestID: *request.RequestID, Target: target}) + case stagerequest.ResolveGroupDM: + usernames, conversionErr := request.ResolveGroupUsernames() + if conversionErr != nil { + return invalidFailure("invalid stage request") + } + result, err = service.ResolveGroup(cmd.Context(), staging.ResolveGroupInput{RequestID: *request.RequestID, Usernames: usernames}) default: return invalidFailure("unsupported stage operation") } @@ -410,6 +467,20 @@ func dispatchStructuredPreview(ctx context.Context, state *rootState, service *s } else { preview, err = service.DryRunUnreact(ctx, input) } + case stagerequest.ResolveDM: + operation = stagestore.ResolveDM + target, conversionErr := request.ResolveDMTarget() + if conversionErr != nil { + return invalidFailure("invalid stage request") + } + preview, err = service.DryRunResolveDM(ctx, target) + case stagerequest.ResolveGroupDM: + operation = stagestore.ResolveGroupDM + usernames, conversionErr := request.ResolveGroupUsernames() + if conversionErr != nil { + return invalidFailure("invalid stage request") + } + preview, err = service.DryRunResolveGroup(ctx, usernames) default: return invalidFailure("unsupported stage operation") } diff --git a/internal/cli/stage_create_test.go b/internal/cli/stage_create_test.go index 166321e..7021ac4 100644 --- a/internal/cli/stage_create_test.go +++ b/internal/cli/stage_create_test.go @@ -229,3 +229,67 @@ func TestStructuredStageReplayUsesStoredTargetAndConflictingReuseFailsClosed(t * t.Fatalf("conflict re-resolved target: %v", got) } } + +func TestConversationCreationStagesExactParticipantsWithoutRemoteMutation(t *testing.T) { + var methods []string + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + methods = append(methods, request.Method+" "+request.URL.Path) + switch request.URL.Path { + case "/api/v4/users/me": + writeJSON(t, writer, `{"id":"self","username":"arda"}`) + case "/api/v4/users/username/alice": + writeJSON(t, writer, `{"id":"z-peer","username":"alice"}`) + case "/api/v4/users/username/bob": + writeJSON(t, writer, `{"id":"a-peer","username":"bob"}`) + case "/api/v4/users/username/hakan": + writeJSON(t, writer, `{"id":"h-peer","username":"hakan"}`) + default: + t.Fatalf("unexpected request: %s %s", request.Method, request.URL.String()) + } + })) + defer server.Close() + t.Run("structured DM preview", func(t *testing.T) { + stateRoot := filepath.Join(t.TempDir(), "state") + setStageEnvironment(t, server.URL, stateRoot) + request := `{"schema":"mm/v2/stage-request","persist":false,"requestId":null,"operation":"resolve_dm","target":{"kind":"user","username":"hakan"},"body":null,"emoji":null,"attachments":[]}` + var stdout, stderr bytes.Buffer + code := Execute(t.Context(), []string{"stage", "--from-json"}, strings.NewReader(request), &stdout, &stderr) + if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), `"channelId":null`) || !strings.Contains(stdout.String(), `"channelType":"dm"`) || !strings.Contains(stdout.String(), `"participantIds":["h-peer"]`) { + t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + registry, _ := mmSchema.Load() + if err := registry.Validate("mm/v2/stage-preview", bytes.NewReader(stdout.Bytes())); err != nil { + t.Fatalf("schema: %v\n%s", err, stdout.String()) + } + if _, err := os.Stat(stateRoot); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("preview created state: %v", err) + } + }) + t.Run("human group stage", func(t *testing.T) { + stateRoot := filepath.Join(t.TempDir(), "state") + setStageEnvironment(t, server.URL, stateRoot) + stdin := new(failOnRead) + var stdout, stderr bytes.Buffer + code := Execute(t.Context(), []string{"--json", "stage", "group-create", "alice", "bob", "--request-id", "group-create-1"}, stdin, &stdout, &stderr) + if code != 0 || stderr.Len() != 0 || stdin.reads.Load() != 0 || !strings.Contains(stdout.String(), `"operation":"resolve_group_dm"`) || !strings.Contains(stdout.String(), `"participantIds":["a-peer","z-peer"]`) { + t.Fatalf("exit=%d reads=%d stdout=%q stderr=%q", code, stdin.reads.Load(), stdout.String(), stderr.String()) + } + registry, _ := mmSchema.Load() + if err := registry.Validate("mm/v2/stage-receipt", bytes.NewReader(stdout.Bytes())); err != nil { + t.Fatalf("schema: %v\n%s", err, stdout.String()) + } + var receipt stageoutputReceipt + if err := json.Unmarshal(stdout.Bytes(), &receipt); err != nil { + t.Fatal(err) + } + detail := loadStoredStage(t, stateRoot, receipt.Stage.StageID) + if detail.Operation != stagestore.ResolveGroupDM || detail.Body != nil || len(detail.Attachments) != 0 { + t.Fatalf("detail=%+v", detail) + } + }) + for _, method := range methods { + if !strings.HasPrefix(method, http.MethodGet+" ") { + t.Fatalf("conversation staging dispatched mutation: %s", method) + } + } +} diff --git a/internal/staging/conversation_stage.go b/internal/staging/conversation_stage.go new file mode 100644 index 0000000..8d378a5 --- /dev/null +++ b/internal/staging/conversation_stage.go @@ -0,0 +1,173 @@ +package staging + +import ( + "context" + "sort" + "strings" + + "github.com/ardasevinc/mattermost-cli/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +type resolveDMIntent struct { + Username string `json:"username"` +} + +type resolveGroupIntent struct { + Usernames []string `json:"usernames"` +} + +func (s *Service) DryRunResolveDM(ctx context.Context, target Target) (Preview, error) { + current, err := s.authenticate(ctx) + if err != nil { + return Preview{}, err + } + return s.resolveDMFor(ctx, current.ID, target) +} + +func (s *Service) ResolveDM(ctx context.Context, in ResolveDMInput) (CreatePostResult, error) { + if nilDependency(s.store) || !validRequestID(in.RequestID) || !validDMCreateTarget(in.Target) { + return CreatePostResult{}, ErrInvalid + } + if contaminated(s.credentials, append([]string{in.RequestID}, targetStrings(in.Target)...)...) { + return CreatePostResult{}, ErrCredential + } + current, err := s.authenticate(ctx) + if err != nil { + return CreatePostResult{}, err + } + digest := intentDigest(stagestore.ResolveDM, resolveDMIntent{in.Target.Value}, nil, "", []stageinput.MetadataIntent{}) + record, found, err := s.findCreate(ctx, current.ID, in.RequestID) + if err != nil { + return CreatePostResult{}, err + } + if found { + return replayResult(record, digest, stagestore.ResolveDM, s.serverURL, current.ID) + } + preview, err := s.resolveDMFor(ctx, current.ID, in.Target) + if err != nil { + return CreatePostResult{}, err + } + return s.persistPost(ctx, in.RequestID, digest, stagestore.ResolveDM, preview, nil, nil) +} + +func (s *Service) resolveDMFor(ctx context.Context, currentUserID string, target Target) (Preview, error) { + if !validDMCreateTarget(target) || contaminated(s.credentials, targetStrings(target)...) { + if contaminated(s.credentials, targetStrings(target)...) { + return Preview{}, ErrCredential + } + return Preview{}, ErrInvalid + } + peer, err := s.users.ByUsernameFresh(ctx, target.Value) + if err != nil { + return Preview{}, targetReadError(err) + } + if !validResolvedUser(peer) || !strings.EqualFold(peer.Username, target.Value) || peer.ID == currentUserID { + return Preview{}, ErrTarget + } + if contaminated(s.credentials, peer.ID, peer.Username) { + return Preview{}, ErrCredential + } + return s.unresolvedConversationPreview(currentUserID, "dm", []string{peer.ID}) +} + +func validDMCreateTarget(target Target) bool { + return target.Conversation == Direct && target.Selector == ByUsername && target.Team == nil && validSelectorValue(target.Value) +} + +func (s *Service) DryRunResolveGroup(ctx context.Context, usernames []string) (Preview, error) { + current, err := s.authenticate(ctx) + if err != nil { + return Preview{}, err + } + return s.resolveGroupFor(ctx, current.ID, usernames) +} + +func (s *Service) ResolveGroup(ctx context.Context, in ResolveGroupInput) (CreatePostResult, error) { + if nilDependency(s.store) || !validRequestID(in.RequestID) || !validGroupUsernames(in.Usernames) { + return CreatePostResult{}, ErrInvalid + } + fields := append([]string{in.RequestID}, in.Usernames...) + if contaminated(s.credentials, fields...) { + return CreatePostResult{}, ErrCredential + } + current, err := s.authenticate(ctx) + if err != nil { + return CreatePostResult{}, err + } + digest := intentDigest(stagestore.ResolveGroupDM, resolveGroupIntent{append([]string(nil), in.Usernames...)}, nil, "", []stageinput.MetadataIntent{}) + record, found, err := s.findCreate(ctx, current.ID, in.RequestID) + if err != nil { + return CreatePostResult{}, err + } + if found { + return replayResult(record, digest, stagestore.ResolveGroupDM, s.serverURL, current.ID) + } + preview, err := s.resolveGroupFor(ctx, current.ID, in.Usernames) + if err != nil { + return CreatePostResult{}, err + } + return s.persistPost(ctx, in.RequestID, digest, stagestore.ResolveGroupDM, preview, nil, nil) +} + +func (s *Service) resolveGroupFor(ctx context.Context, currentUserID string, usernames []string) (Preview, error) { + if !validGroupUsernames(usernames) { + return Preview{}, ErrInvalid + } + if contaminated(s.credentials, usernames...) { + return Preview{}, ErrCredential + } + participants := make([]string, 0, len(usernames)) + seenIDs := make(map[string]struct{}, len(usernames)) + for _, username := range usernames { + user, err := s.users.ByUsernameFresh(ctx, username) + if err != nil { + return Preview{}, targetReadError(err) + } + if !validResolvedUser(user) || !strings.EqualFold(user.Username, username) || user.ID == currentUserID { + return Preview{}, ErrTarget + } + if contaminated(s.credentials, user.ID, user.Username) { + return Preview{}, ErrCredential + } + if _, exists := seenIDs[user.ID]; exists { + return Preview{}, ErrTarget + } + seenIDs[user.ID] = struct{}{} + participants = append(participants, user.ID) + } + sort.Strings(participants) + return s.unresolvedConversationPreview(currentUserID, "group", participants) +} + +func validGroupUsernames(usernames []string) bool { + if len(usernames) < 2 || len(usernames) > 100 { + return false + } + seen := make(map[string]struct{}, len(usernames)) + for _, username := range usernames { + key := strings.ToLower(username) + if !validSelectorValue(username) { + return false + } + if _, exists := seen[key]; exists { + return false + } + seen[key] = struct{}{} + } + return true +} + +func (s *Service) unresolvedConversationPreview(currentUserID, channelType string, participantIDs []string) (Preview, error) { + preview := Preview{ServerURL: s.serverURL, ServerID: s.serverID, UserID: currentUserID, Destination: Destination{ + Kind: "conversation", ChannelType: channelType, ParticipantIDs: append([]string(nil), participantIDs...), + }, Plan: Plan{Steps: []PlanStep{{Ordinal: 1, Type: "resolve_conversation", Condition: "if_missing"}}}} + destination, plan, err := marshalSemantics(preview) + if err != nil { + return Preview{}, ErrInvalid + } + if contaminated(s.credentials, preview.ServerURL, preview.ServerID, preview.UserID, string(destination), string(plan)) { + return Preview{}, ErrCredential + } + return preview, nil +} diff --git a/internal/staging/conversation_stage_test.go b/internal/staging/conversation_stage_test.go new file mode 100644 index 0000000..d2aa397 --- /dev/null +++ b/internal/staging/conversation_stage_test.go @@ -0,0 +1,119 @@ +package staging + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "sync/atomic" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +type directoryUsers struct { + current mattermost.User + byName map[string]mattermost.User + calls atomic.Int64 +} + +func (d *directoryUsers) Current(context.Context) (mattermost.User, error) { return d.current, nil } +func (d *directoryUsers) ByUsernameFresh(_ context.Context, username string) (mattermost.User, error) { + d.calls.Add(1) + user, ok := d.byName[strings.ToLower(username)] + if !ok { + return mattermost.User{}, errors.New("missing") + } + return user, nil +} + +func conversationStageService(t *testing.T, users Users, store Store, credentials []string) *Service { + t.Helper() + service, err := New("https://mattermost.example", "", credentials, users, &fakeChannels{}, emptyTeams{}, emptyPosts{}, store) + if err != nil { + t.Fatal(err) + } + return service +} + +func TestResolveDMPersistsUnresolvedExactParticipantPlan(t *testing.T) { + store := new(recordingStore) + users := &directoryUsers{current: mattermost.User{ID: "self", Username: "arda"}, byName: map[string]mattermost.User{"hakan": {ID: "peer", Username: "Hakan"}}} + service := conversationStageService(t, users, store, nil) + result, err := service.ResolveDM(t.Context(), ResolveDMInput{RequestID: "dm-1", Target: Target{Conversation: Direct, Selector: ByUsername, Value: "hakan"}}) + if err != nil { + t.Fatal(err) + } + if store.calls != 1 || store.in.Operation != stagestore.ResolveDM || result.Preview.UserID != "self" || result.Preview.Destination.ChannelID != "" || result.Preview.Destination.ChannelType != "dm" || !reflect.DeepEqual(result.Preview.Destination.ParticipantIDs, []string{"peer"}) { + t.Fatalf("store=%+v preview=%+v", store.in, result.Preview) + } + wantDestination := `{"kind":"conversation","channelId":null,"channelType":"dm","teamId":null,"postId":null,"rootPostId":null,"participantIds":["peer"],"emoji":null,"postState":null,"reactionPresent":null}` + wantPlan := `{"steps":[{"ordinal":1,"type":"resolve_conversation","condition":"if_missing"}]}` + if string(store.in.Content.Destination) != wantDestination || string(store.in.Content.Plan) != wantPlan || store.in.Content.Body != nil || len(store.in.Content.Attachments) != 0 { + t.Fatalf("destination=%s plan=%s", store.in.Content.Destination, store.in.Content.Plan) + } + var roundTrip Destination + decoder := json.NewDecoder(bytes.NewReader(store.in.Content.Destination)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&roundTrip); err != nil || !reflect.DeepEqual(roundTrip, result.Preview.Destination) { + t.Fatalf("roundtrip=%+v err=%v", roundTrip, err) + } +} + +func TestResolveGroupCanonicalizesParticipantIDsAndRejectsAmbiguity(t *testing.T) { + users := &directoryUsers{current: mattermost.User{ID: "self", Username: "arda"}, byName: map[string]mattermost.User{ + "alice": {ID: "z-peer", Username: "alice"}, "bob": {ID: "a-peer", Username: "bob"}, + }} + store := new(recordingStore) + service := conversationStageService(t, users, store, nil) + result, err := service.ResolveGroup(t.Context(), ResolveGroupInput{RequestID: "group-1", Usernames: []string{"alice", "bob"}}) + if err != nil { + t.Fatal(err) + } + if result.Preview.Destination.ChannelType != "group" || !reflect.DeepEqual(result.Preview.Destination.ParticipantIDs, []string{"a-peer", "z-peer"}) || store.in.Operation != stagestore.ResolveGroupDM { + t.Fatalf("preview=%+v operation=%s", result.Preview, store.in.Operation) + } + before := users.calls.Load() + if _, err := service.ResolveGroup(t.Context(), ResolveGroupInput{Usernames: []string{"Alice", "alice"}}); !errors.Is(err, ErrInvalid) || users.calls.Load() != before { + t.Fatalf("duplicate error=%v calls=%d->%d", err, before, users.calls.Load()) + } + users.byName["alias"] = mattermost.User{ID: "a-peer", Username: "alias"} + if _, err := service.DryRunResolveGroup(t.Context(), []string{"bob", "alias"}); !errors.Is(err, ErrTarget) { + t.Fatalf("duplicate identity error=%v", err) + } +} + +func TestConversationCreateCredentialAndSelfTargetsFailClosed(t *testing.T) { + users := &directoryUsers{current: mattermost.User{ID: "self", Username: "arda"}, byName: map[string]mattermost.User{ + "arda": {ID: "self", Username: "arda"}, "token": {ID: "peer", Username: "active-token"}, + }} + store := new(recordingStore) + service := conversationStageService(t, users, store, []string{"active-token"}) + if _, err := service.ResolveDM(t.Context(), ResolveDMInput{Target: Target{Conversation: Direct, Selector: ByUsername, Value: "arda"}}); !errors.Is(err, ErrTarget) { + t.Fatalf("self error=%v", err) + } + if _, err := service.ResolveDM(t.Context(), ResolveDMInput{Target: Target{Conversation: Direct, Selector: ByUsername, Value: "active-token"}}); !errors.Is(err, ErrCredential) { + t.Fatalf("credential error=%v", err) + } + if store.calls != 0 { + t.Fatalf("persisted=%d", store.calls) + } +} + +func TestDestinationRejectsMissingAndUnknownNullableFields(t *testing.T) { + valid := `{"kind":"conversation","channelId":null,"channelType":"dm","teamId":null,"postId":null,"rootPostId":null,"participantIds":["peer"],"emoji":null,"postState":null,"reactionPresent":null}` + for name, value := range map[string]string{ + "missing": strings.Replace(valid, `"channelId":null,`, "", 1), + "unknown": strings.Replace(valid, `"kind":"conversation"`, `"kind":"conversation","extra":true`, 1), + } { + t.Run(name, func(t *testing.T) { + var destination Destination + if err := json.Unmarshal([]byte(value), &destination); err == nil { + t.Fatalf("accepted %s", value) + } + }) + } +} diff --git a/internal/staging/types.go b/internal/staging/types.go index 5b3ebf5..bf81d97 100644 --- a/internal/staging/types.go +++ b/internal/staging/types.go @@ -1,7 +1,10 @@ package staging import ( + "bytes" "context" + "encoding/json" + "errors" "io" "github.com/ardasevinc/mattermost-cli/internal/mattermost" @@ -68,6 +71,16 @@ type ReactionDryRunInput struct{ PostID, Emoji string } type DryRunInput struct{ Target Target } +type ResolveDMInput struct { + RequestID string + Target Target +} + +type ResolveGroupInput struct { + RequestID string + Usernames []string +} + type Destination struct { Kind string `json:"kind"` ChannelID string `json:"channelId"` @@ -81,6 +94,77 @@ type Destination struct { ReactionPresent *bool `json:"reactionPresent"` } +// MarshalJSON preserves the public nullable channel identity while keeping the +// internal resolved-channel API ergonomic. Empty channel fields mean an exact +// participant set whose conversation still has to be resolved at apply time. +func (d Destination) MarshalJSON() ([]byte, error) { + var channelID, channelType any + if d.ChannelID != "" { + channelID = d.ChannelID + } + if d.ChannelType != "" { + channelType = d.ChannelType + } + return json.Marshal(struct { + Kind string `json:"kind"` + ChannelID any `json:"channelId"` + ChannelType any `json:"channelType"` + TeamID *string `json:"teamId"` + PostID *string `json:"postId"` + RootPostID *string `json:"rootPostId"` + ParticipantIDs []string `json:"participantIds"` + Emoji *string `json:"emoji"` + PostState *PostState `json:"postState"` + ReactionPresent *bool `json:"reactionPresent"` + }{d.Kind, channelID, channelType, d.TeamID, d.PostID, d.RootPostID, d.ParticipantIDs, d.Emoji, d.PostState, d.ReactionPresent}) +} + +func (d *Destination) UnmarshalJSON(data []byte) error { + if d == nil { + return errors.New("nil destination") + } + var raw struct { + Kind json.RawMessage `json:"kind"` + ChannelID json.RawMessage `json:"channelId"` + ChannelType json.RawMessage `json:"channelType"` + TeamID json.RawMessage `json:"teamId"` + PostID json.RawMessage `json:"postId"` + RootPostID json.RawMessage `json:"rootPostId"` + ParticipantIDs json.RawMessage `json:"participantIds"` + Emoji json.RawMessage `json:"emoji"` + PostState json.RawMessage `json:"postState"` + ReactionPresent json.RawMessage `json:"reactionPresent"` + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if decoder.Decode(&raw) != nil || decoder.Decode(new(any)) != io.EOF { + return errors.New("invalid destination") + } + required := []json.RawMessage{raw.Kind, raw.ChannelID, raw.ChannelType, raw.TeamID, raw.PostID, raw.RootPostID, raw.ParticipantIDs, raw.Emoji, raw.PostState, raw.ReactionPresent} + for _, value := range required { + if len(value) == 0 { + return errors.New("incomplete destination") + } + } + var out Destination + if json.Unmarshal(raw.Kind, &out.Kind) != nil || decodeNullableString(raw.ChannelID, &out.ChannelID) != nil || decodeNullableString(raw.ChannelType, &out.ChannelType) != nil || + json.Unmarshal(raw.TeamID, &out.TeamID) != nil || json.Unmarshal(raw.PostID, &out.PostID) != nil || json.Unmarshal(raw.RootPostID, &out.RootPostID) != nil || + json.Unmarshal(raw.ParticipantIDs, &out.ParticipantIDs) != nil || out.ParticipantIDs == nil || json.Unmarshal(raw.Emoji, &out.Emoji) != nil || + json.Unmarshal(raw.PostState, &out.PostState) != nil || json.Unmarshal(raw.ReactionPresent, &out.ReactionPresent) != nil { + return errors.New("invalid destination") + } + *d = out + return nil +} + +func decodeNullableString(raw json.RawMessage, out *string) error { + if bytes.Equal(raw, []byte("null")) { + *out = "" + return nil + } + return json.Unmarshal(raw, out) +} + type PostState struct { AuthorUserID string `json:"authorUserId"` UpdateAt int64 `json:"updateAt"` From 3434161afd0dffc78b97052d6822018e06c2dd88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 04:09:24 +0300 Subject: [PATCH 072/119] fix: rebuild plans for attachment revisions --- internal/cli/store_test.go | 6 +-- internal/stagestore/domain.go | 16 +++++--- internal/stagestore/domain_test.go | 47 ++++++++++++++++++++--- internal/stagestore/schema.go | 6 +++ internal/staging/revision.go | 7 +++- internal/staging/revision_test.go | 25 ++++++++++++ internal/staging/service.go | 7 ++++ schemas/v2/examples/store-doctor.json | 2 +- schemas/v2/examples/store-migrations.json | 2 +- schemas/v2/store-doctor.schema.json | 6 +-- schemas/v2/store-migrations.schema.json | 6 ++- 11 files changed, 108 insertions(+), 22 deletions(-) diff --git a/internal/cli/store_test.go b/internal/cli/store_test.go index 9360ee8..c973090 100644 --- a/internal/cli/store_test.go +++ b/internal/cli/store_test.go @@ -36,7 +36,7 @@ func TestStoreDoctorAbsentIsReadOnlyAndSchemaValid(t *testing.T) { if err := registry.Validate("mm/v2/store-doctor", bytes.NewReader(stdout.Bytes())); err != nil { t.Fatalf("schema: %v\n%s", err, stdout.String()) } - for _, fact := range []string{`"filesystemSafe":null`, `"applicationId":null`, `"integrity":null`, `"applied":null`, `"latest":4`, `"valid":null`, `"journalMode":null`} { + for _, fact := range []string{`"filesystemSafe":null`, `"applicationId":null`, `"integrity":null`, `"applied":null`, `"latest":5`, `"valid":null`, `"journalMode":null`} { if !strings.Contains(stdout.String(), fact) { t.Fatalf("absent report omitted %s: %s", fact, stdout.String()) } @@ -110,7 +110,7 @@ func TestStoreMigrationsIsOfflineAndSchemaValid(t *testing.T) { if err := registry.Validate("mm/v2/store-migrations", bytes.NewReader(stdout.Bytes())); err != nil { t.Fatalf("schema: %v\n%s", err, stdout.String()) } - want := "{\"schema\":\"mm/v2/store-migrations\",\"latest\":4,\"migrations\":[{\"version\":1,\"name\":\"core-stage-state\",\"checksum\":\"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f\"},{\"version\":2,\"name\":\"immutable-local-request-receipts\",\"checksum\":\"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c\"},{\"version\":3,\"name\":\"caller-intent-stage-create-replay\",\"checksum\":\"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5\"},{\"version\":4,\"name\":\"caller-intent-stage-revise-replay\",\"checksum\":\"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4\"}]}\n" + want := "{\"schema\":\"mm/v2/store-migrations\",\"latest\":5,\"migrations\":[{\"version\":1,\"name\":\"core-stage-state\",\"checksum\":\"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f\"},{\"version\":2,\"name\":\"immutable-local-request-receipts\",\"checksum\":\"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c\"},{\"version\":3,\"name\":\"caller-intent-stage-create-replay\",\"checksum\":\"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5\"},{\"version\":4,\"name\":\"caller-intent-stage-revise-replay\",\"checksum\":\"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4\"},{\"version\":5,\"name\":\"revision-plan-follows-composition\",\"checksum\":\"fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0\"}]}\n" if stdout.String() != want { t.Fatalf("stdout does not match golden migration contract: %q", stdout.String()) } @@ -310,7 +310,7 @@ func TestStoreHumanOutputStatesBounds(t *testing.T) { t.Fatalf("stdout=%q", stdout.String()) } stdout.Reset() - if code := Execute(context.Background(), []string{"store", "migrations"}, strings.NewReader(""), &stdout, &stderr); code != 0 || !strings.Contains(stdout.String(), "latest: 4\n1 core-stage-state ") { + if code := Execute(context.Background(), []string{"store", "migrations"}, strings.NewReader(""), &stdout, &stderr); code != 0 || !strings.Contains(stdout.String(), "latest: 5\n1 core-stage-state ") { t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } } diff --git a/internal/stagestore/domain.go b/internal/stagestore/domain.go index 1c5b19b..0a77034 100644 --- a/internal/stagestore/domain.go +++ b/internal/stagestore/domain.go @@ -86,8 +86,9 @@ type RevisionContent struct { Attachments []Attachment } type Composition struct { - Body []byte `json:"body,omitempty"` - Attachments []Attachment `json:"attachments,omitempty"` + Body []byte `json:"body,omitempty"` + Plan json.RawMessage `json:"plan"` + Attachments []Attachment `json:"attachments,omitempty"` } type CreateInput struct { RequestID string @@ -248,7 +249,7 @@ func (s *Store) Revise(ctx context.Context, in ReviseInput) (MutationResult, err if err != nil { return MutationResult{}, ErrInvalid } - content := RevisionContent{composition.Body, bytes.Clone(base.Destination), bytes.Clone(base.Plan), composition.Attachments} + content := RevisionContent{composition.Body, bytes.Clone(base.Destination), composition.Plan, composition.Attachments} requestDigest := in.RequestDigest if in.RequestID != "" && requestDigest == ([32]byte{}) { return MutationResult{}, ErrInvalid @@ -580,15 +581,20 @@ func normalizeContent(op Operation, v RevisionContent) (RevisionContent, error) return v, err } v.Destination, v.Plan = destination, plan - composition, err := normalizeComposition(op, Composition{v.Body, v.Attachments}) + composition, err := normalizeComposition(op, Composition{Body: v.Body, Plan: v.Plan, Attachments: v.Attachments}) if err != nil { return v, err } - v.Body, v.Attachments = composition.Body, composition.Attachments + v.Body, v.Plan, v.Attachments = composition.Body, composition.Plan, composition.Attachments return v, nil } func normalizeComposition(op Operation, v Composition) (Composition, error) { v.Body = bytes.Clone(v.Body) + plan, err := canonicalObject(v.Plan) + if err != nil { + return v, err + } + v.Plan = plan v.Attachments = append([]Attachment(nil), v.Attachments...) if len(v.Attachments) > maxAttachments { return v, ErrInvalid diff --git a/internal/stagestore/domain_test.go b/internal/stagestore/domain_test.go index edde41c..9fe32c1 100644 --- a/internal/stagestore/domain_test.go +++ b/internal/stagestore/domain_test.go @@ -40,7 +40,7 @@ func TestFindCreateUsesExactReceiptRevisionAndFailsClosedOnCorruption(t *testing t.Fatal(err) } revised, err := s.Revise(context.Background(), ReviseInput{StageID: created.Stage.ID, RequestID: "revise-exact", ExpectedRevision: 1, ExpectedDigest: created.Stage.SemanticDigest, - RequestDigest: sha256.Sum256([]byte("revise-exact")), Composition: Composition{Body: []byte("two"), Attachments: in.Content.Attachments}}) + RequestDigest: sha256.Sum256([]byte("revise-exact")), Composition: Composition{Body: []byte("two"), Plan: in.Content.Plan, Attachments: in.Content.Attachments}}) if err != nil || revised.Stage.Revision != 2 { t.Fatalf("revise = %#v/%v", revised, err) } @@ -211,13 +211,48 @@ func TestCallerIntentMigrationsTombstoneLegacyCreateAndReviseReceipts(t *testing t.Fatalf("legacy revise ID reuse = %v", err) } } + +func TestRevisionPlanMigrationAllowsPlanChanges(t *testing.T) { + path := testPath(t) + original := migrations + migrations = append([]migration(nil), original[:4]...) + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + created, err := s.Create(context.Background(), createInput("migration-plan", "one")) + if err != nil { + t.Fatal(err) + } + if err = s.Close(); err != nil { + t.Fatal(err) + } + migrations = original + t.Cleanup(func() { migrations = original }) + s, err = Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + in := reviseInput(created.Stage, "migration-revise", "two") + in.Composition.Plan = json.RawMessage(`{"steps":[{"kind":"create_post","revision":2}]}`) + revised, err := s.Revise(context.Background(), in) + if err != nil { + t.Fatal(err) + } + detail, err := s.Show(context.Background(), revised.Stage.ID) + if err != nil || string(detail.Plan) != `{"steps":[{"kind":"create_post","revision":2}]}` { + t.Fatalf("detail=%+v err=%v", detail, err) + } +} + func reviseInput(stage StageSummary, request, body string) ReviseInput { content := createInput("", body).Content return ReviseInput{StageID: stage.ID, RequestID: request, ExpectedRevision: stage.Revision, ExpectedDigest: stage.SemanticDigest, - RequestDigest: sha256.Sum256([]byte(request + "\x00" + body)), Composition: Composition{Body: content.Body, Attachments: content.Attachments}} + RequestDigest: sha256.Sum256([]byte(request + "\x00" + body)), Composition: Composition{Body: content.Body, Plan: content.Plan, Attachments: content.Attachments}} } -func TestRevisePreservesImmutableDestinationAndPlan(t *testing.T) { +func TestRevisePreservesImmutableDestinationAndAcceptsDerivedPlan(t *testing.T) { s := openDomainStore(t) input := createInput("", "one") input.Content.Destination = json.RawMessage(`{ "binding": {"postId":"post-1"}, "channelId":"channel-1" }`) @@ -238,8 +273,8 @@ func TestRevisePreservesImmutableDestinationAndPlan(t *testing.T) { if err != nil { t.Fatal(err) } - if string(after.Destination) != string(before.Destination) || string(after.Plan) != string(before.Plan) { - t.Fatalf("binding changed: destination %s -> %s, plan %s -> %s", before.Destination, after.Destination, before.Plan, after.Plan) + if string(after.Destination) != string(before.Destination) || string(after.Plan) != `{"steps":[{"kind":"create_post"}]}` || string(after.Plan) == string(before.Plan) { + t.Fatalf("unexpected revision binding: destination %s -> %s, plan %s -> %s", before.Destination, after.Destination, before.Plan, after.Plan) } } @@ -257,7 +292,7 @@ func TestReviseCallerIntentReplayIgnoresBoundFileDrift(t *testing.T) { t.Fatal(err) } retry := first - retry.Composition = Composition{Body: []byte("different bound bytes"), Attachments: []Attachment{attachment("changed.txt")}} + retry.Composition = Composition{Body: []byte("different bound bytes"), Plan: first.Composition.Plan, Attachments: []Attachment{attachment("changed.txt")}} replayed, err := s.Revise(context.Background(), retry) if err != nil || !replayed.Replay || replayed.Stage != revised.Stage { t.Fatalf("replayed=%+v err=%v want=%+v", replayed, err, revised) diff --git a/internal/stagestore/schema.go b/internal/stagestore/schema.go index a2a3ba3..da3199e 100644 --- a/internal/stagestore/schema.go +++ b/internal/stagestore/schema.go @@ -97,4 +97,10 @@ CREATE TRIGGER local_requests_immutable_update BEFORE UPDATE ON local_requests B DROP TRIGGER local_requests_immutable_update; UPDATE local_requests SET request_schema='mm/v2/legacy-stage-revise-conflict' WHERE request_schema='mm/v2/stage-revise-request'; CREATE TRIGGER local_requests_immutable_update BEFORE UPDATE ON local_requests BEGIN SELECT RAISE(ABORT, 'local request receipts are immutable'); END; +`}, {version: 5, name: "revision-plan-follows-composition", sql: ` +DROP TRIGGER stage_revision_binding_immutable; +CREATE TRIGGER stage_revision_binding_immutable BEFORE INSERT ON stage_revisions +WHEN NEW.revision > 1 AND EXISTS(SELECT 1 FROM stage_revisions WHERE stage_id=NEW.stage_id) + AND NEW.destination_json != (SELECT destination_json FROM stage_revisions WHERE stage_id=NEW.stage_id ORDER BY revision LIMIT 1) +BEGIN SELECT RAISE(ABORT, 'stage destination is immutable'); END; `}} diff --git a/internal/staging/revision.go b/internal/staging/revision.go index 67b4a7d..874c261 100644 --- a/internal/staging/revision.go +++ b/internal/staging/revision.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/hex" + "encoding/json" "errors" "github.com/ardasevinc/mattermost-cli/internal/messageinput" @@ -118,8 +119,12 @@ func (r *Reviser) Revise(ctx context.Context, in ReviseInput) (RevisionResult, e if err := ctx.Err(); err != nil { return RevisionResult{}, err } + plan, err := json.Marshal(compositionPlan(detail.Operation, len(attachments))) + if err != nil { + return RevisionResult{}, ErrStore + } stored, err := r.store.Revise(ctx, stagestore.ReviseInput{StageID: in.StageID, RequestID: in.RequestID, ExpectedRevision: in.ExpectedRevision, - ExpectedDigest: in.ExpectedDigest, RequestDigest: requestDigest, Revive: in.Revive, Composition: stagestore.Composition{Body: bytes.Clone(body), Attachments: attachments}}) + ExpectedDigest: in.ExpectedDigest, RequestDigest: requestDigest, Revive: in.Revive, Composition: stagestore.Composition{Body: bytes.Clone(body), Plan: plan, Attachments: attachments}}) if err != nil { return RevisionResult{}, mapRevisionStoreError(err) } diff --git a/internal/staging/revision_test.go b/internal/staging/revision_test.go index b93c2d2..5fc5346 100644 --- a/internal/staging/revision_test.go +++ b/internal/staging/revision_test.go @@ -66,6 +66,9 @@ func TestReviserPreservesBodyAttachmentOrderAndDestinationSnapshot(t *testing.T) if err != nil || store.reviseCalls != 1 || string(store.reviseIn.Composition.Body) != " exact body\n" || store.reviseIn.Composition.Attachments[1].RemoteFilename != "b" { t.Fatalf("unexpected revision: result=%+v err=%v calls=%d input=%+v", result, err, store.reviseCalls, store.reviseIn) } + if string(store.reviseIn.Composition.Plan) != `{"steps":[{"ordinal":1,"type":"upload_attachment","condition":"always"},{"ordinal":2,"type":"upload_attachment","condition":"always"},{"ordinal":3,"type":"create_post","condition":"always"}]}` { + t.Fatalf("revision plan = %s", store.reviseIn.Composition.Plan) + } store.detail.Destination[0] = 'x' if string(result.Destination) != `{"kind":"conversation"}` { t.Fatal("result destination aliases store detail") @@ -147,6 +150,9 @@ func TestReviserNilBodyAndAttachmentsPreserveCurrentComposition(t *testing.T) { if err != nil || string(store.reviseIn.Composition.Body) != "existing body" || len(store.reviseIn.Composition.Attachments) != 1 { t.Fatalf("err=%v input=%+v", err, store.reviseIn) } + if string(store.reviseIn.Composition.Plan) != `{"steps":[{"ordinal":1,"type":"upload_attachment","condition":"always"},{"ordinal":2,"type":"create_post","condition":"always"}]}` { + t.Fatalf("preserved attachment plan = %s", store.reviseIn.Composition.Plan) + } store.detail.Body[0] = 'X' store.detail.Attachments[0].SuppliedPath = "/changed" if string(store.reviseIn.Composition.Body) != "existing body" || store.reviseIn.Composition.Attachments[0].SuppliedPath != "/a" { @@ -154,6 +160,25 @@ func TestReviserNilBodyAndAttachmentsPreserveCurrentComposition(t *testing.T) { } } +func TestReviserClearingAttachmentsRemovesUploadSteps(t *testing.T) { + store, digest := revisionFixture(stagestore.CreatePost) + store.detail.Body = []byte("existing body") + store.detail.Attachments = []stagestore.Attachment{{SuppliedPath: "/a", CanonicalPath: "/a", RemoteFilename: "a", ByteLength: 1, ContentDigest: [32]byte{1}}} + r, _ := NewReviser(nil, store, func(_ context.Context, in []stageinput.Attachment, _ [][]byte) ([]stagestore.Attachment, error) { + if len(in) != 0 { + t.Fatalf("attachment input = %#v", in) + } + return []stagestore.Attachment{}, nil + }) + _, err := r.Revise(context.Background(), ReviseInput{StageID: "stage-1", ExpectedRevision: 1, ExpectedDigest: digest, Attachments: []Attachment{}}) + if err != nil || len(store.reviseIn.Composition.Attachments) != 0 { + t.Fatalf("err=%v input=%+v", err, store.reviseIn) + } + if string(store.reviseIn.Composition.Plan) != `{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}` { + t.Fatalf("cleared attachment plan = %s", store.reviseIn.Composition.Plan) + } +} + func TestReviserRejectsCredentialInExpectedDigest(t *testing.T) { store, digest := revisionFixture(stagestore.Reply) credential := hex.EncodeToString(digest[:]) diff --git a/internal/staging/service.go b/internal/staging/service.go index 272f04c..88ac56b 100644 --- a/internal/staging/service.go +++ b/internal/staging/service.go @@ -228,6 +228,13 @@ func attachmentPlan(count int) Plan { return Plan{steps} } +func compositionPlan(operation stagestore.Operation, attachmentCount int) Plan { + if operation == stagestore.CreatePost || operation == stagestore.Reply { + return attachmentPlan(attachmentCount) + } + return postPlan(operation) +} + func marshalSemantics(preview Preview) ([]byte, []byte, error) { destination, err := json.Marshal(preview.Destination) if err != nil { diff --git a/schemas/v2/examples/store-doctor.json b/schemas/v2/examples/store-doctor.json index 539d428..fe26263 100644 --- a/schemas/v2/examples/store-doctor.json +++ b/schemas/v2/examples/store-doctor.json @@ -1 +1 @@ -{"schema":"mm/v2/store-doctor","path":"/home/arda/.local/state/mattermost-cli/stages.sqlite3","report":{"exists":false,"filesystemSafe":null,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":4,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}} +{"schema":"mm/v2/store-doctor","path":"/home/arda/.local/state/mattermost-cli/stages.sqlite3","report":{"exists":false,"filesystemSafe":null,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":5,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}} diff --git a/schemas/v2/examples/store-migrations.json b/schemas/v2/examples/store-migrations.json index 6201317..84cb3bd 100644 --- a/schemas/v2/examples/store-migrations.json +++ b/schemas/v2/examples/store-migrations.json @@ -1 +1 @@ -{"schema":"mm/v2/store-migrations","latest":4,"migrations":[{"version":1,"name":"core-stage-state","checksum":"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"},{"version":2,"name":"immutable-local-request-receipts","checksum":"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"},{"version":3,"name":"caller-intent-stage-create-replay","checksum":"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5"},{"version":4,"name":"caller-intent-stage-revise-replay","checksum":"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4"}]} +{"schema":"mm/v2/store-migrations","latest":5,"migrations":[{"version":1,"name":"core-stage-state","checksum":"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"},{"version":2,"name":"immutable-local-request-receipts","checksum":"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"},{"version":3,"name":"caller-intent-stage-create-replay","checksum":"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5"},{"version":4,"name":"caller-intent-stage-revise-replay","checksum":"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4"},{"version":5,"name":"revision-plan-follows-composition","checksum":"fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0"}]} diff --git a/schemas/v2/store-doctor.schema.json b/schemas/v2/store-doctor.schema.json index b631590..be76833 100644 --- a/schemas/v2/store-doctor.schema.json +++ b/schemas/v2/store-doctor.schema.json @@ -15,7 +15,7 @@ "exists": { "type": "boolean" }, "filesystemSafe": {}, "applicationId": {}, "integrity": {}, "integrityTruncated": {}, "foreignKeyIssues": {}, "foreignKeyRows": {}, "foreignKeyTruncated": {}, "journalMode": {}, "synchronous": {}, "secureDelete": {}, "foreignKeys": {}, "trustedSchema": {}, "queryOnly": {}, "walFallback": {}, - "migrations": { "type": "object", "additionalProperties": false, "required": ["applied", "latest", "valid"], "properties": { "applied": {}, "latest": { "const": 4 }, "valid": {} } }, + "migrations": { "type": "object", "additionalProperties": false, "required": ["applied", "latest", "valid"], "properties": { "applied": {}, "latest": { "const": 5 }, "valid": {} } }, "permissionModelLimitations": { "type": "array", "items": { "$ref": "#/$defs/safeString" } } }, "allOf": [ @@ -23,14 +23,14 @@ "filesystemSafe": { "const": null }, "applicationId": { "const": null }, "integrity": { "const": null }, "integrityTruncated": { "const": null }, "foreignKeyIssues": { "const": null }, "foreignKeyRows": { "const": null }, "foreignKeyTruncated": { "const": null }, "journalMode": { "const": null }, "synchronous": { "const": null }, "secureDelete": { "const": null }, "foreignKeys": { "const": null }, "trustedSchema": { "const": null }, "queryOnly": { "const": null }, "walFallback": { "const": null }, - "migrations": { "properties": { "applied": { "const": null }, "latest": { "const": 4 }, "valid": { "const": null } } } + "migrations": { "properties": { "applied": { "const": null }, "latest": { "const": 5 }, "valid": { "const": null } } } } } }, { "if": { "properties": { "exists": { "const": true } }, "required": ["exists"] }, "then": { "properties": { "filesystemSafe": { "const": true }, "applicationId": { "const": 1296913970 }, "integrity": { "$ref": "#/$defs/nonemptyBoundedStrings" }, "integrityTruncated": { "type": "boolean" }, "foreignKeyIssues": { "type": "integer", "minimum": 0, "maximum": 21 }, "foreignKeyRows": { "$ref": "#/$defs/boundedStrings" }, "foreignKeyTruncated": { "type": "boolean" }, "journalMode": { "enum": ["wal", "delete", "truncate", "persist"] }, "synchronous": { "type": "integer" }, "secureDelete": { "type": "integer" }, "foreignKeys": { "const": true }, "trustedSchema": { "const": false }, "queryOnly": { "const": true }, "walFallback": { "type": "boolean" }, - "migrations": { "properties": { "applied": { "const": 4 }, "latest": { "const": 4 }, "valid": { "const": true } } } + "migrations": { "properties": { "applied": { "const": 5 }, "latest": { "const": 5 }, "valid": { "const": true } } } }, "allOf": [ { "oneOf": [ { "properties": { "integrity": { "const": ["ok"] }, "integrityTruncated": { "const": false } } }, diff --git a/schemas/v2/store-migrations.schema.json b/schemas/v2/store-migrations.schema.json index abef0d0..869d52c 100644 --- a/schemas/v2/store-migrations.schema.json +++ b/schemas/v2/store-migrations.schema.json @@ -6,9 +6,9 @@ "required": ["schema", "latest", "migrations"], "properties": { "schema": { "const": "mm/v2/store-migrations" }, - "latest": { "const": 4 }, + "latest": { "const": 5 }, "migrations": { - "type": "array", "minItems": 4, "maxItems": 4, + "type": "array", "minItems": 5, "maxItems": 5, "prefixItems": [{ "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { "version": { "const": 1 }, "name": { "const": "core-stage-state" }, "checksum": { "const": "e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f" } } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { @@ -17,6 +17,8 @@ "version": { "const": 3 }, "name": { "const": "caller-intent-stage-create-replay" }, "checksum": { "const": "237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5" } } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { "version": { "const": 4 }, "name": { "const": "caller-intent-stage-revise-replay" }, "checksum": { "const": "c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4" } + } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { + "version": { "const": 5 }, "name": { "const": "revision-plan-follows-composition" }, "checksum": { "const": "fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0" } } }], "items": false } From ad5144a4e7fafe50fff6fb777add06f4c90a8128 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 05:10:57 +0300 Subject: [PATCH 073/119] feat: add durable apply journal --- internal/cli/store_test.go | 6 +- internal/stagestore/apply.go | 930 ++++++++++++++++++++++ internal/stagestore/apply_test.go | 672 ++++++++++++++++ internal/stagestore/domain.go | 55 +- internal/stagestore/domain_test.go | 29 +- internal/stagestore/recovery.go | 131 +++ internal/stagestore/schema.go | 212 +++++ internal/stagestore/store.go | 9 + internal/stagestore/store_test.go | 3 +- schemas/v2/examples/store-doctor.json | 2 +- schemas/v2/examples/store-migrations.json | 2 +- schemas/v2/store-doctor.schema.json | 6 +- schemas/v2/store-migrations.schema.json | 6 +- 13 files changed, 2039 insertions(+), 24 deletions(-) create mode 100644 internal/stagestore/apply.go create mode 100644 internal/stagestore/apply_test.go create mode 100644 internal/stagestore/recovery.go diff --git a/internal/cli/store_test.go b/internal/cli/store_test.go index c973090..1341b22 100644 --- a/internal/cli/store_test.go +++ b/internal/cli/store_test.go @@ -36,7 +36,7 @@ func TestStoreDoctorAbsentIsReadOnlyAndSchemaValid(t *testing.T) { if err := registry.Validate("mm/v2/store-doctor", bytes.NewReader(stdout.Bytes())); err != nil { t.Fatalf("schema: %v\n%s", err, stdout.String()) } - for _, fact := range []string{`"filesystemSafe":null`, `"applicationId":null`, `"integrity":null`, `"applied":null`, `"latest":5`, `"valid":null`, `"journalMode":null`} { + for _, fact := range []string{`"filesystemSafe":null`, `"applicationId":null`, `"integrity":null`, `"applied":null`, `"latest":6`, `"valid":null`, `"journalMode":null`} { if !strings.Contains(stdout.String(), fact) { t.Fatalf("absent report omitted %s: %s", fact, stdout.String()) } @@ -110,7 +110,7 @@ func TestStoreMigrationsIsOfflineAndSchemaValid(t *testing.T) { if err := registry.Validate("mm/v2/store-migrations", bytes.NewReader(stdout.Bytes())); err != nil { t.Fatalf("schema: %v\n%s", err, stdout.String()) } - want := "{\"schema\":\"mm/v2/store-migrations\",\"latest\":5,\"migrations\":[{\"version\":1,\"name\":\"core-stage-state\",\"checksum\":\"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f\"},{\"version\":2,\"name\":\"immutable-local-request-receipts\",\"checksum\":\"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c\"},{\"version\":3,\"name\":\"caller-intent-stage-create-replay\",\"checksum\":\"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5\"},{\"version\":4,\"name\":\"caller-intent-stage-revise-replay\",\"checksum\":\"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4\"},{\"version\":5,\"name\":\"revision-plan-follows-composition\",\"checksum\":\"fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0\"}]}\n" + want := "{\"schema\":\"mm/v2/store-migrations\",\"latest\":6,\"migrations\":[{\"version\":1,\"name\":\"core-stage-state\",\"checksum\":\"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f\"},{\"version\":2,\"name\":\"immutable-local-request-receipts\",\"checksum\":\"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c\"},{\"version\":3,\"name\":\"caller-intent-stage-create-replay\",\"checksum\":\"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5\"},{\"version\":4,\"name\":\"caller-intent-stage-revise-replay\",\"checksum\":\"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4\"},{\"version\":5,\"name\":\"revision-plan-follows-composition\",\"checksum\":\"fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0\"},{\"version\":6,\"name\":\"durable-apply-journal\",\"checksum\":\"4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56\"}]}\n" if stdout.String() != want { t.Fatalf("stdout does not match golden migration contract: %q", stdout.String()) } @@ -310,7 +310,7 @@ func TestStoreHumanOutputStatesBounds(t *testing.T) { t.Fatalf("stdout=%q", stdout.String()) } stdout.Reset() - if code := Execute(context.Background(), []string{"store", "migrations"}, strings.NewReader(""), &stdout, &stderr); code != 0 || !strings.Contains(stdout.String(), "latest: 5\n1 core-stage-state ") { + if code := Execute(context.Background(), []string{"store", "migrations"}, strings.NewReader(""), &stdout, &stderr); code != 0 || !strings.Contains(stdout.String(), "latest: 6\n1 core-stage-state ") { t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } } diff --git a/internal/stagestore/apply.go b/internal/stagestore/apply.go new file mode 100644 index 0000000..8c8b779 --- /dev/null +++ b/internal/stagestore/apply.go @@ -0,0 +1,930 @@ +package stagestore + +import ( + "bytes" + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "io" + "slices" + "time" +) + +type RecoveryMode string +type AttemptOutcome string +type StepState string + +const ( + RecoveryModeOrdinary RecoveryMode = "ordinary" + RecoveryModePartial RecoveryMode = "resume_partial" + RecoveryModeUnknown RecoveryMode = "force_unknown" + + OutcomeSucceeded AttemptOutcome = "succeeded" + OutcomeAlreadySatisfied AttemptOutcome = "already_satisfied" + OutcomeRejected AttemptOutcome = "rejected" + OutcomePartial AttemptOutcome = "partial" + OutcomeUnknown AttemptOutcome = "unknown" + + StepPending StepState = "pending" + StepDispatch StepState = "dispatch_intent" + StepValidated StepState = "response_validated" + StepRejected StepState = "rejected" + StepUnknown StepState = "outcome_unknown" + StepSkipped StepState = "skipped" + StepNotSent StepState = "not_dispatched" +) + +type ApplyClaimInput struct { + StageID, RequestID string + Revision int64 + ExpectedDigest [32]byte + RequestDigest [32]byte + RecoveryMode RecoveryMode +} + +type ApplyStep struct { + Ordinal int `json:"ordinal"` + Kind string `json:"kind"` + Condition string `json:"condition"` + State StepState `json:"state"` + Result json.RawMessage `json:"result"` + StartedAt *time.Time `json:"startedAt"` + EndedAt *time.Time `json:"endedAt"` +} + +type ApplyAttempt struct { + ID string `json:"id"` + StageID string `json:"stageId"` + Revision int64 `json:"revision"` + SemanticDigest [32]byte `json:"semanticDigest"` + RecoveryMode RecoveryMode `json:"recoveryMode"` + PriorRecovery Recovery `json:"priorRecovery"` + ForcedDuplicateRisk bool `json:"forcedDuplicateRisk"` + Plan json.RawMessage `json:"plan"` + PendingPostID string `json:"pendingPostId"` + StartedAt time.Time `json:"startedAt"` + EndedAt *time.Time `json:"endedAt"` + Outcome *AttemptOutcome `json:"outcome"` + Steps []ApplyStep `json:"steps"` + Replay bool `json:"-"` +} + +type ApplyReceipt struct { + Schema string `json:"schema"` + AttemptID string `json:"attemptId"` + StageID string `json:"stageId"` + Revision int64 `json:"revision"` + SemanticDigest string `json:"semanticDigest"` + Operation Operation `json:"operation"` + Destination json.RawMessage `json:"destination"` + Outcome AttemptOutcome `json:"outcome"` + Recovery Recovery `json:"recovery"` + StartedAt time.Time `json:"startedAt"` + RecordedAt time.Time `json:"recordedAt"` + Steps []ApplyStep `json:"steps"` + Replay bool `json:"-"` +} + +type persistedPlan struct { + Steps []struct { + Ordinal int `json:"ordinal"` + Type string `json:"type"` + Condition string `json:"condition"` + } `json:"steps"` +} + +func (s *Store) ClaimApply(ctx context.Context, in ApplyClaimInput) (ApplyAttempt, error) { + if ctx == nil || !bounded(in.StageID, maxIdentityBytes) || in.Revision < 1 || !validRecoveryMode(in.RecoveryMode) || !validRequestID(in.RequestID) || in.ExpectedDigest == ([32]byte{}) || in.RequestID != "" && in.RequestDigest == ([32]byte{}) { + return ApplyAttempt{}, ErrInvalid + } + if err := ctx.Err(); err != nil { + return ApplyAttempt{}, err + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return ApplyAttempt{}, localError(err) + } + defer tx.Rollback() + base, err := scanCurrent(ctx, tx, in.StageID) + if err != nil { + return ApplyAttempt{}, err + } + if in.RequestID != "" { + attempt, found, findErr := findApplyRequest(ctx, tx, base.ServerURL, base.UserID, in.RequestID, in.RequestDigest) + if findErr != nil { + return ApplyAttempt{}, findErr + } + if found { + if attempt.StageID != in.StageID || attempt.Revision != in.Revision || attempt.SemanticDigest != in.ExpectedDigest || attempt.RecoveryMode != in.RecoveryMode { + return ApplyAttempt{}, ErrConflict + } + attempt.Replay = true + return attempt, nil + } + } + if base.Revision != in.Revision || base.SemanticDigest != in.ExpectedDigest { + return ApplyAttempt{}, ErrConflict + } + if base.Lifecycle != LifecycleOpen || !modeMatchesRecovery(in.RecoveryMode, base.Recovery) { + return ApplyAttempt{}, ErrNotEligible + } + // Partial resume needs step input digests plus explicit remote revalidation. + // Refuse it until that proof is part of the claim instead of redispatching a + // previously confirmed effect from ordinal coincidence alone. + if in.RecoveryMode == RecoveryModePartial { + return ApplyAttempt{}, ErrNotEligible + } + plan, steps, err := decodePersistedPlan(base.Plan) + if err != nil { + return ApplyAttempt{}, localError(err) + } + attachments, err := readAttachments(ctx, tx, base.ID, base.Revision) + if err != nil { + return ApplyAttempt{}, err + } + if !validPlanForOperation(base.Operation, steps, len(attachments)) { + return ApplyAttempt{}, localError(errors.New("stored apply plan")) + } + attemptID, err := newIdentity("att_") + if err != nil { + return ApplyAttempt{}, errors.New("stage store: random identity unavailable") + } + pendingID, err := newIdentity("pending_") + if err != nil { + return ApplyAttempt{}, errors.New("stage store: random identity unavailable") + } + now := time.Now().UTC() + stamp := formatTime(now) + forced := in.RecoveryMode == RecoveryModeUnknown + if _, err = tx.ExecContext(ctx, `INSERT INTO apply_attempts(id,stage_id,revision,semantic_digest,recovery_mode,prior_recovery,forced_duplicate_risk,plan_json,pending_post_id,started_at) VALUES(?,?,?,?,?,?,?,?,?,?)`, + attemptID, base.ID, base.Revision, base.SemanticDigest[:], in.RecoveryMode, base.Recovery, boolInt(forced), string(plan), pendingID, stamp); err != nil { + return ApplyAttempt{}, localError(err) + } + for _, step := range steps { + if _, err = tx.ExecContext(ctx, `INSERT INTO apply_steps(attempt_id,ordinal,kind,condition,state) VALUES(?,?,?,?,'pending')`, attemptID, step.Ordinal, step.Kind, step.Condition); err != nil { + return ApplyAttempt{}, localError(err) + } + } + if _, err = tx.ExecContext(ctx, `INSERT INTO apply_events(attempt_id,event,recorded_at) VALUES(?,'claimed',?)`, attemptID, stamp); err != nil { + return ApplyAttempt{}, localError(err) + } + if in.RequestID != "" { + if _, err = tx.ExecContext(ctx, `INSERT INTO apply_requests(server_url,user_id,request_id,request_digest,attempt_id,created_at) VALUES(?,?,?,?,?,?)`, base.ServerURL, base.UserID, in.RequestID, in.RequestDigest[:], attemptID, stamp); err != nil { + return ApplyAttempt{}, localError(err) + } + } + result, err := tx.ExecContext(ctx, `UPDATE stages SET lifecycle='applying',claim_attempt_id=?,updated_at=? WHERE id=? AND current_revision=? AND lifecycle='open' AND recovery=? AND claim_attempt_id IS NULL`, attemptID, stamp, base.ID, base.Revision, base.Recovery) + if err != nil { + return ApplyAttempt{}, localError(err) + } + if !oneRow(result) { + return ApplyAttempt{}, ErrConflict + } + if err = tx.Commit(); err != nil { + return ApplyAttempt{}, localError(err) + } + runCommitHook() + return ApplyAttempt{ID: attemptID, StageID: base.ID, Revision: base.Revision, SemanticDigest: base.SemanticDigest, RecoveryMode: in.RecoveryMode, PriorRecovery: base.Recovery, + ForcedDuplicateRisk: forced, Plan: plan, PendingPostID: pendingID, StartedAt: now, Steps: steps}, nil +} + +func (s *Store) BeginDispatch(ctx context.Context, attemptID string, ordinal int) error { + return s.transitionStep(ctx, attemptID, ordinal, StepPending, StepDispatch, nil) +} + +func (s *Store) MarkStepValidated(ctx context.Context, attemptID string, ordinal int, result json.RawMessage) error { + return s.transitionStep(ctx, attemptID, ordinal, StepDispatch, StepValidated, result) +} + +func (s *Store) MarkStepRejected(ctx context.Context, attemptID string, ordinal int, result json.RawMessage) error { + return s.transitionStep(ctx, attemptID, ordinal, StepDispatch, StepRejected, result) +} + +func (s *Store) MarkStepUnknown(ctx context.Context, attemptID string, ordinal int) error { + return s.transitionStep(ctx, attemptID, ordinal, StepDispatch, StepUnknown, nil) +} + +func (s *Store) MarkStepSkipped(ctx context.Context, attemptID string, ordinal int, result json.RawMessage) error { + return s.transitionStep(ctx, attemptID, ordinal, StepPending, StepSkipped, result) +} + +func (s *Store) transitionStep(ctx context.Context, attemptID string, ordinal int, from, to StepState, result json.RawMessage) error { + if ctx == nil || !bounded(attemptID, maxIdentityBytes) || ordinal < 1 || !validStepTransition(from, to) { + return ErrInvalid + } + if result != nil { + canonical, err := canonicalStepResult(ctx, s.db, attemptID, ordinal, to, result) + if err != nil { + return ErrInvalid + } + result = canonical + } + if (to == StepValidated || to == StepRejected || to == StepSkipped) != (result != nil) { + return ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return localError(err) + } + defer tx.Rollback() + now := time.Now().UTC() + stamp := formatTime(now) + var started, ended any + if to == StepDispatch { + started = stamp + } + if to != StepDispatch { + ended = stamp + } + if from == StepPending { + var blocked int + if err = tx.QueryRowContext(ctx, `SELECT count(*) FROM apply_steps current WHERE current.attempt_id=? AND current.ordinal=? AND ( +EXISTS(SELECT 1 FROM apply_steps prior WHERE prior.attempt_id=current.attempt_id AND prior.ordinalcurrent.ordinal AND later.state!='pending'))`, attemptID, ordinal).Scan(&blocked); err != nil { + return localError(err) + } + if blocked != 0 { + return ErrNotEligible + } + if to == StepSkipped { + var condition string + if err = tx.QueryRowContext(ctx, `SELECT condition FROM apply_steps WHERE attempt_id=? AND ordinal=?`, attemptID, ordinal).Scan(&condition); err != nil { + return localError(err) + } + if condition != "if_missing" { + return ErrNotEligible + } + } + } + res, err := tx.ExecContext(ctx, `UPDATE apply_steps SET state=?,result_json=?,started_at=coalesce(started_at,?),ended_at=? WHERE attempt_id=? AND ordinal=? AND state=? AND EXISTS( +SELECT 1 FROM stages s JOIN apply_attempts a ON a.id=? WHERE s.id=a.stage_id AND s.lifecycle='applying' AND s.claim_attempt_id=a.id AND a.outcome IS NULL)`, + to, nullableRaw(result), started, ended, attemptID, ordinal, from, attemptID) + if err != nil { + return localError(err) + } + if !oneRow(res) { + return ErrConflict + } + if _, err = tx.ExecContext(ctx, `INSERT INTO apply_events(attempt_id,ordinal,event,recorded_at) VALUES(?,?,?,?)`, attemptID, ordinal, to, stamp); err != nil { + return localError(err) + } + if err = tx.Commit(); err != nil { + return localError(err) + } + runCommitHook() + return nil +} + +func (s *Store) FinalizeApply(ctx context.Context, attemptID string) (ApplyReceipt, error) { + if ctx == nil || !bounded(attemptID, maxIdentityBytes) { + return ApplyReceipt{}, ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return ApplyReceipt{}, localError(err) + } + defer tx.Rollback() + attempt, err := scanApplyAttempt(ctx, tx, attemptID) + if err != nil { + return ApplyReceipt{}, err + } + if attempt.Outcome != nil { + receipt, receiptErr := loadApplyReceipt(ctx, tx, attemptID) + if receiptErr != nil { + return ApplyReceipt{}, receiptErr + } + var operation Operation + var destination string + if receiptErr = tx.QueryRowContext(ctx, `SELECT s.operation,r.destination_json FROM stages s JOIN stage_revisions r ON r.stage_id=s.id AND r.revision=? WHERE s.id=?`, attempt.Revision, attempt.StageID).Scan(&operation, &destination); receiptErr != nil { + return ApplyReceipt{}, localError(receiptErr) + } + if !validReceiptForAttempt(receipt, attempt) || receipt.Operation != operation || !bytes.Equal(receipt.Destination, []byte(destination)) { + return ApplyReceipt{}, localError(errors.New("apply receipt binding")) + } + receipt.Replay = true + return receipt, nil + } + if hasStoppedStep(attempt.Steps) { + if err = sealPendingSteps(ctx, tx, &attempt, formatTime(time.Now().UTC()), "not_dispatched"); err != nil { + return ApplyReceipt{}, err + } + } + outcome, recovery, lifecycle, err := deriveAttemptResult(attempt) + if err != nil { + return ApplyReceipt{}, err + } + var operation Operation + var destination string + var currentRevision int64 + var claimed sql.NullString + if err = tx.QueryRowContext(ctx, `SELECT s.operation,r.destination_json,s.current_revision,s.claim_attempt_id FROM stages s JOIN stage_revisions r ON r.stage_id=s.id AND r.revision=a.revision JOIN apply_attempts a ON a.stage_id=s.id WHERE a.id=?`, attemptID).Scan(&operation, &destination, ¤tRevision, &claimed); err != nil { + return ApplyReceipt{}, localError(err) + } + if currentRevision != attempt.Revision || !claimed.Valid || claimed.String != attempt.ID { + return ApplyReceipt{}, ErrConflict + } + now := time.Now().UTC() + stamp := formatTime(now) + if _, err = tx.ExecContext(ctx, `UPDATE apply_attempts SET outcome=?,ended_at=? WHERE id=? AND outcome IS NULL`, outcome, stamp, attemptID); err != nil { + return ApplyReceipt{}, localError(err) + } + res, err := tx.ExecContext(ctx, `UPDATE stages SET lifecycle=?,recovery=?,claim_attempt_id=NULL,updated_at=? WHERE id=? AND lifecycle='applying' AND claim_attempt_id=?`, lifecycle, recovery, stamp, attempt.StageID, attemptID) + if err != nil { + return ApplyReceipt{}, localError(err) + } + if !oneRow(res) { + return ApplyReceipt{}, ErrConflict + } + if lifecycle == LifecycleCompleted { + if _, err = tx.ExecContext(ctx, `UPDATE stage_revisions SET body=NULL WHERE stage_id=?`, attempt.StageID); err != nil { + return ApplyReceipt{}, localError(err) + } + if _, err = tx.ExecContext(ctx, `DELETE FROM stage_attachments WHERE stage_id=?`, attempt.StageID); err != nil { + return ApplyReceipt{}, localError(err) + } + } + attempt.Outcome, attempt.EndedAt = &outcome, &now + receipt := ApplyReceipt{"mm/v2/apply-receipt", attempt.ID, attempt.StageID, attempt.Revision, hex.EncodeToString(attempt.SemanticDigest[:]), operation, json.RawMessage(destination), outcome, recovery, attempt.StartedAt, now, attempt.Steps, false} + raw, err := marshalCanonical(receipt) + if err != nil { + return ApplyReceipt{}, localError(err) + } + if _, err = tx.ExecContext(ctx, `INSERT INTO apply_receipts(attempt_id,receipt_json,recorded_at) VALUES(?,?,?)`, attemptID, string(raw), stamp); err != nil { + return ApplyReceipt{}, localError(err) + } + if _, err = tx.ExecContext(ctx, `INSERT INTO apply_events(attempt_id,event,recorded_at) VALUES(?,'completed',?)`, attemptID, stamp); err != nil { + return ApplyReceipt{}, localError(err) + } + if err = tx.Commit(); err != nil { + return ApplyReceipt{}, localError(err) + } + runCommitHook() + return receipt, nil +} + +func (s *Store) AbandonApplyBeforeDispatch(ctx context.Context, attemptID string) error { + if ctx == nil || !bounded(attemptID, maxIdentityBytes) { + return ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return localError(err) + } + defer tx.Rollback() + var stageID string + var count int + if err = tx.QueryRowContext(ctx, `SELECT a.stage_id,count(CASE WHEN p.state!='pending' THEN 1 END) FROM apply_attempts a JOIN apply_steps p ON p.attempt_id=a.id WHERE a.id=? AND a.outcome IS NULL GROUP BY a.stage_id`, attemptID).Scan(&stageID, &count); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return ErrNotFound + } + return localError(err) + } + if count != 0 { + return ErrNotEligible + } + stamp := formatTime(time.Now().UTC()) + res, err := tx.ExecContext(ctx, `UPDATE stages SET lifecycle='open',claim_attempt_id=NULL,updated_at=? WHERE id=? AND lifecycle='applying' AND claim_attempt_id=?`, stamp, stageID, attemptID) + if err != nil { + return localError(err) + } + if !oneRow(res) { + return ErrConflict + } + if _, err = tx.ExecContext(ctx, `DELETE FROM apply_steps WHERE attempt_id=?`, attemptID); err != nil { + return localError(err) + } + if _, err = tx.ExecContext(ctx, `DELETE FROM apply_events WHERE attempt_id=?`, attemptID); err != nil { + return localError(err) + } + if _, err = tx.ExecContext(ctx, `DELETE FROM apply_requests WHERE attempt_id=?`, attemptID); err != nil { + return localError(err) + } + if _, err = tx.ExecContext(ctx, `DELETE FROM apply_attempts WHERE id=?`, attemptID); err != nil { + return localError(err) + } + if err = tx.Commit(); err != nil { + return localError(err) + } + runCommitHook() + return nil +} + +func (s *Store) FindApply(ctx context.Context, server, user, requestID string, requestDigest [32]byte) (ApplyAttempt, bool, error) { + if ctx == nil || !canonicalServerURL(server) || !bounded(user, maxIdentityBytes) || !validRequestID(requestID) || requestID == "" || requestDigest == ([32]byte{}) { + return ApplyAttempt{}, false, ErrInvalid + } + attempt, found, err := findApplyRequest(ctx, s.db, server, user, requestID, requestDigest) + if found { + attempt.Replay = true + } + return attempt, found, err +} + +func findApplyRequest(ctx context.Context, q queryer, server, user, requestID string, requestDigest [32]byte) (ApplyAttempt, bool, error) { + var storedDigest []byte + var attemptID string + err := q.QueryRowContext(ctx, `SELECT request_digest,attempt_id FROM apply_requests WHERE server_url=? AND user_id=? AND request_id=?`, server, user, requestID).Scan(&storedDigest, &attemptID) + if errors.Is(err, sql.ErrNoRows) { + return ApplyAttempt{}, false, nil + } + if err != nil { + return ApplyAttempt{}, false, localError(err) + } + if len(storedDigest) != sha256.Size || !bytes.Equal(storedDigest, requestDigest[:]) { + return ApplyAttempt{}, false, ErrConflict + } + attempt, err := scanApplyAttempt(ctx, q, attemptID) + return attempt, err == nil, err +} + +func scanApplyAttempt(ctx context.Context, q queryer, attemptID string) (ApplyAttempt, error) { + var a ApplyAttempt + var plan, started string + var ended, outcome sql.NullString + var forced int + var semantic []byte + err := q.QueryRowContext(ctx, `SELECT id,stage_id,revision,semantic_digest,recovery_mode,prior_recovery,forced_duplicate_risk,plan_json,pending_post_id,started_at,ended_at,outcome FROM apply_attempts WHERE id=?`, attemptID). + Scan(&a.ID, &a.StageID, &a.Revision, &semantic, &a.RecoveryMode, &a.PriorRecovery, &forced, &plan, &a.PendingPostID, &started, &ended, &outcome) + if errors.Is(err, sql.ErrNoRows) { + return ApplyAttempt{}, ErrNotFound + } + if err != nil { + return ApplyAttempt{}, localError(err) + } + a.ForcedDuplicateRisk = forced == 1 + if len(semantic) != sha256.Size { + return ApplyAttempt{}, localError(errors.New("attempt semantic digest")) + } + copy(a.SemanticDigest[:], semantic) + a.Plan = json.RawMessage(plan) + if a.StartedAt, err = parseTime(started); err != nil { + return ApplyAttempt{}, err + } + if ended.Valid { + value, parseErr := parseTime(ended.String) + if parseErr != nil { + return ApplyAttempt{}, parseErr + } + a.EndedAt = &value + } + if outcome.Valid { + value := AttemptOutcome(outcome.String) + if !validOutcome(value) { + return ApplyAttempt{}, localError(errors.New("attempt outcome")) + } + a.Outcome = &value + } + rows, err := q.QueryContext(ctx, `SELECT ordinal,kind,condition,state,result_json,started_at,ended_at FROM apply_steps WHERE attempt_id=? ORDER BY ordinal`, attemptID) + if err != nil { + return ApplyAttempt{}, localError(err) + } + for rows.Next() { + var step ApplyStep + var result, stepStarted, stepEnded sql.NullString + if err = rows.Scan(&step.Ordinal, &step.Kind, &step.Condition, &step.State, &result, &stepStarted, &stepEnded); err != nil { + return ApplyAttempt{}, localError(err) + } + if result.Valid { + step.Result = json.RawMessage(result.String) + } + if step.StartedAt, err = parseOptionalTime(stepStarted); err != nil { + return ApplyAttempt{}, err + } + if step.EndedAt, err = parseOptionalTime(stepEnded); err != nil { + return ApplyAttempt{}, err + } + a.Steps = append(a.Steps, step) + } + if err = rows.Err(); err != nil { + rows.Close() + return ApplyAttempt{}, localError(errors.New("apply attempt projection")) + } + if err = rows.Close(); err != nil || !validAttempt(a) { + return ApplyAttempt{}, localError(errors.New("apply attempt projection")) + } + for _, step := range a.Steps { + if step.State != StepValidated && step.State != StepRejected && step.State != StepSkipped { + continue + } + canonical, validationErr := canonicalStepResult(ctx, q, a.ID, step.Ordinal, step.State, step.Result) + if validationErr != nil || !bytes.Equal(canonical, step.Result) { + return ApplyAttempt{}, localError(errors.New("apply step result binding")) + } + } + return a, nil +} + +func loadApplyReceipt(ctx context.Context, q queryer, attemptID string) (ApplyReceipt, error) { + var raw, recorded string + if err := q.QueryRowContext(ctx, `SELECT receipt_json,recorded_at FROM apply_receipts WHERE attempt_id=?`, attemptID).Scan(&raw, &recorded); err != nil { + return ApplyReceipt{}, localError(err) + } + var receipt ApplyReceipt + decoder := json.NewDecoder(bytes.NewBufferString(raw)) + decoder.DisallowUnknownFields() + if decoder.Decode(&receipt) != nil || decoder.Decode(new(any)) != io.EOF || receipt.Schema != "mm/v2/apply-receipt" || receipt.AttemptID != attemptID { + return ApplyReceipt{}, localError(errors.New("apply receipt projection")) + } + for i := range receipt.Steps { + if bytes.Equal(receipt.Steps[i].Result, []byte("null")) { + receipt.Steps[i].Result = nil + } + } + want, err := parseTime(recorded) + if err != nil || !receipt.RecordedAt.Equal(want) { + return ApplyReceipt{}, localError(errors.New("apply receipt timestamp")) + } + return receipt, nil +} + +func decodePersistedPlan(raw json.RawMessage) (json.RawMessage, []ApplyStep, error) { + canonical, err := canonicalObject(raw) + if err != nil { + return nil, nil, err + } + var plan persistedPlan + decoder := json.NewDecoder(bytes.NewReader(canonical)) + decoder.DisallowUnknownFields() + if decoder.Decode(&plan) != nil || decoder.Decode(new(any)) != io.EOF || len(plan.Steps) == 0 || len(plan.Steps) > maxAttachments+2 { + return nil, nil, ErrInvalid + } + steps := make([]ApplyStep, len(plan.Steps)) + for i, input := range plan.Steps { + if input.Ordinal != i+1 || !validStepKind(input.Type) || input.Condition != "always" && input.Condition != "if_missing" { + return nil, nil, ErrInvalid + } + steps[i] = ApplyStep{Ordinal: input.Ordinal, Kind: input.Type, Condition: input.Condition, State: StepPending} + } + return canonical, steps, nil +} + +func canonicalStepResult(ctx context.Context, q queryer, attemptID string, ordinal int, state StepState, raw json.RawMessage) (json.RawMessage, error) { + canonical, err := canonicalObject(raw) + if err != nil { + return nil, err + } + var kind, pendingPostID, userID, destinationRaw string + if err = q.QueryRowContext(ctx, `SELECT p.kind,a.pending_post_id,s.user_id,r.destination_json + FROM apply_steps p JOIN apply_attempts a ON a.id=p.attempt_id JOIN stages s ON s.id=a.stage_id + JOIN stage_revisions r ON r.stage_id=a.stage_id AND r.revision=a.revision + WHERE p.attempt_id=? AND p.ordinal=?`, attemptID, ordinal).Scan(&kind, &pendingPostID, &userID, &destinationRaw); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrInvalid + } + return nil, localError(err) + } + if state == StepRejected { + var result struct { + Status int `json:"status"` + } + if decodeNarrow(canonical, &result) != nil || result.Status < 400 || result.Status > 499 { + return nil, ErrInvalid + } + return marshalCanonical(result) + } + if state == StepSkipped { + var result struct { + Reason string `json:"reason"` + } + if decodeNarrow(canonical, &result) != nil || result.Reason != "already_satisfied" { + return nil, ErrInvalid + } + return marshalCanonical(result) + } + if state != StepValidated { + return nil, ErrInvalid + } + var destination struct { + ChannelID *string `json:"channelId"` + PostID *string `json:"postId"` + ParticipantIDs []string `json:"participantIds"` + } + if json.Unmarshal([]byte(destinationRaw), &destination) != nil { + return nil, ErrInvalid + } + switch kind { + case "upload_attachment": + var result struct { + FileID string `json:"fileId"` + } + if decodeNarrow(canonical, &result) != nil || !validReceiptID(result.FileID) { + return nil, ErrInvalid + } + return marshalCanonical(result) + case "create_post": + var result struct { + PostID string `json:"postId"` + CreateAt int64 `json:"createAt"` + ChannelID string `json:"channelId"` + UserID string `json:"userId"` + PendingPostID string `json:"pendingPostId"` + } + if decodeNarrow(canonical, &result) != nil || !validReceiptID(result.PostID) || !validRemoteTimestamp(result.CreateAt) || destination.ChannelID == nil || result.ChannelID != *destination.ChannelID || result.UserID != userID || result.PendingPostID != pendingPostID { + return nil, ErrInvalid + } + return marshalCanonical(result) + case "edit_post": + var result struct { + PostID string `json:"postId"` + UpdateAt int64 `json:"updateAt"` + } + if decodeNarrow(canonical, &result) != nil || !validReceiptID(result.PostID) || !validRemoteTimestamp(result.UpdateAt) || destination.PostID == nil || result.PostID != *destination.PostID { + return nil, ErrInvalid + } + return marshalCanonical(result) + case "delete_post": + var result struct { + PostID string `json:"postId"` + DeleteAt int64 `json:"deleteAt"` + } + if decodeNarrow(canonical, &result) != nil || !validReceiptID(result.PostID) || !validRemoteTimestamp(result.DeleteAt) || destination.PostID == nil || result.PostID != *destination.PostID { + return nil, ErrInvalid + } + return marshalCanonical(result) + case "add_reaction", "remove_reaction": + var result struct { + PostID string `json:"postId"` + } + if decodeNarrow(canonical, &result) != nil || !validReceiptID(result.PostID) || destination.PostID == nil || result.PostID != *destination.PostID { + return nil, ErrInvalid + } + return marshalCanonical(result) + case "resolve_conversation": + var result struct { + ChannelID string `json:"channelId"` + ParticipantIDs []string `json:"participantIds"` + } + if decodeNarrow(canonical, &result) != nil || !validReceiptID(result.ChannelID) || destination.ChannelID != nil && result.ChannelID != *destination.ChannelID || !slices.Equal(result.ParticipantIDs, destination.ParticipantIDs) { + return nil, ErrInvalid + } + return marshalCanonical(result) + default: + return nil, ErrInvalid + } +} + +func decodeNarrow(raw []byte, out any) error { + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(out); err != nil { + return err + } + if err := decoder.Decode(new(any)); err != io.EOF { + return ErrInvalid + } + return nil +} + +func validReceiptID(value string) bool { + return boundedMetadata(value, maxIdentityBytes) +} + +func validRemoteTimestamp(value int64) bool { + return value > 0 && value <= 9_007_199_254_740_991 +} + +func deriveAttemptResult(a ApplyAttempt) (AttemptOutcome, Recovery, Lifecycle, error) { + validated, skipped, notSent, rejected, unknown, dispatching := 0, 0, 0, 0, 0, 0 + for _, step := range a.Steps { + switch step.State { + case StepValidated: + validated++ + case StepSkipped: + skipped++ + case StepRejected: + rejected++ + case StepUnknown: + unknown++ + case StepNotSent: + notSent++ + case StepDispatch: + dispatching++ + case StepPending: + default: + return "", "", "", ErrInvalid + } + } + if dispatching > 0 { + return "", "", "", ErrNotEligible + } + if unknown > 0 { + return OutcomeUnknown, RecoveryUnknown, LifecycleOpen, nil + } + if rejected > 0 { + if rejected != 1 { + return "", "", "", ErrInvalid + } + if validated > 0 { + return OutcomePartial, maxRecovery(a.PriorRecovery, RecoveryPartial), LifecycleOpen, nil + } + return OutcomeRejected, a.PriorRecovery, LifecycleOpen, nil + } + if notSent > 0 { + if validated+skipped == 0 { + return OutcomeRejected, a.PriorRecovery, LifecycleOpen, nil + } + return OutcomePartial, maxRecovery(a.PriorRecovery, RecoveryPartial), LifecycleOpen, nil + } + if validated+skipped != len(a.Steps) { + return "", "", "", ErrNotEligible + } + if validated == 0 { + return OutcomeAlreadySatisfied, RecoveryForbidden, LifecycleCompleted, nil + } + return OutcomeSucceeded, RecoveryForbidden, LifecycleCompleted, nil +} + +func validAttempt(a ApplyAttempt) bool { + if !bounded(a.ID, maxIdentityBytes) || !bounded(a.StageID, maxIdentityBytes) || a.Revision < 1 || a.SemanticDigest == ([32]byte{}) || !validRecoveryMode(a.RecoveryMode) || !validRecovery(a.PriorRecovery) || a.PriorRecovery == RecoveryForbidden || + len(a.Steps) == 0 || a.StartedAt.IsZero() || !bounded(a.PendingPostID, maxIdentityBytes) || a.ForcedDuplicateRisk != (a.RecoveryMode == RecoveryModeUnknown) || !modeMatchesRecovery(a.RecoveryMode, a.PriorRecovery) || + (a.Outcome == nil) != (a.EndedAt == nil) || a.EndedAt != nil && a.EndedAt.Before(a.StartedAt) { + return false + } + plan, planned, err := decodePersistedPlan(a.Plan) + if err != nil || !bytes.Equal(plan, a.Plan) || len(planned) != len(a.Steps) { + return false + } + for i, step := range a.Steps { + if step.Ordinal != i+1 || step.Kind != planned[i].Kind || step.Condition != planned[i].Condition || !validApplyStep(step) { + return false + } + } + return true +} + +func validApplyStep(step ApplyStep) bool { + if !validStepKind(step.Kind) || step.Condition != "always" && step.Condition != "if_missing" || !validStepState(step.State) { + return false + } + switch step.State { + case StepPending: + return step.StartedAt == nil && step.EndedAt == nil && step.Result == nil + case StepDispatch: + return step.StartedAt != nil && step.EndedAt == nil && step.Result == nil + case StepValidated, StepRejected: + return step.StartedAt != nil && step.EndedAt != nil && step.Result != nil && !step.EndedAt.Before(*step.StartedAt) + case StepSkipped: + return step.Condition == "if_missing" && step.StartedAt == nil && step.EndedAt != nil && step.Result != nil + case StepUnknown: + return step.StartedAt != nil && step.EndedAt != nil && step.Result == nil && !step.EndedAt.Before(*step.StartedAt) + case StepNotSent: + return step.StartedAt == nil && step.EndedAt != nil && step.Result == nil + } + return false +} + +func validReceiptForAttempt(receipt ApplyReceipt, attempt ApplyAttempt) bool { + if receipt.Schema != "mm/v2/apply-receipt" || receipt.AttemptID != attempt.ID || receipt.StageID != attempt.StageID || receipt.Revision != attempt.Revision || receipt.SemanticDigest != hex.EncodeToString(attempt.SemanticDigest[:]) || + attempt.Outcome == nil || receipt.Outcome != *attempt.Outcome || attempt.EndedAt == nil || !receipt.StartedAt.Equal(attempt.StartedAt) || !receipt.RecordedAt.Equal(*attempt.EndedAt) || + !validOperation(receipt.Operation) || !validRecovery(receipt.Recovery) || len(receipt.Steps) != len(attempt.Steps) { + return false + } + if _, err := canonicalObject(receipt.Destination); err != nil { + return false + } + derivedOutcome, derivedRecovery, _, err := deriveAttemptResult(attempt) + if err != nil || derivedOutcome != receipt.Outcome || derivedRecovery != receipt.Recovery { + return false + } + for i := range receipt.Steps { + if receipt.Steps[i].Ordinal != attempt.Steps[i].Ordinal || receipt.Steps[i].Kind != attempt.Steps[i].Kind || receipt.Steps[i].Condition != attempt.Steps[i].Condition || receipt.Steps[i].State != attempt.Steps[i].State || + !bytes.Equal(receipt.Steps[i].Result, attempt.Steps[i].Result) || !timePointerEqual(receipt.Steps[i].StartedAt, attempt.Steps[i].StartedAt) || !timePointerEqual(receipt.Steps[i].EndedAt, attempt.Steps[i].EndedAt) { + return false + } + } + return true +} + +func timePointerEqual(a, b *time.Time) bool { + return a == nil && b == nil || a != nil && b != nil && a.Equal(*b) +} + +func validRecoveryMode(mode RecoveryMode) bool { + return mode == RecoveryModeOrdinary || mode == RecoveryModePartial || mode == RecoveryModeUnknown +} + +func modeMatchesRecovery(mode RecoveryMode, recovery Recovery) bool { + return mode == RecoveryModeOrdinary && recovery == RecoveryNone || mode == RecoveryModePartial && recovery == RecoveryPartial || mode == RecoveryModeUnknown && recovery == RecoveryUnknown +} + +func validStepKind(kind string) bool { + switch kind { + case "upload_attachment", "create_post", "edit_post", "delete_post", "add_reaction", "remove_reaction", "resolve_conversation": + return true + } + return false +} + +func validPlanForOperation(operation Operation, steps []ApplyStep, attachmentCount int) bool { + single := func(kind, condition string) bool { + return len(steps) == 1 && steps[0].Kind == kind && steps[0].Condition == condition + } + switch operation { + case CreatePost, Reply: + if len(steps) != attachmentCount+1 || steps[len(steps)-1].Kind != "create_post" || steps[len(steps)-1].Condition != "always" { + return false + } + for _, step := range steps[:len(steps)-1] { + if step.Kind != "upload_attachment" || step.Condition != "always" { + return false + } + } + return true + case EditPost: + return attachmentCount == 0 && single("edit_post", "always") + case DeletePost: + return attachmentCount == 0 && single("delete_post", "always") + case React: + return attachmentCount == 0 && single("add_reaction", "if_missing") + case Unreact: + return attachmentCount == 0 && single("remove_reaction", "if_missing") + case ResolveDM, ResolveGroupDM: + return attachmentCount == 0 && single("resolve_conversation", "if_missing") + } + return false +} + +func validStepState(state StepState) bool { + switch state { + case StepPending, StepDispatch, StepValidated, StepRejected, StepUnknown, StepSkipped, StepNotSent: + return true + } + return false +} + +func validStepTransition(from, to StepState) bool { + return from == StepPending && (to == StepDispatch || to == StepSkipped) || from == StepDispatch && (to == StepValidated || to == StepRejected || to == StepUnknown) +} + +func validOutcome(outcome AttemptOutcome) bool { + return outcome == OutcomeSucceeded || outcome == OutcomeAlreadySatisfied || outcome == OutcomeRejected || outcome == OutcomePartial || outcome == OutcomeUnknown +} + +func maxRecovery(a, b Recovery) Recovery { + weight := map[Recovery]int{RecoveryNone: 0, RecoveryPartial: 1, RecoveryUnknown: 2, RecoveryForbidden: 3} + if weight[a] >= weight[b] { + return a + } + return b +} + +func parseOptionalTime(raw sql.NullString) (*time.Time, error) { + if !raw.Valid { + return nil, nil + } + value, err := parseTime(raw.String) + if err != nil { + return nil, err + } + return &value, nil +} + +func nullableRaw(raw json.RawMessage) any { + if raw == nil { + return nil + } + return string(raw) +} + +func boolInt(value bool) int { + if value { + return 1 + } + return 0 +} + +func hasStoppedStep(steps []ApplyStep) bool { + for _, step := range steps { + if step.State == StepRejected || step.State == StepUnknown { + return true + } + } + return false +} + +func sealPendingSteps(ctx context.Context, tx *sql.Tx, attempt *ApplyAttempt, stamp, event string) error { + for i := range attempt.Steps { + if attempt.Steps[i].State != StepPending { + continue + } + result, err := tx.ExecContext(ctx, `UPDATE apply_steps SET state='not_dispatched',ended_at=? WHERE attempt_id=? AND ordinal=? AND state='pending'`, stamp, attempt.ID, attempt.Steps[i].Ordinal) + if err != nil { + return localError(err) + } + if !oneRow(result) { + return ErrConflict + } + if _, err = tx.ExecContext(ctx, `INSERT INTO apply_events(attempt_id,ordinal,event,recorded_at) VALUES(?,?,?,?)`, attempt.ID, attempt.Steps[i].Ordinal, event, stamp); err != nil { + return localError(err) + } + ended, parseErr := parseTime(stamp) + if parseErr != nil { + return parseErr + } + attempt.Steps[i].State, attempt.Steps[i].EndedAt = StepNotSent, &ended + } + return nil +} diff --git a/internal/stagestore/apply_test.go b/internal/stagestore/apply_test.go new file mode 100644 index 0000000..850668c --- /dev/null +++ b/internal/stagestore/apply_test.go @@ -0,0 +1,672 @@ +//go:build darwin || linux + +package stagestore + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "strings" + "sync" + "testing" +) + +func createApplyStage(t *testing.T, s *Store, plan string) CreateRecord { + t.Helper() + attachments := make([]Attachment, strings.Count(plan, `"type":"upload_attachment"`)) + for i := range attachments { + attachments[i] = attachment(string(rune('a'+i)) + ".txt") + } + created, err := s.Create(context.Background(), CreateInput{ + RequestDigest: sha256.Sum256([]byte("apply-stage")), Operation: CreatePost, ServerURL: "https://mattermost.example/api/v4", ServerID: "server-1", UserID: "user-1", + Content: RevisionContent{Body: []byte("**reviewed**\n"), Destination: json.RawMessage(`{"kind":"conversation","channelId":"channel-1"}`), Plan: json.RawMessage(plan), Attachments: attachments}, + }) + if err != nil { + t.Fatal(err) + } + return created +} + +func createConversationStage(t *testing.T, s *Store) CreateRecord { + t.Helper() + created, err := s.Create(context.Background(), CreateInput{ + RequestDigest: sha256.Sum256([]byte("conversation-stage")), Operation: ResolveDM, ServerURL: "https://mattermost.example/api/v4", ServerID: "server-1", UserID: "user-1", + Content: RevisionContent{Destination: json.RawMessage(`{"kind":"conversation","channelId":null,"participantIds":["peer-1"]}`), Plan: json.RawMessage(`{"steps":[{"ordinal":1,"type":"resolve_conversation","condition":"if_missing"}]}`)}, + }) + if err != nil { + t.Fatal(err) + } + return created +} + +func claimInput(stage StageSummary, request string, mode RecoveryMode) ApplyClaimInput { + return ApplyClaimInput{StageID: stage.ID, RequestID: request, Revision: stage.Revision, ExpectedDigest: stage.SemanticDigest, + RequestDigest: sha256.Sum256([]byte("apply\x00" + request)), RecoveryMode: mode} +} + +func createPostResult(t *testing.T, attempt ApplyAttempt, postID string) json.RawMessage { + t.Helper() + raw, err := json.Marshal(struct { + PostID string `json:"postId"` + CreateAt int64 `json:"createAt"` + ChannelID string `json:"channelId"` + UserID string `json:"userId"` + PendingPostID string `json:"pendingPostId"` + }{postID, 1784250000000, "channel-1", "user-1", attempt.PendingPostID}) + if err != nil { + t.Fatal(err) + } + return raw +} + +func TestApplyClaimBindsExactRevisionAndReplaysCallerRequest(t *testing.T) { + s := openDomainStore(t) + created := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) + in := claimInput(created.Stage, "apply-request-1", RecoveryModeOrdinary) + claimed, err := s.ClaimApply(context.Background(), in) + if err != nil || claimed.StageID != created.Stage.ID || claimed.Revision != 1 || claimed.RecoveryMode != RecoveryModeOrdinary || claimed.ForcedDuplicateRisk || len(claimed.Steps) != 1 || claimed.Steps[0].State != StepPending { + t.Fatalf("claim=%+v err=%v", claimed, err) + } + replayed, err := s.ClaimApply(context.Background(), in) + if err != nil || !replayed.Replay || replayed.ID != claimed.ID || replayed.PendingPostID != claimed.PendingPostID { + t.Fatalf("replay=%+v err=%v", replayed, err) + } + conflict := in + conflict.RequestDigest[0] ^= 0xff + if _, err = s.ClaimApply(context.Background(), conflict); !errors.Is(err, ErrConflict) { + t.Fatalf("request conflict=%v", err) + } + stale := in + stale.RequestID = "" + stale.RequestDigest = [32]byte{} + stale.ExpectedDigest[0] ^= 0xff + if _, err = s.ClaimApply(context.Background(), stale); !errors.Is(err, ErrConflict) { + t.Fatalf("revision conflict=%v", err) + } +} + +func TestApplyClaimAcceptsConversationResolutionPlan(t *testing.T) { + s := openDomainStore(t) + created := createConversationStage(t, s) + claim, err := s.ClaimApply(context.Background(), claimInput(created.Stage, "", RecoveryModeOrdinary)) + if err != nil || claim.Steps[0].Kind != "resolve_conversation" { + t.Fatalf("claim=%+v err=%v", claim, err) + } +} + +func TestApplySuccessJournalClearsSensitiveCompositionAtomically(t *testing.T) { + s := openDomainStore(t) + created := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) + claimed, err := s.ClaimApply(context.Background(), claimInput(created.Stage, "", RecoveryModeOrdinary)) + if err != nil { + t.Fatal(err) + } + if err = s.BeginDispatch(context.Background(), claimed.ID, 1); err != nil { + t.Fatal(err) + } + result := createPostResult(t, claimed, "post-1") + if err = s.MarkStepValidated(context.Background(), claimed.ID, 1, result); err != nil { + t.Fatal(err) + } + receipt, err := s.FinalizeApply(context.Background(), claimed.ID) + if err != nil || receipt.Outcome != OutcomeSucceeded || receipt.Recovery != RecoveryForbidden || receipt.Steps[0].State != StepValidated || string(receipt.Steps[0].Result) != `{"postId":"post-1","createAt":1784250000000,"channelId":"channel-1","userId":"user-1","pendingPostId":"`+claimed.PendingPostID+`"}` { + t.Fatalf("receipt=%+v err=%v", receipt, err) + } + detail, err := s.Show(context.Background(), created.Stage.ID) + if err != nil || detail.Lifecycle != LifecycleCompleted || detail.Recovery != RecoveryForbidden || detail.Body != nil || len(detail.Attachments) != 0 { + t.Fatalf("detail=%+v err=%v", detail, err) + } + replay, err := s.FinalizeApply(context.Background(), claimed.ID) + if err != nil || !replay.Replay || replay.AttemptID != receipt.AttemptID || replay.RecordedAt != receipt.RecordedAt { + t.Fatalf("receipt replay=%+v err=%v", replay, err) + } + if _, err = s.db.Exec(`UPDATE apply_attempts SET outcome='unknown' WHERE id=?`, claimed.ID); err == nil { + t.Fatal("terminal attempt outcome mutation succeeded") + } + if _, err = s.db.Exec(`UPDATE stage_revisions SET destination_json='{}' WHERE stage_id=?`, created.Stage.ID); err == nil { + t.Fatal("completed destination mutation succeeded") + } + if _, err = s.db.Exec(`UPDATE stage_revisions SET state='superseded' WHERE stage_id=? AND revision=1`, created.Stage.ID); err == nil { + t.Fatal("completed revision replacement transition succeeded") + } + if _, err = s.db.Exec(`INSERT OR REPLACE INTO stage_revisions(stage_id,revision,state,created_at,semantic_digest,body,destination_json,plan_json) + SELECT stage_id,revision,state,created_at,semantic_digest,body,destination_json,plan_json FROM stage_revisions WHERE stage_id=? AND revision=1`, created.Stage.ID); err == nil { + t.Fatal("replace bypass of completed revision succeeded") + } +} + +func TestConfirmedApplyPreservesCreateAndReviseRequestReplayAfterContentErasure(t *testing.T) { + s := openDomainStore(t) + plan := json.RawMessage(`{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) + createInput := CreateInput{ + RequestID: "create-after-apply", RequestDigest: sha256.Sum256([]byte("create caller intent")), Operation: CreatePost, + ServerURL: "https://mattermost.example/api/v4", ServerID: "server-1", UserID: "user-1", + Content: RevisionContent{Body: []byte("original"), Destination: json.RawMessage(`{"kind":"conversation","channelId":"channel-1"}`), Plan: plan}, + } + created, err := s.Create(context.Background(), createInput) + if err != nil { + t.Fatal(err) + } + reviseInput := ReviseInput{ + StageID: created.Stage.ID, RequestID: "revise-after-apply", ExpectedRevision: created.Stage.Revision, ExpectedDigest: created.Stage.SemanticDigest, + RequestDigest: sha256.Sum256([]byte("revise caller intent")), Composition: Composition{Body: []byte("revised"), Plan: plan}, + } + revised, err := s.Revise(context.Background(), reviseInput) + if err != nil { + t.Fatal(err) + } + claim, err := s.ClaimApply(context.Background(), claimInput(revised.Stage, "", RecoveryModeOrdinary)) + if err != nil { + t.Fatal(err) + } + if err = s.BeginDispatch(context.Background(), claim.ID, 1); err != nil { + t.Fatal(err) + } + if _, err = s.db.Exec(`UPDATE apply_steps SET state='response_validated',result_json='{"message":"secret"}',ended_at='2026-01-01T00:00:01.000000Z' WHERE attempt_id=? AND ordinal=1`, claim.ID); err == nil { + t.Fatal("arbitrary result transition succeeded") + } + mismatched := `{"postId":"post-1","createAt":1784250000000,"channelId":"channel-1","userId":"user-1","pendingPostId":"other"}` + if _, err = s.db.Exec(`UPDATE apply_steps SET state='response_validated',result_json=?,ended_at='2026-01-01T00:00:01.000000Z' WHERE attempt_id=? AND ordinal=1`, mismatched, claim.ID); err == nil { + t.Fatal("mismatched result transition succeeded") + } + if _, err = s.db.Exec(`UPDATE stage_revisions SET state='superseded' WHERE stage_id=? AND revision=1`, created.Stage.ID); err == nil { + t.Fatal("applying revision replacement transition succeeded") + } + if err = s.MarkStepValidated(context.Background(), claim.ID, 1, createPostResult(t, claim, "post-replay")); err != nil { + t.Fatal(err) + } + if _, err = s.FinalizeApply(context.Background(), claim.ID); err != nil { + t.Fatal(err) + } + createReplay, err := s.Create(context.Background(), createInput) + if err != nil || !createReplay.Replay || createReplay.Stage.ID != created.Stage.ID { + t.Fatalf("create replay=%+v err=%v", createReplay, err) + } + reviseReplay, err := s.Revise(context.Background(), reviseInput) + if err != nil || !reviseReplay.Replay || reviseReplay.Stage.ID != revised.Stage.ID || reviseReplay.Stage.Revision != revised.Stage.Revision { + t.Fatalf("revise replay=%+v err=%v", reviseReplay, err) + } + if _, err = s.db.Exec(`UPDATE stage_revisions SET semantic_digest=zeroblob(32) WHERE stage_id=? AND revision=1`, created.Stage.ID); err == nil { + t.Fatal("retained semantic digest mutation succeeded") + } + if _, err = s.db.Exec(`DROP TRIGGER stage_revision_semantics_immutable`); err != nil { + t.Fatal(err) + } + if _, err = s.db.Exec(`UPDATE stage_revisions SET semantic_digest=zeroblob(32) WHERE stage_id=? AND revision=1`, created.Stage.ID); err != nil { + t.Fatal(err) + } + if _, _, err = s.FindCreate(context.Background(), created.Stage.ServerURL, created.Stage.UserID, createInput.RequestID); err == nil || errors.Is(err, ErrConflict) { + t.Fatalf("corrupt retained digest=%v", err) + } +} + +func TestApplySuccessClearsSupersededRevisionPlaintextAndPaths(t *testing.T) { + s := openDomainStore(t) + plan := json.RawMessage(`{"steps":[{"ordinal":1,"type":"upload_attachment","condition":"always"},{"ordinal":2,"type":"create_post","condition":"always"}]}`) + created := createApplyStage(t, s, string(plan)) + revised, err := s.Revise(context.Background(), ReviseInput{StageID: created.Stage.ID, ExpectedRevision: 1, ExpectedDigest: created.Stage.SemanticDigest, + Composition: Composition{Body: []byte("revised secret"), Plan: plan, Attachments: []Attachment{attachment("revised.txt")}}}) + if err != nil { + t.Fatal(err) + } + claim, err := s.ClaimApply(context.Background(), claimInput(revised.Stage, "", RecoveryModeOrdinary)) + if err != nil { + t.Fatal(err) + } + if err = s.BeginDispatch(context.Background(), claim.ID, 1); err != nil { + t.Fatal(err) + } + if err = s.MarkStepValidated(context.Background(), claim.ID, 1, json.RawMessage(`{"fileId":"file-1"}`)); err != nil { + t.Fatal(err) + } + if err = s.BeginDispatch(context.Background(), claim.ID, 2); err != nil { + t.Fatal(err) + } + if err = s.MarkStepValidated(context.Background(), claim.ID, 2, createPostResult(t, claim, "post-1")); err != nil { + t.Fatal(err) + } + if _, err = s.FinalizeApply(context.Background(), claim.ID); err != nil { + t.Fatal(err) + } + var bodies, paths int + if err = s.db.QueryRow(`SELECT count(*) FROM stage_revisions WHERE stage_id=? AND body IS NOT NULL`, created.Stage.ID).Scan(&bodies); err != nil { + t.Fatal(err) + } + if err = s.db.QueryRow(`SELECT count(*) FROM stage_attachments WHERE stage_id=?`, created.Stage.ID).Scan(&paths); err != nil { + t.Fatal(err) + } + if bodies != 0 || paths != 0 { + t.Fatalf("retained bodies/paths=%d/%d", bodies, paths) + } +} + +func TestApplyReceiptRejectsBroadResultsAndUnconditionalSkip(t *testing.T) { + s := openDomainStore(t) + created := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) + claim, _ := s.ClaimApply(context.Background(), claimInput(created.Stage, "", RecoveryModeOrdinary)) + if err := s.MarkStepSkipped(context.Background(), claim.ID, 1, json.RawMessage(`{"reason":"already_satisfied"}`)); !errors.Is(err, ErrNotEligible) { + t.Fatalf("unconditional skip=%v", err) + } + if err := s.BeginDispatch(context.Background(), claim.ID, 1); err != nil { + t.Fatal(err) + } + for _, broad := range []json.RawMessage{ + json.RawMessage(`{"postId":"post-1","createAt":1784250000000,"message":"secret"}`), + json.RawMessage(`{"postId":"post-1","createAt":1784250000000,"raw":{"token":"secret"}}`), + } { + if err := s.MarkStepValidated(context.Background(), claim.ID, 1, broad); !errors.Is(err, ErrInvalid) { + t.Fatalf("broad result %s = %v", broad, err) + } + } + edit, err := s.Create(context.Background(), CreateInput{ + RequestDigest: sha256.Sum256([]byte("edit stage")), + Operation: EditPost, ServerURL: "https://mattermost.example/api/v4", ServerID: "server-1", UserID: "user-1", + Content: RevisionContent{Body: []byte("edited"), Destination: json.RawMessage(`{"kind":"post","postId":"target-post"}`), Plan: json.RawMessage(`{"steps":[{"ordinal":1,"type":"edit_post","condition":"always"}]}`)}, + }) + if err != nil { + t.Fatal(err) + } + editClaim, err := s.ClaimApply(context.Background(), claimInput(edit.Stage, "", RecoveryModeOrdinary)) + if err != nil { + t.Fatal(err) + } + if err = s.MarkStepSkipped(context.Background(), editClaim.ID, 1, json.RawMessage(`{"reason":"already_satisfied"}`)); !errors.Is(err, ErrNotEligible) { + t.Fatalf("unconditional edit skip=%v", err) + } +} + +func TestApplyValidatedResultsBindTheClaimedRemoteEffect(t *testing.T) { + s := openDomainStore(t) + created := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) + claim, err := s.ClaimApply(context.Background(), claimInput(created.Stage, "", RecoveryModeOrdinary)) + if err != nil { + t.Fatal(err) + } + if err = s.BeginDispatch(context.Background(), claim.ID, 1); err != nil { + t.Fatal(err) + } + wrong := createPostResult(t, claim, "post-1") + var decoded map[string]any + if err = json.Unmarshal(wrong, &decoded); err != nil { + t.Fatal(err) + } + for name, value := range map[string]string{"channelId": "other-channel", "userId": "other-user", "pendingPostId": "other-pending"} { + copy := make(map[string]any, len(decoded)) + for key, original := range decoded { + copy[key] = original + } + copy[name] = value + raw, marshalErr := json.Marshal(copy) + if marshalErr != nil { + t.Fatal(marshalErr) + } + if err = s.MarkStepValidated(context.Background(), claim.ID, 1, raw); !errors.Is(err, ErrInvalid) { + t.Fatalf("mismatched %s=%v", name, err) + } + } + for _, tc := range []struct { + name string + operation Operation + body []byte + destination string + plan string + result string + }{ + {"edit", EditPost, []byte("edited"), `{"kind":"post","postId":"target-post"}`, `{"steps":[{"ordinal":1,"type":"edit_post","condition":"always"}]}`, `{"postId":"other-post","updateAt":1784250000000}`}, + {"delete", DeletePost, nil, `{"kind":"post","postId":"target-post"}`, `{"steps":[{"ordinal":1,"type":"delete_post","condition":"always"}]}`, `{"postId":"other-post","deleteAt":1784250000000}`}, + {"reaction", React, nil, `{"kind":"reaction","postId":"target-post"}`, `{"steps":[{"ordinal":1,"type":"add_reaction","condition":"if_missing"}]}`, `{"postId":"other-post"}`}, + {"conversation", ResolveDM, nil, `{"kind":"conversation","channelId":null,"participantIds":["peer-1"]}`, `{"steps":[{"ordinal":1,"type":"resolve_conversation","condition":"if_missing"}]}`, `{"channelId":"channel-1","participantIds":["other-peer"]}`}, + } { + t.Run(tc.name, func(t *testing.T) { + stage, createErr := s.Create(context.Background(), CreateInput{RequestDigest: sha256.Sum256([]byte(tc.name)), Operation: tc.operation, + ServerURL: "https://mattermost.example/api/v4", ServerID: "server-1", UserID: "user-1", + Content: RevisionContent{Body: tc.body, Destination: json.RawMessage(tc.destination), Plan: json.RawMessage(tc.plan)}}) + if createErr != nil { + t.Fatal(createErr) + } + attempt, claimErr := s.ClaimApply(context.Background(), claimInput(stage.Stage, "", RecoveryModeOrdinary)) + if claimErr != nil { + t.Fatal(claimErr) + } + if dispatchErr := s.BeginDispatch(context.Background(), attempt.ID, 1); dispatchErr != nil { + t.Fatal(dispatchErr) + } + if validateErr := s.MarkStepValidated(context.Background(), attempt.ID, 1, json.RawMessage(tc.result)); !errors.Is(validateErr, ErrInvalid) { + t.Fatalf("mismatched result=%v", validateErr) + } + }) + } +} + +func TestHistoricalUnknownReceiptReplaysAfterForcedSuccess(t *testing.T) { + s := openDomainStore(t) + created := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) + firstInput := claimInput(created.Stage, "unknown-attempt", RecoveryModeOrdinary) + first, err := s.ClaimApply(context.Background(), firstInput) + if err != nil { + t.Fatal(err) + } + if err = s.BeginDispatch(context.Background(), first.ID, 1); err != nil { + t.Fatal(err) + } + if err = s.MarkStepUnknown(context.Background(), first.ID, 1); err != nil { + t.Fatal(err) + } + unknown, err := s.FinalizeApply(context.Background(), first.ID) + if err != nil || unknown.Outcome != OutcomeUnknown { + t.Fatalf("unknown=%+v err=%v", unknown, err) + } + if _, err = s.db.Exec(`UPDATE stages SET recovery='none' WHERE id=?`, created.Stage.ID); err == nil { + t.Fatal("unknown recovery downgrade succeeded") + } + if _, err = s.db.Exec(`UPDATE stages SET lifecycle='expired',recovery='forbidden' WHERE id=?`, created.Stage.ID); err == nil { + t.Fatal("unknown recovery expiry bypass succeeded") + } + detail, err := s.Show(context.Background(), created.Stage.ID) + if err != nil { + t.Fatal(err) + } + forced, err := s.ClaimApply(context.Background(), claimInput(detail.StageSummary, "", RecoveryModeUnknown)) + if err != nil { + t.Fatal(err) + } + if err = s.BeginDispatch(context.Background(), forced.ID, 1); err != nil { + t.Fatal(err) + } + if err = s.MarkStepValidated(context.Background(), forced.ID, 1, createPostResult(t, forced, "post-forced")); err != nil { + t.Fatal(err) + } + if _, err = s.FinalizeApply(context.Background(), forced.ID); err != nil { + t.Fatal(err) + } + if _, err = s.db.Exec(`UPDATE stages SET lifecycle='open',recovery='none' WHERE id=?`, created.Stage.ID); err == nil { + t.Fatal("completed stage reopen succeeded") + } + storedAttempt, scanErr := scanApplyAttempt(context.Background(), s.db, first.ID) + storedReceipt, receiptErr := loadApplyReceipt(context.Background(), s.db, first.ID) + if scanErr != nil || receiptErr != nil || !validReceiptForAttempt(storedReceipt, storedAttempt) { + t.Fatalf("stored historical binding attempt=%+v receipt=%+v scan=%v load=%v valid=%v", storedAttempt, storedReceipt, scanErr, receiptErr, validReceiptForAttempt(storedReceipt, storedAttempt)) + } + replayed, err := s.FinalizeApply(context.Background(), first.ID) + if err != nil || !replayed.Replay || replayed.Outcome != OutcomeUnknown || replayed.Recovery != RecoveryUnknown { + t.Fatalf("historical replay=%+v err=%v", replayed, err) + } +} + +func TestApplyPartialAndUnknownRecoveryAreMonotonic(t *testing.T) { + t.Run("validated residue then rejection", func(t *testing.T) { + s := openDomainStore(t) + created := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"upload_attachment","condition":"always"},{"ordinal":2,"type":"create_post","condition":"always"}]}`) + claim, _ := s.ClaimApply(context.Background(), claimInput(created.Stage, "", RecoveryModeOrdinary)) + _ = s.BeginDispatch(context.Background(), claim.ID, 1) + _ = s.MarkStepValidated(context.Background(), claim.ID, 1, json.RawMessage(`{"fileId":"file-1"}`)) + _ = s.BeginDispatch(context.Background(), claim.ID, 2) + _ = s.MarkStepRejected(context.Background(), claim.ID, 2, json.RawMessage(`{"status":400}`)) + receipt, err := s.FinalizeApply(context.Background(), claim.ID) + if err != nil || receipt.Outcome != OutcomePartial || receipt.Recovery != RecoveryPartial { + t.Fatalf("receipt=%+v err=%v", receipt, err) + } + }) + + t.Run("later rejection cannot erase uncertainty", func(t *testing.T) { + s := openDomainStore(t) + created := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) + first, _ := s.ClaimApply(context.Background(), claimInput(created.Stage, "", RecoveryModeOrdinary)) + _ = s.BeginDispatch(context.Background(), first.ID, 1) + _ = s.MarkStepUnknown(context.Background(), first.ID, 1) + unknown, err := s.FinalizeApply(context.Background(), first.ID) + if err != nil || unknown.Outcome != OutcomeUnknown || unknown.Recovery != RecoveryUnknown { + t.Fatalf("unknown=%+v err=%v", unknown, err) + } + detail, _ := s.Show(context.Background(), created.Stage.ID) + forced, err := s.ClaimApply(context.Background(), claimInput(detail.StageSummary, "", RecoveryModeUnknown)) + if err != nil || !forced.ForcedDuplicateRisk { + t.Fatalf("forced=%+v err=%v", forced, err) + } + _ = s.BeginDispatch(context.Background(), forced.ID, 1) + _ = s.MarkStepRejected(context.Background(), forced.ID, 1, json.RawMessage(`{"status":403}`)) + rejected, err := s.FinalizeApply(context.Background(), forced.ID) + if err != nil || rejected.Outcome != OutcomeRejected || rejected.Recovery != RecoveryUnknown { + t.Fatalf("rejected=%+v err=%v", rejected, err) + } + }) +} + +func TestApplyPartialResumeIsRefusedUntilReuseProofIsBound(t *testing.T) { + s := openDomainStore(t) + created := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"upload_attachment","condition":"always"},{"ordinal":2,"type":"create_post","condition":"always"}]}`) + first, _ := s.ClaimApply(context.Background(), claimInput(created.Stage, "", RecoveryModeOrdinary)) + _ = s.BeginDispatch(context.Background(), first.ID, 1) + _ = s.MarkStepValidated(context.Background(), first.ID, 1, json.RawMessage(`{"fileId":"file-1"}`)) + _ = s.BeginDispatch(context.Background(), first.ID, 2) + _ = s.MarkStepRejected(context.Background(), first.ID, 2, json.RawMessage(`{"status":400}`)) + _, _ = s.FinalizeApply(context.Background(), first.ID) + detail, _ := s.Show(context.Background(), created.Stage.ID) + if _, err := s.ClaimApply(context.Background(), claimInput(detail.StageSummary, "", RecoveryModePartial)); !errors.Is(err, ErrNotEligible) { + t.Fatalf("unsafe partial resume=%v", err) + } +} + +func TestApplyCanBeReleasedOnlyBeforeDispatch(t *testing.T) { + s := openDomainStore(t) + created := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) + claim, _ := s.ClaimApply(context.Background(), claimInput(created.Stage, "release-1", RecoveryModeOrdinary)) + if err := s.AbandonApplyBeforeDispatch(context.Background(), claim.ID); err != nil { + t.Fatal(err) + } + detail, err := s.Show(context.Background(), created.Stage.ID) + if err != nil || detail.Lifecycle != LifecycleOpen || detail.Recovery != RecoveryNone { + t.Fatalf("detail=%+v err=%v", detail, err) + } + if _, found, err := s.FindApply(context.Background(), created.Stage.ServerURL, created.Stage.UserID, "release-1", claimInput(created.Stage, "release-1", RecoveryModeOrdinary).RequestDigest); err != nil || found { + t.Fatalf("released request found=%v err=%v", found, err) + } + second, _ := s.ClaimApply(context.Background(), claimInput(created.Stage, "", RecoveryModeOrdinary)) + _ = s.BeginDispatch(context.Background(), second.ID, 1) + if err := s.AbandonApplyBeforeDispatch(context.Background(), second.ID); !errors.Is(err, ErrNotEligible) { + t.Fatalf("post-dispatch release=%v", err) + } +} + +func TestApplyAuditHistoryCannotBeDeletedAfterDispatch(t *testing.T) { + s := openDomainStore(t) + created := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) + claim, err := s.ClaimApply(context.Background(), claimInput(created.Stage, "audit-1", RecoveryModeOrdinary)) + if err != nil { + t.Fatal(err) + } + if err = s.BeginDispatch(context.Background(), claim.ID, 1); err != nil { + t.Fatal(err) + } + var recursive int + if err = s.db.QueryRow(`PRAGMA recursive_triggers`).Scan(&recursive); err != nil || recursive != 1 { + t.Fatalf("recursive_triggers=%d err=%v", recursive, err) + } + for _, statement := range []string{ + `DELETE FROM apply_steps WHERE attempt_id=?`, + `DELETE FROM apply_events WHERE attempt_id=?`, + `DELETE FROM apply_requests WHERE attempt_id=?`, + `DELETE FROM apply_attempts WHERE id=?`, + } { + if _, err = s.db.Exec(statement, claim.ID); err == nil { + t.Fatalf("audit deletion succeeded: %s", statement) + } + } + if _, err = s.db.Exec(`UPDATE apply_steps SET state='pending',started_at=NULL WHERE attempt_id=? AND ordinal=1`, claim.ID); err == nil { + t.Fatal("dispatched step state regression succeeded") + } + if _, err = s.db.Exec(`UPDATE apply_steps SET started_at='2026-01-01T00:00:00.000000Z' WHERE attempt_id=? AND ordinal=1`, claim.ID); err == nil { + t.Fatal("dispatched step timestamp mutation succeeded") + } + if _, err = s.db.Exec(`INSERT OR REPLACE INTO apply_steps(attempt_id,ordinal,kind,condition,state,result_json,started_at,ended_at) + SELECT attempt_id,ordinal,kind,condition,state,result_json,started_at,ended_at FROM apply_steps WHERE attempt_id=? AND ordinal=1`, claim.ID); err == nil { + t.Fatal("replace bypass of dispatched step succeeded") + } + if _, err = s.db.Exec(`UPDATE stages SET lifecycle='open',claim_attempt_id=NULL WHERE id=?`, created.Stage.ID); err == nil { + t.Fatal("dispatched stage claim release succeeded") + } +} + +func TestApplyReplayBindsStageRevisionDigestAndMode(t *testing.T) { + s := openDomainStore(t) + first := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) + second := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) + in := claimInput(first.Stage, "bound-request", RecoveryModeOrdinary) + if _, err := s.ClaimApply(context.Background(), in); err != nil { + t.Fatal(err) + } + wrong := in + wrong.StageID, wrong.Revision, wrong.ExpectedDigest = second.Stage.ID, second.Stage.Revision, second.Stage.SemanticDigest + if _, err := s.ClaimApply(context.Background(), wrong); !errors.Is(err, ErrConflict) { + t.Fatalf("cross-stage replay=%v", err) + } +} + +func TestConcurrentApplyHasOneClaimWinner(t *testing.T) { + s := openDomainStore(t) + created := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) + start := make(chan struct{}) + results := make(chan error, 2) + var wg sync.WaitGroup + for range 2 { + wg.Add(1) + go func() { + defer wg.Done() + <-start + _, err := s.ClaimApply(context.Background(), claimInput(created.Stage, "", RecoveryModeOrdinary)) + results <- err + }() + } + close(start) + wg.Wait() + close(results) + success, ineligible := 0, 0 + for err := range results { + if err == nil { + success++ + } else if errors.Is(err, ErrNotEligible) || errors.Is(err, ErrConflict) { + ineligible++ + } else { + t.Fatal(err) + } + } + if success != 1 || ineligible != 1 { + t.Fatalf("success/ineligible=%d/%d", success, ineligible) + } +} + +func TestApplyJournalForbidsOutOfOrderDispatch(t *testing.T) { + s := openDomainStore(t) + created := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"upload_attachment","condition":"always"},{"ordinal":2,"type":"create_post","condition":"always"}]}`) + claim, err := s.ClaimApply(context.Background(), claimInput(created.Stage, "", RecoveryModeOrdinary)) + if err != nil { + t.Fatal(err) + } + if err = s.BeginDispatch(context.Background(), claim.ID, 2); !errors.Is(err, ErrNotEligible) { + t.Fatalf("out-of-order dispatch=%v", err) + } + if err = s.BeginDispatch(context.Background(), claim.ID, 1); err != nil { + t.Fatal(err) + } + if err = s.BeginDispatch(context.Background(), claim.ID, 2); !errors.Is(err, ErrNotEligible) { + t.Fatalf("parallel dispatch=%v", err) + } +} + +func TestWritableOpenRecoversInterruptedApplyFromJournalEvidence(t *testing.T) { + for _, tc := range []struct { + name string + prepare func(*testing.T, *Store, ApplyAttempt) + wantRecovery Recovery + wantOutcome *AttemptOutcome + wantFound bool + wantBody bool + }{ + {name: "no dispatch safely releases", wantRecovery: RecoveryNone, wantFound: false, wantBody: true}, + {name: "unmatched dispatch becomes unknown", prepare: func(t *testing.T, s *Store, a ApplyAttempt) { + t.Helper() + if err := s.BeginDispatch(context.Background(), a.ID, 1); err != nil { + t.Fatal(err) + } + }, wantRecovery: RecoveryUnknown, wantOutcome: outcomePointer(OutcomeUnknown), wantFound: true, wantBody: true}, + {name: "journaled unknown seals pending suffix", prepare: func(t *testing.T, s *Store, a ApplyAttempt) { + t.Helper() + if err := s.BeginDispatch(context.Background(), a.ID, 1); err != nil { + t.Fatal(err) + } + if err := s.MarkStepUnknown(context.Background(), a.ID, 1); err != nil { + t.Fatal(err) + } + }, wantRecovery: RecoveryUnknown, wantOutcome: outcomePointer(OutcomeUnknown), wantFound: true, wantBody: true}, + {name: "validated prefix becomes partial", prepare: func(t *testing.T, s *Store, a ApplyAttempt) { + t.Helper() + if err := s.BeginDispatch(context.Background(), a.ID, 1); err != nil { + t.Fatal(err) + } + if err := s.MarkStepValidated(context.Background(), a.ID, 1, json.RawMessage(`{"fileId":"file-1"}`)); err != nil { + t.Fatal(err) + } + }, wantRecovery: RecoveryPartial, wantOutcome: outcomePointer(OutcomePartial), wantFound: true, wantBody: true}, + {name: "fully validated attempt completes", prepare: func(t *testing.T, s *Store, a ApplyAttempt) { + t.Helper() + for _, step := range a.Steps { + if err := s.BeginDispatch(context.Background(), a.ID, step.Ordinal); err != nil { + t.Fatal(err) + } + result := createPostResult(t, a, "post-1") + if step.Kind == "upload_attachment" { + result = json.RawMessage(`{"fileId":"file-1"}`) + } + if err := s.MarkStepValidated(context.Background(), a.ID, step.Ordinal, result); err != nil { + t.Fatal(err) + } + } + }, wantRecovery: RecoveryForbidden, wantOutcome: outcomePointer(OutcomeSucceeded), wantFound: true, wantBody: false}, + } { + t.Run(tc.name, func(t *testing.T) { + path := testPath(t) + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + plan := `{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}` + if tc.name == "validated prefix becomes partial" || tc.name == "journaled unknown seals pending suffix" { + plan = `{"steps":[{"ordinal":1,"type":"upload_attachment","condition":"always"},{"ordinal":2,"type":"create_post","condition":"always"}]}` + } + created := createApplyStage(t, s, plan) + input := claimInput(created.Stage, "recover-request", RecoveryModeOrdinary) + claim, err := s.ClaimApply(context.Background(), input) + if err != nil { + t.Fatal(err) + } + if tc.prepare != nil { + tc.prepare(t, s, claim) + } + if err = s.Close(); err != nil { + t.Fatal(err) + } + s, err = Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + detail, err := s.Show(context.Background(), created.Stage.ID) + if err != nil || detail.Recovery != tc.wantRecovery || (detail.Body != nil) != tc.wantBody || detail.Lifecycle == LifecycleApplying { + t.Fatalf("detail=%+v err=%v", detail, err) + } + recovered, found, err := s.FindApply(context.Background(), created.Stage.ServerURL, created.Stage.UserID, "recover-request", input.RequestDigest) + if err != nil || found != tc.wantFound { + t.Fatalf("found=%v attempt=%+v err=%v", found, recovered, err) + } + if tc.wantOutcome != nil && (recovered.Outcome == nil || *recovered.Outcome != *tc.wantOutcome) { + t.Fatalf("outcome=%v want=%v", recovered.Outcome, *tc.wantOutcome) + } + if tc.wantOutcome != nil && *tc.wantOutcome == OutcomePartial && recovered.Steps[1].State != StepNotSent { + t.Fatalf("partial suffix=%+v", recovered.Steps) + } + }) + } +} + +func outcomePointer(value AttemptOutcome) *AttemptOutcome { return &value } diff --git a/internal/stagestore/domain.go b/internal/stagestore/domain.go index 0a77034..fa25630 100644 --- a/internal/stagestore/domain.go +++ b/internal/stagestore/domain.go @@ -829,11 +829,13 @@ func loadReviseReplay(ctx context.Context, q queryer, server, user, id string, d return result, found, err } var operation Operation + var lifecycle Lifecycle + var recovery Recovery var storedServer, serverID, storedUser, stageCreated, revisionCreated, destination, plan string var semantic, body []byte - err = q.QueryRowContext(ctx, `SELECT s.operation,s.server_url,coalesce(s.server_id,''),s.user_id,s.created_at,r.created_at,r.semantic_digest,r.body,r.destination_json,r.plan_json + err = q.QueryRowContext(ctx, `SELECT s.operation,s.lifecycle,s.recovery,s.server_url,coalesce(s.server_id,''),s.user_id,s.created_at,r.created_at,r.semantic_digest,r.body,r.destination_json,r.plan_json FROM stages s JOIN stage_revisions r ON r.stage_id=s.id WHERE s.id=? AND r.revision=?`, result.Stage.ID, result.Stage.Revision). - Scan(&operation, &storedServer, &serverID, &storedUser, &stageCreated, &revisionCreated, &semantic, &body, &destination, &plan) + Scan(&operation, &lifecycle, &recovery, &storedServer, &serverID, &storedUser, &stageCreated, &revisionCreated, &semantic, &body, &destination, &plan) if err != nil { return MutationResult{}, false, localError(err) } @@ -843,11 +845,11 @@ func loadReviseReplay(ctx context.Context, q queryer, server, user, id string, d } stageCreatedAt, stageTimeErr := parseTime(stageCreated) revisionCreatedAt, revisionTimeErr := parseTime(revisionCreated) - content, contentErr := normalizeContent(operation, RevisionContent{body, json.RawMessage(destination), json.RawMessage(plan), attachments}) + content, retained, contentErr := replayContentProjection(operation, lifecycle, recovery, body, json.RawMessage(destination), json.RawMessage(plan), attachments) if stageTimeErr != nil || revisionTimeErr != nil || contentErr != nil || len(semantic) != 32 || operation != result.Stage.Operation || storedServer != server || serverID != result.Stage.ServerID || storedUser != user || !stageCreatedAt.Equal(result.Stage.CreatedAt) || !revisionCreatedAt.Equal(result.Stage.UpdatedAt) || !revisionCreatedAt.Equal(result.RecordedAt) || - !bytes.Equal(semantic, result.Stage.SemanticDigest[:]) || semanticDigest(operation, storedServer, serverID, storedUser, content) != result.Stage.SemanticDigest { + !bytes.Equal(semantic, result.Stage.SemanticDigest[:]) || !retained && semanticDigest(operation, storedServer, serverID, storedUser, content) != result.Stage.SemanticDigest { return MutationResult{}, false, localError(errors.New("revise receipt projection")) } return result, true, nil @@ -894,11 +896,13 @@ func findCreate(ctx context.Context, q queryer, server, user, id string) (Create return CreateRecord{}, false, localError(errors.New("create receipt")) } var operation Operation + var lifecycle Lifecycle + var recovery Recovery var stageServer, serverID, stageUser, stageCreated, revisionCreated string var body []byte var destination, plan string var semantic []byte - err = q.QueryRowContext(ctx, `SELECT s.operation,s.server_url,coalesce(s.server_id,''),s.user_id,s.created_at,r.created_at,r.body,r.destination_json,r.plan_json,r.semantic_digest FROM stages s JOIN stage_revisions r ON r.stage_id=s.id AND r.revision=? WHERE s.id=?`, stage.Revision, stage.ID).Scan(&operation, &stageServer, &serverID, &stageUser, &stageCreated, &revisionCreated, &body, &destination, &plan, &semantic) + err = q.QueryRowContext(ctx, `SELECT s.operation,s.lifecycle,s.recovery,s.server_url,coalesce(s.server_id,''),s.user_id,s.created_at,r.created_at,r.body,r.destination_json,r.plan_json,r.semantic_digest FROM stages s JOIN stage_revisions r ON r.stage_id=s.id AND r.revision=? WHERE s.id=?`, stage.Revision, stage.ID).Scan(&operation, &lifecycle, &recovery, &stageServer, &serverID, &stageUser, &stageCreated, &revisionCreated, &body, &destination, &plan, &semantic) if err != nil { return CreateRecord{}, false, localError(err) } @@ -906,12 +910,12 @@ func findCreate(ctx context.Context, q queryer, server, user, id string) (Create if err != nil { return CreateRecord{}, false, err } - content, err := normalizeContent(operation, RevisionContent{body, json.RawMessage(destination), json.RawMessage(plan), attachments}) + content, retained, err := replayContentProjection(operation, lifecycle, recovery, body, json.RawMessage(destination), json.RawMessage(plan), attachments) recordDestination, destinationErr := canonicalObject(record.Destination) recordPlan, planErr := canonicalObject(record.Plan) stageCreatedAt, stageCreatedErr := parseTime(stageCreated) revisionCreatedAt, revisionCreatedErr := parseTime(revisionCreated) - if err != nil || destinationErr != nil || planErr != nil || stageCreatedErr != nil || revisionCreatedErr != nil || !stage.CreatedAt.Equal(stageCreatedAt) || !stage.CreatedAt.Equal(revisionCreatedAt) || len(semantic) != 32 || !bytes.Equal(semantic, stage.SemanticDigest[:]) || semanticDigest(operation, server, serverID, user, content) != stage.SemanticDigest || + if err != nil || destinationErr != nil || planErr != nil || stageCreatedErr != nil || revisionCreatedErr != nil || !stage.CreatedAt.Equal(stageCreatedAt) || !stage.CreatedAt.Equal(revisionCreatedAt) || len(semantic) != 32 || !bytes.Equal(semantic, stage.SemanticDigest[:]) || !retained && semanticDigest(operation, server, serverID, user, content) != stage.SemanticDigest || !bytes.Equal(content.Destination, recordDestination) || !bytes.Equal(content.Plan, recordPlan) || operation != stage.Operation || stageServer != server || stageUser != user || serverID != stage.ServerID { return CreateRecord{}, false, localError(errors.New("create projection")) } @@ -919,6 +923,37 @@ func findCreate(ctx context.Context, q queryer, server, user, id string) (Create return record, true, nil } +// replayContentProjection validates live content normally. After a confirmed +// apply, plaintext and attachment paths are intentionally erased, so the +// immutable historical digest becomes the only possible content binding. +func replayContentProjection(operation Operation, lifecycle Lifecycle, recovery Recovery, body []byte, destination, plan json.RawMessage, attachments []Attachment) (RevisionContent, bool, error) { + if lifecycle != LifecycleCompleted || recovery != RecoveryForbidden { + content, err := normalizeContent(operation, RevisionContent{body, destination, plan, attachments}) + return content, false, err + } + if body != nil || len(attachments) != 0 { + return RevisionContent{}, false, ErrInvalid + } + canonicalDestination, err := canonicalObject(destination) + if err != nil { + return RevisionContent{}, false, err + } + canonicalPlan, steps, err := decodePersistedPlan(plan) + if err != nil { + return RevisionContent{}, false, err + } + uploads := 0 + for _, step := range steps { + if step.Kind == "upload_attachment" { + uploads++ + } + } + if !validPlanForOperation(operation, steps, uploads) { + return RevisionContent{}, false, ErrInvalid + } + return RevisionContent{Destination: canonicalDestination, Plan: canonicalPlan}, true, nil +} + func normalizeCreateProjection(destination, plan json.RawMessage) (json.RawMessage, json.RawMessage, error) { projection := struct { Destination json.RawMessage `json:"destination"` @@ -1022,11 +1057,15 @@ func restoreLineSeparators(data []byte) []byte { return out.Bytes() } func newStageID() (string, error) { + return newIdentity("stg_") +} + +func newIdentity(prefix string) (string, error) { v := make([]byte, 24) if _, err := rand.Read(v); err != nil { return "", err } - return "stg_" + base64.RawURLEncoding.EncodeToString(v), nil + return prefix + base64.RawURLEncoding.EncodeToString(v), nil } func validOperation(v Operation) bool { switch v { diff --git a/internal/stagestore/domain_test.go b/internal/stagestore/domain_test.go index 9fe32c1..41195ee 100644 --- a/internal/stagestore/domain_test.go +++ b/internal/stagestore/domain_test.go @@ -29,7 +29,7 @@ func attachment(name string) Attachment { return Attachment{"/tmp/" + name, "/private/tmp/" + name, name, 3, "text/plain", sha256.Sum256([]byte(name))} } func createInput(request, body string) CreateInput { - return CreateInput{request, sha256.Sum256([]byte(body)), CreatePost, "https://mattermost.example/api/v4", "server-1", "user-1", RevisionContent{[]byte(body), json.RawMessage(`{"kind":"O","channelId":"channel-1"}`), json.RawMessage(`{"steps":[{"kind":"create_post"}]}`), []Attachment{attachment("a.txt"), attachment("b.txt")}}} + return CreateInput{request, sha256.Sum256([]byte(body)), CreatePost, "https://mattermost.example/api/v4", "server-1", "user-1", RevisionContent{[]byte(body), json.RawMessage(`{"kind":"O","channelId":"channel-1"}`), json.RawMessage(`{"steps":[{"ordinal":1,"type":"upload_attachment","condition":"always"},{"ordinal":2,"type":"upload_attachment","condition":"always"},{"ordinal":3,"type":"create_post","condition":"always"}]}`), []Attachment{attachment("a.txt"), attachment("b.txt")}}} } func TestFindCreateUsesExactReceiptRevisionAndFailsClosedOnCorruption(t *testing.T) { @@ -148,14 +148,14 @@ func TestCreateAndReplayReturnCanonicalAuthoritativeProjection(t *testing.T) { s := openDomainStore(t) in := createInput("canonical-projection", "body") in.Content.Destination = json.RawMessage(`{ "kind": "O", "channelId": "channel-1" }`) - in.Content.Plan = json.RawMessage(`{ "steps": [ { "kind": "create_post" } ] }`) + in.Content.Plan = json.RawMessage(`{ "steps": [ { "ordinal": 1, "type": "create_post", "condition": "always" } ] }`) created, err := s.Create(context.Background(), in) if err != nil { t.Fatal(err) } wantDestination := []byte(`{"kind":"O","channelId":"channel-1"}`) - wantPlan := []byte(`{"steps":[{"kind":"create_post"}]}`) + wantPlan := []byte(`{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) if !bytes.Equal(created.Destination, wantDestination) || !bytes.Equal(created.Plan, wantPlan) { t.Fatalf("created projection = %s / %s", created.Destination, created.Plan) } @@ -273,7 +273,7 @@ func TestRevisePreservesImmutableDestinationAndAcceptsDerivedPlan(t *testing.T) if err != nil { t.Fatal(err) } - if string(after.Destination) != string(before.Destination) || string(after.Plan) != `{"steps":[{"kind":"create_post"}]}` || string(after.Plan) == string(before.Plan) { + if string(after.Destination) != string(before.Destination) || string(after.Plan) != `{"steps":[{"condition":"always","ordinal":1,"type":"upload_attachment"},{"condition":"always","ordinal":2,"type":"upload_attachment"},{"condition":"always","ordinal":3,"type":"create_post"}]}` || string(after.Plan) == string(before.Plan) { t.Fatalf("unexpected revision binding: destination %s -> %s, plan %s -> %s", before.Destination, after.Destination, before.Plan, after.Plan) } } @@ -433,6 +433,12 @@ func TestListRecordsFailsClosedWhenCurrentRevisionIsMissing(t *testing.T) { if _, err = s.db.Exec(`PRAGMA foreign_keys=OFF`); err != nil { t.Fatal(err) } + if _, err = s.db.Exec(`UPDATE stages SET current_revision=999 WHERE id=?`, created.Stage.ID); err == nil { + t.Fatal("invalid current revision transition succeeded") + } + if _, err = s.db.Exec(`DROP TRIGGER stage_current_revision_transition_valid`); err != nil { + t.Fatal(err) + } if _, err = s.db.Exec(`UPDATE stages SET current_revision=999 WHERE id=?`, created.Stage.ID); err != nil { t.Fatal(err) } @@ -447,6 +453,12 @@ func TestShowFailsClosedWhenRetainedContentBreaksSemanticDigest(t *testing.T) { if err != nil { t.Fatal(err) } + if _, err = s.db.Exec(`UPDATE stage_revisions SET body=? WHERE stage_id=? AND revision=1`, []byte("modified"), created.Stage.ID); err == nil { + t.Fatal("retained body mutation succeeded") + } + if _, err = s.db.Exec(`DROP TRIGGER stage_revision_body_erasure_only`); err != nil { + t.Fatal(err) + } if _, err = s.db.Exec(`UPDATE stage_revisions SET body=? WHERE stage_id=? AND revision=1`, []byte("modified"), created.Stage.ID); err != nil { t.Fatal(err) } @@ -502,7 +514,8 @@ func TestReviewedStateCASAndApplying(t *testing.T) { if _, err = s.Cancel(context.Background(), staleCancel); !errors.Is(err, ErrConflict) { t.Fatalf("stale cancel=%v", err) } - if _, err = s.db.Exec(`UPDATE stages SET lifecycle='applying' WHERE id=?`, revised.Stage.ID); err != nil { + claim, err := s.ClaimApply(context.Background(), ApplyClaimInput{StageID: revised.Stage.ID, Revision: revised.Stage.Revision, ExpectedDigest: revised.Stage.SemanticDigest, RecoveryMode: RecoveryModeOrdinary}) + if err != nil || claim.StageID != revised.Stage.ID { t.Fatal(err) } if _, err = s.Revise(context.Background(), reviseInput(revised.Stage, "applying", "four")); !errors.Is(err, ErrNotEligible) { @@ -597,6 +610,12 @@ func TestReviveOnlyLegalExpiredForbidden(t *testing.T) { t.Fatalf("revived=%#v err=%v", revived, err) } other, _ := s.Create(context.Background(), createInput("", "x")) + if _, err := s.db.Exec(`UPDATE stages SET lifecycle='expired',recovery='force_unknown' WHERE id=?`, other.Stage.ID); err == nil { + t.Fatal("invalid expired recovery transition succeeded") + } + if _, err := s.db.Exec(`DROP TRIGGER stage_lifecycle_recovery_transition_valid`); err != nil { + t.Fatal(err) + } if _, err := s.db.Exec(`UPDATE stages SET lifecycle='expired',recovery='force_unknown' WHERE id=?`, other.Stage.ID); err != nil { t.Fatal(err) } diff --git a/internal/stagestore/recovery.go b/internal/stagestore/recovery.go new file mode 100644 index 0000000..6257e80 --- /dev/null +++ b/internal/stagestore/recovery.go @@ -0,0 +1,131 @@ +package stagestore + +import ( + "context" + "time" +) + +type RecoveryReport struct { + Released, Finalized, ForcedUnknown, Partial int +} + +// recoverInterruptedApplies classifies every claim left by a process that no +// longer owns the store lock. It never infers non-dispatch from elapsed time. +func (s *Store) recoverInterruptedApplies(ctx context.Context) (RecoveryReport, error) { + if ctx == nil { + return RecoveryReport{}, ErrInvalid + } + rows, err := s.db.QueryContext(ctx, `SELECT a.id FROM apply_attempts a JOIN stages s ON s.claim_attempt_id=a.id WHERE a.outcome IS NULL AND s.lifecycle='applying' ORDER BY a.started_at,a.id`) + if err != nil { + return RecoveryReport{}, localError(err) + } + var ids []string + for rows.Next() { + var id string + if err = rows.Scan(&id); err != nil { + rows.Close() + return RecoveryReport{}, localError(err) + } + ids = append(ids, id) + } + if err = rows.Close(); err != nil { + return RecoveryReport{}, localError(err) + } + var report RecoveryReport + for _, id := range ids { + attempt, scanErr := scanApplyAttempt(ctx, s.db, id) + if scanErr != nil { + return report, scanErr + } + dispatched, stopped, effects, terminal := false, false, false, true + for _, step := range attempt.Steps { + switch step.State { + case StepDispatch: + dispatched = true + terminal = false + case StepPending: + terminal = false + case StepValidated, StepSkipped: + effects = true + case StepRejected, StepUnknown: + stopped = true + } + } + if dispatched { + for _, step := range attempt.Steps { + if step.State == StepDispatch { + if err = s.MarkStepUnknown(ctx, id, step.Ordinal); err != nil { + return report, err + } + } + } + if _, err = s.FinalizeApply(ctx, id); err != nil { + return report, err + } + report.Finalized++ + report.ForcedUnknown++ + continue + } + if stopped { + receipt, finalizeErr := s.FinalizeApply(ctx, id) + if finalizeErr != nil { + return report, finalizeErr + } + report.Finalized++ + if receipt.Outcome == OutcomeUnknown { + report.ForcedUnknown++ + } else if receipt.Outcome == OutcomePartial { + report.Partial++ + } + continue + } + if !effects && !terminal { + if err = s.AbandonApplyBeforeDispatch(ctx, id); err != nil { + return report, err + } + report.Released++ + continue + } + if effects && !terminal { + if err = s.sealInterruptedPending(ctx, id); err != nil { + return report, err + } + report.Partial++ + } + if _, err = s.FinalizeApply(ctx, id); err != nil { + return report, err + } + report.Finalized++ + } + return report, nil +} + +func (s *Store) sealInterruptedPending(ctx context.Context, attemptID string) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return localError(err) + } + defer tx.Rollback() + attempt, err := scanApplyAttempt(ctx, tx, attemptID) + if err != nil { + return err + } + for _, step := range attempt.Steps { + if step.State == StepDispatch { + return ErrNotEligible + } + } + stamp := formatTime(time.Now().UTC()) + if err = sealPendingSteps(ctx, tx, &attempt, stamp, "recovered_partial"); err != nil { + return err + } + if err = tx.Commit(); err != nil { + return localError(err) + } + runCommitHook() + return nil +} + +func applyJournalAvailable() bool { + return len(migrations) >= 6 && migrations[5].version == 6 && migrations[5].name == "durable-apply-journal" +} diff --git a/internal/stagestore/schema.go b/internal/stagestore/schema.go index da3199e..cadefd3 100644 --- a/internal/stagestore/schema.go +++ b/internal/stagestore/schema.go @@ -103,4 +103,216 @@ CREATE TRIGGER stage_revision_binding_immutable BEFORE INSERT ON stage_revisions WHEN NEW.revision > 1 AND EXISTS(SELECT 1 FROM stage_revisions WHERE stage_id=NEW.stage_id) AND NEW.destination_json != (SELECT destination_json FROM stage_revisions WHERE stage_id=NEW.stage_id ORDER BY revision LIMIT 1) BEGIN SELECT RAISE(ABORT, 'stage destination is immutable'); END; +`}, {version: 6, name: "durable-apply-journal", sql: ` +CREATE TABLE apply_attempts ( + id TEXT PRIMARY KEY NOT NULL, + stage_id TEXT NOT NULL, + revision INTEGER NOT NULL CHECK (revision > 0), + semantic_digest BLOB NOT NULL CHECK (length(semantic_digest) = 32), + recovery_mode TEXT NOT NULL CHECK (recovery_mode IN ('ordinary','resume_partial','force_unknown')), + prior_recovery TEXT NOT NULL CHECK (prior_recovery IN ('none','resume_partial','force_unknown')), + forced_duplicate_risk INTEGER NOT NULL CHECK (forced_duplicate_risk IN (0,1)), + plan_json TEXT NOT NULL CHECK (json_valid(plan_json)), + pending_post_id TEXT NOT NULL, + started_at TEXT NOT NULL, + ended_at TEXT, + outcome TEXT CHECK (outcome IN ('succeeded','already_satisfied','rejected','partial','unknown')), + CHECK ((ended_at IS NULL) = (outcome IS NULL)), + FOREIGN KEY (stage_id, revision) REFERENCES stage_revisions(stage_id, revision) +) STRICT; +ALTER TABLE stages ADD COLUMN claim_attempt_id TEXT REFERENCES apply_attempts(id); +UPDATE stages SET lifecycle='open',recovery='force_unknown' WHERE lifecycle='applying'; +CREATE UNIQUE INDEX one_apply_claim_per_stage ON apply_attempts(stage_id) WHERE outcome IS NULL; +CREATE TABLE apply_steps ( + attempt_id TEXT NOT NULL REFERENCES apply_attempts(id) ON DELETE CASCADE, + ordinal INTEGER NOT NULL CHECK (ordinal > 0), + kind TEXT NOT NULL, + condition TEXT NOT NULL CHECK (condition IN ('always','if_missing')), + state TEXT NOT NULL CHECK (state IN ('pending','dispatch_intent','response_validated','rejected','outcome_unknown','skipped','not_dispatched')), + result_json TEXT CHECK (result_json IS NULL OR json_valid(result_json)), + started_at TEXT, + ended_at TEXT, + PRIMARY KEY (attempt_id, ordinal), + CHECK ( + state='pending' AND result_json IS NULL AND started_at IS NULL AND ended_at IS NULL + OR state='dispatch_intent' AND result_json IS NULL AND started_at IS NOT NULL AND ended_at IS NULL + OR state IN ('response_validated','rejected') AND result_json IS NOT NULL AND started_at IS NOT NULL AND ended_at IS NOT NULL + OR state='outcome_unknown' AND result_json IS NULL AND started_at IS NOT NULL AND ended_at IS NOT NULL + OR state='skipped' AND condition='if_missing' AND result_json IS NOT NULL AND started_at IS NULL AND ended_at IS NOT NULL + OR state='not_dispatched' AND result_json IS NULL AND started_at IS NULL AND ended_at IS NOT NULL + ) +) STRICT; +CREATE TABLE apply_events ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + attempt_id TEXT NOT NULL REFERENCES apply_attempts(id) ON DELETE CASCADE, + ordinal INTEGER, + event TEXT NOT NULL CHECK (event IN ('claimed','dispatch_intent','response_validated','rejected','outcome_unknown','skipped','not_dispatched','completed','released_before_dispatch','recovered_unknown','recovered_partial')), + recorded_at TEXT NOT NULL, + CHECK (ordinal IS NULL OR ordinal > 0) +) STRICT; +CREATE TABLE apply_requests ( + server_url TEXT NOT NULL, + user_id TEXT NOT NULL, + request_id TEXT NOT NULL, + request_digest BLOB NOT NULL CHECK (length(request_digest) = 32), + attempt_id TEXT NOT NULL REFERENCES apply_attempts(id) ON DELETE CASCADE, + created_at TEXT NOT NULL, + PRIMARY KEY (server_url, user_id, request_id), + UNIQUE (attempt_id) +) STRICT; +CREATE TABLE apply_receipts ( + attempt_id TEXT PRIMARY KEY NOT NULL REFERENCES apply_attempts(id) ON DELETE CASCADE, + receipt_json TEXT NOT NULL CHECK (json_valid(receipt_json)), + recorded_at TEXT NOT NULL +) STRICT; +CREATE TRIGGER stage_apply_claim_valid BEFORE UPDATE OF lifecycle,claim_attempt_id ON stages +WHEN (NEW.lifecycle='applying' AND (NEW.claim_attempt_id IS NULL OR NOT EXISTS( + SELECT 1 FROM apply_attempts a WHERE a.id=NEW.claim_attempt_id AND a.stage_id=NEW.id AND a.revision=NEW.current_revision AND a.outcome IS NULL +))) OR (NEW.lifecycle!='applying' AND NEW.claim_attempt_id IS NOT NULL) +BEGIN SELECT RAISE(ABORT, 'invalid stage apply claim'); END; +CREATE TRIGGER stage_apply_claim_release_valid BEFORE UPDATE OF lifecycle,claim_attempt_id ON stages +WHEN OLD.lifecycle='applying' AND NEW.lifecycle!='applying' AND EXISTS( + SELECT 1 FROM apply_attempts a WHERE a.id=OLD.claim_attempt_id AND a.outcome IS NULL + AND EXISTS(SELECT 1 FROM apply_steps p WHERE p.attempt_id=a.id AND p.state!='pending') +) +BEGIN SELECT RAISE(ABORT, 'dispatched apply claim cannot be released'); END; +CREATE TRIGGER stage_lifecycle_recovery_transition_valid BEFORE UPDATE OF lifecycle,recovery ON stages +WHEN (NEW.lifecycle IS NOT OLD.lifecycle OR NEW.recovery IS NOT OLD.recovery) AND NOT ( + OLD.lifecycle='open' AND NEW.lifecycle='applying' AND NEW.recovery=OLD.recovery AND NEW.claim_attempt_id IS NOT NULL + AND EXISTS(SELECT 1 FROM apply_attempts a WHERE a.id=NEW.claim_attempt_id AND a.stage_id=OLD.id AND a.outcome IS NULL) + OR OLD.lifecycle='applying' AND NEW.lifecycle='open' AND NEW.recovery=OLD.recovery AND NEW.claim_attempt_id IS NULL + AND EXISTS(SELECT 1 FROM apply_attempts a WHERE a.id=OLD.claim_attempt_id AND a.outcome IS NULL) + AND NOT EXISTS(SELECT 1 FROM apply_steps p WHERE p.attempt_id=OLD.claim_attempt_id AND p.state!='pending') + OR OLD.lifecycle='applying' AND NEW.claim_attempt_id IS NULL AND EXISTS( + SELECT 1 FROM apply_attempts a WHERE a.id=OLD.claim_attempt_id AND a.outcome IS NOT NULL AND ( + a.outcome IN ('succeeded','already_satisfied') AND NEW.lifecycle='completed' AND NEW.recovery='forbidden' + OR a.outcome='rejected' AND NEW.lifecycle='open' AND NEW.recovery=a.prior_recovery + OR a.outcome='partial' AND NEW.lifecycle='open' AND NEW.recovery=CASE WHEN a.prior_recovery='force_unknown' THEN 'force_unknown' ELSE 'resume_partial' END + OR a.outcome='unknown' AND NEW.lifecycle='open' AND NEW.recovery='force_unknown' + ) + ) + OR OLD.lifecycle='open' AND NEW.lifecycle='canceled' AND NEW.recovery='forbidden' AND OLD.claim_attempt_id IS NULL AND NEW.claim_attempt_id IS NULL + OR OLD.lifecycle='open' AND OLD.recovery='none' AND NEW.lifecycle='expired' AND NEW.recovery='forbidden' AND OLD.claim_attempt_id IS NULL AND NEW.claim_attempt_id IS NULL + OR OLD.lifecycle='expired' AND OLD.recovery='forbidden' AND NEW.lifecycle='open' AND NEW.recovery='none' + AND OLD.claim_attempt_id IS NULL AND NEW.claim_attempt_id IS NULL AND NEW.current_revision=OLD.current_revision+1 + AND EXISTS(SELECT 1 FROM stage_revisions r WHERE r.stage_id=OLD.id AND r.revision=OLD.current_revision AND r.state='superseded') + AND EXISTS(SELECT 1 FROM stage_revisions r WHERE r.stage_id=OLD.id AND r.revision=NEW.current_revision AND r.state='current') +) +BEGIN SELECT RAISE(ABORT, 'invalid stage lifecycle or recovery transition'); END; +CREATE TRIGGER apply_attempt_identity_immutable BEFORE UPDATE ON apply_attempts +WHEN NEW.id IS NOT OLD.id OR NEW.stage_id IS NOT OLD.stage_id OR NEW.revision IS NOT OLD.revision OR NEW.semantic_digest IS NOT OLD.semantic_digest + OR NEW.recovery_mode IS NOT OLD.recovery_mode OR NEW.prior_recovery IS NOT OLD.prior_recovery + OR NEW.forced_duplicate_risk IS NOT OLD.forced_duplicate_risk OR NEW.plan_json IS NOT OLD.plan_json + OR NEW.pending_post_id IS NOT OLD.pending_post_id OR NEW.started_at IS NOT OLD.started_at +BEGIN SELECT RAISE(ABORT, 'apply attempt identity is immutable'); END; +CREATE TRIGGER apply_attempt_outcome_immutable BEFORE UPDATE OF outcome,ended_at ON apply_attempts +WHEN OLD.outcome IS NOT NULL +BEGIN SELECT RAISE(ABORT, 'apply attempt outcome is immutable'); END; +CREATE TRIGGER apply_attempt_history_immutable_delete BEFORE DELETE ON apply_attempts +WHEN OLD.outcome IS NOT NULL OR EXISTS(SELECT 1 FROM apply_steps WHERE attempt_id=OLD.id AND state!='pending') +BEGIN SELECT RAISE(ABORT, 'dispatched apply history is immutable'); END; +CREATE TRIGGER apply_step_identity_immutable BEFORE UPDATE ON apply_steps +WHEN NEW.attempt_id IS NOT OLD.attempt_id OR NEW.ordinal IS NOT OLD.ordinal + OR NEW.kind IS NOT OLD.kind OR NEW.condition IS NOT OLD.condition +BEGIN SELECT RAISE(ABORT, 'apply step identity is immutable'); END; +CREATE TRIGGER apply_step_state_transition_valid BEFORE UPDATE OF state ON apply_steps +WHEN NOT ( + OLD.state='pending' AND NEW.state IN ('dispatch_intent','skipped','not_dispatched') + OR OLD.state='dispatch_intent' AND NEW.state IN ('response_validated','rejected','outcome_unknown') +) +BEGIN SELECT RAISE(ABORT, 'invalid apply step transition'); END; +CREATE TRIGGER apply_step_state_transition_required BEFORE UPDATE ON apply_steps +WHEN NEW.state IS OLD.state +BEGIN SELECT RAISE(ABORT, 'apply step history is immutable'); END; +CREATE TRIGGER apply_step_result_transition_valid BEFORE UPDATE ON apply_steps +WHEN NEW.state IN ('response_validated','rejected','skipped') AND NOT EXISTS( + SELECT 1 FROM apply_attempts a JOIN stages s ON s.id=a.stage_id + JOIN stage_revisions r ON r.stage_id=a.stage_id AND r.revision=a.revision + WHERE a.id=NEW.attempt_id AND json_type(NEW.result_json)='object' AND ( + NEW.state='rejected' AND (SELECT count(*) FROM json_each(NEW.result_json))=1 + AND json_type(NEW.result_json,'$.status')='integer' AND json_extract(NEW.result_json,'$.status') BETWEEN 400 AND 499 + OR NEW.state='skipped' AND NEW.condition='if_missing' AND (SELECT count(*) FROM json_each(NEW.result_json))=1 + AND json_type(NEW.result_json,'$.reason')='text' AND json_extract(NEW.result_json,'$.reason')='already_satisfied' + OR NEW.state='response_validated' AND NEW.kind='upload_attachment' AND (SELECT count(*) FROM json_each(NEW.result_json))=1 + AND json_type(NEW.result_json,'$.fileId')='text' AND length(json_extract(NEW.result_json,'$.fileId'))>0 + OR NEW.state='response_validated' AND NEW.kind='create_post' AND (SELECT count(*) FROM json_each(NEW.result_json))=5 + AND json_type(NEW.result_json,'$.postId')='text' AND length(json_extract(NEW.result_json,'$.postId'))>0 + AND json_type(NEW.result_json,'$.createAt')='integer' AND json_extract(NEW.result_json,'$.createAt')>0 + AND json_extract(NEW.result_json,'$.channelId')=json_extract(r.destination_json,'$.channelId') + AND json_extract(NEW.result_json,'$.userId')=s.user_id AND json_extract(NEW.result_json,'$.pendingPostId')=a.pending_post_id + OR NEW.state='response_validated' AND NEW.kind='edit_post' AND (SELECT count(*) FROM json_each(NEW.result_json))=2 + AND json_extract(NEW.result_json,'$.postId')=json_extract(r.destination_json,'$.postId') + AND json_type(NEW.result_json,'$.updateAt')='integer' AND json_extract(NEW.result_json,'$.updateAt')>0 + OR NEW.state='response_validated' AND NEW.kind='delete_post' AND (SELECT count(*) FROM json_each(NEW.result_json))=2 + AND json_extract(NEW.result_json,'$.postId')=json_extract(r.destination_json,'$.postId') + AND json_type(NEW.result_json,'$.deleteAt')='integer' AND json_extract(NEW.result_json,'$.deleteAt')>0 + OR NEW.state='response_validated' AND NEW.kind IN ('add_reaction','remove_reaction') AND (SELECT count(*) FROM json_each(NEW.result_json))=1 + AND json_extract(NEW.result_json,'$.postId')=json_extract(r.destination_json,'$.postId') + OR NEW.state='response_validated' AND NEW.kind='resolve_conversation' AND (SELECT count(*) FROM json_each(NEW.result_json))=2 + AND json_type(NEW.result_json,'$.channelId')='text' AND length(json_extract(NEW.result_json,'$.channelId'))>0 + AND (json_extract(r.destination_json,'$.channelId') IS NULL OR json_extract(NEW.result_json,'$.channelId')=json_extract(r.destination_json,'$.channelId')) + AND json_extract(NEW.result_json,'$.participantIds')=json_extract(r.destination_json,'$.participantIds') + ) +) +BEGIN SELECT RAISE(ABORT, 'invalid apply step result binding'); END; +CREATE TRIGGER apply_steps_history_immutable_delete BEFORE DELETE ON apply_steps +WHEN OLD.state!='pending' OR NOT EXISTS( + SELECT 1 FROM apply_attempts a JOIN stages s ON s.id=a.stage_id + WHERE a.id=OLD.attempt_id AND a.outcome IS NULL AND s.lifecycle='open' AND s.claim_attempt_id IS NULL +) +BEGIN SELECT RAISE(ABORT, 'dispatched apply steps are immutable'); END; +CREATE TRIGGER apply_events_immutable_update BEFORE UPDATE ON apply_events BEGIN SELECT RAISE(ABORT, 'apply events are immutable'); END; +CREATE TRIGGER apply_events_history_immutable_delete BEFORE DELETE ON apply_events +WHEN NOT EXISTS(SELECT 1 FROM apply_attempts a WHERE a.id=OLD.attempt_id AND a.outcome IS NULL) + OR EXISTS(SELECT 1 FROM apply_steps WHERE attempt_id=OLD.attempt_id AND state!='pending') +BEGIN SELECT RAISE(ABORT, 'dispatched apply events are immutable'); END; +CREATE TRIGGER apply_requests_immutable_update BEFORE UPDATE ON apply_requests BEGIN SELECT RAISE(ABORT, 'apply requests are immutable'); END; +CREATE TRIGGER apply_requests_history_immutable_delete BEFORE DELETE ON apply_requests +WHEN NOT EXISTS(SELECT 1 FROM apply_attempts a WHERE a.id=OLD.attempt_id AND a.outcome IS NULL) + OR EXISTS(SELECT 1 FROM apply_steps WHERE attempt_id=OLD.attempt_id AND state!='pending') +BEGIN SELECT RAISE(ABORT, 'dispatched apply requests are immutable'); END; +CREATE TRIGGER apply_receipts_immutable_update BEFORE UPDATE ON apply_receipts BEGIN SELECT RAISE(ABORT, 'apply receipts are immutable'); END; +CREATE TRIGGER apply_receipts_immutable_delete BEFORE DELETE ON apply_receipts BEGIN SELECT RAISE(ABORT, 'apply receipts are immutable'); END; +CREATE TRIGGER stage_revision_semantics_immutable BEFORE UPDATE OF stage_id,revision,created_at,semantic_digest,destination_json,plan_json ON stage_revisions +WHEN NEW.stage_id IS NOT OLD.stage_id OR NEW.revision IS NOT OLD.revision OR NEW.created_at IS NOT OLD.created_at + OR NEW.semantic_digest IS NOT OLD.semantic_digest OR NEW.destination_json IS NOT OLD.destination_json OR NEW.plan_json IS NOT OLD.plan_json +BEGIN SELECT RAISE(ABORT, 'stage revision semantics are immutable'); END; +CREATE TRIGGER stage_revision_state_transition_valid BEFORE UPDATE OF state ON stage_revisions +WHEN NOT (OLD.state='current' AND NEW.state='superseded' AND EXISTS( + SELECT 1 FROM stages s WHERE s.id=OLD.stage_id AND s.claim_attempt_id IS NULL AND s.current_revision=OLD.revision + AND (s.lifecycle='open' AND s.recovery!='forbidden' OR s.lifecycle='expired' AND s.recovery='forbidden') +)) +BEGIN SELECT RAISE(ABORT, 'invalid stage revision state transition'); END; +CREATE TRIGGER stage_revision_insert_lifecycle_valid BEFORE INSERT ON stage_revisions +WHEN EXISTS(SELECT 1 FROM stages s WHERE s.id=NEW.stage_id) AND NOT EXISTS( + SELECT 1 FROM stages s WHERE s.id=NEW.stage_id AND s.claim_attempt_id IS NULL AND NEW.state='current' AND ( + NEW.revision=s.current_revision AND NOT EXISTS(SELECT 1 FROM stage_revisions r WHERE r.stage_id=NEW.stage_id) + OR NEW.revision=s.current_revision+1 AND (s.lifecycle='open' AND s.recovery!='forbidden' OR s.lifecycle='expired' AND s.recovery='forbidden') + ) +) +BEGIN SELECT RAISE(ABORT, 'stage revision insertion is not eligible'); END; +CREATE TRIGGER stage_revision_body_erasure_only BEFORE UPDATE OF body ON stage_revisions +WHEN NEW.body IS NOT OLD.body AND NOT (OLD.body IS NOT NULL AND NEW.body IS NULL AND EXISTS( + SELECT 1 FROM stages s WHERE s.id=OLD.stage_id AND s.lifecycle='completed' AND s.recovery='forbidden' +)) +BEGIN SELECT RAISE(ABORT, 'stage revision body is immutable'); END; +CREATE TRIGGER stage_revisions_history_immutable_delete BEFORE DELETE ON stage_revisions +BEGIN SELECT RAISE(ABORT, 'stage revision history is immutable'); END; +CREATE TRIGGER stage_attachment_immutable_update BEFORE UPDATE ON stage_attachments +BEGIN SELECT RAISE(ABORT, 'stage attachment bindings are immutable'); END; +CREATE TRIGGER stage_attachment_delete_after_completion BEFORE DELETE ON stage_attachments +WHEN NOT EXISTS(SELECT 1 FROM stages s WHERE s.id=OLD.stage_id AND s.lifecycle='completed' AND s.recovery='forbidden') +BEGIN SELECT RAISE(ABORT, 'stage attachment bindings are immutable'); END; +CREATE TRIGGER stage_attachment_insert_before_completion BEFORE INSERT ON stage_attachments +WHEN EXISTS(SELECT 1 FROM stages s WHERE s.id=NEW.stage_id AND s.lifecycle IN ('completed','pruned')) +BEGIN SELECT RAISE(ABORT, 'completed stage attachments are immutable'); END; +CREATE TRIGGER stage_current_revision_transition_valid BEFORE UPDATE OF current_revision ON stages +WHEN NEW.current_revision IS NOT OLD.current_revision AND NOT ( + OLD.claim_attempt_id IS NULL AND NEW.claim_attempt_id IS NULL AND NEW.current_revision=OLD.current_revision+1 + AND NEW.lifecycle='open' AND NEW.recovery!='forbidden' + AND (OLD.lifecycle='open' AND OLD.recovery!='forbidden' OR OLD.lifecycle='expired' AND OLD.recovery='forbidden') + AND EXISTS(SELECT 1 FROM stage_revisions r WHERE r.stage_id=OLD.id AND r.revision=OLD.current_revision AND r.state='superseded') + AND EXISTS(SELECT 1 FROM stage_revisions r WHERE r.stage_id=OLD.id AND r.revision=NEW.current_revision AND r.state='current') +) +BEGIN SELECT RAISE(ABORT, 'invalid current stage revision transition'); END; `}} diff --git a/internal/stagestore/store.go b/internal/stagestore/store.go index d88bb7a..f166fbe 100644 --- a/internal/stagestore/store.go +++ b/internal/stagestore/store.go @@ -80,6 +80,13 @@ func Open(ctx context.Context, path string) (*Store, error) { unlock() return nil, err } + if applyJournalAvailable() { + if _, err := s.recoverInterruptedApplies(ctx); err != nil { + _ = db.Close() + unlock() + return nil, err + } + } return s, nil } @@ -182,6 +189,7 @@ func sqliteURI(path string, readOnly bool) string { } q.Add("_pragma", "busy_timeout("+strconv.Itoa(driverBusyMS)+")") q.Add("_pragma", "foreign_keys(1)") + q.Add("_pragma", "recursive_triggers(1)") q.Add("_pragma", "trusted_schema(0)") q.Add("_pragma", "secure_delete(FAST)") q.Add("_pragma", "synchronous(FULL)") @@ -192,6 +200,7 @@ func sqliteURI(path string, readOnly bool) string { func (s *Store) initialize(ctx context.Context, created bool) error { for _, statement := range []string{ "PRAGMA foreign_keys = ON", "PRAGMA trusted_schema = OFF", + "PRAGMA recursive_triggers = ON", "PRAGMA secure_delete = FAST", "PRAGMA synchronous = FULL", } { if _, err := s.db.ExecContext(ctx, statement); err != nil { diff --git a/internal/stagestore/store_test.go b/internal/stagestore/store_test.go index ba08143..caa2566 100644 --- a/internal/stagestore/store_test.go +++ b/internal/stagestore/store_test.go @@ -835,7 +835,8 @@ func TestCurrentRevisionMustBeCurrent(t *testing.T) { } _, err = tx.Exec(`INSERT INTO stage_revisions(stage_id,revision,state,created_at,semantic_digest,destination_json,plan_json) VALUES('s',1,'superseded','x',zeroblob(32),'{}','{}')`) if err != nil { - t.Fatal(err) + _ = tx.Rollback() + return } if err := tx.Commit(); err == nil { t.Fatal("accepted superseded current revision") diff --git a/schemas/v2/examples/store-doctor.json b/schemas/v2/examples/store-doctor.json index fe26263..38cbbef 100644 --- a/schemas/v2/examples/store-doctor.json +++ b/schemas/v2/examples/store-doctor.json @@ -1 +1 @@ -{"schema":"mm/v2/store-doctor","path":"/home/arda/.local/state/mattermost-cli/stages.sqlite3","report":{"exists":false,"filesystemSafe":null,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":5,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}} +{"schema":"mm/v2/store-doctor","path":"/home/arda/.local/state/mattermost-cli/stages.sqlite3","report":{"exists":false,"filesystemSafe":null,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":6,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}} diff --git a/schemas/v2/examples/store-migrations.json b/schemas/v2/examples/store-migrations.json index 84cb3bd..5894032 100644 --- a/schemas/v2/examples/store-migrations.json +++ b/schemas/v2/examples/store-migrations.json @@ -1 +1 @@ -{"schema":"mm/v2/store-migrations","latest":5,"migrations":[{"version":1,"name":"core-stage-state","checksum":"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"},{"version":2,"name":"immutable-local-request-receipts","checksum":"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"},{"version":3,"name":"caller-intent-stage-create-replay","checksum":"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5"},{"version":4,"name":"caller-intent-stage-revise-replay","checksum":"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4"},{"version":5,"name":"revision-plan-follows-composition","checksum":"fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0"}]} +{"schema":"mm/v2/store-migrations","latest":6,"migrations":[{"version":1,"name":"core-stage-state","checksum":"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"},{"version":2,"name":"immutable-local-request-receipts","checksum":"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"},{"version":3,"name":"caller-intent-stage-create-replay","checksum":"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5"},{"version":4,"name":"caller-intent-stage-revise-replay","checksum":"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4"},{"version":5,"name":"revision-plan-follows-composition","checksum":"fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0"},{"version":6,"name":"durable-apply-journal","checksum":"4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56"}]} diff --git a/schemas/v2/store-doctor.schema.json b/schemas/v2/store-doctor.schema.json index be76833..9a25a67 100644 --- a/schemas/v2/store-doctor.schema.json +++ b/schemas/v2/store-doctor.schema.json @@ -15,7 +15,7 @@ "exists": { "type": "boolean" }, "filesystemSafe": {}, "applicationId": {}, "integrity": {}, "integrityTruncated": {}, "foreignKeyIssues": {}, "foreignKeyRows": {}, "foreignKeyTruncated": {}, "journalMode": {}, "synchronous": {}, "secureDelete": {}, "foreignKeys": {}, "trustedSchema": {}, "queryOnly": {}, "walFallback": {}, - "migrations": { "type": "object", "additionalProperties": false, "required": ["applied", "latest", "valid"], "properties": { "applied": {}, "latest": { "const": 5 }, "valid": {} } }, + "migrations": { "type": "object", "additionalProperties": false, "required": ["applied", "latest", "valid"], "properties": { "applied": {}, "latest": { "const": 6 }, "valid": {} } }, "permissionModelLimitations": { "type": "array", "items": { "$ref": "#/$defs/safeString" } } }, "allOf": [ @@ -23,14 +23,14 @@ "filesystemSafe": { "const": null }, "applicationId": { "const": null }, "integrity": { "const": null }, "integrityTruncated": { "const": null }, "foreignKeyIssues": { "const": null }, "foreignKeyRows": { "const": null }, "foreignKeyTruncated": { "const": null }, "journalMode": { "const": null }, "synchronous": { "const": null }, "secureDelete": { "const": null }, "foreignKeys": { "const": null }, "trustedSchema": { "const": null }, "queryOnly": { "const": null }, "walFallback": { "const": null }, - "migrations": { "properties": { "applied": { "const": null }, "latest": { "const": 5 }, "valid": { "const": null } } } + "migrations": { "properties": { "applied": { "const": null }, "latest": { "const": 6 }, "valid": { "const": null } } } } } }, { "if": { "properties": { "exists": { "const": true } }, "required": ["exists"] }, "then": { "properties": { "filesystemSafe": { "const": true }, "applicationId": { "const": 1296913970 }, "integrity": { "$ref": "#/$defs/nonemptyBoundedStrings" }, "integrityTruncated": { "type": "boolean" }, "foreignKeyIssues": { "type": "integer", "minimum": 0, "maximum": 21 }, "foreignKeyRows": { "$ref": "#/$defs/boundedStrings" }, "foreignKeyTruncated": { "type": "boolean" }, "journalMode": { "enum": ["wal", "delete", "truncate", "persist"] }, "synchronous": { "type": "integer" }, "secureDelete": { "type": "integer" }, "foreignKeys": { "const": true }, "trustedSchema": { "const": false }, "queryOnly": { "const": true }, "walFallback": { "type": "boolean" }, - "migrations": { "properties": { "applied": { "const": 5 }, "latest": { "const": 5 }, "valid": { "const": true } } } + "migrations": { "properties": { "applied": { "const": 6 }, "latest": { "const": 6 }, "valid": { "const": true } } } }, "allOf": [ { "oneOf": [ { "properties": { "integrity": { "const": ["ok"] }, "integrityTruncated": { "const": false } } }, diff --git a/schemas/v2/store-migrations.schema.json b/schemas/v2/store-migrations.schema.json index 869d52c..33704f4 100644 --- a/schemas/v2/store-migrations.schema.json +++ b/schemas/v2/store-migrations.schema.json @@ -6,9 +6,9 @@ "required": ["schema", "latest", "migrations"], "properties": { "schema": { "const": "mm/v2/store-migrations" }, - "latest": { "const": 5 }, + "latest": { "const": 6 }, "migrations": { - "type": "array", "minItems": 5, "maxItems": 5, + "type": "array", "minItems": 6, "maxItems": 6, "prefixItems": [{ "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { "version": { "const": 1 }, "name": { "const": "core-stage-state" }, "checksum": { "const": "e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f" } } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { @@ -19,6 +19,8 @@ "version": { "const": 4 }, "name": { "const": "caller-intent-stage-revise-replay" }, "checksum": { "const": "c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4" } } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { "version": { "const": 5 }, "name": { "const": "revision-plan-follows-composition" }, "checksum": { "const": "fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0" } + } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { + "version": { "const": 6 }, "name": { "const": "durable-apply-journal" }, "checksum": { "const": "4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56" } } }], "items": false } From cbd3c36d69932e4b8d78f04879f849bee84fc544 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 05:38:56 +0300 Subject: [PATCH 074/119] feat: define public apply contracts --- internal/cli/root_test.go | 2 +- internal/schema/apply_semantic.go | 250 +++++++++++++++++++++++++ internal/schema/apply_test.go | 243 ++++++++++++++++++++++++ internal/schema/registry.go | 3 + internal/stagestore/apply.go | 32 ++-- schemas/v2/apply-receipt.schema.json | 125 +++++++++++++ schemas/v2/apply-request.schema.json | 15 ++ schemas/v2/examples/apply-receipt.json | 1 + schemas/v2/examples/apply-request.json | 1 + 9 files changed, 656 insertions(+), 16 deletions(-) create mode 100644 internal/schema/apply_semantic.go create mode 100644 internal/schema/apply_test.go create mode 100644 schemas/v2/apply-receipt.schema.json create mode 100644 schemas/v2/apply-request.schema.json create mode 100644 schemas/v2/examples/apply-receipt.json create mode 100644 schemas/v2/examples/apply-request.json diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index efcdd01..af256c0 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -57,7 +57,7 @@ func TestSchemaList(t *testing.T) { if code != 0 { t.Fatalf("exit code = %d, want 0; stderr = %q", code, stderr.String()) } - if got, want := stdout.String(), "mm/v2/channel\nmm/v2/channels\nmm/v2/config\nmm/v2/dms\nmm/v2/doctor\nmm/v2/error\nmm/v2/group-dms\nmm/v2/mentions\nmm/v2/search\nmm/v2/stage\nmm/v2/stage-cancel-request\nmm/v2/stage-preview\nmm/v2/stage-receipt\nmm/v2/stage-request\nmm/v2/stage-revise-request\nmm/v2/stages\nmm/v2/store-doctor\nmm/v2/store-migrations\nmm/v2/teams\nmm/v2/thread\nmm/v2/unread\nmm/v2/users\nmm/v2/watch-diagnostic\nmm/v2/watch-event\nmm/v2/whoami\n"; got != want { + if got, want := stdout.String(), "mm/v2/apply-receipt\nmm/v2/apply-request\nmm/v2/channel\nmm/v2/channels\nmm/v2/config\nmm/v2/dms\nmm/v2/doctor\nmm/v2/error\nmm/v2/group-dms\nmm/v2/mentions\nmm/v2/search\nmm/v2/stage\nmm/v2/stage-cancel-request\nmm/v2/stage-preview\nmm/v2/stage-receipt\nmm/v2/stage-request\nmm/v2/stage-revise-request\nmm/v2/stages\nmm/v2/store-doctor\nmm/v2/store-migrations\nmm/v2/teams\nmm/v2/thread\nmm/v2/unread\nmm/v2/users\nmm/v2/watch-diagnostic\nmm/v2/watch-event\nmm/v2/whoami\n"; got != want { t.Fatalf("stdout = %q, want %q", got, want) } } diff --git a/internal/schema/apply_semantic.go b/internal/schema/apply_semantic.go new file mode 100644 index 0000000..0cab794 --- /dev/null +++ b/internal/schema/apply_semantic.go @@ -0,0 +1,250 @@ +package schema + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "slices" + "time" +) + +type applyReceiptDocument struct { + Operation string `json:"operation"` + RecoveryMode string `json:"recoveryMode"` + Destination applyReceiptDestination `json:"destination"` + Outcome string `json:"outcome"` + Recovery string `json:"recovery"` + StartedAt time.Time `json:"startedAt"` + RecordedAt time.Time `json:"recordedAt"` + Steps []applyReceiptStep `json:"steps"` +} + +type applyReceiptDestination struct { + Kind string `json:"kind"` + ChannelID *string `json:"channelId"` + ChannelType *string `json:"channelType"` + TeamID *string `json:"teamId"` + PostID *string `json:"postId"` + RootPostID *string `json:"rootPostId"` + ParticipantIDs []string `json:"participantIds"` + PostState json.RawMessage `json:"postState"` +} + +type applyReceiptStep struct { + Ordinal int `json:"ordinal"` + Kind string `json:"kind"` + Condition string `json:"condition"` + State string `json:"state"` + Result json.RawMessage `json:"result"` + StartedAt *time.Time `json:"startedAt"` + EndedAt *time.Time `json:"endedAt"` +} + +func validateSemanticDocument(id string, data []byte) error { + if id != "mm/v2/apply-receipt" { + return nil + } + var receipt applyReceiptDocument + decoder := json.NewDecoder(bytes.NewReader(data)) + if err := decoder.Decode(&receipt); err != nil { + return err + } + return validateApplyReceiptDocument(receipt) +} + +func validateApplyReceiptDocument(receipt applyReceiptDocument) error { + if receipt.RecordedAt.Before(receipt.StartedAt) || !validApplyReceiptPlan(receipt.Operation, receipt.Destination, receipt.Steps) || !validApplyReceiptStepOrder(receipt.Steps) { + return errors.New("invalid apply receipt plan") + } + for i, step := range receipt.Steps { + if step.Ordinal != i+1 || step.StartedAt != nil && step.StartedAt.Before(receipt.StartedAt) || + step.EndedAt != nil && (step.EndedAt.Before(receipt.StartedAt) || step.EndedAt.After(receipt.RecordedAt)) || + step.StartedAt != nil && step.EndedAt != nil && step.EndedAt.Before(*step.StartedAt) || + !validApplyReceiptResultTarget(step, receipt.Destination) { + return fmt.Errorf("invalid apply receipt step %d", i+1) + } + } + outcome, recovery, ok := deriveApplyReceiptResult(receipt.RecoveryMode, receipt.Steps) + if !ok || receipt.Outcome != outcome || receipt.Recovery != recovery { + return errors.New("invalid apply receipt outcome") + } + return nil +} + +func validApplyReceiptStepOrder(steps []applyReceiptStep) bool { + stopped := false + for _, step := range steps { + switch step.State { + case "response_validated", "skipped": + if stopped { + return false + } + case "rejected", "outcome_unknown": + if stopped { + return false + } + stopped = true + case "not_dispatched": + stopped = true + default: + return false + } + } + return true +} + +func deriveApplyReceiptResult(mode string, steps []applyReceiptStep) (string, string, bool) { + prior := map[string]string{"ordinary": "none", "resume_partial": "resume_partial", "force_unknown": "force_unknown"}[mode] + if prior == "" { + return "", "", false + } + validated, skipped, notSent, rejected, unknown := 0, 0, 0, 0, 0 + for _, step := range steps { + switch step.State { + case "response_validated": + validated++ + case "skipped": + skipped++ + case "not_dispatched": + notSent++ + case "rejected": + rejected++ + case "outcome_unknown": + unknown++ + default: + return "", "", false + } + } + if unknown > 0 { + return "unknown", "force_unknown", true + } + if rejected > 0 { + if rejected != 1 { + return "", "", false + } + if validated > 0 { + return "partial", maxApplyRecovery(prior, "resume_partial"), true + } + return "rejected", prior, true + } + if notSent > 0 { + if validated+skipped == 0 { + return "rejected", prior, true + } + return "partial", maxApplyRecovery(prior, "resume_partial"), true + } + if validated+skipped != len(steps) { + return "", "", false + } + if validated == 0 { + return "already_satisfied", "forbidden", true + } + return "succeeded", "forbidden", true +} + +func maxApplyRecovery(left, right string) string { + weight := map[string]int{"none": 0, "resume_partial": 1, "force_unknown": 2, "forbidden": 3} + if weight[left] >= weight[right] { + return left + } + return right +} + +func validApplyReceiptPlan(operation string, destination applyReceiptDestination, steps []applyReceiptStep) bool { + single := func(kind, condition string) bool { + return len(steps) == 1 && steps[0].Kind == kind && steps[0].Condition == condition + } + switch operation { + case "create_post": + return validResolvedConversation(destination) && validCreateSteps(steps) + case "reply": + return validChannelDestination(destination) && destination.Kind == "post" && destination.PostID != nil && destination.RootPostID != nil && isJSONNull(destination.PostState) && validCreateSteps(steps) + case "edit_post": + return validChannelDestination(destination) && destination.Kind == "post" && destination.PostID != nil && !isJSONNull(destination.PostState) && single("edit_post", "always") + case "delete_post": + return validChannelDestination(destination) && destination.Kind == "post" && destination.PostID != nil && !isJSONNull(destination.PostState) && single("delete_post", "always") + case "react": + return validChannelDestination(destination) && destination.Kind == "reaction" && destination.PostID != nil && single("add_reaction", "if_missing") + case "unreact": + return validChannelDestination(destination) && destination.Kind == "reaction" && destination.PostID != nil && single("remove_reaction", "if_missing") + case "resolve_dm": + return validUnresolvedConversation(destination, "dm", 1) && single("resolve_conversation", "if_missing") + case "resolve_group_dm": + return validUnresolvedConversation(destination, "group", 2) && single("resolve_conversation", "if_missing") + default: + return false + } +} + +func validCreateSteps(steps []applyReceiptStep) bool { + if len(steps) == 0 || steps[len(steps)-1].Kind != "create_post" || steps[len(steps)-1].Condition != "always" { + return false + } + for _, step := range steps[:len(steps)-1] { + if step.Kind != "upload_attachment" || step.Condition != "always" { + return false + } + } + return true +} + +func validResolvedConversation(destination applyReceiptDestination) bool { + return destination.Kind == "conversation" && validChannelDestination(destination) +} + +func validChannelDestination(destination applyReceiptDestination) bool { + if destination.ChannelID == nil || destination.ChannelType == nil { + return false + } + switch *destination.ChannelType { + case "public", "private": + return destination.TeamID != nil && len(destination.ParticipantIDs) == 0 + case "dm": + return destination.TeamID == nil && len(destination.ParticipantIDs) == 1 + case "group": + return destination.TeamID == nil && len(destination.ParticipantIDs) == 0 + default: + return false + } +} + +func validUnresolvedConversation(destination applyReceiptDestination, channelType string, minimumParticipants int) bool { + if destination.Kind != "conversation" || destination.ChannelID != nil || destination.ChannelType == nil || *destination.ChannelType != channelType || destination.TeamID != nil { + return false + } + if channelType == "dm" { + return len(destination.ParticipantIDs) == minimumParticipants + } + return len(destination.ParticipantIDs) >= minimumParticipants +} + +func validApplyReceiptResultTarget(step applyReceiptStep, destination applyReceiptDestination) bool { + if step.State != "response_validated" { + return true + } + var result struct { + PostID string `json:"postId"` + ChannelID string `json:"channelId"` + ParticipantIDs []string `json:"participantIds"` + } + if err := json.Unmarshal(step.Result, &result); err != nil { + return false + } + switch step.Kind { + case "create_post": + return destination.ChannelID != nil && result.ChannelID == *destination.ChannelID + case "edit_post", "delete_post", "add_reaction", "remove_reaction": + return destination.PostID != nil && result.PostID == *destination.PostID + case "resolve_conversation": + return (destination.ChannelID == nil || result.ChannelID == *destination.ChannelID) && slices.Equal(result.ParticipantIDs, destination.ParticipantIDs) + case "upload_attachment": + return true + default: + return false + } +} + +func isJSONNull(raw json.RawMessage) bool { + return bytes.Equal(bytes.TrimSpace(raw), []byte("null")) +} diff --git a/internal/schema/apply_test.go b/internal/schema/apply_test.go new file mode 100644 index 0000000..d4a12b2 --- /dev/null +++ b/internal/schema/apply_test.go @@ -0,0 +1,243 @@ +package schema + +import ( + "bytes" + "encoding/json" + "io/fs" + "testing" + + publicschemas "github.com/ardasevinc/mattermost-cli/schemas" +) + +func TestApplySchemasAndExamples(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + for _, name := range []string{"apply-request.json", "apply-receipt.json"} { + raw, readErr := fs.ReadFile(publicschemas.FS, "v2/examples/"+name) + if readErr != nil { + t.Fatal(readErr) + } + var envelope struct { + Schema string `json:"schema"` + } + if json.Unmarshal(raw, &envelope) != nil { + t.Fatalf("decode apply example %s", name) + } + if validateErr := registry.Validate(envelope.Schema, bytes.NewReader(raw)); validateErr != nil { + if envelope.Schema == "mm/v2/apply-receipt" { + var receipt applyReceiptDocument + _ = json.Unmarshal(raw, &receipt) + var document any + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + _ = decoder.Decode(&document) + t.Fatalf("invalid apply example %s: structural=%v semantic=%v", name, registry.compiled[envelope.Schema].Validate(document), validateApplyReceiptDocument(receipt)) + } + t.Fatalf("invalid apply example %s", name) + } + } +} + +func TestApplySchemasRejectUnsafeOrContradictoryDocuments(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + request, _ := fs.ReadFile(publicschemas.FS, "v2/examples/apply-request.json") + receipt, _ := fs.ReadFile(publicschemas.FS, "v2/examples/apply-receipt.json") + cases := []struct { + id string + raw []byte + }{ + {"mm/v2/apply-request", bytes.Replace(request, []byte(`"ordinary"`), []byte(`"unsafe"`), 1)}, + {"mm/v2/apply-request", bytes.Replace(request, []byte(`"requestId":"apply-2026-07-17-1"`), []byte(`"requestId":""`), 1)}, + {"mm/v2/apply-request", bytes.Replace(request, []byte(`"requestId":"apply-2026-07-17-1"`), []byte(`"requestId":".apply"`), 1)}, + {"mm/v2/apply-receipt", bytes.Replace(receipt, []byte(`"forcedDuplicateRisk":false`), []byte(`"forcedDuplicateRisk":true`), 1)}, + {"mm/v2/apply-receipt", bytes.Replace(receipt, []byte(`"recovery":"forbidden"`), []byte(`"recovery":"none"`), 1)}, + } + for _, tc := range cases { + if err := registry.Validate(tc.id, bytes.NewReader(tc.raw)); err == nil { + t.Fatalf("accepted contradictory %s: %s", tc.id, tc.raw) + } + } +} + +func TestApplyRequestAcceptsStoreRequestIDGrammar(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + raw, err := fs.ReadFile(publicschemas.FS, "v2/examples/apply-request.json") + if err != nil { + t.Fatal(err) + } + raw = bytes.Replace(raw, []byte(`"requestId":"apply-2026-07-17-1"`), []byte(`"requestId":"a~b"`), 1) + if err := registry.Validate("mm/v2/apply-request", bytes.NewReader(raw)); err != nil { + t.Fatalf("rejected store-valid request ID: %s", raw) + } +} + +func TestApplyReceiptBindsOutcomeResultAndOperation(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + raw, err := fs.ReadFile(publicschemas.FS, "v2/examples/apply-receipt.json") + if err != nil { + t.Fatal(err) + } + var base map[string]any + if err := json.Unmarshal(raw, &base); err != nil { + t.Fatal(err) + } + + cases := map[string]func(map[string]any){ + "succeeded with pending step": func(doc map[string]any) { + step := doc["steps"].([]any)[0].(map[string]any) + step["state"] = "pending" + step["result"] = nil + step["startedAt"] = nil + step["endedAt"] = nil + }, + "partial without recovery": func(doc map[string]any) { + doc["outcome"] = "partial" + doc["recovery"] = "none" + }, + "validated create with status result": func(doc map[string]any) { + doc["steps"].([]any)[0].(map[string]any)["result"] = map[string]any{"status": float64(409)} + }, + "rejected create with success result": func(doc map[string]any) { + doc["outcome"] = "rejected" + doc["recovery"] = "none" + doc["steps"].([]any)[0].(map[string]any)["state"] = "rejected" + }, + "edit operation with create plan": func(doc map[string]any) { + doc["operation"] = "edit_post" + }, + "unknown with pending residue": func(doc map[string]any) { + doc["outcome"] = "unknown" + doc["recovery"] = "force_unknown" + doc["steps"] = []any{ + map[string]any{"ordinal": float64(1), "kind": "upload_attachment", "condition": "always", "state": "outcome_unknown", "result": nil, "startedAt": "2026-07-17T02:00:00Z", "endedAt": "2026-07-17T02:00:01Z"}, + map[string]any{"ordinal": float64(2), "kind": "create_post", "condition": "always", "state": "pending", "result": nil, "startedAt": nil, "endedAt": nil}, + } + }, + "partial with multiple rejections": func(doc map[string]any) { + doc["outcome"] = "partial" + doc["recovery"] = "resume_partial" + doc["steps"] = []any{ + map[string]any{"ordinal": float64(1), "kind": "upload_attachment", "condition": "always", "state": "rejected", "result": map[string]any{"status": float64(409)}, "startedAt": "2026-07-17T02:00:00Z", "endedAt": "2026-07-17T02:00:01Z"}, + map[string]any{"ordinal": float64(2), "kind": "upload_attachment", "condition": "always", "state": "rejected", "result": map[string]any{"status": float64(409)}, "startedAt": "2026-07-17T02:00:00Z", "endedAt": "2026-07-17T02:00:01Z"}, + doc["steps"].([]any)[0], + } + doc["steps"].([]any)[2].(map[string]any)["ordinal"] = float64(3) + }, + "ordinary rejection with unknown recovery": func(doc map[string]any) { + doc["outcome"] = "rejected" + doc["recovery"] = "force_unknown" + step := doc["steps"].([]any)[0].(map[string]any) + step["state"] = "rejected" + step["result"] = map[string]any{"status": float64(409)} + }, + "validated effect after rejection": func(doc map[string]any) { + doc["outcome"] = "partial" + doc["recovery"] = "resume_partial" + doc["steps"] = []any{ + map[string]any{"ordinal": float64(1), "kind": "upload_attachment", "condition": "always", "state": "rejected", "result": map[string]any{"status": float64(409)}, "startedAt": "2026-07-17T02:00:00Z", "endedAt": "2026-07-17T02:00:01Z"}, + doc["steps"].([]any)[0], + } + doc["steps"].([]any)[1].(map[string]any)["ordinal"] = float64(2) + }, + "validated effect after not dispatched": func(doc map[string]any) { + doc["outcome"] = "partial" + doc["recovery"] = "resume_partial" + doc["steps"] = []any{ + map[string]any{"ordinal": float64(1), "kind": "upload_attachment", "condition": "always", "state": "not_dispatched", "result": nil, "startedAt": nil, "endedAt": "2026-07-17T02:00:01Z"}, + doc["steps"].([]any)[0], + } + doc["steps"].([]any)[1].(map[string]any)["ordinal"] = float64(2) + }, + "create before upload": func(doc map[string]any) { + doc["steps"] = append(doc["steps"].([]any), map[string]any{"ordinal": float64(2), "kind": "upload_attachment", "condition": "always", "state": "response_validated", "result": map[string]any{"fileId": "file-1"}, "startedAt": "2026-07-17T02:00:00Z", "endedAt": "2026-07-17T02:00:01Z"}) + }, + "reply without post binding": func(doc map[string]any) { + doc["operation"] = "reply" + destination := doc["destination"].(map[string]any) + destination["kind"] = "post" + destination["postId"] = nil + destination["rootPostId"] = nil + }, + "dm resolution with group target": func(doc map[string]any) { + doc["operation"] = "resolve_dm" + destination := doc["destination"].(map[string]any) + destination["channelId"] = nil + destination["channelType"] = "group" + destination["participantIds"] = []any{"user-2", "user-3", "user-4"} + doc["steps"] = []any{map[string]any{"ordinal": float64(1), "kind": "resolve_conversation", "condition": "if_missing", "state": "response_validated", "result": map[string]any{"channelId": "channel-2", "participantIds": []any{"user-2", "user-3", "user-4"}}, "startedAt": "2026-07-17T02:00:00Z", "endedAt": "2026-07-17T02:00:01Z"}} + }, + "create result for another channel": func(doc map[string]any) { + doc["steps"].([]any)[0].(map[string]any)["result"].(map[string]any)["channelId"] = "channel-2" + }, + "edit result for another post": func(doc map[string]any) { + makeEditReceipt(doc, "post-2") + }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + var doc map[string]any + encoded, marshalErr := json.Marshal(base) + if marshalErr != nil { + t.Fatal(marshalErr) + } + if unmarshalErr := json.Unmarshal(encoded, &doc); unmarshalErr != nil { + t.Fatal(unmarshalErr) + } + mutate(doc) + encoded, marshalErr = json.Marshal(doc) + if marshalErr != nil { + t.Fatal(marshalErr) + } + if validateErr := registry.Validate("mm/v2/apply-receipt", bytes.NewReader(encoded)); validateErr == nil { + t.Fatalf("accepted contradictory receipt: %s", encoded) + } + }) + } +} + +func TestApplyReceiptAcceptsRealisticEditProjection(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + raw, err := fs.ReadFile(publicschemas.FS, "v2/examples/apply-receipt.json") + if err != nil { + t.Fatal(err) + } + var doc map[string]any + if err := json.Unmarshal(raw, &doc); err != nil { + t.Fatal(err) + } + makeEditReceipt(doc, "post-1") + encoded, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/apply-receipt", bytes.NewReader(encoded)); err != nil { + t.Fatalf("rejected realistic edit receipt: %s", encoded) + } +} + +func makeEditReceipt(doc map[string]any, resultPostID string) { + doc["operation"] = "edit_post" + destination := doc["destination"].(map[string]any) + destination["kind"] = "post" + destination["channelType"] = "private" + destination["teamId"] = "team-1" + destination["participantIds"] = []any{} + destination["postId"] = "post-1" + destination["rootPostId"] = nil + destination["postState"] = map[string]any{"authorUserId": "user-1", "updateAt": float64(1784253599000), "contentDigest": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"} + doc["steps"] = []any{map[string]any{"ordinal": float64(1), "kind": "edit_post", "condition": "always", "state": "response_validated", "result": map[string]any{"postId": resultPostID, "updateAt": float64(1784253600000)}, "startedAt": "2026-07-17T02:00:00Z", "endedAt": "2026-07-17T02:00:01Z"}} +} diff --git a/internal/schema/registry.go b/internal/schema/registry.go index 2f07d86..3cd3d24 100644 --- a/internal/schema/registry.go +++ b/internal/schema/registry.go @@ -143,6 +143,9 @@ func (r *Registry) ReadAndValidate(id string, input io.Reader) ([]byte, error) { if err := compiled.Validate(document); err != nil { return nil, fmt.Errorf("document does not match %s", id) } + if err := validateSemanticDocument(id, data); err != nil { + return nil, fmt.Errorf("document does not match %s", id) + } return bytes.Clone(data), nil } diff --git a/internal/stagestore/apply.go b/internal/stagestore/apply.go index 8c8b779..4d8713a 100644 --- a/internal/stagestore/apply.go +++ b/internal/stagestore/apply.go @@ -73,19 +73,21 @@ type ApplyAttempt struct { } type ApplyReceipt struct { - Schema string `json:"schema"` - AttemptID string `json:"attemptId"` - StageID string `json:"stageId"` - Revision int64 `json:"revision"` - SemanticDigest string `json:"semanticDigest"` - Operation Operation `json:"operation"` - Destination json.RawMessage `json:"destination"` - Outcome AttemptOutcome `json:"outcome"` - Recovery Recovery `json:"recovery"` - StartedAt time.Time `json:"startedAt"` - RecordedAt time.Time `json:"recordedAt"` - Steps []ApplyStep `json:"steps"` - Replay bool `json:"-"` + Schema string `json:"schema"` + AttemptID string `json:"attemptId"` + StageID string `json:"stageId"` + Revision int64 `json:"revision"` + SemanticDigest string `json:"semanticDigest"` + Operation Operation `json:"operation"` + RecoveryMode RecoveryMode `json:"recoveryMode"` + ForcedDuplicateRisk bool `json:"forcedDuplicateRisk"` + Destination json.RawMessage `json:"destination"` + Outcome AttemptOutcome `json:"outcome"` + Recovery Recovery `json:"recovery"` + StartedAt time.Time `json:"startedAt"` + RecordedAt time.Time `json:"recordedAt"` + Steps []ApplyStep `json:"steps"` + Replay bool `json:"-"` } type persistedPlan struct { @@ -347,7 +349,7 @@ func (s *Store) FinalizeApply(ctx context.Context, attemptID string) (ApplyRecei } } attempt.Outcome, attempt.EndedAt = &outcome, &now - receipt := ApplyReceipt{"mm/v2/apply-receipt", attempt.ID, attempt.StageID, attempt.Revision, hex.EncodeToString(attempt.SemanticDigest[:]), operation, json.RawMessage(destination), outcome, recovery, attempt.StartedAt, now, attempt.Steps, false} + receipt := ApplyReceipt{"mm/v2/apply-receipt", attempt.ID, attempt.StageID, attempt.Revision, hex.EncodeToString(attempt.SemanticDigest[:]), operation, attempt.RecoveryMode, attempt.ForcedDuplicateRisk, json.RawMessage(destination), outcome, recovery, attempt.StartedAt, now, attempt.Steps, false} raw, err := marshalCanonical(receipt) if err != nil { return ApplyReceipt{}, localError(err) @@ -777,7 +779,7 @@ func validApplyStep(step ApplyStep) bool { } func validReceiptForAttempt(receipt ApplyReceipt, attempt ApplyAttempt) bool { - if receipt.Schema != "mm/v2/apply-receipt" || receipt.AttemptID != attempt.ID || receipt.StageID != attempt.StageID || receipt.Revision != attempt.Revision || receipt.SemanticDigest != hex.EncodeToString(attempt.SemanticDigest[:]) || + if receipt.Schema != "mm/v2/apply-receipt" || receipt.AttemptID != attempt.ID || receipt.StageID != attempt.StageID || receipt.Revision != attempt.Revision || receipt.SemanticDigest != hex.EncodeToString(attempt.SemanticDigest[:]) || receipt.RecoveryMode != attempt.RecoveryMode || receipt.ForcedDuplicateRisk != attempt.ForcedDuplicateRisk || attempt.Outcome == nil || receipt.Outcome != *attempt.Outcome || attempt.EndedAt == nil || !receipt.StartedAt.Equal(attempt.StartedAt) || !receipt.RecordedAt.Equal(*attempt.EndedAt) || !validOperation(receipt.Operation) || !validRecovery(receipt.Recovery) || len(receipt.Steps) != len(attempt.Steps) { return false diff --git a/schemas/v2/apply-receipt.schema.json b/schemas/v2/apply-receipt.schema.json new file mode 100644 index 0000000..be42832 --- /dev/null +++ b/schemas/v2/apply-receipt.schema.json @@ -0,0 +1,125 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:apply-receipt", + "$comment": "mm schema validate also enforces terminal outcome derivation, operation plan ordering, destination shape, timestamp ordering, and result-to-target equality because JSON Schema cannot express all cross-field equalities.", + "type": "object", + "additionalProperties": false, + "required": ["schema", "attemptId", "stageId", "revision", "semanticDigest", "operation", "recoveryMode", "forcedDuplicateRisk", "destination", "outcome", "recovery", "startedAt", "recordedAt", "steps"], + "properties": { + "schema": { "const": "mm/v2/apply-receipt" }, + "attemptId": { "type": "string", "pattern": "^att_[A-Za-z0-9_-]{32}$" }, + "stageId": { "type": "string", "pattern": "^stg_[A-Za-z0-9_-]{32}$" }, + "revision": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, + "semanticDigest": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "operation": { "enum": ["create_post", "reply", "edit_post", "delete_post", "react", "unreact", "resolve_dm", "resolve_group_dm"] }, + "recoveryMode": { "enum": ["ordinary", "resume_partial", "force_unknown"] }, + "forcedDuplicateRisk": { "type": "boolean" }, + "destination": { "$ref": "#/$defs/destination" }, + "outcome": { "enum": ["succeeded", "already_satisfied", "rejected", "partial", "unknown"] }, + "recovery": { "enum": ["none", "resume_partial", "force_unknown", "forbidden"] }, + "startedAt": { "type": "string", "format": "date-time" }, + "recordedAt": { "type": "string", "format": "date-time" }, + "steps": { + "type": "array", + "minItems": 1, + "maxItems": 102, + "items": { "$ref": "#/$defs/step" } + } + }, + "allOf": [ + { "if": { "properties": { "recoveryMode": { "const": "force_unknown" } } }, "then": { "properties": { "forcedDuplicateRisk": { "const": true } } }, "else": { "properties": { "forcedDuplicateRisk": { "const": false } } } }, + { "if": { "properties": { "outcome": { "enum": ["succeeded", "already_satisfied"] } } }, "then": { "properties": { "recovery": { "const": "forbidden" } } } }, + { "if": { "properties": { "outcome": { "const": "unknown" } } }, "then": { "properties": { "recovery": { "const": "force_unknown" }, "steps": { "contains": { "properties": { "state": { "const": "outcome_unknown" } } }, "minContains": 1 } } } }, + { "if": { "properties": { "outcome": { "const": "succeeded" } } }, "then": { "properties": { "steps": { "items": { "properties": { "state": { "enum": ["response_validated", "skipped"] } } }, "contains": { "properties": { "state": { "const": "response_validated" } } }, "minContains": 1 } } } }, + { "if": { "properties": { "outcome": { "const": "already_satisfied" } } }, "then": { "properties": { "steps": { "items": { "properties": { "state": { "const": "skipped" } } } } } } }, + { "if": { "properties": { "outcome": { "const": "rejected" } } }, "then": { "properties": { "recovery": { "enum": ["none", "resume_partial", "force_unknown"] }, "steps": { "items": { "properties": { "state": { "enum": ["rejected", "not_dispatched"] } } }, "contains": { "properties": { "state": { "const": "rejected" } } }, "minContains": 1 } } } }, + { "if": { "properties": { "outcome": { "const": "partial" } } }, "then": { "properties": { "recovery": { "enum": ["resume_partial", "force_unknown"] }, "steps": { "items": { "properties": { "state": { "enum": ["response_validated", "skipped", "rejected", "not_dispatched"] } } }, "allOf": [{ "contains": { "properties": { "state": { "const": "response_validated" } } }, "minContains": 1 }, { "contains": { "properties": { "state": { "enum": ["rejected", "not_dispatched"] } } }, "minContains": 1 }] } } } }, + { "if": { "properties": { "operation": { "enum": ["create_post", "reply"] } } }, "then": { "properties": { "steps": { "items": { "properties": { "kind": { "enum": ["upload_attachment", "create_post"] }, "condition": { "const": "always" } }, "allOf": [{ "if": { "properties": { "kind": { "const": "upload_attachment" } } }, "then": { "properties": { "kind": { "const": "upload_attachment" } } } }] }, "contains": { "properties": { "kind": { "const": "create_post" } } }, "minContains": 1, "maxContains": 1 } } } }, + { "if": { "properties": { "operation": { "const": "create_post" } } }, "then": { "properties": { "destination": { "properties": { "kind": { "const": "conversation" } } } } } }, + { "if": { "properties": { "operation": { "const": "reply" } } }, "then": { "properties": { "destination": { "properties": { "kind": { "const": "post" }, "postState": { "type": "null" } } } } } }, + { "if": { "properties": { "operation": { "const": "edit_post" } } }, "then": { "$ref": "#/$defs/editOperation" } }, + { "if": { "properties": { "operation": { "const": "delete_post" } } }, "then": { "$ref": "#/$defs/deleteOperation" } }, + { "if": { "properties": { "operation": { "const": "react" } } }, "then": { "$ref": "#/$defs/reactOperation" } }, + { "if": { "properties": { "operation": { "const": "unreact" } } }, "then": { "$ref": "#/$defs/unreactOperation" } }, + { "if": { "properties": { "operation": { "enum": ["resolve_dm", "resolve_group_dm"] } } }, "then": { "$ref": "#/$defs/resolveOperation" } } + ], + "$defs": { + "id": { "type": "string", "minLength": 1, "maxLength": 4096 }, + "destination": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "channelId", "channelType", "teamId", "postId", "rootPostId", "participantIds", "emoji", "postState", "reactionPresent"], + "properties": { + "kind": { "enum": ["conversation", "post", "reaction"] }, + "channelId": { "anyOf": [{ "$ref": "#/$defs/id" }, { "type": "null" }] }, + "channelType": { "enum": ["dm", "group", "public", "private", null] }, + "teamId": { "anyOf": [{ "$ref": "#/$defs/id" }, { "type": "null" }] }, + "postId": { "anyOf": [{ "$ref": "#/$defs/id" }, { "type": "null" }] }, + "rootPostId": { "anyOf": [{ "$ref": "#/$defs/id" }, { "type": "null" }] }, + "participantIds": { "type": "array", "maxItems": 100, "uniqueItems": true, "items": { "$ref": "#/$defs/id" } }, + "emoji": { "anyOf": [{ "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9_+-]+$" }, { "type": "null" }] }, + "postState": { + "anyOf": [ + { "type": "null" }, + { "type": "object", "additionalProperties": false, "required": ["authorUserId", "updateAt", "contentDigest"], "properties": { "authorUserId": { "$ref": "#/$defs/id" }, "updateAt": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, "contentDigest": { "type": "string", "pattern": "^[0-9a-f]{64}$" } } } + ] + }, + "reactionPresent": { "type": ["boolean", "null"] } + }, + "allOf": [ + { "if": { "properties": { "kind": { "const": "conversation" } } }, "then": { "properties": { "postId": { "type": "null" }, "rootPostId": { "type": "null" }, "emoji": { "type": "null" }, "postState": { "type": "null" }, "reactionPresent": { "type": "null" } } } }, + { "if": { "properties": { "kind": { "const": "reaction" } } }, "then": { "properties": { "postId": { "$ref": "#/$defs/id" }, "emoji": { "type": "string" }, "postState": { "type": "null" }, "reactionPresent": { "type": "boolean" } } } } + ] + }, + "fileResult": { "type": "object", "additionalProperties": false, "required": ["fileId"], "properties": { "fileId": { "type": "string", "minLength": 1, "maxLength": 4096 } } }, + "createResult": { "type": "object", "additionalProperties": false, "required": ["postId", "createAt", "channelId", "userId", "pendingPostId"], "properties": { "postId": { "type": "string", "minLength": 1, "maxLength": 4096 }, "createAt": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, "channelId": { "type": "string", "minLength": 1, "maxLength": 4096 }, "userId": { "type": "string", "minLength": 1, "maxLength": 4096 }, "pendingPostId": { "type": "string", "minLength": 1, "maxLength": 4096 } } }, + "editResult": { "type": "object", "additionalProperties": false, "required": ["postId", "updateAt"], "properties": { "postId": { "type": "string", "minLength": 1, "maxLength": 4096 }, "updateAt": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 } } }, + "deleteResult": { "type": "object", "additionalProperties": false, "required": ["postId", "deleteAt"], "properties": { "postId": { "type": "string", "minLength": 1, "maxLength": 4096 }, "deleteAt": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 } } }, + "postResult": { "type": "object", "additionalProperties": false, "required": ["postId"], "properties": { "postId": { "type": "string", "minLength": 1, "maxLength": 4096 } } }, + "conversationResult": { "type": "object", "additionalProperties": false, "required": ["channelId", "participantIds"], "properties": { "channelId": { "type": "string", "minLength": 1, "maxLength": 4096 }, "participantIds": { "type": "array", "maxItems": 100, "uniqueItems": true, "items": { "type": "string", "minLength": 1, "maxLength": 4096 } } } }, + "rejectedResult": { "type": "object", "additionalProperties": false, "required": ["status"], "properties": { "status": { "type": "integer", "minimum": 400, "maximum": 499 } } }, + "skippedResult": { "type": "object", "additionalProperties": false, "required": ["reason"], "properties": { "reason": { "const": "already_satisfied" } } }, + "result": { + "anyOf": [ + { "type": "null" }, + { "$ref": "#/$defs/fileResult" }, { "$ref": "#/$defs/createResult" }, { "$ref": "#/$defs/editResult" }, { "$ref": "#/$defs/deleteResult" }, + { "$ref": "#/$defs/postResult" }, { "$ref": "#/$defs/conversationResult" }, { "$ref": "#/$defs/rejectedResult" }, { "$ref": "#/$defs/skippedResult" } + ] + }, + "step": { + "type": "object", + "additionalProperties": false, + "required": ["ordinal", "kind", "condition", "state", "result", "startedAt", "endedAt"], + "properties": { + "ordinal": { "type": "integer", "minimum": 1, "maximum": 102 }, + "kind": { "enum": ["upload_attachment", "create_post", "edit_post", "delete_post", "add_reaction", "remove_reaction", "resolve_conversation"] }, + "condition": { "enum": ["always", "if_missing"] }, + "state": { "enum": ["pending", "dispatch_intent", "response_validated", "rejected", "outcome_unknown", "skipped", "not_dispatched"] }, + "result": { "$ref": "#/$defs/result" }, + "startedAt": { "type": ["string", "null"], "format": "date-time" }, + "endedAt": { "type": ["string", "null"], "format": "date-time" } + }, + "allOf": [ + { "if": { "properties": { "state": { "const": "pending" } } }, "then": { "properties": { "result": { "type": "null" }, "startedAt": { "type": "null" }, "endedAt": { "type": "null" } } } }, + { "if": { "properties": { "state": { "const": "dispatch_intent" } } }, "then": { "properties": { "result": { "type": "null" }, "startedAt": { "type": "string" }, "endedAt": { "type": "null" } } } }, + { "if": { "properties": { "state": { "const": "response_validated" } } }, "then": { "properties": { "result": { "not": { "type": "null" } }, "startedAt": { "type": "string" }, "endedAt": { "type": "string" } } } }, + { "if": { "properties": { "state": { "const": "rejected" } } }, "then": { "properties": { "result": { "$ref": "#/$defs/rejectedResult" }, "startedAt": { "type": "string" }, "endedAt": { "type": "string" } } } }, + { "if": { "properties": { "state": { "const": "outcome_unknown" } } }, "then": { "properties": { "result": { "type": "null" }, "startedAt": { "type": "string" }, "endedAt": { "type": "string" } } } }, + { "if": { "properties": { "state": { "const": "skipped" } } }, "then": { "properties": { "condition": { "const": "if_missing" }, "result": { "$ref": "#/$defs/skippedResult" }, "startedAt": { "type": "null" }, "endedAt": { "type": "string" } } } }, + { "if": { "properties": { "state": { "const": "not_dispatched" } } }, "then": { "properties": { "result": { "type": "null" }, "startedAt": { "type": "null" }, "endedAt": { "type": "string" } } } }, + { "if": { "properties": { "state": { "const": "response_validated" }, "kind": { "const": "upload_attachment" } } }, "then": { "properties": { "result": { "$ref": "#/$defs/fileResult" } } } }, + { "if": { "properties": { "state": { "const": "response_validated" }, "kind": { "const": "create_post" } } }, "then": { "properties": { "result": { "$ref": "#/$defs/createResult" } } } }, + { "if": { "properties": { "state": { "const": "response_validated" }, "kind": { "const": "edit_post" } } }, "then": { "properties": { "result": { "$ref": "#/$defs/editResult" } } } }, + { "if": { "properties": { "state": { "const": "response_validated" }, "kind": { "const": "delete_post" } } }, "then": { "properties": { "result": { "$ref": "#/$defs/deleteResult" } } } }, + { "if": { "properties": { "state": { "const": "response_validated" }, "kind": { "enum": ["add_reaction", "remove_reaction"] } } }, "then": { "properties": { "result": { "$ref": "#/$defs/postResult" } } } }, + { "if": { "properties": { "state": { "const": "response_validated" }, "kind": { "const": "resolve_conversation" } } }, "then": { "properties": { "result": { "$ref": "#/$defs/conversationResult" } } } } + ] + }, + "singleStep": { "properties": { "steps": { "minItems": 1, "maxItems": 1 } } }, + "editOperation": { "allOf": [{ "$ref": "#/$defs/singleStep" }, { "properties": { "destination": { "properties": { "kind": { "const": "post" }, "postState": { "type": "object" } } }, "steps": { "items": { "properties": { "kind": { "const": "edit_post" }, "condition": { "const": "always" } } } } } }] }, + "deleteOperation": { "allOf": [{ "$ref": "#/$defs/singleStep" }, { "properties": { "destination": { "properties": { "kind": { "const": "post" }, "postState": { "type": "object" } } }, "steps": { "items": { "properties": { "kind": { "const": "delete_post" }, "condition": { "const": "always" } } } } } }] }, + "reactOperation": { "allOf": [{ "$ref": "#/$defs/singleStep" }, { "properties": { "destination": { "properties": { "kind": { "const": "reaction" } } }, "steps": { "items": { "properties": { "kind": { "const": "add_reaction" }, "condition": { "const": "if_missing" } } } } } }] }, + "unreactOperation": { "allOf": [{ "$ref": "#/$defs/singleStep" }, { "properties": { "destination": { "properties": { "kind": { "const": "reaction" } } }, "steps": { "items": { "properties": { "kind": { "const": "remove_reaction" }, "condition": { "const": "if_missing" } } } } } }] }, + "resolveOperation": { "allOf": [{ "$ref": "#/$defs/singleStep" }, { "properties": { "destination": { "properties": { "kind": { "const": "conversation" } } }, "steps": { "items": { "properties": { "kind": { "const": "resolve_conversation" }, "condition": { "const": "if_missing" } } } } } }] } + } +} diff --git a/schemas/v2/apply-request.schema.json b/schemas/v2/apply-request.schema.json new file mode 100644 index 0000000..b0f678c --- /dev/null +++ b/schemas/v2/apply-request.schema.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:apply-request", + "type": "object", + "additionalProperties": false, + "required": ["schema", "requestId", "stageId", "revision", "expectedDigest", "recoveryMode"], + "properties": { + "schema": { "const": "mm/v2/apply-request" }, + "requestId": { "type": "string", "minLength": 1, "maxLength": 256, "pattern": "^[A-Za-z0-9][A-Za-z0-9._~:-]*$" }, + "stageId": { "type": "string", "pattern": "^stg_[A-Za-z0-9_-]{32}$" }, + "revision": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, + "expectedDigest": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "recoveryMode": { "enum": ["ordinary", "resume_partial", "force_unknown"] } + } +} diff --git a/schemas/v2/examples/apply-receipt.json b/schemas/v2/examples/apply-receipt.json new file mode 100644 index 0000000..f7c84bf --- /dev/null +++ b/schemas/v2/examples/apply-receipt.json @@ -0,0 +1 @@ +{"schema":"mm/v2/apply-receipt","attemptId":"att_abcdefghijklmnopqrstuvwxyzABCDEF","stageId":"stg_abcdefghijklmnopqrstuvwxyzABCDEF","revision":2,"semanticDigest":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","operation":"create_post","recoveryMode":"ordinary","forcedDuplicateRisk":false,"destination":{"kind":"conversation","channelId":"channel-1","channelType":"dm","teamId":null,"postId":null,"rootPostId":null,"participantIds":["user-2"],"emoji":null,"postState":null,"reactionPresent":null},"outcome":"succeeded","recovery":"forbidden","startedAt":"2026-07-17T02:00:00Z","recordedAt":"2026-07-17T02:00:01Z","steps":[{"ordinal":1,"kind":"create_post","condition":"always","state":"response_validated","result":{"postId":"post-1","createAt":1784253600000,"channelId":"channel-1","userId":"user-1","pendingPostId":"pending_abcdefghijklmnopqrstuvwxyzAB"},"startedAt":"2026-07-17T02:00:00Z","endedAt":"2026-07-17T02:00:01Z"}]} diff --git a/schemas/v2/examples/apply-request.json b/schemas/v2/examples/apply-request.json new file mode 100644 index 0000000..f1b6b5f --- /dev/null +++ b/schemas/v2/examples/apply-request.json @@ -0,0 +1 @@ +{"schema":"mm/v2/apply-request","requestId":"apply-2026-07-17-1","stageId":"stg_abcdefghijklmnopqrstuvwxyzABCDEF","revision":2,"expectedDigest":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","recoveryMode":"ordinary"} From a46df29856a4b76149916ad238b7e097e50e3734 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 05:50:35 +0300 Subject: [PATCH 075/119] feat: add prepared mutation transport --- internal/api/client.go | 105 ++++++++++++++++++- internal/api/client_test.go | 199 ++++++++++++++++++++++++++++++++++++ 2 files changed, 303 insertions(+), 1 deletion(-) diff --git a/internal/api/client.go b/internal/api/client.go index 4f32166..cc5d5aa 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -33,6 +33,7 @@ var ( ErrInvalidJSON = errors.New("Mattermost returned an invalid JSON response") ErrBodyTooLarge = errors.New("Mattermost response exceeded the size limit") ErrClientClosed = errors.New("Mattermost client is closed") + ErrMutationUsed = errors.New("prepared Mattermost mutation was already consumed") ) type APIError struct{ Status int } @@ -100,6 +101,24 @@ type Client struct { closed bool } +// PreparedMutation is an immutable, one-shot JSON mutation. URL resolution and +// body encoding happen before it is returned so callers can durably record +// dispatch intent immediately before Execute. Once Execute starts, every local, +// transport, response, or cancellation failure is conservatively outcome +// unknown; callers must never retry the same effect automatically. +type PreparedMutation struct { + client *Client + method string + endpoint *url.URL + payload []byte + state *preparedMutationState +} + +type preparedMutationState struct { + mu sync.Mutex + used bool +} + type requestContextKey uint8 const mutationRequestKey requestContextKey = 1 @@ -157,6 +176,79 @@ func (c *Client) Delete(ctx context.Context, path string, out any) error { return c.request(ctx, http.MethodDelete, path, nil, out, true, true) } +func (c *Client) PreparePost(path string, body any) (*PreparedMutation, error) { + return c.prepareMutation(http.MethodPost, path, body) +} + +func (c *Client) PreparePut(path string, body any) (*PreparedMutation, error) { + return c.prepareMutation(http.MethodPut, path, body) +} + +func (c *Client) PrepareDelete(path string) (*PreparedMutation, error) { + return c.prepareMutation(http.MethodDelete, path, nil) +} + +func (c *Client) prepareMutation(method, path string, body any) (*PreparedMutation, error) { + c.lifecycle.RLock() + defer c.lifecycle.RUnlock() + if c.closed { + return nil, ErrClientClosed + } + endpoint, err := c.endpoint(path) + if err != nil { + return nil, err + } + payload, err := encodeBody(body) + if err != nil { + return nil, errors.New("unable to encode Mattermost request") + } + return &PreparedMutation{client: c, method: method, endpoint: endpoint, payload: bytes.Clone(payload), state: &preparedMutationState{}}, nil +} + +// Execute consumes the prepared mutation before inspecting cancellation or +// client state. This makes a durable dispatch intent conservative: any failure +// after the caller records it is unknown and the mutation cannot be replayed. +func (p *PreparedMutation) Execute(ctx context.Context, out any) error { + if p == nil || p.state == nil || p.client == nil || p.endpoint == nil { + return consumedMutationError() + } + p.state.mu.Lock() + if p.state.used { + p.state.mu.Unlock() + return consumedMutationError() + } + p.state.used = true + p.state.mu.Unlock() + + p.client.lifecycle.RLock() + defer p.client.lifecycle.RUnlock() + if p.client.closed || ctx == nil { + return &OutcomeUnknownError{} + } + status, _, data, failure := p.client.attempt(ctx, p.method, p.endpoint, p.payload, true, true) + if failure != nil { + return &OutcomeUnknownError{} + } + if status >= 400 && status < 500 { + return &APIError{Status: status} + } + if status < 200 || status >= 300 { + return &OutcomeUnknownError{} + } + decodeTarget := out + if decodeTarget == nil { + decodeTarget = new(any) + } + if err := decodeJSON(data, decodeTarget); err != nil { + return &OutcomeUnknownError{} + } + return nil +} + +func consumedMutationError() error { + return errors.Join(&OutcomeUnknownError{}, ErrMutationUsed) +} + func (c *Client) request(ctx context.Context, method, path string, body, out any, mutation, authenticated bool) error { c.lifecycle.RLock() defer c.lifecycle.RUnlock() @@ -234,10 +326,21 @@ func (c *Client) attempt(parent context.Context, method string, endpoint *url.UR if mutation { ctx = context.WithValue(ctx, mutationRequestKey, true) } - req, err := http.NewRequestWithContext(ctx, method, endpoint.String(), bytes.NewReader(payload)) + var body io.Reader = bytes.NewReader(payload) + if mutation { + // bytes.Reader makes requests replayable by populating GetBody, and a + // nil body becomes http.NoBody. Both permit transparent HTTP/2 retries. + // A distinct ReadCloser keeps even empty mutations non-replayable. + body = io.NopCloser(bytes.NewReader(payload)) + } + req, err := http.NewRequestWithContext(ctx, method, endpoint.String(), body) if err != nil { return 0, nil, nil, &attemptFailure{kind: "transport"} } + if mutation { + req.ContentLength = int64(len(payload)) + req.GetBody = nil + } if authenticated { req.Header.Set("Authorization", "Bearer "+c.token) } diff --git a/internal/api/client_test.go b/internal/api/client_test.go index 05978e4..784fd53 100644 --- a/internal/api/client_test.go +++ b/internal/api/client_test.go @@ -1,12 +1,14 @@ package api import ( + "bytes" "context" "errors" "io" "net/http" "net/http/httptest" "strings" + "sync" "sync/atomic" "testing" "time" @@ -14,6 +16,203 @@ import ( "github.com/ardasevinc/mattermost-cli/internal/presentation" ) +func TestPreparedMutationFreezesRequestBeforeDispatch(t *testing.T) { + body := map[string]string{"message": "before", "other": "\u2028"} + var gotMethod, gotURI, gotAuth, gotType string + var gotBody []byte + transport := roundTripFunc(func(request *http.Request) (*http.Response, error) { + if request.GetBody != nil || request.Body == nil || request.Body == http.NoBody { + t.Errorf("mutation body is replayable: GetBody=%v Body=%T", request.GetBody != nil, request.Body) + } + gotMethod, gotURI = request.Method, request.URL.RequestURI() + gotAuth, gotType = request.Header.Get("Authorization"), request.Header.Get("Content-Type") + gotBody, _ = io.ReadAll(request.Body) + return &http.Response{StatusCode: http.StatusCreated, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"id":"post-1"}`))}, nil + }) + c := newTestClient(t, "https://mattermost.example/base", WithRoundTripper(transport)) + prepared, err := c.PreparePost("/posts?set=pinned", body) + if err != nil { + t.Fatal(err) + } + body["message"] = "after" + var result struct { + ID string `json:"id"` + } + if err := prepared.Execute(context.Background(), &result); err != nil { + t.Fatal(err) + } + if gotMethod != http.MethodPost || gotURI != "/base/api/v4/posts?set=pinned" || gotAuth != "Bearer token-value" || gotType != "application/json" || result.ID != "post-1" { + t.Fatalf("request=%s %s auth=%q type=%q result=%+v", gotMethod, gotURI, gotAuth, gotType, result) + } + if !bytes.Equal(gotBody, []byte("{\"message\":\"before\",\"other\":\"\u2028\"}")) { + t.Fatalf("body = %q", gotBody) + } +} + +func TestPreparedEmptyDeleteRemainsNonReplayable(t *testing.T) { + var attempts atomic.Int32 + transport := roundTripFunc(func(request *http.Request) (*http.Response, error) { + attempts.Add(1) + if request.GetBody != nil || request.Body == nil || request.Body == http.NoBody || request.ContentLength != 0 { + t.Fatalf("replayable empty mutation: GetBody=%v Body=%T ContentLength=%d", request.GetBody != nil, request.Body, request.ContentLength) + } + body, err := io.ReadAll(request.Body) + if err != nil || len(body) != 0 { + t.Fatalf("body=%q error=%v", body, err) + } + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{}`))}, nil + }) + c := newTestClient(t, "https://mattermost.example", WithRoundTripper(transport)) + prepared, err := c.PrepareDelete("/posts/post-1") + if err != nil { + t.Fatal(err) + } + if err := prepared.Execute(context.Background(), &struct{}{}); err != nil { + t.Fatal(err) + } + if attempts.Load() != 1 { + t.Fatalf("attempts = %d", attempts.Load()) + } +} + +func TestPreparedMutationRejectsLocalFailuresBeforeDispatch(t *testing.T) { + transport := &countingTransport{err: errors.New("must not be called")} + c := newTestClient(t, "https://mattermost.example", WithRoundTripper(transport)) + if _, err := c.PreparePost("posts", map[string]string{"message": "hi"}); err == nil { + t.Fatal("invalid path prepared") + } + if _, err := c.PreparePost("/posts", map[string]any{"bad": make(chan int)}); err == nil || err.Error() != "unable to encode Mattermost request" { + t.Fatalf("encoding error = %v", err) + } + c.Close() + if _, err := c.PrepareDelete("/posts/post-1"); !errors.Is(err, ErrClientClosed) { + t.Fatalf("closed error = %v", err) + } + if transport.count.Load() != 0 { + t.Fatalf("attempts = %d", transport.count.Load()) + } +} + +func TestPreparedMutationIsOneShotAcrossConcurrentCallers(t *testing.T) { + var attempts atomic.Int32 + transport := roundTripFunc(func(*http.Request) (*http.Response, error) { + attempts.Add(1) + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{}`))}, nil + }) + c := newTestClient(t, "https://mattermost.example", WithRoundTripper(transport)) + prepared, err := c.PreparePut("/posts/post-1", map[string]string{"message": "hi"}) + if err != nil { + t.Fatal(err) + } + const callers = 16 + start := make(chan struct{}) + errorsSeen := make(chan error, callers) + var wait sync.WaitGroup + for range callers { + wait.Add(1) + go func() { + defer wait.Done() + <-start + errorsSeen <- prepared.Execute(context.Background(), &struct{}{}) + }() + } + close(start) + wait.Wait() + close(errorsSeen) + succeeded, consumed := 0, 0 + for err := range errorsSeen { + switch { + case err == nil: + succeeded++ + case errors.Is(err, ErrMutationUsed): + var unknown *OutcomeUnknownError + if !errors.As(err, &unknown) { + t.Fatalf("consumed mutation was not unknown: %v", err) + } + consumed++ + default: + t.Fatalf("unexpected error = %v", err) + } + } + if succeeded != 1 || consumed != callers-1 || attempts.Load() != 1 { + t.Fatalf("succeeded=%d consumed=%d attempts=%d", succeeded, consumed, attempts.Load()) + } +} + +func TestPreparedMutationClassifiesEveryPostIntentFailureConservatively(t *testing.T) { + for _, tc := range []struct { + name string + status int + body string + transportErr error + want4x bool + }{ + {"rejected", 409, `ignored`, nil, true}, + {"redirect", 307, `ignored`, nil, false}, + {"informational", 101, `ignored`, nil, false}, + {"server", 503, `ignored`, nil, false}, + {"network", 0, ``, errors.New("network failed"), false}, + {"empty success", 204, ``, nil, false}, + {"malformed success", 200, `{`, nil, false}, + } { + t.Run(tc.name, func(t *testing.T) { + var attempts atomic.Int32 + transport := roundTripFunc(func(*http.Request) (*http.Response, error) { + attempts.Add(1) + if tc.transportErr != nil { + return nil, tc.transportErr + } + return &http.Response{StatusCode: tc.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(tc.body))}, nil + }) + c := newTestClient(t, "https://mattermost.example", WithRoundTripper(transport)) + prepared, err := c.PrepareDelete("/posts/post-1") + if err != nil { + t.Fatal(err) + } + err = prepared.Execute(context.Background(), &struct{}{}) + var rejected *APIError + var unknown *OutcomeUnknownError + if tc.want4x { + if !errors.As(err, &rejected) || rejected.Status != tc.status { + t.Fatalf("error = %v", err) + } + } else if !errors.As(err, &unknown) { + t.Fatalf("error = %v", err) + } + if attempts.Load() != 1 { + t.Fatalf("attempts = %d", attempts.Load()) + } + }) + } +} + +func TestPreparedMutationFailureAfterPreparationIsUnknownAndConsumed(t *testing.T) { + transport := &countingTransport{err: errors.New("network failed")} + c := newTestClient(t, "https://mattermost.example", WithRoundTripper(transport)) + prepared, err := c.PreparePost("/posts", map[string]string{"message": "hi"}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + var unknown *OutcomeUnknownError + if err := prepared.Execute(ctx, &struct{}{}); !errors.As(err, &unknown) { + t.Fatalf("canceled error = %v", err) + } + if err := prepared.Execute(context.Background(), &struct{}{}); !errors.Is(err, ErrMutationUsed) || !errors.As(err, &unknown) { + t.Fatalf("replay error = %v", err) + } + + closed, err := c.PreparePost("/posts", map[string]string{"message": "next"}) + if err != nil { + t.Fatal(err) + } + c.Close() + if err := closed.Execute(context.Background(), &struct{}{}); !errors.As(err, &unknown) { + t.Fatalf("closed error = %v", err) + } +} + func TestReadAuthJSONAndRetryCounts(t *testing.T) { var attempts atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { From a38bb743f8e3702dce7ca7250c901a0c711d5bf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 06:11:02 +0300 Subject: [PATCH 076/119] fix: align delete receipts with Mattermost --- internal/cli/store_test.go | 6 +- internal/schema/apply_test.go | 45 +++++++ internal/stagestore/apply.go | 5 +- internal/stagestore/apply_test.go | 153 +++++++++++++++++++++- internal/stagestore/schema.go | 47 +++++++ schemas/v2/apply-receipt.schema.json | 5 +- schemas/v2/examples/store-doctor.json | 2 +- schemas/v2/examples/store-migrations.json | 2 +- schemas/v2/store-doctor.schema.json | 6 +- schemas/v2/store-migrations.schema.json | 6 +- 10 files changed, 260 insertions(+), 17 deletions(-) diff --git a/internal/cli/store_test.go b/internal/cli/store_test.go index 1341b22..f79fbac 100644 --- a/internal/cli/store_test.go +++ b/internal/cli/store_test.go @@ -36,7 +36,7 @@ func TestStoreDoctorAbsentIsReadOnlyAndSchemaValid(t *testing.T) { if err := registry.Validate("mm/v2/store-doctor", bytes.NewReader(stdout.Bytes())); err != nil { t.Fatalf("schema: %v\n%s", err, stdout.String()) } - for _, fact := range []string{`"filesystemSafe":null`, `"applicationId":null`, `"integrity":null`, `"applied":null`, `"latest":6`, `"valid":null`, `"journalMode":null`} { + for _, fact := range []string{`"filesystemSafe":null`, `"applicationId":null`, `"integrity":null`, `"applied":null`, `"latest":7`, `"valid":null`, `"journalMode":null`} { if !strings.Contains(stdout.String(), fact) { t.Fatalf("absent report omitted %s: %s", fact, stdout.String()) } @@ -110,7 +110,7 @@ func TestStoreMigrationsIsOfflineAndSchemaValid(t *testing.T) { if err := registry.Validate("mm/v2/store-migrations", bytes.NewReader(stdout.Bytes())); err != nil { t.Fatalf("schema: %v\n%s", err, stdout.String()) } - want := "{\"schema\":\"mm/v2/store-migrations\",\"latest\":6,\"migrations\":[{\"version\":1,\"name\":\"core-stage-state\",\"checksum\":\"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f\"},{\"version\":2,\"name\":\"immutable-local-request-receipts\",\"checksum\":\"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c\"},{\"version\":3,\"name\":\"caller-intent-stage-create-replay\",\"checksum\":\"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5\"},{\"version\":4,\"name\":\"caller-intent-stage-revise-replay\",\"checksum\":\"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4\"},{\"version\":5,\"name\":\"revision-plan-follows-composition\",\"checksum\":\"fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0\"},{\"version\":6,\"name\":\"durable-apply-journal\",\"checksum\":\"4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56\"}]}\n" + want := "{\"schema\":\"mm/v2/store-migrations\",\"latest\":7,\"migrations\":[{\"version\":1,\"name\":\"core-stage-state\",\"checksum\":\"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f\"},{\"version\":2,\"name\":\"immutable-local-request-receipts\",\"checksum\":\"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c\"},{\"version\":3,\"name\":\"caller-intent-stage-create-replay\",\"checksum\":\"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5\"},{\"version\":4,\"name\":\"caller-intent-stage-revise-replay\",\"checksum\":\"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4\"},{\"version\":5,\"name\":\"revision-plan-follows-composition\",\"checksum\":\"fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0\"},{\"version\":6,\"name\":\"durable-apply-journal\",\"checksum\":\"4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56\"},{\"version\":7,\"name\":\"status-confirmed-delete-results\",\"checksum\":\"bf980f24ff24fdf9d6a3a28b9ded97b51336fa1c8ef290d1bdcf91f1b593c5ce\"}]}\n" if stdout.String() != want { t.Fatalf("stdout does not match golden migration contract: %q", stdout.String()) } @@ -310,7 +310,7 @@ func TestStoreHumanOutputStatesBounds(t *testing.T) { t.Fatalf("stdout=%q", stdout.String()) } stdout.Reset() - if code := Execute(context.Background(), []string{"store", "migrations"}, strings.NewReader(""), &stdout, &stderr); code != 0 || !strings.Contains(stdout.String(), "latest: 6\n1 core-stage-state ") { + if code := Execute(context.Background(), []string{"store", "migrations"}, strings.NewReader(""), &stdout, &stderr); code != 0 || !strings.Contains(stdout.String(), "latest: 7\n1 core-stage-state ") { t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } } diff --git a/internal/schema/apply_test.go b/internal/schema/apply_test.go index d4a12b2..8fc0ad0 100644 --- a/internal/schema/apply_test.go +++ b/internal/schema/apply_test.go @@ -229,6 +229,38 @@ func TestApplyReceiptAcceptsRealisticEditProjection(t *testing.T) { } } +func TestApplyReceiptAcceptsStatusConfirmedDeleteProjection(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + raw, err := fs.ReadFile(publicschemas.FS, "v2/examples/apply-receipt.json") + if err != nil { + t.Fatal(err) + } + var doc map[string]any + if err := json.Unmarshal(raw, &doc); err != nil { + t.Fatal(err) + } + makeDeleteReceipt(doc) + encoded, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/apply-receipt", bytes.NewReader(encoded)); err != nil { + t.Fatalf("rejected status-confirmed delete receipt: %s", encoded) + } + step := doc["steps"].([]any)[0].(map[string]any) + step["result"].(map[string]any)["deleteAt"] = float64(1784253600000) + encoded, err = json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/apply-receipt", bytes.NewReader(encoded)); err == nil { + t.Fatalf("accepted invented delete timestamp: %s", encoded) + } +} + func makeEditReceipt(doc map[string]any, resultPostID string) { doc["operation"] = "edit_post" destination := doc["destination"].(map[string]any) @@ -241,3 +273,16 @@ func makeEditReceipt(doc map[string]any, resultPostID string) { destination["postState"] = map[string]any{"authorUserId": "user-1", "updateAt": float64(1784253599000), "contentDigest": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"} doc["steps"] = []any{map[string]any{"ordinal": float64(1), "kind": "edit_post", "condition": "always", "state": "response_validated", "result": map[string]any{"postId": resultPostID, "updateAt": float64(1784253600000)}, "startedAt": "2026-07-17T02:00:00Z", "endedAt": "2026-07-17T02:00:01Z"}} } + +func makeDeleteReceipt(doc map[string]any) { + doc["operation"] = "delete_post" + destination := doc["destination"].(map[string]any) + destination["kind"] = "post" + destination["channelType"] = "private" + destination["teamId"] = "team-1" + destination["participantIds"] = []any{} + destination["postId"] = "post-1" + destination["rootPostId"] = nil + destination["postState"] = map[string]any{"authorUserId": "user-1", "updateAt": float64(1784253599000), "contentDigest": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"} + doc["steps"] = []any{map[string]any{"ordinal": float64(1), "kind": "delete_post", "condition": "always", "state": "response_validated", "result": map[string]any{"postId": "post-1"}, "startedAt": "2026-07-17T02:00:00Z", "endedAt": "2026-07-17T02:00:01Z"}} +} diff --git a/internal/stagestore/apply.go b/internal/stagestore/apply.go index 4d8713a..c31c4a0 100644 --- a/internal/stagestore/apply.go +++ b/internal/stagestore/apply.go @@ -639,10 +639,9 @@ func canonicalStepResult(ctx context.Context, q queryer, attemptID string, ordin return marshalCanonical(result) case "delete_post": var result struct { - PostID string `json:"postId"` - DeleteAt int64 `json:"deleteAt"` + PostID string `json:"postId"` } - if decodeNarrow(canonical, &result) != nil || !validReceiptID(result.PostID) || !validRemoteTimestamp(result.DeleteAt) || destination.PostID == nil || result.PostID != *destination.PostID { + if decodeNarrow(canonical, &result) != nil || !validReceiptID(result.PostID) || destination.PostID == nil || result.PostID != *destination.PostID { return nil, ErrInvalid } return marshalCanonical(result) diff --git a/internal/stagestore/apply_test.go b/internal/stagestore/apply_test.go index 850668c..af31ec2 100644 --- a/internal/stagestore/apply_test.go +++ b/internal/stagestore/apply_test.go @@ -5,11 +5,13 @@ package stagestore import ( "context" "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "strings" "sync" "testing" + "time" ) func createApplyStage(t *testing.T, s *Store, plan string) CreateRecord { @@ -314,7 +316,7 @@ func TestApplyValidatedResultsBindTheClaimedRemoteEffect(t *testing.T) { result string }{ {"edit", EditPost, []byte("edited"), `{"kind":"post","postId":"target-post"}`, `{"steps":[{"ordinal":1,"type":"edit_post","condition":"always"}]}`, `{"postId":"other-post","updateAt":1784250000000}`}, - {"delete", DeletePost, nil, `{"kind":"post","postId":"target-post"}`, `{"steps":[{"ordinal":1,"type":"delete_post","condition":"always"}]}`, `{"postId":"other-post","deleteAt":1784250000000}`}, + {"delete", DeletePost, nil, `{"kind":"post","postId":"target-post"}`, `{"steps":[{"ordinal":1,"type":"delete_post","condition":"always"}]}`, `{"postId":"other-post"}`}, {"reaction", React, nil, `{"kind":"reaction","postId":"target-post"}`, `{"steps":[{"ordinal":1,"type":"add_reaction","condition":"if_missing"}]}`, `{"postId":"other-post"}`}, {"conversation", ResolveDM, nil, `{"kind":"conversation","channelId":null,"participantIds":["peer-1"]}`, `{"steps":[{"ordinal":1,"type":"resolve_conversation","condition":"if_missing"}]}`, `{"channelId":"channel-1","participantIds":["other-peer"]}`}, } { @@ -337,6 +339,155 @@ func TestApplyValidatedResultsBindTheClaimedRemoteEffect(t *testing.T) { } }) } + deleteStage, err := s.Create(context.Background(), CreateInput{RequestDigest: sha256.Sum256([]byte("valid-delete")), Operation: DeletePost, + ServerURL: "https://mattermost.example/api/v4", ServerID: "server-1", UserID: "user-1", + Content: RevisionContent{Destination: json.RawMessage(`{"kind":"post","postId":"target-post"}`), Plan: json.RawMessage(`{"steps":[{"ordinal":1,"type":"delete_post","condition":"always"}]}`)}}) + if err != nil { + t.Fatal(err) + } + deleteAttempt, err := s.ClaimApply(context.Background(), claimInput(deleteStage.Stage, "", RecoveryModeOrdinary)) + if err != nil { + t.Fatal(err) + } + if err = s.BeginDispatch(context.Background(), deleteAttempt.ID, 1); err != nil { + t.Fatal(err) + } + if err = s.MarkStepValidated(context.Background(), deleteAttempt.ID, 1, json.RawMessage(`{"postId":"target-post"}`)); err != nil { + t.Fatalf("valid delete result rejected: %v", err) + } +} + +func TestStatusConfirmedDeleteResultMigrationUpgradesVersionSixStore(t *testing.T) { + path := testPath(t) + original := migrations + migrations = append([]migration(nil), original[:6]...) + t.Cleanup(func() { migrations = original }) + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + createDelete := func(label string) CreateRecord { + t.Helper() + created, createErr := s.Create(context.Background(), CreateInput{RequestDigest: sha256.Sum256([]byte(label)), Operation: DeletePost, + ServerURL: "https://mattermost.example/api/v4", ServerID: "server-1", UserID: "user-1", + Content: RevisionContent{Destination: json.RawMessage(`{"kind":"post","postId":"target-post"}`), Plan: json.RawMessage(`{"steps":[{"ordinal":1,"type":"delete_post","condition":"always"}]}`)}}) + if createErr != nil { + t.Fatal(createErr) + } + return created + } + created := createDelete("migration-delete-success") + attempt, err := s.ClaimApply(context.Background(), claimInput(created.Stage, "", RecoveryModeOrdinary)) + if err != nil { + t.Fatal(err) + } + if err = s.BeginDispatch(context.Background(), attempt.ID, 1); err != nil { + t.Fatal(err) + } + var startedRaw string + if err = s.db.QueryRow(`SELECT started_at FROM apply_steps WHERE attempt_id=? AND ordinal=1`, attempt.ID).Scan(&startedRaw); err != nil { + t.Fatal(err) + } + started, err := parseTime(startedRaw) + if err != nil { + t.Fatal(err) + } + ended := time.Now().UTC() + legacyResult := json.RawMessage(`{"postId":"target-post","deleteAt":1784250000000}`) + legacyReceipt := ApplyReceipt{ + Schema: "mm/v2/apply-receipt", AttemptID: attempt.ID, StageID: attempt.StageID, Revision: attempt.Revision, + SemanticDigest: hex.EncodeToString(attempt.SemanticDigest[:]), Operation: DeletePost, RecoveryMode: RecoveryModeOrdinary, + Destination: created.Destination, Outcome: OutcomeSucceeded, Recovery: RecoveryForbidden, StartedAt: attempt.StartedAt, RecordedAt: ended, + Steps: []ApplyStep{{Ordinal: 1, Kind: "delete_post", Condition: "always", State: StepValidated, Result: legacyResult, StartedAt: &started, EndedAt: &ended}}, + } + legacyReceiptRaw, err := json.Marshal(legacyReceipt) + if err != nil { + t.Fatal(err) + } + tx, err := s.db.Begin() + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() + stamp := formatTime(ended) + if _, err = tx.Exec(`UPDATE apply_steps SET state='response_validated',result_json=?,ended_at=? WHERE attempt_id=? AND ordinal=1`, string(legacyResult), stamp, attempt.ID); err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(`INSERT INTO apply_events(attempt_id,ordinal,event,recorded_at) VALUES(?,1,'response_validated',?)`, attempt.ID, stamp); err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(`UPDATE apply_attempts SET outcome='succeeded',ended_at=? WHERE id=?`, stamp, attempt.ID); err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(`UPDATE stages SET lifecycle='completed',recovery='forbidden',claim_attempt_id=NULL,updated_at=? WHERE id=?`, stamp, attempt.StageID); err != nil { + t.Fatal(err) + } + if _, err = tx.Exec(`INSERT INTO apply_receipts(attempt_id,receipt_json,recorded_at) VALUES(?,?,?)`, attempt.ID, string(legacyReceiptRaw), stamp); err != nil { + t.Fatal(err) + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + rejectedStage := createDelete("migration-delete-rejected") + rejectedAttempt, err := s.ClaimApply(context.Background(), claimInput(rejectedStage.Stage, "", RecoveryModeOrdinary)) + if err != nil { + t.Fatal(err) + } + if err = s.BeginDispatch(context.Background(), rejectedAttempt.ID, 1); err != nil { + t.Fatal(err) + } + if err = s.MarkStepRejected(context.Background(), rejectedAttempt.ID, 1, json.RawMessage(`{"status":403}`)); err != nil { + t.Fatal(err) + } + if _, err = s.FinalizeApply(context.Background(), rejectedAttempt.ID); err != nil { + t.Fatal(err) + } + unknownStage := createDelete("migration-delete-unknown") + unknownAttempt, err := s.ClaimApply(context.Background(), claimInput(unknownStage.Stage, "", RecoveryModeOrdinary)) + if err != nil { + t.Fatal(err) + } + if err = s.BeginDispatch(context.Background(), unknownAttempt.ID, 1); err != nil { + t.Fatal(err) + } + if err = s.MarkStepUnknown(context.Background(), unknownAttempt.ID, 1); err != nil { + t.Fatal(err) + } + if _, err = s.FinalizeApply(context.Background(), unknownAttempt.ID); err != nil { + t.Fatal(err) + } + if err = s.Close(); err != nil { + t.Fatal(err) + } + + migrations = original + s, err = Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + replayed, err := s.FinalizeApply(context.Background(), attempt.ID) + if err != nil || !replayed.Replay || len(replayed.Steps) != 1 || string(replayed.Steps[0].Result) != `{"postId":"target-post"}` { + t.Fatalf("normalized replay = %+v / %v", replayed, err) + } + var storedReceipt string + if err = s.db.QueryRow(`SELECT receipt_json FROM apply_receipts WHERE attempt_id=?`, attempt.ID).Scan(&storedReceipt); err != nil || strings.Contains(storedReceipt, "deleteAt") { + t.Fatalf("stored legacy receipt = %q / %v", storedReceipt, err) + } + if _, err = s.db.Exec(`UPDATE apply_receipts SET receipt_json=receipt_json WHERE attempt_id=?`, attempt.ID); err == nil { + t.Fatal("receipt immutability trigger was not restored") + } + for _, preserved := range []struct { + name string + attemptID string + state StepState + result string + }{{"rejected", rejectedAttempt.ID, StepRejected, `{"status":403}`}, {"unknown", unknownAttempt.ID, StepUnknown, ""}} { + replay, replayErr := s.FinalizeApply(context.Background(), preserved.attemptID) + if replayErr != nil || !replay.Replay || len(replay.Steps) != 1 || replay.Steps[0].State != preserved.state || string(replay.Steps[0].Result) != preserved.result { + t.Fatalf("%s replay = %+v / %v", preserved.name, replay, replayErr) + } + } } func TestHistoricalUnknownReceiptReplaysAfterForcedSuccess(t *testing.T) { diff --git a/internal/stagestore/schema.go b/internal/stagestore/schema.go index cadefd3..4289d78 100644 --- a/internal/stagestore/schema.go +++ b/internal/stagestore/schema.go @@ -315,4 +315,51 @@ WHEN NEW.current_revision IS NOT OLD.current_revision AND NOT ( AND EXISTS(SELECT 1 FROM stage_revisions r WHERE r.stage_id=OLD.id AND r.revision=NEW.current_revision AND r.state='current') ) BEGIN SELECT RAISE(ABORT, 'invalid current stage revision transition'); END; +`}, {version: 7, name: "status-confirmed-delete-results", sql: ` +DROP TRIGGER apply_step_result_transition_valid; +DROP TRIGGER apply_step_state_transition_required; +DROP TRIGGER apply_receipts_immutable_update; +UPDATE apply_steps +SET result_json=json_object('postId',json_extract(result_json,'$.postId')) +WHERE state='response_validated' AND kind='delete_post'; +UPDATE apply_receipts +SET receipt_json=json_set(receipt_json,'$.steps[0].result',json_object('postId',json_extract(receipt_json,'$.steps[0].result.postId'))) +WHERE json_extract(receipt_json,'$.operation')='delete_post' + AND json_extract(receipt_json,'$.steps[0].kind')='delete_post' + AND json_extract(receipt_json,'$.steps[0].state')='response_validated' + AND json_type(receipt_json,'$.steps[0].result.deleteAt')='integer'; +CREATE TRIGGER apply_step_state_transition_required BEFORE UPDATE ON apply_steps +WHEN NEW.state IS OLD.state +BEGIN SELECT RAISE(ABORT, 'apply step history is immutable'); END; +CREATE TRIGGER apply_receipts_immutable_update BEFORE UPDATE ON apply_receipts BEGIN SELECT RAISE(ABORT, 'apply receipts are immutable'); END; +CREATE TRIGGER apply_step_result_transition_valid BEFORE UPDATE ON apply_steps +WHEN NEW.state IN ('response_validated','rejected','skipped') AND NOT EXISTS( + SELECT 1 FROM apply_attempts a JOIN stages s ON s.id=a.stage_id + JOIN stage_revisions r ON r.stage_id=a.stage_id AND r.revision=a.revision + WHERE a.id=NEW.attempt_id AND json_type(NEW.result_json)='object' AND ( + NEW.state='rejected' AND (SELECT count(*) FROM json_each(NEW.result_json))=1 + AND json_type(NEW.result_json,'$.status')='integer' AND json_extract(NEW.result_json,'$.status') BETWEEN 400 AND 499 + OR NEW.state='skipped' AND NEW.condition='if_missing' AND (SELECT count(*) FROM json_each(NEW.result_json))=1 + AND json_type(NEW.result_json,'$.reason')='text' AND json_extract(NEW.result_json,'$.reason')='already_satisfied' + OR NEW.state='response_validated' AND NEW.kind='upload_attachment' AND (SELECT count(*) FROM json_each(NEW.result_json))=1 + AND json_type(NEW.result_json,'$.fileId')='text' AND length(json_extract(NEW.result_json,'$.fileId'))>0 + OR NEW.state='response_validated' AND NEW.kind='create_post' AND (SELECT count(*) FROM json_each(NEW.result_json))=5 + AND json_type(NEW.result_json,'$.postId')='text' AND length(json_extract(NEW.result_json,'$.postId'))>0 + AND json_type(NEW.result_json,'$.createAt')='integer' AND json_extract(NEW.result_json,'$.createAt')>0 + AND json_extract(NEW.result_json,'$.channelId')=json_extract(r.destination_json,'$.channelId') + AND json_extract(NEW.result_json,'$.userId')=s.user_id AND json_extract(NEW.result_json,'$.pendingPostId')=a.pending_post_id + OR NEW.state='response_validated' AND NEW.kind='edit_post' AND (SELECT count(*) FROM json_each(NEW.result_json))=2 + AND json_extract(NEW.result_json,'$.postId')=json_extract(r.destination_json,'$.postId') + AND json_type(NEW.result_json,'$.updateAt')='integer' AND json_extract(NEW.result_json,'$.updateAt')>0 + OR NEW.state='response_validated' AND NEW.kind='delete_post' AND (SELECT count(*) FROM json_each(NEW.result_json))=1 + AND json_extract(NEW.result_json,'$.postId')=json_extract(r.destination_json,'$.postId') + OR NEW.state='response_validated' AND NEW.kind IN ('add_reaction','remove_reaction') AND (SELECT count(*) FROM json_each(NEW.result_json))=1 + AND json_extract(NEW.result_json,'$.postId')=json_extract(r.destination_json,'$.postId') + OR NEW.state='response_validated' AND NEW.kind='resolve_conversation' AND (SELECT count(*) FROM json_each(NEW.result_json))=2 + AND json_type(NEW.result_json,'$.channelId')='text' AND length(json_extract(NEW.result_json,'$.channelId'))>0 + AND (json_extract(r.destination_json,'$.channelId') IS NULL OR json_extract(NEW.result_json,'$.channelId')=json_extract(r.destination_json,'$.channelId')) + AND json_extract(NEW.result_json,'$.participantIds')=json_extract(r.destination_json,'$.participantIds') + ) +) +BEGIN SELECT RAISE(ABORT, 'invalid apply step result binding'); END; `}} diff --git a/schemas/v2/apply-receipt.schema.json b/schemas/v2/apply-receipt.schema.json index be42832..e90493f 100644 --- a/schemas/v2/apply-receipt.schema.json +++ b/schemas/v2/apply-receipt.schema.json @@ -74,7 +74,6 @@ "fileResult": { "type": "object", "additionalProperties": false, "required": ["fileId"], "properties": { "fileId": { "type": "string", "minLength": 1, "maxLength": 4096 } } }, "createResult": { "type": "object", "additionalProperties": false, "required": ["postId", "createAt", "channelId", "userId", "pendingPostId"], "properties": { "postId": { "type": "string", "minLength": 1, "maxLength": 4096 }, "createAt": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, "channelId": { "type": "string", "minLength": 1, "maxLength": 4096 }, "userId": { "type": "string", "minLength": 1, "maxLength": 4096 }, "pendingPostId": { "type": "string", "minLength": 1, "maxLength": 4096 } } }, "editResult": { "type": "object", "additionalProperties": false, "required": ["postId", "updateAt"], "properties": { "postId": { "type": "string", "minLength": 1, "maxLength": 4096 }, "updateAt": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 } } }, - "deleteResult": { "type": "object", "additionalProperties": false, "required": ["postId", "deleteAt"], "properties": { "postId": { "type": "string", "minLength": 1, "maxLength": 4096 }, "deleteAt": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 } } }, "postResult": { "type": "object", "additionalProperties": false, "required": ["postId"], "properties": { "postId": { "type": "string", "minLength": 1, "maxLength": 4096 } } }, "conversationResult": { "type": "object", "additionalProperties": false, "required": ["channelId", "participantIds"], "properties": { "channelId": { "type": "string", "minLength": 1, "maxLength": 4096 }, "participantIds": { "type": "array", "maxItems": 100, "uniqueItems": true, "items": { "type": "string", "minLength": 1, "maxLength": 4096 } } } }, "rejectedResult": { "type": "object", "additionalProperties": false, "required": ["status"], "properties": { "status": { "type": "integer", "minimum": 400, "maximum": 499 } } }, @@ -82,7 +81,7 @@ "result": { "anyOf": [ { "type": "null" }, - { "$ref": "#/$defs/fileResult" }, { "$ref": "#/$defs/createResult" }, { "$ref": "#/$defs/editResult" }, { "$ref": "#/$defs/deleteResult" }, + { "$ref": "#/$defs/fileResult" }, { "$ref": "#/$defs/createResult" }, { "$ref": "#/$defs/editResult" }, { "$ref": "#/$defs/postResult" }, { "$ref": "#/$defs/conversationResult" }, { "$ref": "#/$defs/rejectedResult" }, { "$ref": "#/$defs/skippedResult" } ] }, @@ -110,7 +109,7 @@ { "if": { "properties": { "state": { "const": "response_validated" }, "kind": { "const": "upload_attachment" } } }, "then": { "properties": { "result": { "$ref": "#/$defs/fileResult" } } } }, { "if": { "properties": { "state": { "const": "response_validated" }, "kind": { "const": "create_post" } } }, "then": { "properties": { "result": { "$ref": "#/$defs/createResult" } } } }, { "if": { "properties": { "state": { "const": "response_validated" }, "kind": { "const": "edit_post" } } }, "then": { "properties": { "result": { "$ref": "#/$defs/editResult" } } } }, - { "if": { "properties": { "state": { "const": "response_validated" }, "kind": { "const": "delete_post" } } }, "then": { "properties": { "result": { "$ref": "#/$defs/deleteResult" } } } }, + { "if": { "properties": { "state": { "const": "response_validated" }, "kind": { "const": "delete_post" } } }, "then": { "properties": { "result": { "$ref": "#/$defs/postResult" } } } }, { "if": { "properties": { "state": { "const": "response_validated" }, "kind": { "enum": ["add_reaction", "remove_reaction"] } } }, "then": { "properties": { "result": { "$ref": "#/$defs/postResult" } } } }, { "if": { "properties": { "state": { "const": "response_validated" }, "kind": { "const": "resolve_conversation" } } }, "then": { "properties": { "result": { "$ref": "#/$defs/conversationResult" } } } } ] diff --git a/schemas/v2/examples/store-doctor.json b/schemas/v2/examples/store-doctor.json index 38cbbef..ec62e46 100644 --- a/schemas/v2/examples/store-doctor.json +++ b/schemas/v2/examples/store-doctor.json @@ -1 +1 @@ -{"schema":"mm/v2/store-doctor","path":"/home/arda/.local/state/mattermost-cli/stages.sqlite3","report":{"exists":false,"filesystemSafe":null,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":6,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}} +{"schema":"mm/v2/store-doctor","path":"/home/arda/.local/state/mattermost-cli/stages.sqlite3","report":{"exists":false,"filesystemSafe":null,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":7,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}} diff --git a/schemas/v2/examples/store-migrations.json b/schemas/v2/examples/store-migrations.json index 5894032..88869d2 100644 --- a/schemas/v2/examples/store-migrations.json +++ b/schemas/v2/examples/store-migrations.json @@ -1 +1 @@ -{"schema":"mm/v2/store-migrations","latest":6,"migrations":[{"version":1,"name":"core-stage-state","checksum":"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"},{"version":2,"name":"immutable-local-request-receipts","checksum":"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"},{"version":3,"name":"caller-intent-stage-create-replay","checksum":"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5"},{"version":4,"name":"caller-intent-stage-revise-replay","checksum":"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4"},{"version":5,"name":"revision-plan-follows-composition","checksum":"fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0"},{"version":6,"name":"durable-apply-journal","checksum":"4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56"}]} +{"schema":"mm/v2/store-migrations","latest":7,"migrations":[{"version":1,"name":"core-stage-state","checksum":"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"},{"version":2,"name":"immutable-local-request-receipts","checksum":"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"},{"version":3,"name":"caller-intent-stage-create-replay","checksum":"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5"},{"version":4,"name":"caller-intent-stage-revise-replay","checksum":"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4"},{"version":5,"name":"revision-plan-follows-composition","checksum":"fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0"},{"version":6,"name":"durable-apply-journal","checksum":"4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56"},{"version":7,"name":"status-confirmed-delete-results","checksum":"bf980f24ff24fdf9d6a3a28b9ded97b51336fa1c8ef290d1bdcf91f1b593c5ce"}]} diff --git a/schemas/v2/store-doctor.schema.json b/schemas/v2/store-doctor.schema.json index 9a25a67..478fc9b 100644 --- a/schemas/v2/store-doctor.schema.json +++ b/schemas/v2/store-doctor.schema.json @@ -15,7 +15,7 @@ "exists": { "type": "boolean" }, "filesystemSafe": {}, "applicationId": {}, "integrity": {}, "integrityTruncated": {}, "foreignKeyIssues": {}, "foreignKeyRows": {}, "foreignKeyTruncated": {}, "journalMode": {}, "synchronous": {}, "secureDelete": {}, "foreignKeys": {}, "trustedSchema": {}, "queryOnly": {}, "walFallback": {}, - "migrations": { "type": "object", "additionalProperties": false, "required": ["applied", "latest", "valid"], "properties": { "applied": {}, "latest": { "const": 6 }, "valid": {} } }, + "migrations": { "type": "object", "additionalProperties": false, "required": ["applied", "latest", "valid"], "properties": { "applied": {}, "latest": { "const": 7 }, "valid": {} } }, "permissionModelLimitations": { "type": "array", "items": { "$ref": "#/$defs/safeString" } } }, "allOf": [ @@ -23,14 +23,14 @@ "filesystemSafe": { "const": null }, "applicationId": { "const": null }, "integrity": { "const": null }, "integrityTruncated": { "const": null }, "foreignKeyIssues": { "const": null }, "foreignKeyRows": { "const": null }, "foreignKeyTruncated": { "const": null }, "journalMode": { "const": null }, "synchronous": { "const": null }, "secureDelete": { "const": null }, "foreignKeys": { "const": null }, "trustedSchema": { "const": null }, "queryOnly": { "const": null }, "walFallback": { "const": null }, - "migrations": { "properties": { "applied": { "const": null }, "latest": { "const": 6 }, "valid": { "const": null } } } + "migrations": { "properties": { "applied": { "const": null }, "latest": { "const": 7 }, "valid": { "const": null } } } } } }, { "if": { "properties": { "exists": { "const": true } }, "required": ["exists"] }, "then": { "properties": { "filesystemSafe": { "const": true }, "applicationId": { "const": 1296913970 }, "integrity": { "$ref": "#/$defs/nonemptyBoundedStrings" }, "integrityTruncated": { "type": "boolean" }, "foreignKeyIssues": { "type": "integer", "minimum": 0, "maximum": 21 }, "foreignKeyRows": { "$ref": "#/$defs/boundedStrings" }, "foreignKeyTruncated": { "type": "boolean" }, "journalMode": { "enum": ["wal", "delete", "truncate", "persist"] }, "synchronous": { "type": "integer" }, "secureDelete": { "type": "integer" }, "foreignKeys": { "const": true }, "trustedSchema": { "const": false }, "queryOnly": { "const": true }, "walFallback": { "type": "boolean" }, - "migrations": { "properties": { "applied": { "const": 6 }, "latest": { "const": 6 }, "valid": { "const": true } } } + "migrations": { "properties": { "applied": { "const": 7 }, "latest": { "const": 7 }, "valid": { "const": true } } } }, "allOf": [ { "oneOf": [ { "properties": { "integrity": { "const": ["ok"] }, "integrityTruncated": { "const": false } } }, diff --git a/schemas/v2/store-migrations.schema.json b/schemas/v2/store-migrations.schema.json index 33704f4..2261899 100644 --- a/schemas/v2/store-migrations.schema.json +++ b/schemas/v2/store-migrations.schema.json @@ -6,9 +6,9 @@ "required": ["schema", "latest", "migrations"], "properties": { "schema": { "const": "mm/v2/store-migrations" }, - "latest": { "const": 6 }, + "latest": { "const": 7 }, "migrations": { - "type": "array", "minItems": 6, "maxItems": 6, + "type": "array", "minItems": 7, "maxItems": 7, "prefixItems": [{ "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { "version": { "const": 1 }, "name": { "const": "core-stage-state" }, "checksum": { "const": "e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f" } } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { @@ -21,6 +21,8 @@ "version": { "const": 5 }, "name": { "const": "revision-plan-follows-composition" }, "checksum": { "const": "fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0" } } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { "version": { "const": 6 }, "name": { "const": "durable-apply-journal" }, "checksum": { "const": "4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56" } + } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { + "version": { "const": 7 }, "name": { "const": "status-confirmed-delete-results" }, "checksum": { "const": "bf980f24ff24fdf9d6a3a28b9ded97b51336fa1c8ef290d1bdcf91f1b593c5ce" } } }], "items": false } From 1130379371ec780bc7f45568ee6b58d35964da20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 06:30:52 +0300 Subject: [PATCH 077/119] feat: add prepared post mutations --- internal/api/client.go | 39 +++-- internal/api/client_test.go | 16 ++ internal/mattermost/mutations.go | 235 ++++++++++++++++++++++++++ internal/mattermost/mutations_test.go | 221 ++++++++++++++++++++++++ 4 files changed, 501 insertions(+), 10 deletions(-) create mode 100644 internal/mattermost/mutations.go create mode 100644 internal/mattermost/mutations_test.go diff --git a/internal/api/client.go b/internal/api/client.go index cc5d5aa..3aae717 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -107,11 +107,12 @@ type Client struct { // transport, response, or cancellation failure is conservatively outcome // unknown; callers must never retry the same effect automatically. type PreparedMutation struct { - client *Client - method string - endpoint *url.URL - payload []byte - state *preparedMutationState + client *Client + method string + endpoint *url.URL + payload []byte + expectedStatus int + state *preparedMutationState } type preparedMutationState struct { @@ -177,23 +178,38 @@ func (c *Client) Delete(ctx context.Context, path string, out any) error { } func (c *Client) PreparePost(path string, body any) (*PreparedMutation, error) { - return c.prepareMutation(http.MethodPost, path, body) + return c.prepareMutation(http.MethodPost, path, body, 0) } func (c *Client) PreparePut(path string, body any) (*PreparedMutation, error) { - return c.prepareMutation(http.MethodPut, path, body) + return c.prepareMutation(http.MethodPut, path, body, 0) } func (c *Client) PrepareDelete(path string) (*PreparedMutation, error) { - return c.prepareMutation(http.MethodDelete, path, nil) + return c.prepareMutation(http.MethodDelete, path, nil, 0) } -func (c *Client) prepareMutation(method, path string, body any) (*PreparedMutation, error) { +func (c *Client) PreparePostStatus(path string, body any, expectedStatus int) (*PreparedMutation, error) { + return c.prepareMutation(http.MethodPost, path, body, expectedStatus) +} + +func (c *Client) PreparePutStatus(path string, body any, expectedStatus int) (*PreparedMutation, error) { + return c.prepareMutation(http.MethodPut, path, body, expectedStatus) +} + +func (c *Client) PrepareDeleteStatus(path string, expectedStatus int) (*PreparedMutation, error) { + return c.prepareMutation(http.MethodDelete, path, nil, expectedStatus) +} + +func (c *Client) prepareMutation(method, path string, body any, expectedStatus int) (*PreparedMutation, error) { c.lifecycle.RLock() defer c.lifecycle.RUnlock() if c.closed { return nil, ErrClientClosed } + if expectedStatus != 0 && (expectedStatus < 200 || expectedStatus >= 300) { + return nil, errors.New("invalid expected Mattermost status") + } endpoint, err := c.endpoint(path) if err != nil { return nil, err @@ -202,7 +218,7 @@ func (c *Client) prepareMutation(method, path string, body any) (*PreparedMutati if err != nil { return nil, errors.New("unable to encode Mattermost request") } - return &PreparedMutation{client: c, method: method, endpoint: endpoint, payload: bytes.Clone(payload), state: &preparedMutationState{}}, nil + return &PreparedMutation{client: c, method: method, endpoint: endpoint, payload: bytes.Clone(payload), expectedStatus: expectedStatus, state: &preparedMutationState{}}, nil } // Execute consumes the prepared mutation before inspecting cancellation or @@ -235,6 +251,9 @@ func (p *PreparedMutation) Execute(ctx context.Context, out any) error { if status < 200 || status >= 300 { return &OutcomeUnknownError{} } + if p.expectedStatus != 0 && status != p.expectedStatus { + return &OutcomeUnknownError{} + } decodeTarget := out if decodeTarget == nil { decodeTarget = new(any) diff --git a/internal/api/client_test.go b/internal/api/client_test.go index 784fd53..dd9f729 100644 --- a/internal/api/client_test.go +++ b/internal/api/client_test.go @@ -93,6 +93,22 @@ func TestPreparedMutationRejectsLocalFailuresBeforeDispatch(t *testing.T) { } } +func TestPreparedMutationFreezesExactExpectedStatus(t *testing.T) { + transport := &statusTransport{status: http.StatusOK} + c := newTestClient(t, "https://mattermost.example", WithRoundTripper(transport)) + prepared, err := c.PreparePostStatus("/posts", map[string]string{"message": "hi"}, http.StatusCreated) + if err != nil { + t.Fatal(err) + } + var unknown *OutcomeUnknownError + if err = prepared.Execute(context.Background(), new(map[string]any)); !errors.As(err, &unknown) { + t.Fatalf("unexpected status = %v", err) + } + if _, err = c.PrepareDeleteStatus("/posts/id", http.StatusBadRequest); err == nil { + t.Fatal("accepted non-success expected status") + } +} + func TestPreparedMutationIsOneShotAcrossConcurrentCallers(t *testing.T) { var attempts atomic.Int32 transport := roundTripFunc(func(*http.Request) (*http.Response, error) { diff --git a/internal/mattermost/mutations.go b/internal/mattermost/mutations.go new file mode 100644 index 0000000..3a4a794 --- /dev/null +++ b/internal/mattermost/mutations.go @@ -0,0 +1,235 @@ +package mattermost + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/url" + "slices" + "unicode/utf8" + + "github.com/ardasevinc/mattermost-cli/internal/api" +) + +const maxMutationMessageBytes = 65_535 + +var ErrInvalidMutationRequest = errors.New("invalid Mattermost mutation request") + +type PostMutations struct{ client *api.Client } + +func NewPostMutations(client *api.Client) *PostMutations { return &PostMutations{client: client} } + +type CreatePostMutationInput struct { + ChannelID string + UserID string + Message string + RootID string + FileIDs []string + PendingPostID string +} + +type EditPostMutationInput struct { + PostID string + ChannelID string + UserID string + Message string + RootID string + FileIDs []string +} + +type DeletePostMutationInput struct{ PostID string } + +type CreatePostMutationResult struct { + PostID string `json:"postId"` + CreateAt int64 `json:"createAt"` + ChannelID string `json:"channelId"` + UserID string `json:"userId"` + PendingPostID string `json:"pendingPostId"` +} + +type EditPostMutationResult struct { + PostID string `json:"postId"` + UpdateAt int64 `json:"updateAt"` +} + +type DeletePostMutationResult struct { + PostID string `json:"postId"` +} + +type PreparedCreatePost struct { + mutation *api.PreparedMutation + expected mutationPostExpectation +} + +type PreparedEditPost struct { + mutation *api.PreparedMutation + expected mutationPostExpectation +} + +type PreparedDeletePost struct { + mutation *api.PreparedMutation + postID string +} + +func (m *PostMutations) PrepareCreate(in CreatePostMutationInput) (*PreparedCreatePost, error) { + if m == nil || m.client == nil || !validMutationPostInput(in.ChannelID, in.UserID, in.Message, in.RootID, in.FileIDs) || !isSafePostID(in.PendingPostID) { + return nil, ErrInvalidMutationRequest + } + body := struct { + ChannelID string `json:"channel_id"` + Message string `json:"message"` + RootID string `json:"root_id,omitempty"` + FileIDs []string `json:"file_ids,omitempty"` + PendingPostID string `json:"pending_post_id"` + }{in.ChannelID, in.Message, in.RootID, slices.Clone(in.FileIDs), in.PendingPostID} + prepared, err := m.client.PreparePostStatus("/posts", body, http.StatusCreated) + if err != nil { + return nil, err + } + return &PreparedCreatePost{prepared, mutationPostExpectation{channelID: in.ChannelID, userID: in.UserID, message: in.Message, rootID: in.RootID, fileIDs: slices.Clone(in.FileIDs), pendingPostID: in.PendingPostID}}, nil +} + +func (m *PostMutations) PrepareEdit(in EditPostMutationInput) (*PreparedEditPost, error) { + if m == nil || m.client == nil || !isSafePostID(in.PostID) || !validMutationPostInput(in.ChannelID, in.UserID, in.Message, in.RootID, in.FileIDs) { + return nil, ErrInvalidMutationRequest + } + body := struct { + Message string `json:"message"` + }{in.Message} + prepared, err := m.client.PreparePutStatus("/posts/"+url.PathEscape(in.PostID)+"/patch", body, http.StatusOK) + if err != nil { + return nil, err + } + return &PreparedEditPost{prepared, mutationPostExpectation{postID: in.PostID, channelID: in.ChannelID, userID: in.UserID, message: in.Message, rootID: in.RootID, fileIDs: slices.Clone(in.FileIDs)}}, nil +} + +func (m *PostMutations) PrepareDelete(in DeletePostMutationInput) (*PreparedDeletePost, error) { + if m == nil || m.client == nil || !isSafePostID(in.PostID) { + return nil, ErrInvalidMutationRequest + } + prepared, err := m.client.PrepareDeleteStatus("/posts/"+url.PathEscape(in.PostID), http.StatusOK) + if err != nil { + return nil, err + } + return &PreparedDeletePost{prepared, in.PostID}, nil +} + +func (p *PreparedCreatePost) Execute(ctx context.Context) (CreatePostMutationResult, error) { + if p == nil || p.mutation == nil { + return CreatePostMutationResult{}, &api.OutcomeUnknownError{} + } + var response mutationPostResponse + if err := p.mutation.Execute(ctx, &response); err != nil { + return CreatePostMutationResult{}, err + } + if !response.matches(p.expected) || response.PendingPostID != p.expected.pendingPostID { + return CreatePostMutationResult{}, &api.OutcomeUnknownError{} + } + return CreatePostMutationResult{response.ID, response.CreateAt, response.ChannelID, response.UserID, response.PendingPostID}, nil +} + +func (p *PreparedEditPost) Execute(ctx context.Context) (EditPostMutationResult, error) { + if p == nil || p.mutation == nil { + return EditPostMutationResult{}, &api.OutcomeUnknownError{} + } + var response mutationPostResponse + if err := p.mutation.Execute(ctx, &response); err != nil { + return EditPostMutationResult{}, err + } + if !response.matches(p.expected) { + return EditPostMutationResult{}, &api.OutcomeUnknownError{} + } + return EditPostMutationResult{response.ID, response.UpdateAt}, nil +} + +func (p *PreparedDeletePost) Execute(ctx context.Context) (DeletePostMutationResult, error) { + if p == nil || p.mutation == nil || !isSafePostID(p.postID) { + return DeletePostMutationResult{}, &api.OutcomeUnknownError{} + } + if err := p.mutation.Execute(ctx, new(statusOKResponse)); err != nil { + return DeletePostMutationResult{}, err + } + return DeletePostMutationResult{p.postID}, nil +} + +type mutationPostExpectation struct { + postID, channelID, userID, message, rootID string + fileIDs []string + pendingPostID string +} + +type mutationPostResponse struct { + ID, ChannelID, UserID, Message, MessageSource, RootID, PendingPostID string + FileIDs []string + CreateAt, UpdateAt int64 +} + +func (p *mutationPostResponse) UnmarshalJSON(data []byte) error { + raw, ok := uniqueJSONObject(data) + if !ok { + return ErrInvalidPostResponse + } + id, idOK := safePostID(raw["id"]) + channelID, channelOK := safePostID(raw["channel_id"]) + userID, userOK := safePostID(raw["user_id"]) + message, messageOK := strictString(raw["message"]) + createAt, createOK := nonnegativeInteger(raw["create_at"]) + updateAt, updateOK := nonnegativeInteger(raw["update_at"]) + deleteAt, deleteOK := nonnegativeInteger(raw["delete_at"]) + rootID, rootOK := strictString(raw["root_id"]) + fileIDs, fileIDsOK := canonicalPostIDs(raw["file_ids"], 5) + pendingPostID, pendingOK := strictString(raw["pending_post_id"]) + postType, typeOK := strictString(raw["type"]) + messageSource := "" + if source, present := raw["message_source"]; present { + var sourceOK bool + messageSource, sourceOK = strictString(source) + if !sourceOK { + return ErrInvalidPostResponse + } + } + if !idOK || !channelOK || !userOK || !messageOK || !createOK || createAt == 0 || createAt > maxDateMilliseconds || + !updateOK || updateAt == 0 || updateAt > maxDateMilliseconds || updateAt < createAt || !deleteOK || deleteAt != 0 || + !rootOK || rootID != "" && !isSafePostID(rootID) || !fileIDsOK || !pendingOK || pendingPostID != "" && !isSafePostID(pendingPostID) || !typeOK || postType != "" { + return ErrInvalidPostResponse + } + *p = mutationPostResponse{id, channelID, userID, message, messageSource, rootID, pendingPostID, fileIDs, createAt, updateAt} + return nil +} + +func (p mutationPostResponse) matches(expected mutationPostExpectation) bool { + return (expected.postID == "" || p.ID == expected.postID) && p.ChannelID == expected.channelID && p.UserID == expected.userID && (p.Message == expected.message || p.MessageSource == expected.message) && + p.RootID == expected.rootID && slices.Equal(p.FileIDs, expected.fileIDs) +} + +type statusOKResponse struct{} + +func (*statusOKResponse) UnmarshalJSON(data []byte) error { + raw, ok := uniqueJSONObject(data) + status, statusOK := strictString(raw["status"]) + if !ok || len(raw) != 1 || !statusOK || status != "OK" { + return ErrInvalidPostResponse + } + return nil +} + +func validMutationPostInput(channelID, userID, message, rootID string, fileIDs []string) bool { + if !isSafePostID(channelID) || !isSafePostID(userID) || message == "" || len(message) > maxMutationMessageBytes || !utf8.ValidString(message) || utf8.RuneCountInString(message) > 16_383 || rootID != "" && !isSafePostID(rootID) || len(fileIDs) > 5 { + return false + } + seen := make(map[string]struct{}, len(fileIDs)) + for _, id := range fileIDs { + if !isSafePostID(id) { + return false + } + if _, exists := seen[id]; exists { + return false + } + seen[id] = struct{}{} + } + return true +} + +var _ json.Unmarshaler = (*mutationPostResponse)(nil) +var _ json.Unmarshaler = (*statusOKResponse)(nil) diff --git a/internal/mattermost/mutations_test.go b/internal/mattermost/mutations_test.go new file mode 100644 index 0000000..8d74c7a --- /dev/null +++ b/internal/mattermost/mutations_test.go @@ -0,0 +1,221 @@ +package mattermost + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "sync/atomic" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/api" +) + +type mutationRoundTripFunc func(*http.Request) (*http.Response, error) + +func (f mutationRoundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { + return f(request) +} + +func mutationClient(t *testing.T, transport http.RoundTripper) *api.Client { + t.Helper() + client, err := api.New("https://mattermost.example", "active-token", api.WithRoundTripper(transport)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(client.Close) + return client +} + +func mutationResponse(status int, body string) *http.Response { + return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body))} +} + +func postMutationJSON(id, channelID, userID, message, rootID, pendingID string, fileIDs []string, createAt, updateAt int64) string { + raw, _ := json.Marshal(map[string]any{ + "id": id, "channel_id": channelID, "user_id": userID, "message": message, "create_at": createAt, "update_at": updateAt, + "delete_at": 0, "root_id": rootID, "file_ids": fileIDs, "pending_post_id": pendingID, "type": "", + }) + return string(raw) +} + +func TestPreparedCreatePreservesShortAndLongMarkdownExactly(t *testing.T) { + for name, message := range map[string]string{ + "short": "# heading\n\n**bold** and `code`\n", + "long": strings.Repeat("界", 16_382) + "\n", + } { + t.Run(name, func(t *testing.T) { + var calls atomic.Int32 + transport := mutationRoundTripFunc(func(request *http.Request) (*http.Response, error) { + calls.Add(1) + if request.Method != http.MethodPost || request.URL.Path != "/api/v4/posts" || request.GetBody != nil { + t.Fatalf("request = %s %s getBody=%v", request.Method, request.URL.Path, request.GetBody != nil) + } + body, err := io.ReadAll(request.Body) + if err != nil { + t.Fatal(err) + } + var decoded struct { + ChannelID string `json:"channel_id"` + Message string `json:"message"` + RootID string `json:"root_id"` + FileIDs []string `json:"file_ids"` + PendingPostID string `json:"pending_post_id"` + } + if err = json.Unmarshal(body, &decoded); err != nil || decoded.Message != message || decoded.ChannelID != "channel-1" || decoded.RootID != "root-1" || !slicesEqual(decoded.FileIDs, []string{"file-1"}) || decoded.PendingPostID != "pending_1" { + t.Fatalf("body mismatch: %s / %v", body, err) + } + return mutationResponse(http.StatusCreated, postMutationJSON("post-1", "channel-1", "user-1", message, "root-1", "pending_1", []string{"file-1"}, 100, 100)), nil + }) + service := NewPostMutations(mutationClient(t, transport)) + prepared, err := service.PrepareCreate(CreatePostMutationInput{"channel-1", "user-1", message, "root-1", []string{"file-1"}, "pending_1"}) + if err != nil || calls.Load() != 0 { + t.Fatalf("prepare = %v calls=%d", err, calls.Load()) + } + result, err := prepared.Execute(context.Background()) + if err != nil || result != (CreatePostMutationResult{"post-1", 100, "channel-1", "user-1", "pending_1"}) || calls.Load() != 1 { + t.Fatalf("result=%+v err=%v calls=%d", result, err, calls.Load()) + } + }) + } +} + +func TestPreparedEditFreezesBodyAndValidatesBoundPost(t *testing.T) { + message := "edited **markdown**\n" + transport := mutationRoundTripFunc(func(request *http.Request) (*http.Response, error) { + body, _ := io.ReadAll(request.Body) + if request.Method != http.MethodPut || request.URL.Path != "/api/v4/posts/post-1/patch" || string(body) != `{"message":"edited **markdown**\n"}` { + t.Fatalf("request = %s %s %s", request.Method, request.URL.Path, body) + } + return mutationResponse(http.StatusOK, postMutationJSON("post-1", "channel-1", "user-1", message, "root-1", "", []string{"file-1"}, 100, 101)), nil + }) + prepared, err := NewPostMutations(mutationClient(t, transport)).PrepareEdit(EditPostMutationInput{"post-1", "channel-1", "user-1", message, "root-1", []string{"file-1"}}) + if err != nil { + t.Fatal(err) + } + result, err := prepared.Execute(context.Background()) + if err != nil || result != (EditPostMutationResult{"post-1", 101}) { + t.Fatalf("result=%+v err=%v", result, err) + } +} + +func TestPreparedEditRejectsDifferentResponsePost(t *testing.T) { + transport := mutationRoundTripFunc(func(*http.Request) (*http.Response, error) { + return mutationResponse(http.StatusOK, postMutationJSON("post-2", "channel-1", "user-1", "edited", "", "", nil, 100, 101)), nil + }) + prepared, err := NewPostMutations(mutationClient(t, transport)).PrepareEdit(EditPostMutationInput{"post-1", "channel-1", "user-1", "edited", "", nil}) + if err != nil { + t.Fatal(err) + } + if _, err = prepared.Execute(context.Background()); !outcomeUnknown(err) { + t.Fatalf("error = %v", err) + } +} + +func TestPreparedMutationsClassifyUnvalidatedSuccessAsUnknown(t *testing.T) { + valid := postMutationJSON("post-1", "channel-1", "user-1", "hello", "", "pending_1", nil, 100, 100) + for name, test := range map[string]struct { + status int + body string + }{ + "wrong status": {http.StatusOK, valid}, + "wrong channel": {http.StatusCreated, postMutationJSON("post-1", "other", "user-1", "hello", "", "pending_1", nil, 100, 100)}, + "wrong message": {http.StatusCreated, postMutationJSON("post-1", "channel-1", "user-1", "changed", "", "pending_1", nil, 100, 100)}, + "wrong pending id": {http.StatusCreated, postMutationJSON("post-1", "channel-1", "user-1", "hello", "", "other", nil, 100, 100)}, + "duplicate field": {http.StatusCreated, strings.Replace(valid, `"id":"post-1"`, `"id":"post-1","id":"post-2"`, 1)}, + } { + t.Run(name, func(t *testing.T) { + transport := mutationRoundTripFunc(func(*http.Request) (*http.Response, error) { return mutationResponse(test.status, test.body), nil }) + prepared, err := NewPostMutations(mutationClient(t, transport)).PrepareCreate(CreatePostMutationInput{"channel-1", "user-1", "hello", "", nil, "pending_1"}) + if err != nil { + t.Fatal(err) + } + if _, err = prepared.Execute(context.Background()); !outcomeUnknown(err) { + t.Fatalf("error = %v", err) + } + }) + } +} + +func TestPreparedDeleteRequiresExactStatusOK(t *testing.T) { + for name, test := range map[string]struct { + body string + unknown bool + }{"exact": {`{"status":"OK"}`, false}, "wrong case": {`{"status":"ok"}`, true}, "extra": {`{"status":"OK","post_id":"post-1"}`, true}} { + t.Run(name, func(t *testing.T) { + transport := mutationRoundTripFunc(func(request *http.Request) (*http.Response, error) { + if request.Method != http.MethodDelete || request.URL.Path != "/api/v4/posts/post-1" || request.GetBody != nil { + t.Fatalf("request = %s %s getBody=%v", request.Method, request.URL.Path, request.GetBody != nil) + } + return mutationResponse(http.StatusOK, test.body), nil + }) + prepared, err := NewPostMutations(mutationClient(t, transport)).PrepareDelete(DeletePostMutationInput{"post-1"}) + if err != nil { + t.Fatal(err) + } + result, err := prepared.Execute(context.Background()) + if test.unknown != outcomeUnknown(err) || !test.unknown && result.PostID != "post-1" { + t.Fatalf("result=%+v err=%v", result, err) + } + }) + } +} + +func TestPostMutationPreparationRejectsInvalidInputsWithoutDispatch(t *testing.T) { + var calls atomic.Int32 + transport := mutationRoundTripFunc(func(*http.Request) (*http.Response, error) { calls.Add(1); return nil, fmt.Errorf("unexpected") }) + service := NewPostMutations(mutationClient(t, transport)) + invalidUTF8 := string([]byte{0xff}) + for name, prepare := range map[string]func() error{ + "invalid channel": func() error { + _, err := service.PrepareCreate(CreatePostMutationInput{"bad/channel", "user-1", "hi", "", nil, "pending_1"}) + return err + }, + "invalid utf8": func() error { + _, err := service.PrepareCreate(CreatePostMutationInput{"channel-1", "user-1", invalidUTF8, "", nil, "pending_1"}) + return err + }, + "too many runes": func() error { + _, err := service.PrepareCreate(CreatePostMutationInput{"channel-1", "user-1", strings.Repeat("a", 16_384), "", nil, "pending_1"}) + return err + }, + "duplicate files": func() error { + _, err := service.PrepareEdit(EditPostMutationInput{"post-1", "channel-1", "user-1", "hi", "", []string{"file-1", "file-1"}}) + return err + }, + "too many files": func() error { + _, err := service.PrepareCreate(CreatePostMutationInput{"channel-1", "user-1", "hi", "", []string{"a", "b", "c", "d", "e", "f"}, "pending_1"}) + return err + }, + "invalid post": func() error { _, err := service.PrepareDelete(DeletePostMutationInput{"../post"}); return err }, + } { + t.Run(name, func(t *testing.T) { + if err := prepare(); !errors.Is(err, ErrInvalidMutationRequest) { + t.Fatalf("error = %v", err) + } + }) + } + if calls.Load() != 0 { + t.Fatalf("calls = %d", calls.Load()) + } +} + +func outcomeUnknown(err error) bool { + var unknown *api.OutcomeUnknownError + return errors.As(err, &unknown) +} + +func slicesEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} From 6c01eb9b537073c10146b81176d56ab8c1a431fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 06:40:19 +0300 Subject: [PATCH 078/119] feat: add prepared reaction mutations --- internal/mattermost/reaction_mutations.go | 112 ++++++++++++++++++ .../mattermost/reaction_mutations_test.go | 97 +++++++++++++++ 2 files changed, 209 insertions(+) create mode 100644 internal/mattermost/reaction_mutations.go create mode 100644 internal/mattermost/reaction_mutations_test.go diff --git a/internal/mattermost/reaction_mutations.go b/internal/mattermost/reaction_mutations.go new file mode 100644 index 0000000..7aea6bc --- /dev/null +++ b/internal/mattermost/reaction_mutations.go @@ -0,0 +1,112 @@ +package mattermost + +import ( + "context" + "net/http" + "net/url" + "strings" + + "github.com/ardasevinc/mattermost-cli/internal/api" +) + +type ReactionMutationInput struct { + PostID, ChannelID, UserID, Emoji string +} + +type ReactionMutationResult struct { + PostID string `json:"postId"` +} + +type PreparedAddReaction struct { + mutation *api.PreparedMutation + postID, channelID, userID, emoji string +} + +type PreparedRemoveReaction struct { + mutation *api.PreparedMutation + postID string +} + +func (m *PostMutations) PrepareAddReaction(in ReactionMutationInput) (*PreparedAddReaction, error) { + if !validReactionMutation(m, in) { + return nil, ErrInvalidMutationRequest + } + emoji := strings.ToLower(in.Emoji) + body := struct { + UserID string `json:"user_id"` + PostID string `json:"post_id"` + Emoji string `json:"emoji_name"` + }{in.UserID, in.PostID, emoji} + prepared, err := m.client.PreparePostStatus("/reactions", body, http.StatusOK) + if err != nil { + return nil, err + } + return &PreparedAddReaction{prepared, in.PostID, in.ChannelID, in.UserID, emoji}, nil +} + +func (m *PostMutations) PrepareRemoveReaction(in ReactionMutationInput) (*PreparedRemoveReaction, error) { + if !validReactionMutation(m, in) { + return nil, ErrInvalidMutationRequest + } + path := "/users/" + url.PathEscape(in.UserID) + "/posts/" + url.PathEscape(in.PostID) + "/reactions/" + url.PathEscape(strings.ToLower(in.Emoji)) + prepared, err := m.client.PrepareDeleteStatus(path, http.StatusOK) + if err != nil { + return nil, err + } + return &PreparedRemoveReaction{prepared, in.PostID}, nil +} + +func (p *PreparedAddReaction) Execute(ctx context.Context) (ReactionMutationResult, error) { + if p == nil || p.mutation == nil { + return ReactionMutationResult{}, &api.OutcomeUnknownError{} + } + var response reactionMutationResponse + if err := p.mutation.Execute(ctx, &response); err != nil { + return ReactionMutationResult{}, err + } + if response.PostID != p.postID || response.ChannelID != p.channelID || response.UserID != p.userID || response.Emoji != p.emoji { + return ReactionMutationResult{}, &api.OutcomeUnknownError{} + } + return ReactionMutationResult{p.postID}, nil +} + +func (p *PreparedRemoveReaction) Execute(ctx context.Context) (ReactionMutationResult, error) { + if p == nil || p.mutation == nil || !isSafePostID(p.postID) { + return ReactionMutationResult{}, &api.OutcomeUnknownError{} + } + if err := p.mutation.Execute(ctx, new(statusOKResponse)); err != nil { + return ReactionMutationResult{}, err + } + return ReactionMutationResult{p.postID}, nil +} + +type reactionMutationResponse struct { + UserID, PostID, ChannelID, Emoji string +} + +func (r *reactionMutationResponse) UnmarshalJSON(data []byte) error { + fields, ok := uniqueJSONObject(data) + if !ok { + return ErrInvalidReactionsResponse + } + userID, userOK := safePostID(fields["user_id"]) + postID, postOK := safePostID(fields["post_id"]) + channelID, channelOK := safePostID(fields["channel_id"]) + emoji, emojiOK := strictString(fields["emoji_name"]) + createAt, createOK := nonnegativeInteger(fields["create_at"]) + updateAt, updateOK := nonnegativeInteger(fields["update_at"]) + deleteAt, deleteOK := nonnegativeInteger(fields["delete_at"]) + remoteRaw, remotePresent := fields["remote_id"] + remoteID, remoteOK := strictString(remoteRaw) + remoteOK = remotePresent && !isJSONNull(remoteRaw) && remoteOK + if !userOK || !postOK || !channelOK || !emojiOK || !validEmojiName(emoji) || !createOK || createAt == 0 || createAt > maxDateMilliseconds || + !updateOK || updateAt == 0 || updateAt > maxDateMilliseconds || updateAt < createAt || !deleteOK || deleteAt != 0 || !remoteOK || remoteID != "" { + return ErrInvalidReactionsResponse + } + *r = reactionMutationResponse{userID, postID, channelID, emoji} + return nil +} + +func validReactionMutation(m *PostMutations, in ReactionMutationInput) bool { + return m != nil && m.client != nil && isSafePostID(in.PostID) && isSafePostID(in.ChannelID) && isSafePostID(in.UserID) && validEmojiName(in.Emoji) +} diff --git a/internal/mattermost/reaction_mutations_test.go b/internal/mattermost/reaction_mutations_test.go new file mode 100644 index 0000000..a36f624 --- /dev/null +++ b/internal/mattermost/reaction_mutations_test.go @@ -0,0 +1,97 @@ +package mattermost + +import ( + "context" + "io" + "net/http" + "strings" + "sync/atomic" + "testing" +) + +const validReactionResponse = `{"user_id":"user-1","post_id":"post-1","channel_id":"channel-1","emoji_name":"ship_it+1","create_at":100,"update_at":100,"delete_at":0,"remote_id":""}` + +func TestPreparedAddReactionBindsExactRequestAndResponse(t *testing.T) { + var calls atomic.Int32 + transport := mutationRoundTripFunc(func(request *http.Request) (*http.Response, error) { + calls.Add(1) + body, _ := io.ReadAll(request.Body) + if request.Method != http.MethodPost || request.URL.Path != "/api/v4/reactions" || string(body) != `{"user_id":"user-1","post_id":"post-1","emoji_name":"ship_it+1"}` { + t.Fatalf("request = %s %s %s", request.Method, request.URL.Path, body) + } + return mutationResponse(http.StatusOK, validReactionResponse), nil + }) + prepared, err := NewPostMutations(mutationClient(t, transport)).PrepareAddReaction(ReactionMutationInput{"post-1", "channel-1", "user-1", "Ship_It+1"}) + if err != nil || calls.Load() != 0 { + t.Fatalf("prepare=%v calls=%d", err, calls.Load()) + } + result, err := prepared.Execute(context.Background()) + if err != nil || result.PostID != "post-1" || calls.Load() != 1 { + t.Fatalf("result=%+v err=%v calls=%d", result, err, calls.Load()) + } +} + +func TestPreparedAddReactionRejectsUnboundSuccess(t *testing.T) { + for name, body := range map[string]string{ + "wrong user": strings.Replace(validReactionResponse, `"user_id":"user-1"`, `"user_id":"user-2"`, 1), + "wrong post": strings.Replace(validReactionResponse, `"post_id":"post-1"`, `"post_id":"post-2"`, 1), + "wrong channel": strings.Replace(validReactionResponse, `"channel_id":"channel-1"`, `"channel_id":"channel-2"`, 1), + "wrong emoji": strings.Replace(validReactionResponse, `"emoji_name":"ship_it+1"`, `"emoji_name":"other"`, 1), + "deleted": strings.Replace(validReactionResponse, `"delete_at":0`, `"delete_at":1`, 1), + "remote": strings.Replace(validReactionResponse, `"remote_id":""`, `"remote_id":"remote-1"`, 1), + "null remote": strings.Replace(validReactionResponse, `"remote_id":""`, `"remote_id":null`, 1), + "duplicate field": strings.Replace(validReactionResponse, `"user_id":"user-1"`, `"user_id":"user-1","user_id":"user-2"`, 1), + } { + t.Run(name, func(t *testing.T) { + transport := mutationRoundTripFunc(func(*http.Request) (*http.Response, error) { return mutationResponse(http.StatusOK, body), nil }) + prepared, err := NewPostMutations(mutationClient(t, transport)).PrepareAddReaction(ReactionMutationInput{"post-1", "channel-1", "user-1", "ship_it+1"}) + if err != nil { + t.Fatal(err) + } + if _, err = prepared.Execute(context.Background()); !outcomeUnknown(err) { + t.Fatalf("error = %v", err) + } + }) + } +} + +func TestPreparedRemoveReactionUsesExactPathAndStatus(t *testing.T) { + transport := mutationRoundTripFunc(func(request *http.Request) (*http.Response, error) { + if request.Method != http.MethodDelete || request.URL.Path != "/api/v4/users/user-1/posts/post-1/reactions/ship_it+1" || request.GetBody != nil { + t.Fatalf("request = %s %s getBody=%v", request.Method, request.URL.Path, request.GetBody != nil) + } + return mutationResponse(http.StatusOK, `{"status":"OK"}`), nil + }) + prepared, err := NewPostMutations(mutationClient(t, transport)).PrepareRemoveReaction(ReactionMutationInput{"post-1", "channel-1", "user-1", "Ship_It+1"}) + if err != nil { + t.Fatal(err) + } + result, err := prepared.Execute(context.Background()) + if err != nil || result.PostID != "post-1" { + t.Fatalf("result=%+v err=%v", result, err) + } +} + +func TestReactionPreparationRejectsInvalidIdentityWithoutDispatch(t *testing.T) { + var calls atomic.Int32 + transport := mutationRoundTripFunc(func(*http.Request) (*http.Response, error) { + calls.Add(1) + return mutationResponse(http.StatusOK, validReactionResponse), nil + }) + service := NewPostMutations(mutationClient(t, transport)) + for name, input := range map[string]ReactionMutationInput{ + "post": {"bad/post", "channel-1", "user-1", "ship_it"}, + "channel": {"post-1", "bad/channel", "user-1", "ship_it"}, + "user": {"post-1", "channel-1", "bad/user", "ship_it"}, + "emoji": {"post-1", "channel-1", "user-1", "not:emoji"}, + } { + t.Run(name, func(t *testing.T) { + if _, err := service.PrepareAddReaction(input); err != ErrInvalidMutationRequest { + t.Fatalf("error = %v", err) + } + }) + } + if calls.Load() != 0 { + t.Fatalf("calls = %d", calls.Load()) + } +} From 533162242782ab60c567a6763875af0c672cc2fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 06:47:40 +0300 Subject: [PATCH 079/119] fix: enforce Mattermost group limits --- internal/staging/conversation_stage.go | 2 +- internal/staging/conversation_stage_test.go | 23 +++++++++++++++++++++ schemas/v2/apply-receipt.schema.json | 4 ++-- schemas/v2/stage-preview.schema.json | 2 +- schemas/v2/stage-receipt.schema.json | 2 +- schemas/v2/stage-request.schema.json | 2 +- schemas/v2/stage.schema.json | 2 +- schemas/v2/stages.schema.json | 2 +- 8 files changed, 31 insertions(+), 8 deletions(-) diff --git a/internal/staging/conversation_stage.go b/internal/staging/conversation_stage.go index 8d378a5..4600f08 100644 --- a/internal/staging/conversation_stage.go +++ b/internal/staging/conversation_stage.go @@ -141,7 +141,7 @@ func (s *Service) resolveGroupFor(ctx context.Context, currentUserID string, use } func validGroupUsernames(usernames []string) bool { - if len(usernames) < 2 || len(usernames) > 100 { + if len(usernames) < 2 || len(usernames) > 7 { return false } seen := make(map[string]struct{}, len(usernames)) diff --git a/internal/staging/conversation_stage_test.go b/internal/staging/conversation_stage_test.go index d2aa397..6877b9e 100644 --- a/internal/staging/conversation_stage_test.go +++ b/internal/staging/conversation_stage_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "reflect" "strings" "sync/atomic" @@ -86,6 +87,28 @@ func TestResolveGroupCanonicalizesParticipantIDsAndRejectsAmbiguity(t *testing.T } } +func TestResolveGroupEnforcesMattermostPeerLimitsBeforeLookup(t *testing.T) { + users := &directoryUsers{current: mattermost.User{ID: "self", Username: "arda"}, byName: make(map[string]mattermost.User)} + seven := make([]string, 7) + for index := range seven { + username := fmt.Sprintf("peer-%d", index) + seven[index] = username + users.byName[username] = mattermost.User{ID: fmt.Sprintf("user-%d", index), Username: username} + } + service := conversationStageService(t, users, new(recordingStore), nil) + if _, err := service.DryRunResolveGroup(t.Context(), seven); err != nil { + t.Fatalf("seven peers rejected: %v", err) + } + before := users.calls.Load() + tooMany := append(append([]string(nil), seven...), "peer-7") + if _, err := service.DryRunResolveGroup(t.Context(), tooMany); !errors.Is(err, ErrInvalid) { + t.Fatalf("eight peers error=%v", err) + } + if got := users.calls.Load(); got != before { + t.Fatalf("invalid request performed lookups: %d -> %d", before, got) + } +} + func TestConversationCreateCredentialAndSelfTargetsFailClosed(t *testing.T) { users := &directoryUsers{current: mattermost.User{ID: "self", Username: "arda"}, byName: map[string]mattermost.User{ "arda": {ID: "self", Username: "arda"}, "token": {ID: "peer", Username: "active-token"}, diff --git a/schemas/v2/apply-receipt.schema.json b/schemas/v2/apply-receipt.schema.json index e90493f..2c19770 100644 --- a/schemas/v2/apply-receipt.schema.json +++ b/schemas/v2/apply-receipt.schema.json @@ -56,7 +56,7 @@ "teamId": { "anyOf": [{ "$ref": "#/$defs/id" }, { "type": "null" }] }, "postId": { "anyOf": [{ "$ref": "#/$defs/id" }, { "type": "null" }] }, "rootPostId": { "anyOf": [{ "$ref": "#/$defs/id" }, { "type": "null" }] }, - "participantIds": { "type": "array", "maxItems": 100, "uniqueItems": true, "items": { "$ref": "#/$defs/id" } }, + "participantIds": { "type": "array", "maxItems": 7, "uniqueItems": true, "items": { "$ref": "#/$defs/id" } }, "emoji": { "anyOf": [{ "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z0-9_+-]+$" }, { "type": "null" }] }, "postState": { "anyOf": [ @@ -75,7 +75,7 @@ "createResult": { "type": "object", "additionalProperties": false, "required": ["postId", "createAt", "channelId", "userId", "pendingPostId"], "properties": { "postId": { "type": "string", "minLength": 1, "maxLength": 4096 }, "createAt": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, "channelId": { "type": "string", "minLength": 1, "maxLength": 4096 }, "userId": { "type": "string", "minLength": 1, "maxLength": 4096 }, "pendingPostId": { "type": "string", "minLength": 1, "maxLength": 4096 } } }, "editResult": { "type": "object", "additionalProperties": false, "required": ["postId", "updateAt"], "properties": { "postId": { "type": "string", "minLength": 1, "maxLength": 4096 }, "updateAt": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 } } }, "postResult": { "type": "object", "additionalProperties": false, "required": ["postId"], "properties": { "postId": { "type": "string", "minLength": 1, "maxLength": 4096 } } }, - "conversationResult": { "type": "object", "additionalProperties": false, "required": ["channelId", "participantIds"], "properties": { "channelId": { "type": "string", "minLength": 1, "maxLength": 4096 }, "participantIds": { "type": "array", "maxItems": 100, "uniqueItems": true, "items": { "type": "string", "minLength": 1, "maxLength": 4096 } } } }, + "conversationResult": { "type": "object", "additionalProperties": false, "required": ["channelId", "participantIds"], "properties": { "channelId": { "type": "string", "minLength": 1, "maxLength": 4096 }, "participantIds": { "type": "array", "maxItems": 7, "uniqueItems": true, "items": { "type": "string", "minLength": 1, "maxLength": 4096 } } } }, "rejectedResult": { "type": "object", "additionalProperties": false, "required": ["status"], "properties": { "status": { "type": "integer", "minimum": 400, "maximum": 499 } } }, "skippedResult": { "type": "object", "additionalProperties": false, "required": ["reason"], "properties": { "reason": { "const": "already_satisfied" } } }, "result": { diff --git a/schemas/v2/stage-preview.schema.json b/schemas/v2/stage-preview.schema.json index e3a2d54..5ec0001 100644 --- a/schemas/v2/stage-preview.schema.json +++ b/schemas/v2/stage-preview.schema.json @@ -409,7 +409,7 @@ }, "participantIds": { "type": "array", - "maxItems": 100, + "maxItems": 7, "uniqueItems": true, "items": { "$ref": "#/$defs/id" diff --git a/schemas/v2/stage-receipt.schema.json b/schemas/v2/stage-receipt.schema.json index 08761db..bf44724 100644 --- a/schemas/v2/stage-receipt.schema.json +++ b/schemas/v2/stage-receipt.schema.json @@ -261,7 +261,7 @@ }, "participantIds": { "type": "array", - "maxItems": 100, + "maxItems": 7, "uniqueItems": true, "items": { "$ref": "#/$defs/id" diff --git a/schemas/v2/stage-request.schema.json b/schemas/v2/stage-request.schema.json index c069277..c259b9c 100644 --- a/schemas/v2/stage-request.schema.json +++ b/schemas/v2/stage-request.schema.json @@ -529,7 +529,7 @@ "usernames": { "type": "array", "minItems": 2, - "maxItems": 100, + "maxItems": 7, "uniqueItems": true, "items": { "$ref": "#/$defs/id" diff --git a/schemas/v2/stage.schema.json b/schemas/v2/stage.schema.json index 214435a..dfb84d0 100644 --- a/schemas/v2/stage.schema.json +++ b/schemas/v2/stage.schema.json @@ -582,7 +582,7 @@ }, "participantIds": { "type": "array", - "maxItems": 100, + "maxItems": 7, "uniqueItems": true, "items": { "$ref": "#/$defs/id" diff --git a/schemas/v2/stages.schema.json b/schemas/v2/stages.schema.json index 6cd8fa2..bec3dde 100644 --- a/schemas/v2/stages.schema.json +++ b/schemas/v2/stages.schema.json @@ -153,7 +153,7 @@ }, "participantIds": { "type": "array", - "maxItems": 100, + "maxItems": 7, "uniqueItems": true, "items": { "$ref": "#/$defs/id" From d3617b7deec50d5cc0670e2deecba8893687e46c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 06:47:47 +0300 Subject: [PATCH 080/119] test: cover Mattermost group limits --- internal/schema/stage_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/schema/stage_test.go b/internal/schema/stage_test.go index 76b91fe..ba9564b 100644 --- a/internal/schema/stage_test.go +++ b/internal/schema/stage_test.go @@ -54,6 +54,7 @@ func TestStageMachineSchemasRejectContradictionsAndLeaks(t *testing.T) { stageRequest(true, "r1", "resolve_dm", `{"kind":"user","username":"arda"}`, "null", `"wave"`, `[]`), stageRequest(true, "r1", "resolve_group_dm", `{"kind":"users","usernames":["a","a"]}`, "null", "null", `[]`), stageRequest(true, "r1", "resolve_group_dm", `{"kind":"users","usernames":["a"]}`, "null", "null", `[]`), + stageRequest(true, "r1", "resolve_group_dm", `{"kind":"users","usernames":["a","b","c","d","e","f","g","h"]}`, "null", "null", `[]`), stageRequest(true, "r1", "create_post", conversationTarget("dm", "username", "arda", "null"), `"hello"`, "null", `[{"path":"/tmp/a","remoteFilename":null,"mediaType":null,"contentDigest":"`+digest+`"}]`), }, "mm/v2/stages": { @@ -102,6 +103,7 @@ func TestStageRequestAcceptsEveryTargetBranch(t *testing.T) { stageRequest(true, "r9", "unreact", `{"kind":"post","postId":"p"}`, "null", `"wave"`, `[]`), stageRequest(true, "r10", "resolve_dm", `{"kind":"user","username":"arda"}`, "null", "null", `[]`), stageRequest(true, "r11", "resolve_group_dm", `{"kind":"users","usernames":["arda","hakan"]}`, "null", "null", `[]`), + stageRequest(true, "r12", "resolve_group_dm", `{"kind":"users","usernames":["a","b","c","d","e","f","g"]}`, "null", "null", `[]`), stageRequest(false, "null", "create_post", conversationTarget("channel", "id", "channel-id", "null"), "null", "null", `[]`), } for _, document := range valid { From dbbd6f8695ea264308795638590be05f6f2a4687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 06:53:27 +0300 Subject: [PATCH 081/119] feat: add prepared conversation mutations --- internal/mattermost/conversation_mutations.go | 111 ++++++++++++ .../mattermost/conversation_mutations_test.go | 166 ++++++++++++++++++ 2 files changed, 277 insertions(+) create mode 100644 internal/mattermost/conversation_mutations.go create mode 100644 internal/mattermost/conversation_mutations_test.go diff --git a/internal/mattermost/conversation_mutations.go b/internal/mattermost/conversation_mutations.go new file mode 100644 index 0000000..fd8fde3 --- /dev/null +++ b/internal/mattermost/conversation_mutations.go @@ -0,0 +1,111 @@ +package mattermost + +import ( + "context" + "crypto/sha1" // Mattermost defines group channel names as SHA-1 over sorted member IDs. + "encoding/hex" + "encoding/json" + "net/http" + "slices" + "strings" + + "github.com/ardasevinc/mattermost-cli/internal/api" +) + +type ConversationMutations struct{ client *api.Client } + +func NewConversationMutations(client *api.Client) *ConversationMutations { + return &ConversationMutations{client: client} +} + +type ResolveConversationMutationInput struct { + CurrentUserID string + ParticipantIDs []string +} + +type ResolveConversationMutationResult struct { + ChannelID string `json:"channelId"` + ParticipantIDs []string `json:"participantIds"` +} + +type PreparedResolveConversation struct { + mutation *api.PreparedMutation + channelType string + channelName string + participantIDs []string +} + +func (m *ConversationMutations) PrepareDirect(in ResolveConversationMutationInput) (*PreparedResolveConversation, error) { + return m.prepare(in, "D", 1, 1, "/channels/direct") +} + +func (m *ConversationMutations) PrepareGroup(in ResolveConversationMutationInput) (*PreparedResolveConversation, error) { + return m.prepare(in, "G", 2, 7, "/channels/group") +} + +func (m *ConversationMutations) prepare(in ResolveConversationMutationInput, channelType string, minimum, maximum int, path string) (*PreparedResolveConversation, error) { + if m == nil || m.client == nil || !isSafePostID(in.CurrentUserID) || len(in.ParticipantIDs) < minimum || len(in.ParticipantIDs) > maximum { + return nil, ErrInvalidMutationRequest + } + peers := slices.Clone(in.ParticipantIDs) + slices.Sort(peers) + for index, id := range peers { + if !isSafePostID(id) || id == in.CurrentUserID || index > 0 && id == peers[index-1] { + return nil, ErrInvalidMutationRequest + } + } + members := append(slices.Clone(peers), in.CurrentUserID) + slices.Sort(members) + name := strings.Join(members, "__") + if channelType == "G" { + digest := sha1.Sum([]byte(strings.Join(members, ""))) + name = hex.EncodeToString(digest[:]) + } + prepared, err := m.client.PreparePostStatus(path, members, http.StatusCreated) + if err != nil { + return nil, err + } + return &PreparedResolveConversation{prepared, channelType, name, peers}, nil +} + +func (p *PreparedResolveConversation) Execute(ctx context.Context) (ResolveConversationMutationResult, error) { + if p == nil || p.mutation == nil || (p.channelType != "D" && p.channelType != "G") || p.channelName == "" { + return ResolveConversationMutationResult{}, &api.OutcomeUnknownError{} + } + var response conversationMutationResponse + if err := p.mutation.Execute(ctx, &response); err != nil { + return ResolveConversationMutationResult{}, err + } + if response.Type != p.channelType || response.Name != p.channelName { + return ResolveConversationMutationResult{}, &api.OutcomeUnknownError{} + } + return ResolveConversationMutationResult{response.ID, slices.Clone(p.participantIDs)}, nil +} + +type conversationMutationResponse struct { + ID, Type, Name string +} + +func (c *conversationMutationResponse) UnmarshalJSON(data []byte) error { + fields, ok := uniqueJSONObject(data) + if !ok { + return ErrInvalidChannelResponse + } + id, idOK := safePostID(fields["id"]) + channelType, typeOK := strictString(fields["type"]) + name, nameOK := strictString(fields["name"]) + teamID, teamOK := strictString(fields["team_id"]) + displayName, displayOK := strictString(fields["display_name"]) + createAt, createOK := nonnegativeInteger(fields["create_at"]) + updateAt, updateOK := nonnegativeInteger(fields["update_at"]) + deleteAt, deleteOK := nonnegativeInteger(fields["delete_at"]) + if !idOK || !typeOK || channelType != "D" && channelType != "G" || !nameOK || name == "" || !teamOK || teamID != "" || !displayOK || + channelType == "D" && displayName != "" || !createOK || createAt == 0 || createAt > maxDateMilliseconds || + !updateOK || updateAt == 0 || updateAt > maxDateMilliseconds || updateAt < createAt || !deleteOK || deleteAt != 0 { + return ErrInvalidChannelResponse + } + *c = conversationMutationResponse{id, channelType, name} + return nil +} + +var _ json.Unmarshaler = (*conversationMutationResponse)(nil) diff --git a/internal/mattermost/conversation_mutations_test.go b/internal/mattermost/conversation_mutations_test.go new file mode 100644 index 0000000..fe7557e --- /dev/null +++ b/internal/mattermost/conversation_mutations_test.go @@ -0,0 +1,166 @@ +package mattermost + +import ( + "context" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "slices" + "strings" + "sync/atomic" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/api" +) + +func conversationMutationJSON(id, channelType, name, displayName string) string { + raw, _ := json.Marshal(map[string]any{ + "id": id, "create_at": 100, "update_at": 100, "delete_at": 0, + "team_id": "", "type": channelType, "name": name, "display_name": displayName, + }) + return string(raw) +} + +func TestPreparedDirectFreezesExactMembersAndBindsChannelIdentity(t *testing.T) { + var calls atomic.Int32 + peers := []string{"peer"} + transport := mutationRoundTripFunc(func(request *http.Request) (*http.Response, error) { + calls.Add(1) + body, _ := io.ReadAll(request.Body) + if request.Method != http.MethodPost || request.URL.Path != "/api/v4/channels/direct" || string(body) != `["peer","self"]` || request.GetBody != nil { + t.Fatalf("request = %s %s %s getBody=%v", request.Method, request.URL.Path, body, request.GetBody != nil) + } + return mutationResponse(http.StatusCreated, conversationMutationJSON("channel-1", "D", "peer__self", "")), nil + }) + prepared, err := NewConversationMutations(mutationClient(t, transport)).PrepareDirect(ResolveConversationMutationInput{"self", peers}) + if err != nil || calls.Load() != 0 { + t.Fatalf("prepare=%v calls=%d", err, calls.Load()) + } + peers[0] = "changed-after-prepare" + result, err := prepared.Execute(context.Background()) + if err != nil || result.ChannelID != "channel-1" || !slices.Equal(result.ParticipantIDs, []string{"peer"}) || calls.Load() != 1 { + t.Fatalf("result=%+v err=%v calls=%d", result, err, calls.Load()) + } +} + +func TestPreparedConversationPreservesKnownRejection(t *testing.T) { + transport := mutationRoundTripFunc(func(*http.Request) (*http.Response, error) { + return mutationResponse(http.StatusForbidden, `{"message":"not allowed"}`), nil + }) + prepared, err := NewConversationMutations(mutationClient(t, transport)).PrepareDirect(ResolveConversationMutationInput{"self", []string{"peer"}}) + if err != nil { + t.Fatal(err) + } + _, err = prepared.Execute(context.Background()) + var apiErr *api.APIError + if !errors.As(err, &apiErr) || apiErr.Status != http.StatusForbidden || outcomeUnknown(err) { + t.Fatalf("error = %v", err) + } +} + +func TestPreparedGroupAcceptsSevenPeersAndFreezesCanonicalMembership(t *testing.T) { + peers := []string{"u7", "u2", "u6", "u1", "u5", "u3", "u4"} + members := []string{"self", "u1", "u2", "u3", "u4", "u5", "u6", "u7"} + digest := sha1.Sum([]byte(strings.Join(members, ""))) + name := hex.EncodeToString(digest[:]) + transport := mutationRoundTripFunc(func(request *http.Request) (*http.Response, error) { + body, _ := io.ReadAll(request.Body) + if request.URL.Path != "/api/v4/channels/group" || string(body) != `["self","u1","u2","u3","u4","u5","u6","u7"]` { + t.Fatalf("request = %s %s", request.URL.Path, body) + } + return mutationResponse(http.StatusCreated, conversationMutationJSON("group-1", "G", name, "arda, peers")), nil + }) + prepared, err := NewConversationMutations(mutationClient(t, transport)).PrepareGroup(ResolveConversationMutationInput{"self", peers}) + if err != nil { + t.Fatal(err) + } + peers[0] = "changed-after-prepare" + result, err := prepared.Execute(context.Background()) + if err != nil || result.ChannelID != "group-1" || !slices.Equal(result.ParticipantIDs, members[1:]) { + t.Fatalf("result=%+v err=%v", result, err) + } +} + +func TestPreparedConversationRejectsUnboundSuccessAsUnknown(t *testing.T) { + valid := conversationMutationJSON("channel-1", "D", "peer__self", "") + for name, test := range map[string]struct { + status int + body string + }{ + "wrong status": {http.StatusOK, valid}, + "wrong type": {http.StatusCreated, conversationMutationJSON("channel-1", "G", "peer__self", "peers")}, + "wrong name": {http.StatusCreated, conversationMutationJSON("channel-1", "D", "other__self", "")}, + "nonempty team": {http.StatusCreated, strings.Replace(valid, `"team_id":""`, `"team_id":"team"`, 1)}, + "deleted": {http.StatusCreated, strings.Replace(valid, `"delete_at":0`, `"delete_at":1`, 1)}, + "duplicate id": {http.StatusCreated, strings.Replace(valid, `"id":"channel-1"`, `"id":"channel-1","id":"other"`, 1)}, + } { + t.Run(name, func(t *testing.T) { + transport := mutationRoundTripFunc(func(*http.Request) (*http.Response, error) { + return mutationResponse(test.status, test.body), nil + }) + prepared, err := NewConversationMutations(mutationClient(t, transport)).PrepareDirect(ResolveConversationMutationInput{"self", []string{"peer"}}) + if err != nil { + t.Fatal(err) + } + if _, err = prepared.Execute(context.Background()); !outcomeUnknown(err) { + t.Fatalf("error = %v", err) + } + }) + } +} + +func TestConversationPreparationRejectsInvalidMembershipWithoutDispatch(t *testing.T) { + var calls atomic.Int32 + transport := mutationRoundTripFunc(func(*http.Request) (*http.Response, error) { + calls.Add(1) + return nil, fmt.Errorf("unexpected") + }) + service := NewConversationMutations(mutationClient(t, transport)) + if _, err := service.PrepareGroup(ResolveConversationMutationInput{"self", []string{"a", "b"}}); err != nil { + t.Fatalf("two peers rejected: %v", err) + } + eight := []string{"a", "b", "c", "d", "e", "f", "g", "h"} + for name, prepare := range map[string]func() error{ + "direct missing peer": func() error { + _, err := service.PrepareDirect(ResolveConversationMutationInput{"self", nil}) + return err + }, + "direct extra peer": func() error { + _, err := service.PrepareDirect(ResolveConversationMutationInput{"self", []string{"a", "b"}}) + return err + }, + "group missing peer": func() error { + _, err := service.PrepareGroup(ResolveConversationMutationInput{"self", []string{"a"}}) + return err + }, + "group too many peers": func() error { + _, err := service.PrepareGroup(ResolveConversationMutationInput{"self", eight}) + return err + }, + "self peer": func() error { + _, err := service.PrepareDirect(ResolveConversationMutationInput{"self", []string{"self"}}) + return err + }, + "duplicate peer": func() error { + _, err := service.PrepareGroup(ResolveConversationMutationInput{"self", []string{"a", "a"}}) + return err + }, + "unsafe peer": func() error { + _, err := service.PrepareDirect(ResolveConversationMutationInput{"self", []string{"../peer"}}) + return err + }, + } { + t.Run(name, func(t *testing.T) { + if err := prepare(); !errors.Is(err, ErrInvalidMutationRequest) { + t.Fatalf("error = %v", err) + } + }) + } + if calls.Load() != 0 { + t.Fatalf("calls = %d", calls.Load()) + } +} From e9a065bae72054d0a4c228024f686cd13a8016e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 07:02:06 +0300 Subject: [PATCH 082/119] fix: verify live group membership --- internal/mattermost/channels.go | 95 +++++++++++++++++-- internal/mattermost/channels_test.go | 51 ++++++++++ internal/mattermost/conversation_mutations.go | 10 +- 3 files changed, 147 insertions(+), 9 deletions(-) diff --git a/internal/mattermost/channels.go b/internal/mattermost/channels.go index 0243768..c25f2ed 100644 --- a/internal/mattermost/channels.go +++ b/internal/mattermost/channels.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "net/url" + "slices" "sort" "strings" ) @@ -101,16 +102,24 @@ type ChannelMember struct { UserID string } -func (m *ChannelMember) UnmarshalJSON(data []byte) error { - var raw struct { - ChannelID json.RawMessage `json:"channel_id"` - UserID json.RawMessage `json:"user_id"` +type groupMemberList []ChannelMember + +func (l *groupMemberList) UnmarshalJSON(data []byte) error { + var members []ChannelMember + if err := json.Unmarshal(data, &members); err != nil || members == nil || len(members) > 8 { + return ErrInvalidChannelsResponse } - if err := json.Unmarshal(data, &raw); err != nil { + *l = members + return nil +} + +func (m *ChannelMember) UnmarshalJSON(data []byte) error { + fields, ok := uniqueJSONObject(data) + if !ok { return ErrInvalidChannelResponse } - channelID, channelOK := requiredString(raw.ChannelID) - userID, userOK := requiredString(raw.UserID) + channelID, channelOK := requiredString(fields["channel_id"]) + userID, userOK := requiredString(fields["user_id"]) if !channelOK || !userOK { return ErrInvalidChannelResponse } @@ -529,6 +538,78 @@ func (s *Channels) ExistingDirect(ctx context.Context, currentUserID, peerID str return match, found, nil } +func (s *Channels) exactGroupMemberIDs(ctx context.Context, channelID string) ([]string, error) { + if !isSafePostID(channelID) { + return nil, ErrInvalidChannelRequest + } + var decoded groupMemberList + path := "/channels/" + url.PathEscape(channelID) + "/members?page=0&per_page=9" + if err := s.client.Get(ctx, path, &decoded); err != nil { + return nil, err + } + members := []ChannelMember(decoded) + ids := make([]string, 0, len(members)) + seen := make(map[string]struct{}, len(members)) + for _, member := range members { + if member.ChannelID != channelID || !isSafePostID(member.UserID) { + return nil, ErrInvalidChannelsResponse + } + if _, duplicate := seen[member.UserID]; duplicate { + return nil, ErrInvalidChannelsResponse + } + seen[member.UserID] = struct{}{} + ids = append(ids, member.UserID) + } + slices.Sort(ids) + return ids, nil +} + +// ExistingGroup finds the unique current-user-bound G channel for an exact +// peer set. Mattermost defines the channel name as SHA-1 over the sorted full +// member IDs, so the canonical listing binds both membership and identity +// without an unbounded per-channel member fan-out. +func (s *Channels) ExistingGroup(ctx context.Context, currentUserID string, peerIDs []string) (Channel, bool, error) { + if !isSafePostID(currentUserID) || len(peerIDs) < 2 || len(peerIDs) > 7 { + return Channel{}, false, ErrInvalidChannelRequest + } + peers := slices.Clone(peerIDs) + slices.Sort(peers) + for index, id := range peers { + if !isSafePostID(id) || id == currentUserID || index > 0 && id == peers[index-1] { + return Channel{}, false, ErrInvalidChannelRequest + } + } + wanted := canonicalGroupChannelName(append(peers, currentUserID)) + channels, err := s.GroupList(ctx, currentUserID) + if err != nil { + return Channel{}, false, err + } + var match Channel + found := false + for _, channel := range channels { + if channel.Name != wanted { + continue + } + if !isSafePostID(channel.ID) || found { + return Channel{}, false, ErrInvalidChannelsResponse + } + match, found = channel, true + } + if !found { + return Channel{}, false, nil + } + members, err := s.exactGroupMemberIDs(ctx, match.ID) + if err != nil { + return Channel{}, false, err + } + expected := append(slices.Clone(peers), currentUserID) + slices.Sort(expected) + if !slices.Equal(members, expected) { + return Channel{}, false, ErrInvalidChannelsResponse + } + return match, found, nil +} + // GroupList returns only G channels from the canonical current-user channel // listing. That authenticated, same-session endpoint is itself the membership // proof for discovered channels; per-channel Member calls would turn one diff --git a/internal/mattermost/channels_test.go b/internal/mattermost/channels_test.go index d329ecf..fc53ef1 100644 --- a/internal/mattermost/channels_test.go +++ b/internal/mattermost/channels_test.go @@ -508,6 +508,57 @@ func TestGroupListUsesCanonicalListingAsBoundedMembershipProof(t *testing.T) { } } +func TestExistingGroupFindsExactCanonicalPeerSet(t *testing.T) { + wanted := canonicalGroupChannelName([]string{"self", "a", "b"}) + f := &fakeChannelTransport{responses: map[string]string{ + "/users/self/channels": `[{"id":"other","team_id":"","type":"G","name":"opaque","display_name":"Other"},{"id":"wanted","team_id":"","type":"G","name":"` + wanted + `","display_name":"A, B"}]`, + "/channels/wanted/members?page=0&per_page=9": `[{"channel_id":"wanted","user_id":"b"},{"channel_id":"wanted","user_id":"self"},{"channel_id":"wanted","user_id":"a"}]`, + }} + got, found, err := NewChannels(f).ExistingGroup(context.Background(), "self", []string{"b", "a"}) + if err != nil || !found || got.ID != "wanted" { + t.Fatalf("channel=%#v found=%v error=%v", got, found, err) + } + if !reflect.DeepEqual(f.paths, []string{"/users/self/channels", "/channels/wanted/members?page=0&per_page=9"}) { + t.Fatalf("paths=%v", f.paths) + } +} + +func TestExistingGroupRejectsStaleNameWithPartialLiveMembership(t *testing.T) { + wanted := canonicalGroupChannelName([]string{"self", "a", "b"}) + f := &fakeChannelTransport{responses: map[string]string{ + "/users/self/channels": `[{"id":"wanted","team_id":"","type":"G","name":"` + wanted + `","display_name":"A, B"}]`, + "/channels/wanted/members?page=0&per_page=9": `[{"channel_id":"wanted","user_id":"self"},{"channel_id":"wanted","user_id":"a"}]`, + }} + if _, _, err := NewChannels(f).ExistingGroup(context.Background(), "self", []string{"a", "b"}); !errors.Is(err, ErrInvalidChannelsResponse) { + t.Fatalf("error=%v", err) + } +} + +func TestExistingGroupReturnsNoneAndRejectsAmbiguousOrInvalidSets(t *testing.T) { + wanted := canonicalGroupChannelName([]string{"self", "a", "b"}) + for name, payload := range map[string]string{ + "none": `[{"id":"other","team_id":"","type":"G","name":"opaque","display_name":"Other"}]`, + "ambiguous": `[{"id":"one","team_id":"","type":"G","name":"` + wanted + `","display_name":"A"},{"id":"two","team_id":"","type":"G","name":"` + wanted + `","display_name":"B"}]`, + } { + t.Run(name, func(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{"/users/self/channels": payload}} + got, found, err := NewChannels(f).ExistingGroup(context.Background(), "self", []string{"a", "b"}) + if name == "none" && (err != nil || found || got != (Channel{})) { + t.Fatalf("channel=%#v found=%v error=%v", got, found, err) + } + if name == "ambiguous" && !errors.Is(err, ErrInvalidChannelsResponse) { + t.Fatalf("error=%v", err) + } + }) + } + for _, peers := range [][]string{{"a"}, {"a", "a"}, {"self", "a"}, {"a", "../b"}, {"a", "b", "c", "d", "e", "f", "g", "h"}} { + f := &fakeChannelTransport{} + if _, _, err := NewChannels(f).ExistingGroup(context.Background(), "self", peers); !errors.Is(err, ErrInvalidChannelRequest) || len(f.paths) != 0 { + t.Fatalf("peers=%q error=%v paths=%v", peers, err, f.paths) + } + } +} + func TestGroupListRejectsMalformedFocusedChannelsAndMembership(t *testing.T) { for name, payload := range map[string]string{ "malformed group": `[{"id":"group","team_id":"","type":"G","display_name":"Crew"}]`, diff --git a/internal/mattermost/conversation_mutations.go b/internal/mattermost/conversation_mutations.go index fd8fde3..b23807c 100644 --- a/internal/mattermost/conversation_mutations.go +++ b/internal/mattermost/conversation_mutations.go @@ -58,8 +58,7 @@ func (m *ConversationMutations) prepare(in ResolveConversationMutationInput, cha slices.Sort(members) name := strings.Join(members, "__") if channelType == "G" { - digest := sha1.Sum([]byte(strings.Join(members, ""))) - name = hex.EncodeToString(digest[:]) + name = canonicalGroupChannelName(members) } prepared, err := m.client.PreparePostStatus(path, members, http.StatusCreated) if err != nil { @@ -68,6 +67,13 @@ func (m *ConversationMutations) prepare(in ResolveConversationMutationInput, cha return &PreparedResolveConversation{prepared, channelType, name, peers}, nil } +func canonicalGroupChannelName(memberIDs []string) string { + members := slices.Clone(memberIDs) + slices.Sort(members) + digest := sha1.Sum([]byte(strings.Join(members, ""))) + return hex.EncodeToString(digest[:]) +} + func (p *PreparedResolveConversation) Execute(ctx context.Context) (ResolveConversationMutationResult, error) { if p == nil || p.mutation == nil || (p.channelType != "D" && p.channelType != "G") || p.channelName == "" { return ResolveConversationMutationResult{}, &api.OutcomeUnknownError{} From 69c1d7140433bf960837f793caec7eb22d76697d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 07:05:17 +0300 Subject: [PATCH 083/119] fix: verify live direct membership --- internal/mattermost/channels.go | 25 ++++++++++++++++++++----- internal/mattermost/channels_test.go | 22 +++++++++++++++++++--- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/internal/mattermost/channels.go b/internal/mattermost/channels.go index c25f2ed..38ac111 100644 --- a/internal/mattermost/channels.go +++ b/internal/mattermost/channels.go @@ -102,9 +102,9 @@ type ChannelMember struct { UserID string } -type groupMemberList []ChannelMember +type conversationMemberList []ChannelMember -func (l *groupMemberList) UnmarshalJSON(data []byte) error { +func (l *conversationMemberList) UnmarshalJSON(data []byte) error { var members []ChannelMember if err := json.Unmarshal(data, &members); err != nil || members == nil || len(members) > 8 { return ErrInvalidChannelsResponse @@ -535,14 +535,29 @@ func (s *Channels) ExistingDirect(ctx context.Context, currentUserID, peerID str } match, found = channel, true } + if !found { + return Channel{}, false, nil + } + members, err := s.exactConversationMemberIDs(ctx, match.ID) + if err != nil { + return Channel{}, false, err + } + expected := []string{currentUserID} + if peerID != currentUserID { + expected = append(expected, peerID) + } + slices.Sort(expected) + if !slices.Equal(members, expected) { + return Channel{}, false, ErrInvalidChannelsResponse + } return match, found, nil } -func (s *Channels) exactGroupMemberIDs(ctx context.Context, channelID string) ([]string, error) { +func (s *Channels) exactConversationMemberIDs(ctx context.Context, channelID string) ([]string, error) { if !isSafePostID(channelID) { return nil, ErrInvalidChannelRequest } - var decoded groupMemberList + var decoded conversationMemberList path := "/channels/" + url.PathEscape(channelID) + "/members?page=0&per_page=9" if err := s.client.Get(ctx, path, &decoded); err != nil { return nil, err @@ -598,7 +613,7 @@ func (s *Channels) ExistingGroup(ctx context.Context, currentUserID string, peer if !found { return Channel{}, false, nil } - members, err := s.exactGroupMemberIDs(ctx, match.ID) + members, err := s.exactConversationMemberIDs(ctx, match.ID) if err != nil { return Channel{}, false, err } diff --git a/internal/mattermost/channels_test.go b/internal/mattermost/channels_test.go index fc53ef1..c621e54 100644 --- a/internal/mattermost/channels_test.go +++ b/internal/mattermost/channels_test.go @@ -439,12 +439,15 @@ func TestExistingDirectFindsExactPairInEitherOrder(t *testing.T) { for _, name := range []string{"user__peer", "peer__user"} { t.Run(name, func(t *testing.T) { payload := `[{"id":"other","team_id":"","type":"D","name":"user__someone","display_name":""},{"id":"wanted","team_id":"","type":"D","name":"` + name + `","display_name":""}]` - f := &fakeChannelTransport{responses: map[string]string{"/users/user/channels": payload}} + f := &fakeChannelTransport{responses: map[string]string{ + "/users/user/channels": payload, + "/channels/wanted/members?page=0&per_page=9": `[{"channel_id":"wanted","user_id":"peer"},{"channel_id":"wanted","user_id":"user"}]`, + }} got, found, err := NewChannels(f).ExistingDirect(context.Background(), "user", "peer") if err != nil || !found || got.ID != "wanted" { t.Fatalf("channel=%#v found=%v error=%v", got, found, err) } - if !reflect.DeepEqual(f.paths, []string{"/users/user/channels"}) { + if !reflect.DeepEqual(f.paths, []string{"/users/user/channels", "/channels/wanted/members?page=0&per_page=9"}) { t.Fatalf("paths=%v", f.paths) } }) @@ -460,13 +463,26 @@ func TestExistingDirectReturnsCleanNone(t *testing.T) { } func TestExistingDirectPreservesCanonicalSelfDM(t *testing.T) { - f := &fakeChannelTransport{responses: map[string]string{"/users/user/channels": `[{"id":"self","team_id":"","type":"D","name":"user__user","display_name":""}]`}} + f := &fakeChannelTransport{responses: map[string]string{ + "/users/user/channels": `[{"id":"self","team_id":"","type":"D","name":"user__user","display_name":""}]`, + "/channels/self/members?page=0&per_page=9": `[{"channel_id":"self","user_id":"user"}]`, + }} got, found, err := NewChannels(f).ExistingDirect(context.Background(), "user", "user") if err != nil || !found || got.ID != "self" { t.Fatalf("channel=%#v found=%v error=%v", got, found, err) } } +func TestExistingDirectRejectsStaleNameWithMissingPeerMembership(t *testing.T) { + f := &fakeChannelTransport{responses: map[string]string{ + "/users/user/channels": `[{"id":"wanted","team_id":"","type":"D","name":"user__peer","display_name":""}]`, + "/channels/wanted/members?page=0&per_page=9": `[{"channel_id":"wanted","user_id":"user"}]`, + }} + if _, _, err := NewChannels(f).ExistingDirect(context.Background(), "user", "peer"); !errors.Is(err, ErrInvalidChannelsResponse) { + t.Fatalf("error=%v", err) + } +} + func TestExistingDirectRejectsNonCanonicalMatchingChannelID(t *testing.T) { f := &fakeChannelTransport{responses: map[string]string{"/users/user/channels": `[{"id":" bad ","team_id":"","type":"D","name":"user__peer","display_name":""}]`}} if _, _, err := NewChannels(f).ExistingDirect(context.Background(), "user", "peer"); !errors.Is(err, ErrInvalidChannelsResponse) { From 1825e15aff17d1fc61b914000de3ac0d2fed983c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 07:21:57 +0300 Subject: [PATCH 084/119] feat: execute conversation apply plans --- internal/apply/service.go | 259 ++++++++++++++++++++ internal/apply/service_test.go | 432 +++++++++++++++++++++++++++++++++ 2 files changed, 691 insertions(+) create mode 100644 internal/apply/service.go create mode 100644 internal/apply/service_test.go diff --git a/internal/apply/service.go b/internal/apply/service.go new file mode 100644 index 0000000..52f6f74 --- /dev/null +++ b/internal/apply/service.go @@ -0,0 +1,259 @@ +// Package apply executes reviewed stage plans through the durable apply journal. +package apply + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "slices" + + "github.com/ardasevinc/mattermost-cli/internal/api" + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/internal/staging" +) + +var ( + ErrInvalid = errors.New("apply: invalid request") + ErrTargetDrift = errors.New("apply: staged target no longer matches Mattermost") + ErrUnsupportedOperation = errors.New("apply: operation is not implemented") + ErrJournal = errors.New("apply: durable journal update failed") +) + +// ConfirmedEffectError means Mattermost confirmed the effect but its durable +// receipt could not be recorded. The caller must not retry automatically. +type ConfirmedEffectError struct{ err error } + +func (e *ConfirmedEffectError) Error() string { + return "apply: effect confirmed but receipt was not recorded; do not retry" +} +func (e *ConfirmedEffectError) Unwrap() error { return e.err } + +type Store interface { + Show(context.Context, string) (stagestore.StageDetail, error) + FindApply(context.Context, string, string, string, [32]byte) (stagestore.ApplyAttempt, bool, error) + ClaimApply(context.Context, stagestore.ApplyClaimInput) (stagestore.ApplyAttempt, error) + AbandonApplyBeforeDispatch(context.Context, string) error + BeginDispatch(context.Context, string, int) error + MarkStepValidated(context.Context, string, int, json.RawMessage) error + MarkStepRejected(context.Context, string, int, json.RawMessage) error + MarkStepUnknown(context.Context, string, int) error + MarkStepSkipped(context.Context, string, int, json.RawMessage) error + FinalizeApply(context.Context, string) (stagestore.ApplyReceipt, error) +} + +type CurrentUser interface { + Current(context.Context) (mattermost.User, error) +} + +type Conversations interface { + ExistingDirect(context.Context, string, string) (mattermost.Channel, bool, error) + ExistingGroup(context.Context, string, []string) (mattermost.Channel, bool, error) +} + +type Service struct { + serverURL, serverID string + store Store + users CurrentUser + channels Conversations + writes *mattermost.ConversationMutations +} + +func New(serverURL, serverID string, store Store, users CurrentUser, channels Conversations, writes *mattermost.ConversationMutations) (*Service, error) { + if serverURL == "" || store == nil || users == nil || channels == nil || writes == nil { + return nil, ErrInvalid + } + return &Service{serverURL: serverURL, serverID: serverID, store: store, users: users, channels: channels, writes: writes}, nil +} + +func (s *Service) Apply(ctx context.Context, in stagestore.ApplyClaimInput) (stagestore.ApplyReceipt, error) { + if ctx == nil || s == nil { + return stagestore.ApplyReceipt{}, ErrInvalid + } + detail, err := s.store.Show(ctx, in.StageID) + if err != nil { + return stagestore.ApplyReceipt{}, err + } + if detail.ServerURL != s.serverURL || detail.ServerID != s.serverID { + return stagestore.ApplyReceipt{}, ErrTargetDrift + } + if in.RequestID != "" { + replay, found, findErr := s.store.FindApply(ctx, detail.ServerURL, detail.UserID, in.RequestID, in.RequestDigest) + if findErr != nil { + return stagestore.ApplyReceipt{}, findErr + } + if found { + if replay.StageID != in.StageID || replay.Revision != in.Revision || replay.SemanticDigest != in.ExpectedDigest || replay.RecoveryMode != in.RecoveryMode { + return stagestore.ApplyReceipt{}, stagestore.ErrConflict + } + return s.store.FinalizeApply(ctx, replay.ID) + } + } + if detail.Revision != in.Revision || detail.SemanticDigest != in.ExpectedDigest { + return stagestore.ApplyReceipt{}, stagestore.ErrConflict + } + if detail.Operation != stagestore.ResolveDM && detail.Operation != stagestore.ResolveGroupDM { + return stagestore.ApplyReceipt{}, ErrUnsupportedOperation + } + destination, err := decodeResolveDestination(detail.Operation, detail.Destination, detail.UserID) + if err != nil { + return stagestore.ApplyReceipt{}, err + } + attempt, err := s.store.ClaimApply(ctx, in) + if err != nil { + return stagestore.ApplyReceipt{}, err + } + if attempt.Replay { + return s.store.FinalizeApply(ctx, attempt.ID) + } + current, err := s.users.Current(ctx) + if err != nil { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, err) + } + if current.ID != detail.UserID { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, ErrTargetDrift) + } + return s.applyConversation(ctx, attempt, detail.Operation, current.ID, destination) +} + +func (s *Service) applyConversation(ctx context.Context, attempt stagestore.ApplyAttempt, operation stagestore.Operation, currentUserID string, destination staging.Destination) (stagestore.ApplyReceipt, error) { + var prepared *mattermost.PreparedResolveConversation + var err error + input := mattermost.ResolveConversationMutationInput{CurrentUserID: currentUserID, ParticipantIDs: destination.ParticipantIDs} + if operation == stagestore.ResolveDM { + prepared, err = s.writes.PrepareDirect(input) + } else { + prepared, err = s.writes.PrepareGroup(input) + } + if err != nil { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, err) + } + _, found, err := s.findConversation(ctx, operation, currentUserID, destination.ParticipantIDs) + if err != nil { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, errors.Join(ErrTargetDrift, err)) + } + if found { + journalCtx := context.WithoutCancel(ctx) + if err = s.store.MarkStepSkipped(journalCtx, attempt.ID, 1, json.RawMessage(`{"reason":"already_satisfied"}`)); err != nil { + return stagestore.ApplyReceipt{}, s.abandon(journalCtx, attempt.ID, fmt.Errorf("%w: %v", ErrJournal, err)) + } + return s.store.FinalizeApply(journalCtx, attempt.ID) + } + if err = s.store.BeginDispatch(ctx, attempt.ID, 1); err != nil { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, err) + } + result, remoteErr := prepared.Execute(ctx) + if remoteErr != nil { + return s.recordRemoteFailure(ctx, attempt.ID, remoteErr) + } + validated, found, validationErr := s.findConversation(ctx, operation, currentUserID, destination.ParticipantIDs) + if validationErr != nil || !found || validated.ID != result.ChannelID { + if err = s.store.MarkStepUnknown(context.WithoutCancel(ctx), attempt.ID, 1); err != nil { + return stagestore.ApplyReceipt{}, errors.Join(&api.OutcomeUnknownError{}, fmt.Errorf("%w: %v", ErrJournal, err)) + } + receipt, finalizeErr := s.store.FinalizeApply(context.WithoutCancel(ctx), attempt.ID) + if finalizeErr != nil { + return stagestore.ApplyReceipt{}, errors.Join(&api.OutcomeUnknownError{}, fmt.Errorf("%w: %v", ErrJournal, finalizeErr)) + } + return receipt, nil + } + encoded, err := json.Marshal(result) + if err != nil { + return stagestore.ApplyReceipt{}, &ConfirmedEffectError{err} + } + if err = s.store.MarkStepValidated(context.WithoutCancel(ctx), attempt.ID, 1, encoded); err != nil { + return stagestore.ApplyReceipt{}, &ConfirmedEffectError{err} + } + receipt, err := s.store.FinalizeApply(context.WithoutCancel(ctx), attempt.ID) + if err != nil { + return stagestore.ApplyReceipt{}, &ConfirmedEffectError{err} + } + return receipt, nil +} + +func (s *Service) findConversation(ctx context.Context, operation stagestore.Operation, currentUserID string, participantIDs []string) (mattermost.Channel, bool, error) { + if operation == stagestore.ResolveDM { + return s.channels.ExistingDirect(ctx, currentUserID, participantIDs[0]) + } + return s.channels.ExistingGroup(ctx, currentUserID, participantIDs) +} + +func (s *Service) recordRemoteFailure(ctx context.Context, attemptID string, remoteErr error) (stagestore.ApplyReceipt, error) { + journalCtx := context.WithoutCancel(ctx) + var rejected *api.APIError + var err error + if errors.As(remoteErr, &rejected) && rejected.Status >= 400 && rejected.Status <= 499 { + result, marshalErr := json.Marshal(struct { + Status int `json:"status"` + }{rejected.Status}) + if marshalErr != nil { + return stagestore.ApplyReceipt{}, fmt.Errorf("%w: %v", ErrJournal, marshalErr) + } + err = s.store.MarkStepRejected(journalCtx, attemptID, 1, result) + } else { + err = s.store.MarkStepUnknown(journalCtx, attemptID, 1) + } + if err != nil { + return stagestore.ApplyReceipt{}, errors.Join(remoteErr, fmt.Errorf("%w: %v", ErrJournal, err)) + } + receipt, finalizeErr := s.store.FinalizeApply(journalCtx, attemptID) + if finalizeErr != nil { + return stagestore.ApplyReceipt{}, errors.Join(remoteErr, fmt.Errorf("%w: %v", ErrJournal, finalizeErr)) + } + return receipt, nil +} + +func (s *Service) abandon(ctx context.Context, attemptID string, cause error) error { + if err := s.store.AbandonApplyBeforeDispatch(context.WithoutCancel(ctx), attemptID); err != nil { + return fmt.Errorf("%w: %v", ErrJournal, err) + } + return cause +} + +func decodeResolveDestination(operation stagestore.Operation, raw json.RawMessage, currentUserID string) (staging.Destination, error) { + var canonical map[string]any + if json.Unmarshal(raw, &canonical) != nil || len(canonical) != 10 { + return staging.Destination{}, ErrInvalid + } + canonicalRaw, err := json.Marshal(canonical) + if err != nil || !bytes.Equal(canonicalRaw, raw) { + return staging.Destination{}, ErrInvalid + } + var wire struct { + Kind string `json:"kind"` + ChannelID *string `json:"channelId"` + ChannelType *string `json:"channelType"` + TeamID *string `json:"teamId"` + PostID *string `json:"postId"` + RootPostID *string `json:"rootPostId"` + ParticipantIDs []string `json:"participantIds"` + Emoji *string `json:"emoji"` + PostState *staging.PostState `json:"postState"` + ReactionPresent *bool `json:"reactionPresent"` + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if decoder.Decode(&wire) != nil || decoder.Decode(new(any)) != io.EOF || wire.ChannelType == nil { + return staging.Destination{}, ErrInvalid + } + destination := staging.Destination{Kind: wire.Kind, ChannelType: *wire.ChannelType, TeamID: wire.TeamID, PostID: wire.PostID, RootPostID: wire.RootPostID, ParticipantIDs: wire.ParticipantIDs, Emoji: wire.Emoji, PostState: wire.PostState, ReactionPresent: wire.ReactionPresent} + if destination.Kind != "conversation" || wire.ChannelID != nil || destination.TeamID != nil || destination.PostID != nil || destination.RootPostID != nil || destination.Emoji != nil || destination.PostState != nil || destination.ReactionPresent != nil { + return staging.Destination{}, ErrInvalid + } + wantType, wantCount := "dm", 1 + if operation == stagestore.ResolveGroupDM { + wantType, wantCount = "group", 2 + } + if destination.ChannelType != wantType || len(destination.ParticipantIDs) < wantCount || operation == stagestore.ResolveDM && len(destination.ParticipantIDs) != 1 || operation == stagestore.ResolveGroupDM && len(destination.ParticipantIDs) > 7 || !slices.IsSorted(destination.ParticipantIDs) { + return staging.Destination{}, ErrInvalid + } + for index, id := range destination.ParticipantIDs { + if id == "" || id == currentUserID || index > 0 && id == destination.ParticipantIDs[index-1] { + return staging.Destination{}, ErrInvalid + } + } + return destination, nil +} diff --git a/internal/apply/service_test.go b/internal/apply/service_test.go new file mode 100644 index 0000000..de2f80e --- /dev/null +++ b/internal/apply/service_test.go @@ -0,0 +1,432 @@ +package apply + +import ( + "context" + "crypto/sha1" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "slices" + "strings" + "sync/atomic" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/api" + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +func TestApplyResolveDMCreatesOnceAndReplaysDurableReceipt(t *testing.T) { + var created atomic.Bool + var writes atomic.Int32 + var invalidWrite atomic.Bool + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch request.URL.RequestURI() { + case "/api/v4/users/me": + _, _ = io.WriteString(response, `{"id":"self","username":"arda"}`) + case "/api/v4/users/self/channels": + if !created.Load() { + _, _ = io.WriteString(response, `[]`) + return + } + _, _ = io.WriteString(response, `[{"id":"dm-1","team_id":"","type":"D","name":"peer__self","display_name":""}]`) + case "/api/v4/channels/dm-1/members?page=0&per_page=9": + _, _ = io.WriteString(response, `[{"channel_id":"dm-1","user_id":"self"},{"channel_id":"dm-1","user_id":"peer"}]`) + case "/api/v4/channels/direct": + body, _ := io.ReadAll(request.Body) + if request.Method != http.MethodPost || string(body) != `["peer","self"]` { + invalidWrite.Store(true) + response.WriteHeader(http.StatusBadRequest) + return + } + writes.Add(1) + created.Store(true) + response.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(response, `{"id":"dm-1","create_at":100,"update_at":100,"delete_at":0,"team_id":"","type":"D","name":"peer__self","display_name":""}`) + default: + http.NotFound(response, request) + } + })) + defer server.Close() + store := openApplyStore(t) + createdStage := createResolveStage(t, store, server.URL+"/api/v4", stagestore.ResolveDM, "dm", []string{"peer"}) + detail, err := store.Show(context.Background(), createdStage.Stage.ID) + if err != nil { + t.Fatal(err) + } + if _, err = decodeResolveDestination(stagestore.ResolveDM, detail.Destination, "self"); err != nil { + t.Fatalf("stored destination=%s err=%v", detail.Destination, err) + } + client, err := api.New(server.URL, "token") + if err != nil { + t.Fatal(err) + } + defer client.Close() + service, err := New(server.URL+"/api/v4", "", store, mattermost.NewUsers(client), mattermost.NewChannels(client), mattermost.NewConversationMutations(client)) + if err != nil { + t.Fatal(err) + } + claim := applyClaim(createdStage, "apply-1") + receipt, err := service.Apply(context.Background(), claim) + if err != nil || receipt.Outcome != stagestore.OutcomeSucceeded || receipt.Recovery != stagestore.RecoveryForbidden || receipt.Steps[0].State != stagestore.StepValidated || writes.Load() != 1 || invalidWrite.Load() { + t.Fatalf("receipt=%+v err=%v writes=%d invalid=%v", receipt, err, writes.Load(), invalidWrite.Load()) + } + server.Close() + replay, err := service.Apply(context.Background(), claim) + if err != nil || !replay.Replay || replay.AttemptID != receipt.AttemptID || writes.Load() != 1 { + t.Fatalf("replay=%+v err=%v writes=%d", replay, err, writes.Load()) + } +} + +func TestDecodeResolveDestinationAcceptsCanonicalUnresolvedDM(t *testing.T) { + raw := json.RawMessage(`{"channelId":null,"channelType":"dm","emoji":null,"kind":"conversation","participantIds":["peer"],"postId":null,"postState":null,"reactionPresent":null,"rootPostId":null,"teamId":null}`) + destination, err := decodeResolveDestination(stagestore.ResolveDM, raw, "self") + if err != nil || destination.ChannelType != "dm" || !slices.Equal(destination.ParticipantIDs, []string{"peer"}) { + t.Fatalf("destination=%+v err=%v", destination, err) + } +} + +func TestApplyResolveDMSkipsExactExistingConversation(t *testing.T) { + var writes atomic.Int32 + server := existingDirectServer(t, &writes) + defer server.Close() + store := openApplyStore(t) + stage := createResolveStage(t, store, server.URL+"/api/v4", stagestore.ResolveDM, "dm", []string{"peer"}) + service, client := applyService(t, server.URL, store) + defer client.Close() + receipt, err := service.Apply(context.Background(), applyClaim(stage, "apply-skip")) + if err != nil || receipt.Outcome != stagestore.OutcomeAlreadySatisfied || receipt.Steps[0].State != stagestore.StepSkipped || writes.Load() != 0 { + t.Fatalf("receipt=%+v err=%v writes=%d", receipt, err, writes.Load()) + } +} + +func TestApplyResolveGroupCreatesExactCanonicalConversation(t *testing.T) { + peers := []string{"alpha", "beta"} + nameDigest := sha1.Sum([]byte("alphabetaself")) + groupName := fmt.Sprintf("%x", nameDigest) + var created atomic.Bool + var writes atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch request.URL.RequestURI() { + case "/api/v4/users/me": + _, _ = io.WriteString(response, `{"id":"self","username":"arda"}`) + case "/api/v4/users/self/channels": + if !created.Load() { + _, _ = io.WriteString(response, `[]`) + return + } + _, _ = fmt.Fprintf(response, `[{"id":"group-1","team_id":"","type":"G","name":%q,"display_name":"group"}]`, groupName) + case "/api/v4/channels/group-1/members?page=0&per_page=9": + _, _ = io.WriteString(response, `[{"channel_id":"group-1","user_id":"alpha"},{"channel_id":"group-1","user_id":"beta"},{"channel_id":"group-1","user_id":"self"}]`) + case "/api/v4/channels/group": + body, _ := io.ReadAll(request.Body) + if request.Method != http.MethodPost || string(body) != `["alpha","beta","self"]` { + response.WriteHeader(http.StatusBadRequest) + return + } + writes.Add(1) + created.Store(true) + response.WriteHeader(http.StatusCreated) + _, _ = fmt.Fprintf(response, `{"id":"group-1","create_at":100,"update_at":100,"delete_at":0,"team_id":"","type":"G","name":%q,"display_name":"group"}`, groupName) + default: + http.NotFound(response, request) + } + })) + defer server.Close() + store := openApplyStore(t) + stage := createResolveStage(t, store, server.URL+"/api/v4", stagestore.ResolveGroupDM, "group", peers) + service, client := applyService(t, server.URL, store) + defer client.Close() + receipt, err := service.Apply(context.Background(), applyClaim(stage, "apply-group")) + if err != nil || receipt.Outcome != stagestore.OutcomeSucceeded || receipt.Steps[0].State != stagestore.StepValidated || writes.Load() != 1 { + t.Fatalf("receipt=%+v err=%v writes=%d", receipt, err, writes.Load()) + } +} + +func TestApplyResolveDMAbandonsBeforeDispatchOnStaleMembership(t *testing.T) { + var writes atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch request.URL.RequestURI() { + case "/api/v4/users/me": + _, _ = io.WriteString(response, `{"id":"self","username":"arda"}`) + case "/api/v4/users/self/channels": + _, _ = io.WriteString(response, `[{"id":"dm-1","team_id":"","type":"D","name":"peer__self","display_name":""}]`) + case "/api/v4/channels/dm-1/members?page=0&per_page=9": + _, _ = io.WriteString(response, `[{"channel_id":"dm-1","user_id":"self"}]`) + case "/api/v4/channels/direct": + writes.Add(1) + response.WriteHeader(http.StatusInternalServerError) + default: + http.NotFound(response, request) + } + })) + defer server.Close() + store := openApplyStore(t) + stage := createResolveStage(t, store, server.URL+"/api/v4", stagestore.ResolveDM, "dm", []string{"peer"}) + service, client := applyService(t, server.URL, store) + defer client.Close() + _, err := service.Apply(context.Background(), applyClaim(stage, "apply-stale")) + detail, showErr := store.Show(context.Background(), stage.Stage.ID) + if !errors.Is(err, ErrTargetDrift) || showErr != nil || detail.Lifecycle != stagestore.LifecycleOpen || detail.Recovery != stagestore.RecoveryNone || writes.Load() != 0 { + t.Fatalf("detail=%+v applyErr=%v showErr=%v writes=%d", detail.StageSummary, err, showErr, writes.Load()) + } +} + +func TestApplyResolveDMClassifiesRejectedAndUnknown(t *testing.T) { + for name, status := range map[string]int{"rejected": http.StatusForbidden, "unknown": http.StatusInternalServerError} { + t.Run(name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch request.URL.RequestURI() { + case "/api/v4/users/me": + _, _ = io.WriteString(response, `{"id":"self","username":"arda"}`) + case "/api/v4/users/self/channels": + _, _ = io.WriteString(response, `[]`) + case "/api/v4/channels/direct": + response.WriteHeader(status) + _, _ = io.WriteString(response, `{}`) + default: + http.NotFound(response, request) + } + })) + defer server.Close() + store := openApplyStore(t) + stage := createResolveStage(t, store, server.URL+"/api/v4", stagestore.ResolveDM, "dm", []string{"peer"}) + service, client := applyService(t, server.URL, store) + defer client.Close() + receipt, err := service.Apply(context.Background(), applyClaim(stage, "apply-"+name)) + wantOutcome, wantRecovery, wantState := stagestore.OutcomeRejected, stagestore.RecoveryNone, stagestore.StepRejected + if name == "unknown" { + wantOutcome, wantRecovery, wantState = stagestore.OutcomeUnknown, stagestore.RecoveryUnknown, stagestore.StepUnknown + } + if err != nil || receipt.Outcome != wantOutcome || receipt.Recovery != wantRecovery || receipt.Steps[0].State != wantState { + t.Fatalf("receipt=%+v err=%v", receipt, err) + } + }) + } +} + +func TestApplyResolveDMAbandonsSkippedClaimWhenJournalWriteFails(t *testing.T) { + var writes atomic.Int32 + server := existingDirectServer(t, &writes) + defer server.Close() + store := openApplyStore(t) + stage := createResolveStage(t, store, server.URL+"/api/v4", stagestore.ResolveDM, "dm", []string{"peer"}) + faults := &faultStore{Store: store, skipErr: errors.New("disk unavailable")} + service, client := applyServiceWithStore(t, server.URL, faults) + defer client.Close() + _, err := service.Apply(context.Background(), applyClaim(stage, "apply-skip-fault")) + detail, showErr := store.Show(context.Background(), stage.Stage.ID) + if !errors.Is(err, ErrJournal) || showErr != nil || detail.Lifecycle != stagestore.LifecycleOpen || detail.Recovery != stagestore.RecoveryNone || writes.Load() != 0 { + t.Fatalf("detail=%+v applyErr=%v showErr=%v writes=%d", detail.StageSummary, err, showErr, writes.Load()) + } +} + +func TestApplyResolveDMFinishesSkippedJournalAfterCallerCancellation(t *testing.T) { + var writes atomic.Int32 + server := existingDirectServer(t, &writes) + defer server.Close() + store := openApplyStore(t) + stage := createResolveStage(t, store, server.URL+"/api/v4", stagestore.ResolveDM, "dm", []string{"peer"}) + ctx, cancel := context.WithCancel(context.Background()) + faults := &faultStore{Store: store, beforeSkip: cancel} + service, client := applyServiceWithStore(t, server.URL, faults) + defer client.Close() + receipt, err := service.Apply(ctx, applyClaim(stage, "apply-skip-canceled")) + if err != nil || receipt.Outcome != stagestore.OutcomeAlreadySatisfied || receipt.Recovery != stagestore.RecoveryForbidden || writes.Load() != 0 || ctx.Err() != context.Canceled { + t.Fatalf("receipt=%+v err=%v writes=%d context=%v", receipt, err, writes.Load(), ctx.Err()) + } +} + +func TestApplyResolveDMPreservesUnknownWhenJournalCannotRecordDispatchOutcome(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch request.URL.RequestURI() { + case "/api/v4/users/me": + _, _ = io.WriteString(response, `{"id":"self","username":"arda"}`) + case "/api/v4/users/self/channels": + _, _ = io.WriteString(response, `[]`) + case "/api/v4/channels/direct": + response.WriteHeader(http.StatusInternalServerError) + _, _ = io.WriteString(response, `{}`) + default: + http.NotFound(response, request) + } + })) + defer server.Close() + store := openApplyStore(t) + stage := createResolveStage(t, store, server.URL+"/api/v4", stagestore.ResolveDM, "dm", []string{"peer"}) + faults := &faultStore{Store: store, unknownErr: errors.New("disk unavailable")} + service, client := applyServiceWithStore(t, server.URL, faults) + defer client.Close() + _, err := service.Apply(context.Background(), applyClaim(stage, "apply-unknown-fault")) + var unknown *api.OutcomeUnknownError + detail, showErr := store.Show(context.Background(), stage.Stage.ID) + if !errors.As(err, &unknown) || !errors.Is(err, ErrJournal) || showErr != nil || detail.Lifecycle != stagestore.LifecycleApplying || detail.Recovery != stagestore.RecoveryNone { + t.Fatalf("detail=%+v applyErr=%v showErr=%v", detail.StageSummary, err, showErr) + } +} + +func TestApplyResolveDMPreservesUnknownWhenFinalizationFailsAfterUnvalidatedSuccess(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch request.URL.RequestURI() { + case "/api/v4/users/me": + _, _ = io.WriteString(response, `{"id":"self","username":"arda"}`) + case "/api/v4/users/self/channels": + _, _ = io.WriteString(response, `[]`) + case "/api/v4/channels/direct": + response.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(response, `{"id":"dm-1","create_at":100,"update_at":100,"delete_at":0,"team_id":"","type":"D","name":"peer__self","display_name":""}`) + default: + http.NotFound(response, request) + } + })) + defer server.Close() + store := openApplyStore(t) + stage := createResolveStage(t, store, server.URL+"/api/v4", stagestore.ResolveDM, "dm", []string{"peer"}) + faults := &faultStore{Store: store, finalizeErr: errors.New("disk unavailable")} + service, client := applyServiceWithStore(t, server.URL, faults) + defer client.Close() + _, err := service.Apply(context.Background(), applyClaim(stage, "apply-finalize-fault")) + var unknown *api.OutcomeUnknownError + if !errors.As(err, &unknown) || !errors.Is(err, ErrJournal) { + t.Fatalf("error=%v", err) + } +} + +func TestApplyResolveDMReportsConfirmedEffectWhenValidatedResultCannotBeJournaled(t *testing.T) { + var created atomic.Bool + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch request.URL.RequestURI() { + case "/api/v4/users/me": + _, _ = io.WriteString(response, `{"id":"self","username":"arda"}`) + case "/api/v4/users/self/channels": + if !created.Load() { + _, _ = io.WriteString(response, `[]`) + return + } + _, _ = io.WriteString(response, `[{"id":"dm-1","team_id":"","type":"D","name":"peer__self","display_name":""}]`) + case "/api/v4/channels/dm-1/members?page=0&per_page=9": + _, _ = io.WriteString(response, `[{"channel_id":"dm-1","user_id":"peer"},{"channel_id":"dm-1","user_id":"self"}]`) + case "/api/v4/channels/direct": + created.Store(true) + response.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(response, `{"id":"dm-1","create_at":100,"update_at":100,"delete_at":0,"team_id":"","type":"D","name":"peer__self","display_name":""}`) + default: + http.NotFound(response, request) + } + })) + defer server.Close() + store := openApplyStore(t) + stage := createResolveStage(t, store, server.URL+"/api/v4", stagestore.ResolveDM, "dm", []string{"peer"}) + faults := &faultStore{Store: store, validatedErr: errors.New("disk unavailable")} + service, client := applyServiceWithStore(t, server.URL, faults) + defer client.Close() + _, err := service.Apply(context.Background(), applyClaim(stage, "apply-validated-fault")) + var confirmed *ConfirmedEffectError + if !errors.As(err, &confirmed) { + t.Fatalf("error=%v", err) + } +} + +func openApplyStore(t *testing.T) *stagestore.Store { + t.Helper() + store, err := stagestore.Open(context.Background(), filepath.Join(t.TempDir(), "state", "mattermost-cli", stagestore.DatabaseFilename)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + return store +} + +func createResolveStage(t *testing.T, store *stagestore.Store, serverURL string, operation stagestore.Operation, channelType string, peers []string) stagestore.MutationResult { + t.Helper() + destination := []byte(`{"kind":"conversation","channelId":null,"channelType":"` + channelType + `","teamId":null,"postId":null,"rootPostId":null,"participantIds":["` + strings.Join(peers, `","`) + `"],"emoji":null,"postState":null,"reactionPresent":null}`) + plan := []byte(`{"steps":[{"ordinal":1,"type":"resolve_conversation","condition":"if_missing"}]}`) + result, err := store.Create(context.Background(), stagestore.CreateInput{RequestDigest: sha256.Sum256([]byte("stage")), Operation: operation, ServerURL: serverURL, UserID: "self", Content: stagestore.RevisionContent{Destination: destination, Plan: plan}}) + if err != nil { + t.Fatal(err) + } + return result.MutationResult +} + +func applyClaim(stage stagestore.MutationResult, requestID string) stagestore.ApplyClaimInput { + return stagestore.ApplyClaimInput{StageID: stage.Stage.ID, RequestID: requestID, Revision: stage.Stage.Revision, ExpectedDigest: stage.Stage.SemanticDigest, RequestDigest: sha256.Sum256([]byte(requestID)), RecoveryMode: stagestore.RecoveryModeOrdinary} +} + +func applyService(t *testing.T, serverURL string, store *stagestore.Store) (*Service, *api.Client) { + return applyServiceWithStore(t, serverURL, store) +} + +func applyServiceWithStore(t *testing.T, serverURL string, store Store) (*Service, *api.Client) { + t.Helper() + client, err := api.New(serverURL, "token") + if err != nil { + t.Fatal(err) + } + service, err := New(serverURL+"/api/v4", "", store, mattermost.NewUsers(client), mattermost.NewChannels(client), mattermost.NewConversationMutations(client)) + if err != nil { + client.Close() + t.Fatal(err) + } + return service, client +} + +type faultStore struct { + *stagestore.Store + skipErr, unknownErr, validatedErr, finalizeErr error + beforeSkip func() +} + +func (s *faultStore) MarkStepSkipped(ctx context.Context, attemptID string, ordinal int, result json.RawMessage) error { + if s.beforeSkip != nil { + s.beforeSkip() + } + if s.skipErr != nil { + return s.skipErr + } + return s.Store.MarkStepSkipped(ctx, attemptID, ordinal, result) +} + +func (s *faultStore) MarkStepUnknown(ctx context.Context, attemptID string, ordinal int) error { + if s.unknownErr != nil { + return s.unknownErr + } + return s.Store.MarkStepUnknown(ctx, attemptID, ordinal) +} + +func (s *faultStore) MarkStepValidated(ctx context.Context, attemptID string, ordinal int, result json.RawMessage) error { + if s.validatedErr != nil { + return s.validatedErr + } + return s.Store.MarkStepValidated(ctx, attemptID, ordinal, result) +} + +func (s *faultStore) FinalizeApply(ctx context.Context, attemptID string) (stagestore.ApplyReceipt, error) { + if s.finalizeErr != nil { + return stagestore.ApplyReceipt{}, s.finalizeErr + } + return s.Store.FinalizeApply(ctx, attemptID) +} + +func existingDirectServer(t *testing.T, writes *atomic.Int32) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch request.URL.RequestURI() { + case "/api/v4/users/me": + _, _ = io.WriteString(response, `{"id":"self","username":"arda"}`) + case "/api/v4/users/self/channels": + _, _ = io.WriteString(response, `[{"id":"dm-1","team_id":"","type":"D","name":"peer__self","display_name":""}]`) + case "/api/v4/channels/dm-1/members?page=0&per_page=9": + _, _ = io.WriteString(response, `[{"channel_id":"dm-1","user_id":"self"},{"channel_id":"dm-1","user_id":"peer"}]`) + case "/api/v4/channels/direct": + writes.Add(1) + response.WriteHeader(http.StatusInternalServerError) + default: + http.NotFound(response, request) + } + })) +} From 0673402a0f6e6821c99d8769ec5520e821ca50a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 07:25:54 +0300 Subject: [PATCH 085/119] fix: canonicalize staged reaction names --- internal/staging/post.go | 20 ++++++++++++++++---- internal/staging/post_test.go | 6 +++--- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/internal/staging/post.go b/internal/staging/post.go index 8911030..c2515e6 100644 --- a/internal/staging/post.go +++ b/internal/staging/post.go @@ -151,19 +151,31 @@ func (s *Service) DeletePost(ctx context.Context, in DeletePostInput) (CreatePos } func (s *Service) DryRunReact(ctx context.Context, in ReactionDryRunInput) (Preview, error) { - return s.resolvePost(ctx, postOperation{operation: stagestore.React, postID: in.PostID, emoji: in.Emoji}) + if contaminated(s.credentials, in.Emoji) { + return Preview{}, ErrCredential + } + return s.resolvePost(ctx, postOperation{operation: stagestore.React, postID: in.PostID, emoji: strings.ToLower(in.Emoji)}) } func (s *Service) React(ctx context.Context, in ReactionInput) (CreatePostResult, error) { - return s.persistContentless(ctx, in.RequestID, postOperation{operation: stagestore.React, postID: in.PostID, emoji: in.Emoji}) + if contaminated(s.credentials, in.Emoji) { + return CreatePostResult{}, ErrCredential + } + return s.persistContentless(ctx, in.RequestID, postOperation{operation: stagestore.React, postID: in.PostID, emoji: strings.ToLower(in.Emoji)}) } func (s *Service) DryRunUnreact(ctx context.Context, in ReactionDryRunInput) (Preview, error) { - return s.resolvePost(ctx, postOperation{operation: stagestore.Unreact, postID: in.PostID, emoji: in.Emoji}) + if contaminated(s.credentials, in.Emoji) { + return Preview{}, ErrCredential + } + return s.resolvePost(ctx, postOperation{operation: stagestore.Unreact, postID: in.PostID, emoji: strings.ToLower(in.Emoji)}) } func (s *Service) Unreact(ctx context.Context, in ReactionInput) (CreatePostResult, error) { - return s.persistContentless(ctx, in.RequestID, postOperation{operation: stagestore.Unreact, postID: in.PostID, emoji: in.Emoji}) + if contaminated(s.credentials, in.Emoji) { + return CreatePostResult{}, ErrCredential + } + return s.persistContentless(ctx, in.RequestID, postOperation{operation: stagestore.Unreact, postID: in.PostID, emoji: strings.ToLower(in.Emoji)}) } func (s *Service) persistContentless(ctx context.Context, requestID string, op postOperation) (CreatePostResult, error) { diff --git a/internal/staging/post_test.go b/internal/staging/post_test.go index e99cbdb..7c1191c 100644 --- a/internal/staging/post_test.go +++ b/internal/staging/post_test.go @@ -159,7 +159,7 @@ func TestReactionUsesAuthoritativeStateAndExactBinding(t *testing.T) { if err != nil || result.Preview.Destination.ReactionPresent == nil || !*result.Preview.Destination.ReactionPresent { t.Fatalf("result/error = %#v/%v", result, err) } - if !reflect.DeepEqual(posts.reaction, []string{"post-1", "channel-1", "user-1", "Eyes"}) || !reflect.DeepEqual(result.Preview.Destination.ParticipantIDs, []string{"user-1"}) { + if !reflect.DeepEqual(posts.reaction, []string{"post-1", "channel-1", "user-1", "eyes"}) || result.Preview.Destination.Emoji == nil || *result.Preview.Destination.Emoji != "eyes" || !reflect.DeepEqual(result.Preview.Destination.ParticipantIDs, []string{"user-1"}) { t.Fatalf("reaction/participants = %v/%v", posts.reaction, result.Preview.Destination.ParticipantIDs) } if result.Preview.Destination.RootPostID == nil || *result.Preview.Destination.RootPostID != "root-1" { @@ -272,8 +272,8 @@ func TestEditCanRemediateCredentialInExistingRemoteMessage(t *testing.T) { func TestPostMutationRejectsCallerCredentialBeforeNetwork(t *testing.T) { post := ordinaryPost() service, posts, calls := postService(t, post, mattermost.Channel{ID: "channel-1", Type: "G", Name: "group"}, &recordingStore{}) - service.credentials = [][]byte{[]byte("secret")} - _, err := service.React(context.Background(), ReactionInput{RequestID: "request", PostID: "post-1", Emoji: "secret"}) + service.credentials = [][]byte{[]byte("Secret")} + _, err := service.React(context.Background(), ReactionInput{RequestID: "request", PostID: "post-1", Emoji: "Secret"}) if !errors.Is(err, ErrCredential) || len(*calls) != 0 || len(posts.reaction) != 0 { t.Fatalf("error/calls = %v/%v", err, *calls) } From 788b4ee4a0940e851b539a8d7b3e27f4559568bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 07:44:28 +0300 Subject: [PATCH 086/119] feat: execute reaction apply plans --- internal/apply/reaction.go | 198 +++++++++++++++++++++++++++ internal/apply/service.go | 168 +++++++++++++++++++---- internal/apply/service_test.go | 237 ++++++++++++++++++++++++++++++++- 3 files changed, 575 insertions(+), 28 deletions(-) create mode 100644 internal/apply/reaction.go diff --git a/internal/apply/reaction.go b/internal/apply/reaction.go new file mode 100644 index 0000000..f27ab7e --- /dev/null +++ b/internal/apply/reaction.go @@ -0,0 +1,198 @@ +package apply + +import ( + "context" + "encoding/json" + "errors" + "slices" + "strings" + + "github.com/ardasevinc/mattermost-cli/internal/api" + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/internal/staging" +) + +type preparedReaction interface { + Execute(context.Context) (mattermost.ReactionMutationResult, error) +} + +func (s *Service) applyReaction(ctx context.Context, attempt stagestore.ApplyAttempt, operation stagestore.Operation, currentUserID string, destination staging.Destination) (stagestore.ApplyReceipt, error) { + input := mattermost.ReactionMutationInput{PostID: *destination.PostID, ChannelID: destination.ChannelID, UserID: currentUserID, Emoji: *destination.Emoji} + var prepared preparedReaction + var err error + if operation == stagestore.React { + prepared, err = s.postWrites.PrepareAddReaction(input) + } else { + prepared, err = s.postWrites.PrepareRemoveReaction(input) + } + if err != nil { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, err) + } + present, err := s.revalidateReaction(ctx, currentUserID, destination) + if err != nil { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, errors.Join(ErrTargetDrift, err)) + } + if operation == stagestore.React && present || operation == stagestore.Unreact && !present { + return s.skip(ctx, attempt.ID) + } + if err = s.store.BeginDispatch(ctx, attempt.ID, 1); err != nil { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, err) + } + result, remoteErr := prepared.Execute(ctx) + if remoteErr != nil { + return s.recordRemoteFailure(ctx, attempt.ID, remoteErr) + } + present, validationErr := s.revalidateReaction(ctx, currentUserID, destination) + if validationErr != nil || operation == stagestore.React && !present || operation == stagestore.Unreact && present { + return s.recordUnvalidatedSuccess(ctx, attempt.ID) + } + encoded, err := json.Marshal(result) + if err != nil { + return stagestore.ApplyReceipt{}, &ConfirmedEffectError{err} + } + journalCtx := context.WithoutCancel(ctx) + if err = s.store.MarkStepValidated(journalCtx, attempt.ID, 1, encoded); err != nil { + return stagestore.ApplyReceipt{}, &ConfirmedEffectError{err} + } + receipt, err := s.finalizeReceipt(journalCtx, attempt.ID) + if err != nil { + return stagestore.ApplyReceipt{}, &ConfirmedEffectError{err} + } + return receipt, nil +} + +func (s *Service) recordUnvalidatedSuccess(ctx context.Context, attemptID string) (stagestore.ApplyReceipt, error) { + journalCtx := context.WithoutCancel(ctx) + if err := s.store.MarkStepUnknown(journalCtx, attemptID, 1); err != nil { + return stagestore.ApplyReceipt{}, errors.Join(&api.OutcomeUnknownError{}, errors.Join(ErrJournal, err)) + } + receipt, err := s.finalizeReceipt(journalCtx, attemptID) + if err != nil { + return stagestore.ApplyReceipt{}, errors.Join(&api.OutcomeUnknownError{}, errors.Join(ErrJournal, err)) + } + return receipt, nil +} + +func (s *Service) revalidateReaction(ctx context.Context, currentUserID string, destination staging.Destination) (bool, error) { + post, err := s.posts.ByID(ctx, *destination.PostID) + if err != nil { + return false, err + } + wantRoot := "" + if destination.RootPostID != nil { + wantRoot = *destination.RootPostID + } + if post.ID != *destination.PostID || post.ChannelID != destination.ChannelID || post.RootID != wantRoot { + return false, ErrTargetDrift + } + if destination.ChannelType == "dm" { + channel, found, directErr := s.channels.ExistingDirect(ctx, currentUserID, destination.ParticipantIDs[0]) + if directErr != nil { + return false, directErr + } + if !found || channel.ID != destination.ChannelID { + return false, ErrTargetDrift + } + return s.posts.ReactionState(ctx, *destination.PostID, destination.ChannelID, currentUserID, *destination.Emoji) + } + channel, err := s.channels.ByID(ctx, destination.ChannelID) + if err != nil { + return false, err + } + if !channelMatchesDestination(channel, destination, currentUserID) { + return false, ErrTargetDrift + } + member, err := s.channels.Member(ctx, destination.ChannelID, currentUserID) + if err != nil { + return false, err + } + if member.ChannelID != destination.ChannelID || member.UserID != currentUserID { + return false, ErrTargetDrift + } + return s.posts.ReactionState(ctx, *destination.PostID, destination.ChannelID, currentUserID, *destination.Emoji) +} + +func channelMatchesDestination(channel mattermost.Channel, destination staging.Destination, currentUserID string) bool { + if channel.ID != destination.ChannelID || map[string]string{"D": "dm", "G": "group", "O": "public", "P": "private"}[channel.Type] != destination.ChannelType { + return false + } + if channel.Type == "O" || channel.Type == "P" { + return destination.TeamID != nil && *destination.TeamID == channel.TeamID && len(destination.ParticipantIDs) == 0 + } + if destination.TeamID != nil || channel.TeamID != "" { + return false + } + if channel.Type != "D" { + return len(destination.ParticipantIDs) == 0 + } + parts := strings.Split(channel.Name, "__") + if len(parts) != 2 { + return false + } + participants := []string{} + if parts[0] == currentUserID && parts[1] == currentUserID { + participants = []string{currentUserID} + } else if parts[0] == currentUserID { + participants = []string{parts[1]} + } else if parts[1] == currentUserID { + participants = []string{parts[0]} + } + return slices.Equal(participants, destination.ParticipantIDs) +} + +func decodeReactionDestination(raw json.RawMessage) (staging.Destination, error) { + if !canonicalDestination(raw) { + return staging.Destination{}, ErrInvalid + } + var destination staging.Destination + if json.Unmarshal(raw, &destination) != nil || destination.Kind != "reaction" || !safeID(destination.ChannelID) || destination.PostID == nil || !safeID(*destination.PostID) || destination.Emoji == nil || !safeEmoji(*destination.Emoji) || destination.PostState != nil || destination.ReactionPresent == nil { + return staging.Destination{}, ErrInvalid + } + if destination.RootPostID != nil && !safeID(*destination.RootPostID) { + return staging.Destination{}, ErrInvalid + } + switch destination.ChannelType { + case "dm": + if destination.TeamID != nil || len(destination.ParticipantIDs) != 1 || !safeID(destination.ParticipantIDs[0]) { + return staging.Destination{}, ErrInvalid + } + case "group": + if destination.TeamID != nil || len(destination.ParticipantIDs) != 0 { + return staging.Destination{}, ErrInvalid + } + case "public", "private": + if destination.TeamID == nil || !safeID(*destination.TeamID) || len(destination.ParticipantIDs) != 0 { + return staging.Destination{}, ErrInvalid + } + default: + return staging.Destination{}, ErrInvalid + } + return destination, nil +} + +func safeID(value string) bool { + if len(value) == 0 || len(value) > 128 { + return false + } + for index := range len(value) { + character := value[index] + if !((character >= 'a' && character <= 'z') || (character >= 'A' && character <= 'Z') || (character >= '0' && character <= '9') || character == '_' || character == '-') { + return false + } + } + return true +} + +func safeEmoji(value string) bool { + if value != strings.ToLower(value) || len(value) == 0 || len(value) > 64 { + return false + } + for index := range len(value) { + character := value[index] + if !((character >= 'a' && character <= 'z') || (character >= '0' && character <= '9') || character == '_' || character == '-' || character == '+') { + return false + } + } + return true +} diff --git a/internal/apply/service.go b/internal/apply/service.go index 52f6f74..e327919 100644 --- a/internal/apply/service.go +++ b/internal/apply/service.go @@ -21,6 +21,7 @@ var ( ErrTargetDrift = errors.New("apply: staged target no longer matches Mattermost") ErrUnsupportedOperation = errors.New("apply: operation is not implemented") ErrJournal = errors.New("apply: durable journal update failed") + ErrCredential = errors.New("apply: active credential in outbound content") ) // ConfirmedEffectError means Mattermost confirmed the effect but its durable @@ -52,6 +53,13 @@ type CurrentUser interface { type Conversations interface { ExistingDirect(context.Context, string, string) (mattermost.Channel, bool, error) ExistingGroup(context.Context, string, []string) (mattermost.Channel, bool, error) + ByID(context.Context, string) (mattermost.Channel, error) + Member(context.Context, string, string) (mattermost.ChannelMember, error) +} + +type PostTargets interface { + ByID(context.Context, string) (mattermost.Post, error) + ReactionState(context.Context, string, string, string, string) (bool, error) } type Service struct { @@ -59,20 +67,27 @@ type Service struct { store Store users CurrentUser channels Conversations + posts PostTargets writes *mattermost.ConversationMutations + postWrites *mattermost.PostMutations + credentials [][]byte } -func New(serverURL, serverID string, store Store, users CurrentUser, channels Conversations, writes *mattermost.ConversationMutations) (*Service, error) { - if serverURL == "" || store == nil || users == nil || channels == nil || writes == nil { +func New(serverURL, serverID string, credentials [][]byte, store Store, users CurrentUser, channels Conversations, posts PostTargets, writes *mattermost.ConversationMutations, postWrites *mattermost.PostMutations) (*Service, error) { + protected, validCredentials := cloneCredentials(credentials) + if serverURL == "" || !validCredentials || store == nil || users == nil || channels == nil || posts == nil || writes == nil || postWrites == nil { return nil, ErrInvalid } - return &Service{serverURL: serverURL, serverID: serverID, store: store, users: users, channels: channels, writes: writes}, nil + return &Service{serverURL: serverURL, serverID: serverID, store: store, users: users, channels: channels, posts: posts, writes: writes, postWrites: postWrites, credentials: protected}, nil } func (s *Service) Apply(ctx context.Context, in stagestore.ApplyClaimInput) (stagestore.ApplyReceipt, error) { if ctx == nil || s == nil { return stagestore.ApplyReceipt{}, ErrInvalid } + if s.containsCredential(in.RequestID) { + return stagestore.ApplyReceipt{}, ErrCredential + } detail, err := s.store.Show(ctx, in.StageID) if err != nil { return stagestore.ApplyReceipt{}, err @@ -80,6 +95,21 @@ func (s *Service) Apply(ctx context.Context, in stagestore.ApplyClaimInput) (sta if detail.ServerURL != s.serverURL || detail.ServerID != s.serverID { return stagestore.ApplyReceipt{}, ErrTargetDrift } + if detail.Operation != stagestore.ResolveDM && detail.Operation != stagestore.ResolveGroupDM && detail.Operation != stagestore.React && detail.Operation != stagestore.Unreact { + return stagestore.ApplyReceipt{}, ErrUnsupportedOperation + } + var destination staging.Destination + if detail.Operation == stagestore.ResolveDM || detail.Operation == stagestore.ResolveGroupDM { + destination, err = decodeResolveDestination(detail.Operation, detail.Destination, detail.UserID) + } else { + destination, err = decodeReactionDestination(detail.Destination) + } + if err != nil { + return stagestore.ApplyReceipt{}, err + } + if s.destinationContainsCredential(destination, detail.UserID) { + return stagestore.ApplyReceipt{}, ErrCredential + } if in.RequestID != "" { replay, found, findErr := s.store.FindApply(ctx, detail.ServerURL, detail.UserID, in.RequestID, in.RequestDigest) if findErr != nil { @@ -89,25 +119,18 @@ func (s *Service) Apply(ctx context.Context, in stagestore.ApplyClaimInput) (sta if replay.StageID != in.StageID || replay.Revision != in.Revision || replay.SemanticDigest != in.ExpectedDigest || replay.RecoveryMode != in.RecoveryMode { return stagestore.ApplyReceipt{}, stagestore.ErrConflict } - return s.store.FinalizeApply(ctx, replay.ID) + return s.finalizeReceipt(ctx, replay.ID) } } if detail.Revision != in.Revision || detail.SemanticDigest != in.ExpectedDigest { return stagestore.ApplyReceipt{}, stagestore.ErrConflict } - if detail.Operation != stagestore.ResolveDM && detail.Operation != stagestore.ResolveGroupDM { - return stagestore.ApplyReceipt{}, ErrUnsupportedOperation - } - destination, err := decodeResolveDestination(detail.Operation, detail.Destination, detail.UserID) - if err != nil { - return stagestore.ApplyReceipt{}, err - } attempt, err := s.store.ClaimApply(ctx, in) if err != nil { return stagestore.ApplyReceipt{}, err } if attempt.Replay { - return s.store.FinalizeApply(ctx, attempt.ID) + return s.finalizeReceipt(ctx, attempt.ID) } current, err := s.users.Current(ctx) if err != nil { @@ -116,6 +139,9 @@ func (s *Service) Apply(ctx context.Context, in stagestore.ApplyClaimInput) (sta if current.ID != detail.UserID { return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, ErrTargetDrift) } + if detail.Operation == stagestore.React || detail.Operation == stagestore.Unreact { + return s.applyReaction(ctx, attempt, detail.Operation, current.ID, destination) + } return s.applyConversation(ctx, attempt, detail.Operation, current.ID, destination) } @@ -136,11 +162,7 @@ func (s *Service) applyConversation(ctx context.Context, attempt stagestore.Appl return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, errors.Join(ErrTargetDrift, err)) } if found { - journalCtx := context.WithoutCancel(ctx) - if err = s.store.MarkStepSkipped(journalCtx, attempt.ID, 1, json.RawMessage(`{"reason":"already_satisfied"}`)); err != nil { - return stagestore.ApplyReceipt{}, s.abandon(journalCtx, attempt.ID, fmt.Errorf("%w: %v", ErrJournal, err)) - } - return s.store.FinalizeApply(journalCtx, attempt.ID) + return s.skip(ctx, attempt.ID) } if err = s.store.BeginDispatch(ctx, attempt.ID, 1); err != nil { return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, err) @@ -154,7 +176,7 @@ func (s *Service) applyConversation(ctx context.Context, attempt stagestore.Appl if err = s.store.MarkStepUnknown(context.WithoutCancel(ctx), attempt.ID, 1); err != nil { return stagestore.ApplyReceipt{}, errors.Join(&api.OutcomeUnknownError{}, fmt.Errorf("%w: %v", ErrJournal, err)) } - receipt, finalizeErr := s.store.FinalizeApply(context.WithoutCancel(ctx), attempt.ID) + receipt, finalizeErr := s.finalizeReceipt(context.WithoutCancel(ctx), attempt.ID) if finalizeErr != nil { return stagestore.ApplyReceipt{}, errors.Join(&api.OutcomeUnknownError{}, fmt.Errorf("%w: %v", ErrJournal, finalizeErr)) } @@ -167,7 +189,7 @@ func (s *Service) applyConversation(ctx context.Context, attempt stagestore.Appl if err = s.store.MarkStepValidated(context.WithoutCancel(ctx), attempt.ID, 1, encoded); err != nil { return stagestore.ApplyReceipt{}, &ConfirmedEffectError{err} } - receipt, err := s.store.FinalizeApply(context.WithoutCancel(ctx), attempt.ID) + receipt, err := s.finalizeReceipt(context.WithoutCancel(ctx), attempt.ID) if err != nil { return stagestore.ApplyReceipt{}, &ConfirmedEffectError{err} } @@ -199,7 +221,7 @@ func (s *Service) recordRemoteFailure(ctx context.Context, attemptID string, rem if err != nil { return stagestore.ApplyReceipt{}, errors.Join(remoteErr, fmt.Errorf("%w: %v", ErrJournal, err)) } - receipt, finalizeErr := s.store.FinalizeApply(journalCtx, attemptID) + receipt, finalizeErr := s.finalizeReceipt(journalCtx, attemptID) if finalizeErr != nil { return stagestore.ApplyReceipt{}, errors.Join(remoteErr, fmt.Errorf("%w: %v", ErrJournal, finalizeErr)) } @@ -213,13 +235,31 @@ func (s *Service) abandon(ctx context.Context, attemptID string, cause error) er return cause } -func decodeResolveDestination(operation stagestore.Operation, raw json.RawMessage, currentUserID string) (staging.Destination, error) { - var canonical map[string]any - if json.Unmarshal(raw, &canonical) != nil || len(canonical) != 10 { - return staging.Destination{}, ErrInvalid +func (s *Service) skip(ctx context.Context, attemptID string) (stagestore.ApplyReceipt, error) { + journalCtx := context.WithoutCancel(ctx) + if err := s.store.MarkStepSkipped(journalCtx, attemptID, 1, json.RawMessage(`{"reason":"already_satisfied"}`)); err != nil { + return stagestore.ApplyReceipt{}, s.abandon(journalCtx, attemptID, fmt.Errorf("%w: %v", ErrJournal, err)) + } + return s.finalizeReceipt(journalCtx, attemptID) +} + +func (s *Service) finalizeReceipt(ctx context.Context, attemptID string) (stagestore.ApplyReceipt, error) { + receipt, err := s.store.FinalizeApply(ctx, attemptID) + if err != nil { + return stagestore.ApplyReceipt{}, err + } + encoded, err := json.Marshal(receipt) + if err != nil { + return stagestore.ApplyReceipt{}, fmt.Errorf("%w: encode receipt: %v", ErrJournal, err) } - canonicalRaw, err := json.Marshal(canonical) - if err != nil || !bytes.Equal(canonicalRaw, raw) { + if s.rawContainsCredentialValue(encoded) { + return stagestore.ApplyReceipt{}, ErrCredential + } + return receipt, nil +} + +func decodeResolveDestination(operation stagestore.Operation, raw json.RawMessage, currentUserID string) (staging.Destination, error) { + if !canonicalDestination(raw) { return staging.Destination{}, ErrInvalid } var wire struct { @@ -257,3 +297,79 @@ func decodeResolveDestination(operation stagestore.Operation, raw json.RawMessag } return destination, nil } + +func canonicalDestination(raw json.RawMessage) bool { + var canonical map[string]any + if json.Unmarshal(raw, &canonical) != nil || len(canonical) != 10 { + return false + } + encoded, err := json.Marshal(canonical) + return err == nil && bytes.Equal(encoded, raw) +} + +func cloneCredentials(values [][]byte) ([][]byte, bool) { + if len(values) == 0 { + return nil, false + } + cloned := make([][]byte, len(values)) + for index, value := range values { + if len(value) == 0 { + return nil, false + } + cloned[index] = bytes.Clone(value) + } + return cloned, true +} + +func (s *Service) destinationContainsCredential(destination staging.Destination, currentUserID string) bool { + values := []string{currentUserID, destination.ChannelID, destination.ChannelType} + values = append(values, destination.ParticipantIDs...) + for _, optional := range []*string{destination.TeamID, destination.PostID, destination.RootPostID, destination.Emoji} { + if optional != nil { + values = append(values, *optional) + } + } + return s.containsCredential(values...) +} + +func (s *Service) containsCredential(values ...string) bool { + for _, value := range values { + for _, credential := range s.credentials { + if bytes.Contains([]byte(value), credential) { + return true + } + } + } + return false +} + +func (s *Service) rawContainsCredentialValue(raw json.RawMessage) bool { + if len(raw) == 0 { + return false + } + var value any + if json.Unmarshal(raw, &value) != nil { + return true + } + var visit func(any) bool + visit = func(candidate any) bool { + switch typed := candidate.(type) { + case string: + return s.containsCredential(typed) + case []any: + for _, item := range typed { + if visit(item) { + return true + } + } + case map[string]any: + for _, item := range typed { + if visit(item) { + return true + } + } + } + return false + } + return visit(value) +} diff --git a/internal/apply/service_test.go b/internal/apply/service_test.go index de2f80e..dbd99b4 100644 --- a/internal/apply/service_test.go +++ b/internal/apply/service_test.go @@ -67,7 +67,7 @@ func TestApplyResolveDMCreatesOnceAndReplaysDurableReceipt(t *testing.T) { t.Fatal(err) } defer client.Close() - service, err := New(server.URL+"/api/v4", "", store, mattermost.NewUsers(client), mattermost.NewChannels(client), mattermost.NewConversationMutations(client)) + service, err := New(server.URL+"/api/v4", "", [][]byte{[]byte("token")}, store, mattermost.NewUsers(client), mattermost.NewChannels(client), mattermost.NewPosts(client), mattermost.NewConversationMutations(client), mattermost.NewPostMutations(client)) if err != nil { t.Fatal(err) } @@ -81,6 +81,27 @@ func TestApplyResolveDMCreatesOnceAndReplaysDurableReceipt(t *testing.T) { if err != nil || !replay.Replay || replay.AttemptID != receipt.AttemptID || writes.Load() != 1 { t.Fatalf("replay=%+v err=%v writes=%d", replay, err, writes.Load()) } + rotated, err := New(server.URL+"/api/v4", "", [][]byte{[]byte("peer")}, store, mattermost.NewUsers(client), mattermost.NewChannels(client), mattermost.NewPosts(client), mattermost.NewConversationMutations(client), mattermost.NewPostMutations(client)) + if err != nil { + t.Fatal(err) + } + if _, err = rotated.Apply(context.Background(), claim); !errors.Is(err, ErrCredential) { + t.Fatalf("rotated credential replay error=%v", err) + } + rotatedResult, err := New(server.URL+"/api/v4", "", [][]byte{[]byte("dm-1")}, store, mattermost.NewUsers(client), mattermost.NewChannels(client), mattermost.NewPosts(client), mattermost.NewConversationMutations(client), mattermost.NewPostMutations(client)) + if err != nil { + t.Fatal(err) + } + if _, err = rotatedResult.Apply(context.Background(), claim); !errors.Is(err, ErrCredential) { + t.Fatalf("rotated result credential replay error=%v", err) + } + rotatedTimestamp, err := New(server.URL+"/api/v4", "", [][]byte{[]byte("2026")}, store, mattermost.NewUsers(client), mattermost.NewChannels(client), mattermost.NewPosts(client), mattermost.NewConversationMutations(client), mattermost.NewPostMutations(client)) + if err != nil { + t.Fatal(err) + } + if _, err = rotatedTimestamp.Apply(context.Background(), claim); !errors.Is(err, ErrCredential) { + t.Fatalf("rotated timestamp credential replay error=%v", err) + } } func TestDecodeResolveDestinationAcceptsCanonicalUnresolvedDM(t *testing.T) { @@ -91,6 +112,57 @@ func TestDecodeResolveDestinationAcceptsCanonicalUnresolvedDM(t *testing.T) { } } +func TestDecodeReactionDestinationRejectsUnknownMissingAndNullMembers(t *testing.T) { + valid := `{"channelId":"channel-1","channelType":"public","emoji":"eyes","kind":"reaction","participantIds":[],"postId":"post-1","postState":null,"reactionPresent":false,"rootPostId":null,"teamId":"team-1"}` + for name, raw := range map[string]string{ + "unknown-replaces-required": strings.Replace(valid, `"postState":null`, `"extra":null`, 1), + "null-participants": strings.Replace(valid, `"participantIds":[]`, `"participantIds":null`, 1), + "missing-required": strings.Replace(valid, `,"postState":null`, ``, 1), + } { + t.Run(name, func(t *testing.T) { + if _, err := decodeReactionDestination(json.RawMessage(raw)); !errors.Is(err, ErrInvalid) { + t.Fatalf("error=%v raw=%s", err, raw) + } + }) + } +} + +func TestApplyBlocksCurrentCredentialInStoredOutboundFieldsBeforeNetworkOrClaim(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + calls.Add(1) + http.NotFound(response, request) + })) + defer server.Close() + store := openApplyStore(t) + stage := createResolveStage(t, store, server.URL+"/api/v4", stagestore.ResolveDM, "dm", []string{"peer"}) + service, client := applyServiceWithCredentials(t, server.URL, store, [][]byte{[]byte("peer")}) + defer client.Close() + _, err := service.Apply(context.Background(), applyClaim(stage, "apply-credential")) + detail, showErr := store.Show(context.Background(), stage.Stage.ID) + if !errors.Is(err, ErrCredential) || showErr != nil || detail.Lifecycle != stagestore.LifecycleOpen || calls.Load() != 0 { + t.Fatalf("detail=%+v applyErr=%v showErr=%v calls=%d", detail.StageSummary, err, showErr, calls.Load()) + } +} + +func TestApplyBlocksCurrentCredentialAsRequestIDBeforePersistenceOrNetwork(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + calls.Add(1) + http.NotFound(response, request) + })) + defer server.Close() + store := openApplyStore(t) + stage := createResolveStage(t, store, server.URL+"/api/v4", stagestore.ResolveDM, "dm", []string{"peer"}) + service, client := applyServiceWithCredentials(t, server.URL, store, [][]byte{[]byte("request-secret")}) + defer client.Close() + _, err := service.Apply(context.Background(), applyClaim(stage, "request-secret")) + detail, showErr := store.Show(context.Background(), stage.Stage.ID) + if !errors.Is(err, ErrCredential) || showErr != nil || detail.Lifecycle != stagestore.LifecycleOpen || calls.Load() != 0 { + t.Fatalf("detail=%+v applyErr=%v showErr=%v calls=%d", detail.StageSummary, err, showErr, calls.Load()) + } +} + func TestApplyResolveDMSkipsExactExistingConversation(t *testing.T) { var writes atomic.Int32 server := existingDirectServer(t, &writes) @@ -332,6 +404,137 @@ func TestApplyResolveDMReportsConfirmedEffectWhenValidatedResultCannotBeJournale } } +func TestApplyReactionExecutesOrSkipsFromFreshAuthoritativeState(t *testing.T) { + for _, test := range []struct { + name, operation string + initial, skip bool + }{ + {"add", string(stagestore.React), false, false}, + {"remove", string(stagestore.Unreact), true, false}, + {"already-present", string(stagestore.React), true, true}, + } { + t.Run(test.name, func(t *testing.T) { + var present atomic.Bool + present.Store(test.initial) + var writes atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch request.URL.RequestURI() { + case "/api/v4/users/me": + _, _ = io.WriteString(response, `{"id":"self","username":"arda"}`) + case "/api/v4/posts/post-1": + _, _ = io.WriteString(response, `{"id":"post-1","channel_id":"channel-1","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":"","type":"","file_ids":[]}`) + case "/api/v4/channels/channel-1": + _, _ = io.WriteString(response, `{"id":"channel-1","team_id":"team-1","type":"O","name":"town-square","display_name":"Town Square"}`) + case "/api/v4/channels/channel-1/members/self": + _, _ = io.WriteString(response, `{"channel_id":"channel-1","user_id":"self"}`) + case "/api/v4/posts/post-1/reactions": + if present.Load() { + _, _ = io.WriteString(response, `[{"user_id":"self","post_id":"post-1","emoji_name":"eyes","create_at":1,"update_at":1,"delete_at":0,"remote_id":null,"channel_id":"channel-1"}]`) + } else { + _, _ = io.WriteString(response, `[]`) + } + case "/api/v4/reactions": + body, _ := io.ReadAll(request.Body) + if request.Method != http.MethodPost || string(body) != `{"user_id":"self","post_id":"post-1","emoji_name":"eyes"}` { + response.WriteHeader(http.StatusBadRequest) + return + } + writes.Add(1) + present.Store(true) + _, _ = io.WriteString(response, `{"user_id":"self","post_id":"post-1","emoji_name":"eyes","create_at":1,"update_at":1,"delete_at":0,"remote_id":"","channel_id":"channel-1"}`) + case "/api/v4/users/self/posts/post-1/reactions/eyes": + if request.Method != http.MethodDelete { + response.WriteHeader(http.StatusBadRequest) + return + } + writes.Add(1) + present.Store(false) + _, _ = io.WriteString(response, `{"status":"OK"}`) + default: + http.NotFound(response, request) + } + })) + defer server.Close() + store := openApplyStore(t) + operation := stagestore.Operation(test.operation) + stage := createReactionStage(t, store, server.URL+"/api/v4", operation, test.initial) + service, client := applyService(t, server.URL, store) + defer client.Close() + receipt, err := service.Apply(context.Background(), applyClaim(stage, "apply-reaction-"+test.name)) + wantOutcome, wantState, wantWrites := stagestore.OutcomeSucceeded, stagestore.StepValidated, int32(1) + if test.skip { + wantOutcome, wantState, wantWrites = stagestore.OutcomeAlreadySatisfied, stagestore.StepSkipped, 0 + } + if err != nil || receipt.Outcome != wantOutcome || receipt.Steps[0].State != wantState || writes.Load() != wantWrites { + t.Fatalf("receipt=%+v err=%v writes=%d", receipt, err, writes.Load()) + } + }) + } +} + +func TestApplyReactionTreatsUnconfirmedSuccessfulResponseAsUnknown(t *testing.T) { + var writes atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch request.URL.RequestURI() { + case "/api/v4/users/me": + _, _ = io.WriteString(response, `{"id":"self","username":"arda"}`) + case "/api/v4/posts/post-1": + _, _ = io.WriteString(response, `{"id":"post-1","channel_id":"channel-1","user_id":"author","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":"","type":"","file_ids":[]}`) + case "/api/v4/channels/channel-1": + _, _ = io.WriteString(response, `{"id":"channel-1","team_id":"team-1","type":"O","name":"town-square","display_name":"Town Square"}`) + case "/api/v4/channels/channel-1/members/self": + _, _ = io.WriteString(response, `{"channel_id":"channel-1","user_id":"self"}`) + case "/api/v4/posts/post-1/reactions": + _, _ = io.WriteString(response, `[]`) + case "/api/v4/reactions": + writes.Add(1) + _, _ = io.WriteString(response, `{"user_id":"self","post_id":"post-1","emoji_name":"eyes","create_at":1,"update_at":1,"delete_at":0,"remote_id":"","channel_id":"channel-1"}`) + default: + http.NotFound(response, request) + } + })) + defer server.Close() + store := openApplyStore(t) + stage := createReactionStage(t, store, server.URL+"/api/v4", stagestore.React, false) + service, client := applyService(t, server.URL, store) + defer client.Close() + receipt, err := service.Apply(context.Background(), applyClaim(stage, "apply-reaction-unconfirmed")) + if err != nil || receipt.Outcome != stagestore.OutcomeUnknown || receipt.Recovery != stagestore.RecoveryUnknown || receipt.Steps[0].State != stagestore.StepUnknown || writes.Load() != 1 { + t.Fatalf("receipt=%+v err=%v writes=%d", receipt, err, writes.Load()) + } +} + +func TestApplyReactionRejectsStaleDMMembershipBeforeDispatch(t *testing.T) { + var writes atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch request.URL.RequestURI() { + case "/api/v4/users/me": + _, _ = io.WriteString(response, `{"id":"self","username":"arda"}`) + case "/api/v4/posts/post-1": + _, _ = io.WriteString(response, `{"id":"post-1","channel_id":"dm-1","user_id":"peer","message":"hello","create_at":1,"update_at":1,"delete_at":0,"root_id":"","type":"","file_ids":[]}`) + case "/api/v4/users/self/channels": + _, _ = io.WriteString(response, `[{"id":"dm-1","team_id":"","type":"D","name":"peer__self","display_name":""}]`) + case "/api/v4/channels/dm-1/members?page=0&per_page=9": + _, _ = io.WriteString(response, `[{"channel_id":"dm-1","user_id":"self"}]`) + case "/api/v4/reactions": + writes.Add(1) + response.WriteHeader(http.StatusInternalServerError) + default: + http.NotFound(response, request) + } + })) + defer server.Close() + store := openApplyStore(t) + stage := createDMReactionStage(t, store, server.URL+"/api/v4") + service, client := applyService(t, server.URL, store) + defer client.Close() + _, err := service.Apply(context.Background(), applyClaim(stage, "apply-stale-dm-reaction")) + detail, showErr := store.Show(context.Background(), stage.Stage.ID) + if !errors.Is(err, ErrTargetDrift) || showErr != nil || detail.Lifecycle != stagestore.LifecycleOpen || writes.Load() != 0 { + t.Fatalf("detail=%+v applyErr=%v showErr=%v writes=%d", detail.StageSummary, err, showErr, writes.Load()) + } +} + func openApplyStore(t *testing.T) *stagestore.Store { t.Helper() store, err := stagestore.Open(context.Background(), filepath.Join(t.TempDir(), "state", "mattermost-cli", stagestore.DatabaseFilename)) @@ -353,6 +556,32 @@ func createResolveStage(t *testing.T, store *stagestore.Store, serverURL string, return result.MutationResult } +func createReactionStage(t *testing.T, store *stagestore.Store, serverURL string, operation stagestore.Operation, present bool) stagestore.MutationResult { + t.Helper() + destination := []byte(fmt.Sprintf(`{"channelId":"channel-1","channelType":"public","emoji":"eyes","kind":"reaction","participantIds":[],"postId":"post-1","postState":null,"reactionPresent":%t,"rootPostId":null,"teamId":"team-1"}`, present)) + stepType := "add_reaction" + if operation == stagestore.Unreact { + stepType = "remove_reaction" + } + plan := []byte(`{"steps":[{"ordinal":1,"type":"` + stepType + `","condition":"if_missing"}]}`) + result, err := store.Create(context.Background(), stagestore.CreateInput{RequestDigest: sha256.Sum256([]byte("reaction-stage\x00" + string(operation))), Operation: operation, ServerURL: serverURL, UserID: "self", Content: stagestore.RevisionContent{Destination: destination, Plan: plan}}) + if err != nil { + t.Fatal(err) + } + return result.MutationResult +} + +func createDMReactionStage(t *testing.T, store *stagestore.Store, serverURL string) stagestore.MutationResult { + t.Helper() + destination := []byte(`{"channelId":"dm-1","channelType":"dm","emoji":"eyes","kind":"reaction","participantIds":["peer"],"postId":"post-1","postState":null,"reactionPresent":false,"rootPostId":null,"teamId":null}`) + plan := []byte(`{"steps":[{"ordinal":1,"type":"add_reaction","condition":"if_missing"}]}`) + result, err := store.Create(context.Background(), stagestore.CreateInput{RequestDigest: sha256.Sum256([]byte("dm-reaction-stage")), Operation: stagestore.React, ServerURL: serverURL, UserID: "self", Content: stagestore.RevisionContent{Destination: destination, Plan: plan}}) + if err != nil { + t.Fatal(err) + } + return result.MutationResult +} + func applyClaim(stage stagestore.MutationResult, requestID string) stagestore.ApplyClaimInput { return stagestore.ApplyClaimInput{StageID: stage.Stage.ID, RequestID: requestID, Revision: stage.Stage.Revision, ExpectedDigest: stage.Stage.SemanticDigest, RequestDigest: sha256.Sum256([]byte(requestID)), RecoveryMode: stagestore.RecoveryModeOrdinary} } @@ -362,12 +591,16 @@ func applyService(t *testing.T, serverURL string, store *stagestore.Store) (*Ser } func applyServiceWithStore(t *testing.T, serverURL string, store Store) (*Service, *api.Client) { + return applyServiceWithCredentials(t, serverURL, store, [][]byte{[]byte("token")}) +} + +func applyServiceWithCredentials(t *testing.T, serverURL string, store Store, credentials [][]byte) (*Service, *api.Client) { t.Helper() client, err := api.New(serverURL, "token") if err != nil { t.Fatal(err) } - service, err := New(serverURL+"/api/v4", "", store, mattermost.NewUsers(client), mattermost.NewChannels(client), mattermost.NewConversationMutations(client)) + service, err := New(serverURL+"/api/v4", "", credentials, store, mattermost.NewUsers(client), mattermost.NewChannels(client), mattermost.NewPosts(client), mattermost.NewConversationMutations(client), mattermost.NewPostMutations(client)) if err != nil { client.Close() t.Fatal(err) From a5d1971147ff181982e835ec266f69c529574fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 08:04:07 +0300 Subject: [PATCH 087/119] feat: execute post apply plans --- internal/apply/post.go | 217 ++++++++++++++++++ internal/apply/post_test.go | 264 ++++++++++++++++++++++ internal/apply/service.go | 34 ++- internal/cli/store_test.go | 6 +- internal/schema/apply_semantic.go | 12 +- internal/schema/apply_test.go | 12 + internal/stagestore/apply.go | 8 +- internal/stagestore/apply_test.go | 10 +- internal/stagestore/schema.go | 123 ++++++++++ internal/staging/post.go | 30 ++- internal/staging/post_test.go | 8 + schemas/v2/apply-receipt.schema.json | 3 +- schemas/v2/examples/store-doctor.json | 2 +- schemas/v2/examples/store-migrations.json | 2 +- schemas/v2/store-doctor.schema.json | 6 +- schemas/v2/store-migrations.schema.json | 6 +- 16 files changed, 711 insertions(+), 32 deletions(-) create mode 100644 internal/apply/post.go create mode 100644 internal/apply/post_test.go diff --git a/internal/apply/post.go b/internal/apply/post.go new file mode 100644 index 0000000..9bee8d2 --- /dev/null +++ b/internal/apply/post.go @@ -0,0 +1,217 @@ +package apply + +import ( + "context" + "encoding/hex" + "encoding/json" + "errors" + "strings" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/internal/staging" +) + +func (s *Service) applyPost(ctx context.Context, attempt stagestore.ApplyAttempt, operation stagestore.Operation, currentUserID string, destination staging.Destination, body []byte) (stagestore.ApplyReceipt, error) { + switch operation { + case stagestore.CreatePost, stagestore.Reply: + rootID := "" + if destination.RootPostID != nil { + rootID = *destination.RootPostID + } + prepared, err := s.postWrites.PrepareCreate(mattermost.CreatePostMutationInput{ + ChannelID: destination.ChannelID, UserID: currentUserID, Message: string(body), RootID: rootID, PendingPostID: attempt.PendingPostID, + }) + if err != nil { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, err) + } + if err = s.revalidatePostTarget(ctx, operation, currentUserID, destination); err != nil { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, errors.Join(ErrTargetDrift, err)) + } + return executePrepared(s, ctx, attempt.ID, prepared.Execute) + case stagestore.EditPost: + post, err := s.revalidateBoundPost(ctx, currentUserID, destination) + if err != nil { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, errors.Join(ErrTargetDrift, err)) + } + if post.Message == string(body) { + return s.skip(ctx, attempt.ID) + } + prepared, err := s.postWrites.PrepareEdit(mattermost.EditPostMutationInput{ + PostID: post.ID, ChannelID: post.ChannelID, UserID: currentUserID, Message: string(body), RootID: post.RootID, FileIDs: post.FileIDs, + }) + if err != nil { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, err) + } + return executePrepared(s, ctx, attempt.ID, prepared.Execute) + case stagestore.DeletePost: + prepared, err := s.postWrites.PrepareDelete(mattermost.DeletePostMutationInput{PostID: *destination.PostID}) + if err != nil { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, err) + } + if _, err = s.revalidateBoundPost(ctx, currentUserID, destination); err != nil { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, errors.Join(ErrTargetDrift, err)) + } + return executePrepared(s, ctx, attempt.ID, prepared.Execute) + default: + return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, ErrUnsupportedOperation) + } +} + +func executePrepared[T any](s *Service, ctx context.Context, attemptID string, execute func(context.Context) (T, error)) (stagestore.ApplyReceipt, error) { + if err := s.store.BeginDispatch(ctx, attemptID, 1); err != nil { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attemptID, err) + } + result, remoteErr := execute(ctx) + if remoteErr != nil { + return s.recordRemoteFailure(ctx, attemptID, remoteErr) + } + encoded, err := json.Marshal(result) + if err != nil { + return stagestore.ApplyReceipt{}, &ConfirmedEffectError{err} + } + journalCtx := context.WithoutCancel(ctx) + if err = s.store.MarkStepValidated(journalCtx, attemptID, 1, encoded); err != nil { + return stagestore.ApplyReceipt{}, &ConfirmedEffectError{err} + } + receipt, err := s.finalizeReceipt(journalCtx, attemptID) + if err != nil { + return stagestore.ApplyReceipt{}, &ConfirmedEffectError{err} + } + return receipt, nil +} + +func (s *Service) revalidatePostTarget(ctx context.Context, operation stagestore.Operation, currentUserID string, destination staging.Destination) error { + if err := s.revalidateChannelAccess(ctx, currentUserID, destination); err != nil { + return err + } + if operation != stagestore.Reply { + return nil + } + post, err := s.posts.ByID(ctx, *destination.PostID) + if err != nil { + return err + } + if post.ID != *destination.PostID || post.ChannelID != destination.ChannelID { + return ErrTargetDrift + } + if post.RootID == "" { + if destination.RootPostID == nil || *destination.RootPostID != post.ID { + return ErrTargetDrift + } + } else if destination.RootPostID == nil || *destination.RootPostID != post.RootID { + return ErrTargetDrift + } + root, err := s.posts.ByID(ctx, *destination.RootPostID) + if err != nil { + return err + } + if root.ID != *destination.RootPostID || root.ChannelID != destination.ChannelID || root.RootID != "" { + return ErrTargetDrift + } + return nil +} + +func (s *Service) revalidateBoundPost(ctx context.Context, currentUserID string, destination staging.Destination) (mattermost.Post, error) { + post, err := s.posts.ByID(ctx, *destination.PostID) + if err != nil { + return mattermost.Post{}, err + } + state := destination.PostState + if post.ID != *destination.PostID || post.ChannelID != destination.ChannelID || post.UserID != currentUserID || post.Type != "" || state == nil || + post.UserID != state.AuthorUserID || post.UpdateAt != state.UpdateAt || staging.PostContentDigest(post, s.credentials) != state.ContentDigest { + return mattermost.Post{}, ErrTargetDrift + } + wantRoot := "" + if destination.RootPostID != nil { + wantRoot = *destination.RootPostID + } + if post.RootID != wantRoot { + return mattermost.Post{}, ErrTargetDrift + } + if err = s.revalidateChannelAccess(ctx, currentUserID, destination); err != nil { + return mattermost.Post{}, err + } + return post, nil +} + +func (s *Service) revalidateChannelAccess(ctx context.Context, currentUserID string, destination staging.Destination) error { + if destination.ChannelType == "dm" { + channel, found, err := s.channels.ExistingDirect(ctx, currentUserID, destination.ParticipantIDs[0]) + if err != nil { + return err + } + if !found || channel.ID != destination.ChannelID { + return ErrTargetDrift + } + return nil + } + channel, err := s.channels.ByID(ctx, destination.ChannelID) + if err != nil { + return err + } + if !channelMatchesDestination(channel, destination, currentUserID) { + return ErrTargetDrift + } + member, err := s.channels.Member(ctx, destination.ChannelID, currentUserID) + if err != nil { + return err + } + if member.ChannelID != destination.ChannelID || member.UserID != currentUserID { + return ErrTargetDrift + } + return nil +} + +func decodePostDestination(operation stagestore.Operation, raw json.RawMessage) (staging.Destination, error) { + if !canonicalDestination(raw) { + return staging.Destination{}, ErrInvalid + } + var destination staging.Destination + if json.Unmarshal(raw, &destination) != nil || !validDestinationChannel(destination) || destination.Emoji != nil || destination.ReactionPresent != nil { + return staging.Destination{}, ErrInvalid + } + switch operation { + case stagestore.CreatePost: + if destination.Kind != "conversation" || destination.PostID != nil || destination.RootPostID != nil || destination.PostState != nil { + return staging.Destination{}, ErrInvalid + } + case stagestore.Reply: + if destination.Kind != "post" || destination.PostID == nil || !safeID(*destination.PostID) || destination.RootPostID == nil || !safeID(*destination.RootPostID) || destination.PostState != nil { + return staging.Destination{}, ErrInvalid + } + case stagestore.EditPost, stagestore.DeletePost: + if destination.Kind != "post" || destination.PostID == nil || !safeID(*destination.PostID) || destination.PostState == nil || !validPostState(*destination.PostState) { + return staging.Destination{}, ErrInvalid + } + if destination.RootPostID != nil && !safeID(*destination.RootPostID) { + return staging.Destination{}, ErrInvalid + } + default: + return staging.Destination{}, ErrInvalid + } + return destination, nil +} + +func validDestinationChannel(destination staging.Destination) bool { + if !safeID(destination.ChannelID) { + return false + } + switch destination.ChannelType { + case "dm": + return destination.TeamID == nil && len(destination.ParticipantIDs) == 1 && safeID(destination.ParticipantIDs[0]) + case "group": + return destination.TeamID == nil && len(destination.ParticipantIDs) == 0 + case "public", "private": + return destination.TeamID != nil && safeID(*destination.TeamID) && len(destination.ParticipantIDs) == 0 + } + return false +} + +func validPostState(state staging.PostState) bool { + if !safeID(state.AuthorUserID) || state.UpdateAt <= 0 || len(state.ContentDigest) != 64 { + return false + } + decoded, err := hex.DecodeString(state.ContentDigest) + return err == nil && len(decoded) == 32 && state.ContentDigest == strings.ToLower(state.ContentDigest) +} diff --git a/internal/apply/post_test.go b/internal/apply/post_test.go new file mode 100644 index 0000000..55fac67 --- /dev/null +++ b/internal/apply/post_test.go @@ -0,0 +1,264 @@ +package apply + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/internal/staging" +) + +func TestApplyCreatePostPreservesStagedMarkdownExactly(t *testing.T) { + for name, message := range map[string]string{ + "short": "# heading\n\n**bold** and `code`\n", + "long": strings.Repeat("界", 16_382) + "\n", + } { + t.Run(name, func(t *testing.T) { + var writes atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch request.URL.RequestURI() { + case "/api/v4/users/me": + _, _ = io.WriteString(response, `{"id":"self","username":"arda"}`) + case "/api/v4/channels/channel-1": + _, _ = io.WriteString(response, `{"id":"channel-1","team_id":"team-1","type":"O","name":"town-square","display_name":"Town Square"}`) + case "/api/v4/channels/channel-1/members/self": + _, _ = io.WriteString(response, `{"channel_id":"channel-1","user_id":"self"}`) + case "/api/v4/posts": + writes.Add(1) + var input struct { + ChannelID string `json:"channel_id"` + Message string `json:"message"` + PendingPostID string `json:"pending_post_id"` + } + decoder := json.NewDecoder(request.Body) + if request.Method != http.MethodPost || decoder.Decode(&input) != nil || input.ChannelID != "channel-1" || input.Message != message || input.PendingPostID == "" { + response.WriteHeader(http.StatusBadRequest) + return + } + response.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(response, mutationPostResponse("created-1", "channel-1", "self", message, "", input.PendingPostID, nil, 100, 100)) + default: + http.NotFound(response, request) + } + })) + defer server.Close() + store := openApplyStore(t) + stage := createPostApplyStage(t, store, server.URL+"/api/v4", stagestore.CreatePost, message, staging.Destination{ + Kind: "conversation", ChannelID: "channel-1", ChannelType: "public", TeamID: stringPointer("team-1"), ParticipantIDs: []string{}, + }) + service, client := applyService(t, server.URL, store) + defer client.Close() + receipt, err := service.Apply(context.Background(), applyClaim(stage, "apply-create-"+name)) + if err != nil || receipt.Outcome != stagestore.OutcomeSucceeded || receipt.Steps[0].State != stagestore.StepValidated || writes.Load() != 1 { + t.Fatalf("receipt=%+v err=%v writes=%d", receipt, err, writes.Load()) + } + }) + } +} + +func TestApplyReplyRevalidatesSelectedPostAndCanonicalRoot(t *testing.T) { + const message = "thread **reply**\n" + var writes atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch request.URL.RequestURI() { + case "/api/v4/users/me": + _, _ = io.WriteString(response, `{"id":"self","username":"arda"}`) + case "/api/v4/posts/reply-1": + _, _ = io.WriteString(response, livePostJSON("reply-1", "channel-1", "peer", "old", "root-1", nil, 2)) + case "/api/v4/posts/root-1": + _, _ = io.WriteString(response, livePostJSON("root-1", "channel-1", "peer", "root", "", nil, 1)) + case "/api/v4/channels/channel-1": + _, _ = io.WriteString(response, `{"id":"channel-1","team_id":"team-1","type":"P","name":"private","display_name":"Private"}`) + case "/api/v4/channels/channel-1/members/self": + _, _ = io.WriteString(response, `{"channel_id":"channel-1","user_id":"self"}`) + case "/api/v4/posts": + writes.Add(1) + var input struct { + ChannelID string `json:"channel_id"` + Message string `json:"message"` + RootID string `json:"root_id"` + PendingPostID string `json:"pending_post_id"` + } + if json.NewDecoder(request.Body).Decode(&input) != nil || input.ChannelID != "channel-1" || input.Message != message || input.RootID != "root-1" { + response.WriteHeader(http.StatusBadRequest) + return + } + response.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(response, mutationPostResponse("new-reply", "channel-1", "self", message, "root-1", input.PendingPostID, nil, 100, 100)) + default: + http.NotFound(response, request) + } + })) + defer server.Close() + store := openApplyStore(t) + stage := createPostApplyStage(t, store, server.URL+"/api/v4", stagestore.Reply, message, staging.Destination{ + Kind: "post", ChannelID: "channel-1", ChannelType: "private", TeamID: stringPointer("team-1"), PostID: stringPointer("reply-1"), RootPostID: stringPointer("root-1"), ParticipantIDs: []string{}, + }) + service, client := applyService(t, server.URL, store) + defer client.Close() + receipt, err := service.Apply(context.Background(), applyClaim(stage, "apply-reply")) + if err != nil || receipt.Outcome != stagestore.OutcomeSucceeded || writes.Load() != 1 { + t.Fatalf("receipt=%+v err=%v writes=%d", receipt, err, writes.Load()) + } +} + +func TestApplyEditSkipsSatisfiedAndRejectsBoundStateDrift(t *testing.T) { + for _, test := range []struct { + name, liveMessage, stagedMessage string + liveUpdate int64 + wantSkip bool + wantDrift bool + }{ + {name: "already-satisfied", liveMessage: "same", stagedMessage: "same", liveUpdate: 2, wantSkip: true}, + {name: "stale-update", liveMessage: "old", stagedMessage: "new", liveUpdate: 3, wantDrift: true}, + } { + t.Run(test.name, func(t *testing.T) { + var writes atomic.Int32 + bound := mattermost.Post{ID: "post-1", ChannelID: "channel-1", UserID: "self", Message: test.liveMessage, CreateAt: 1, UpdateAt: 2, RootID: "", Type: "", FileIDs: []string{}} + server := postTargetServer(t, func(response http.ResponseWriter, request *http.Request) bool { + if request.URL.RequestURI() == "/api/v4/posts/post-1" { + _, _ = io.WriteString(response, livePostJSON("post-1", "channel-1", "self", test.liveMessage, "", nil, test.liveUpdate)) + return true + } + if request.URL.RequestURI() == "/api/v4/posts/post-1/patch" { + writes.Add(1) + response.WriteHeader(http.StatusInternalServerError) + return true + } + return false + }) + defer server.Close() + store := openApplyStore(t) + stage := createPostApplyStage(t, store, server.URL+"/api/v4", stagestore.EditPost, test.stagedMessage, boundPostDestination(bound)) + service, client := applyService(t, server.URL, store) + defer client.Close() + receipt, err := service.Apply(context.Background(), applyClaim(stage, "apply-edit-"+test.name)) + if test.wantDrift { + detail, showErr := store.Show(context.Background(), stage.Stage.ID) + if !errors.Is(err, ErrTargetDrift) || showErr != nil || detail.Lifecycle != stagestore.LifecycleOpen || writes.Load() != 0 { + t.Fatalf("detail=%+v receipt=%+v err=%v showErr=%v writes=%d", detail.StageSummary, receipt, err, showErr, writes.Load()) + } + return + } + if err != nil || !test.wantSkip || receipt.Outcome != stagestore.OutcomeAlreadySatisfied || writes.Load() != 0 { + t.Fatalf("receipt=%+v err=%v writes=%d", receipt, err, writes.Load()) + } + }) + } +} + +func TestApplyEditAndDeleteExecuteOneBoundMutation(t *testing.T) { + for _, operation := range []stagestore.Operation{stagestore.EditPost, stagestore.DeletePost} { + t.Run(string(operation), func(t *testing.T) { + var writes atomic.Int32 + post := mattermost.Post{ID: "post-1", ChannelID: "channel-1", UserID: "self", Message: "old", CreateAt: 1, UpdateAt: 2, RootID: "", Type: "", FileIDs: []string{}} + server := postTargetServer(t, func(response http.ResponseWriter, request *http.Request) bool { + if request.URL.RequestURI() == "/api/v4/posts/post-1" && request.Method == http.MethodGet { + _, _ = io.WriteString(response, livePostJSON("post-1", "channel-1", "self", "old", "", nil, 2)) + return true + } + if request.URL.RequestURI() == "/api/v4/posts/post-1/patch" && request.Method == http.MethodPut { + writes.Add(1) + _, _ = io.WriteString(response, mutationPostResponse("post-1", "channel-1", "self", "new", "", "", nil, 1, 3)) + return true + } + return false + }) + if operation == stagestore.DeletePost { + server.Close() + server = postTargetServer(t, func(response http.ResponseWriter, request *http.Request) bool { + if request.URL.RequestURI() == "/api/v4/posts/post-1" && request.Method == http.MethodGet { + _, _ = io.WriteString(response, livePostJSON("post-1", "channel-1", "self", "old", "", nil, 2)) + return true + } + if request.URL.RequestURI() == "/api/v4/posts/post-1" && request.Method == http.MethodDelete { + writes.Add(1) + _, _ = io.WriteString(response, `{"status":"OK"}`) + return true + } + return false + }) + } + defer server.Close() + body := "new" + if operation == stagestore.DeletePost { + body = "" + } + store := openApplyStore(t) + stage := createPostApplyStage(t, store, server.URL+"/api/v4", operation, body, boundPostDestination(post)) + service, client := applyService(t, server.URL, store) + defer client.Close() + receipt, err := service.Apply(context.Background(), applyClaim(stage, "apply-"+string(operation))) + if err != nil || receipt.Outcome != stagestore.OutcomeSucceeded || writes.Load() != 1 { + t.Fatalf("receipt=%+v err=%v writes=%d", receipt, err, writes.Load()) + } + }) + } +} + +func createPostApplyStage(t *testing.T, store *stagestore.Store, serverURL string, operation stagestore.Operation, body string, destination staging.Destination) stagestore.MutationResult { + t.Helper() + destinationJSON, err := json.Marshal(destination) + if err != nil { + t.Fatal(err) + } + step := "create_post" + if operation == stagestore.EditPost { + step = "edit_post" + } else if operation == stagestore.DeletePost { + step = "delete_post" + } + plan := json.RawMessage(fmt.Sprintf(`{"steps":[{"ordinal":1,"type":%q,"condition":"always"}]}`, step)) + result, err := store.Create(context.Background(), stagestore.CreateInput{RequestDigest: sha256.Sum256([]byte(string(operation) + "\x00" + body)), Operation: operation, ServerURL: serverURL, UserID: "self", Content: stagestore.RevisionContent{Body: []byte(body), Destination: destinationJSON, Plan: plan}}) + if err != nil { + t.Fatal(err) + } + return result.MutationResult +} + +func boundPostDestination(post mattermost.Post) staging.Destination { + postID := post.ID + return staging.Destination{Kind: "post", ChannelID: post.ChannelID, ChannelType: "public", TeamID: stringPointer("team-1"), PostID: &postID, ParticipantIDs: []string{}, PostState: &staging.PostState{ + AuthorUserID: post.UserID, UpdateAt: post.UpdateAt, ContentDigest: staging.PostContentDigest(post, [][]byte{[]byte("token")}), + }} +} + +func postTargetServer(t *testing.T, handle func(http.ResponseWriter, *http.Request) bool) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch request.URL.RequestURI() { + case "/api/v4/users/me": + _, _ = io.WriteString(response, `{"id":"self","username":"arda"}`) + case "/api/v4/channels/channel-1": + _, _ = io.WriteString(response, `{"id":"channel-1","team_id":"team-1","type":"O","name":"town-square","display_name":"Town Square"}`) + case "/api/v4/channels/channel-1/members/self": + _, _ = io.WriteString(response, `{"channel_id":"channel-1","user_id":"self"}`) + default: + if !handle(response, request) { + http.NotFound(response, request) + } + } + })) +} + +func livePostJSON(id, channelID, userID, message, rootID string, fileIDs []string, updateAt int64) string { + raw, _ := json.Marshal(map[string]any{"id": id, "channel_id": channelID, "user_id": userID, "message": message, "create_at": 1, "update_at": updateAt, "delete_at": 0, "root_id": rootID, "type": "", "file_ids": fileIDs}) + return string(raw) +} + +func mutationPostResponse(id, channelID, userID, message, rootID, pendingID string, fileIDs []string, createAt, updateAt int64) string { + raw, _ := json.Marshal(map[string]any{"id": id, "channel_id": channelID, "user_id": userID, "message": message, "create_at": createAt, "update_at": updateAt, "delete_at": 0, "root_id": rootID, "file_ids": fileIDs, "pending_post_id": pendingID, "type": ""}) + return string(raw) +} + +func stringPointer(value string) *string { return &value } diff --git a/internal/apply/service.go b/internal/apply/service.go index e327919..f058c39 100644 --- a/internal/apply/service.go +++ b/internal/apply/service.go @@ -95,19 +95,22 @@ func (s *Service) Apply(ctx context.Context, in stagestore.ApplyClaimInput) (sta if detail.ServerURL != s.serverURL || detail.ServerID != s.serverID { return stagestore.ApplyReceipt{}, ErrTargetDrift } - if detail.Operation != stagestore.ResolveDM && detail.Operation != stagestore.ResolveGroupDM && detail.Operation != stagestore.React && detail.Operation != stagestore.Unreact { + if !supportedOperation(detail.Operation) { return stagestore.ApplyReceipt{}, ErrUnsupportedOperation } var destination staging.Destination - if detail.Operation == stagestore.ResolveDM || detail.Operation == stagestore.ResolveGroupDM { + switch detail.Operation { + case stagestore.ResolveDM, stagestore.ResolveGroupDM: destination, err = decodeResolveDestination(detail.Operation, detail.Destination, detail.UserID) - } else { + case stagestore.React, stagestore.Unreact: destination, err = decodeReactionDestination(detail.Destination) + default: + destination, err = decodePostDestination(detail.Operation, detail.Destination) } if err != nil { return stagestore.ApplyReceipt{}, err } - if s.destinationContainsCredential(destination, detail.UserID) { + if s.destinationContainsCredential(destination, detail.UserID) || s.rawContainsCredentialValue(detail.Plan) { return stagestore.ApplyReceipt{}, ErrCredential } if in.RequestID != "" { @@ -125,6 +128,12 @@ func (s *Service) Apply(ctx context.Context, in stagestore.ApplyClaimInput) (sta if detail.Revision != in.Revision || detail.SemanticDigest != in.ExpectedDigest { return stagestore.ApplyReceipt{}, stagestore.ErrConflict } + if s.containsCredential(string(detail.Body)) { + return stagestore.ApplyReceipt{}, ErrCredential + } + if len(detail.Attachments) > 0 { + return stagestore.ApplyReceipt{}, ErrUnsupportedOperation + } attempt, err := s.store.ClaimApply(ctx, in) if err != nil { return stagestore.ApplyReceipt{}, err @@ -132,6 +141,9 @@ func (s *Service) Apply(ctx context.Context, in stagestore.ApplyClaimInput) (sta if attempt.Replay { return s.finalizeReceipt(ctx, attempt.ID) } + if s.containsCredential(attempt.ID, attempt.PendingPostID) { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, ErrCredential) + } current, err := s.users.Current(ctx) if err != nil { return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, err) @@ -142,9 +154,20 @@ func (s *Service) Apply(ctx context.Context, in stagestore.ApplyClaimInput) (sta if detail.Operation == stagestore.React || detail.Operation == stagestore.Unreact { return s.applyReaction(ctx, attempt, detail.Operation, current.ID, destination) } + if detail.Operation == stagestore.CreatePost || detail.Operation == stagestore.Reply || detail.Operation == stagestore.EditPost || detail.Operation == stagestore.DeletePost { + return s.applyPost(ctx, attempt, detail.Operation, current.ID, destination, detail.Body) + } return s.applyConversation(ctx, attempt, detail.Operation, current.ID, destination) } +func supportedOperation(operation stagestore.Operation) bool { + switch operation { + case stagestore.CreatePost, stagestore.Reply, stagestore.EditPost, stagestore.DeletePost, stagestore.React, stagestore.Unreact, stagestore.ResolveDM, stagestore.ResolveGroupDM: + return true + } + return false +} + func (s *Service) applyConversation(ctx context.Context, attempt stagestore.ApplyAttempt, operation stagestore.Operation, currentUserID string, destination staging.Destination) (stagestore.ApplyReceipt, error) { var prepared *mattermost.PreparedResolveConversation var err error @@ -329,6 +352,9 @@ func (s *Service) destinationContainsCredential(destination staging.Destination, values = append(values, *optional) } } + if destination.PostState != nil { + values = append(values, destination.PostState.AuthorUserID, destination.PostState.ContentDigest) + } return s.containsCredential(values...) } diff --git a/internal/cli/store_test.go b/internal/cli/store_test.go index f79fbac..4a87f23 100644 --- a/internal/cli/store_test.go +++ b/internal/cli/store_test.go @@ -36,7 +36,7 @@ func TestStoreDoctorAbsentIsReadOnlyAndSchemaValid(t *testing.T) { if err := registry.Validate("mm/v2/store-doctor", bytes.NewReader(stdout.Bytes())); err != nil { t.Fatalf("schema: %v\n%s", err, stdout.String()) } - for _, fact := range []string{`"filesystemSafe":null`, `"applicationId":null`, `"integrity":null`, `"applied":null`, `"latest":7`, `"valid":null`, `"journalMode":null`} { + for _, fact := range []string{`"filesystemSafe":null`, `"applicationId":null`, `"integrity":null`, `"applied":null`, `"latest":8`, `"valid":null`, `"journalMode":null`} { if !strings.Contains(stdout.String(), fact) { t.Fatalf("absent report omitted %s: %s", fact, stdout.String()) } @@ -110,7 +110,7 @@ func TestStoreMigrationsIsOfflineAndSchemaValid(t *testing.T) { if err := registry.Validate("mm/v2/store-migrations", bytes.NewReader(stdout.Bytes())); err != nil { t.Fatalf("schema: %v\n%s", err, stdout.String()) } - want := "{\"schema\":\"mm/v2/store-migrations\",\"latest\":7,\"migrations\":[{\"version\":1,\"name\":\"core-stage-state\",\"checksum\":\"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f\"},{\"version\":2,\"name\":\"immutable-local-request-receipts\",\"checksum\":\"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c\"},{\"version\":3,\"name\":\"caller-intent-stage-create-replay\",\"checksum\":\"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5\"},{\"version\":4,\"name\":\"caller-intent-stage-revise-replay\",\"checksum\":\"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4\"},{\"version\":5,\"name\":\"revision-plan-follows-composition\",\"checksum\":\"fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0\"},{\"version\":6,\"name\":\"durable-apply-journal\",\"checksum\":\"4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56\"},{\"version\":7,\"name\":\"status-confirmed-delete-results\",\"checksum\":\"bf980f24ff24fdf9d6a3a28b9ded97b51336fa1c8ef290d1bdcf91f1b593c5ce\"}]}\n" + want := "{\"schema\":\"mm/v2/store-migrations\",\"latest\":8,\"migrations\":[{\"version\":1,\"name\":\"core-stage-state\",\"checksum\":\"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f\"},{\"version\":2,\"name\":\"immutable-local-request-receipts\",\"checksum\":\"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c\"},{\"version\":3,\"name\":\"caller-intent-stage-create-replay\",\"checksum\":\"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5\"},{\"version\":4,\"name\":\"caller-intent-stage-revise-replay\",\"checksum\":\"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4\"},{\"version\":5,\"name\":\"revision-plan-follows-composition\",\"checksum\":\"fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0\"},{\"version\":6,\"name\":\"durable-apply-journal\",\"checksum\":\"4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56\"},{\"version\":7,\"name\":\"status-confirmed-delete-results\",\"checksum\":\"bf980f24ff24fdf9d6a3a28b9ded97b51336fa1c8ef290d1bdcf91f1b593c5ce\"},{\"version\":8,\"name\":\"already-satisfied-edit-apply\",\"checksum\":\"cff30dd4fcc987876e092e2de1f5afd21ebeadbcb792fd550a56eb4c221edd10\"}]}\n" if stdout.String() != want { t.Fatalf("stdout does not match golden migration contract: %q", stdout.String()) } @@ -310,7 +310,7 @@ func TestStoreHumanOutputStatesBounds(t *testing.T) { t.Fatalf("stdout=%q", stdout.String()) } stdout.Reset() - if code := Execute(context.Background(), []string{"store", "migrations"}, strings.NewReader(""), &stdout, &stderr); code != 0 || !strings.Contains(stdout.String(), "latest: 7\n1 core-stage-state ") { + if code := Execute(context.Background(), []string{"store", "migrations"}, strings.NewReader(""), &stdout, &stderr); code != 0 || !strings.Contains(stdout.String(), "latest: 8\n1 core-stage-state ") { t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } } diff --git a/internal/schema/apply_semantic.go b/internal/schema/apply_semantic.go index 0cab794..b351782 100644 --- a/internal/schema/apply_semantic.go +++ b/internal/schema/apply_semantic.go @@ -28,7 +28,9 @@ type applyReceiptDestination struct { PostID *string `json:"postId"` RootPostID *string `json:"rootPostId"` ParticipantIDs []string `json:"participantIds"` + Emoji json.RawMessage `json:"emoji"` PostState json.RawMessage `json:"postState"` + ReactionState json.RawMessage `json:"reactionPresent"` } type applyReceiptStep struct { @@ -159,11 +161,11 @@ func validApplyReceiptPlan(operation string, destination applyReceiptDestination case "create_post": return validResolvedConversation(destination) && validCreateSteps(steps) case "reply": - return validChannelDestination(destination) && destination.Kind == "post" && destination.PostID != nil && destination.RootPostID != nil && isJSONNull(destination.PostState) && validCreateSteps(steps) + return validChannelDestination(destination) && validPostOnlyDestination(destination) && destination.PostID != nil && destination.RootPostID != nil && isJSONNull(destination.PostState) && validCreateSteps(steps) case "edit_post": - return validChannelDestination(destination) && destination.Kind == "post" && destination.PostID != nil && !isJSONNull(destination.PostState) && single("edit_post", "always") + return validChannelDestination(destination) && validPostOnlyDestination(destination) && destination.PostID != nil && !isJSONNull(destination.PostState) && single("edit_post", "always") case "delete_post": - return validChannelDestination(destination) && destination.Kind == "post" && destination.PostID != nil && !isJSONNull(destination.PostState) && single("delete_post", "always") + return validChannelDestination(destination) && validPostOnlyDestination(destination) && destination.PostID != nil && !isJSONNull(destination.PostState) && single("delete_post", "always") case "react": return validChannelDestination(destination) && destination.Kind == "reaction" && destination.PostID != nil && single("add_reaction", "if_missing") case "unreact": @@ -177,6 +179,10 @@ func validApplyReceiptPlan(operation string, destination applyReceiptDestination } } +func validPostOnlyDestination(destination applyReceiptDestination) bool { + return destination.Kind == "post" && isJSONNull(destination.Emoji) && isJSONNull(destination.ReactionState) +} + func validCreateSteps(steps []applyReceiptStep) bool { if len(steps) == 0 || steps[len(steps)-1].Kind != "create_post" || steps[len(steps)-1].Condition != "always" { return false diff --git a/internal/schema/apply_test.go b/internal/schema/apply_test.go index 8fc0ad0..b8e9c3d 100644 --- a/internal/schema/apply_test.go +++ b/internal/schema/apply_test.go @@ -227,6 +227,18 @@ func TestApplyReceiptAcceptsRealisticEditProjection(t *testing.T) { if err := registry.Validate("mm/v2/apply-receipt", bytes.NewReader(encoded)); err != nil { t.Fatalf("rejected realistic edit receipt: %s", encoded) } + for field, hostile := range map[string]any{"emoji": "eyes", "reactionPresent": true} { + destination := doc["destination"].(map[string]any) + destination[field] = hostile + encoded, err = json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/apply-receipt", bytes.NewReader(encoded)); err == nil { + t.Fatalf("accepted post receipt with %s: %s", field, encoded) + } + destination[field] = nil + } } func TestApplyReceiptAcceptsStatusConfirmedDeleteProjection(t *testing.T) { diff --git a/internal/stagestore/apply.go b/internal/stagestore/apply.go index c31c4a0..e87e2f6 100644 --- a/internal/stagestore/apply.go +++ b/internal/stagestore/apply.go @@ -252,11 +252,11 @@ OR EXISTS(SELECT 1 FROM apply_steps later WHERE later.attempt_id=current.attempt return ErrNotEligible } if to == StepSkipped { - var condition string - if err = tx.QueryRowContext(ctx, `SELECT condition FROM apply_steps WHERE attempt_id=? AND ordinal=?`, attemptID, ordinal).Scan(&condition); err != nil { + var kind, condition string + if err = tx.QueryRowContext(ctx, `SELECT kind,condition FROM apply_steps WHERE attempt_id=? AND ordinal=?`, attemptID, ordinal).Scan(&kind, &condition); err != nil { return localError(err) } - if condition != "if_missing" { + if condition != "if_missing" && !(kind == "edit_post" && condition == "always") { return ErrNotEligible } } @@ -768,7 +768,7 @@ func validApplyStep(step ApplyStep) bool { case StepValidated, StepRejected: return step.StartedAt != nil && step.EndedAt != nil && step.Result != nil && !step.EndedAt.Before(*step.StartedAt) case StepSkipped: - return step.Condition == "if_missing" && step.StartedAt == nil && step.EndedAt != nil && step.Result != nil + return (step.Condition == "if_missing" || step.Kind == "edit_post" && step.Condition == "always") && step.StartedAt == nil && step.EndedAt != nil && step.Result != nil case StepUnknown: return step.StartedAt != nil && step.EndedAt != nil && step.Result == nil && !step.EndedAt.Before(*step.StartedAt) case StepNotSent: diff --git a/internal/stagestore/apply_test.go b/internal/stagestore/apply_test.go index af31ec2..c7ce697 100644 --- a/internal/stagestore/apply_test.go +++ b/internal/stagestore/apply_test.go @@ -243,7 +243,7 @@ func TestApplySuccessClearsSupersededRevisionPlaintextAndPaths(t *testing.T) { } } -func TestApplyReceiptRejectsBroadResultsAndUnconditionalSkip(t *testing.T) { +func TestApplyReceiptRejectsBroadResultsAndAllowsOnlySatisfiedEditUnconditionalSkip(t *testing.T) { s := openDomainStore(t) created := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) claim, _ := s.ClaimApply(context.Background(), claimInput(created.Stage, "", RecoveryModeOrdinary)) @@ -273,8 +273,12 @@ func TestApplyReceiptRejectsBroadResultsAndUnconditionalSkip(t *testing.T) { if err != nil { t.Fatal(err) } - if err = s.MarkStepSkipped(context.Background(), editClaim.ID, 1, json.RawMessage(`{"reason":"already_satisfied"}`)); !errors.Is(err, ErrNotEligible) { - t.Fatalf("unconditional edit skip=%v", err) + if err = s.MarkStepSkipped(context.Background(), editClaim.ID, 1, json.RawMessage(`{"reason":"already_satisfied"}`)); err != nil { + t.Fatalf("satisfied edit skip=%v", err) + } + receipt, err := s.FinalizeApply(context.Background(), editClaim.ID) + if err != nil || receipt.Outcome != OutcomeAlreadySatisfied || receipt.Steps[0].Condition != "always" || receipt.Steps[0].State != StepSkipped { + t.Fatalf("receipt=%+v err=%v", receipt, err) } } diff --git a/internal/stagestore/schema.go b/internal/stagestore/schema.go index 4289d78..1526f49 100644 --- a/internal/stagestore/schema.go +++ b/internal/stagestore/schema.go @@ -362,4 +362,127 @@ WHEN NEW.state IN ('response_validated','rejected','skipped') AND NOT EXISTS( ) ) BEGIN SELECT RAISE(ABORT, 'invalid apply step result binding'); END; +`}, {version: 8, name: "already-satisfied-edit-apply", sql: ` +DROP TRIGGER apply_step_identity_immutable; +DROP TRIGGER apply_step_state_transition_valid; +DROP TRIGGER apply_step_state_transition_required; +DROP TRIGGER apply_step_result_transition_valid; +DROP TRIGGER apply_steps_history_immutable_delete; +DROP TRIGGER stage_apply_claim_release_valid; +DROP TRIGGER stage_lifecycle_recovery_transition_valid; +DROP TRIGGER apply_attempt_history_immutable_delete; +DROP TRIGGER apply_events_history_immutable_delete; +DROP TRIGGER apply_requests_history_immutable_delete; +CREATE TABLE apply_steps_v8 ( + attempt_id TEXT NOT NULL REFERENCES apply_attempts(id) ON DELETE CASCADE, + ordinal INTEGER NOT NULL CHECK (ordinal > 0), + kind TEXT NOT NULL, + condition TEXT NOT NULL CHECK (condition IN ('always','if_missing')), + state TEXT NOT NULL CHECK (state IN ('pending','dispatch_intent','response_validated','rejected','outcome_unknown','skipped','not_dispatched')), + result_json TEXT CHECK (result_json IS NULL OR json_valid(result_json)), + started_at TEXT, + ended_at TEXT, + PRIMARY KEY (attempt_id, ordinal), + CHECK ( + state='pending' AND result_json IS NULL AND started_at IS NULL AND ended_at IS NULL + OR state='dispatch_intent' AND result_json IS NULL AND started_at IS NOT NULL AND ended_at IS NULL + OR state IN ('response_validated','rejected') AND result_json IS NOT NULL AND started_at IS NOT NULL AND ended_at IS NOT NULL + OR state='outcome_unknown' AND result_json IS NULL AND started_at IS NOT NULL AND ended_at IS NOT NULL + OR state='skipped' AND (condition='if_missing' OR kind='edit_post' AND condition='always') AND result_json IS NOT NULL AND started_at IS NULL AND ended_at IS NOT NULL + OR state='not_dispatched' AND result_json IS NULL AND started_at IS NULL AND ended_at IS NOT NULL + ) +) STRICT; +INSERT INTO apply_steps_v8(attempt_id,ordinal,kind,condition,state,result_json,started_at,ended_at) +SELECT attempt_id,ordinal,kind,condition,state,result_json,started_at,ended_at FROM apply_steps; +DROP TABLE apply_steps; +ALTER TABLE apply_steps_v8 RENAME TO apply_steps; +CREATE TRIGGER apply_step_identity_immutable BEFORE UPDATE ON apply_steps +WHEN NEW.attempt_id IS NOT OLD.attempt_id OR NEW.ordinal IS NOT OLD.ordinal + OR NEW.kind IS NOT OLD.kind OR NEW.condition IS NOT OLD.condition +BEGIN SELECT RAISE(ABORT, 'apply step identity is immutable'); END; +CREATE TRIGGER apply_step_state_transition_valid BEFORE UPDATE OF state ON apply_steps +WHEN NOT ( + OLD.state='pending' AND NEW.state IN ('dispatch_intent','skipped','not_dispatched') + OR OLD.state='dispatch_intent' AND NEW.state IN ('response_validated','rejected','outcome_unknown') +) +BEGIN SELECT RAISE(ABORT, 'invalid apply step transition'); END; +CREATE TRIGGER apply_step_state_transition_required BEFORE UPDATE ON apply_steps +WHEN NEW.state IS OLD.state +BEGIN SELECT RAISE(ABORT, 'apply step history is immutable'); END; +CREATE TRIGGER apply_step_result_transition_valid BEFORE UPDATE ON apply_steps +WHEN NEW.state IN ('response_validated','rejected','skipped') AND NOT EXISTS( + SELECT 1 FROM apply_attempts a JOIN stages s ON s.id=a.stage_id + JOIN stage_revisions r ON r.stage_id=a.stage_id AND r.revision=a.revision + WHERE a.id=NEW.attempt_id AND json_type(NEW.result_json)='object' AND ( + NEW.state='rejected' AND (SELECT count(*) FROM json_each(NEW.result_json))=1 + AND json_type(NEW.result_json,'$.status')='integer' AND json_extract(NEW.result_json,'$.status') BETWEEN 400 AND 499 + OR NEW.state='skipped' AND (NEW.condition='if_missing' OR NEW.kind='edit_post' AND NEW.condition='always') AND (SELECT count(*) FROM json_each(NEW.result_json))=1 + AND json_type(NEW.result_json,'$.reason')='text' AND json_extract(NEW.result_json,'$.reason')='already_satisfied' + OR NEW.state='response_validated' AND NEW.kind='upload_attachment' AND (SELECT count(*) FROM json_each(NEW.result_json))=1 + AND json_type(NEW.result_json,'$.fileId')='text' AND length(json_extract(NEW.result_json,'$.fileId'))>0 + OR NEW.state='response_validated' AND NEW.kind='create_post' AND (SELECT count(*) FROM json_each(NEW.result_json))=5 + AND json_type(NEW.result_json,'$.postId')='text' AND length(json_extract(NEW.result_json,'$.postId'))>0 + AND json_type(NEW.result_json,'$.createAt')='integer' AND json_extract(NEW.result_json,'$.createAt')>0 + AND json_extract(NEW.result_json,'$.channelId')=json_extract(r.destination_json,'$.channelId') + AND json_extract(NEW.result_json,'$.userId')=s.user_id AND json_extract(NEW.result_json,'$.pendingPostId')=a.pending_post_id + OR NEW.state='response_validated' AND NEW.kind='edit_post' AND (SELECT count(*) FROM json_each(NEW.result_json))=2 + AND json_extract(NEW.result_json,'$.postId')=json_extract(r.destination_json,'$.postId') + AND json_type(NEW.result_json,'$.updateAt')='integer' AND json_extract(NEW.result_json,'$.updateAt')>0 + OR NEW.state='response_validated' AND NEW.kind='delete_post' AND (SELECT count(*) FROM json_each(NEW.result_json))=1 + AND json_extract(NEW.result_json,'$.postId')=json_extract(r.destination_json,'$.postId') + OR NEW.state='response_validated' AND NEW.kind IN ('add_reaction','remove_reaction') AND (SELECT count(*) FROM json_each(NEW.result_json))=1 + AND json_extract(NEW.result_json,'$.postId')=json_extract(r.destination_json,'$.postId') + OR NEW.state='response_validated' AND NEW.kind='resolve_conversation' AND (SELECT count(*) FROM json_each(NEW.result_json))=2 + AND json_type(NEW.result_json,'$.channelId')='text' AND length(json_extract(NEW.result_json,'$.channelId'))>0 + AND (json_extract(r.destination_json,'$.channelId') IS NULL OR json_extract(NEW.result_json,'$.channelId')=json_extract(r.destination_json,'$.channelId')) + AND json_extract(NEW.result_json,'$.participantIds')=json_extract(r.destination_json,'$.participantIds') + ) +) +BEGIN SELECT RAISE(ABORT, 'invalid apply step result binding'); END; +CREATE TRIGGER apply_steps_history_immutable_delete BEFORE DELETE ON apply_steps +WHEN OLD.state!='pending' OR NOT EXISTS( + SELECT 1 FROM apply_attempts a JOIN stages s ON s.id=a.stage_id + WHERE a.id=OLD.attempt_id AND a.outcome IS NULL AND s.lifecycle='open' AND s.claim_attempt_id IS NULL +) +BEGIN SELECT RAISE(ABORT, 'dispatched apply steps are immutable'); END; +CREATE TRIGGER stage_apply_claim_release_valid BEFORE UPDATE OF lifecycle,claim_attempt_id ON stages +WHEN OLD.lifecycle='applying' AND NEW.lifecycle!='applying' AND EXISTS( + SELECT 1 FROM apply_attempts a WHERE a.id=OLD.claim_attempt_id AND a.outcome IS NULL + AND EXISTS(SELECT 1 FROM apply_steps p WHERE p.attempt_id=a.id AND p.state!='pending') +) +BEGIN SELECT RAISE(ABORT, 'dispatched apply claim cannot be released'); END; +CREATE TRIGGER stage_lifecycle_recovery_transition_valid BEFORE UPDATE OF lifecycle,recovery ON stages +WHEN (NEW.lifecycle IS NOT OLD.lifecycle OR NEW.recovery IS NOT OLD.recovery) AND NOT ( + OLD.lifecycle='open' AND NEW.lifecycle='applying' AND NEW.recovery=OLD.recovery AND NEW.claim_attempt_id IS NOT NULL + AND EXISTS(SELECT 1 FROM apply_attempts a WHERE a.id=NEW.claim_attempt_id AND a.stage_id=OLD.id AND a.outcome IS NULL) + OR OLD.lifecycle='applying' AND NEW.lifecycle='open' AND NEW.recovery=OLD.recovery AND NEW.claim_attempt_id IS NULL + AND EXISTS(SELECT 1 FROM apply_attempts a WHERE a.id=OLD.claim_attempt_id AND a.outcome IS NULL) + AND NOT EXISTS(SELECT 1 FROM apply_steps p WHERE p.attempt_id=OLD.claim_attempt_id AND p.state!='pending') + OR OLD.lifecycle='applying' AND NEW.claim_attempt_id IS NULL AND EXISTS( + SELECT 1 FROM apply_attempts a WHERE a.id=OLD.claim_attempt_id AND a.outcome IS NOT NULL AND ( + a.outcome IN ('succeeded','already_satisfied') AND NEW.lifecycle='completed' AND NEW.recovery='forbidden' + OR a.outcome='rejected' AND NEW.lifecycle='open' AND NEW.recovery=a.prior_recovery + OR a.outcome='partial' AND NEW.lifecycle='open' AND NEW.recovery=CASE WHEN a.prior_recovery='force_unknown' THEN 'force_unknown' ELSE 'resume_partial' END + OR a.outcome='unknown' AND NEW.lifecycle='open' AND NEW.recovery='force_unknown' + ) + ) + OR OLD.lifecycle='open' AND NEW.lifecycle='canceled' AND NEW.recovery='forbidden' AND OLD.claim_attempt_id IS NULL AND NEW.claim_attempt_id IS NULL + OR OLD.lifecycle='open' AND OLD.recovery='none' AND NEW.lifecycle='expired' AND NEW.recovery='forbidden' AND OLD.claim_attempt_id IS NULL AND NEW.claim_attempt_id IS NULL + OR OLD.lifecycle='expired' AND OLD.recovery='forbidden' AND NEW.lifecycle='open' AND NEW.recovery='none' + AND OLD.claim_attempt_id IS NULL AND NEW.claim_attempt_id IS NULL AND NEW.current_revision=OLD.current_revision+1 + AND EXISTS(SELECT 1 FROM stage_revisions r WHERE r.stage_id=OLD.id AND r.revision=OLD.current_revision AND r.state='superseded') + AND EXISTS(SELECT 1 FROM stage_revisions r WHERE r.stage_id=OLD.id AND r.revision=NEW.current_revision AND r.state='current') +) +BEGIN SELECT RAISE(ABORT, 'invalid stage lifecycle or recovery transition'); END; +CREATE TRIGGER apply_attempt_history_immutable_delete BEFORE DELETE ON apply_attempts +WHEN OLD.outcome IS NOT NULL OR EXISTS(SELECT 1 FROM apply_steps WHERE attempt_id=OLD.id AND state!='pending') +BEGIN SELECT RAISE(ABORT, 'dispatched apply history is immutable'); END; +CREATE TRIGGER apply_events_history_immutable_delete BEFORE DELETE ON apply_events +WHEN NOT EXISTS(SELECT 1 FROM apply_attempts a WHERE a.id=OLD.attempt_id AND a.outcome IS NULL) + OR EXISTS(SELECT 1 FROM apply_steps WHERE attempt_id=OLD.attempt_id AND state!='pending') +BEGIN SELECT RAISE(ABORT, 'dispatched apply events are immutable'); END; +CREATE TRIGGER apply_requests_history_immutable_delete BEFORE DELETE ON apply_requests +WHEN NOT EXISTS(SELECT 1 FROM apply_attempts a WHERE a.id=OLD.attempt_id AND a.outcome IS NULL) + OR EXISTS(SELECT 1 FROM apply_steps WHERE attempt_id=OLD.attempt_id AND state!='pending') +BEGIN SELECT RAISE(ABORT, 'dispatched apply requests are immutable'); END; `}} diff --git a/internal/staging/post.go b/internal/staging/post.go index c2515e6..1403890 100644 --- a/internal/staging/post.go +++ b/internal/staging/post.go @@ -241,7 +241,9 @@ func (s *Service) resolvePostFor(ctx context.Context, op postOperation, current if post.ID != op.postID || !validResolvedPost(post) { return Preview{}, ErrTarget } - if contaminated(s.credentials, post.ID, post.ChannelID, post.UserID, post.RootID) { + postIdentity := []string{post.ID, post.ChannelID, post.UserID, post.RootID} + postIdentity = append(postIdentity, post.FileIDs...) + if contaminated(s.credentials, postIdentity...) { return Preview{}, ErrCredential } if (op.operation == stagestore.EditPost || op.operation == stagestore.DeletePost) && (post.UserID != current.ID || post.Type != "") { @@ -357,13 +359,15 @@ func (s *Service) findCreate(ctx context.Context, userID, requestID string) (sta return stagestore.CreateRecord{}, false, ErrStore } -func digestPost(post mattermost.Post, credentials [][]byte) string { +// PostContentDigest binds the exact mutable Mattermost post content while +// eliding active credentials so the digest cannot become an offline verifier. +func PostContentDigest(post mattermost.Post, credentials [][]byte) string { canonical := struct { - Message any `json:"message"` - FileIDs []string `json:"fileIds"` - RootID string `json:"rootId"` - Type string `json:"type"` - }{credentialSafePostMessage(post.Message, credentials), post.FileIDs, post.RootID, post.Type} + Message any `json:"message"` + FileIDs []any `json:"fileIds"` + RootID string `json:"rootId"` + Type string `json:"type"` + }{credentialSafePostMessage(post.Message, credentials), credentialSafeStrings(post.FileIDs, credentials), post.RootID, post.Type} var buffer bytes.Buffer encoder := json.NewEncoder(&buffer) encoder.SetEscapeHTML(false) @@ -374,6 +378,18 @@ func digestPost(post mattermost.Post, credentials [][]byte) string { return hex.EncodeToString(digest[:]) } +func credentialSafeStrings(values []string, credentials [][]byte) []any { + out := make([]any, len(values)) + for index, value := range values { + out[index] = credentialSafePostMessage(value, credentials) + } + return out +} + +func digestPost(post mattermost.Post, credentials [][]byte) string { + return PostContentDigest(post, credentials) +} + func credentialSafePostMessage(message string, credentials [][]byte) any { source := []byte(message) protected := credentialsByLength(credentials) diff --git a/internal/staging/post_test.go b/internal/staging/post_test.go index 7c1191c..f071fa3 100644 --- a/internal/staging/post_test.go +++ b/internal/staging/post_test.go @@ -186,6 +186,14 @@ func TestPostDigestUsesCanonicalUTF8JSONAndPreservesFileOrder(t *testing.T) { } } +func TestPostDigestCannotVerifyCredentialBearingFileID(t *testing.T) { + first := mattermost.Post{Message: "safe", FileIDs: []string{"token-a"}} + second := mattermost.Post{Message: "safe", FileIDs: []string{"token-b"}} + if left, right := PostContentDigest(first, [][]byte{[]byte("token-a")}), PostContentDigest(second, [][]byte{[]byte("token-b")}); left != right { + t.Fatalf("credential-elided file IDs produced distinct digests: %s/%s", left, right) + } +} + func TestInvalidPostIDsAreZeroNetwork(t *testing.T) { service, _, calls := postService(t, ordinaryPost(), mattermost.Channel{ID: "channel-1", Type: "G", Name: "group"}, &recordingStore{}) for _, id := range []string{"bad/id", "bad\u202e", strings.Repeat("a", 129)} { diff --git a/schemas/v2/apply-receipt.schema.json b/schemas/v2/apply-receipt.schema.json index 2c19770..2265bcb 100644 --- a/schemas/v2/apply-receipt.schema.json +++ b/schemas/v2/apply-receipt.schema.json @@ -68,6 +68,7 @@ }, "allOf": [ { "if": { "properties": { "kind": { "const": "conversation" } } }, "then": { "properties": { "postId": { "type": "null" }, "rootPostId": { "type": "null" }, "emoji": { "type": "null" }, "postState": { "type": "null" }, "reactionPresent": { "type": "null" } } } }, + { "if": { "properties": { "kind": { "const": "post" } } }, "then": { "properties": { "emoji": { "type": "null" }, "reactionPresent": { "type": "null" } } } }, { "if": { "properties": { "kind": { "const": "reaction" } } }, "then": { "properties": { "postId": { "$ref": "#/$defs/id" }, "emoji": { "type": "string" }, "postState": { "type": "null" }, "reactionPresent": { "type": "boolean" } } } } ] }, @@ -104,7 +105,7 @@ { "if": { "properties": { "state": { "const": "response_validated" } } }, "then": { "properties": { "result": { "not": { "type": "null" } }, "startedAt": { "type": "string" }, "endedAt": { "type": "string" } } } }, { "if": { "properties": { "state": { "const": "rejected" } } }, "then": { "properties": { "result": { "$ref": "#/$defs/rejectedResult" }, "startedAt": { "type": "string" }, "endedAt": { "type": "string" } } } }, { "if": { "properties": { "state": { "const": "outcome_unknown" } } }, "then": { "properties": { "result": { "type": "null" }, "startedAt": { "type": "string" }, "endedAt": { "type": "string" } } } }, - { "if": { "properties": { "state": { "const": "skipped" } } }, "then": { "properties": { "condition": { "const": "if_missing" }, "result": { "$ref": "#/$defs/skippedResult" }, "startedAt": { "type": "null" }, "endedAt": { "type": "string" } } } }, + { "if": { "properties": { "state": { "const": "skipped" } } }, "then": { "allOf": [{ "anyOf": [{ "properties": { "condition": { "const": "if_missing" } } }, { "properties": { "kind": { "const": "edit_post" }, "condition": { "const": "always" } } }] }, { "properties": { "result": { "$ref": "#/$defs/skippedResult" }, "startedAt": { "type": "null" }, "endedAt": { "type": "string" } } }] } }, { "if": { "properties": { "state": { "const": "not_dispatched" } } }, "then": { "properties": { "result": { "type": "null" }, "startedAt": { "type": "null" }, "endedAt": { "type": "string" } } } }, { "if": { "properties": { "state": { "const": "response_validated" }, "kind": { "const": "upload_attachment" } } }, "then": { "properties": { "result": { "$ref": "#/$defs/fileResult" } } } }, { "if": { "properties": { "state": { "const": "response_validated" }, "kind": { "const": "create_post" } } }, "then": { "properties": { "result": { "$ref": "#/$defs/createResult" } } } }, diff --git a/schemas/v2/examples/store-doctor.json b/schemas/v2/examples/store-doctor.json index ec62e46..b10bb3f 100644 --- a/schemas/v2/examples/store-doctor.json +++ b/schemas/v2/examples/store-doctor.json @@ -1 +1 @@ -{"schema":"mm/v2/store-doctor","path":"/home/arda/.local/state/mattermost-cli/stages.sqlite3","report":{"exists":false,"filesystemSafe":null,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":7,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}} +{"schema":"mm/v2/store-doctor","path":"/home/arda/.local/state/mattermost-cli/stages.sqlite3","report":{"exists":false,"filesystemSafe":null,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":8,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}} diff --git a/schemas/v2/examples/store-migrations.json b/schemas/v2/examples/store-migrations.json index 88869d2..550b5b2 100644 --- a/schemas/v2/examples/store-migrations.json +++ b/schemas/v2/examples/store-migrations.json @@ -1 +1 @@ -{"schema":"mm/v2/store-migrations","latest":7,"migrations":[{"version":1,"name":"core-stage-state","checksum":"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"},{"version":2,"name":"immutable-local-request-receipts","checksum":"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"},{"version":3,"name":"caller-intent-stage-create-replay","checksum":"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5"},{"version":4,"name":"caller-intent-stage-revise-replay","checksum":"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4"},{"version":5,"name":"revision-plan-follows-composition","checksum":"fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0"},{"version":6,"name":"durable-apply-journal","checksum":"4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56"},{"version":7,"name":"status-confirmed-delete-results","checksum":"bf980f24ff24fdf9d6a3a28b9ded97b51336fa1c8ef290d1bdcf91f1b593c5ce"}]} +{"schema":"mm/v2/store-migrations","latest":8,"migrations":[{"version":1,"name":"core-stage-state","checksum":"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"},{"version":2,"name":"immutable-local-request-receipts","checksum":"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"},{"version":3,"name":"caller-intent-stage-create-replay","checksum":"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5"},{"version":4,"name":"caller-intent-stage-revise-replay","checksum":"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4"},{"version":5,"name":"revision-plan-follows-composition","checksum":"fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0"},{"version":6,"name":"durable-apply-journal","checksum":"4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56"},{"version":7,"name":"status-confirmed-delete-results","checksum":"bf980f24ff24fdf9d6a3a28b9ded97b51336fa1c8ef290d1bdcf91f1b593c5ce"},{"version":8,"name":"already-satisfied-edit-apply","checksum":"cff30dd4fcc987876e092e2de1f5afd21ebeadbcb792fd550a56eb4c221edd10"}]} diff --git a/schemas/v2/store-doctor.schema.json b/schemas/v2/store-doctor.schema.json index 478fc9b..73222ba 100644 --- a/schemas/v2/store-doctor.schema.json +++ b/schemas/v2/store-doctor.schema.json @@ -15,7 +15,7 @@ "exists": { "type": "boolean" }, "filesystemSafe": {}, "applicationId": {}, "integrity": {}, "integrityTruncated": {}, "foreignKeyIssues": {}, "foreignKeyRows": {}, "foreignKeyTruncated": {}, "journalMode": {}, "synchronous": {}, "secureDelete": {}, "foreignKeys": {}, "trustedSchema": {}, "queryOnly": {}, "walFallback": {}, - "migrations": { "type": "object", "additionalProperties": false, "required": ["applied", "latest", "valid"], "properties": { "applied": {}, "latest": { "const": 7 }, "valid": {} } }, + "migrations": { "type": "object", "additionalProperties": false, "required": ["applied", "latest", "valid"], "properties": { "applied": {}, "latest": { "const": 8 }, "valid": {} } }, "permissionModelLimitations": { "type": "array", "items": { "$ref": "#/$defs/safeString" } } }, "allOf": [ @@ -23,14 +23,14 @@ "filesystemSafe": { "const": null }, "applicationId": { "const": null }, "integrity": { "const": null }, "integrityTruncated": { "const": null }, "foreignKeyIssues": { "const": null }, "foreignKeyRows": { "const": null }, "foreignKeyTruncated": { "const": null }, "journalMode": { "const": null }, "synchronous": { "const": null }, "secureDelete": { "const": null }, "foreignKeys": { "const": null }, "trustedSchema": { "const": null }, "queryOnly": { "const": null }, "walFallback": { "const": null }, - "migrations": { "properties": { "applied": { "const": null }, "latest": { "const": 7 }, "valid": { "const": null } } } + "migrations": { "properties": { "applied": { "const": null }, "latest": { "const": 8 }, "valid": { "const": null } } } } } }, { "if": { "properties": { "exists": { "const": true } }, "required": ["exists"] }, "then": { "properties": { "filesystemSafe": { "const": true }, "applicationId": { "const": 1296913970 }, "integrity": { "$ref": "#/$defs/nonemptyBoundedStrings" }, "integrityTruncated": { "type": "boolean" }, "foreignKeyIssues": { "type": "integer", "minimum": 0, "maximum": 21 }, "foreignKeyRows": { "$ref": "#/$defs/boundedStrings" }, "foreignKeyTruncated": { "type": "boolean" }, "journalMode": { "enum": ["wal", "delete", "truncate", "persist"] }, "synchronous": { "type": "integer" }, "secureDelete": { "type": "integer" }, "foreignKeys": { "const": true }, "trustedSchema": { "const": false }, "queryOnly": { "const": true }, "walFallback": { "type": "boolean" }, - "migrations": { "properties": { "applied": { "const": 7 }, "latest": { "const": 7 }, "valid": { "const": true } } } + "migrations": { "properties": { "applied": { "const": 8 }, "latest": { "const": 8 }, "valid": { "const": true } } } }, "allOf": [ { "oneOf": [ { "properties": { "integrity": { "const": ["ok"] }, "integrityTruncated": { "const": false } } }, diff --git a/schemas/v2/store-migrations.schema.json b/schemas/v2/store-migrations.schema.json index 2261899..227e1a9 100644 --- a/schemas/v2/store-migrations.schema.json +++ b/schemas/v2/store-migrations.schema.json @@ -6,9 +6,9 @@ "required": ["schema", "latest", "migrations"], "properties": { "schema": { "const": "mm/v2/store-migrations" }, - "latest": { "const": 7 }, + "latest": { "const": 8 }, "migrations": { - "type": "array", "minItems": 7, "maxItems": 7, + "type": "array", "minItems": 8, "maxItems": 8, "prefixItems": [{ "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { "version": { "const": 1 }, "name": { "const": "core-stage-state" }, "checksum": { "const": "e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f" } } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { @@ -23,6 +23,8 @@ "version": { "const": 6 }, "name": { "const": "durable-apply-journal" }, "checksum": { "const": "4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56" } } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { "version": { "const": 7 }, "name": { "const": "status-confirmed-delete-results" }, "checksum": { "const": "bf980f24ff24fdf9d6a3a28b9ded97b51336fa1c8ef290d1bdcf91f1b593c5ce" } + } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { + "version": { "const": 8 }, "name": { "const": "already-satisfied-edit-apply" }, "checksum": { "const": "cff30dd4fcc987876e092e2de1f5afd21ebeadbcb792fd550a56eb4c221edd10" } } }], "items": false } From 63c8d44e06f4a031c89eb2d976bd6411082693ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 08:50:37 +0300 Subject: [PATCH 088/119] feat: execute attachment apply plans --- docs/V2_CONTRACT.md | 13 +- internal/api/client.go | 71 +++++- internal/api/client_test.go | 64 +++++ internal/apply/post.go | 114 ++++++++- internal/apply/post_test.go | 262 ++++++++++++++++++++ internal/apply/reaction.go | 2 +- internal/apply/service.go | 63 ++++- internal/apply/service_test.go | 12 +- internal/cli/store_test.go | 6 +- internal/mattermost/file_mutations.go | 188 ++++++++++++++ internal/mattermost/file_mutations_test.go | 107 ++++++++ internal/stageinput/file_other.go | 1 + internal/stageinput/file_unix.go | 14 ++ internal/stageinput/file_unix_test.go | 23 ++ internal/stageinput/input.go | 14 +- internal/stageinput/input_test.go | 112 +++++++++ internal/stageinput/space_other.go | 9 + internal/stageinput/space_unix.go | 23 ++ internal/stageinput/spool.go | 155 ++++++++++++ internal/stagestore/apply.go | 39 +++ internal/stagestore/domain.go | 56 ++++- internal/stagestore/domain_test.go | 63 ++++- internal/stagestore/schema.go | 18 ++ internal/stagestore/store.go | 4 + internal/staging/revision.go | 3 + internal/staging/validation.go | 4 +- schemas/v2/examples/store-doctor.json | 2 +- schemas/v2/examples/store-migrations.json | 16 +- schemas/v2/stage-preview.schema.json | 6 +- schemas/v2/stage-receipt.schema.json | 6 +- schemas/v2/stage-request.schema.json | 2 +- schemas/v2/stage-revise-request.schema.json | 2 +- schemas/v2/stage.schema.json | 8 +- schemas/v2/stages.schema.json | 6 +- schemas/v2/store-doctor.schema.json | 6 +- schemas/v2/store-migrations.schema.json | 6 +- 36 files changed, 1420 insertions(+), 80 deletions(-) create mode 100644 internal/mattermost/file_mutations.go create mode 100644 internal/mattermost/file_mutations_test.go create mode 100644 internal/stageinput/file_unix_test.go create mode 100644 internal/stageinput/space_other.go create mode 100644 internal/stageinput/space_unix.go create mode 100644 internal/stageinput/spool.go diff --git a/docs/V2_CONTRACT.md b/docs/V2_CONTRACT.md index 29df5bb..d06b0ef 100644 --- a/docs/V2_CONTRACT.md +++ b/docs/V2_CONTRACT.md @@ -248,7 +248,7 @@ Only operations with mutable composition may be revised. Top-level posts and rep Attachments are path-backed, not copied into SQLite or managed blob storage. -At staging time v2 records, at minimum: +At staging time v2 accepts at most five ordered attachments, matching the maximum file-ID set accepted by a Mattermost post. It records, at minimum: - caller-provided path; - canonical path information needed for later safe reopening; @@ -256,14 +256,17 @@ At staging time v2 records, at minimum: - byte length; - detected or explicit media type; - cryptographic content digest. +- local file identity needed to reject replacement even when replacement bytes are identical. Apply securely reopens and rehashes the file. Missing, inaccessible, non-regular, replaced, symlink-swapped, resized, or digest-mismatched files cause a local conflict before upload. v2 never silently uploads bytes different from the staged revision. -To close the hash-to-upload race without retaining attachment copies between commands, apply copies each securely opened source into a private `0600` spool file while hashing it. Only a complete spool whose digest and length match the staged revision may be uploaded. Upload reads the spool, not the original path. Thus path-backed staging remains low-storage at rest while dispatched bytes are an immutable snapshot of the reviewed file. +Development databases upgraded from a pre-identity schema cannot safely infer this binding. Their existing attachment revisions remain inspectable but ineligible for apply until an ordinary revision securely rebinds the attachment sources. -Spools live under a private per-attempt directory. A spool is deleted after validated success or definitive rejection once the journal commit is durable. It is retained for partial/unknown recovery when it represents bytes that may need exact reuse. On startup, unreferenced spools and spools whose journal proves no dispatch are removed; spools referenced by a stale applying claim are retained while that claim becomes unknown. Explicit prune removes retained spools only under the recovery-destructive rules below. +To close the hash-to-upload race without retaining attachment copies between commands, apply copies every securely opened source into a private `0600` spool file while hashing it before the first remote dispatch. Only a complete spool whose identity, digest, and length match the staged revision may be uploaded. Each completed spool is unlinked while its descriptor remains open; upload reads that descriptor, not the original path. Thus path-backed staging remains low-storage at rest while dispatched bytes are an immutable snapshot of the reviewed file. -Server upload limits are checked before upload when discoverable. Upload and post creation are separate remote substeps and therefore use the compound-operation journal. +Spools are execution-only snapshots and never durable recovery artifacts. Process exit closes their descriptors and leaves no pathname to reconcile. A later explicit recovery securely reopens and respools the staged source; if the exact bound source is unavailable, recovery fails closed until the source is restored or recovery is explicitly abandoned. Validated upload steps recover from their journaled file IDs after fresh remote metadata revalidation, not by uploading the file again. + +Attachments are non-empty and limited to five per post. Server per-file upload limits are checked before upload when discoverable; a dispatched `413` is a definitive rejection. Checked arithmetic caps an attempt's aggregate spool bytes at 512 MiB, and apply refuses to begin unless the private state filesystem can retain that snapshot while preserving a 64 MiB free-space reserve. Upload and post creation are separate remote substeps and therefore use the compound-operation journal. ## 11. Supported remote mutation lifecycle @@ -366,7 +369,7 @@ Normal transitions are: An interrupted or stale `applying` claim creates an `unknown` attempt outcome unless the journal proves no remote mutation dispatch was handed to the transport. The stage returns to `open` with aggregate `force_unknown`. Lease expiry never makes a mutation ordinarily replayable. -Revise is refused while `applying` or after `completed`, `canceled`, or `pruned`. An expired stage may be revised only with `stage revise --revive`, which atomically returns it to `open` with a new revision. Revision never clears recovery history: revising after partial or unknown carries `resume_partial` or `force_unknown` to the new revision. Revised content is therefore still gated by the unresolved risk of prior effects. +Revise is refused while `applying`, while recovery is `resume_partial`, or after `completed`, `canceled`, or `pruned`. Partial recovery is bound to the exact immutable attachment ordinals that produced its confirmed effects; it must be completed or explicitly abandoned before composition can change. An expired stage may be revised only with `stage revise --revive`, which atomically returns it to `open` with a new revision. Revision never clears unknown recovery history: revising after an unknown attempt carries `force_unknown` to the new revision. Revised content is therefore still gated by the unresolved risk of prior effects. Ordinary `mm apply @` requires the exact current revision, lifecycle `open`, and recovery requirement `none`. diff --git a/internal/api/client.go b/internal/api/client.go index 3aae717..e26d655 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "io" + "mime" "net/http" "net/url" "strconv" @@ -111,6 +112,9 @@ type PreparedMutation struct { method string endpoint *url.URL payload []byte + body io.ReadCloser + contentLength int64 + contentType string expectedStatus int state *preparedMutationState } @@ -201,6 +205,26 @@ func (c *Client) PrepareDeleteStatus(path string, expectedStatus int) (*Prepared return c.prepareMutation(http.MethodDelete, path, nil, expectedStatus) } +// PrepareRawPostStatus takes ownership of body on success. The body is +// consumed at most once and is closed by Execute or Close. +func (c *Client) PrepareRawPostStatus(path, contentType string, body io.ReadCloser, contentLength int64, expectedStatus int) (*PreparedMutation, error) { + c.lifecycle.RLock() + defer c.lifecycle.RUnlock() + if c.closed { + return nil, ErrClientClosed + } + mediaType, _, err := mime.ParseMediaType(contentType) + if err != nil || mediaType == "" || body == nil || contentLength <= 0 || expectedStatus < 200 || expectedStatus >= 300 { + return nil, errors.New("invalid raw Mattermost mutation") + } + endpoint, err := c.endpoint(path) + if err != nil { + return nil, err + } + return &PreparedMutation{client: c, method: http.MethodPost, endpoint: endpoint, body: body, contentLength: contentLength, + contentType: contentType, expectedStatus: expectedStatus, state: &preparedMutationState{}}, nil +} + func (c *Client) prepareMutation(method, path string, body any, expectedStatus int) (*PreparedMutation, error) { c.lifecycle.RLock() defer c.lifecycle.RUnlock() @@ -218,7 +242,8 @@ func (c *Client) prepareMutation(method, path string, body any, expectedStatus i if err != nil { return nil, errors.New("unable to encode Mattermost request") } - return &PreparedMutation{client: c, method: method, endpoint: endpoint, payload: bytes.Clone(payload), expectedStatus: expectedStatus, state: &preparedMutationState{}}, nil + return &PreparedMutation{client: c, method: method, endpoint: endpoint, payload: bytes.Clone(payload), contentLength: int64(len(payload)), + contentType: "application/json", expectedStatus: expectedStatus, state: &preparedMutationState{}}, nil } // Execute consumes the prepared mutation before inspecting cancellation or @@ -236,12 +261,20 @@ func (p *PreparedMutation) Execute(ctx context.Context, out any) error { p.state.used = true p.state.mu.Unlock() + var body io.ReadCloser + if p.body != nil { + body = p.body + p.body = nil + } else { + body = io.NopCloser(bytes.NewReader(p.payload)) + } + defer body.Close() p.client.lifecycle.RLock() defer p.client.lifecycle.RUnlock() if p.client.closed || ctx == nil { return &OutcomeUnknownError{} } - status, _, data, failure := p.client.attempt(ctx, p.method, p.endpoint, p.payload, true, true) + status, _, data, failure := p.client.attemptReader(ctx, p.method, p.endpoint, body, p.contentLength, p.contentType, true, true) if failure != nil { return &OutcomeUnknownError{} } @@ -264,6 +297,25 @@ func (p *PreparedMutation) Execute(ctx context.Context, out any) error { return nil } +// Close discards an unexecuted prepared mutation without dispatching it. +func (p *PreparedMutation) Close() error { + if p == nil || p.state == nil { + return nil + } + p.state.mu.Lock() + defer p.state.mu.Unlock() + if p.state.used { + return nil + } + p.state.used = true + if p.body != nil { + err := p.body.Close() + p.body = nil + return err + } + return nil +} + func consumedMutationError() error { return errors.Join(&OutcomeUnknownError{}, ErrMutationUsed) } @@ -340,30 +392,27 @@ type attemptFailure struct{ kind string } func (e *attemptFailure) Error() string { return e.kind } func (c *Client) attempt(parent context.Context, method string, endpoint *url.URL, payload []byte, mutation, authenticated bool) (int, http.Header, []byte, error) { + return c.attemptReader(parent, method, endpoint, io.NopCloser(bytes.NewReader(payload)), int64(len(payload)), "application/json", mutation, authenticated) +} + +func (c *Client) attemptReader(parent context.Context, method string, endpoint *url.URL, body io.ReadCloser, contentLength int64, contentType string, mutation, authenticated bool) (int, http.Header, []byte, error) { ctx, cancel := context.WithTimeout(parent, c.timeout) defer cancel() if mutation { ctx = context.WithValue(ctx, mutationRequestKey, true) } - var body io.Reader = bytes.NewReader(payload) - if mutation { - // bytes.Reader makes requests replayable by populating GetBody, and a - // nil body becomes http.NoBody. Both permit transparent HTTP/2 retries. - // A distinct ReadCloser keeps even empty mutations non-replayable. - body = io.NopCloser(bytes.NewReader(payload)) - } req, err := http.NewRequestWithContext(ctx, method, endpoint.String(), body) if err != nil { return 0, nil, nil, &attemptFailure{kind: "transport"} } if mutation { - req.ContentLength = int64(len(payload)) + req.ContentLength = contentLength req.GetBody = nil } if authenticated { req.Header.Set("Authorization", "Bearer "+c.token) } - req.Header.Set("Content-Type", "application/json") + req.Header.Set("Content-Type", contentType) resp, err := c.http.Do(req) if err != nil { if parent.Err() != nil { diff --git a/internal/api/client_test.go b/internal/api/client_test.go index dd9f729..c0fe8cb 100644 --- a/internal/api/client_test.go +++ b/internal/api/client_test.go @@ -75,6 +75,60 @@ func TestPreparedEmptyDeleteRemainsNonReplayable(t *testing.T) { } } +func TestPreparedRawMutationOwnsExactOneShotBody(t *testing.T) { + payload := []byte{0, 1, 2, 0xff, '\n'} + body := &trackedReadCloser{Reader: bytes.NewReader(payload)} + var got []byte + transport := roundTripFunc(func(request *http.Request) (*http.Response, error) { + if request.GetBody != nil || request.ContentLength != int64(len(payload)) || request.Header.Get("Content-Type") != "application/octet-stream" { + t.Fatalf("GetBody=%v length=%d type=%q", request.GetBody != nil, request.ContentLength, request.Header.Get("Content-Type")) + } + got, _ = io.ReadAll(request.Body) + return &http.Response{StatusCode: http.StatusCreated, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"ok":true}`))}, nil + }) + c := newTestClient(t, "https://mattermost.example", WithRoundTripper(transport)) + prepared, err := c.PrepareRawPostStatus("/files?channel_id=channel&filename=a.bin", "application/octet-stream", body, int64(len(payload)), http.StatusCreated) + if err != nil { + t.Fatal(err) + } + var response struct { + OK bool `json:"ok"` + } + if err = prepared.Execute(context.Background(), &response); err != nil || !response.OK || !bytes.Equal(got, payload) || !body.closed.Load() { + t.Fatalf("response=%+v exact=%v closed=%v err=%v", response, bytes.Equal(got, payload), body.closed.Load(), err) + } + if err = prepared.Execute(context.Background(), &response); !errors.Is(err, ErrMutationUsed) { + t.Fatalf("replay=%v", err) + } +} + +func TestPreparedRawMutationCanBeDiscardedWithoutDispatch(t *testing.T) { + body := &trackedReadCloser{Reader: strings.NewReader("x")} + transport := &countingTransport{err: errors.New("must not dispatch")} + c := newTestClient(t, "https://mattermost.example", WithRoundTripper(transport)) + prepared, err := c.PrepareRawPostStatus("/files", "text/plain", body, 1, http.StatusCreated) + if err != nil { + t.Fatal(err) + } + if err = prepared.Close(); err != nil || !body.closed.Load() || transport.count.Load() != 0 { + t.Fatalf("closed=%v attempts=%d err=%v", body.closed.Load(), transport.count.Load(), err) + } +} + +func TestPreparedRawMutationClosesBodyWhenClientClosedBeforeExecute(t *testing.T) { + body := &trackedReadCloser{Reader: strings.NewReader("x")} + c := newTestClient(t, "https://mattermost.example") + prepared, err := c.PrepareRawPostStatus("/files", "text/plain", body, 1, http.StatusCreated) + if err != nil { + t.Fatal(err) + } + c.Close() + var unknown *OutcomeUnknownError + if err = prepared.Execute(context.Background(), &struct{}{}); !errors.As(err, &unknown) || !body.closed.Load() { + t.Fatalf("closed=%v err=%v", body.closed.Load(), err) + } +} + func TestPreparedMutationRejectsLocalFailuresBeforeDispatch(t *testing.T) { transport := &countingTransport{err: errors.New("must not be called")} c := newTestClient(t, "https://mattermost.example", WithRoundTripper(transport)) @@ -688,6 +742,16 @@ type roundTripFunc func(*http.Request) (*http.Response, error) func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { return f(request) } +type trackedReadCloser struct { + io.Reader + closed atomic.Bool +} + +func (b *trackedReadCloser) Close() error { + b.closed.Store(true) + return nil +} + type trackingBody struct { reads atomic.Int32 closed atomic.Bool diff --git a/internal/apply/post.go b/internal/apply/post.go index 9bee8d2..0a1b5cd 100644 --- a/internal/apply/post.go +++ b/internal/apply/post.go @@ -8,11 +8,15 @@ import ( "strings" "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/stageinput" "github.com/ardasevinc/mattermost-cli/internal/stagestore" "github.com/ardasevinc/mattermost-cli/internal/staging" ) -func (s *Service) applyPost(ctx context.Context, attempt stagestore.ApplyAttempt, operation stagestore.Operation, currentUserID string, destination staging.Destination, body []byte) (stagestore.ApplyReceipt, error) { +func (s *Service) applyPost(ctx context.Context, attempt stagestore.ApplyAttempt, operation stagestore.Operation, currentUserID string, destination staging.Destination, body []byte, attachments []stagestore.Attachment) (stagestore.ApplyReceipt, error) { + if len(attachments) > 0 { + return s.applyPostWithAttachments(ctx, attempt, operation, currentUserID, destination, body, attachments) + } switch operation { case stagestore.CreatePost, stagestore.Reply: rootID := "" @@ -28,7 +32,7 @@ func (s *Service) applyPost(ctx context.Context, attempt stagestore.ApplyAttempt if err = s.revalidatePostTarget(ctx, operation, currentUserID, destination); err != nil { return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, errors.Join(ErrTargetDrift, err)) } - return executePrepared(s, ctx, attempt.ID, prepared.Execute) + return executePrepared(s, ctx, attempt.ID, 1, prepared.Execute) case stagestore.EditPost: post, err := s.revalidateBoundPost(ctx, currentUserID, destination) if err != nil { @@ -43,7 +47,7 @@ func (s *Service) applyPost(ctx context.Context, attempt stagestore.ApplyAttempt if err != nil { return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, err) } - return executePrepared(s, ctx, attempt.ID, prepared.Execute) + return executePrepared(s, ctx, attempt.ID, 1, prepared.Execute) case stagestore.DeletePost: prepared, err := s.postWrites.PrepareDelete(mattermost.DeletePostMutationInput{PostID: *destination.PostID}) if err != nil { @@ -52,26 +56,120 @@ func (s *Service) applyPost(ctx context.Context, attempt stagestore.ApplyAttempt if _, err = s.revalidateBoundPost(ctx, currentUserID, destination); err != nil { return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, errors.Join(ErrTargetDrift, err)) } - return executePrepared(s, ctx, attempt.ID, prepared.Execute) + return executePrepared(s, ctx, attempt.ID, 1, prepared.Execute) default: return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, ErrUnsupportedOperation) } } -func executePrepared[T any](s *Service, ctx context.Context, attemptID string, execute func(context.Context) (T, error)) (stagestore.ApplyReceipt, error) { - if err := s.store.BeginDispatch(ctx, attemptID, 1); err != nil { +func executePrepared[T any](s *Service, ctx context.Context, attemptID string, ordinal int, execute func(context.Context) (T, error)) (stagestore.ApplyReceipt, error) { + if err := s.store.BeginDispatch(ctx, attemptID, ordinal); err != nil { + if ordinal > 1 { + return s.stopCompoundBeforeDispatch(ctx, attemptID, err) + } return stagestore.ApplyReceipt{}, s.abandon(ctx, attemptID, err) } result, remoteErr := execute(ctx) if remoteErr != nil { - return s.recordRemoteFailure(ctx, attemptID, remoteErr) + return s.recordRemoteFailure(ctx, attemptID, ordinal, remoteErr) } encoded, err := json.Marshal(result) if err != nil { return stagestore.ApplyReceipt{}, &ConfirmedEffectError{err} } journalCtx := context.WithoutCancel(ctx) - if err = s.store.MarkStepValidated(journalCtx, attemptID, 1, encoded); err != nil { + if err = s.store.MarkStepValidated(journalCtx, attemptID, ordinal, encoded); err != nil { + return stagestore.ApplyReceipt{}, &ConfirmedEffectError{err} + } + receipt, err := s.finalizeReceipt(journalCtx, attemptID) + if err != nil { + return stagestore.ApplyReceipt{}, &ConfirmedEffectError{err} + } + return receipt, nil +} + +func (s *Service) applyPostWithAttachments(ctx context.Context, attempt stagestore.ApplyAttempt, operation stagestore.Operation, currentUserID string, destination staging.Destination, body []byte, attachments []stagestore.Attachment) (stagestore.ApplyReceipt, error) { + if operation != stagestore.CreatePost && operation != stagestore.Reply || len(attachments) == 0 || len(attachments) > stageinput.MaxAttachments { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, ErrUnsupportedOperation) + } + if err := s.revalidatePostTarget(ctx, operation, currentUserID, destination); err != nil { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, errors.Join(ErrTargetDrift, err)) + } + spools := make([]*stageinput.Spool, len(attachments)) + defer func() { + for _, spool := range spools { + if spool != nil { + _ = spool.Close() + } + } + }() + for i, attachment := range attachments { + spool, err := stageinput.Snapshot(ctx, attachment, s.credentials, s.spoolDirectory) + if err != nil { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, errors.Join(ErrTargetDrift, err)) + } + spools[i] = spool + } + + fileIDs := make([]string, 0, len(spools)) + for i, spool := range spools { + ordinal := i + 1 + prepared, err := s.fileWrites.PrepareUpload(mattermost.UploadMutationInput{ + ChannelID: destination.ChannelID, UserID: currentUserID, Filename: spool.RemoteFilename, MediaType: spool.MediaType, Length: spool.Length, Body: spool, + }) + if err != nil { + return s.stopCompoundBeforeDispatch(ctx, attempt.ID, err) + } + spools[i] = nil // ownership transferred to the prepared mutation + if err = s.store.BeginDispatch(ctx, attempt.ID, ordinal); err != nil { + _ = prepared.Close() + return s.stopCompoundBeforeDispatch(ctx, attempt.ID, err) + } + result, remoteErr := prepared.Execute(ctx) + if remoteErr != nil { + return s.recordRemoteFailure(ctx, attempt.ID, ordinal, remoteErr) + } + if s.containsCredential(result.FileID) { + journalCtx := context.WithoutCancel(ctx) + if err = s.store.MarkStepUnknown(journalCtx, attempt.ID, ordinal); err != nil { + return stagestore.ApplyReceipt{}, &ConfirmedEffectError{err} + } + return s.finalizeReceipt(journalCtx, attempt.ID) + } + encoded, err := json.Marshal(result) + if err != nil { + return stagestore.ApplyReceipt{}, &ConfirmedEffectError{err} + } + if err = s.store.MarkStepValidated(context.WithoutCancel(ctx), attempt.ID, ordinal, encoded); err != nil { + return stagestore.ApplyReceipt{}, &ConfirmedEffectError{err} + } + fileIDs = append(fileIDs, result.FileID) + } + + if err := s.revalidatePostTarget(ctx, operation, currentUserID, destination); err != nil { + if sealErr := s.store.SealRemainingNotDispatched(context.WithoutCancel(ctx), attempt.ID); sealErr != nil { + return stagestore.ApplyReceipt{}, &ConfirmedEffectError{sealErr} + } + return s.finalizeReceipt(context.WithoutCancel(ctx), attempt.ID) + } + rootID := "" + if destination.RootPostID != nil { + rootID = *destination.RootPostID + } + prepared, err := s.postWrites.PrepareCreate(mattermost.CreatePostMutationInput{ChannelID: destination.ChannelID, UserID: currentUserID, + Message: string(body), RootID: rootID, FileIDs: fileIDs, PendingPostID: attempt.PendingPostID}) + if err != nil { + return s.stopCompoundBeforeDispatch(ctx, attempt.ID, err) + } + return executePrepared(s, ctx, attempt.ID, len(attachments)+1, prepared.Execute) +} + +func (s *Service) stopCompoundBeforeDispatch(ctx context.Context, attemptID string, cause error) (stagestore.ApplyReceipt, error) { + journalCtx := context.WithoutCancel(ctx) + if err := s.store.SealRemainingNotDispatched(journalCtx, attemptID); err != nil { + if errors.Is(err, stagestore.ErrNotEligible) { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attemptID, cause) + } return stagestore.ApplyReceipt{}, &ConfirmedEffectError{err} } receipt, err := s.finalizeReceipt(journalCtx, attemptID) diff --git a/internal/apply/post_test.go b/internal/apply/post_test.go index 55fac67..d598aa1 100644 --- a/internal/apply/post_test.go +++ b/internal/apply/post_test.go @@ -1,6 +1,7 @@ package apply import ( + "bytes" "context" "crypto/sha256" "encoding/json" @@ -9,11 +10,16 @@ import ( "io" "net/http" "net/http/httptest" + "os" + "path/filepath" + "slices" "strings" "sync/atomic" "testing" + "github.com/ardasevinc/mattermost-cli/internal/api" "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/stageinput" "github.com/ardasevinc/mattermost-cli/internal/stagestore" "github.com/ardasevinc/mattermost-cli/internal/staging" ) @@ -66,6 +72,179 @@ func TestApplyCreatePostPreservesStagedMarkdownExactly(t *testing.T) { } } +func TestApplyAttachmentPostPreservesShortAndLongMarkdownAndOrderedBytes(t *testing.T) { + for name, message := range map[string]string{ + "short": "# heading\n\n- one\n- **two**\n", + "long": strings.Repeat("界", 16_382) + "\n", + } { + t.Run(name, func(t *testing.T) { + var uploads, posts atomic.Int32 + files := [][]byte{[]byte("first\x00file"), []byte("second file\n")} + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/users/me": + _, _ = io.WriteString(response, `{"id":"self","username":"arda"}`) + case "/api/v4/channels/channel-1": + _, _ = io.WriteString(response, `{"id":"channel-1","team_id":"team-1","type":"O","name":"town-square","display_name":"Town Square"}`) + case "/api/v4/channels/channel-1/members/self": + _, _ = io.WriteString(response, `{"channel_id":"channel-1","user_id":"self"}`) + case "/api/v4/files": + index := int(uploads.Add(1)) - 1 + got, _ := io.ReadAll(request.Body) + filename := request.URL.Query().Get("filename") + if index >= len(files) || !bytes.Equal(got, files[index]) || request.URL.Query().Get("channel_id") != "channel-1" || filename != fmt.Sprintf("file-%d.bin", index+1) { + response.WriteHeader(http.StatusBadRequest) + return + } + response.WriteHeader(http.StatusCreated) + _, _ = fmt.Fprintf(response, `{"file_infos":[{"id":"file-%d","user_id":"self","channel_id":"channel-1","create_at":100,"update_at":100,"delete_at":0,"name":%q,"size":%d}]}`, index+1, filename, len(got)) + case "/api/v4/posts": + posts.Add(1) + var input struct { + ChannelID string `json:"channel_id"` + Message string `json:"message"` + PendingPostID string `json:"pending_post_id"` + FileIDs []string `json:"file_ids"` + } + if json.NewDecoder(request.Body).Decode(&input) != nil || input.ChannelID != "channel-1" || input.Message != message || !slices.Equal(input.FileIDs, []string{"file-1", "file-2"}) { + response.WriteHeader(http.StatusBadRequest) + return + } + response.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(response, mutationPostResponse("post-1", "channel-1", "self", message, "", input.PendingPostID, input.FileIDs, 101, 101)) + default: + http.NotFound(response, request) + } + })) + defer server.Close() + store := openApplyStore(t) + stage := createAttachmentApplyStage(t, store, server.URL+"/api/v4", message, files) + service, client := applyAttachmentService(t, server.URL, store) + defer client.Close() + receipt, err := service.Apply(context.Background(), applyClaim(stage, "apply-attachments-"+name)) + if err != nil || receipt.Outcome != stagestore.OutcomeSucceeded || len(receipt.Steps) != 3 || uploads.Load() != 2 || posts.Load() != 1 { + t.Fatalf("receipt=%+v uploads=%d posts=%d err=%v", receipt, uploads.Load(), posts.Load(), err) + } + }) + } +} + +func TestApplyPreSpoolsEveryAttachmentBeforeFirstDispatch(t *testing.T) { + var writes atomic.Int32 + server := attachmentTargetServer(t, func(response http.ResponseWriter, request *http.Request) { + writes.Add(1) + response.WriteHeader(http.StatusInternalServerError) + }) + defer server.Close() + store := openApplyStore(t) + files := [][]byte{[]byte("first"), []byte("second")} + stage, paths := createAttachmentApplyStageWithPaths(t, store, server.URL+"/api/v4", "body", files) + if err := os.WriteFile(paths[1], []byte("drifted"), 0o600); err != nil { + t.Fatal(err) + } + service, client := applyAttachmentService(t, server.URL, store) + defer client.Close() + _, err := service.Apply(context.Background(), applyClaim(stage, "apply-drifted-attachment")) + detail, showErr := store.Show(context.Background(), stage.Stage.ID) + if !errors.Is(err, ErrTargetDrift) || showErr != nil || detail.Lifecycle != stagestore.LifecycleOpen || writes.Load() != 0 { + t.Fatalf("detail=%+v writes=%d err=%v show=%v", detail.StageSummary, writes.Load(), err, showErr) + } +} + +func TestApplyAttachmentRejectionAfterValidatedUploadIsPartial(t *testing.T) { + var uploads, posts atomic.Int32 + server := attachmentTargetServer(t, func(response http.ResponseWriter, request *http.Request) { + if request.URL.Path == "/api/v4/posts" { + posts.Add(1) + response.WriteHeader(http.StatusInternalServerError) + return + } + index := uploads.Add(1) + if index == 2 { + response.WriteHeader(http.StatusForbidden) + return + } + name := request.URL.Query().Get("filename") + body, _ := io.ReadAll(request.Body) + response.WriteHeader(http.StatusCreated) + _, _ = fmt.Fprintf(response, `{"file_infos":[{"id":"file-1","user_id":"self","channel_id":"channel-1","create_at":100,"update_at":100,"delete_at":0,"name":%q,"size":%d}]}`, name, len(body)) + }) + defer server.Close() + store := openApplyStore(t) + stage := createAttachmentApplyStage(t, store, server.URL+"/api/v4", "body", [][]byte{[]byte("first"), []byte("second")}) + service, client := applyAttachmentService(t, server.URL, store) + defer client.Close() + receipt, err := service.Apply(context.Background(), applyClaim(stage, "apply-partial-upload")) + if err != nil || receipt.Outcome != stagestore.OutcomePartial || receipt.Recovery != stagestore.RecoveryPartial || receipt.Steps[0].State != stagestore.StepValidated || receipt.Steps[1].State != stagestore.StepRejected || receipt.Steps[2].State != stagestore.StepNotSent || posts.Load() != 0 { + t.Fatalf("receipt=%+v uploads=%d posts=%d err=%v", receipt, uploads.Load(), posts.Load(), err) + } +} + +func TestApplyAttachmentTargetDriftAfterUploadsLeavesPartialWithoutPost(t *testing.T) { + var channelReads, uploads, posts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/users/me": + _, _ = io.WriteString(response, `{"id":"self","username":"arda"}`) + case "/api/v4/channels/channel-1": + team := "team-1" + if channelReads.Add(1) > 1 { + team = "drifted" + } + _, _ = fmt.Fprintf(response, `{"id":"channel-1","team_id":%q,"type":"O","name":"town-square","display_name":"Town Square"}`, team) + case "/api/v4/channels/channel-1/members/self": + _, _ = io.WriteString(response, `{"channel_id":"channel-1","user_id":"self"}`) + case "/api/v4/files": + uploads.Add(1) + name := request.URL.Query().Get("filename") + body, _ := io.ReadAll(request.Body) + response.WriteHeader(http.StatusCreated) + _, _ = fmt.Fprintf(response, `{"file_infos":[{"id":"file-1","user_id":"self","channel_id":"channel-1","create_at":100,"update_at":100,"delete_at":0,"name":%q,"size":%d}]}`, name, len(body)) + case "/api/v4/posts": + posts.Add(1) + response.WriteHeader(http.StatusInternalServerError) + default: + http.NotFound(response, request) + } + })) + defer server.Close() + store := openApplyStore(t) + stage := createAttachmentApplyStage(t, store, server.URL+"/api/v4", "body", [][]byte{[]byte("file")}) + service, client := applyAttachmentService(t, server.URL, store) + defer client.Close() + receipt, err := service.Apply(context.Background(), applyClaim(stage, "apply-post-upload-drift")) + if err != nil || receipt.Outcome != stagestore.OutcomePartial || receipt.Steps[0].State != stagestore.StepValidated || receipt.Steps[1].State != stagestore.StepNotSent || uploads.Load() != 1 || posts.Load() != 0 { + t.Fatalf("receipt=%+v reads=%d uploads=%d posts=%d err=%v", receipt, channelReads.Load(), uploads.Load(), posts.Load(), err) + } +} + +func TestApplyAttachmentJournalHandoffAfterUploadReturnsDurablePartialReceipt(t *testing.T) { + var uploads, posts atomic.Int32 + server := attachmentTargetServer(t, func(response http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/files": + uploads.Add(1) + name := request.URL.Query().Get("filename") + body, _ := io.ReadAll(request.Body) + response.WriteHeader(http.StatusCreated) + _, _ = fmt.Fprintf(response, `{"file_infos":[{"id":"file-1","user_id":"self","channel_id":"channel-1","create_at":100,"update_at":100,"delete_at":0,"name":%q,"size":%d}]}`, name, len(body)) + case "/api/v4/posts": + posts.Add(1) + response.WriteHeader(http.StatusInternalServerError) + } + }) + defer server.Close() + store := openApplyStore(t) + stage := createAttachmentApplyStage(t, store, server.URL+"/api/v4", "body", [][]byte{[]byte("file")}) + faults := &faultStore{Store: store, beginOrdinal: 2, beginErr: errors.New("journal unavailable")} + service, client := applyAttachmentServiceWithStore(t, server.URL, faults, store.StateDir()) + defer client.Close() + receipt, err := service.Apply(context.Background(), applyClaim(stage, "apply-post-handoff-failure")) + if err != nil || receipt.Outcome != stagestore.OutcomePartial || receipt.Recovery != stagestore.RecoveryPartial || receipt.Steps[0].State != stagestore.StepValidated || receipt.Steps[1].State != stagestore.StepNotSent || uploads.Load() != 1 || posts.Load() != 0 { + t.Fatalf("receipt=%+v uploads=%d posts=%d err=%v", receipt, uploads.Load(), posts.Load(), err) + } +} + func TestApplyReplyRevalidatesSelectedPostAndCanonicalRoot(t *testing.T) { const message = "thread **reply**\n" var writes atomic.Int32 @@ -262,3 +441,86 @@ func mutationPostResponse(id, channelID, userID, message, rootID, pendingID stri } func stringPointer(value string) *string { return &value } + +func createAttachmentApplyStage(t *testing.T, store *stagestore.Store, serverURL, body string, files [][]byte) stagestore.MutationResult { + stage, _ := createAttachmentApplyStageWithPaths(t, store, serverURL, body, files) + return stage +} + +func createAttachmentApplyStageWithPaths(t *testing.T, store *stagestore.Store, serverURL, body string, files [][]byte) (stagestore.MutationResult, []string) { + t.Helper() + dir, err := os.MkdirTemp(".", ".apply-attachment-") + if err != nil { + t.Fatal(err) + } + dir, err = filepath.Abs(dir) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + inputs := make([]stageinput.Attachment, len(files)) + paths := make([]string, len(files)) + for i, content := range files { + paths[i] = filepath.Join(dir, fmt.Sprintf("file-%d.bin", i+1)) + if err = os.WriteFile(paths[i], content, 0o600); err != nil { + t.Fatal(err) + } + inputs[i] = stageinput.Attachment{Path: paths[i], MediaType: "application/octet-stream"} + } + attachments, err := stageinput.Bind(context.Background(), inputs, [][]byte{[]byte("token")}) + if err != nil { + t.Fatal(err) + } + destination, err := json.Marshal(staging.Destination{Kind: "conversation", ChannelID: "channel-1", ChannelType: "public", TeamID: stringPointer("team-1"), ParticipantIDs: []string{}}) + if err != nil { + t.Fatal(err) + } + steps := make([]string, 0, len(files)+1) + for i := range files { + steps = append(steps, fmt.Sprintf(`{"ordinal":%d,"type":"upload_attachment","condition":"always"}`, i+1)) + } + steps = append(steps, fmt.Sprintf(`{"ordinal":%d,"type":"create_post","condition":"always"}`, len(files)+1)) + created, err := store.Create(context.Background(), stagestore.CreateInput{RequestDigest: sha256.Sum256([]byte(body)), Operation: stagestore.CreatePost, ServerURL: serverURL, UserID: "self", + Content: stagestore.RevisionContent{Body: []byte(body), Destination: destination, Plan: json.RawMessage(`{"steps":[` + strings.Join(steps, ",") + `]}`), Attachments: attachments}}) + if err != nil { + t.Fatal(err) + } + return created.MutationResult, paths +} + +func applyAttachmentService(t *testing.T, serverURL string, store *stagestore.Store) (*Service, *api.Client) { + return applyAttachmentServiceWithStore(t, serverURL, store, store.StateDir()) +} + +func applyAttachmentServiceWithStore(t *testing.T, serverURL string, store Store, stateDirectory string) (*Service, *api.Client) { + t.Helper() + client, err := api.New(serverURL, "token") + if err != nil { + t.Fatal(err) + } + service, err := New(serverURL+"/api/v4", "", [][]byte{[]byte("token")}, store, mattermost.NewUsers(client), mattermost.NewChannels(client), mattermost.NewPosts(client), + mattermost.NewConversationMutations(client), mattermost.NewPostMutations(client), WithAttachmentExecution(stateDirectory, mattermost.NewFileMutations(client))) + if err != nil { + client.Close() + t.Fatal(err) + } + return service, client +} + +func attachmentTargetServer(t *testing.T, write func(http.ResponseWriter, *http.Request)) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/api/v4/users/me": + _, _ = io.WriteString(response, `{"id":"self","username":"arda"}`) + case "/api/v4/channels/channel-1": + _, _ = io.WriteString(response, `{"id":"channel-1","team_id":"team-1","type":"O","name":"town-square","display_name":"Town Square"}`) + case "/api/v4/channels/channel-1/members/self": + _, _ = io.WriteString(response, `{"channel_id":"channel-1","user_id":"self"}`) + case "/api/v4/files", "/api/v4/posts": + write(response, request) + default: + http.NotFound(response, request) + } + })) +} diff --git a/internal/apply/reaction.go b/internal/apply/reaction.go index f27ab7e..aea2a0f 100644 --- a/internal/apply/reaction.go +++ b/internal/apply/reaction.go @@ -41,7 +41,7 @@ func (s *Service) applyReaction(ctx context.Context, attempt stagestore.ApplyAtt } result, remoteErr := prepared.Execute(ctx) if remoteErr != nil { - return s.recordRemoteFailure(ctx, attempt.ID, remoteErr) + return s.recordRemoteFailure(ctx, attempt.ID, 1, remoteErr) } present, validationErr := s.revalidateReaction(ctx, currentUserID, destination) if validationErr != nil || operation == stagestore.React && !present || operation == stagestore.Unreact && present { diff --git a/internal/apply/service.go b/internal/apply/service.go index f058c39..14acb7d 100644 --- a/internal/apply/service.go +++ b/internal/apply/service.go @@ -8,10 +8,12 @@ import ( "errors" "fmt" "io" + "path/filepath" "slices" "github.com/ardasevinc/mattermost-cli/internal/api" "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/stageinput" "github.com/ardasevinc/mattermost-cli/internal/stagestore" "github.com/ardasevinc/mattermost-cli/internal/staging" ) @@ -22,6 +24,7 @@ var ( ErrUnsupportedOperation = errors.New("apply: operation is not implemented") ErrJournal = errors.New("apply: durable journal update failed") ErrCredential = errors.New("apply: active credential in outbound content") + ErrAttachmentBudget = errors.New("apply: attachment exceeds the safe spool or server budget") ) // ConfirmedEffectError means Mattermost confirmed the effect but its durable @@ -43,6 +46,7 @@ type Store interface { MarkStepRejected(context.Context, string, int, json.RawMessage) error MarkStepUnknown(context.Context, string, int) error MarkStepSkipped(context.Context, string, int, json.RawMessage) error + SealRemainingNotDispatched(context.Context, string) error FinalizeApply(context.Context, string) (stagestore.ApplyReceipt, error) } @@ -70,15 +74,35 @@ type Service struct { posts PostTargets writes *mattermost.ConversationMutations postWrites *mattermost.PostMutations + fileWrites *mattermost.FileMutations + spoolDirectory string credentials [][]byte } -func New(serverURL, serverID string, credentials [][]byte, store Store, users CurrentUser, channels Conversations, posts PostTargets, writes *mattermost.ConversationMutations, postWrites *mattermost.PostMutations) (*Service, error) { +type Option func(*Service) error + +func WithAttachmentExecution(directory string, writes *mattermost.FileMutations) Option { + return func(service *Service) error { + if !filepath.IsAbs(directory) || filepath.Clean(directory) != directory || writes == nil { + return ErrInvalid + } + service.spoolDirectory, service.fileWrites = directory, writes + return nil + } +} + +func New(serverURL, serverID string, credentials [][]byte, store Store, users CurrentUser, channels Conversations, posts PostTargets, writes *mattermost.ConversationMutations, postWrites *mattermost.PostMutations, options ...Option) (*Service, error) { protected, validCredentials := cloneCredentials(credentials) if serverURL == "" || !validCredentials || store == nil || users == nil || channels == nil || posts == nil || writes == nil || postWrites == nil { return nil, ErrInvalid } - return &Service{serverURL: serverURL, serverID: serverID, store: store, users: users, channels: channels, posts: posts, writes: writes, postWrites: postWrites, credentials: protected}, nil + service := &Service{serverURL: serverURL, serverID: serverID, store: store, users: users, channels: channels, posts: posts, writes: writes, postWrites: postWrites, credentials: protected} + for _, option := range options { + if option == nil || option(service) != nil { + return nil, ErrInvalid + } + } + return service, nil } func (s *Service) Apply(ctx context.Context, in stagestore.ApplyClaimInput) (stagestore.ApplyReceipt, error) { @@ -132,7 +156,21 @@ func (s *Service) Apply(ctx context.Context, in stagestore.ApplyClaimInput) (sta return stagestore.ApplyReceipt{}, ErrCredential } if len(detail.Attachments) > 0 { - return stagestore.ApplyReceipt{}, ErrUnsupportedOperation + if detail.Operation != stagestore.CreatePost && detail.Operation != stagestore.Reply || s.fileWrites == nil || s.spoolDirectory == "" { + return stagestore.ApplyReceipt{}, ErrUnsupportedOperation + } + if s.attachmentsContainCredential(detail.Attachments) { + return stagestore.ApplyReceipt{}, ErrCredential + } + for _, attachment := range detail.Attachments { + if attachment.FileIdentity == ([32]byte{}) || attachment.ByteLength <= 0 { + return stagestore.ApplyReceipt{}, ErrTargetDrift + } + } + serverMax, _ := s.fileWrites.DiscoverMaxUploadBytes(ctx) + if err := stageinput.ValidateSpoolBudget(detail.Attachments, s.spoolDirectory, serverMax); err != nil { + return stagestore.ApplyReceipt{}, errors.Join(ErrAttachmentBudget, err) + } } attempt, err := s.store.ClaimApply(ctx, in) if err != nil { @@ -155,11 +193,20 @@ func (s *Service) Apply(ctx context.Context, in stagestore.ApplyClaimInput) (sta return s.applyReaction(ctx, attempt, detail.Operation, current.ID, destination) } if detail.Operation == stagestore.CreatePost || detail.Operation == stagestore.Reply || detail.Operation == stagestore.EditPost || detail.Operation == stagestore.DeletePost { - return s.applyPost(ctx, attempt, detail.Operation, current.ID, destination, detail.Body) + return s.applyPost(ctx, attempt, detail.Operation, current.ID, destination, detail.Body, detail.Attachments) } return s.applyConversation(ctx, attempt, detail.Operation, current.ID, destination) } +func (s *Service) attachmentsContainCredential(values []stagestore.Attachment) bool { + for _, value := range values { + if s.containsCredential(value.SuppliedPath, value.CanonicalPath, value.RemoteFilename, value.MediaType) { + return true + } + } + return false +} + func supportedOperation(operation stagestore.Operation) bool { switch operation { case stagestore.CreatePost, stagestore.Reply, stagestore.EditPost, stagestore.DeletePost, stagestore.React, stagestore.Unreact, stagestore.ResolveDM, stagestore.ResolveGroupDM: @@ -192,7 +239,7 @@ func (s *Service) applyConversation(ctx context.Context, attempt stagestore.Appl } result, remoteErr := prepared.Execute(ctx) if remoteErr != nil { - return s.recordRemoteFailure(ctx, attempt.ID, remoteErr) + return s.recordRemoteFailure(ctx, attempt.ID, 1, remoteErr) } validated, found, validationErr := s.findConversation(ctx, operation, currentUserID, destination.ParticipantIDs) if validationErr != nil || !found || validated.ID != result.ChannelID { @@ -226,7 +273,7 @@ func (s *Service) findConversation(ctx context.Context, operation stagestore.Ope return s.channels.ExistingGroup(ctx, currentUserID, participantIDs) } -func (s *Service) recordRemoteFailure(ctx context.Context, attemptID string, remoteErr error) (stagestore.ApplyReceipt, error) { +func (s *Service) recordRemoteFailure(ctx context.Context, attemptID string, ordinal int, remoteErr error) (stagestore.ApplyReceipt, error) { journalCtx := context.WithoutCancel(ctx) var rejected *api.APIError var err error @@ -237,9 +284,9 @@ func (s *Service) recordRemoteFailure(ctx context.Context, attemptID string, rem if marshalErr != nil { return stagestore.ApplyReceipt{}, fmt.Errorf("%w: %v", ErrJournal, marshalErr) } - err = s.store.MarkStepRejected(journalCtx, attemptID, 1, result) + err = s.store.MarkStepRejected(journalCtx, attemptID, ordinal, result) } else { - err = s.store.MarkStepUnknown(journalCtx, attemptID, 1) + err = s.store.MarkStepUnknown(journalCtx, attemptID, ordinal) } if err != nil { return stagestore.ApplyReceipt{}, errors.Join(remoteErr, fmt.Errorf("%w: %v", ErrJournal, err)) diff --git a/internal/apply/service_test.go b/internal/apply/service_test.go index dbd99b4..1cf80d9 100644 --- a/internal/apply/service_test.go +++ b/internal/apply/service_test.go @@ -610,8 +610,16 @@ func applyServiceWithCredentials(t *testing.T, serverURL string, store Store, cr type faultStore struct { *stagestore.Store - skipErr, unknownErr, validatedErr, finalizeErr error - beforeSkip func() + skipErr, unknownErr, validatedErr, finalizeErr, beginErr error + beginOrdinal int + beforeSkip func() +} + +func (s *faultStore) BeginDispatch(ctx context.Context, attemptID string, ordinal int) error { + if s.beginErr != nil && ordinal == s.beginOrdinal { + return s.beginErr + } + return s.Store.BeginDispatch(ctx, attemptID, ordinal) } func (s *faultStore) MarkStepSkipped(ctx context.Context, attemptID string, ordinal int, result json.RawMessage) error { diff --git a/internal/cli/store_test.go b/internal/cli/store_test.go index 4a87f23..a8ca6ed 100644 --- a/internal/cli/store_test.go +++ b/internal/cli/store_test.go @@ -36,7 +36,7 @@ func TestStoreDoctorAbsentIsReadOnlyAndSchemaValid(t *testing.T) { if err := registry.Validate("mm/v2/store-doctor", bytes.NewReader(stdout.Bytes())); err != nil { t.Fatalf("schema: %v\n%s", err, stdout.String()) } - for _, fact := range []string{`"filesystemSafe":null`, `"applicationId":null`, `"integrity":null`, `"applied":null`, `"latest":8`, `"valid":null`, `"journalMode":null`} { + for _, fact := range []string{`"filesystemSafe":null`, `"applicationId":null`, `"integrity":null`, `"applied":null`, `"latest":9`, `"valid":null`, `"journalMode":null`} { if !strings.Contains(stdout.String(), fact) { t.Fatalf("absent report omitted %s: %s", fact, stdout.String()) } @@ -110,7 +110,7 @@ func TestStoreMigrationsIsOfflineAndSchemaValid(t *testing.T) { if err := registry.Validate("mm/v2/store-migrations", bytes.NewReader(stdout.Bytes())); err != nil { t.Fatalf("schema: %v\n%s", err, stdout.String()) } - want := "{\"schema\":\"mm/v2/store-migrations\",\"latest\":8,\"migrations\":[{\"version\":1,\"name\":\"core-stage-state\",\"checksum\":\"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f\"},{\"version\":2,\"name\":\"immutable-local-request-receipts\",\"checksum\":\"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c\"},{\"version\":3,\"name\":\"caller-intent-stage-create-replay\",\"checksum\":\"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5\"},{\"version\":4,\"name\":\"caller-intent-stage-revise-replay\",\"checksum\":\"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4\"},{\"version\":5,\"name\":\"revision-plan-follows-composition\",\"checksum\":\"fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0\"},{\"version\":6,\"name\":\"durable-apply-journal\",\"checksum\":\"4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56\"},{\"version\":7,\"name\":\"status-confirmed-delete-results\",\"checksum\":\"bf980f24ff24fdf9d6a3a28b9ded97b51336fa1c8ef290d1bdcf91f1b593c5ce\"},{\"version\":8,\"name\":\"already-satisfied-edit-apply\",\"checksum\":\"cff30dd4fcc987876e092e2de1f5afd21ebeadbcb792fd550a56eb4c221edd10\"}]}\n" + want := "{\"schema\":\"mm/v2/store-migrations\",\"latest\":9,\"migrations\":[{\"version\":1,\"name\":\"core-stage-state\",\"checksum\":\"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f\"},{\"version\":2,\"name\":\"immutable-local-request-receipts\",\"checksum\":\"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c\"},{\"version\":3,\"name\":\"caller-intent-stage-create-replay\",\"checksum\":\"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5\"},{\"version\":4,\"name\":\"caller-intent-stage-revise-replay\",\"checksum\":\"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4\"},{\"version\":5,\"name\":\"revision-plan-follows-composition\",\"checksum\":\"fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0\"},{\"version\":6,\"name\":\"durable-apply-journal\",\"checksum\":\"4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56\"},{\"version\":7,\"name\":\"status-confirmed-delete-results\",\"checksum\":\"bf980f24ff24fdf9d6a3a28b9ded97b51336fa1c8ef290d1bdcf91f1b593c5ce\"},{\"version\":8,\"name\":\"already-satisfied-edit-apply\",\"checksum\":\"cff30dd4fcc987876e092e2de1f5afd21ebeadbcb792fd550a56eb4c221edd10\"},{\"version\":9,\"name\":\"attachment-identity-binding\",\"checksum\":\"d0782e01014d010e5978b39bf6f04ef26508bec9318c7ace3be8bfbbcab4999d\"}]}\n" if stdout.String() != want { t.Fatalf("stdout does not match golden migration contract: %q", stdout.String()) } @@ -310,7 +310,7 @@ func TestStoreHumanOutputStatesBounds(t *testing.T) { t.Fatalf("stdout=%q", stdout.String()) } stdout.Reset() - if code := Execute(context.Background(), []string{"store", "migrations"}, strings.NewReader(""), &stdout, &stderr); code != 0 || !strings.Contains(stdout.String(), "latest: 8\n1 core-stage-state ") { + if code := Execute(context.Background(), []string{"store", "migrations"}, strings.NewReader(""), &stdout, &stderr); code != 0 || !strings.Contains(stdout.String(), "latest: 9\n1 core-stage-state ") { t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } } diff --git a/internal/mattermost/file_mutations.go b/internal/mattermost/file_mutations.go new file mode 100644 index 0000000..729489f --- /dev/null +++ b/internal/mattermost/file_mutations.go @@ -0,0 +1,188 @@ +package mattermost + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/url" + "strconv" + "unicode/utf8" + + "github.com/ardasevinc/mattermost-cli/internal/api" +) + +type FileMutations struct{ client *api.Client } + +func NewFileMutations(client *api.Client) *FileMutations { return &FileMutations{client: client} } + +// DiscoverMaxUploadBytes returns the public server limit when the client +// configuration exposes a valid MaxFileSize. Absence or retrieval failure is +// intentionally non-fatal because older servers may omit this hint. +func (m *FileMutations) DiscoverMaxUploadBytes(ctx context.Context) (int64, bool) { + if m == nil || m.client == nil || ctx == nil { + return 0, false + } + config := map[string]string{} + if m.client.GetPublic(ctx, "/config/client", &config) != nil { + return 0, false + } + raw, ok := config["MaxFileSize"] + if !ok { + return 0, false + } + value, err := strconv.ParseInt(raw, 10, 64) + if err != nil || value <= 0 { + return 0, false + } + return value, true +} + +type UploadMutationInput struct { + ChannelID, UserID, Filename, MediaType string + Length int64 + Body io.ReadCloser +} + +type UploadMutationResult struct { + FileID string `json:"fileId"` +} + +type PreparedUpload struct { + mutation *api.PreparedMutation + channelID, userID, filename string + length int64 +} + +func (m *FileMutations) PrepareUpload(in UploadMutationInput) (*PreparedUpload, error) { + if m == nil || m.client == nil || !isSafePostID(in.ChannelID) || !isSafePostID(in.UserID) || !validUploadFilename(in.Filename) || + in.MediaType == "" || in.Length <= 0 || in.Body == nil { + return nil, ErrInvalidMutationRequest + } + query := url.Values{} + query.Set("channel_id", in.ChannelID) + query.Set("filename", in.Filename) + prepared, err := m.client.PrepareRawPostStatus("/files?"+query.Encode(), in.MediaType, in.Body, in.Length, http.StatusCreated) + if err != nil { + return nil, err + } + return &PreparedUpload{prepared, in.ChannelID, in.UserID, in.Filename, in.Length}, nil +} + +func (p *PreparedUpload) Execute(ctx context.Context) (UploadMutationResult, error) { + if p == nil || p.mutation == nil { + return UploadMutationResult{}, &api.OutcomeUnknownError{} + } + var response uploadResponse + if err := p.mutation.Execute(ctx, &response); err != nil { + return UploadMutationResult{}, err + } + if len(response.FileInfos) != 1 || len(response.ClientIDs) != 0 { + return UploadMutationResult{}, &api.OutcomeUnknownError{} + } + file := response.FileInfos[0] + if !isSafePostID(file.ID) || file.UserID != p.userID || file.ChannelID != p.channelID || file.PostID != "" || file.Name != p.filename || + file.Size != p.length || file.CreateAt <= 0 || file.CreateAt > maxDateMilliseconds || file.UpdateAt < file.CreateAt || file.UpdateAt > maxDateMilliseconds || file.DeleteAt != 0 { + return UploadMutationResult{}, &api.OutcomeUnknownError{} + } + return UploadMutationResult{file.ID}, nil +} + +func (p *PreparedUpload) Close() error { + if p == nil || p.mutation == nil { + return nil + } + return p.mutation.Close() +} + +type uploadResponse struct { + FileInfos []uploadFileInfo + ClientIDs []string +} + +type uploadFileInfo struct { + ID, UserID, PostID, ChannelID, Name string + CreateAt, UpdateAt, DeleteAt, Size int64 +} + +func (r *uploadResponse) UnmarshalJSON(data []byte) error { + _, ok := uniqueJSONObject(data) + if !ok { + return ErrInvalidPostResponse + } + var envelope struct { + FileInfos json.RawMessage `json:"file_infos"` + ClientIDs json.RawMessage `json:"client_ids"` + } + if json.Unmarshal(data, &envelope) != nil || envelope.FileInfos == nil { + return ErrInvalidPostResponse + } + var infos []json.RawMessage + if json.Unmarshal(envelope.FileInfos, &infos) != nil || len(infos) != 1 { + return ErrInvalidPostResponse + } + infoRaw, ok := uniqueJSONObject(infos[0]) + if !ok { + return ErrInvalidPostResponse + } + file := uploadFileInfo{} + var fieldsOK bool + file.ID, fieldsOK = safePostID(infoRaw["id"]) + if !fieldsOK { + return ErrInvalidPostResponse + } + file.UserID, fieldsOK = safePostID(infoRaw["user_id"]) + if !fieldsOK { + return ErrInvalidPostResponse + } + file.ChannelID, fieldsOK = safePostID(infoRaw["channel_id"]) + if !fieldsOK { + return ErrInvalidPostResponse + } + if rawPostID, present := infoRaw["post_id"]; present { + file.PostID, fieldsOK = strictString(rawPostID) + if !fieldsOK { + return ErrInvalidPostResponse + } + } + file.Name, fieldsOK = strictString(infoRaw["name"]) + if !fieldsOK { + return ErrInvalidPostResponse + } + file.CreateAt, fieldsOK = nonnegativeInteger(infoRaw["create_at"]) + if !fieldsOK { + return ErrInvalidPostResponse + } + file.UpdateAt, fieldsOK = nonnegativeInteger(infoRaw["update_at"]) + if !fieldsOK { + return ErrInvalidPostResponse + } + file.DeleteAt, fieldsOK = nonnegativeInteger(infoRaw["delete_at"]) + if !fieldsOK { + return ErrInvalidPostResponse + } + file.Size, fieldsOK = nonnegativeInteger(infoRaw["size"]) + if !fieldsOK { + return ErrInvalidPostResponse + } + clientIDs := []string(nil) + if envelope.ClientIDs != nil && string(envelope.ClientIDs) != "null" && json.Unmarshal(envelope.ClientIDs, &clientIDs) != nil { + return ErrInvalidPostResponse + } + *r = uploadResponse{[]uploadFileInfo{file}, clientIDs} + return nil +} + +func validUploadFilename(value string) bool { + if value == "" || len(value) > 255 || !utf8.ValidString(value) || value == "." || value == ".." { + return false + } + for _, r := range value { + if r < 0x20 || r == 0x7f || r == '/' || r == '\\' { + return false + } + } + return true +} + +var _ json.Unmarshaler = (*uploadResponse)(nil) diff --git a/internal/mattermost/file_mutations_test.go b/internal/mattermost/file_mutations_test.go new file mode 100644 index 0000000..cf13d7b --- /dev/null +++ b/internal/mattermost/file_mutations_test.go @@ -0,0 +1,107 @@ +package mattermost + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "strings" + "sync/atomic" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/api" +) + +func TestUploadMutationSendsExactRawFileAndValidatesIdentity(t *testing.T) { + payload := []byte{0, 1, 2, 0xff, '\n'} + var calls atomic.Int32 + client := mutationClient(t, mutationRoundTripFunc(func(request *http.Request) (*http.Response, error) { + calls.Add(1) + if request.URL.RequestURI() != "/api/v4/files?channel_id=channel-1&filename=report.bin" || request.Header.Get("Content-Type") != "application/octet-stream" || request.ContentLength != int64(len(payload)) || request.GetBody != nil { + t.Fatalf("request=%s type=%q length=%d replay=%v", request.URL.RequestURI(), request.Header.Get("Content-Type"), request.ContentLength, request.GetBody != nil) + } + got, _ := io.ReadAll(request.Body) + if !bytes.Equal(got, payload) { + t.Fatalf("body=%x", got) + } + response := `{"file_infos":[{"id":"file-1","user_id":"user-1","channel_id":"channel-1","create_at":100,"update_at":100,"delete_at":0,"name":"report.bin","size":5,"mime_type":"application/octet-stream"}]}` + return mutationResponse(http.StatusCreated, response), nil + })) + prepared, err := NewFileMutations(client).PrepareUpload(UploadMutationInput{ + ChannelID: "channel-1", UserID: "user-1", Filename: "report.bin", MediaType: "application/octet-stream", Length: int64(len(payload)), Body: io.NopCloser(bytes.NewReader(payload)), + }) + if err != nil { + t.Fatal(err) + } + result, err := prepared.Execute(context.Background()) + if err != nil || result.FileID != "file-1" || calls.Load() != 1 { + t.Fatalf("result=%+v calls=%d err=%v", result, calls.Load(), err) + } +} + +func TestUploadMutationRejectsUnvalidatedSuccessAndBadInput(t *testing.T) { + valid := `{"file_infos":[{"id":"file-1","user_id":"user-1","channel_id":"channel-1","create_at":100,"update_at":100,"delete_at":0,"name":"a.txt","size":1}]}` + for name, response := range map[string]string{ + "wrong user": strings.Replace(valid, `"user-1"`, `"other"`, 1), + "wrong channel": strings.Replace(valid, `"channel-1"`, `"other"`, 1), + "wrong name": strings.Replace(valid, `"a.txt"`, `"b.txt"`, 1), + "wrong size": strings.Replace(valid, `"size":1`, `"size":2`, 1), + "two files": strings.Replace(valid, `]}`, `,{}]}`, 1), + "client id": strings.Replace(valid, `]}`, `],"client_ids":["x"]}`, 1), + } { + t.Run(name, func(t *testing.T) { + client := mutationClient(t, mutationRoundTripFunc(func(*http.Request) (*http.Response, error) { + return mutationResponse(http.StatusCreated, response), nil + })) + prepared, err := NewFileMutations(client).PrepareUpload(UploadMutationInput{"channel-1", "user-1", "a.txt", "text/plain", 1, io.NopCloser(strings.NewReader("x"))}) + if err != nil { + t.Fatal(err) + } + _, err = prepared.Execute(context.Background()) + var unknown *api.OutcomeUnknownError + if !errors.As(err, &unknown) { + t.Fatalf("error=%v", err) + } + }) + } + if _, err := NewFileMutations(nil).PrepareUpload(UploadMutationInput{}); !errors.Is(err, ErrInvalidMutationRequest) { + t.Fatalf("invalid=%v", err) + } +} + +func TestPreparedUploadCloseDoesNotDispatch(t *testing.T) { + var calls atomic.Int32 + client := mutationClient(t, mutationRoundTripFunc(func(*http.Request) (*http.Response, error) { + calls.Add(1) + return nil, errors.New("unexpected") + })) + prepared, err := NewFileMutations(client).PrepareUpload(UploadMutationInput{"channel-1", "user-1", "a.txt", "text/plain", 1, io.NopCloser(strings.NewReader("x"))}) + if err != nil { + t.Fatal(err) + } + if err = prepared.Close(); err != nil || calls.Load() != 0 { + t.Fatalf("calls=%d err=%v", calls.Load(), err) + } +} + +func TestDiscoverMaxUploadBytesUsesOnlyValidPublicHint(t *testing.T) { + for name, response := range map[string]string{ + "valid": `{"MaxFileSize":"12345"}`, + "missing": `{"Version":"10.0"}`, + "invalid": `{"MaxFileSize":"unbounded"}`, + } { + t.Run(name, func(t *testing.T) { + client := mutationClient(t, mutationRoundTripFunc(func(request *http.Request) (*http.Response, error) { + if request.URL.RequestURI() != "/api/v4/config/client" || request.Header.Get("Authorization") != "" { + t.Fatalf("uri=%s authorization=%q", request.URL.RequestURI(), request.Header.Get("Authorization")) + } + return mutationResponse(http.StatusOK, response), nil + })) + value, found := NewFileMutations(client).DiscoverMaxUploadBytes(context.Background()) + if name == "valid" && (!found || value != 12345) || name != "valid" && (found || value != 0) { + t.Fatalf("value=%d found=%v", value, found) + } + }) + } +} diff --git a/internal/stageinput/file_other.go b/internal/stageinput/file_other.go index 6cbb320..09584ec 100644 --- a/internal/stageinput/file_other.go +++ b/internal/stageinput/file_other.go @@ -8,5 +8,6 @@ type fileIdentity struct{ size int64 } func (a fileIdentity) sameFile(fileIdentity) bool { return false } func (a fileIdentity) stable(fileIdentity) bool { return false } +func (a fileIdentity) binding() [32]byte { return [32]byte{} } func openSecure(string) (*os.File, fileIdentity, error) { return nil, fileIdentity{}, ErrUnsupported } func fileIdentityOf(*os.File) (fileIdentity, error) { return fileIdentity{}, ErrUnsupported } diff --git a/internal/stageinput/file_unix.go b/internal/stageinput/file_unix.go index 914f7c0..974ad71 100644 --- a/internal/stageinput/file_unix.go +++ b/internal/stageinput/file_unix.go @@ -3,6 +3,8 @@ package stageinput import ( + "crypto/sha256" + "encoding/binary" "errors" "os" "path/filepath" @@ -25,6 +27,18 @@ func (a fileIdentity) stable(b fileIdentity) bool { return a.sameFile(b) && a.mode == b.mode && a.nlink == b.nlink && a.size == b.size && a.mtimeNsec == b.mtimeNsec && a.ctimeNsec == b.ctimeNsec } +func (a fileIdentity) binding() [32]byte { + var encoded [52]byte + binary.BigEndian.PutUint64(encoded[0:8], a.dev) + binary.BigEndian.PutUint64(encoded[8:16], a.ino) + binary.BigEndian.PutUint32(encoded[16:20], a.mode) + binary.BigEndian.PutUint64(encoded[20:28], a.nlink) + binary.BigEndian.PutUint64(encoded[28:36], uint64(a.size)) + binary.BigEndian.PutUint64(encoded[36:44], uint64(a.mtimeNsec)) + binary.BigEndian.PutUint64(encoded[44:52], uint64(a.ctimeNsec)) + return sha256.Sum256(encoded[:]) +} + func openSecure(path string) (*os.File, fileIdentity, error) { if !filepath.IsAbs(path) { return nil, fileIdentity{}, ErrInvalid diff --git a/internal/stageinput/file_unix_test.go b/internal/stageinput/file_unix_test.go new file mode 100644 index 0000000..2a51bd5 --- /dev/null +++ b/internal/stageinput/file_unix_test.go @@ -0,0 +1,23 @@ +//go:build darwin || linux + +package stageinput + +import "testing" + +func TestDurableFileBindingCoversCompleteStableIdentity(t *testing.T) { + base := fileIdentity{dev: 1, ino: 2, mode: 0o100600, nlink: 1, size: 3, mtimeNsec: 4, ctimeNsec: 5} + mutations := []fileIdentity{ + {dev: 9, ino: 2, mode: 0o100600, nlink: 1, size: 3, mtimeNsec: 4, ctimeNsec: 5}, + {dev: 1, ino: 9, mode: 0o100600, nlink: 1, size: 3, mtimeNsec: 4, ctimeNsec: 5}, + {dev: 1, ino: 2, mode: 0o100400, nlink: 1, size: 3, mtimeNsec: 4, ctimeNsec: 5}, + {dev: 1, ino: 2, mode: 0o100600, nlink: 2, size: 3, mtimeNsec: 4, ctimeNsec: 5}, + {dev: 1, ino: 2, mode: 0o100600, nlink: 1, size: 9, mtimeNsec: 4, ctimeNsec: 5}, + {dev: 1, ino: 2, mode: 0o100600, nlink: 1, size: 3, mtimeNsec: 9, ctimeNsec: 5}, + {dev: 1, ino: 2, mode: 0o100600, nlink: 1, size: 3, mtimeNsec: 4, ctimeNsec: 9}, + } + for i, changed := range mutations { + if base.binding() == changed.binding() { + t.Fatalf("mutation %d did not change durable binding", i) + } + } +} diff --git a/internal/stageinput/input.go b/internal/stageinput/input.go index e28208b..034f0e6 100644 --- a/internal/stageinput/input.go +++ b/internal/stageinput/input.go @@ -16,10 +16,13 @@ import ( "unicode/utf8" "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "golang.org/x/text/unicode/norm" ) const ( - MaxAttachments = 100 + MaxAttachments = 5 + MaxSpoolBytes = int64(512 << 20) + MinSpoolFreeReserve = int64(64 << 20) maxPathBytes = 4096 maxFilenameBytes = 255 maxMediaTypeBytes = 255 @@ -35,6 +38,8 @@ var ( ErrFileChanged = errors.New("stage input: attachment changed while binding") ErrUnsupported = errors.New("stage input: secure attachment binding unsupported on this platform") ErrTooMany = errors.New("stage input: too many attachments") + ErrTooLarge = errors.New("stage input: attachment spool budget exceeded") + ErrNoSpoolSpace = errors.New("stage input: insufficient private spool space") ErrCredentialSet = errors.New("stage input: invalid protected credential set") ) @@ -117,6 +122,9 @@ func Bind(ctx context.Context, inputs []Attachment, credentials [][]byte) ([]sta if scanErr != nil { return nil, scanErr } + if length == 0 { + return nil, ErrInvalid + } if statErr != nil || closeErr != nil { return nil, ErrUnsafeFile } @@ -139,7 +147,8 @@ func Bind(ctx context.Context, inputs []Attachment, credentials [][]byte) ([]sta return nil, ErrFileChanged } bound = append(bound, stagestore.Attachment{SuppliedPath: input.supplied, CanonicalPath: input.canonical, - RemoteFilename: input.filename, ByteLength: length, MediaType: mediaType, ContentDigest: digest}) + RemoteFilename: input.filename, ByteLength: length, MediaType: mediaType, ContentDigest: digest, + FileIdentity: before.binding()}) } return bound, nil } @@ -162,6 +171,7 @@ func prepareMetadata(input Attachment) (preparedAttachment, error) { if filename == "" { filename = filepath.Base(canonical) } + filename = norm.NFC.String(filename) if !validText(filename, maxFilenameBytes) || strings.TrimSpace(filename) != filename || filename == "." || filename == ".." || strings.ContainsAny(filename, `/\`) || filepath.Base(filename) != filename { return preparedAttachment{}, ErrInvalid } diff --git a/internal/stageinput/input_test.go b/internal/stageinput/input_test.go index 359108b..5a7e1c6 100644 --- a/internal/stageinput/input_test.go +++ b/internal/stageinput/input_test.go @@ -12,6 +12,8 @@ import ( "strings" "testing" "time" + + "github.com/ardasevinc/mattermost-cli/internal/stagestore" ) func TestTokenScannerEveryChunkBoundary(t *testing.T) { @@ -60,6 +62,43 @@ func TestBindCapturesMetadataAndRejectsMetadataCredential(t *testing.T) { } } +func TestBindRejectsEmptyAndMoreThanFiveAttachments(t *testing.T) { + dir := localTempDir(t) + empty := filepath.Join(dir, "empty") + if err := os.WriteFile(empty, nil, 0o600); err != nil { + t.Fatal(err) + } + if result, err := Bind(context.Background(), []Attachment{{Path: empty}}, nil); !errors.Is(err, ErrInvalid) || result != nil { + t.Fatalf("empty result=%v err=%v", result, err) + } + inputs := make([]Attachment, MaxAttachments+1) + for i := range inputs { + inputs[i] = Attachment{Path: empty} + } + if result, err := Bind(context.Background(), inputs, nil); !errors.Is(err, ErrTooMany) || result != nil { + t.Fatalf("too many result=%v err=%v", result, err) + } +} + +func TestValidateSpoolBudgetChecksServerAggregateAndFreeSpace(t *testing.T) { + directory := localTempDir(t) + attachment := func(length int64) stagestore.Attachment { + return stagestore.Attachment{ByteLength: length} + } + if err := ValidateSpoolBudget([]stagestore.Attachment{attachment(1), attachment(2)}, directory, 2); err != nil { + t.Fatal(err) + } + if err := ValidateSpoolBudget([]stagestore.Attachment{attachment(3)}, directory, 2); !errors.Is(err, ErrTooLarge) { + t.Fatalf("server limit error=%v", err) + } + if err := ValidateSpoolBudget([]stagestore.Attachment{attachment(MaxSpoolBytes), attachment(1)}, directory, 0); !errors.Is(err, ErrTooLarge) { + t.Fatalf("aggregate error=%v", err) + } + if err := ValidateSpoolBudget([]stagestore.Attachment{attachment(1)}, filepath.Join(directory, "missing"), 0); !errors.Is(err, ErrNoSpoolSpace) { + t.Fatalf("space error=%v", err) + } +} + func TestBindDerivesSafeFilenameAndMediaType(t *testing.T) { path := filepath.Join(localTempDir(t), "note.txt") if err := os.WriteFile(path, []byte("plain text\n"), 0o600); err != nil { @@ -109,6 +148,79 @@ func TestBindCanonicalizesMediaType(t *testing.T) { } } +func TestSnapshotRevalidatesExactBindingAndLeavesNoResidue(t *testing.T) { + dir := localTempDir(t) + path := filepath.Join(dir, "payload.bin") + content := append(bytes.Repeat([]byte("chunk-"), 9000), []byte("tail")...) + if err := os.WriteFile(path, content, 0o600); err != nil { + t.Fatal(err) + } + bound, err := Bind(context.Background(), []Attachment{{Path: path}}, [][]byte{[]byte("absent-token")}) + if err != nil || bound[0].FileIdentity == ([32]byte{}) { + t.Fatalf("bound=%+v err=%v", bound, err) + } + spool, err := Snapshot(context.Background(), bound[0], [][]byte{[]byte("absent-token")}, dir) + if err != nil { + t.Fatal(err) + } + got, readErr := io.ReadAll(spool) + if readErr != nil || !bytes.Equal(got, content) || spool.Length != int64(len(content)) || spool.RemoteFilename != "payload.bin" { + t.Fatalf("length=%d name=%q read=%v exact=%v", spool.Length, spool.RemoteFilename, readErr, bytes.Equal(got, content)) + } + entries, err := os.ReadDir(dir) + if err != nil || len(entries) != 1 || entries[0].Name() != "payload.bin" { + t.Fatalf("spool residue=%v err=%v", entries, err) + } + if err = spool.Close(); err != nil { + t.Fatal(err) + } +} + +func TestSnapshotRejectsReplacementDriftAndCredentialBytes(t *testing.T) { + for name, mutate := range map[string]func(string) error{ + "replacement same bytes": func(path string) error { + replacement := path + ".new" + if err := os.WriteFile(replacement, []byte("original"), 0o600); err != nil { + return err + } + return os.Rename(replacement, path) + }, + "changed bytes": func(path string) error { return os.WriteFile(path, []byte("different"), 0o600) }, + } { + t.Run(name, func(t *testing.T) { + dir := localTempDir(t) + path := filepath.Join(dir, "payload") + if err := os.WriteFile(path, []byte("original"), 0o600); err != nil { + t.Fatal(err) + } + bound, err := Bind(context.Background(), []Attachment{{Path: path}}, nil) + if err != nil { + t.Fatal(err) + } + if err = mutate(path); err != nil { + t.Fatal(err) + } + if spool, snapshotErr := Snapshot(context.Background(), bound[0], nil, dir); !errors.Is(snapshotErr, ErrFileChanged) || spool != nil { + t.Fatalf("spool=%v err=%v", spool, snapshotErr) + } + }) + } + + dir := localTempDir(t) + path := filepath.Join(dir, "credential") + if err := os.WriteFile(path, []byte("safe-original"), 0o600); err != nil { + t.Fatal(err) + } + bound, err := Bind(context.Background(), []Attachment{{Path: path}}, nil) + if err != nil { + t.Fatal(err) + } + credential := []byte("safe-original") + if spool, snapshotErr := Snapshot(context.Background(), bound[0], [][]byte{credential}, dir); !errors.Is(snapshotErr, ErrCredential) || spool != nil { + t.Fatalf("spool=%v err=%v", spool, snapshotErr) + } +} + func TestPreflightReturnsNormalizedIntentWithoutOpeningFile(t *testing.T) { missing := filepath.Join(t.TempDir(), "missing.txt") intent, err := Preflight([]Attachment{{Path: missing, RemoteFilename: "Report.TXT", MediaType: `Text/Plain; Charset="UTF-8"`}}) diff --git a/internal/stageinput/space_other.go b/internal/stageinput/space_other.go new file mode 100644 index 0000000..5d17a3e --- /dev/null +++ b/internal/stageinput/space_other.go @@ -0,0 +1,9 @@ +//go:build !darwin && !linux + +package stageinput + +import "errors" + +func availableSpoolBytes(string) (uint64, error) { + return 0, errors.New("spool filesystem accounting unsupported") +} diff --git a/internal/stageinput/space_unix.go b/internal/stageinput/space_unix.go new file mode 100644 index 0000000..fa8f5b9 --- /dev/null +++ b/internal/stageinput/space_unix.go @@ -0,0 +1,23 @@ +//go:build darwin || linux + +package stageinput + +import ( + "errors" + "math" + + "golang.org/x/sys/unix" +) + +func availableSpoolBytes(directory string) (uint64, error) { + var stat unix.Statfs_t + if err := unix.Statfs(directory, &stat); err != nil || stat.Bsize <= 0 { + return 0, errors.New("spool filesystem unavailable") + } + blockSize := uint64(stat.Bsize) + availableBlocks := uint64(stat.Bavail) + if availableBlocks > math.MaxUint64/blockSize { + return math.MaxUint64, nil + } + return availableBlocks * blockSize, nil +} diff --git a/internal/stageinput/spool.go b/internal/stageinput/spool.go new file mode 100644 index 0000000..02ef924 --- /dev/null +++ b/internal/stageinput/spool.go @@ -0,0 +1,155 @@ +package stageinput + +import ( + "bytes" + "context" + "crypto/sha256" + "io" + "os" + "path/filepath" + + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +// Spool is an unlinked private snapshot. It survives only while its descriptor +// is open, so neither success nor process failure can leave plaintext residue. +type Spool struct { + file *os.File + Length int64 + RemoteFilename string + MediaType string +} + +func (s *Spool) Read(p []byte) (int, error) { + if s == nil || s.file == nil { + return 0, os.ErrClosed + } + return s.file.Read(p) +} + +func (s *Spool) Close() error { + if s == nil || s.file == nil { + return nil + } + err := s.file.Close() + s.file = nil + return err +} + +// ValidateSpoolBudget proves that the complete immutable snapshot set fits +// both the local v2 cap and the currently available private state filesystem. +// A positive serverMax additionally enforces a discovered per-file limit. +func ValidateSpoolBudget(attachments []stagestore.Attachment, directory string, serverMax int64) error { + if len(attachments) == 0 || len(attachments) > MaxAttachments || !filepath.IsAbs(directory) || filepath.Clean(directory) != directory || serverMax < 0 { + return ErrInvalid + } + var total int64 + for _, attachment := range attachments { + if attachment.ByteLength <= 0 || serverMax > 0 && attachment.ByteLength > serverMax || attachment.ByteLength > MaxSpoolBytes-total { + return ErrTooLarge + } + total += attachment.ByteLength + } + available, err := availableSpoolBytes(directory) + if err != nil { + return ErrNoSpoolSpace + } + if uint64(total+MinSpoolFreeReserve) > available { + return ErrNoSpoolSpace + } + return nil +} + +// Snapshot securely reopens, rescans, rehashes, and copies one stored binding +// into a private immutable-by-convention descriptor for a single upload. +func Snapshot(ctx context.Context, bound stagestore.Attachment, credentials [][]byte, directory string) (*Spool, error) { + if ctx == nil || !filepath.IsAbs(directory) || filepath.Clean(directory) != directory || bound.FileIdentity == ([32]byte{}) || bound.ByteLength <= 0 { + return nil, ErrInvalid + } + if err := ctx.Err(); err != nil { + return nil, err + } + scanner, err := newScanner(credentials) + if err != nil { + return nil, err + } + metadata, err := prepareMetadata(Attachment{Path: bound.CanonicalPath, RemoteFilename: bound.RemoteFilename, MediaType: bound.MediaType}) + if err != nil || metadata.canonical != bound.CanonicalPath || scanner.contains([]byte(bound.SuppliedPath)) || scanner.contains([]byte(bound.CanonicalPath)) || + scanner.contains([]byte(bound.RemoteFilename)) || scanner.contains([]byte(bound.MediaType)) { + return nil, ErrCredential + } + source, before, err := openSecure(bound.CanonicalPath) + if err != nil { + return nil, ErrFileChanged + } + if before.binding() != bound.FileIdentity { + _ = source.Close() + return nil, ErrFileChanged + } + + spool, err := os.CreateTemp(directory, ".mm-spool-") + if err != nil { + _ = source.Close() + return nil, ErrUnsafeFile + } + name := spool.Name() + cleanup := func() { + _ = source.Close() + _ = spool.Close() + _ = os.Remove(name) + } + if err = spool.Chmod(0o600); err != nil || os.Remove(name) != nil { + cleanup() + return nil, ErrUnsafeFile + } + + hash := sha256.New() + stream := scanner.stream() + buffer := make([]byte, 32*1024) + var length int64 + for { + if err = ctx.Err(); err != nil { + cleanup() + return nil, err + } + n, readErr := source.Read(buffer) + if n > 0 { + chunk := buffer[:n] + length += int64(n) + _, _ = hash.Write(chunk) + if stream.write(chunk) { + cleanup() + return nil, ErrCredential + } + written, writeErr := spool.Write(chunk) + if writeErr != nil || written != n { + cleanup() + return nil, ErrUnsafeFile + } + } + if readErr == io.EOF { + break + } + if readErr != nil || n == 0 { + cleanup() + return nil, ErrUnsafeFile + } + } + after, statErr := fileIdentityOf(source) + closeErr := source.Close() + if statErr != nil || closeErr != nil || !before.stable(after) || length != bound.ByteLength || !bytes.Equal(hash.Sum(nil), bound.ContentDigest[:]) { + _ = spool.Close() + return nil, ErrFileChanged + } + if err = spool.Sync(); err != nil { + _ = spool.Close() + return nil, ErrUnsafeFile + } + if _, err = spool.Seek(0, io.SeekStart); err != nil { + _ = spool.Close() + return nil, ErrUnsafeFile + } + return &Spool{file: spool, Length: length, RemoteFilename: bound.RemoteFilename, MediaType: bound.MediaType}, nil +} + +var _ io.ReadCloser = (*Spool)(nil) diff --git a/internal/stagestore/apply.go b/internal/stagestore/apply.go index e87e2f6..63996c7 100644 --- a/internal/stagestore/apply.go +++ b/internal/stagestore/apply.go @@ -213,6 +213,45 @@ func (s *Store) MarkStepSkipped(ctx context.Context, attemptID string, ordinal i return s.transitionStep(ctx, attemptID, ordinal, StepPending, StepSkipped, result) } +// SealRemainingNotDispatched closes a compound attempt after confirmed early +// effects when a fresh local precondition prevents the next dispatch. +func (s *Store) SealRemainingNotDispatched(ctx context.Context, attemptID string) error { + if ctx == nil || !bounded(attemptID, maxIdentityBytes) { + return ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return localError(err) + } + defer tx.Rollback() + attempt, err := scanApplyAttempt(ctx, tx, attemptID) + if err != nil { + return err + } + hasEffect, hasPending := false, false + for _, step := range attempt.Steps { + switch step.State { + case StepValidated, StepSkipped: + hasEffect = true + case StepPending: + hasPending = true + default: + return ErrNotEligible + } + } + if !hasEffect || !hasPending { + return ErrNotEligible + } + if err = sealPendingSteps(ctx, tx, &attempt, formatTime(time.Now().UTC()), "not_dispatched"); err != nil { + return err + } + if err = tx.Commit(); err != nil { + return localError(err) + } + runCommitHook() + return nil +} + func (s *Store) transitionStep(ctx context.Context, attemptID string, ordinal int, from, to StepState, result json.RawMessage) error { if ctx == nil || !bounded(attemptID, maxIdentityBytes) || ordinal < 1 || !validStepTransition(from, to) { return ErrInvalid diff --git a/internal/stagestore/domain.go b/internal/stagestore/domain.go index fa25630..2286596 100644 --- a/internal/stagestore/domain.go +++ b/internal/stagestore/domain.go @@ -26,7 +26,7 @@ const ( maxJSONBytes = 1 << 20 maxRequestID = 256 maxListLimit = 100 - maxAttachments = 100 + maxAttachments = 5 maxFilenameBytes = 255 maxMediaTypeBytes = 256 ) @@ -78,6 +78,7 @@ type Attachment struct { ByteLength int64 `json:"byteLength"` MediaType string `json:"mediaType,omitempty"` ContentDigest [32]byte `json:"contentDigest"` + FileIdentity [32]byte `json:"-"` } type RevisionContent struct { Body []byte @@ -270,7 +271,7 @@ func (s *Store) Revise(ctx context.Context, in ReviseInput) (MutationResult, err return MutationResult{}, ErrNotEligible } recovery = RecoveryNone - } else if base.Lifecycle != LifecycleOpen || base.Recovery == RecoveryForbidden { + } else if base.Lifecycle != LifecycleOpen || base.Recovery == RecoveryForbidden || base.Recovery == RecoveryPartial { return MutationResult{}, ErrNotEligible } next := base.Revision + 1 @@ -522,14 +523,33 @@ func scanCurrent(ctx context.Context, tx *sql.Tx, id string) (StageDetail, error } type semanticContent struct { - Body []byte `json:"body,omitempty"` - Destination json.RawMessage `json:"destination"` - Plan json.RawMessage `json:"plan"` - Attachments []Attachment `json:"attachments,omitempty"` + Body []byte `json:"body,omitempty"` + Destination json.RawMessage `json:"destination"` + Plan json.RawMessage `json:"plan"` + Attachments []semanticAttachment `json:"attachments,omitempty"` +} + +type semanticAttachment struct { + SuppliedPath string `json:"suppliedPath"` + CanonicalPath string `json:"canonicalPath"` + RemoteFilename string `json:"remoteFilename"` + ByteLength int64 `json:"byteLength"` + MediaType string `json:"mediaType,omitempty"` + ContentDigest [32]byte `json:"contentDigest"` + FileIdentity []byte `json:"fileIdentity,omitempty"` } func (v RevisionContent) semantic() semanticContent { - return semanticContent{v.Body, v.Destination, v.Plan, v.Attachments} + attachments := make([]semanticAttachment, len(v.Attachments)) + for i, attachment := range v.Attachments { + var identity []byte + if attachment.FileIdentity != ([32]byte{}) { + identity = bytes.Clone(attachment.FileIdentity[:]) + } + attachments[i] = semanticAttachment{attachment.SuppliedPath, attachment.CanonicalPath, attachment.RemoteFilename, attachment.ByteLength, + attachment.MediaType, attachment.ContentDigest, identity} + } + return semanticContent{v.Body, v.Destination, v.Plan, attachments} } func semanticDigest(op Operation, server, serverID, user string, c RevisionContent) [32]byte { return digestValue(struct { @@ -613,7 +633,7 @@ func normalizeComposition(op Operation, v Composition) (Composition, error) { return v, ErrInvalid } for _, a := range v.Attachments { - if !boundedMetadata(a.SuppliedPath, maxIdentityBytes) || !boundedMetadata(a.CanonicalPath, maxIdentityBytes) || !boundedMetadata(a.RemoteFilename, maxFilenameBytes) || a.ByteLength < 0 || a.ContentDigest == ([32]byte{}) || (a.MediaType != "" && !boundedMetadata(a.MediaType, maxMediaTypeBytes)) { + if !boundedMetadata(a.SuppliedPath, maxIdentityBytes) || !boundedMetadata(a.CanonicalPath, maxIdentityBytes) || !boundedMetadata(a.RemoteFilename, maxFilenameBytes) || a.ByteLength <= 0 || a.ContentDigest == ([32]byte{}) || (a.MediaType != "" && !boundedMetadata(a.MediaType, maxMediaTypeBytes)) { return v, ErrInvalid } } @@ -761,6 +781,11 @@ func insertAttachments(ctx context.Context, tx *sql.Tx, stage string, revision i if _, err := tx.ExecContext(ctx, `INSERT INTO stage_attachments(stage_id,revision,ordinal,supplied_path,canonical_path,remote_filename,byte_length,media_type,content_digest) VALUES(?,?,?,?,?,?,?,?,?)`, stage, revision, i, a.SuppliedPath, a.CanonicalPath, a.RemoteFilename, a.ByteLength, nullable(a.MediaType), a.ContentDigest[:]); err != nil { return localError(err) } + if attachmentIdentityAvailable() { + if _, err := tx.ExecContext(ctx, `INSERT INTO stage_attachment_identities(stage_id,revision,ordinal,file_identity) VALUES(?,?,?,?)`, stage, revision, i, a.FileIdentity[:]); err != nil { + return localError(err) + } + } } return nil } @@ -771,7 +796,11 @@ type queryer interface { } func readAttachments(ctx context.Context, q queryer, stage string, revision int64) ([]Attachment, error) { - rows, err := q.QueryContext(ctx, `SELECT supplied_path,canonical_path,remote_filename,byte_length,coalesce(media_type,''),content_digest FROM stage_attachments WHERE stage_id=? AND revision=? ORDER BY ordinal`, stage, revision) + query := `SELECT supplied_path,canonical_path,remote_filename,byte_length,coalesce(media_type,''),content_digest,NULL FROM stage_attachments WHERE stage_id=? AND revision=? ORDER BY ordinal` + if attachmentIdentityAvailable() { + query = `SELECT a.supplied_path,a.canonical_path,a.remote_filename,a.byte_length,coalesce(a.media_type,''),a.content_digest,i.file_identity FROM stage_attachments a LEFT JOIN stage_attachment_identities i USING(stage_id,revision,ordinal) WHERE a.stage_id=? AND a.revision=? ORDER BY a.ordinal` + } + rows, err := q.QueryContext(ctx, query, stage, revision) if err != nil { return nil, localError(err) } @@ -779,14 +808,17 @@ func readAttachments(ctx context.Context, q queryer, stage string, revision int6 out := make([]Attachment, 0) for rows.Next() { var a Attachment - var digest []byte - if err = rows.Scan(&a.SuppliedPath, &a.CanonicalPath, &a.RemoteFilename, &a.ByteLength, &a.MediaType, &digest); err != nil { + var digest, identity []byte + if err = rows.Scan(&a.SuppliedPath, &a.CanonicalPath, &a.RemoteFilename, &a.ByteLength, &a.MediaType, &digest, &identity); err != nil { return nil, localError(err) } - if len(digest) != 32 { + if len(digest) != 32 || len(identity) != 0 && len(identity) != 32 { return nil, localError(errors.New("digest")) } copy(a.ContentDigest[:], digest) + if len(identity) == 32 { + copy(a.FileIdentity[:], identity) + } out = append(out, a) } return out, localError(rows.Err()) diff --git a/internal/stagestore/domain_test.go b/internal/stagestore/domain_test.go index 41195ee..5a803e9 100644 --- a/internal/stagestore/domain_test.go +++ b/internal/stagestore/domain_test.go @@ -26,7 +26,20 @@ func openDomainStore(t *testing.T) *Store { return s } func attachment(name string) Attachment { - return Attachment{"/tmp/" + name, "/private/tmp/" + name, name, 3, "text/plain", sha256.Sum256([]byte(name))} + return Attachment{SuppliedPath: "/tmp/" + name, CanonicalPath: "/private/tmp/" + name, RemoteFilename: name, ByteLength: 3, MediaType: "text/plain", ContentDigest: sha256.Sum256([]byte(name)), FileIdentity: [32]byte{1}} +} + +func TestSemanticDigestBindsAttachmentFileIdentity(t *testing.T) { + in := createInput("identity-digest", "body") + first, err := ComputeSemanticDigest(in.Operation, in.ServerURL, in.ServerID, in.UserID, in.Content) + if err != nil { + t.Fatal(err) + } + in.Content.Attachments[0].FileIdentity[0]++ + second, err := ComputeSemanticDigest(in.Operation, in.ServerURL, in.ServerID, in.UserID, in.Content) + if err != nil || first == second { + t.Fatalf("first=%x second=%x err=%v", first, second, err) + } } func createInput(request, body string) CreateInput { return CreateInput{request, sha256.Sum256([]byte(body)), CreatePost, "https://mattermost.example/api/v4", "server-1", "user-1", RevisionContent{[]byte(body), json.RawMessage(`{"kind":"O","channelId":"channel-1"}`), json.RawMessage(`{"steps":[{"ordinal":1,"type":"upload_attachment","condition":"always"},{"ordinal":2,"type":"upload_attachment","condition":"always"},{"ordinal":3,"type":"create_post","condition":"always"}]}`), []Attachment{attachment("a.txt"), attachment("b.txt")}}} @@ -212,6 +225,49 @@ func TestCallerIntentMigrationsTombstoneLegacyCreateAndReviseReceipts(t *testing } } +func TestAttachmentIdentityMigrationRequiresOrdinaryRevisionRebind(t *testing.T) { + path := testPath(t) + original := migrations + migrations = append([]migration(nil), original[:8]...) + in := createInput("legacy-attachment-identity", "body") + for i := range in.Content.Attachments { + in.Content.Attachments[i].FileIdentity = [32]byte{} + } + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + created, err := s.Create(context.Background(), in) + if err != nil { + t.Fatal(err) + } + if err = s.Close(); err != nil { + t.Fatal(err) + } + migrations = original + t.Cleanup(func() { migrations = original }) + s, err = Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + detail, err := s.Show(context.Background(), created.Stage.ID) + if err != nil || len(detail.Attachments) != 2 || detail.Attachments[0].FileIdentity != ([32]byte{}) { + t.Fatalf("legacy detail=%+v err=%v", detail.StageSummary, err) + } + rebound := []Attachment{attachment("a.txt"), attachment("b.txt")} + revised, err := s.Revise(context.Background(), ReviseInput{StageID: created.Stage.ID, RequestID: "rebind-legacy-attachments", ExpectedRevision: created.Stage.Revision, + ExpectedDigest: created.Stage.SemanticDigest, RequestDigest: sha256.Sum256([]byte("rebind-legacy-attachments")), + Composition: Composition{Body: []byte("body"), Plan: in.Content.Plan, Attachments: rebound}}) + if err != nil || revised.Stage.Revision != 2 { + t.Fatalf("revised=%+v err=%v", revised, err) + } + detail, err = s.Show(context.Background(), created.Stage.ID) + if err != nil || detail.Attachments[0].FileIdentity == ([32]byte{}) { + t.Fatalf("rebound detail=%+v err=%v", detail.StageSummary, err) + } +} + func TestRevisionPlanMigrationAllowsPlanChanges(t *testing.T) { path := testPath(t) original := migrations @@ -629,9 +685,8 @@ func TestReviveOnlyLegalExpiredForbidden(t *testing.T) { t.Fatal(err) } normal := reviseInput(partial.Stage, "partial", "q") - got, err := s.Revise(context.Background(), normal) - if err != nil || got.Stage.Recovery != RecoveryPartial { - t.Fatalf("monotonic=%#v err=%v", got, err) + if _, err := s.Revise(context.Background(), normal); !errors.Is(err, ErrNotEligible) { + t.Fatalf("partial revision=%v", err) } } diff --git a/internal/stagestore/schema.go b/internal/stagestore/schema.go index 1526f49..80fe229 100644 --- a/internal/stagestore/schema.go +++ b/internal/stagestore/schema.go @@ -485,4 +485,22 @@ CREATE TRIGGER apply_requests_history_immutable_delete BEFORE DELETE ON apply_re WHEN NOT EXISTS(SELECT 1 FROM apply_attempts a WHERE a.id=OLD.attempt_id AND a.outcome IS NULL) OR EXISTS(SELECT 1 FROM apply_steps WHERE attempt_id=OLD.attempt_id AND state!='pending') BEGIN SELECT RAISE(ABORT, 'dispatched apply requests are immutable'); END; +`}, {version: 9, name: "attachment-identity-binding", sql: ` +CREATE TABLE stage_attachment_identities ( + stage_id TEXT NOT NULL, + revision INTEGER NOT NULL, + ordinal INTEGER NOT NULL, + file_identity BLOB NOT NULL CHECK (length(file_identity) = 32), + PRIMARY KEY (stage_id,revision,ordinal), + FOREIGN KEY (stage_id,revision,ordinal) REFERENCES stage_attachments(stage_id,revision,ordinal) ON DELETE CASCADE +) STRICT; +CREATE TRIGGER stage_attachment_identity_immutable_update BEFORE UPDATE ON stage_attachment_identities +BEGIN SELECT RAISE(ABORT, 'stage attachment identity is immutable'); END; +CREATE TRIGGER stage_attachment_identity_immutable_delete BEFORE DELETE ON stage_attachment_identities +WHEN EXISTS(SELECT 1 FROM stage_attachments a WHERE a.stage_id=OLD.stage_id AND a.revision=OLD.revision AND a.ordinal=OLD.ordinal) +BEGIN SELECT RAISE(ABORT, 'stage attachment identity is immutable'); END; `}} + +func attachmentIdentityAvailable() bool { + return len(migrations) >= 9 && migrations[8].version == 9 && migrations[8].name == "attachment-identity-binding" +} diff --git a/internal/stagestore/store.go b/internal/stagestore/store.go index f166fbe..cda0874 100644 --- a/internal/stagestore/store.go +++ b/internal/stagestore/store.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "net/url" + "path/filepath" "strconv" "strings" "sync" @@ -175,6 +176,9 @@ func (s *Store) Close() error { return s.closeErr } +// StateDir returns the already validated private directory containing the store. +func (s *Store) StateDir() string { return filepath.Dir(s.path) } + func sqliteURI(path string, readOnly bool) string { u := &url.URL{Scheme: "file", Path: path} q := u.Query() diff --git a/internal/staging/revision.go b/internal/staging/revision.go index 874c261..7067e95 100644 --- a/internal/staging/revision.go +++ b/internal/staging/revision.go @@ -65,6 +65,9 @@ func (r *Reviser) Revise(ctx context.Context, in ReviseInput) (RevisionResult, e if detail.ID != in.StageID { return RevisionResult{}, ErrStore } + if detail.Recovery == stagestore.RecoveryPartial { + return RevisionResult{}, ErrNotEligible + } if detail.Operation != stagestore.CreatePost && detail.Operation != stagestore.Reply && detail.Operation != stagestore.EditPost { return RevisionResult{}, ErrNotEligible } diff --git a/internal/staging/validation.go b/internal/staging/validation.go index 1205b45..250c47d 100644 --- a/internal/staging/validation.go +++ b/internal/staging/validation.go @@ -117,13 +117,13 @@ func validResolvedTeam(team mattermost.Team) bool { } func validBoundAttachments(values []stagestore.Attachment) bool { - if len(values) > 100 { + if len(values) > 5 { return false } for _, value := range values { if !validBoundText(value.SuppliedPath, 4096) || !validBoundText(value.CanonicalPath, 4096) || !validBoundText(value.RemoteFilename, 255) || (value.MediaType != "" && !validBoundText(value.MediaType, 255)) || - value.ByteLength < 0 || value.ContentDigest == ([32]byte{}) { + value.ByteLength <= 0 || value.ContentDigest == ([32]byte{}) { return false } } diff --git a/schemas/v2/examples/store-doctor.json b/schemas/v2/examples/store-doctor.json index b10bb3f..937002d 100644 --- a/schemas/v2/examples/store-doctor.json +++ b/schemas/v2/examples/store-doctor.json @@ -1 +1 @@ -{"schema":"mm/v2/store-doctor","path":"/home/arda/.local/state/mattermost-cli/stages.sqlite3","report":{"exists":false,"filesystemSafe":null,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":8,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}} +{"schema":"mm/v2/store-doctor","path":"/home/arda/.local/state/mattermost-cli/stages.sqlite3","report":{"exists":false,"filesystemSafe":null,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":9,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}} diff --git a/schemas/v2/examples/store-migrations.json b/schemas/v2/examples/store-migrations.json index 550b5b2..7749550 100644 --- a/schemas/v2/examples/store-migrations.json +++ b/schemas/v2/examples/store-migrations.json @@ -1 +1,15 @@ -{"schema":"mm/v2/store-migrations","latest":8,"migrations":[{"version":1,"name":"core-stage-state","checksum":"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"},{"version":2,"name":"immutable-local-request-receipts","checksum":"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"},{"version":3,"name":"caller-intent-stage-create-replay","checksum":"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5"},{"version":4,"name":"caller-intent-stage-revise-replay","checksum":"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4"},{"version":5,"name":"revision-plan-follows-composition","checksum":"fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0"},{"version":6,"name":"durable-apply-journal","checksum":"4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56"},{"version":7,"name":"status-confirmed-delete-results","checksum":"bf980f24ff24fdf9d6a3a28b9ded97b51336fa1c8ef290d1bdcf91f1b593c5ce"},{"version":8,"name":"already-satisfied-edit-apply","checksum":"cff30dd4fcc987876e092e2de1f5afd21ebeadbcb792fd550a56eb4c221edd10"}]} +{ + "schema": "mm/v2/store-migrations", + "latest": 9, + "migrations": [ + {"version": 1, "name": "core-stage-state", "checksum": "e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"}, + {"version": 2, "name": "immutable-local-request-receipts", "checksum": "ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"}, + {"version": 3, "name": "caller-intent-stage-create-replay", "checksum": "237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5"}, + {"version": 4, "name": "caller-intent-stage-revise-replay", "checksum": "c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4"}, + {"version": 5, "name": "revision-plan-follows-composition", "checksum": "fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0"}, + {"version": 6, "name": "durable-apply-journal", "checksum": "4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56"}, + {"version": 7, "name": "status-confirmed-delete-results", "checksum": "bf980f24ff24fdf9d6a3a28b9ded97b51336fa1c8ef290d1bdcf91f1b593c5ce"}, + {"version": 8, "name": "already-satisfied-edit-apply", "checksum": "cff30dd4fcc987876e092e2de1f5afd21ebeadbcb792fd550a56eb4c221edd10"}, + {"version": 9, "name": "attachment-identity-binding", "checksum": "d0782e01014d010e5978b39bf6f04ef26508bec9318c7ace3be8bfbbcab4999d"} + ] +} diff --git a/schemas/v2/stage-preview.schema.json b/schemas/v2/stage-preview.schema.json index 5ec0001..8b76132 100644 --- a/schemas/v2/stage-preview.schema.json +++ b/schemas/v2/stage-preview.schema.json @@ -674,7 +674,7 @@ }, "byteLength": { "type": "integer", - "minimum": 0, + "minimum": 1, "maximum": 9007199254740991 }, "mediaType": { @@ -731,7 +731,7 @@ "steps": { "type": "array", "minItems": 1, - "maxItems": 102, + "maxItems": 7, "items": { "$ref": "#/$defs/step" } @@ -774,7 +774,7 @@ "steps": { "type": "array", "minItems": 1, - "maxItems": 102, + "maxItems": 7, "items": { "allOf": [ { diff --git a/schemas/v2/stage-receipt.schema.json b/schemas/v2/stage-receipt.schema.json index bf44724..cb3fccc 100644 --- a/schemas/v2/stage-receipt.schema.json +++ b/schemas/v2/stage-receipt.schema.json @@ -526,7 +526,7 @@ }, "byteLength": { "type": "integer", - "minimum": 0, + "minimum": 1, "maximum": 9007199254740991 }, "mediaType": { @@ -583,7 +583,7 @@ "steps": { "type": "array", "minItems": 1, - "maxItems": 102, + "maxItems": 7, "items": { "$ref": "#/$defs/step" } @@ -626,7 +626,7 @@ "steps": { "type": "array", "minItems": 1, - "maxItems": 102, + "maxItems": 7, "items": { "allOf": [ { diff --git a/schemas/v2/stage-request.schema.json b/schemas/v2/stage-request.schema.json index c259b9c..c252a3d 100644 --- a/schemas/v2/stage-request.schema.json +++ b/schemas/v2/stage-request.schema.json @@ -71,7 +71,7 @@ }, "attachments": { "type": "array", - "maxItems": 100, + "maxItems": 5, "items": { "$ref": "#/$defs/attachment" } diff --git a/schemas/v2/stage-revise-request.schema.json b/schemas/v2/stage-revise-request.schema.json index 00a4020..adc489e 100644 --- a/schemas/v2/stage-revise-request.schema.json +++ b/schemas/v2/stage-revise-request.schema.json @@ -46,7 +46,7 @@ "anyOf": [ { "type": "array", - "maxItems": 100, + "maxItems": 5, "items": { "$ref": "#/$defs/attachment" } diff --git a/schemas/v2/stage.schema.json b/schemas/v2/stage.schema.json index dfb84d0..47bded7 100644 --- a/schemas/v2/stage.schema.json +++ b/schemas/v2/stage.schema.json @@ -37,7 +37,7 @@ }, "attachments": { "type": "array", - "maxItems": 100, + "maxItems": 5, "items": { "$ref": "#/$defs/storedAttachment" } @@ -847,7 +847,7 @@ }, "byteLength": { "type": "integer", - "minimum": 0, + "minimum": 1, "maximum": 9007199254740991 }, "mediaType": { @@ -904,7 +904,7 @@ "steps": { "type": "array", "minItems": 1, - "maxItems": 102, + "maxItems": 7, "items": { "$ref": "#/$defs/step" } @@ -947,7 +947,7 @@ "steps": { "type": "array", "minItems": 1, - "maxItems": 102, + "maxItems": 7, "items": { "allOf": [ { diff --git a/schemas/v2/stages.schema.json b/schemas/v2/stages.schema.json index bec3dde..aa21a1b 100644 --- a/schemas/v2/stages.schema.json +++ b/schemas/v2/stages.schema.json @@ -418,7 +418,7 @@ }, "byteLength": { "type": "integer", - "minimum": 0, + "minimum": 1, "maximum": 9007199254740991 }, "mediaType": { @@ -475,7 +475,7 @@ "steps": { "type": "array", "minItems": 1, - "maxItems": 102, + "maxItems": 7, "items": { "$ref": "#/$defs/step" } @@ -518,7 +518,7 @@ "steps": { "type": "array", "minItems": 1, - "maxItems": 102, + "maxItems": 7, "items": { "allOf": [ { diff --git a/schemas/v2/store-doctor.schema.json b/schemas/v2/store-doctor.schema.json index 73222ba..77bb39d 100644 --- a/schemas/v2/store-doctor.schema.json +++ b/schemas/v2/store-doctor.schema.json @@ -15,7 +15,7 @@ "exists": { "type": "boolean" }, "filesystemSafe": {}, "applicationId": {}, "integrity": {}, "integrityTruncated": {}, "foreignKeyIssues": {}, "foreignKeyRows": {}, "foreignKeyTruncated": {}, "journalMode": {}, "synchronous": {}, "secureDelete": {}, "foreignKeys": {}, "trustedSchema": {}, "queryOnly": {}, "walFallback": {}, - "migrations": { "type": "object", "additionalProperties": false, "required": ["applied", "latest", "valid"], "properties": { "applied": {}, "latest": { "const": 8 }, "valid": {} } }, + "migrations": { "type": "object", "additionalProperties": false, "required": ["applied", "latest", "valid"], "properties": { "applied": {}, "latest": { "const": 9 }, "valid": {} } }, "permissionModelLimitations": { "type": "array", "items": { "$ref": "#/$defs/safeString" } } }, "allOf": [ @@ -23,14 +23,14 @@ "filesystemSafe": { "const": null }, "applicationId": { "const": null }, "integrity": { "const": null }, "integrityTruncated": { "const": null }, "foreignKeyIssues": { "const": null }, "foreignKeyRows": { "const": null }, "foreignKeyTruncated": { "const": null }, "journalMode": { "const": null }, "synchronous": { "const": null }, "secureDelete": { "const": null }, "foreignKeys": { "const": null }, "trustedSchema": { "const": null }, "queryOnly": { "const": null }, "walFallback": { "const": null }, - "migrations": { "properties": { "applied": { "const": null }, "latest": { "const": 8 }, "valid": { "const": null } } } + "migrations": { "properties": { "applied": { "const": null }, "latest": { "const": 9 }, "valid": { "const": null } } } } } }, { "if": { "properties": { "exists": { "const": true } }, "required": ["exists"] }, "then": { "properties": { "filesystemSafe": { "const": true }, "applicationId": { "const": 1296913970 }, "integrity": { "$ref": "#/$defs/nonemptyBoundedStrings" }, "integrityTruncated": { "type": "boolean" }, "foreignKeyIssues": { "type": "integer", "minimum": 0, "maximum": 21 }, "foreignKeyRows": { "$ref": "#/$defs/boundedStrings" }, "foreignKeyTruncated": { "type": "boolean" }, "journalMode": { "enum": ["wal", "delete", "truncate", "persist"] }, "synchronous": { "type": "integer" }, "secureDelete": { "type": "integer" }, "foreignKeys": { "const": true }, "trustedSchema": { "const": false }, "queryOnly": { "const": true }, "walFallback": { "type": "boolean" }, - "migrations": { "properties": { "applied": { "const": 8 }, "latest": { "const": 8 }, "valid": { "const": true } } } + "migrations": { "properties": { "applied": { "const": 9 }, "latest": { "const": 9 }, "valid": { "const": true } } } }, "allOf": [ { "oneOf": [ { "properties": { "integrity": { "const": ["ok"] }, "integrityTruncated": { "const": false } } }, diff --git a/schemas/v2/store-migrations.schema.json b/schemas/v2/store-migrations.schema.json index 227e1a9..b423e62 100644 --- a/schemas/v2/store-migrations.schema.json +++ b/schemas/v2/store-migrations.schema.json @@ -6,9 +6,9 @@ "required": ["schema", "latest", "migrations"], "properties": { "schema": { "const": "mm/v2/store-migrations" }, - "latest": { "const": 8 }, + "latest": { "const": 9 }, "migrations": { - "type": "array", "minItems": 8, "maxItems": 8, + "type": "array", "minItems": 9, "maxItems": 9, "prefixItems": [{ "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { "version": { "const": 1 }, "name": { "const": "core-stage-state" }, "checksum": { "const": "e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f" } } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { @@ -25,6 +25,8 @@ "version": { "const": 7 }, "name": { "const": "status-confirmed-delete-results" }, "checksum": { "const": "bf980f24ff24fdf9d6a3a28b9ded97b51336fa1c8ef290d1bdcf91f1b593c5ce" } } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { "version": { "const": 8 }, "name": { "const": "already-satisfied-edit-apply" }, "checksum": { "const": "cff30dd4fcc987876e092e2de1f5afd21ebeadbcb792fd550a56eb4c221edd10" } + } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { + "version": { "const": 9 }, "name": { "const": "attachment-identity-binding" }, "checksum": { "const": "d0782e01014d010e5978b39bf6f04ef26508bec9318c7ace3be8bfbbcab4999d" } } }], "items": false } From 001c20eff05c61ff8cb50d00ca63ea9104799a9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 09:17:14 +0300 Subject: [PATCH 089/119] feat: resume partial attachment plans --- docs/V2_CONTRACT.md | 5 +- internal/apply/post.go | 31 +++- internal/apply/post_test.go | 162 +++++++++++++++++++++ internal/apply/service.go | 1 + internal/apply/service_test.go | 9 ++ internal/cli/store_test.go | 6 +- internal/mattermost/file_mutations.go | 72 ++++++--- internal/mattermost/file_mutations_test.go | 61 ++++++++ internal/schema/apply_semantic.go | 31 +++- internal/schema/apply_test.go | 58 ++++++++ internal/stagestore/apply.go | 160 +++++++++++++++----- internal/stagestore/apply_test.go | 75 +++++++++- internal/stagestore/schema.go | 46 ++++++ schemas/v2/apply-receipt.schema.json | 12 +- schemas/v2/examples/store-doctor.json | 2 +- schemas/v2/examples/store-migrations.json | 5 +- schemas/v2/store-doctor.schema.json | 6 +- schemas/v2/store-migrations.schema.json | 6 +- 18 files changed, 672 insertions(+), 76 deletions(-) diff --git a/docs/V2_CONTRACT.md b/docs/V2_CONTRACT.md index d06b0ef..0da7f5b 100644 --- a/docs/V2_CONTRACT.md +++ b/docs/V2_CONTRACT.md @@ -264,7 +264,7 @@ Development databases upgraded from a pre-identity schema cannot safely infer th To close the hash-to-upload race without retaining attachment copies between commands, apply copies every securely opened source into a private `0600` spool file while hashing it before the first remote dispatch. Only a complete spool whose identity, digest, and length match the staged revision may be uploaded. Each completed spool is unlinked while its descriptor remains open; upload reads that descriptor, not the original path. Thus path-backed staging remains low-storage at rest while dispatched bytes are an immutable snapshot of the reviewed file. -Spools are execution-only snapshots and never durable recovery artifacts. Process exit closes their descriptors and leaves no pathname to reconcile. A later explicit recovery securely reopens and respools the staged source; if the exact bound source is unavailable, recovery fails closed until the source is restored or recovery is explicitly abandoned. Validated upload steps recover from their journaled file IDs after fresh remote metadata revalidation, not by uploading the file again. +Spools are execution-only snapshots and never durable recovery artifacts. Process exit closes their descriptors and leaves no pathname to reconcile. A later explicit recovery securely reopens and respools every staged source before any new dispatch; if any exact bound source is unavailable or changed, recovery fails closed until the source is restored or recovery is explicitly abandoned. A contiguous prefix of directly validated upload steps from terminal attempts on the same stage revision and semantic digest may recover from its journaled file IDs after fresh remote metadata revalidation. Each ordinal binds directly to the attempt that dispatched and validated that upload; reused provenance cannot chain through another recovery attempt. Only the proven-not-applied suffix is uploaded, in original ordinal order, and every reused or newly uploaded file is revalidated again immediately before the final post dispatch. Attachments are non-empty and limited to five per post. Server per-file upload limits are checked before upload when discoverable; a dispatched `413` is a definitive rejection. Checked arithmetic caps an attempt's aggregate spool bytes at 512 MiB, and apply refuses to begin unless the private state filesystem can retain that snapshot while preserving a 64 MiB free-space reserve. Upload and post creation are separate remote substeps and therefore use the compound-operation journal. @@ -373,7 +373,7 @@ Revise is refused while `applying`, while recovery is `resume_partial`, or after Ordinary `mm apply @` requires the exact current revision, lifecycle `open`, and recovery requirement `none`. -`mm apply @ --resume-partial` requires recovery `resume_partial`. It preserves the prior attempt, assigns fresh idempotency/pending IDs to new substep attempts where applicable, revalidates reusable effects, and continues only effects proven not applied. A definitively rejected request is proven not applied and may be retried this way. Resume is forbidden when any attempt in the stage's history remains uncertain. +`mm apply @ --resume-partial` requires recovery `resume_partial`. It preserves the prior attempt, claims the exact same revision and semantic digest, assigns fresh idempotency/pending IDs to new substep attempts where applicable, and continues only effects proven not applied. Attachment recovery reuses only a contiguous direct validated-upload prefix after exact remote file metadata revalidation; no uncertain or already-reused source step qualifies. A definitively rejected request is proven not applied and may be retried this way. Resume is forbidden when any attempt in the stage's history remains uncertain. `mm apply @ --force-unknown` requires aggregate recovery `force_unknown`. It: @@ -411,6 +411,7 @@ Receipts include only fields needed to establish: - stage ID, revision, and attempt ID; - destination and authenticated identity in sanitized narrow form; - each planned substep's known state; +- direct reused-upload provenance as `reusedFrom.attemptId` and the identical source ordinal when partial recovery did not redispatch that upload; - canonical post/channel/file identifiers when validated; - server creation/update timestamps when validated; - overall `succeeded`, `rejected`, `partial`, `unknown`, or `already_satisfied` outcome; diff --git a/internal/apply/post.go b/internal/apply/post.go index 0a1b5cd..4a14fbe 100644 --- a/internal/apply/post.go +++ b/internal/apply/post.go @@ -112,7 +112,26 @@ func (s *Service) applyPostWithAttachments(ctx context.Context, attempt stagesto } fileIDs := make([]string, 0, len(spools)) - for i, spool := range spools { + for _, reusable := range attempt.ReusableUploads { + attachment := attachments[reusable.Ordinal-1] + if err := s.fileWrites.ValidateUpload(ctx, mattermost.UploadMutationInput{ChannelID: destination.ChannelID, UserID: currentUserID, + Filename: attachment.RemoteFilename, Length: attachment.ByteLength}, reusable.FileID); err != nil { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, errors.Join(ErrTargetDrift, err)) + } + } + for i, reusable := range attempt.ReusableUploads { + if err := s.store.MarkStepReused(context.WithoutCancel(ctx), attempt.ID, reusable.Ordinal, reusable.SourceAttemptID, reusable.SourceOrdinal, reusable.FileID); err != nil { + if i == 0 { + return stagestore.ApplyReceipt{}, s.abandon(ctx, attempt.ID, err) + } + return s.stopCompoundBeforeDispatch(ctx, attempt.ID, err) + } + _ = spools[reusable.Ordinal-1].Close() + spools[reusable.Ordinal-1] = nil + fileIDs = append(fileIDs, reusable.FileID) + } + for i := len(attempt.ReusableUploads); i < len(spools); i++ { + spool := spools[i] ordinal := i + 1 prepared, err := s.fileWrites.PrepareUpload(mattermost.UploadMutationInput{ ChannelID: destination.ChannelID, UserID: currentUserID, Filename: spool.RemoteFilename, MediaType: spool.MediaType, Length: spool.Length, Body: spool, @@ -152,6 +171,16 @@ func (s *Service) applyPostWithAttachments(ctx context.Context, attempt stagesto } return s.finalizeReceipt(context.WithoutCancel(ctx), attempt.ID) } + for i, fileID := range fileIDs { + attachment := attachments[i] + if err := s.fileWrites.ValidateUpload(ctx, mattermost.UploadMutationInput{ChannelID: destination.ChannelID, UserID: currentUserID, + Filename: attachment.RemoteFilename, Length: attachment.ByteLength}, fileID); err != nil { + if sealErr := s.store.SealRemainingNotDispatched(context.WithoutCancel(ctx), attempt.ID); sealErr != nil { + return stagestore.ApplyReceipt{}, &ConfirmedEffectError{sealErr} + } + return s.finalizeReceipt(context.WithoutCancel(ctx), attempt.ID) + } + } rootID := "" if destination.RootPostID != nil { rootID = *destination.RootPostID diff --git a/internal/apply/post_test.go b/internal/apply/post_test.go index d598aa1..4e63229 100644 --- a/internal/apply/post_test.go +++ b/internal/apply/post_test.go @@ -98,6 +98,9 @@ func TestApplyAttachmentPostPreservesShortAndLongMarkdownAndOrderedBytes(t *test } response.WriteHeader(http.StatusCreated) _, _ = fmt.Fprintf(response, `{"file_infos":[{"id":"file-%d","user_id":"self","channel_id":"channel-1","create_at":100,"update_at":100,"delete_at":0,"name":%q,"size":%d}]}`, index+1, filename, len(got)) + case "/api/v4/files/file-1/info", "/api/v4/files/file-2/info": + index := int(request.URL.Path[len("/api/v4/files/file-")] - '1') + _, _ = fmt.Fprintf(response, `{"id":"file-%d","user_id":"self","channel_id":"channel-1","create_at":100,"update_at":100,"delete_at":0,"name":"file-%d.bin","size":%d}`, index+1, index+1, len(files[index])) case "/api/v4/posts": posts.Add(1) var input struct { @@ -180,6 +183,161 @@ func TestApplyAttachmentRejectionAfterValidatedUploadIsPartial(t *testing.T) { } } +func TestApplyAttachmentPartialResumeReusesValidatedPrefixAndUploadsOnlySuffix(t *testing.T) { + var uploads, metadataReads, posts atomic.Int32 + var uploadedNames []string + server := attachmentTargetServer(t, func(response http.ResponseWriter, request *http.Request) { + switch { + case request.URL.Path == "/api/v4/files": + call := uploads.Add(1) + name := request.URL.Query().Get("filename") + uploadedNames = append(uploadedNames, name) + body, _ := io.ReadAll(request.Body) + if call == 2 { + response.WriteHeader(http.StatusForbidden) + return + } + fileID := "file-1" + if call == 3 { + fileID = "file-2" + } + response.WriteHeader(http.StatusCreated) + _, _ = fmt.Fprintf(response, `{"file_infos":[{"id":%q,"user_id":"self","channel_id":"channel-1","create_at":100,"update_at":100,"delete_at":0,"name":%q,"size":%d}]}`, fileID, name, len(body)) + case strings.HasPrefix(request.URL.Path, "/api/v4/files/file-"): + metadataReads.Add(1) + fileID := strings.TrimSuffix(strings.TrimPrefix(request.URL.Path, "/api/v4/files/"), "/info") + index := 0 + if fileID == "file-2" { + index = 1 + } + _, _ = fmt.Fprintf(response, `{"id":%q,"user_id":"self","channel_id":"channel-1","create_at":100,"update_at":100,"delete_at":0,"name":"file-%d.bin","size":%d}`, fileID, index+1, len([][]byte{[]byte("first"), []byte("second")}[index])) + case request.URL.Path == "/api/v4/posts": + posts.Add(1) + var input struct { + FileIDs []string `json:"file_ids"` + Message string `json:"message"` + PendingPostID string `json:"pending_post_id"` + } + if json.NewDecoder(request.Body).Decode(&input) != nil || !slices.Equal(input.FileIDs, []string{"file-1", "file-2"}) || input.Message != "body" { + response.WriteHeader(http.StatusBadRequest) + return + } + response.WriteHeader(http.StatusCreated) + _, _ = io.WriteString(response, mutationPostResponse("post-1", "channel-1", "self", "body", "", input.PendingPostID, input.FileIDs, 101, 101)) + } + }) + defer server.Close() + store := openApplyStore(t) + stage := createAttachmentApplyStage(t, store, server.URL+"/api/v4", "body", [][]byte{[]byte("first"), []byte("second")}) + service, client := applyAttachmentService(t, server.URL, store) + defer client.Close() + first, err := service.Apply(context.Background(), applyClaim(stage, "apply-partial-first")) + if err != nil || first.Outcome != stagestore.OutcomePartial { + t.Fatalf("first=%+v err=%v", first, err) + } + detail, err := store.Show(context.Background(), stage.Stage.ID) + if err != nil { + t.Fatal(err) + } + claim := applyClaim(stagestore.MutationResult{Stage: detail.StageSummary}, "apply-partial-resume") + claim.RecoveryMode = stagestore.RecoveryModePartial + resumed, err := service.Apply(context.Background(), claim) + if err != nil || resumed.Outcome != stagestore.OutcomeSucceeded || resumed.Recovery != stagestore.RecoveryForbidden || resumed.Steps[0].ReusedFrom == nil || resumed.Steps[0].ReusedFrom.AttemptID != first.AttemptID || resumed.Steps[1].ReusedFrom != nil || uploads.Load() != 3 || metadataReads.Load() != 3 || posts.Load() != 1 || !slices.Equal(uploadedNames, []string{"file-1.bin", "file-2.bin", "file-2.bin"}) { + t.Fatalf("resumed=%+v uploads=%d metadata=%d posts=%d names=%v err=%v", resumed, uploads.Load(), metadataReads.Load(), posts.Load(), uploadedNames, err) + } + replayed, err := service.Apply(context.Background(), claim) + if err != nil || !replayed.Replay || replayed.AttemptID != resumed.AttemptID || uploads.Load() != 3 || metadataReads.Load() != 3 || posts.Load() != 1 { + t.Fatalf("replayed=%+v uploads=%d metadata=%d posts=%d err=%v", replayed, uploads.Load(), metadataReads.Load(), posts.Load(), err) + } +} + +func TestApplyAttachmentPartialResumeRejectsRemoteMetadataDriftBeforeMutation(t *testing.T) { + var uploads, metadataReads, posts atomic.Int32 + server := attachmentTargetServer(t, func(response http.ResponseWriter, request *http.Request) { + switch { + case request.URL.Path == "/api/v4/files": + call := uploads.Add(1) + if call == 2 { + response.WriteHeader(http.StatusForbidden) + return + } + name := request.URL.Query().Get("filename") + body, _ := io.ReadAll(request.Body) + response.WriteHeader(http.StatusCreated) + _, _ = fmt.Fprintf(response, `{"file_infos":[{"id":"file-1","user_id":"self","channel_id":"channel-1","create_at":100,"update_at":100,"delete_at":0,"name":%q,"size":%d}]}`, name, len(body)) + case request.URL.Path == "/api/v4/files/file-1/info": + metadataReads.Add(1) + _, _ = io.WriteString(response, `{"id":"file-1","user_id":"other","channel_id":"channel-1","create_at":100,"update_at":100,"delete_at":0,"name":"file-1.bin","size":5}`) + case request.URL.Path == "/api/v4/posts": + posts.Add(1) + } + }) + defer server.Close() + store := openApplyStore(t) + stage := createAttachmentApplyStage(t, store, server.URL+"/api/v4", "body", [][]byte{[]byte("first"), []byte("second")}) + service, client := applyAttachmentService(t, server.URL, store) + defer client.Close() + first, err := service.Apply(context.Background(), applyClaim(stage, "apply-drift-first")) + if err != nil || first.Outcome != stagestore.OutcomePartial { + t.Fatalf("first=%+v err=%v", first, err) + } + detail, err := store.Show(context.Background(), stage.Stage.ID) + if err != nil { + t.Fatal(err) + } + claim := applyClaim(stagestore.MutationResult{Stage: detail.StageSummary}, "apply-drift-resume") + claim.RecoveryMode = stagestore.RecoveryModePartial + _, err = service.Apply(context.Background(), claim) + after, showErr := store.Show(context.Background(), stage.Stage.ID) + if !errors.Is(err, ErrTargetDrift) || showErr != nil || after.Lifecycle != stagestore.LifecycleOpen || after.Recovery != stagestore.RecoveryPartial || uploads.Load() != 2 || metadataReads.Load() != 1 || posts.Load() != 0 { + t.Fatalf("after=%+v uploads=%d metadata=%d posts=%d err=%v show=%v", after.StageSummary, uploads.Load(), metadataReads.Load(), posts.Load(), err, showErr) + } +} + +func TestApplyAttachmentPartialResumeSealsAfterReuseJournalFailure(t *testing.T) { + var uploads, metadataReads, posts atomic.Int32 + server := attachmentTargetServer(t, func(response http.ResponseWriter, request *http.Request) { + switch { + case request.URL.Path == "/api/v4/files": + index := uploads.Add(1) + name := request.URL.Query().Get("filename") + body, _ := io.ReadAll(request.Body) + response.WriteHeader(http.StatusCreated) + _, _ = fmt.Fprintf(response, `{"file_infos":[{"id":"file-%d","user_id":"self","channel_id":"channel-1","create_at":100,"update_at":100,"delete_at":0,"name":%q,"size":%d}]}`, index, name, len(body)) + case strings.HasPrefix(request.URL.Path, "/api/v4/files/file-"): + metadataReads.Add(1) + fileID := strings.TrimSuffix(strings.TrimPrefix(request.URL.Path, "/api/v4/files/"), "/info") + index := int(fileID[len(fileID)-1] - '1') + _, _ = fmt.Fprintf(response, `{"id":%q,"user_id":"self","channel_id":"channel-1","create_at":100,"update_at":100,"delete_at":0,"name":"file-%d.bin","size":%d}`, fileID, index+1, len([][]byte{[]byte("first"), []byte("second")}[index])) + case request.URL.Path == "/api/v4/posts": + posts.Add(1) + response.WriteHeader(http.StatusForbidden) + } + }) + defer server.Close() + store := openApplyStore(t) + stage := createAttachmentApplyStage(t, store, server.URL+"/api/v4", "body", [][]byte{[]byte("first"), []byte("second")}) + service, client := applyAttachmentService(t, server.URL, store) + first, err := service.Apply(context.Background(), applyClaim(stage, "apply-reuse-fault-first")) + client.Close() + if err != nil || first.Outcome != stagestore.OutcomePartial || uploads.Load() != 2 || posts.Load() != 1 { + t.Fatalf("first=%+v uploads=%d posts=%d err=%v", first, uploads.Load(), posts.Load(), err) + } + detail, err := store.Show(context.Background(), stage.Stage.ID) + if err != nil { + t.Fatal(err) + } + faults := &faultStore{Store: store, reuseOrdinal: 2, reuseErr: errors.New("journal unavailable")} + service, client = applyAttachmentServiceWithStore(t, server.URL, faults, store.StateDir()) + defer client.Close() + claim := applyClaim(stagestore.MutationResult{Stage: detail.StageSummary}, "apply-reuse-fault-resume") + claim.RecoveryMode = stagestore.RecoveryModePartial + receipt, err := service.Apply(context.Background(), claim) + if err != nil || receipt.Outcome != stagestore.OutcomePartial || receipt.Steps[0].ReusedFrom == nil || receipt.Steps[1].State != stagestore.StepNotSent || receipt.Steps[2].State != stagestore.StepNotSent || uploads.Load() != 2 || metadataReads.Load() != 4 || posts.Load() != 1 { + t.Fatalf("receipt=%+v uploads=%d metadata=%d posts=%d err=%v", receipt, uploads.Load(), metadataReads.Load(), posts.Load(), err) + } +} + func TestApplyAttachmentTargetDriftAfterUploadsLeavesPartialWithoutPost(t *testing.T) { var channelReads, uploads, posts atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { @@ -520,6 +678,10 @@ func attachmentTargetServer(t *testing.T, write func(http.ResponseWriter, *http. case "/api/v4/files", "/api/v4/posts": write(response, request) default: + if strings.HasPrefix(request.URL.Path, "/api/v4/files/") && strings.HasSuffix(request.URL.Path, "/info") { + write(response, request) + return + } http.NotFound(response, request) } })) diff --git a/internal/apply/service.go b/internal/apply/service.go index 14acb7d..9e85bab 100644 --- a/internal/apply/service.go +++ b/internal/apply/service.go @@ -46,6 +46,7 @@ type Store interface { MarkStepRejected(context.Context, string, int, json.RawMessage) error MarkStepUnknown(context.Context, string, int) error MarkStepSkipped(context.Context, string, int, json.RawMessage) error + MarkStepReused(context.Context, string, int, string, int, string) error SealRemainingNotDispatched(context.Context, string) error FinalizeApply(context.Context, string) (stagestore.ApplyReceipt, error) } diff --git a/internal/apply/service_test.go b/internal/apply/service_test.go index 1cf80d9..aa9b915 100644 --- a/internal/apply/service_test.go +++ b/internal/apply/service_test.go @@ -611,7 +611,9 @@ func applyServiceWithCredentials(t *testing.T, serverURL string, store Store, cr type faultStore struct { *stagestore.Store skipErr, unknownErr, validatedErr, finalizeErr, beginErr error + reuseErr error beginOrdinal int + reuseOrdinal int beforeSkip func() } @@ -646,6 +648,13 @@ func (s *faultStore) MarkStepValidated(ctx context.Context, attemptID string, or return s.Store.MarkStepValidated(ctx, attemptID, ordinal, result) } +func (s *faultStore) MarkStepReused(ctx context.Context, attemptID string, ordinal int, sourceAttemptID string, sourceOrdinal int, fileID string) error { + if s.reuseErr != nil && ordinal == s.reuseOrdinal { + return s.reuseErr + } + return s.Store.MarkStepReused(ctx, attemptID, ordinal, sourceAttemptID, sourceOrdinal, fileID) +} + func (s *faultStore) FinalizeApply(ctx context.Context, attemptID string) (stagestore.ApplyReceipt, error) { if s.finalizeErr != nil { return stagestore.ApplyReceipt{}, s.finalizeErr diff --git a/internal/cli/store_test.go b/internal/cli/store_test.go index a8ca6ed..28b25a2 100644 --- a/internal/cli/store_test.go +++ b/internal/cli/store_test.go @@ -36,7 +36,7 @@ func TestStoreDoctorAbsentIsReadOnlyAndSchemaValid(t *testing.T) { if err := registry.Validate("mm/v2/store-doctor", bytes.NewReader(stdout.Bytes())); err != nil { t.Fatalf("schema: %v\n%s", err, stdout.String()) } - for _, fact := range []string{`"filesystemSafe":null`, `"applicationId":null`, `"integrity":null`, `"applied":null`, `"latest":9`, `"valid":null`, `"journalMode":null`} { + for _, fact := range []string{`"filesystemSafe":null`, `"applicationId":null`, `"integrity":null`, `"applied":null`, `"latest":10`, `"valid":null`, `"journalMode":null`} { if !strings.Contains(stdout.String(), fact) { t.Fatalf("absent report omitted %s: %s", fact, stdout.String()) } @@ -110,7 +110,7 @@ func TestStoreMigrationsIsOfflineAndSchemaValid(t *testing.T) { if err := registry.Validate("mm/v2/store-migrations", bytes.NewReader(stdout.Bytes())); err != nil { t.Fatalf("schema: %v\n%s", err, stdout.String()) } - want := "{\"schema\":\"mm/v2/store-migrations\",\"latest\":9,\"migrations\":[{\"version\":1,\"name\":\"core-stage-state\",\"checksum\":\"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f\"},{\"version\":2,\"name\":\"immutable-local-request-receipts\",\"checksum\":\"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c\"},{\"version\":3,\"name\":\"caller-intent-stage-create-replay\",\"checksum\":\"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5\"},{\"version\":4,\"name\":\"caller-intent-stage-revise-replay\",\"checksum\":\"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4\"},{\"version\":5,\"name\":\"revision-plan-follows-composition\",\"checksum\":\"fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0\"},{\"version\":6,\"name\":\"durable-apply-journal\",\"checksum\":\"4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56\"},{\"version\":7,\"name\":\"status-confirmed-delete-results\",\"checksum\":\"bf980f24ff24fdf9d6a3a28b9ded97b51336fa1c8ef290d1bdcf91f1b593c5ce\"},{\"version\":8,\"name\":\"already-satisfied-edit-apply\",\"checksum\":\"cff30dd4fcc987876e092e2de1f5afd21ebeadbcb792fd550a56eb4c221edd10\"},{\"version\":9,\"name\":\"attachment-identity-binding\",\"checksum\":\"d0782e01014d010e5978b39bf6f04ef26508bec9318c7ace3be8bfbbcab4999d\"}]}\n" + want := "{\"schema\":\"mm/v2/store-migrations\",\"latest\":10,\"migrations\":[{\"version\":1,\"name\":\"core-stage-state\",\"checksum\":\"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f\"},{\"version\":2,\"name\":\"immutable-local-request-receipts\",\"checksum\":\"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c\"},{\"version\":3,\"name\":\"caller-intent-stage-create-replay\",\"checksum\":\"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5\"},{\"version\":4,\"name\":\"caller-intent-stage-revise-replay\",\"checksum\":\"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4\"},{\"version\":5,\"name\":\"revision-plan-follows-composition\",\"checksum\":\"fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0\"},{\"version\":6,\"name\":\"durable-apply-journal\",\"checksum\":\"4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56\"},{\"version\":7,\"name\":\"status-confirmed-delete-results\",\"checksum\":\"bf980f24ff24fdf9d6a3a28b9ded97b51336fa1c8ef290d1bdcf91f1b593c5ce\"},{\"version\":8,\"name\":\"already-satisfied-edit-apply\",\"checksum\":\"cff30dd4fcc987876e092e2de1f5afd21ebeadbcb792fd550a56eb4c221edd10\"},{\"version\":9,\"name\":\"attachment-identity-binding\",\"checksum\":\"d0782e01014d010e5978b39bf6f04ef26508bec9318c7ace3be8bfbbcab4999d\"},{\"version\":10,\"name\":\"validated-upload-reuse\",\"checksum\":\"75088d75d41eab9d7d1b1c7b1fa6fc4ce604849321b88feeb110d7ada3447947\"}]}\n" if stdout.String() != want { t.Fatalf("stdout does not match golden migration contract: %q", stdout.String()) } @@ -310,7 +310,7 @@ func TestStoreHumanOutputStatesBounds(t *testing.T) { t.Fatalf("stdout=%q", stdout.String()) } stdout.Reset() - if code := Execute(context.Background(), []string{"store", "migrations"}, strings.NewReader(""), &stdout, &stderr); code != 0 || !strings.Contains(stdout.String(), "latest: 9\n1 core-stage-state ") { + if code := Execute(context.Background(), []string{"store", "migrations"}, strings.NewReader(""), &stdout, &stderr); code != 0 || !strings.Contains(stdout.String(), "latest: 10\n1 core-stage-state ") { t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } } diff --git a/internal/mattermost/file_mutations.go b/internal/mattermost/file_mutations.go index 729489f..c1a7cf5 100644 --- a/internal/mattermost/file_mutations.go +++ b/internal/mattermost/file_mutations.go @@ -3,6 +3,7 @@ package mattermost import ( "context" "encoding/json" + "errors" "io" "net/http" "net/url" @@ -12,6 +13,8 @@ import ( "github.com/ardasevinc/mattermost-cli/internal/api" ) +var ErrUploadBinding = errors.New("Mattermost file no longer matches the validated upload") + type FileMutations struct{ client *api.Client } func NewFileMutations(client *api.Client) *FileMutations { return &FileMutations{client: client} } @@ -38,6 +41,22 @@ func (m *FileMutations) DiscoverMaxUploadBytes(ctx context.Context) (int64, bool return value, true } +func (m *FileMutations) ValidateUpload(ctx context.Context, in UploadMutationInput, fileID string) error { + if m == nil || m.client == nil || ctx == nil || !isSafePostID(fileID) || !isSafePostID(in.ChannelID) || !isSafePostID(in.UserID) || !validUploadFilename(in.Filename) || in.Length <= 0 { + return ErrInvalidMutationRequest + } + var response fileInfoResponse + if err := m.client.Get(ctx, "/files/"+url.PathEscape(fileID)+"/info", &response); err != nil { + return err + } + file := uploadFileInfo(response) + if file.ID != fileID || file.UserID != in.UserID || file.ChannelID != in.ChannelID || file.PostID != "" || file.Name != in.Filename || file.Size != in.Length || + file.CreateAt <= 0 || file.CreateAt > maxDateMilliseconds || file.UpdateAt < file.CreateAt || file.UpdateAt > maxDateMilliseconds || file.DeleteAt != 0 { + return ErrUploadBinding + } + return nil +} + type UploadMutationInput struct { ChannelID, UserID, Filename, MediaType string Length int64 @@ -121,56 +140,75 @@ func (r *uploadResponse) UnmarshalJSON(data []byte) error { if json.Unmarshal(envelope.FileInfos, &infos) != nil || len(infos) != 1 { return ErrInvalidPostResponse } - infoRaw, ok := uniqueJSONObject(infos[0]) - if !ok { + file, err := decodeUploadFileInfo(infos[0]) + if err != nil { + return err + } + clientIDs := []string(nil) + if envelope.ClientIDs != nil && string(envelope.ClientIDs) != "null" && json.Unmarshal(envelope.ClientIDs, &clientIDs) != nil { return ErrInvalidPostResponse } + *r = uploadResponse{[]uploadFileInfo{file}, clientIDs} + return nil +} + +type fileInfoResponse uploadFileInfo + +func (r *fileInfoResponse) UnmarshalJSON(data []byte) error { + file, err := decodeUploadFileInfo(data) + if err != nil { + return err + } + *r = fileInfoResponse(file) + return nil +} + +func decodeUploadFileInfo(data []byte) (uploadFileInfo, error) { + infoRaw, ok := uniqueJSONObject(data) + if !ok { + return uploadFileInfo{}, ErrInvalidPostResponse + } file := uploadFileInfo{} var fieldsOK bool file.ID, fieldsOK = safePostID(infoRaw["id"]) if !fieldsOK { - return ErrInvalidPostResponse + return uploadFileInfo{}, ErrInvalidPostResponse } file.UserID, fieldsOK = safePostID(infoRaw["user_id"]) if !fieldsOK { - return ErrInvalidPostResponse + return uploadFileInfo{}, ErrInvalidPostResponse } file.ChannelID, fieldsOK = safePostID(infoRaw["channel_id"]) if !fieldsOK { - return ErrInvalidPostResponse + return uploadFileInfo{}, ErrInvalidPostResponse } if rawPostID, present := infoRaw["post_id"]; present { file.PostID, fieldsOK = strictString(rawPostID) if !fieldsOK { - return ErrInvalidPostResponse + return uploadFileInfo{}, ErrInvalidPostResponse } } file.Name, fieldsOK = strictString(infoRaw["name"]) if !fieldsOK { - return ErrInvalidPostResponse + return uploadFileInfo{}, ErrInvalidPostResponse } file.CreateAt, fieldsOK = nonnegativeInteger(infoRaw["create_at"]) if !fieldsOK { - return ErrInvalidPostResponse + return uploadFileInfo{}, ErrInvalidPostResponse } file.UpdateAt, fieldsOK = nonnegativeInteger(infoRaw["update_at"]) if !fieldsOK { - return ErrInvalidPostResponse + return uploadFileInfo{}, ErrInvalidPostResponse } file.DeleteAt, fieldsOK = nonnegativeInteger(infoRaw["delete_at"]) if !fieldsOK { - return ErrInvalidPostResponse + return uploadFileInfo{}, ErrInvalidPostResponse } file.Size, fieldsOK = nonnegativeInteger(infoRaw["size"]) if !fieldsOK { - return ErrInvalidPostResponse + return uploadFileInfo{}, ErrInvalidPostResponse } - clientIDs := []string(nil) - if envelope.ClientIDs != nil && string(envelope.ClientIDs) != "null" && json.Unmarshal(envelope.ClientIDs, &clientIDs) != nil { - return ErrInvalidPostResponse - } - *r = uploadResponse{[]uploadFileInfo{file}, clientIDs} - return nil + return file, nil } func validUploadFilename(value string) bool { diff --git a/internal/mattermost/file_mutations_test.go b/internal/mattermost/file_mutations_test.go index cf13d7b..3a95275 100644 --- a/internal/mattermost/file_mutations_test.go +++ b/internal/mattermost/file_mutations_test.go @@ -85,6 +85,67 @@ func TestPreparedUploadCloseDoesNotDispatch(t *testing.T) { } } +func TestValidateUploadReadsExactRemoteBindingWithoutMutation(t *testing.T) { + var calls atomic.Int32 + client := mutationClient(t, mutationRoundTripFunc(func(request *http.Request) (*http.Response, error) { + calls.Add(1) + if request.Method != http.MethodGet || request.URL.RequestURI() != "/api/v4/files/file-1/info" || request.ContentLength != 0 { + t.Fatalf("method=%s uri=%s body=%v", request.Method, request.URL.RequestURI(), request.Body) + } + return mutationResponse(http.StatusOK, `{"id":"file-1","user_id":"user-1","channel_id":"channel-1","post_id":"","create_at":100,"update_at":100,"delete_at":0,"name":"a.txt","size":1}`), nil + })) + err := NewFileMutations(client).ValidateUpload(context.Background(), UploadMutationInput{ChannelID: "channel-1", UserID: "user-1", Filename: "a.txt", Length: 1}, "file-1") + if err != nil || calls.Load() != 1 { + t.Fatalf("calls=%d err=%v", calls.Load(), err) + } +} + +func TestValidateUploadRejectsChangedOrUnparseableRemoteBinding(t *testing.T) { + valid := `{"id":"file-1","user_id":"user-1","channel_id":"channel-1","post_id":"","create_at":100,"update_at":100,"delete_at":0,"name":"a.txt","size":1}` + for name, response := range map[string]string{ + "wrong id": strings.Replace(valid, `"file-1"`, `"file-2"`, 1), + "wrong user": strings.Replace(valid, `"user-1"`, `"user-2"`, 1), + "wrong channel": strings.Replace(valid, `"channel-1"`, `"channel-2"`, 1), + "attached": strings.Replace(valid, `"post_id":""`, `"post_id":"post-1"`, 1), + "wrong name": strings.Replace(valid, `"a.txt"`, `"b.txt"`, 1), + "wrong size": strings.Replace(valid, `"size":1`, `"size":2`, 1), + "deleted": strings.Replace(valid, `"delete_at":0`, `"delete_at":101`, 1), + "stale update": strings.Replace(valid, `"update_at":100`, `"update_at":99`, 1), + } { + t.Run(name, func(t *testing.T) { + client := mutationClient(t, mutationRoundTripFunc(func(request *http.Request) (*http.Response, error) { + if request.Method != http.MethodGet { + t.Fatalf("method=%s", request.Method) + } + return mutationResponse(http.StatusOK, response), nil + })) + err := NewFileMutations(client).ValidateUpload(context.Background(), UploadMutationInput{ChannelID: "channel-1", UserID: "user-1", Filename: "a.txt", Length: 1}, "file-1") + if !errors.Is(err, ErrUploadBinding) { + t.Fatalf("error=%v", err) + } + }) + } + for name, response := range map[string]string{ + "malformed": `{`, + "duplicate": strings.Replace(valid, `"id":"file-1"`, `"id":"file-1","id":"file-1"`, 1), + } { + t.Run(name, func(t *testing.T) { + client := mutationClient(t, mutationRoundTripFunc(func(*http.Request) (*http.Response, error) { + return mutationResponse(http.StatusOK, response), nil + })) + if err := NewFileMutations(client).ValidateUpload(context.Background(), UploadMutationInput{ChannelID: "channel-1", UserID: "user-1", Filename: "a.txt", Length: 1}, "file-1"); !errors.Is(err, api.ErrInvalidJSON) { + t.Fatalf("error=%v", err) + } + }) + } + client := mutationClient(t, mutationRoundTripFunc(func(*http.Request) (*http.Response, error) { + return mutationResponse(http.StatusNotFound, `{}`), nil + })) + if err := NewFileMutations(client).ValidateUpload(context.Background(), UploadMutationInput{ChannelID: "channel-1", UserID: "user-1", Filename: "a.txt", Length: 1}, "file-1"); err == nil || errors.Is(err, ErrUploadBinding) { + t.Fatalf("404 error=%v", err) + } +} + func TestDiscoverMaxUploadBytesUsesOnlyValidPublicHint(t *testing.T) { for name, response := range map[string]string{ "valid": `{"MaxFileSize":"12345"}`, diff --git a/internal/schema/apply_semantic.go b/internal/schema/apply_semantic.go index b351782..cd53a11 100644 --- a/internal/schema/apply_semantic.go +++ b/internal/schema/apply_semantic.go @@ -10,6 +10,7 @@ import ( ) type applyReceiptDocument struct { + AttemptID string `json:"attemptId"` Operation string `json:"operation"` RecoveryMode string `json:"recoveryMode"` Destination applyReceiptDestination `json:"destination"` @@ -34,13 +35,19 @@ type applyReceiptDestination struct { } type applyReceiptStep struct { - Ordinal int `json:"ordinal"` - Kind string `json:"kind"` - Condition string `json:"condition"` - State string `json:"state"` - Result json.RawMessage `json:"result"` - StartedAt *time.Time `json:"startedAt"` - EndedAt *time.Time `json:"endedAt"` + Ordinal int `json:"ordinal"` + Kind string `json:"kind"` + Condition string `json:"condition"` + State string `json:"state"` + Result json.RawMessage `json:"result"` + StartedAt *time.Time `json:"startedAt"` + EndedAt *time.Time `json:"endedAt"` + ReusedFrom *applyReceiptStepSource `json:"reusedFrom"` +} + +type applyReceiptStepSource struct { + AttemptID string `json:"attemptId"` + Ordinal int `json:"ordinal"` } func validateSemanticDocument(id string, data []byte) error { @@ -63,7 +70,7 @@ func validateApplyReceiptDocument(receipt applyReceiptDocument) error { if step.Ordinal != i+1 || step.StartedAt != nil && step.StartedAt.Before(receipt.StartedAt) || step.EndedAt != nil && (step.EndedAt.Before(receipt.StartedAt) || step.EndedAt.After(receipt.RecordedAt)) || step.StartedAt != nil && step.EndedAt != nil && step.EndedAt.Before(*step.StartedAt) || - !validApplyReceiptResultTarget(step, receipt.Destination) { + !validApplyReceiptResultTarget(step, receipt.Destination) || !validApplyReceiptReuse(receipt, step) { return fmt.Errorf("invalid apply receipt step %d", i+1) } } @@ -74,6 +81,14 @@ func validateApplyReceiptDocument(receipt applyReceiptDocument) error { return nil } +func validApplyReceiptReuse(receipt applyReceiptDocument, step applyReceiptStep) bool { + if step.ReusedFrom == nil { + return true + } + return receipt.RecoveryMode == "resume_partial" && step.State == "response_validated" && step.Kind == "upload_attachment" && + step.ReusedFrom.AttemptID != receipt.AttemptID && step.ReusedFrom.Ordinal == step.Ordinal +} + func validApplyReceiptStepOrder(steps []applyReceiptStep) bool { stopped := false for _, step := range steps { diff --git a/internal/schema/apply_test.go b/internal/schema/apply_test.go index b8e9c3d..6f609c5 100644 --- a/internal/schema/apply_test.go +++ b/internal/schema/apply_test.go @@ -273,6 +273,64 @@ func TestApplyReceiptAcceptsStatusConfirmedDeleteProjection(t *testing.T) { } } +func TestApplyReceiptBindsReusedUploadProvenance(t *testing.T) { + registry, err := Load() + if err != nil { + t.Fatal(err) + } + raw, err := fs.ReadFile(publicschemas.FS, "v2/examples/apply-receipt.json") + if err != nil { + t.Fatal(err) + } + var base map[string]any + if err := json.Unmarshal(raw, &base); err != nil { + t.Fatal(err) + } + base["recoveryMode"] = "resume_partial" + base["steps"] = []any{ + map[string]any{"ordinal": float64(1), "kind": "upload_attachment", "condition": "always", "state": "response_validated", "result": map[string]any{"fileId": "file-1"}, "startedAt": "2026-07-17T02:00:00Z", "endedAt": "2026-07-17T02:00:00Z", "reusedFrom": map[string]any{"attemptId": "att_11111111111111111111111111111111", "ordinal": float64(1)}}, + base["steps"].([]any)[0], + } + base["steps"].([]any)[1].(map[string]any)["ordinal"] = float64(2) + valid, err := json.Marshal(base) + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/apply-receipt", bytes.NewReader(valid)); err != nil { + t.Fatalf("rejected reused upload receipt: %v\n%s", err, valid) + } + + for name, mutate := range map[string]func(map[string]any){ + "ordinary mode": func(doc map[string]any) { doc["recoveryMode"] = "ordinary" }, + "same attempt": func(doc map[string]any) { + doc["steps"].([]any)[0].(map[string]any)["reusedFrom"].(map[string]any)["attemptId"] = doc["attemptId"] + }, + "wrong ordinal": func(doc map[string]any) { + doc["steps"].([]any)[0].(map[string]any)["reusedFrom"].(map[string]any)["ordinal"] = float64(2) + }, + "create step": func(doc map[string]any) { + source := doc["steps"].([]any)[0].(map[string]any)["reusedFrom"] + delete(doc["steps"].([]any)[0].(map[string]any), "reusedFrom") + doc["steps"].([]any)[1].(map[string]any)["reusedFrom"] = source + }, + } { + t.Run(name, func(t *testing.T) { + var doc map[string]any + if err := json.Unmarshal(valid, &doc); err != nil { + t.Fatal(err) + } + mutate(doc) + encoded, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/apply-receipt", bytes.NewReader(encoded)); err == nil { + t.Fatalf("accepted invalid reuse provenance: %s", encoded) + } + }) + } +} + func makeEditReceipt(doc map[string]any, resultPostID string) { doc["operation"] = "edit_post" destination := doc["destination"].(map[string]any) diff --git a/internal/stagestore/apply.go b/internal/stagestore/apply.go index 63996c7..f498cf2 100644 --- a/internal/stagestore/apply.go +++ b/internal/stagestore/apply.go @@ -10,6 +10,7 @@ import ( "errors" "io" "slices" + "strconv" "time" ) @@ -46,30 +47,42 @@ type ApplyClaimInput struct { } type ApplyStep struct { - Ordinal int `json:"ordinal"` - Kind string `json:"kind"` - Condition string `json:"condition"` - State StepState `json:"state"` - Result json.RawMessage `json:"result"` - StartedAt *time.Time `json:"startedAt"` - EndedAt *time.Time `json:"endedAt"` + Ordinal int `json:"ordinal"` + Kind string `json:"kind"` + Condition string `json:"condition"` + State StepState `json:"state"` + Result json.RawMessage `json:"result"` + StartedAt *time.Time `json:"startedAt"` + EndedAt *time.Time `json:"endedAt"` + ReusedFrom *StepSource `json:"reusedFrom,omitempty"` +} + +type StepSource struct { + AttemptID string `json:"attemptId"` + Ordinal int `json:"ordinal"` +} + +type ReusableUpload struct { + Ordinal, SourceOrdinal int + SourceAttemptID, FileID string } type ApplyAttempt struct { - ID string `json:"id"` - StageID string `json:"stageId"` - Revision int64 `json:"revision"` - SemanticDigest [32]byte `json:"semanticDigest"` - RecoveryMode RecoveryMode `json:"recoveryMode"` - PriorRecovery Recovery `json:"priorRecovery"` - ForcedDuplicateRisk bool `json:"forcedDuplicateRisk"` - Plan json.RawMessage `json:"plan"` - PendingPostID string `json:"pendingPostId"` - StartedAt time.Time `json:"startedAt"` - EndedAt *time.Time `json:"endedAt"` - Outcome *AttemptOutcome `json:"outcome"` - Steps []ApplyStep `json:"steps"` - Replay bool `json:"-"` + ID string `json:"id"` + StageID string `json:"stageId"` + Revision int64 `json:"revision"` + SemanticDigest [32]byte `json:"semanticDigest"` + RecoveryMode RecoveryMode `json:"recoveryMode"` + PriorRecovery Recovery `json:"priorRecovery"` + ForcedDuplicateRisk bool `json:"forcedDuplicateRisk"` + Plan json.RawMessage `json:"plan"` + PendingPostID string `json:"pendingPostId"` + StartedAt time.Time `json:"startedAt"` + EndedAt *time.Time `json:"endedAt"` + Outcome *AttemptOutcome `json:"outcome"` + Steps []ApplyStep `json:"steps"` + Replay bool `json:"-"` + ReusableUploads []ReusableUpload `json:"-"` } type ApplyReceipt struct { @@ -133,12 +146,6 @@ func (s *Store) ClaimApply(ctx context.Context, in ApplyClaimInput) (ApplyAttemp if base.Lifecycle != LifecycleOpen || !modeMatchesRecovery(in.RecoveryMode, base.Recovery) { return ApplyAttempt{}, ErrNotEligible } - // Partial resume needs step input digests plus explicit remote revalidation. - // Refuse it until that proof is part of the claim instead of redispatching a - // previously confirmed effect from ordinal coincidence alone. - if in.RecoveryMode == RecoveryModePartial { - return ApplyAttempt{}, ErrNotEligible - } plan, steps, err := decodePersistedPlan(base.Plan) if err != nil { return ApplyAttempt{}, localError(err) @@ -150,6 +157,19 @@ func (s *Store) ClaimApply(ctx context.Context, in ApplyClaimInput) (ApplyAttemp if !validPlanForOperation(base.Operation, steps, len(attachments)) { return ApplyAttempt{}, localError(errors.New("stored apply plan")) } + var reusable []ReusableUpload + if in.RecoveryMode == RecoveryModePartial { + if !validatedUploadReuseAvailable() { + return ApplyAttempt{}, ErrNotEligible + } + reusable, err = findReusableUploads(ctx, tx, base.ID, base.Revision, base.SemanticDigest, len(attachments)) + if err != nil { + return ApplyAttempt{}, err + } + if len(reusable) == 0 { + return ApplyAttempt{}, ErrNotEligible + } + } attemptID, err := newIdentity("att_") if err != nil { return ApplyAttempt{}, errors.New("stage store: random identity unavailable") @@ -190,7 +210,32 @@ func (s *Store) ClaimApply(ctx context.Context, in ApplyClaimInput) (ApplyAttemp } runCommitHook() return ApplyAttempt{ID: attemptID, StageID: base.ID, Revision: base.Revision, SemanticDigest: base.SemanticDigest, RecoveryMode: in.RecoveryMode, PriorRecovery: base.Recovery, - ForcedDuplicateRisk: forced, Plan: plan, PendingPostID: pendingID, StartedAt: now, Steps: steps}, nil + ForcedDuplicateRisk: forced, Plan: plan, PendingPostID: pendingID, StartedAt: now, Steps: steps, ReusableUploads: reusable}, nil +} + +func findReusableUploads(ctx context.Context, q queryer, stageID string, revision int64, digest [32]byte, attachmentCount int) ([]ReusableUpload, error) { + out := make([]ReusableUpload, 0, attachmentCount) + for ordinal := 1; ordinal <= attachmentCount; ordinal++ { + var source, fileID string + err := q.QueryRowContext(ctx, `SELECT p.attempt_id,json_extract(p.result_json,'$.fileId') +FROM apply_steps p JOIN apply_attempts a ON a.id=p.attempt_id +WHERE a.stage_id=? AND a.revision=? AND a.semantic_digest=? AND a.outcome IS NOT NULL + AND p.ordinal=? AND p.kind='upload_attachment' AND p.state='response_validated' + AND NOT EXISTS(SELECT 1 FROM apply_step_reuse r WHERE r.attempt_id=p.attempt_id AND r.ordinal=p.ordinal) + AND NOT EXISTS(SELECT 1 FROM apply_steps u WHERE u.attempt_id=a.id AND u.state='outcome_unknown') +ORDER BY a.started_at DESC,a.id DESC LIMIT 1`, stageID, revision, digest[:], ordinal).Scan(&source, &fileID) + if errors.Is(err, sql.ErrNoRows) { + break + } + if err != nil { + return nil, localError(err) + } + if !bounded(source, maxIdentityBytes) || !bounded(fileID, maxIdentityBytes) { + return nil, localError(errors.New("reusable upload binding")) + } + out = append(out, ReusableUpload{Ordinal: ordinal, SourceOrdinal: ordinal, SourceAttemptID: source, FileID: fileID}) + } + return out, nil } func (s *Store) BeginDispatch(ctx context.Context, attemptID string, ordinal int) error { @@ -213,6 +258,37 @@ func (s *Store) MarkStepSkipped(ctx context.Context, attemptID string, ordinal i return s.transitionStep(ctx, attemptID, ordinal, StepPending, StepSkipped, result) } +func (s *Store) MarkStepReused(ctx context.Context, attemptID string, ordinal int, sourceAttemptID string, sourceOrdinal int, fileID string) error { + if ctx == nil || !bounded(attemptID, maxIdentityBytes) || !bounded(sourceAttemptID, maxIdentityBytes) || !bounded(fileID, maxIdentityBytes) || ordinal < 1 || sourceOrdinal < 1 { + return ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return localError(err) + } + defer tx.Rollback() + stamp := formatTime(time.Now().UTC()) + if _, err = tx.ExecContext(ctx, `INSERT INTO apply_step_reuse(attempt_id,ordinal,source_attempt_id,source_ordinal,file_id) VALUES(?,?,?,?,?)`, attemptID, ordinal, sourceAttemptID, sourceOrdinal, fileID); err != nil { + return localError(err) + } + result := json.RawMessage(`{"fileId":` + strconv.Quote(fileID) + `}`) + updated, err := tx.ExecContext(ctx, `UPDATE apply_steps SET state='response_validated',result_json=?,started_at=?,ended_at=? WHERE attempt_id=? AND ordinal=? AND state='pending'`, string(result), stamp, stamp, attemptID, ordinal) + if err != nil { + return localError(err) + } + if !oneRow(updated) { + return ErrConflict + } + if _, err = tx.ExecContext(ctx, `INSERT INTO apply_events(attempt_id,ordinal,event,recorded_at) VALUES(?,?,'response_validated',?)`, attemptID, ordinal, stamp); err != nil { + return localError(err) + } + if err = tx.Commit(); err != nil { + return localError(err) + } + runCommitHook() + return nil +} + // SealRemainingNotDispatched closes a compound attempt after confirmed early // effects when a fresh local precondition prevents the next dispatch. func (s *Store) SealRemainingNotDispatched(ctx context.Context, attemptID string) error { @@ -518,16 +594,27 @@ func scanApplyAttempt(ctx context.Context, q queryer, attemptID string) (ApplyAt } a.Outcome = &value } - rows, err := q.QueryContext(ctx, `SELECT ordinal,kind,condition,state,result_json,started_at,ended_at FROM apply_steps WHERE attempt_id=? ORDER BY ordinal`, attemptID) + stepQuery := `SELECT ordinal,kind,condition,state,result_json,started_at,ended_at,NULL,NULL FROM apply_steps WHERE attempt_id=? ORDER BY ordinal` + if validatedUploadReuseAvailable() { + stepQuery = `SELECT p.ordinal,p.kind,p.condition,p.state,p.result_json,p.started_at,p.ended_at,r.source_attempt_id,r.source_ordinal FROM apply_steps p LEFT JOIN apply_step_reuse r ON r.attempt_id=p.attempt_id AND r.ordinal=p.ordinal WHERE p.attempt_id=? ORDER BY p.ordinal` + } + rows, err := q.QueryContext(ctx, stepQuery, attemptID) if err != nil { return ApplyAttempt{}, localError(err) } for rows.Next() { var step ApplyStep - var result, stepStarted, stepEnded sql.NullString - if err = rows.Scan(&step.Ordinal, &step.Kind, &step.Condition, &step.State, &result, &stepStarted, &stepEnded); err != nil { + var result, stepStarted, stepEnded, sourceAttempt sql.NullString + var sourceOrdinal sql.NullInt64 + if err = rows.Scan(&step.Ordinal, &step.Kind, &step.Condition, &step.State, &result, &stepStarted, &stepEnded, &sourceAttempt, &sourceOrdinal); err != nil { return ApplyAttempt{}, localError(err) } + if sourceAttempt.Valid != sourceOrdinal.Valid || sourceOrdinal.Valid && (sourceOrdinal.Int64 < 1 || sourceOrdinal.Int64 > 102) { + return ApplyAttempt{}, localError(errors.New("apply reuse projection")) + } + if sourceAttempt.Valid { + step.ReusedFrom = &StepSource{AttemptID: sourceAttempt.String, Ordinal: int(sourceOrdinal.Int64)} + } if result.Valid { step.Result = json.RawMessage(result.String) } @@ -788,7 +875,7 @@ func validAttempt(a ApplyAttempt) bool { return false } for i, step := range a.Steps { - if step.Ordinal != i+1 || step.Kind != planned[i].Kind || step.Condition != planned[i].Condition || !validApplyStep(step) { + if step.Ordinal != i+1 || step.Kind != planned[i].Kind || step.Condition != planned[i].Condition || !validApplyStep(step) || step.ReusedFrom != nil && a.RecoveryMode != RecoveryModePartial { return false } } @@ -799,6 +886,9 @@ func validApplyStep(step ApplyStep) bool { if !validStepKind(step.Kind) || step.Condition != "always" && step.Condition != "if_missing" || !validStepState(step.State) { return false } + if step.ReusedFrom != nil && (step.State != StepValidated || step.Kind != "upload_attachment" || !bounded(step.ReusedFrom.AttemptID, maxIdentityBytes) || step.ReusedFrom.Ordinal != step.Ordinal) { + return false + } switch step.State { case StepPending: return step.StartedAt == nil && step.EndedAt == nil && step.Result == nil @@ -831,13 +921,17 @@ func validReceiptForAttempt(receipt ApplyReceipt, attempt ApplyAttempt) bool { } for i := range receipt.Steps { if receipt.Steps[i].Ordinal != attempt.Steps[i].Ordinal || receipt.Steps[i].Kind != attempt.Steps[i].Kind || receipt.Steps[i].Condition != attempt.Steps[i].Condition || receipt.Steps[i].State != attempt.Steps[i].State || - !bytes.Equal(receipt.Steps[i].Result, attempt.Steps[i].Result) || !timePointerEqual(receipt.Steps[i].StartedAt, attempt.Steps[i].StartedAt) || !timePointerEqual(receipt.Steps[i].EndedAt, attempt.Steps[i].EndedAt) { + !bytes.Equal(receipt.Steps[i].Result, attempt.Steps[i].Result) || !timePointerEqual(receipt.Steps[i].StartedAt, attempt.Steps[i].StartedAt) || !timePointerEqual(receipt.Steps[i].EndedAt, attempt.Steps[i].EndedAt) || !stepSourceEqual(receipt.Steps[i].ReusedFrom, attempt.Steps[i].ReusedFrom) { return false } } return true } +func stepSourceEqual(a, b *StepSource) bool { + return a == nil && b == nil || a != nil && b != nil && *a == *b +} + func timePointerEqual(a, b *time.Time) bool { return a == nil && b == nil || a != nil && b != nil && a.Equal(*b) } diff --git a/internal/stagestore/apply_test.go b/internal/stagestore/apply_test.go index c7ce697..e8cc2ef 100644 --- a/internal/stagestore/apply_test.go +++ b/internal/stagestore/apply_test.go @@ -588,7 +588,7 @@ func TestApplyPartialAndUnknownRecoveryAreMonotonic(t *testing.T) { }) } -func TestApplyPartialResumeIsRefusedUntilReuseProofIsBound(t *testing.T) { +func TestApplyPartialResumeBindsValidatedUploadProvenance(t *testing.T) { s := openDomainStore(t) created := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"upload_attachment","condition":"always"},{"ordinal":2,"type":"create_post","condition":"always"}]}`) first, _ := s.ClaimApply(context.Background(), claimInput(created.Stage, "", RecoveryModeOrdinary)) @@ -598,8 +598,77 @@ func TestApplyPartialResumeIsRefusedUntilReuseProofIsBound(t *testing.T) { _ = s.MarkStepRejected(context.Background(), first.ID, 2, json.RawMessage(`{"status":400}`)) _, _ = s.FinalizeApply(context.Background(), first.ID) detail, _ := s.Show(context.Background(), created.Stage.ID) - if _, err := s.ClaimApply(context.Background(), claimInput(detail.StageSummary, "", RecoveryModePartial)); !errors.Is(err, ErrNotEligible) { - t.Fatalf("unsafe partial resume=%v", err) + resume, err := s.ClaimApply(context.Background(), claimInput(detail.StageSummary, "", RecoveryModePartial)) + if err != nil || len(resume.ReusableUploads) != 1 || resume.ReusableUploads[0].SourceAttemptID != first.ID || resume.ReusableUploads[0].FileID != "file-1" { + t.Fatalf("resume=%+v err=%v", resume, err) + } + for name, args := range map[string][]any{ + "wrong file": {resume.ID, 1, first.ID, 1, "file-2"}, + "wrong source ordinal": {resume.ID, 1, first.ID, 2, "file-1"}, + "wrong destination": {resume.ID, 2, first.ID, 1, "file-1"}, + "self source": {resume.ID, 1, resume.ID, 1, "file-1"}, + } { + t.Run(name, func(t *testing.T) { + if _, err := s.db.Exec(`INSERT INTO apply_step_reuse(attempt_id,ordinal,source_attempt_id,source_ordinal,file_id) VALUES(?,?,?,?,?)`, args...); err == nil { + t.Fatal("invalid provenance insertion succeeded") + } + }) + } + if err = s.MarkStepReused(context.Background(), resume.ID, 1, first.ID, 1, "file-1"); err != nil { + t.Fatal(err) + } + if _, err = s.db.Exec(`UPDATE apply_step_reuse SET file_id='file-2' WHERE attempt_id=? AND ordinal=1`, resume.ID); err == nil { + t.Fatal("reuse provenance update succeeded") + } + if _, err = s.db.Exec(`DELETE FROM apply_step_reuse WHERE attempt_id=? AND ordinal=1`, resume.ID); err == nil { + t.Fatal("reuse provenance deletion succeeded") + } + stored, err := scanApplyAttempt(context.Background(), s.db, resume.ID) + if err != nil || stored.Steps[0].ReusedFrom == nil || stored.Steps[0].ReusedFrom.AttemptID != first.ID || stored.Steps[0].StartedAt == nil { + t.Fatalf("stored=%+v err=%v", stored, err) + } +} + +func TestApplyPartialResumeAccumulatesOnlyDirectValidatedUploads(t *testing.T) { + s := openDomainStore(t) + must := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + created := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"upload_attachment","condition":"always"},{"ordinal":2,"type":"upload_attachment","condition":"always"},{"ordinal":3,"type":"create_post","condition":"always"}]}`) + first, err := s.ClaimApply(context.Background(), claimInput(created.Stage, "", RecoveryModeOrdinary)) + must(err) + must(s.BeginDispatch(context.Background(), first.ID, 1)) + must(s.MarkStepValidated(context.Background(), first.ID, 1, json.RawMessage(`{"fileId":"file-1"}`))) + must(s.BeginDispatch(context.Background(), first.ID, 2)) + must(s.MarkStepRejected(context.Background(), first.ID, 2, json.RawMessage(`{"status":403}`))) + _, err = s.FinalizeApply(context.Background(), first.ID) + must(err) + + detail, err := s.Show(context.Background(), created.Stage.ID) + must(err) + second, err := s.ClaimApply(context.Background(), claimInput(detail.StageSummary, "", RecoveryModePartial)) + if err != nil { + t.Fatal(err) + } + must(s.MarkStepReused(context.Background(), second.ID, 1, first.ID, 1, "file-1")) + must(s.BeginDispatch(context.Background(), second.ID, 2)) + must(s.MarkStepValidated(context.Background(), second.ID, 2, json.RawMessage(`{"fileId":"file-2"}`))) + must(s.BeginDispatch(context.Background(), second.ID, 3)) + must(s.MarkStepRejected(context.Background(), second.ID, 3, json.RawMessage(`{"status":403}`))) + _, err = s.FinalizeApply(context.Background(), second.ID) + must(err) + + detail, err = s.Show(context.Background(), created.Stage.ID) + must(err) + third, err := s.ClaimApply(context.Background(), claimInput(detail.StageSummary, "", RecoveryModePartial)) + if err != nil || len(third.ReusableUploads) != 2 || third.ReusableUploads[0].SourceAttemptID != first.ID || third.ReusableUploads[1].SourceAttemptID != second.ID { + t.Fatalf("third=%+v err=%v", third, err) + } + if err = s.MarkStepReused(context.Background(), third.ID, 1, second.ID, 1, "file-1"); err == nil { + t.Fatal("reuse chain succeeded") } } diff --git a/internal/stagestore/schema.go b/internal/stagestore/schema.go index 80fe229..32567ca 100644 --- a/internal/stagestore/schema.go +++ b/internal/stagestore/schema.go @@ -499,8 +499,54 @@ BEGIN SELECT RAISE(ABORT, 'stage attachment identity is immutable'); END; CREATE TRIGGER stage_attachment_identity_immutable_delete BEFORE DELETE ON stage_attachment_identities WHEN EXISTS(SELECT 1 FROM stage_attachments a WHERE a.stage_id=OLD.stage_id AND a.revision=OLD.revision AND a.ordinal=OLD.ordinal) BEGIN SELECT RAISE(ABORT, 'stage attachment identity is immutable'); END; +`}, {version: 10, name: "validated-upload-reuse", sql: ` +DROP TRIGGER apply_step_state_transition_valid; +CREATE TABLE apply_step_reuse ( + attempt_id TEXT NOT NULL, + ordinal INTEGER NOT NULL CHECK (ordinal > 0), + source_attempt_id TEXT NOT NULL, + source_ordinal INTEGER NOT NULL CHECK (source_ordinal > 0), + file_id TEXT NOT NULL CHECK (length(file_id) > 0), + PRIMARY KEY (attempt_id,ordinal), + FOREIGN KEY (attempt_id,ordinal) REFERENCES apply_steps(attempt_id,ordinal) ON DELETE CASCADE, + FOREIGN KEY (source_attempt_id,source_ordinal) REFERENCES apply_steps(attempt_id,ordinal), + CHECK (attempt_id != source_attempt_id), + CHECK (ordinal = source_ordinal) +) STRICT; +CREATE TRIGGER apply_step_reuse_insert_valid BEFORE INSERT ON apply_step_reuse +WHEN NOT EXISTS( + SELECT 1 FROM apply_steps d + JOIN apply_attempts da ON da.id=d.attempt_id + JOIN apply_steps s ON s.attempt_id=NEW.source_attempt_id AND s.ordinal=NEW.source_ordinal + JOIN apply_attempts sa ON sa.id=s.attempt_id + WHERE d.attempt_id=NEW.attempt_id AND d.ordinal=NEW.ordinal + AND d.kind='upload_attachment' AND d.state='pending' AND da.recovery_mode='resume_partial' + AND s.kind='upload_attachment' AND s.state='response_validated' + AND json_extract(s.result_json,'$.fileId')=NEW.file_id + AND sa.outcome IS NOT NULL + AND da.stage_id=sa.stage_id AND da.revision=sa.revision AND da.semantic_digest=sa.semantic_digest + AND NOT EXISTS(SELECT 1 FROM apply_step_reuse prior WHERE prior.attempt_id=s.attempt_id AND prior.ordinal=s.ordinal) + AND NOT EXISTS(SELECT 1 FROM apply_steps uncertain WHERE uncertain.attempt_id=sa.id AND uncertain.state='outcome_unknown') +) +BEGIN SELECT RAISE(ABORT, 'invalid validated upload reuse'); END; +CREATE TRIGGER apply_step_reuse_immutable_update BEFORE UPDATE ON apply_step_reuse +BEGIN SELECT RAISE(ABORT, 'validated upload reuse is immutable'); END; +CREATE TRIGGER apply_step_reuse_immutable_delete BEFORE DELETE ON apply_step_reuse +BEGIN SELECT RAISE(ABORT, 'validated upload reuse is immutable'); END; +CREATE TRIGGER apply_step_state_transition_valid BEFORE UPDATE OF state ON apply_steps +WHEN NOT ( + OLD.state='pending' AND NEW.state IN ('dispatch_intent','skipped','not_dispatched') + OR OLD.state='pending' AND NEW.state='response_validated' AND NEW.kind='upload_attachment' + AND EXISTS(SELECT 1 FROM apply_step_reuse r WHERE r.attempt_id=NEW.attempt_id AND r.ordinal=NEW.ordinal AND json_extract(NEW.result_json,'$.fileId')=r.file_id) + OR OLD.state='dispatch_intent' AND NEW.state IN ('response_validated','rejected','outcome_unknown') +) +BEGIN SELECT RAISE(ABORT, 'invalid apply step transition'); END; `}} func attachmentIdentityAvailable() bool { return len(migrations) >= 9 && migrations[8].version == 9 && migrations[8].name == "attachment-identity-binding" } + +func validatedUploadReuseAvailable() bool { + return len(migrations) >= 10 && migrations[9].version == 10 && migrations[9].name == "validated-upload-reuse" +} diff --git a/schemas/v2/apply-receipt.schema.json b/schemas/v2/apply-receipt.schema.json index 2265bcb..c6d11a9 100644 --- a/schemas/v2/apply-receipt.schema.json +++ b/schemas/v2/apply-receipt.schema.json @@ -97,7 +97,16 @@ "state": { "enum": ["pending", "dispatch_intent", "response_validated", "rejected", "outcome_unknown", "skipped", "not_dispatched"] }, "result": { "$ref": "#/$defs/result" }, "startedAt": { "type": ["string", "null"], "format": "date-time" }, - "endedAt": { "type": ["string", "null"], "format": "date-time" } + "endedAt": { "type": ["string", "null"], "format": "date-time" }, + "reusedFrom": { + "type": "object", + "additionalProperties": false, + "required": ["attemptId", "ordinal"], + "properties": { + "attemptId": { "type": "string", "pattern": "^att_[A-Za-z0-9_-]{32}$" }, + "ordinal": { "type": "integer", "minimum": 1, "maximum": 102 } + } + } }, "allOf": [ { "if": { "properties": { "state": { "const": "pending" } } }, "then": { "properties": { "result": { "type": "null" }, "startedAt": { "type": "null" }, "endedAt": { "type": "null" } } } }, @@ -113,6 +122,7 @@ { "if": { "properties": { "state": { "const": "response_validated" }, "kind": { "const": "delete_post" } } }, "then": { "properties": { "result": { "$ref": "#/$defs/postResult" } } } }, { "if": { "properties": { "state": { "const": "response_validated" }, "kind": { "enum": ["add_reaction", "remove_reaction"] } } }, "then": { "properties": { "result": { "$ref": "#/$defs/postResult" } } } }, { "if": { "properties": { "state": { "const": "response_validated" }, "kind": { "const": "resolve_conversation" } } }, "then": { "properties": { "result": { "$ref": "#/$defs/conversationResult" } } } } + ,{ "if": { "required": ["reusedFrom"] }, "then": { "properties": { "state": { "const": "response_validated" }, "kind": { "const": "upload_attachment" } } } } ] }, "singleStep": { "properties": { "steps": { "minItems": 1, "maxItems": 1 } } }, diff --git a/schemas/v2/examples/store-doctor.json b/schemas/v2/examples/store-doctor.json index 937002d..8473a3c 100644 --- a/schemas/v2/examples/store-doctor.json +++ b/schemas/v2/examples/store-doctor.json @@ -1 +1 @@ -{"schema":"mm/v2/store-doctor","path":"/home/arda/.local/state/mattermost-cli/stages.sqlite3","report":{"exists":false,"filesystemSafe":null,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":9,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}} +{"schema":"mm/v2/store-doctor","path":"/home/arda/.local/state/mattermost-cli/stages.sqlite3","report":{"exists":false,"filesystemSafe":null,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":10,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}} diff --git a/schemas/v2/examples/store-migrations.json b/schemas/v2/examples/store-migrations.json index 7749550..a93d1ec 100644 --- a/schemas/v2/examples/store-migrations.json +++ b/schemas/v2/examples/store-migrations.json @@ -1,6 +1,6 @@ { "schema": "mm/v2/store-migrations", - "latest": 9, + "latest": 10, "migrations": [ {"version": 1, "name": "core-stage-state", "checksum": "e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"}, {"version": 2, "name": "immutable-local-request-receipts", "checksum": "ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"}, @@ -10,6 +10,7 @@ {"version": 6, "name": "durable-apply-journal", "checksum": "4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56"}, {"version": 7, "name": "status-confirmed-delete-results", "checksum": "bf980f24ff24fdf9d6a3a28b9ded97b51336fa1c8ef290d1bdcf91f1b593c5ce"}, {"version": 8, "name": "already-satisfied-edit-apply", "checksum": "cff30dd4fcc987876e092e2de1f5afd21ebeadbcb792fd550a56eb4c221edd10"}, - {"version": 9, "name": "attachment-identity-binding", "checksum": "d0782e01014d010e5978b39bf6f04ef26508bec9318c7ace3be8bfbbcab4999d"} + {"version": 9, "name": "attachment-identity-binding", "checksum": "d0782e01014d010e5978b39bf6f04ef26508bec9318c7ace3be8bfbbcab4999d"}, + {"version": 10, "name": "validated-upload-reuse", "checksum": "75088d75d41eab9d7d1b1c7b1fa6fc4ce604849321b88feeb110d7ada3447947"} ] } diff --git a/schemas/v2/store-doctor.schema.json b/schemas/v2/store-doctor.schema.json index 77bb39d..255a5e6 100644 --- a/schemas/v2/store-doctor.schema.json +++ b/schemas/v2/store-doctor.schema.json @@ -15,7 +15,7 @@ "exists": { "type": "boolean" }, "filesystemSafe": {}, "applicationId": {}, "integrity": {}, "integrityTruncated": {}, "foreignKeyIssues": {}, "foreignKeyRows": {}, "foreignKeyTruncated": {}, "journalMode": {}, "synchronous": {}, "secureDelete": {}, "foreignKeys": {}, "trustedSchema": {}, "queryOnly": {}, "walFallback": {}, - "migrations": { "type": "object", "additionalProperties": false, "required": ["applied", "latest", "valid"], "properties": { "applied": {}, "latest": { "const": 9 }, "valid": {} } }, + "migrations": { "type": "object", "additionalProperties": false, "required": ["applied", "latest", "valid"], "properties": { "applied": {}, "latest": { "const": 10 }, "valid": {} } }, "permissionModelLimitations": { "type": "array", "items": { "$ref": "#/$defs/safeString" } } }, "allOf": [ @@ -23,14 +23,14 @@ "filesystemSafe": { "const": null }, "applicationId": { "const": null }, "integrity": { "const": null }, "integrityTruncated": { "const": null }, "foreignKeyIssues": { "const": null }, "foreignKeyRows": { "const": null }, "foreignKeyTruncated": { "const": null }, "journalMode": { "const": null }, "synchronous": { "const": null }, "secureDelete": { "const": null }, "foreignKeys": { "const": null }, "trustedSchema": { "const": null }, "queryOnly": { "const": null }, "walFallback": { "const": null }, - "migrations": { "properties": { "applied": { "const": null }, "latest": { "const": 9 }, "valid": { "const": null } } } + "migrations": { "properties": { "applied": { "const": null }, "latest": { "const": 10 }, "valid": { "const": null } } } } } }, { "if": { "properties": { "exists": { "const": true } }, "required": ["exists"] }, "then": { "properties": { "filesystemSafe": { "const": true }, "applicationId": { "const": 1296913970 }, "integrity": { "$ref": "#/$defs/nonemptyBoundedStrings" }, "integrityTruncated": { "type": "boolean" }, "foreignKeyIssues": { "type": "integer", "minimum": 0, "maximum": 21 }, "foreignKeyRows": { "$ref": "#/$defs/boundedStrings" }, "foreignKeyTruncated": { "type": "boolean" }, "journalMode": { "enum": ["wal", "delete", "truncate", "persist"] }, "synchronous": { "type": "integer" }, "secureDelete": { "type": "integer" }, "foreignKeys": { "const": true }, "trustedSchema": { "const": false }, "queryOnly": { "const": true }, "walFallback": { "type": "boolean" }, - "migrations": { "properties": { "applied": { "const": 9 }, "latest": { "const": 9 }, "valid": { "const": true } } } + "migrations": { "properties": { "applied": { "const": 10 }, "latest": { "const": 10 }, "valid": { "const": true } } } }, "allOf": [ { "oneOf": [ { "properties": { "integrity": { "const": ["ok"] }, "integrityTruncated": { "const": false } } }, diff --git a/schemas/v2/store-migrations.schema.json b/schemas/v2/store-migrations.schema.json index b423e62..68f7147 100644 --- a/schemas/v2/store-migrations.schema.json +++ b/schemas/v2/store-migrations.schema.json @@ -6,9 +6,9 @@ "required": ["schema", "latest", "migrations"], "properties": { "schema": { "const": "mm/v2/store-migrations" }, - "latest": { "const": 9 }, + "latest": { "const": 10 }, "migrations": { - "type": "array", "minItems": 9, "maxItems": 9, + "type": "array", "minItems": 10, "maxItems": 10, "prefixItems": [{ "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { "version": { "const": 1 }, "name": { "const": "core-stage-state" }, "checksum": { "const": "e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f" } } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { @@ -27,6 +27,8 @@ "version": { "const": 8 }, "name": { "const": "already-satisfied-edit-apply" }, "checksum": { "const": "cff30dd4fcc987876e092e2de1f5afd21ebeadbcb792fd550a56eb4c221edd10" } } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { "version": { "const": 9 }, "name": { "const": "attachment-identity-binding" }, "checksum": { "const": "d0782e01014d010e5978b39bf6f04ef26508bec9318c7ace3be8bfbbcab4999d" } + } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { + "version": { "const": 10 }, "name": { "const": "validated-upload-reuse" }, "checksum": { "const": "75088d75d41eab9d7d1b1c7b1fa6fc4ce604849321b88feeb110d7ada3447947" } } }], "items": false } From bf0d34df81a7e7da6cee7012dec8431fc71a1eec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 09:59:26 +0300 Subject: [PATCH 090/119] feat: expose durable apply command --- docs/V2_CONTRACT.md | 2 + internal/apply/service.go | 23 +- internal/apply/service_test.go | 41 +++ internal/cli/apply.go | 376 ++++++++++++++++++++++++ internal/cli/apply_test.go | 393 ++++++++++++++++++++++++++ internal/cli/root.go | 76 ++++- internal/cli/root_test.go | 25 +- internal/cli/runtime.go | 4 + internal/cli/stage_create_test.go | 2 +- internal/cli/stage_inspect_test.go | 2 +- internal/output/machine.go | 1 + internal/stagerequest/request.go | 49 ++++ internal/stagerequest/request_test.go | 42 +++ internal/stagestore/apply.go | 42 ++- internal/stagestore/apply_test.go | 20 ++ 15 files changed, 1084 insertions(+), 14 deletions(-) create mode 100644 internal/cli/apply.go create mode 100644 internal/cli/apply_test.go diff --git a/docs/V2_CONTRACT.md b/docs/V2_CONTRACT.md index 0da7f5b..eece071 100644 --- a/docs/V2_CONTRACT.md +++ b/docs/V2_CONTRACT.md @@ -215,6 +215,8 @@ mm apply --from-json It consumes one `mm/v2/apply-request` containing `requestId`, `stageId`, `revision`, expected semantic digest, and exactly one `recoveryMode`: `ordinary`, `resume_partial`, or `force_unknown`. Identical replay returns the existing attempt receipt without dispatch. Reuse of a request ID with different content conflicts. Flags and structured recovery mode cannot be combined. +Apply replay equality uses the digest domain `mm/v2/apply-request/caller-intent/v1`. It binds the stage ID, exact revision, expected semantic digest, and recovery mode, and excludes the request ID and server scope. Human `--request-id` and structured apply therefore share one replay identity without allowing a caller-generated key to alter the intent it names. + Every machine-issued local state mutation, including revise, cancel, revive, and destructive prune, has a versioned request schema and required caller request ID with the same identical-replay/conflicting-reuse semantics. Human subcommands may opt into those semantics with `--request-id`. Input methods are mutually exclusive. Empty or whitespace-only content is rejected. UTF-8 failure is fatal. The current Mattermost limits remain locally enforced before any remote mutation: at most 16,383 Unicode code points and 65,535 UTF-8 bytes, unless a verified server capability establishes a lower bound. diff --git a/internal/apply/service.go b/internal/apply/service.go index 9e85bab..4191ceb 100644 --- a/internal/apply/service.go +++ b/internal/apply/service.go @@ -36,6 +36,21 @@ func (e *ConfirmedEffectError) Error() string { } func (e *ConfirmedEffectError) Unwrap() error { return e.err } +// UnsafeReceiptError means a terminal receipt was durably recorded but cannot +// be emitted because one of its narrow remote fields contains an active +// credential. Callers must classify from the receipt outcome without printing +// the receipt itself. +type UnsafeReceiptError struct { + receipt stagestore.ApplyReceipt + err error +} + +func (e *UnsafeReceiptError) Error() string { return "apply: durable receipt is unsafe to emit" } +func (e *UnsafeReceiptError) Unwrap() error { return e.err } +func (e *UnsafeReceiptError) UnsafeReceipt() stagestore.ApplyReceipt { + return e.receipt +} + type Store interface { Show(context.Context, string) (stagestore.StageDetail, error) FindApply(context.Context, string, string, string, [32]byte) (stagestore.ApplyAttempt, bool, error) @@ -249,7 +264,7 @@ func (s *Service) applyConversation(ctx context.Context, attempt stagestore.Appl } receipt, finalizeErr := s.finalizeReceipt(context.WithoutCancel(ctx), attempt.ID) if finalizeErr != nil { - return stagestore.ApplyReceipt{}, errors.Join(&api.OutcomeUnknownError{}, fmt.Errorf("%w: %v", ErrJournal, finalizeErr)) + return receipt, errors.Join(&api.OutcomeUnknownError{}, ErrJournal, finalizeErr) } return receipt, nil } @@ -294,7 +309,7 @@ func (s *Service) recordRemoteFailure(ctx context.Context, attemptID string, ord } receipt, finalizeErr := s.finalizeReceipt(journalCtx, attemptID) if finalizeErr != nil { - return stagestore.ApplyReceipt{}, errors.Join(remoteErr, fmt.Errorf("%w: %v", ErrJournal, finalizeErr)) + return receipt, errors.Join(remoteErr, ErrJournal, finalizeErr) } return receipt, nil } @@ -321,10 +336,10 @@ func (s *Service) finalizeReceipt(ctx context.Context, attemptID string) (stages } encoded, err := json.Marshal(receipt) if err != nil { - return stagestore.ApplyReceipt{}, fmt.Errorf("%w: encode receipt: %v", ErrJournal, err) + return receipt, &ConfirmedEffectError{fmt.Errorf("%w: encode receipt: %v", ErrJournal, err)} } if s.rawContainsCredentialValue(encoded) { - return stagestore.ApplyReceipt{}, ErrCredential + return receipt, &UnsafeReceiptError{receipt: receipt, err: ErrCredential} } return receipt, nil } diff --git a/internal/apply/service_test.go b/internal/apply/service_test.go index aa9b915..ba7bc04 100644 --- a/internal/apply/service_test.go +++ b/internal/apply/service_test.go @@ -95,6 +95,10 @@ func TestApplyResolveDMCreatesOnceAndReplaysDurableReceipt(t *testing.T) { if _, err = rotatedResult.Apply(context.Background(), claim); !errors.Is(err, ErrCredential) { t.Fatalf("rotated result credential replay error=%v", err) } + var unsafeReceipt *UnsafeReceiptError + if !errors.As(err, &unsafeReceipt) || unsafeReceipt.UnsafeReceipt().Outcome != stagestore.OutcomeSucceeded { + t.Fatalf("rotated result credential classification=%T receipt=%+v", err, unsafeReceipt) + } rotatedTimestamp, err := New(server.URL+"/api/v4", "", [][]byte{[]byte("2026")}, store, mattermost.NewUsers(client), mattermost.NewChannels(client), mattermost.NewPosts(client), mattermost.NewConversationMutations(client), mattermost.NewPostMutations(client)) if err != nil { t.Fatal(err) @@ -282,6 +286,39 @@ func TestApplyResolveDMClassifiesRejectedAndUnknown(t *testing.T) { } } +func TestRecordRemoteFailurePreservesUnsafePartialReceiptClassification(t *testing.T) { + store := openApplyStore(t) + stage := createResolveStage(t, store, "https://mattermost.example.com/api/v4", stagestore.ResolveDM, "dm", []string{"peer"}) + attempt, err := store.ClaimApply(context.Background(), applyClaim(stage, "unsafe-partial")) + if err != nil { + t.Fatal(err) + } + if err = store.BeginDispatch(context.Background(), attempt.ID, 1); err != nil { + t.Fatal(err) + } + unsafeReceipt := stagestore.ApplyReceipt{ + Outcome: stagestore.OutcomePartial, + Recovery: stagestore.RecoveryPartial, + Steps: []stagestore.ApplyStep{{ + Ordinal: 1, + State: stagestore.StepValidated, + Result: json.RawMessage(`{"fileId":"active-token"}`), + }}, + } + service := &Service{ + store: &faultStore{Store: store, finalizeReceipt: &unsafeReceipt}, + credentials: [][]byte{[]byte("active-token")}, + } + receipt, err := service.recordRemoteFailure(context.Background(), attempt.ID, 1, &api.APIError{Status: http.StatusBadRequest}) + var unsafeErr *UnsafeReceiptError + if !errors.As(err, &unsafeErr) || !errors.Is(err, ErrJournal) { + t.Fatalf("classification=%T err=%v", err, err) + } + if receipt.Outcome != stagestore.OutcomePartial || unsafeErr.UnsafeReceipt().Recovery != stagestore.RecoveryPartial { + t.Fatalf("receipt=%+v unsafe=%+v", receipt, unsafeErr.UnsafeReceipt()) + } +} + func TestApplyResolveDMAbandonsSkippedClaimWhenJournalWriteFails(t *testing.T) { var writes atomic.Int32 server := existingDirectServer(t, &writes) @@ -612,6 +649,7 @@ type faultStore struct { *stagestore.Store skipErr, unknownErr, validatedErr, finalizeErr, beginErr error reuseErr error + finalizeReceipt *stagestore.ApplyReceipt beginOrdinal int reuseOrdinal int beforeSkip func() @@ -656,6 +694,9 @@ func (s *faultStore) MarkStepReused(ctx context.Context, attemptID string, ordin } func (s *faultStore) FinalizeApply(ctx context.Context, attemptID string) (stagestore.ApplyReceipt, error) { + if s.finalizeReceipt != nil { + return *s.finalizeReceipt, nil + } if s.finalizeErr != nil { return stagestore.ApplyReceipt{}, s.finalizeErr } diff --git a/internal/cli/apply.go b/internal/cli/apply.go new file mode 100644 index 0000000..9ca1903 --- /dev/null +++ b/internal/cli/apply.go @@ -0,0 +1,376 @@ +package cli + +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/spf13/cobra" + + "github.com/ardasevinc/mattermost-cli/internal/api" + applyservice "github.com/ardasevinc/mattermost-cli/internal/apply" + "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/internal/stagerequest" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" +) + +var stageReferencePattern = regexp.MustCompile(`^(stg_[A-Za-z0-9_-]{32})@([1-9][0-9]{0,15})$`) +var applyRequestIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._~:-]{0,255}$`) + +type applyCommandFailure struct { + code, recovery, stageRef string + exit int + err error +} + +func (e applyCommandFailure) Error() string { return e.err.Error() } +func (e applyCommandFailure) Unwrap() error { return e.err } + +func newApplyCommand(state *rootState) *cobra.Command { + var fromJSON, resumePartial, forceUnknown bool + var requestID string + command := &cobra.Command{ + Use: "apply @", + Short: "Apply one exact reviewed stage revision", + Args: func(_ *cobra.Command, args []string) error { + if fromJSON { + state.flags.json = true + } + if fromJSON && len(args) != 0 { + return invalidFailure("--from-json cannot be combined with a stage reference") + } + if !fromJSON && len(args) != 1 { + return invalidFailure("apply requires one exact @ reference") + } + return nil + }, + PreRunE: func(cmd *cobra.Command, _ []string) error { + if resumePartial && forceUnknown { + return invalidFailure("--resume-partial and --force-unknown cannot be combined") + } + if fromJSON { + state.flags.json = true + if flagChanged(cmd, "request-id") || flagChanged(cmd, "resume-partial") || flagChanged(cmd, "force-unknown") { + return invalidFailure("structured apply cannot be combined with human apply flags") + } + } else if flagChanged(cmd, "request-id") && !applyRequestIDPattern.MatchString(requestID) { + return invalidFailure("invalid --request-id") + } + return nil + }, + } + command.Flags().BoolVar(&fromJSON, "from-json", false, "read one versioned apply request from stdin") + command.Flags().StringVar(&requestID, "request-id", "", "caller-generated replay key") + command.Flags().BoolVar(&resumePartial, "resume-partial", false, "resume only effects proven not applied") + command.Flags().BoolVar(&forceUnknown, "force-unknown", false, "accept duplicate risk after an unknown outcome") + command.RunE = func(cmd *cobra.Command, args []string) error { + if fromJSON { + decoder, err := stagerequest.NewDecoder() + if err != nil { + return internalFailure(err) + } + request, err := decoder.DecodeApply(state.streams.in) + if err != nil { + if schema.IsInputReadError(err) { + return readFailure(errors.New("could not read structured apply request")) + } + return invalidFailure("invalid structured apply request") + } + claim, err := request.ApplyClaimInput() + if err != nil { + return invalidFailure("invalid structured apply request") + } + return executeApply(cmd, state, claim, false) + } + + stageID, revision, err := parseStageReference(args[0]) + if err != nil { + return err + } + mode := stagestore.RecoveryModeOrdinary + if resumePartial { + mode = stagestore.RecoveryModePartial + } else if forceUnknown { + mode = stagestore.RecoveryModeUnknown + } + store, err := openExistingStageStore(cmd, state) + if err != nil { + return err + } + detail, showErr := store.Show(cmd.Context(), stageID) + if showErr != nil { + _ = store.Close() + return classifyApplyError(args[0], showErr) + } + if requestID != "" { + binding, found, lookupErr := store.LookupApplyRequest(cmd.Context(), detail.ServerURL, detail.UserID, requestID) + if lookupErr != nil { + _ = store.Close() + return classifyApplyErrorWithRecovery(args[0], string(detail.Recovery), lookupErr) + } + if found { + claim := stagerequest.NewApplyClaimInput(stageID, requestID, revision, binding.Attempt.SemanticDigest, mode) + if binding.Attempt.StageID != stageID || binding.Attempt.Revision != revision || binding.Attempt.RecoveryMode != mode || binding.RequestDigest != claim.RequestDigest { + _ = store.Close() + return applyStateConflictWithRecovery(args[0], string(detail.Recovery), "request id conflicts with a different apply intent") + } + if closeErr := store.Close(); closeErr != nil { + return applyStateConflictWithRecovery(args[0], string(detail.Recovery), "could not close stage store safely") + } + return executeApply(cmd, state, claim, false) + } + } + if detail.Revision != revision { + _ = store.Close() + return applyStateConflict(args[0], "stage revision changed; inspect the stage again") + } + if closeErr := store.Close(); closeErr != nil { + return applyStateConflict(args[0], "could not close stage store safely") + } + claim := stagerequest.NewApplyClaimInput(stageID, requestID, revision, detail.SemanticDigest, mode) + if requestID != "" { + contract := stagerequest.ApplyRequest{Schema: stagerequest.ApplySchema, RequestID: requestID, StageID: stageID, Revision: stagerequest.ExactInt64(revision), ExpectedDigest: hex.EncodeToString(detail.SemanticDigest[:]), RecoveryMode: string(mode)} + claim, err = contract.ApplyClaimInput() + if err != nil { + return invalidFailure("invalid --request-id") + } + } + return executeApply(cmd, state, claim, true) + } + return command +} + +func parseStageReference(value string) (string, int64, error) { + match := stageReferencePattern.FindStringSubmatch(value) + if match == nil { + return "", 0, invalidFailure("invalid stage reference; expected @") + } + revision, err := strconv.ParseInt(match[2], 10, 64) + if err != nil || revision < 1 || revision > 9007199254740991 { + return "", 0, invalidFailure("invalid stage revision") + } + return match[1], revision, nil +} + +func executeApply(cmd *cobra.Command, state *rootState, claim stagestore.ApplyClaimInput, human bool) error { + stageRef := fmt.Sprintf("%s@%d", claim.StageID, claim.Revision) + store, err := openExistingStageStore(cmd, state) + if err != nil { + return err + } + runtime, err := state.runtimeFor(cmd) + if err != nil { + _ = store.Close() + return err + } + credentials := make([][]byte, 0, len(state.credentials)) + for _, credential := range state.credentials { + if credential != "" { + credentials = append(credentials, []byte(credential)) + } + } + service, err := applyservice.New( + strings.TrimRight(runtime.Config.URL, "/")+"/api/v4", "", credentials, store, + runtime.Users, runtime.Channels, runtime.Posts, + mattermost.NewConversationMutations(runtime.Client), mattermost.NewPostMutations(runtime.Client), + applyservice.WithAttachmentExecution(store.StateDir(), mattermost.NewFileMutations(runtime.Client)), + ) + if err != nil { + _ = store.Close() + return internalFailure(errors.New("could not initialize apply service")) + } + if human && claim.RecoveryMode == stagestore.RecoveryModeUnknown { + warning := "warning: forcing an unknown stage may duplicate a real Mattermost side effect; inspect the destination first\n" + if err := writeAll(state.streams.err, []byte(warning)); err != nil { + _ = store.Close() + return err + } + } + receipt, applyErr := service.Apply(cmd.Context(), claim) + recoveryHint := "none" + if applyErr != nil { + if detail, showErr := store.Show(context.WithoutCancel(cmd.Context()), claim.StageID); showErr == nil { + recoveryHint = string(detail.Recovery) + } + } + closeErr := store.Close() + if applyErr != nil { + return classifyApplyErrorWithRecovery(stageRef, recoveryHint, applyErr) + } + if closeErr != nil { + return classifyApplyReceiptCloseFailure(stageRef, receipt) + } + if err := writeApplyReceipt(state, receipt); err != nil { + if receiptConfirmsEffect(receipt) { + return applyConfirmedFailureWithRecovery(stageRef, string(receipt.Recovery), errors.New("effect confirmed but receipt output failed; do not retry")) + } + if receipt.Outcome == stagestore.OutcomeUnknown { + return applyCommandFailure{"mutation_unknown", "force_unknown", stageRef, 5, errors.New("mutation outcome is unknown and receipt output failed")} + } + return err + } + switch receipt.Outcome { + case stagestore.OutcomeSucceeded, stagestore.OutcomeAlreadySatisfied: + state.setSemanticExit(0) + case stagestore.OutcomeRejected: + state.setSemanticExit(4) + case stagestore.OutcomePartial, stagestore.OutcomeUnknown: + state.setSemanticExit(5) + default: + return internalFailure(errors.New("stored apply receipt has an invalid outcome")) + } + return nil +} + +func classifyApplyReceiptCloseFailure(stageRef string, receipt stagestore.ApplyReceipt) error { + switch receipt.Outcome { + case stagestore.OutcomeUnknown: + return applyCommandFailure{"mutation_unknown", "force_unknown", stageRef, 5, errors.New("mutation outcome is unknown and the stage store did not close safely; inspect the destination before forcing recovery")} + case stagestore.OutcomeRejected: + return applyCommandFailure{"mutation_rejected", string(receipt.Recovery), stageRef, 4, errors.New("mutation was rejected but the stage store did not close safely")} + } + if receiptConfirmsEffect(receipt) { + return applyConfirmedFailureWithRecovery(stageRef, string(receipt.Recovery), errors.New("effect confirmed but the stage store did not close safely; do not retry")) + } + return applyStateConflictWithRecovery(stageRef, string(receipt.Recovery), "could not close stage store safely") +} + +func classifyApplyError(stageRef string, err error) error { + return classifyApplyErrorWithRecovery(stageRef, "none", err) +} + +func classifyApplyErrorWithRecovery(stageRef, recovery string, err error) error { + var confirmed *applyservice.ConfirmedEffectError + var unsafeReceipt *applyservice.UnsafeReceiptError + var unknown *api.OutcomeUnknownError + switch { + case errors.As(err, &unsafeReceipt): + return classifyUnsafeApplyReceipt(stageRef, unsafeReceipt.UnsafeReceipt()) + case errors.As(err, &confirmed): + return applyConfirmedFailure(stageRef, err) + case errors.As(err, &unknown): + return applyCommandFailure{"mutation_unknown", "force_unknown", stageRef, 5, errors.New("mutation outcome is unknown; inspect the destination before forcing recovery")} + case errors.Is(err, stagestore.ErrConflict), errors.Is(err, stagestore.ErrNotEligible), errors.Is(err, stagestore.ErrNotFound), + errors.Is(err, applyservice.ErrTargetDrift), errors.Is(err, applyservice.ErrAttachmentBudget): + return applyStateConflictWithRecovery(stageRef, recovery, "stage state or remote target changed; inspect the stage again") + case errors.Is(err, applyservice.ErrCredential): + return invalidFailure("protected Mattermost credential present in apply input") + case errors.Is(err, applyservice.ErrInvalid): + return invalidFailure("invalid apply request") + case errors.Is(err, applyservice.ErrUnsupportedOperation): + return applyStateConflict(stageRef, "staged operation is not supported by this build") + case errors.Is(err, applyservice.ErrJournal): + return applyStateConflict(stageRef, "could not persist the apply journal") + case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): + return readFailure(errors.New("apply canceled before a confirmed mutation outcome")) + default: + var remote *api.APIError + if errors.As(err, &remote) || errors.Is(err, api.ErrNetwork) || errors.Is(err, api.ErrTimeout) || errors.Is(err, api.ErrInvalidJSON) || errors.Is(err, api.ErrBodyTooLarge) { + return readFailure(errors.New("could not validate the staged Mattermost target")) + } + return applyStateConflict(stageRef, "could not safely apply the staged change") + } +} + +func classifyUnsafeApplyReceipt(stageRef string, receipt stagestore.ApplyReceipt) error { + switch receipt.Outcome { + case stagestore.OutcomeSucceeded, stagestore.OutcomeAlreadySatisfied, stagestore.OutcomePartial: + return applyConfirmedFailureWithRecovery(stageRef, string(receipt.Recovery), errors.New("effect confirmed but its receipt is unsafe to emit; do not retry")) + case stagestore.OutcomeUnknown: + return applyCommandFailure{"mutation_unknown", "force_unknown", stageRef, 5, errors.New("mutation outcome is unknown and its receipt is unsafe to emit; inspect the destination before forcing recovery")} + case stagestore.OutcomeRejected: + return applyCommandFailure{"mutation_rejected", string(receipt.Recovery), stageRef, 4, errors.New("mutation was rejected but its receipt is unsafe to emit")} + default: + return applyStateConflictWithRecovery(stageRef, string(receipt.Recovery), "stored apply receipt has an invalid outcome") + } +} + +func applyStateConflict(stageRef, message string) error { + return applyStateConflictWithRecovery(stageRef, "none", message) +} + +func applyStateConflictWithRecovery(stageRef, recovery, message string) error { + if recovery != "none" && recovery != "resume_partial" && recovery != "force_unknown" && recovery != "forbidden" { + recovery = "none" + } + return applyCommandFailure{"state_conflict", recovery, stageRef, 6, errors.New(message)} +} + +func applyConfirmedFailure(stageRef string, err error) error { + return applyConfirmedFailureWithRecovery(stageRef, "forbidden", err) + +} + +func applyConfirmedFailureWithRecovery(stageRef, recovery string, err error) error { + if recovery != "resume_partial" && recovery != "force_unknown" && recovery != "forbidden" { + recovery = "forbidden" + } + return applyCommandFailure{"confirmed_effect_local_failure", recovery, stageRef, 7, err} +} + +func receiptConfirmsEffect(receipt stagestore.ApplyReceipt) bool { + if receipt.Outcome == stagestore.OutcomeSucceeded || receipt.Outcome == stagestore.OutcomeAlreadySatisfied { + return true + } + for _, step := range receipt.Steps { + if step.State == stagestore.StepValidated || step.State == stagestore.StepSkipped { + return true + } + } + return false +} + +func writeApplyReceipt(state *rootState, receipt stagestore.ApplyReceipt) error { + raw, err := json.Marshal(receipt) + if err != nil { + return internalFailure(errors.New("could not encode apply receipt")) + } + registry, err := schema.Load() + if err != nil { + return internalFailure(err) + } + if err := registry.Validate("mm/v2/apply-receipt", bytes.NewReader(raw)); err != nil { + return internalFailure(errors.New("stored apply receipt is invalid")) + } + if state.flags.json { + return writeAll(state.streams.out, append(raw, '\n')) + } + lines := []string{ + "attempt: " + safeStoreValue(state, receipt.AttemptID), + fmt.Sprintf("stage: %s@%d", safeStoreValue(state, receipt.StageID), receipt.Revision), + "operation: " + safeStoreValue(state, string(receipt.Operation)), + "outcome: " + safeStoreValue(state, string(receipt.Outcome)), + "recovery: " + safeStoreValue(state, string(receipt.Recovery)), + "replayed: " + strconv.FormatBool(receipt.Replay), + "steps:", + } + for _, step := range receipt.Steps { + line := fmt.Sprintf(" %d. %s %s", step.Ordinal, safeStoreValue(state, step.Kind), safeStoreValue(state, string(step.State))) + if step.ReusedFrom != nil { + line += fmt.Sprintf(" reused-from=%s/%d", safeStoreValue(state, step.ReusedFrom.AttemptID), step.ReusedFrom.Ordinal) + } + if len(step.Result) != 0 && string(step.Result) != "null" { + line += " result=" + safeStoreValue(state, string(step.Result)) + } + lines = append(lines, line) + } + stageRef := fmt.Sprintf("%s@%d", safeStoreValue(state, receipt.StageID), receipt.Revision) + next := "none (do not retry)" + switch receipt.Recovery { + case stagestore.RecoveryNone: + next = "mm apply " + stageRef + case stagestore.RecoveryPartial: + next = "mm apply " + stageRef + " --resume-partial" + case stagestore.RecoveryUnknown: + next = "mm apply " + stageRef + " --force-unknown" + } + lines = append(lines, "next: "+next) + return writeAll(state.streams.out, []byte(strings.Join(lines, "\n")+"\n")) +} diff --git a/internal/cli/apply_test.go b/internal/cli/apply_test.go new file mode 100644 index 0000000..13b8f59 --- /dev/null +++ b/internal/cli/apply_test.go @@ -0,0 +1,393 @@ +//go:build darwin || linux + +package cli + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/internal/stagerequest" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/internal/staging" +) + +func createCLIApplyStage(t *testing.T, stateRoot, serverURL, body string) stagestore.MutationResult { + t.Helper() + paths, err := stagestore.ResolvePaths(t.TempDir(), func(key string) (string, bool) { return stateRoot, key == "XDG_STATE_HOME" }) + if err != nil { + t.Fatal(err) + } + store, err := stagestore.Open(t.Context(), paths.DBPath) + if err != nil { + t.Fatal(err) + } + destination, err := json.Marshal(staging.Destination{Kind: "conversation", ChannelID: "channel-1", ChannelType: "public", TeamID: stringPointerCLI("team-1"), ParticipantIDs: []string{}}) + if err != nil { + t.Fatal(err) + } + result, err := store.Create(t.Context(), stagestore.CreateInput{ + RequestDigest: sha256.Sum256([]byte("cli-apply\x00" + body)), Operation: stagestore.CreatePost, + ServerURL: serverURL + "/api/v4", UserID: "self", + Content: stagestore.RevisionContent{Body: []byte(body), Destination: destination, Plan: json.RawMessage(`{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`)}, + }) + if closeErr := store.Close(); err == nil { + err = closeErr + } + if err != nil { + t.Fatal(err) + } + return result.MutationResult +} + +func stringPointerCLI(value string) *string { return &value } + +func setApplyEnvironment(t *testing.T, serverURL, stateRoot string) { + t.Helper() + t.Setenv("HOME", t.TempDir()) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(t.TempDir(), "config")) + t.Setenv("XDG_STATE_HOME", stateRoot) + t.Setenv("MM_URL", serverURL) + t.Setenv("MM_TOKEN", "test-token") +} + +func postApplyServer(t *testing.T, message string, status int, writes *atomic.Int32, warningSeen *atomic.Bool) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch request.URL.RequestURI() { + case "/api/v4/users/me": + _, _ = io.WriteString(response, `{"id":"self","username":"arda"}`) + case "/api/v4/channels/channel-1": + _, _ = io.WriteString(response, `{"id":"channel-1","team_id":"team-1","type":"O","name":"town-square","display_name":"Town Square"}`) + case "/api/v4/channels/channel-1/members/self": + _, _ = io.WriteString(response, `{"channel_id":"channel-1","user_id":"self"}`) + case "/api/v4/posts": + writes.Add(1) + if warningSeen != nil && !warningSeen.Load() { + t.Error("force warning was not emitted before mutation dispatch") + } + var input struct { + ChannelID string `json:"channel_id"` + Message string `json:"message"` + PendingPostID string `json:"pending_post_id"` + } + if request.Method != http.MethodPost || json.NewDecoder(request.Body).Decode(&input) != nil || input.ChannelID != "channel-1" || input.Message != message || input.PendingPostID == "" { + response.WriteHeader(http.StatusBadRequest) + return + } + if status != http.StatusCreated { + response.WriteHeader(status) + return + } + response.WriteHeader(http.StatusCreated) + raw, _ := json.Marshal(map[string]any{"id": "post-1", "channel_id": "channel-1", "user_id": "self", "message": message, "create_at": 100, "update_at": 100, "delete_at": 0, "root_id": "", "file_ids": []string{}, "pending_post_id": input.PendingPostID, "type": ""}) + _, _ = response.Write(raw) + default: + http.NotFound(response, request) + } + })) +} + +func applyRequestJSON(t *testing.T, stage stagestore.MutationResult, requestID string, mode stagestore.RecoveryMode) []byte { + t.Helper() + request := stagerequest.ApplyRequest{Schema: stagerequest.ApplySchema, RequestID: requestID, StageID: stage.Stage.ID, Revision: stagerequest.ExactInt64(stage.Stage.Revision), ExpectedDigest: hex.EncodeToString(stage.Stage.SemanticDigest[:]), RecoveryMode: string(mode)} + raw, err := json.Marshal(request) + if err != nil { + t.Fatal(err) + } + return raw +} + +func validateApplyReceipt(t *testing.T, raw []byte) { + t.Helper() + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/apply-receipt", bytes.NewReader(raw)); err != nil { + t.Fatalf("invalid receipt: %v\n%s", err, raw) + } +} + +func TestStructuredApplyPreservesShortAndLongMarkdownAndReplaysWithoutNetwork(t *testing.T) { + for name, message := range map[string]string{ + "short": "# heading\n\n- **bold** and `code`\n", + "long": strings.Repeat("界", 16_382) + "\n", + } { + t.Run(name, func(t *testing.T) { + var writes atomic.Int32 + server := postApplyServer(t, message, http.StatusCreated, &writes, nil) + defer server.Close() + stateRoot := filepath.Join(t.TempDir(), "state") + setApplyEnvironment(t, server.URL, stateRoot) + stage := createCLIApplyStage(t, stateRoot, server.URL, message) + request := applyRequestJSON(t, stage, "apply-"+name, stagestore.RecoveryModeOrdinary) + + var stdout, stderr bytes.Buffer + if code := Execute(t.Context(), []string{"apply", "--from-json"}, bytes.NewReader(request), &stdout, &stderr); code != 0 || stderr.Len() != 0 { + t.Fatalf("first exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + validateApplyReceipt(t, stdout.Bytes()) + if strings.Contains(stdout.String(), message) || writes.Load() != 1 { + t.Fatalf("receipt leaked content or wrong writes: %d %q", writes.Load(), stdout.String()) + } + + stdout.Reset() + stderr.Reset() + if code := Execute(t.Context(), []string{"apply", "--from-json"}, bytes.NewReader(request), &stdout, &stderr); code != 0 || stderr.Len() != 0 || writes.Load() != 1 { + t.Fatalf("replay exit=%d writes=%d stdout=%q stderr=%q", code, writes.Load(), stdout.String(), stderr.String()) + } + validateApplyReceipt(t, stdout.Bytes()) + }) + } +} + +func TestHumanApplyRejectedReturnsReceiptAndSafeRetryCommand(t *testing.T) { + var writes atomic.Int32 + server := postApplyServer(t, "rejected", http.StatusBadRequest, &writes, nil) + defer server.Close() + stateRoot := filepath.Join(t.TempDir(), "state") + setApplyEnvironment(t, server.URL, stateRoot) + stage := createCLIApplyStage(t, stateRoot, server.URL, "rejected") + stageRef := stage.Stage.ID + "@1" + + var stdout, stderr bytes.Buffer + code := Execute(t.Context(), []string{"apply", stageRef}, strings.NewReader(""), &stdout, &stderr) + if code != 4 || stderr.Len() != 0 || writes.Load() != 1 || !strings.Contains(stdout.String(), "outcome: rejected") || !strings.Contains(stdout.String(), "next: mm apply "+stageRef) { + t.Fatalf("exit=%d writes=%d stdout=%q stderr=%q", code, writes.Load(), stdout.String(), stderr.String()) + } +} + +func TestApplyConfirmedEffectOutputFailureExitsSevenAndDoesNotRetry(t *testing.T) { + var writes atomic.Int32 + server := postApplyServer(t, "one shot", http.StatusCreated, &writes, nil) + defer server.Close() + stateRoot := filepath.Join(t.TempDir(), "state") + setApplyEnvironment(t, server.URL, stateRoot) + stage := createCLIApplyStage(t, stateRoot, server.URL, "one shot") + stageRef := stage.Stage.ID + "@1" + + var stderr bytes.Buffer + if code := Execute(t.Context(), []string{"apply", stageRef}, strings.NewReader(""), shortWriter{}, &stderr); code != 7 || writes.Load() != 1 || !strings.Contains(stderr.String(), "do not retry") { + t.Fatalf("exit=%d writes=%d stderr=%q", code, writes.Load(), stderr.String()) + } + var stdout bytes.Buffer + stderr.Reset() + if code := Execute(t.Context(), []string{"apply", stageRef}, strings.NewReader(""), &stdout, &stderr); code != 6 || writes.Load() != 1 { + t.Fatalf("retry exit=%d writes=%d stdout=%q stderr=%q", code, writes.Load(), stdout.String(), stderr.String()) + } + + for _, machine := range []bool{false, true} { + stage = createCLIApplyStage(t, stateRoot, server.URL, "one shot") + args := []string{"apply", stage.Stage.ID + "@1"} + if machine { + args = append([]string{"--json"}, args...) + } + if code := Execute(t.Context(), args, strings.NewReader(""), zeroFailWriter{}, zeroFailWriter{}); code != 7 { + t.Fatalf("machine=%v exit=%d writes=%d", machine, code, writes.Load()) + } + } + if writes.Load() != 3 { + t.Fatalf("writes=%d, want 3", writes.Load()) + } +} + +func TestUnsafeConfirmedReceiptExitsSevenWithoutLeakingCredentialOrRetrying(t *testing.T) { + var writes atomic.Int32 + server := postApplyServer(t, "safe body", http.StatusCreated, &writes, nil) + defer server.Close() + stateRoot := filepath.Join(t.TempDir(), "state") + setApplyEnvironment(t, server.URL, stateRoot) + t.Setenv("MM_TOKEN", "post-1") + stage := createCLIApplyStage(t, stateRoot, server.URL, "safe body") + args := []string{"apply", stage.Stage.ID + "@1", "--request-id", "unsafe-receipt-replay"} + + for attempt := 1; attempt <= 2; attempt++ { + var stdout, stderr bytes.Buffer + if code := Execute(t.Context(), args, strings.NewReader(""), &stdout, &stderr); code != 7 || stdout.Len() != 0 || writes.Load() != 1 || !strings.Contains(stderr.String(), "do not retry") || strings.Contains(stderr.String(), "post-1") { + t.Fatalf("attempt=%d exit=%d writes=%d stdout=%q stderr=%q", attempt, code, writes.Load(), stdout.String(), stderr.String()) + } + } +} + +type zeroFailWriter struct{} + +func (zeroFailWriter) Write([]byte) (int, error) { return 0, errors.New("closed") } + +type warningObserver struct { + buffer *bytes.Buffer + seen *atomic.Bool +} + +func (w warningObserver) Write(data []byte) (int, error) { + if bytes.Contains(data, []byte("may duplicate a real Mattermost side effect")) { + w.seen.Store(true) + } + return w.buffer.Write(data) +} + +func TestForceUnknownWarnsBeforeDispatch(t *testing.T) { + var writes atomic.Int32 + var warningSeen atomic.Bool + server := postApplyServer(t, "forced", http.StatusCreated, &writes, &warningSeen) + defer server.Close() + stateRoot := filepath.Join(t.TempDir(), "state") + setApplyEnvironment(t, server.URL, stateRoot) + stage := createCLIApplyStage(t, stateRoot, server.URL, "forced") + paths, err := stagestore.ResolvePaths(t.TempDir(), func(key string) (string, bool) { return stateRoot, key == "XDG_STATE_HOME" }) + if err != nil { + t.Fatal(err) + } + store, err := stagestore.Open(t.Context(), paths.DBPath) + if err != nil { + t.Fatal(err) + } + claim := stagerequest.NewApplyClaimInput(stage.Stage.ID, "", 1, stage.Stage.SemanticDigest, stagestore.RecoveryModeOrdinary) + attempt, err := store.ClaimApply(t.Context(), claim) + if err == nil { + err = store.BeginDispatch(t.Context(), attempt.ID, 1) + } + if closeErr := store.Close(); err == nil { + err = closeErr + } + if err != nil { + t.Fatal(err) + } + + var stdout, stderr bytes.Buffer + code := Execute(t.Context(), []string{"apply", stage.Stage.ID + "@1", "--force-unknown"}, strings.NewReader(""), &stdout, warningObserver{&stderr, &warningSeen}) + if code != 0 || writes.Load() != 1 || !warningSeen.Load() || !strings.Contains(stdout.String(), "forcedDuplicateRisk") && !strings.Contains(stdout.String(), "outcome: succeeded") { + t.Fatalf("exit=%d writes=%d stdout=%q stderr=%q", code, writes.Load(), stdout.String(), stderr.String()) + } +} + +func TestHumanRequestIDReplaysOldUnknownReceiptAfterRevision(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { requests.Add(1) })) + defer server.Close() + stateRoot := filepath.Join(t.TempDir(), "state") + setApplyEnvironment(t, server.URL, stateRoot) + stage := createCLIApplyStage(t, stateRoot, server.URL, "original") + paths, err := stagestore.ResolvePaths(t.TempDir(), func(key string) (string, bool) { return stateRoot, key == "XDG_STATE_HOME" }) + if err != nil { + t.Fatal(err) + } + store, err := stagestore.Open(t.Context(), paths.DBPath) + if err != nil { + t.Fatal(err) + } + claim := stagerequest.NewApplyClaimInput(stage.Stage.ID, "human-old-replay", 1, stage.Stage.SemanticDigest, stagestore.RecoveryModeOrdinary) + attempt, err := store.ClaimApply(t.Context(), claim) + if err == nil { + err = store.BeginDispatch(t.Context(), attempt.ID, 1) + } + if err == nil { + err = store.MarkStepUnknown(t.Context(), attempt.ID, 1) + } + if err == nil { + _, err = store.FinalizeApply(t.Context(), attempt.ID) + } + detail, showErr := store.Show(t.Context(), stage.Stage.ID) + if err == nil { + err = showErr + } + if err == nil { + _, err = store.Revise(t.Context(), stagestore.ReviseInput{StageID: stage.Stage.ID, ExpectedRevision: 1, ExpectedDigest: stage.Stage.SemanticDigest, RequestDigest: sha256.Sum256([]byte("revise after unknown")), Composition: stagestore.Composition{Body: []byte("revised"), Plan: detail.Plan}}) + } + if closeErr := store.Close(); err == nil { + err = closeErr + } + if err != nil { + t.Fatal(err) + } + + var stdout, stderr bytes.Buffer + code := Execute(t.Context(), []string{"apply", stage.Stage.ID + "@1", "--request-id", "human-old-replay"}, strings.NewReader(""), &stdout, &stderr) + if code != 5 || stderr.Len() != 0 || requests.Load() != 0 || !strings.Contains(stdout.String(), "outcome: unknown") || !strings.Contains(stdout.String(), "replayed: true") || !strings.Contains(stdout.String(), "--force-unknown") { + t.Fatalf("exit=%d requests=%d stdout=%q stderr=%q", code, requests.Load(), stdout.String(), stderr.String()) + } +} + +func TestApplyRejectsInvalidCombinationsAndStructuredInputBeforeStateOrNetwork(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { requests.Add(1) })) + defer server.Close() + stateRoot := filepath.Join(t.TempDir(), "absent") + setApplyEnvironment(t, server.URL, stateRoot) + for _, test := range []struct { + args []string + input string + }{ + {[]string{"apply", "stg_0123456789abcdefghijklmnopqrstuv@1", "--resume-partial", "--force-unknown"}, ""}, + {[]string{"apply", "stg_0123456789abcdefghijklmnopqrstuv@1", "--request-id", "bad id"}, ""}, + {[]string{"apply", "--from-json", "--request-id", "x"}, `{}`}, + {[]string{"apply", "--from-json", "--resume-partial=false"}, `{}`}, + {[]string{"apply", "--from-json"}, `{"schema":"mm/v2/apply-request"}`}, + } { + var stdout, stderr bytes.Buffer + if code := Execute(t.Context(), test.args, strings.NewReader(test.input), &stdout, &stderr); code != 2 || stdout.Len() != 0 { + t.Fatalf("args=%q exit=%d stdout=%q stderr=%q", test.args, code, stdout.String(), stderr.String()) + } + } + if requests.Load() != 0 { + t.Fatalf("invalid apply used network %d times", requests.Load()) + } +} + +func TestStructuredApplyArgumentFailureIsMachineError(t *testing.T) { + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + for _, args := range [][]string{{"apply", "--from-json", "unexpected"}, {"apply", "--from-json", "--unknown-flag"}, {"apply", "--from-json=wat"}} { + var stdout, stderr bytes.Buffer + if code := Execute(t.Context(), args, strings.NewReader(""), &stdout, &stderr); code != 2 || stdout.Len() != 0 { + t.Fatalf("args=%q exit=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String()) + } + if err := registry.Validate("mm/v2/error", bytes.NewReader(stderr.Bytes())); err != nil { + t.Fatalf("args=%q invalid machine error: %v\n%s", args, err, stderr.String()) + } + } +} + +func TestStructuredApplyStateConflictUsesSchemaValidRecoveryError(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { requests.Add(1) })) + defer server.Close() + stateRoot := filepath.Join(t.TempDir(), "state") + setApplyEnvironment(t, server.URL, stateRoot) + stage := createCLIApplyStage(t, stateRoot, server.URL, "not dispatched") + request := applyRequestJSON(t, stage, "wrong-recovery", stagestore.RecoveryModePartial) + + var stdout, stderr bytes.Buffer + if code := Execute(t.Context(), []string{"apply", "--from-json"}, bytes.NewReader(request), &stdout, &stderr); code != 6 || stdout.Len() != 0 || requests.Load() != 0 { + t.Fatalf("exit=%d requests=%d stdout=%q stderr=%q", code, requests.Load(), stdout.String(), stderr.String()) + } + registry, err := mmSchema.Load() + if err != nil { + t.Fatal(err) + } + if err := registry.Validate("mm/v2/error", bytes.NewReader(stderr.Bytes())); err != nil { + t.Fatalf("invalid machine error: %v\n%s", err, stderr.String()) + } + if !strings.Contains(stderr.String(), `"code":"state_conflict"`) || !strings.Contains(stderr.String(), `"recovery":"none"`) || !strings.Contains(stderr.String(), `"stageRef":"`+stage.Stage.ID+`@1"`) { + t.Fatalf("stderr=%s", stderr.String()) + } +} + +func TestDurableUnknownReceiptDominatesStoreCloseFailure(t *testing.T) { + err := classifyApplyReceiptCloseFailure("stg_0123456789abcdefghijklmnopqrstuv@1", stagestore.ApplyReceipt{Outcome: stagestore.OutcomeUnknown, Recovery: stagestore.RecoveryUnknown}) + if exitCode(err) != 5 || machineErrorCode(err) != "mutation_unknown" || machineRecovery(err) != "force_unknown" { + t.Fatalf("exit=%d code=%q recovery=%q err=%v", exitCode(err), machineErrorCode(err), machineRecovery(err), err) + } +} diff --git a/internal/cli/root.go b/internal/cli/root.go index c8866af..ae5dd87 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -35,6 +35,7 @@ func Execute(ctx context.Context, args []string, in io.Reader, out, errOut io.Wr defer state.releaseCredentials() defer state.close() cmd := newRootWithState(state) + state.flags.json = earlyStructuredMachineMode(args) cmd.SetArgs(args) if err := cmd.ExecuteContext(ctx); err != nil { var corrupted watchOutputFailure @@ -48,17 +49,29 @@ func Execute(ctx context.Context, args []string, in io.Reader, out, errOut io.Wr message := presentation.SanitizeLabel(presentation.Preprocess(err.Error(), state.credentials).Text) code := exitCode(err) if state.flags.json && trackedOut.BytesWritten() > 0 { + if code >= 4 { + return code + } return 3 } if state.flags.json { - document := output.ErrorEnvelope{Schema: "mm/v2/error", Code: machineErrorCode(err), Message: message, ExitCode: code, Recovery: "none"} + document := output.ErrorEnvelope{Schema: "mm/v2/error", Code: machineErrorCode(err), Message: message, ExitCode: code, StageRef: machineStageRef(err), Recovery: machineRecovery(err)} if _, writeErr := output.WriteMachineJSON(errOut, document); writeErr != nil { + if code >= 4 { + return code + } return 3 } } else if writeErr := writeAll(errOut, []byte(fmt.Sprintf("error: %s\n", message))); writeErr != nil { + if code >= 4 { + return code + } return 3 } if trackedOut.Failed() { + if code >= 4 { + return code + } return 3 } return code @@ -74,7 +87,49 @@ func Execute(ctx context.Context, args []string, in io.Reader, out, errOut io.Wr return state.semanticExitCode() } +func earlyStructuredMachineMode(args []string) bool { + for index := 0; index < len(args); index++ { + arg := args[index] + if arg == "--" { + return false + } + if arg == "--url" || arg == "--token" || arg == "-t" { + index++ + continue + } + if strings.HasPrefix(arg, "--url=") || strings.HasPrefix(arg, "--token=") || strings.HasPrefix(arg, "-t") || strings.HasPrefix(arg, "-") { + continue + } + if arg != "apply" { + return false + } + for _, applyArg := range args[index+1:] { + if applyArg == "--" { + return false + } + if applyArg == "--from-json" { + return true + } + if strings.HasPrefix(applyArg, "--from-json=") { + value := strings.TrimPrefix(applyArg, "--from-json=") + switch value { + case "0", "f", "F", "false", "FALSE", "False": + return false + default: + return true + } + } + } + return false + } + return false +} + func machineErrorCode(err error) string { + var applyFailure applyCommandFailure + if errors.As(err, &applyFailure) { + return applyFailure.code + } var outputFailure outputError if errors.As(err, &outputFailure) { return "internal" @@ -89,11 +144,27 @@ func machineErrorCode(err error) string { } var local localStateFailure if errors.As(err, &local) { - return "local_state" + return "state_conflict" } return "invalid_invocation" } +func machineRecovery(err error) string { + var applyFailure applyCommandFailure + if errors.As(err, &applyFailure) { + return applyFailure.recovery + } + return "none" +} + +func machineStageRef(err error) string { + var applyFailure applyCommandFailure + if errors.As(err, &applyFailure) { + return applyFailure.stageRef + } + return "" +} + func newRoot(s streams) *cobra.Command { return newRootWithState(&rootState{streams: s, deps: defaultDependencies(s.out)}) } @@ -130,6 +201,7 @@ func newRootWithState(state *rootState) *cobra.Command { cmd.AddCommand(newSchemaCommand(state)) cmd.AddCommand(newStoreCommand(state)) cmd.AddCommand(newStageCommand(state)) + cmd.AddCommand(newApplyCommand(state)) cmd.AddCommand(newConfigCommand(state)) cmd.AddCommand(newDoctorCommand(state)) cmd.AddCommand(newWhoAmICommand(state)) diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index af256c0..0e2a44f 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -174,7 +174,7 @@ func TestMachineErrorCodePreservesSemantics(t *testing.T) { {readFailure(&api.APIError{Status: 401}), "authentication"}, {readFailure(&api.APIError{Status: 403}), "authorization"}, {outputError{err: errors.New("bad")}, "internal"}, - {localStateFailure{err: errors.New("bad")}, "local_state"}, + {localStateFailure{err: errors.New("bad")}, "state_conflict"}, } for _, test := range tests { if got := machineErrorCode(test.err); got != test.want { @@ -183,6 +183,29 @@ func TestMachineErrorCodePreservesSemantics(t *testing.T) { } } +func TestConfirmedEffectFailurePreservesSafePartialRecovery(t *testing.T) { + err := applyConfirmedFailureWithRecovery("stg_0123456789abcdefghijklmnopqrstuv@1", "resume_partial", errors.New("output failed")) + if exitCode(err) != 7 || machineErrorCode(err) != "confirmed_effect_local_failure" || machineRecovery(err) != "resume_partial" { + t.Fatalf("exit=%d code=%q recovery=%q", exitCode(err), machineErrorCode(err), machineRecovery(err)) + } +} + +func TestEarlyStructuredMachineModeOnlyRecognizesApplyFlag(t *testing.T) { + for _, test := range []struct { + args []string + want bool + }{ + {[]string{"apply", "--from-json"}, true}, + {[]string{"--url", "https://example.com", "apply", "--from-json", "--unknown"}, true}, + {[]string{"stage", "send", "dm", "alice", "--message", "--from-json"}, false}, + {[]string{"apply", "--", "--from-json"}, false}, + } { + if got := earlyStructuredMachineMode(test.args); got != test.want { + t.Fatalf("args=%q got=%v want=%v", test.args, got, test.want) + } + } +} + func TestErrorOutputShortWriteReturnsOutputFailure(t *testing.T) { var stdout bytes.Buffer code := Execute(context.Background(), []string{"unknown"}, strings.NewReader(""), &stdout, shortWriter{}) diff --git a/internal/cli/runtime.go b/internal/cli/runtime.go index 9a79004..f20dd8c 100644 --- a/internal/cli/runtime.go +++ b/internal/cli/runtime.go @@ -309,6 +309,10 @@ func internalFailure(err error) error { } func exitCode(err error) int { + var applyFailure applyCommandFailure + if errors.As(err, &applyFailure) { + return applyFailure.exit + } var outputFailure outputError if errors.As(err, &outputFailure) { return 3 diff --git a/internal/cli/stage_create_test.go b/internal/cli/stage_create_test.go index 7021ac4..8c44773 100644 --- a/internal/cli/stage_create_test.go +++ b/internal/cli/stage_create_test.go @@ -222,7 +222,7 @@ func TestStructuredStageReplayUsesStoredTargetAndConflictingReuseFailsClosed(t * } beforeConflict := len(methods) code, conflictOut, conflictErr := run("changed **markdown**") - if code != 6 || conflictOut != "" || !strings.Contains(conflictErr, `"code":"local_state"`) { + if code != 6 || conflictOut != "" || !strings.Contains(conflictErr, `"code":"state_conflict"`) { t.Fatalf("conflict exit=%d stdout=%q stderr=%q", code, conflictOut, conflictErr) } if got := methods[beforeConflict:]; len(got) != 1 || got[0] != "GET /api/v4/users/me" { diff --git a/internal/cli/stage_inspect_test.go b/internal/cli/stage_inspect_test.go index 786cf5c..0a7514e 100644 --- a/internal/cli/stage_inspect_test.go +++ b/internal/cli/stage_inspect_test.go @@ -205,7 +205,7 @@ func TestStageInspectionIsWiredThroughRootWithStableMachineErrors(t *testing.T) if code := Execute(t.Context(), []string{"--json", "stage", "show", stageID}, strings.NewReader(""), &stdout, &stderr); code != 6 { t.Fatalf("show exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } - if stdout.Len() != 0 || !strings.Contains(stderr.String(), `"code":"local_state"`) || !strings.Contains(stderr.String(), `"exitCode":6`) { + if stdout.Len() != 0 || !strings.Contains(stderr.String(), `"code":"state_conflict"`) || !strings.Contains(stderr.String(), `"exitCode":6`) { t.Fatalf("show stdout=%q stderr=%q", stdout.String(), stderr.String()) } } diff --git a/internal/output/machine.go b/internal/output/machine.go index 4333777..564caf9 100644 --- a/internal/output/machine.go +++ b/internal/output/machine.go @@ -120,6 +120,7 @@ type ErrorEnvelope struct { Code string `json:"code"` Message string `json:"message"` ExitCode int `json:"exitCode"` + StageRef string `json:"stageRef,omitempty"` Recovery string `json:"recovery"` } diff --git a/internal/stagerequest/request.go b/internal/stagerequest/request.go index 37ba6d0..53fd833 100644 --- a/internal/stagerequest/request.go +++ b/internal/stagerequest/request.go @@ -4,6 +4,7 @@ package stagerequest import ( "bytes" + "crypto/sha256" "encoding/hex" "encoding/json" "errors" @@ -17,6 +18,7 @@ import ( "github.com/ardasevinc/mattermost-cli/internal/schema" "github.com/ardasevinc/mattermost-cli/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" "github.com/ardasevinc/mattermost-cli/internal/staging" ) @@ -24,6 +26,7 @@ const ( StageSchema = "mm/v2/stage-request" ReviseSchema = "mm/v2/stage-revise-request" CancelSchema = "mm/v2/stage-cancel-request" + ApplySchema = "mm/v2/apply-request" ) var ErrInvalid = errors.New("invalid stage request") @@ -132,6 +135,15 @@ type CancelRequest struct { ExpectedDigest string `json:"expectedDigest"` } +type ApplyRequest struct { + Schema string `json:"schema"` + RequestID string `json:"requestId"` + StageID string `json:"stageId"` + Revision ExactInt64 `json:"revision"` + ExpectedDigest string `json:"expectedDigest"` + RecoveryMode string `json:"recoveryMode"` +} + type ExactInt64 int64 func (n *ExactInt64) UnmarshalJSON(data []byte) error { @@ -177,6 +189,10 @@ func (d *Decoder) DecodeCancel(input io.Reader) (CancelRequest, error) { return decode[CancelRequest](d, CancelSchema, input) } +func (d *Decoder) DecodeApply(input io.Reader) (ApplyRequest, error) { + return decode[ApplyRequest](d, ApplySchema, input) +} + func decode[T any](d *Decoder, id string, input io.Reader) (T, error) { var zero T if d == nil || d.registry == nil || input == nil { @@ -453,3 +469,36 @@ func (r CancelRequest) CancelInput() (staging.CancelInput, error) { } return staging.CancelInput{StageID: r.StageID, RequestID: r.RequestID, ExpectedRevision: int64(r.ExpectedRevision), ExpectedDigest: digest}, nil } + +// ApplyClaimInput converts the public request and derives caller-intent replay +// identity. The request ID is deliberately excluded from the digest. +func (r ApplyRequest) ApplyClaimInput() (stagestore.ApplyClaimInput, error) { + digest, err := decodeDigest(r.ExpectedDigest) + mode := map[string]stagestore.RecoveryMode{ + string(stagestore.RecoveryModeOrdinary): stagestore.RecoveryModeOrdinary, + string(stagestore.RecoveryModePartial): stagestore.RecoveryModePartial, + string(stagestore.RecoveryModeUnknown): stagestore.RecoveryModeUnknown, + }[r.RecoveryMode] + if err != nil || mode == "" || validateConversion(ApplySchema, r) != nil { + return stagestore.ApplyClaimInput{}, ErrInvalid + } + return NewApplyClaimInput(r.StageID, r.RequestID, int64(r.Revision), digest, mode), nil +} + +// NewApplyClaimInput builds the same replay identity for human and structured +// callers. Human callers may omit requestID and therefore opt out of replay. +func NewApplyClaimInput(stageID, requestID string, revision int64, expectedDigest [32]byte, mode stagestore.RecoveryMode) stagestore.ApplyClaimInput { + intent := struct { + Domain string `json:"domain"` + StageID string `json:"stageId"` + Revision int64 `json:"revision"` + ExpectedDigest string `json:"expectedDigest"` + RecoveryMode stagestore.RecoveryMode `json:"recoveryMode"` + }{"mm/v2/apply-request/caller-intent/v1", stageID, revision, hex.EncodeToString(expectedDigest[:]), mode} + raw, _ := json.Marshal(intent) + requestDigest := [32]byte{} + if requestID != "" { + requestDigest = sha256.Sum256(raw) + } + return stagestore.ApplyClaimInput{StageID: stageID, RequestID: requestID, Revision: revision, ExpectedDigest: expectedDigest, RequestDigest: requestDigest, RecoveryMode: mode} +} diff --git a/internal/stagerequest/request_test.go b/internal/stagerequest/request_test.go index c442ac3..1c1cd11 100644 --- a/internal/stagerequest/request_test.go +++ b/internal/stagerequest/request_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/internal/stagestore" "github.com/ardasevinc/mattermost-cli/internal/staging" ) @@ -174,6 +175,47 @@ func TestReviseAndCancelDecodeAndStoreConversions(t *testing.T) { } } +func TestApplyDecodeConversionAndCallerIntentReplayDigest(t *testing.T) { + digestText := strings.Repeat("ab", 32) + raw := `{"schema":"mm/v2/apply-request","requestId":"apply-1","stageId":"stg_abcdefghijklmnopqrstuvwxyzABCDEF","revision":2,"expectedDigest":"` + digestText + `","recoveryMode":"resume_partial"}` + request, err := decoder(t).DecodeApply(strings.NewReader(raw)) + if err != nil { + t.Fatal(err) + } + input, err := request.ApplyClaimInput() + if err != nil || input.Revision != 2 || input.ExpectedDigest[0] != 0xab || input.RecoveryMode != stagestore.RecoveryModePartial || input.RequestDigest == ([32]byte{}) { + t.Fatalf("input=%#v err=%v", input, err) + } + replayed := request + replayed.RequestID = "apply-2" + replayInput, err := replayed.ApplyClaimInput() + if err != nil || replayInput.RequestDigest != input.RequestDigest { + t.Fatalf("request id changed caller intent: %#v %v", replayInput, err) + } + replayed.RecoveryMode = string(stagestore.RecoveryModeUnknown) + changed, err := replayed.ApplyClaimInput() + if err != nil || changed.RequestDigest == input.RequestDigest { + t.Fatalf("recovery mode did not change caller intent: %#v %v", changed, err) + } +} + +func TestApplyDecodeRejectsDuplicateAndMutatedContracts(t *testing.T) { + valid := `{"schema":"mm/v2/apply-request","requestId":"apply-1","stageId":"stg_abcdefghijklmnopqrstuvwxyzABCDEF","revision":1,"expectedDigest":"` + strings.Repeat("ab", 32) + `","recoveryMode":"ordinary"}` + for _, raw := range []string{strings.Replace(valid, `"requestId":"apply-1"`, `"requestId":"apply-1","requestId":"apply-1"`, 1), valid + `{}`} { + if _, err := decoder(t).DecodeApply(strings.NewReader(raw)); !errors.Is(err, ErrInvalid) { + t.Fatalf("malformed apply accepted: %v", err) + } + } + request, err := decoder(t).DecodeApply(strings.NewReader(valid)) + if err != nil { + t.Fatal(err) + } + request.ExpectedDigest = strings.ToUpper(request.ExpectedDigest) + if _, err := request.ApplyClaimInput(); !errors.Is(err, ErrInvalid) { + t.Fatalf("mutated apply accepted: %v", err) + } +} + func TestResolveConversionsCloneUsernames(t *testing.T) { dm := StageRequest{Schema: StageSchema, Operation: ResolveDM, Target: Target{Kind: "user", Username: "alice"}, Attachments: []Attachment{}} target, err := dm.ResolveDMTarget() diff --git a/internal/stagestore/apply.go b/internal/stagestore/apply.go index f498cf2..3743f3a 100644 --- a/internal/stagestore/apply.go +++ b/internal/stagestore/apply.go @@ -103,6 +103,11 @@ type ApplyReceipt struct { Replay bool `json:"-"` } +type ApplyRequestBinding struct { + RequestDigest [32]byte + Attempt ApplyAttempt +} + type persistedPlan struct { Steps []struct { Ordinal int `json:"ordinal"` @@ -540,21 +545,48 @@ func (s *Store) FindApply(ctx context.Context, server, user, requestID string, r return attempt, found, err } +// LookupApplyRequest returns the immutable caller-intent binding for a human +// replay that does not carry an expected semantic digest on the command line. +func (s *Store) LookupApplyRequest(ctx context.Context, server, user, requestID string) (ApplyRequestBinding, bool, error) { + if ctx == nil || !canonicalServerURL(server) || !bounded(user, maxIdentityBytes) || !validRequestID(requestID) || requestID == "" { + return ApplyRequestBinding{}, false, ErrInvalid + } + storedDigest, attempt, found, err := lookupApplyRequest(ctx, s.db, server, user, requestID) + if err != nil || !found { + return ApplyRequestBinding{}, found, err + } + attempt.Replay = true + return ApplyRequestBinding{RequestDigest: storedDigest, Attempt: attempt}, true, nil +} + func findApplyRequest(ctx context.Context, q queryer, server, user, requestID string, requestDigest [32]byte) (ApplyAttempt, bool, error) { + storedDigest, attempt, found, err := lookupApplyRequest(ctx, q, server, user, requestID) + if err != nil || !found { + return ApplyAttempt{}, found, err + } + if storedDigest != requestDigest { + return ApplyAttempt{}, false, ErrConflict + } + return attempt, true, nil +} + +func lookupApplyRequest(ctx context.Context, q queryer, server, user, requestID string) ([32]byte, ApplyAttempt, bool, error) { + var digest [32]byte var storedDigest []byte var attemptID string err := q.QueryRowContext(ctx, `SELECT request_digest,attempt_id FROM apply_requests WHERE server_url=? AND user_id=? AND request_id=?`, server, user, requestID).Scan(&storedDigest, &attemptID) if errors.Is(err, sql.ErrNoRows) { - return ApplyAttempt{}, false, nil + return digest, ApplyAttempt{}, false, nil } if err != nil { - return ApplyAttempt{}, false, localError(err) + return digest, ApplyAttempt{}, false, localError(err) } - if len(storedDigest) != sha256.Size || !bytes.Equal(storedDigest, requestDigest[:]) { - return ApplyAttempt{}, false, ErrConflict + if len(storedDigest) != sha256.Size { + return digest, ApplyAttempt{}, false, localError(errors.New("apply request digest")) } + copy(digest[:], storedDigest) attempt, err := scanApplyAttempt(ctx, q, attemptID) - return attempt, err == nil, err + return digest, attempt, err == nil, err } func scanApplyAttempt(ctx context.Context, q queryer, attemptID string) (ApplyAttempt, error) { diff --git a/internal/stagestore/apply_test.go b/internal/stagestore/apply_test.go index e8cc2ef..6ad501c 100644 --- a/internal/stagestore/apply_test.go +++ b/internal/stagestore/apply_test.go @@ -693,6 +693,26 @@ func TestApplyCanBeReleasedOnlyBeforeDispatch(t *testing.T) { } } +func TestLookupApplyRequestReturnsImmutableHumanReplayBinding(t *testing.T) { + s := openDomainStore(t) + created := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) + input := claimInput(created.Stage, "human-replay", RecoveryModeUnknown) + // Force mode is not eligible on a fresh stage, so bind an ordinary attempt. + input.RecoveryMode = RecoveryModeOrdinary + input.RequestDigest = sha256.Sum256([]byte("human replay intent")) + attempt, err := s.ClaimApply(context.Background(), input) + if err != nil { + t.Fatal(err) + } + binding, found, err := s.LookupApplyRequest(context.Background(), created.Stage.ServerURL, created.Stage.UserID, input.RequestID) + if err != nil || !found || binding.RequestDigest != input.RequestDigest || !binding.Attempt.Replay || binding.Attempt.ID != attempt.ID { + t.Fatalf("binding=%+v found=%v err=%v", binding, found, err) + } + if _, found, err = s.LookupApplyRequest(context.Background(), created.Stage.ServerURL, created.Stage.UserID, "missing"); err != nil || found { + t.Fatalf("missing found=%v err=%v", found, err) + } +} + func TestApplyAuditHistoryCannotBeDeletedAfterDispatch(t *testing.T) { s := openDomainStore(t) created := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) From 199e39b9b5e0a85083c91408c3e27b4d5a6e1222 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 10:07:16 +0300 Subject: [PATCH 091/119] test: add Go Docker mutation acceptance --- justfile | 8 +- scripts/test-e2e.mjs | 8 ++ tests/e2e/go-live_test.go | 257 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 272 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/go-live_test.go diff --git a/justfile b/justfile index e6aa4d6..bd88c20 100644 --- a/justfile +++ b/justfile @@ -21,11 +21,17 @@ go-modules: go-build: go build -o "${TMPDIR:-/tmp}/mattermost-cli-mm" ./cmd/mm +go-cross-build: + @tmp="${TMPDIR:-/tmp}/mattermost-cli-cross"; mkdir -p "$tmp"; for target in darwin/arm64 darwin/amd64 linux/arm64 linux/amd64; do os="${target%/*}"; arch="${target#*/}"; echo "building $target"; CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" go build -trimpath -o "$tmp/mm-$os-$arch" ./cmd/mm; done + +docker-e2e: + bun run test:e2e + oracle-smoke: git diff --quiet v1.6.0 -- src package.json bun.lock tsconfig.json go run ./cmd/conformance --scenario conformance/scenarios/v1/whoami.json --cwd . -- bun src/index.ts -go-gate: go-format-check go-test go-race go-vet go-modules go-build +go-gate: go-format-check go-test go-race go-vet go-modules go-build go-cross-build git diff --check legacy-gate: diff --git a/scripts/test-e2e.mjs b/scripts/test-e2e.mjs index 064727c..104c99d 100644 --- a/scripts/test-e2e.mjs +++ b/scripts/test-e2e.mjs @@ -1,6 +1,8 @@ import { spawnSync } from 'node:child_process' import { randomUUID } from 'node:crypto' +import { rmSync } from 'node:fs' import { fileURLToPath } from 'node:url' +import os from 'node:os' import path from 'node:path' const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') @@ -8,6 +10,7 @@ const composeFile = path.join(root, 'tests/e2e/compose.yml') const project = `mattermost-cli-e2e-${process.pid}-${randomUUID().slice(0, 8)}` const requestedPort = process.env.MM_E2E_PORT || '0' const compose = ['compose', '-p', project, '-f', composeFile] +const goBinary = path.join(os.tmpdir(), `${project}-mm`) function run(command, args, options = {}) { const result = spawnSync(command, args, { @@ -98,6 +101,11 @@ try { run('bunx', ['vitest', 'run', '--config', 'vitest.e2e.config.ts'], { env: { MM_E2E_URL: url, MM_E2E_TOKEN: token }, }) + run('go', ['build', '-o', goBinary, './cmd/mm']) + run('go', ['test', '-tags=e2e', '-count=1', './tests/e2e'], { + env: { MM_E2E_URL: url, MM_E2E_TOKEN: token, MM_E2E_BINARY: goBinary }, + }) } finally { + rmSync(goBinary, { force: true }) cleanup() } diff --git a/tests/e2e/go-live_test.go b/tests/e2e/go-live_test.go new file mode 100644 index 0000000..e8d7eb1 --- /dev/null +++ b/tests/e2e/go-live_test.go @@ -0,0 +1,257 @@ +//go:build e2e + +package e2e_test + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + "unicode/utf8" +) + +type liveHarness struct { + t *testing.T + url string + token string + binary string + home string + client *http.Client +} + +type stageReceipt struct { + Schema string `json:"schema"` + Stage struct { + StageRef string `json:"stageRef"` + } `json:"stage"` +} + +type applyReceipt struct { + Schema string `json:"schema"` + AttemptID string `json:"attemptId"` + Outcome string `json:"outcome"` + Steps []struct { + Kind string `json:"kind"` + Result json.RawMessage `json:"result"` + } `json:"steps"` +} + +type postPage struct { + Order []string `json:"order"` + Posts map[string]struct { + Message string `json:"message"` + } `json:"posts"` +} + +func TestGoStageApplyPreservesShortMarkdownAndReplaysWithoutAnotherPost(t *testing.T) { + h := newLiveHarness(t) + self := h.user("me") + alice := h.user("username/alice") + channel := h.createChannel("direct", []string{self.ID, alice.ID}) + message := "# review\n\n- **bold**\n- `code`\n\n> final line\n" + h.assertStageApplyExactlyOnce("dm", "alice", channel.ID, self.ID, message, "short-markdown") +} + +func TestGoStageApplyPreservesMaximumLongMarkdownInGroup(t *testing.T) { + h := newLiveHarness(t) + users := []user{h.user("me"), h.user("username/alice"), h.user("username/bob")} + ids := []string{users[0].ID, users[1].ID, users[2].ID} + channel := h.createChannel("group", ids) + prefix := "# long markdown\n\n" + message := prefix + strings.Repeat("λ", 16_383-utf8.RuneCountInString(prefix)) + if utf8.RuneCountInString(message) != 16_383 { + t.Fatalf("long fixture has %d runes", utf8.RuneCountInString(message)) + } + h.assertStageApplyExactlyOnce("group", channel.ID, channel.ID, users[0].ID, message, "long-markdown") +} + +func newLiveHarness(t *testing.T) *liveHarness { + t.Helper() + rawURL, token, binary := os.Getenv("MM_E2E_URL"), os.Getenv("MM_E2E_TOKEN"), os.Getenv("MM_E2E_BINARY") + parsed, err := url.Parse(rawURL) + if err != nil || parsed.Scheme != "http" || parsed.Hostname() != "127.0.0.1" || parsed.Port() == "" { + t.Fatal("refusing to run Go mutation E2E without an explicit loopback Mattermost URL") + } + if token == "" || binary == "" || !filepath.IsAbs(binary) { + t.Fatal("disposable Mattermost token and absolute Go binary path are required") + } + transport := &http.Transport{Proxy: nil} + t.Cleanup(transport.CloseIdleConnections) + return &liveHarness{ + t: t, url: strings.TrimRight(rawURL, "/"), token: token, binary: binary, home: t.TempDir(), + client: &http.Client{Transport: transport, Timeout: 15 * time.Second}, + } +} + +type user struct { + ID string `json:"id"` +} + +type channel struct { + ID string `json:"id"` +} + +func (h *liveHarness) user(selector string) user { + h.t.Helper() + var result user + h.api(http.MethodGet, "/users/"+selector, nil, &result) + if result.ID == "" { + h.t.Fatal("Mattermost returned an empty user id") + } + return result +} + +func (h *liveHarness) createChannel(kind string, userIDs []string) channel { + h.t.Helper() + body, err := json.Marshal(userIDs) + if err != nil { + h.t.Fatal(err) + } + var result channel + h.api(http.MethodPost, "/channels/"+kind, body, &result) + if result.ID == "" { + h.t.Fatal("Mattermost returned an empty channel id") + } + return result +} + +func (h *liveHarness) assertStageApplyExactlyOnce(kind, target, channelID, userID, message, requestPrefix string) { + h.t.Helper() + before := h.posts(channelID) + stageRaw := h.cli(message, "--json", "stage", "send", kind, target, "--request-id", requestPrefix+"-stage") + var staged stageReceipt + if err := json.Unmarshal(stageRaw, &staged); err != nil || staged.Schema != "mm/v2/stage-receipt" || staged.Stage.StageRef == "" { + h.t.Fatalf("invalid stage receipt: %v", err) + } + if afterStage := h.posts(channelID); len(afterStage.Order) != len(before.Order) { + h.t.Fatalf("staging mutated Mattermost: before=%d after=%d", len(before.Order), len(afterStage.Order)) + } + + applyRaw := h.cli("", "--json", "apply", staged.Stage.StageRef, "--request-id", requestPrefix+"-apply") + var applied applyReceipt + if err := json.Unmarshal(applyRaw, &applied); err != nil || applied.Schema != "mm/v2/apply-receipt" || applied.Outcome != "succeeded" { + h.t.Fatalf("invalid apply receipt: %v outcome=%q", err, applied.Outcome) + } + postID := createPostID(h.t, applied) + var posted struct { + ID string `json:"id"` + ChannelID string `json:"channel_id"` + UserID string `json:"user_id"` + Message string `json:"message"` + } + h.api(http.MethodGet, "/posts/"+postID, nil, &posted) + if posted.ID != postID || posted.ChannelID != channelID || posted.UserID != userID || posted.Message != message { + h.t.Fatalf("stored post mismatch: id=%q channel=%q user=%q bytes=%d", posted.ID, posted.ChannelID, posted.UserID, len(posted.Message)) + } + afterApply := h.posts(channelID) + if countMessage(afterApply, message) != 1 || len(afterApply.Order) != len(before.Order)+1 { + h.t.Fatalf("apply count mismatch: before=%d after=%d matches=%d", len(before.Order), len(afterApply.Order), countMessage(afterApply, message)) + } + + replayRaw := h.cli("", "--json", "apply", staged.Stage.StageRef, "--request-id", requestPrefix+"-apply") + var replay applyReceipt + if err := json.Unmarshal(replayRaw, &replay); err != nil || replay.AttemptID != applied.AttemptID || replay.Outcome != "succeeded" { + h.t.Fatalf("invalid apply replay: %v attempt=%q outcome=%q", err, replay.AttemptID, replay.Outcome) + } + afterReplay := h.posts(channelID) + if countMessage(afterReplay, message) != 1 || len(afterReplay.Order) != len(afterApply.Order) { + h.t.Fatalf("apply replay created another post: before=%d after=%d matches=%d", len(afterApply.Order), len(afterReplay.Order), countMessage(afterReplay, message)) + } +} + +func createPostID(t *testing.T, receipt applyReceipt) string { + t.Helper() + for _, step := range receipt.Steps { + if step.Kind != "create_post" { + continue + } + var result struct { + PostID string `json:"postId"` + } + if err := json.Unmarshal(step.Result, &result); err == nil && result.PostID != "" { + return result.PostID + } + } + t.Fatal("apply receipt has no validated create_post result") + return "" +} + +func (h *liveHarness) posts(channelID string) postPage { + h.t.Helper() + var result postPage + h.api(http.MethodGet, "/channels/"+channelID+"/posts?per_page=200", nil, &result) + if result.Order == nil || result.Posts == nil { + h.t.Fatal("Mattermost returned an invalid post page") + } + return result +} + +func countMessage(page postPage, message string) int { + count := 0 + for _, id := range page.Order { + if page.Posts[id].Message == message { + count++ + } + } + return count +} + +func (h *liveHarness) api(method, path string, body []byte, target any) { + h.t.Helper() + request, err := http.NewRequest(method, h.url+"/api/v4"+path, bytes.NewReader(body)) + if err != nil { + h.t.Fatal(err) + } + request.Header.Set("Authorization", "Bearer "+h.token) + request.Header.Set("Content-Type", "application/json") + response, err := h.client.Do(request) + if err != nil { + h.t.Fatalf("Mattermost E2E request failed: %v", err) + } + defer response.Body.Close() + if response.StatusCode < 200 || response.StatusCode >= 300 { + _, _ = io.Copy(io.Discard, response.Body) + h.t.Fatalf("Mattermost E2E request returned status %d", response.StatusCode) + } + decoder := json.NewDecoder(io.LimitReader(response.Body, 4<<20)) + if err := decoder.Decode(target); err != nil { + h.t.Fatalf("Mattermost E2E response decode failed: %v", err) + } +} + +func (h *liveHarness) cli(stdin string, args ...string) []byte { + h.t.Helper() + command := exec.Command(h.binary, args...) + command.Stdin = strings.NewReader(stdin) + command.Env = []string{ + "HOME=" + h.home, + "XDG_CONFIG_HOME=" + filepath.Join(h.home, ".config"), + "XDG_STATE_HOME=" + filepath.Join(h.home, ".local", "state"), + "MM_URL=" + h.url, + "MM_TOKEN=" + h.token, + "PATH=" + os.Getenv("PATH"), + "TMPDIR=" + h.home, + "LANG=C.UTF-8", + "LC_ALL=C.UTF-8", + "TZ=UTC", + "NO_COLOR=1", + "TERM=dumb", + } + var stdout, stderr bytes.Buffer + command.Stdout, command.Stderr = &stdout, &stderr + if err := command.Run(); err != nil { + h.t.Fatalf("mm %s failed: %v, stderr=%s", strings.Join(args, " "), err, fmt.Sprintf("%q", stderr.String())) + } + if stderr.Len() != 0 { + h.t.Fatalf("mm %s emitted stderr on success: %q", strings.Join(args, " "), stderr.String()) + } + return stdout.Bytes() +} From 45fe96f8d2f95895bf3421e17cbfa22cfaa3668e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 10:17:24 +0300 Subject: [PATCH 092/119] test: harden Docker mutation acceptance --- justfile | 10 ++- scripts/test-e2e.mjs | 156 +++++++++++++++++++++++++++++--------- tests/e2e/go-live_test.go | 70 ++++++++++++++++- 3 files changed, 192 insertions(+), 44 deletions(-) diff --git a/justfile b/justfile index bd88c20..d22278f 100644 --- a/justfile +++ b/justfile @@ -4,7 +4,7 @@ default: @just --list go-format-check: - @unformatted="$(gofmt -l cmd internal)"; if [[ -n "$unformatted" ]]; then print -r -- "$unformatted"; exit 1; fi + @unformatted="$(gofmt -l cmd internal tests/e2e)"; if [[ -n "$unformatted" ]]; then print -r -- "$unformatted"; exit 1; fi go-test: go test ./... @@ -21,8 +21,12 @@ go-modules: go-build: go build -o "${TMPDIR:-/tmp}/mattermost-cli-mm" ./cmd/mm +go-e2e-compile: + go test -tags=e2e -run '^$' ./tests/e2e + go vet -tags=e2e ./tests/e2e + go-cross-build: - @tmp="${TMPDIR:-/tmp}/mattermost-cli-cross"; mkdir -p "$tmp"; for target in darwin/arm64 darwin/amd64 linux/arm64 linux/amd64; do os="${target%/*}"; arch="${target#*/}"; echo "building $target"; CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" go build -trimpath -o "$tmp/mm-$os-$arch" ./cmd/mm; done + @tmp="$(mktemp -d "${TMPDIR:-/tmp}/mattermost-cli-cross.XXXXXX")"; trap 'find "$tmp" -type f -delete; rmdir "$tmp"' EXIT; for target in darwin/arm64 darwin/amd64 linux/arm64 linux/amd64; do os="${target%/*}"; arch="${target#*/}"; echo "building $target"; CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" go build -trimpath -o "$tmp/mm-$os-$arch" ./cmd/mm; done docker-e2e: bun run test:e2e @@ -31,7 +35,7 @@ oracle-smoke: git diff --quiet v1.6.0 -- src package.json bun.lock tsconfig.json go run ./cmd/conformance --scenario conformance/scenarios/v1/whoami.json --cwd . -- bun src/index.ts -go-gate: go-format-check go-test go-race go-vet go-modules go-build go-cross-build +go-gate: go-format-check go-test go-race go-vet go-modules go-build go-e2e-compile go-cross-build git diff --check legacy-gate: diff --git a/scripts/test-e2e.mjs b/scripts/test-e2e.mjs index 104c99d..6101302 100644 --- a/scripts/test-e2e.mjs +++ b/scripts/test-e2e.mjs @@ -1,4 +1,4 @@ -import { spawnSync } from 'node:child_process' +import { spawn } from 'node:child_process' import { randomUUID } from 'node:crypto' import { rmSync } from 'node:fs' import { fileURLToPath } from 'node:url' @@ -8,28 +8,52 @@ import path from 'node:path' const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') const composeFile = path.join(root, 'tests/e2e/compose.yml') const project = `mattermost-cli-e2e-${process.pid}-${randomUUID().slice(0, 8)}` +const markerTeam = `mm-e2e-${randomUUID().replaceAll('-', '').slice(0, 16)}` const requestedPort = process.env.MM_E2E_PORT || '0' const compose = ['compose', '-p', project, '-f', composeFile] const goBinary = path.join(os.tmpdir(), `${project}-mm`) +const children = new Set() function run(command, args, options = {}) { - const result = spawnSync(command, args, { - cwd: root, - encoding: 'utf8', - stdio: options.capture ? 'pipe' : 'inherit', - env: { ...process.env, MM_E2E_PORT: requestedPort, ...options.env }, - }) - if (result.status !== 0) { - if (options.capture && !options.sensitive) { - if (result.stdout) process.stderr.write(result.stdout) - if (result.stderr) process.stderr.write(result.stderr) + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: root, + stdio: options.capture ? ['ignore', 'pipe', 'pipe'] : 'inherit', + env: { ...process.env, MM_E2E_PORT: requestedPort, ...options.env }, + }) + children.add(child) + let stdout = '' + let stderr = '' + if (options.capture) { + child.stdout.setEncoding('utf8') + child.stderr.setEncoding('utf8') + child.stdout.on('data', (chunk) => { + stdout += chunk + }) + child.stderr.on('data', (chunk) => { + stderr += chunk + }) } - throw new Error(`${command} exited with status ${result.status ?? 'unknown'}`) - } - return result.stdout || '' + child.once('error', (error) => { + children.delete(child) + reject(error) + }) + child.once('close', (code, signal) => { + children.delete(child) + if (code === 0) { + resolve(stdout) + return + } + if (options.capture && !options.sensitive) { + if (stdout) process.stderr.write(stdout) + if (stderr) process.stderr.write(stderr) + } + reject(new Error(`${command} exited with status ${code ?? signal ?? 'unknown'}`)) + }) + }) } -function mmctl(args, capture = false, sensitive = false) { +async function mmctl(args, capture = false, sensitive = false) { return run( 'docker', [...compose, 'exec', '-T', 'mattermost', 'mmctl', '--local', ...args], @@ -37,29 +61,69 @@ function mmctl(args, capture = false, sensitive = false) { ) } +let cleanupPromise + function cleanup() { - run('docker', [...compose, 'down', '--volumes', '--remove-orphans']) - const containers = run('docker', [...compose, 'ps', '-aq'], { capture: true }).trim() - const volumes = run( - 'docker', - ['volume', 'ls', '-q', '--filter', `label=com.docker.compose.project=${project}`], - { capture: true }, + cleanupPromise ??= cleanupOnce() + return cleanupPromise +} + +async function cleanupOnce() { + await run('docker', [...compose, 'down', '--volumes', '--remove-orphans']) + const containers = (await run('docker', [...compose, 'ps', '-aq'], { capture: true })).trim() + const volumes = ( + await run( + 'docker', + ['volume', 'ls', '-q', '--filter', `label=com.docker.compose.project=${project}`], + { capture: true }, + ) + ).trim() + const networks = ( + await run( + 'docker', + ['network', 'ls', '-q', '--filter', `label=com.docker.compose.project=${project}`], + { capture: true }, + ) ).trim() - if (containers || volumes) { + if (containers || volumes || networks) { throw new Error('Docker E2E cleanup left project resources behind') } } +let terminating = false +for (const [signal, exitCode] of [ + ['SIGINT', 130], + ['SIGTERM', 143], +]) { + process.once(signal, () => { + void terminate(exitCode) + }) +} + +async function terminate(exitCode) { + if (terminating) return + terminating = true + for (const child of children) child.kill('SIGTERM') + try { + await cleanup() + } finally { + rmSync(goBinary, { force: true }) + process.exit(exitCode) + } +} + try { - run('docker', [...compose, 'up', '-d', '--wait', '--wait-timeout', '180']) - const published = run('docker', [...compose, 'port', 'mattermost', '8065'], { - capture: true, - }).trim() + await run('docker', [...compose, 'up', '-d', '--wait', '--wait-timeout', '180']) + const published = ( + await run('docker', [...compose, 'port', 'mattermost', '8065'], { + capture: true, + }) + ).trim() const port = published.match(/:(\d+)$/)?.[1] if (!port) throw new Error('Docker did not report the Mattermost E2E port') const url = `http://127.0.0.1:${port}` - mmctl([ + await mmctl([ '--quiet', 'user', 'create', @@ -74,7 +138,7 @@ try { '--disable-welcome-email', ]) for (const username of ['alice', 'bob']) { - mmctl([ + await mmctl([ '--quiet', 'user', 'create', @@ -88,24 +152,42 @@ try { '--disable-welcome-email', ]) } - mmctl(['--quiet', 'team', 'create', '--name', 'e2e', '--display-name', 'E2E']) - mmctl(['--quiet', 'team', 'users', 'add', 'e2e', 'sender', 'alice', 'bob']) + await mmctl(['--quiet', 'team', 'create', '--name', 'e2e', '--display-name', 'E2E']) + await mmctl(['--quiet', 'team', 'users', 'add', 'e2e', 'sender', 'alice', 'bob']) + await mmctl([ + '--quiet', + 'team', + 'create', + '--name', + markerTeam, + '--display-name', + `Mattermost CLI E2E ${markerTeam}`, + ]) + await mmctl(['--quiet', 'team', 'users', 'add', markerTeam, 'sender']) const generated = JSON.parse( - mmctl(['--json', 'token', 'generate', 'sender', 'mattermost-cli-e2e'], true, true), + await mmctl(['--json', 'token', 'generate', 'sender', 'mattermost-cli-e2e'], true, true), ) const token = generated?.[0]?.token if (typeof token !== 'string' || token.length === 0) { throw new Error('Mattermost did not return an E2E access token') } - run('bunx', ['vitest', 'run', '--config', 'vitest.e2e.config.ts'], { + await run('bunx', ['vitest', 'run', '--config', 'vitest.e2e.config.ts'], { env: { MM_E2E_URL: url, MM_E2E_TOKEN: token }, }) - run('go', ['build', '-o', goBinary, './cmd/mm']) - run('go', ['test', '-tags=e2e', '-count=1', './tests/e2e'], { - env: { MM_E2E_URL: url, MM_E2E_TOKEN: token, MM_E2E_BINARY: goBinary }, + await run('go', ['build', '-o', goBinary, './cmd/mm']) + await run('go', ['test', '-tags=e2e', '-count=1', './tests/e2e'], { + env: { + MM_E2E_URL: url, + MM_E2E_TOKEN: token, + MM_E2E_BINARY: goBinary, + MM_E2E_MARKER_TEAM: markerTeam, + }, }) } finally { - rmSync(goBinary, { force: true }) - cleanup() + try { + await cleanup() + } finally { + rmSync(goBinary, { force: true }) + } } diff --git a/tests/e2e/go-live_test.go b/tests/e2e/go-live_test.go index e8d7eb1..7f9ab39 100644 --- a/tests/e2e/go-live_test.go +++ b/tests/e2e/go-live_test.go @@ -7,12 +7,17 @@ import ( "encoding/json" "fmt" "io" + "maps" "net/http" + "net/http/httptest" + "net/http/httputil" "net/url" "os" "os/exec" "path/filepath" + "regexp" "strings" + "sync" "testing" "time" "unicode/utf8" @@ -21,10 +26,13 @@ import ( type liveHarness struct { t *testing.T url string + cliURL string token string binary string home string client *http.Client + mu sync.Mutex + writes map[string]int } type stageReceipt struct { @@ -76,19 +84,46 @@ func TestGoStageApplyPreservesMaximumLongMarkdownInGroup(t *testing.T) { func newLiveHarness(t *testing.T) *liveHarness { t.Helper() rawURL, token, binary := os.Getenv("MM_E2E_URL"), os.Getenv("MM_E2E_TOKEN"), os.Getenv("MM_E2E_BINARY") + markerTeam := os.Getenv("MM_E2E_MARKER_TEAM") parsed, err := url.Parse(rawURL) if err != nil || parsed.Scheme != "http" || parsed.Hostname() != "127.0.0.1" || parsed.Port() == "" { t.Fatal("refusing to run Go mutation E2E without an explicit loopback Mattermost URL") } - if token == "" || binary == "" || !filepath.IsAbs(binary) { - t.Fatal("disposable Mattermost token and absolute Go binary path are required") + if token == "" || binary == "" || !filepath.IsAbs(binary) || !regexp.MustCompile(`^mm-e2e-[0-9a-f]{16}$`).MatchString(markerTeam) { + t.Fatal("disposable Mattermost token, marker team, and absolute Go binary path are required") } transport := &http.Transport{Proxy: nil} t.Cleanup(transport.CloseIdleConnections) - return &liveHarness{ + harness := &liveHarness{ t: t, url: strings.TrimRight(rawURL, "/"), token: token, binary: binary, home: t.TempDir(), client: &http.Client{Transport: transport, Timeout: 15 * time.Second}, + writes: make(map[string]int), } + proxyTransport := &http.Transport{Proxy: nil} + proxy := httputil.NewSingleHostReverseProxy(parsed) + proxy.Transport = proxyTransport + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodGet && request.Method != http.MethodHead && request.Method != http.MethodOptions { + harness.mu.Lock() + harness.writes[request.Method+" "+request.URL.Path]++ + harness.mu.Unlock() + } + proxy.ServeHTTP(response, request) + })) + harness.cliURL = server.URL + t.Cleanup(func() { + server.Close() + proxyTransport.CloseIdleConnections() + }) + var marker struct { + Name string `json:"name"` + DisplayName string `json:"display_name"` + } + harness.api(http.MethodGet, "/teams/name/"+url.PathEscape(markerTeam), nil, &marker) + if marker.Name != markerTeam || marker.DisplayName != "Mattermost CLI E2E "+markerTeam { + t.Fatal("refusing to mutate a Mattermost server without this run's random marker team") + } + return harness } type user struct { @@ -126,7 +161,9 @@ func (h *liveHarness) createChannel(kind string, userIDs []string) channel { func (h *liveHarness) assertStageApplyExactlyOnce(kind, target, channelID, userID, message, requestPrefix string) { h.t.Helper() before := h.posts(channelID) + writesBeforeStage := h.mutationSnapshot() stageRaw := h.cli(message, "--json", "stage", "send", kind, target, "--request-id", requestPrefix+"-stage") + h.assertMutationDelta(writesBeforeStage, nil) var staged stageReceipt if err := json.Unmarshal(stageRaw, &staged); err != nil || staged.Schema != "mm/v2/stage-receipt" || staged.Stage.StageRef == "" { h.t.Fatalf("invalid stage receipt: %v", err) @@ -135,7 +172,9 @@ func (h *liveHarness) assertStageApplyExactlyOnce(kind, target, channelID, userI h.t.Fatalf("staging mutated Mattermost: before=%d after=%d", len(before.Order), len(afterStage.Order)) } + writesBeforeApply := h.mutationSnapshot() applyRaw := h.cli("", "--json", "apply", staged.Stage.StageRef, "--request-id", requestPrefix+"-apply") + h.assertMutationDelta(writesBeforeApply, map[string]int{"POST /api/v4/posts": 1}) var applied applyReceipt if err := json.Unmarshal(applyRaw, &applied); err != nil || applied.Schema != "mm/v2/apply-receipt" || applied.Outcome != "succeeded" { h.t.Fatalf("invalid apply receipt: %v outcome=%q", err, applied.Outcome) @@ -156,7 +195,9 @@ func (h *liveHarness) assertStageApplyExactlyOnce(kind, target, channelID, userI h.t.Fatalf("apply count mismatch: before=%d after=%d matches=%d", len(before.Order), len(afterApply.Order), countMessage(afterApply, message)) } + writesBeforeReplay := h.mutationSnapshot() replayRaw := h.cli("", "--json", "apply", staged.Stage.StageRef, "--request-id", requestPrefix+"-apply") + h.assertMutationDelta(writesBeforeReplay, nil) var replay applyReceipt if err := json.Unmarshal(replayRaw, &replay); err != nil || replay.AttemptID != applied.AttemptID || replay.Outcome != "succeeded" { h.t.Fatalf("invalid apply replay: %v attempt=%q outcome=%q", err, replay.AttemptID, replay.Outcome) @@ -167,6 +208,27 @@ func (h *liveHarness) assertStageApplyExactlyOnce(kind, target, channelID, userI } } +func (h *liveHarness) mutationSnapshot() map[string]int { + h.t.Helper() + h.mu.Lock() + defer h.mu.Unlock() + return maps.Clone(h.writes) +} + +func (h *liveHarness) assertMutationDelta(before, expected map[string]int) { + h.t.Helper() + after := h.mutationSnapshot() + actual := make(map[string]int) + for key, count := range after { + if delta := count - before[key]; delta != 0 { + actual[key] = delta + } + } + if !maps.Equal(actual, expected) { + h.t.Fatalf("CLI mutation dispatch delta=%v, want %v", actual, expected) + } +} + func createPostID(t *testing.T, receipt applyReceipt) string { t.Helper() for _, step := range receipt.Steps { @@ -235,7 +297,7 @@ func (h *liveHarness) cli(stdin string, args ...string) []byte { "HOME=" + h.home, "XDG_CONFIG_HOME=" + filepath.Join(h.home, ".config"), "XDG_STATE_HOME=" + filepath.Join(h.home, ".local", "state"), - "MM_URL=" + h.url, + "MM_URL=" + h.cliURL, "MM_TOKEN=" + h.token, "PATH=" + os.Getenv("PATH"), "TMPDIR=" + h.home, From 2a542f0d8dcabf3627459dbf3575c44ad1f31434 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 10:24:32 +0300 Subject: [PATCH 093/119] test: verify live conversation creation --- scripts/test-e2e.mjs | 15 ++- tests/e2e/go-conversation-live_test.go | 147 +++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/go-conversation-live_test.go diff --git a/scripts/test-e2e.mjs b/scripts/test-e2e.mjs index 6101302..2f3a3a5 100644 --- a/scripts/test-e2e.mjs +++ b/scripts/test-e2e.mjs @@ -137,7 +137,7 @@ try { '--email-verified', '--disable-welcome-email', ]) - for (const username of ['alice', 'bob']) { + for (const username of ['alice', 'bob', 'carol', 'dave']) { await mmctl([ '--quiet', 'user', @@ -153,7 +153,18 @@ try { ]) } await mmctl(['--quiet', 'team', 'create', '--name', 'e2e', '--display-name', 'E2E']) - await mmctl(['--quiet', 'team', 'users', 'add', 'e2e', 'sender', 'alice', 'bob']) + await mmctl([ + '--quiet', + 'team', + 'users', + 'add', + 'e2e', + 'sender', + 'alice', + 'bob', + 'carol', + 'dave', + ]) await mmctl([ '--quiet', 'team', diff --git a/tests/e2e/go-conversation-live_test.go b/tests/e2e/go-conversation-live_test.go new file mode 100644 index 0000000..85a2dac --- /dev/null +++ b/tests/e2e/go-conversation-live_test.go @@ -0,0 +1,147 @@ +//go:build e2e + +package e2e_test + +import ( + "encoding/json" + "net/http" + "slices" + "sort" + "testing" +) + +func TestGoApplyCreatesExactDirectAndGroupConversationsOnce(t *testing.T) { + h := newLiveHarness(t) + self := h.user("me") + carol := h.user("username/carol") + dave := h.user("username/dave") + tests := []struct { + name, operation, endpoint string + peers []string + args []string + }{ + {"direct", "resolve_dm", "POST /api/v4/channels/direct", []string{carol.ID}, []string{"dm-create", "carol"}}, + {"group", "resolve_group_dm", "POST /api/v4/channels/group", []string{carol.ID, dave.ID}, []string{"group-create", "carol", "dave"}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + requestPrefix := "conversation-" + test.name + beforeChannels := h.conversationIDs(self.ID) + stageArgs := append([]string{"--json", "stage"}, test.args...) + stageArgs = append(stageArgs, "--request-id", requestPrefix+"-stage") + writesBeforeStage := h.mutationSnapshot() + var staged stageReceipt + decodeCLI(t, h.cli("", stageArgs...), &staged) + h.assertMutationDelta(writesBeforeStage, nil) + if staged.Schema != "mm/v2/stage-receipt" || staged.Stage.StageRef == "" { + t.Fatalf("invalid stage receipt: %+v", staged) + } + + writesBeforeApply := h.mutationSnapshot() + var applied applyReceipt + decodeCLI(t, h.cli("", "--json", "apply", staged.Stage.StageRef, "--request-id", requestPrefix+"-apply"), &applied) + h.assertMutationDelta(writesBeforeApply, map[string]int{test.endpoint: 1}) + if applied.Schema != "mm/v2/apply-receipt" || applied.Outcome != "succeeded" { + t.Fatalf("invalid apply receipt: %+v", applied) + } + channelID, receiptPeers := conversationResult(t, applied, test.operation) + wantReceiptPeers := slices.Clone(test.peers) + sort.Strings(wantReceiptPeers) + if !slices.IsSorted(receiptPeers) || !slices.Equal(receiptPeers, wantReceiptPeers) { + t.Fatalf("receipt peers=%v, want canonical %v", receiptPeers, wantReceiptPeers) + } + var created struct { + ID string `json:"id"` + Type string `json:"type"` + } + h.api(http.MethodGet, "/channels/"+channelID, nil, &created) + wantType := "D" + if test.name == "group" { + wantType = "G" + } + if created.ID != channelID || created.Type != wantType { + t.Fatalf("channel=%+v, want id=%q type=%q", created, channelID, wantType) + } + if _, existed := beforeChannels[channelID]; existed { + t.Fatalf("apply resolved pre-existing conversation %q instead of creating it", channelID) + } + if afterType := h.conversationIDs(self.ID)[channelID]; afterType != wantType { + t.Fatalf("new conversation %q absent from live user channels or has type %q", channelID, afterType) + } + var members []struct { + ChannelID string `json:"channel_id"` + UserID string `json:"user_id"` + } + h.api(http.MethodGet, "/channels/"+channelID+"/members?page=0&per_page=9", nil, &members) + gotMembers := make([]string, 0, len(members)) + for _, member := range members { + if member.ChannelID != channelID || member.UserID == "" { + t.Fatalf("invalid member: %+v", member) + } + gotMembers = append(gotMembers, member.UserID) + } + wantMembers := append([]string{self.ID}, test.peers...) + sort.Strings(gotMembers) + sort.Strings(wantMembers) + if !slices.Equal(gotMembers, wantMembers) { + t.Fatalf("live members=%v, want %v", gotMembers, wantMembers) + } + + writesBeforeReplay := h.mutationSnapshot() + var replay applyReceipt + decodeCLI(t, h.cli("", "--json", "apply", staged.Stage.StageRef, "--request-id", requestPrefix+"-apply"), &replay) + h.assertMutationDelta(writesBeforeReplay, nil) + if replay.AttemptID != applied.AttemptID || replay.Outcome != "succeeded" { + t.Fatalf("invalid replay: %+v", replay) + } + }) + } +} + +func (h *liveHarness) conversationIDs(userID string) map[string]string { + h.t.Helper() + var channels []struct { + ID string `json:"id"` + Type string `json:"type"` + } + h.api(http.MethodGet, "/users/"+userID+"/channels", nil, &channels) + result := make(map[string]string) + for _, channel := range channels { + if channel.Type != "D" && channel.Type != "G" { + continue + } + if channel.ID == "" { + h.t.Fatal("live conversation list contains an empty channel id") + } + if _, duplicate := result[channel.ID]; duplicate { + h.t.Fatalf("live conversation list duplicates %q", channel.ID) + } + result[channel.ID] = channel.Type + } + return result +} + +func decodeCLI(t *testing.T, raw []byte, target any) { + t.Helper() + if err := json.Unmarshal(raw, target); err != nil { + t.Fatalf("invalid CLI JSON: %v", err) + } +} + +func conversationResult(t *testing.T, receipt applyReceipt, operation string) (string, []string) { + t.Helper() + for _, step := range receipt.Steps { + if step.Kind != "resolve_conversation" { + continue + } + var result struct { + ChannelID string `json:"channelId"` + ParticipantID []string `json:"participantIds"` + } + if err := json.Unmarshal(step.Result, &result); err == nil && result.ChannelID != "" { + return result.ChannelID, result.ParticipantID + } + } + t.Fatalf("%s receipt has no validated conversation result", operation) + return "", nil +} From 50f780ab803caa605098f2407a0c63de7e64a171 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 10:34:24 +0300 Subject: [PATCH 094/119] test: verify live post lifecycle --- tests/e2e/go-lifecycle-live_test.go | 217 ++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 tests/e2e/go-lifecycle-live_test.go diff --git a/tests/e2e/go-lifecycle-live_test.go b/tests/e2e/go-lifecycle-live_test.go new file mode 100644 index 0000000..eadb02a --- /dev/null +++ b/tests/e2e/go-lifecycle-live_test.go @@ -0,0 +1,217 @@ +//go:build e2e + +package e2e_test + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/stageinput" +) + +func TestGoApplyFullPostLifecycleWithAttachment(t *testing.T) { + h := newLiveHarness(t) + self := h.user("me") + var team struct { + ID string `json:"id"` + } + h.api(http.MethodGet, "/teams/name/e2e", nil, &team) + if team.ID == "" { + t.Fatal("fixture team has no id") + } + channelBody, _ := json.Marshal(map[string]string{ + "team_id": team.ID, "name": "go-lifecycle", "display_name": "Go Lifecycle", "type": "O", + }) + var target channel + h.api(http.MethodPost, "/channels", channelBody, &target) + if target.ID == "" { + t.Fatal("fixture channel has no id") + } + + attachment := []byte("mattermost-cli Go v2 attachment\n\x00exact bytes\n") + attachmentDir, err := os.MkdirTemp(".", ".mm-e2e-attachment-") + if err != nil { + t.Fatal(err) + } + attachmentDir, err = filepath.Abs(attachmentDir) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(attachmentDir) }) + attachmentPath := filepath.Join(attachmentDir, "proof.bin") + if err := os.WriteFile(attachmentPath, attachment, 0o600); err != nil { + t.Fatal(err) + } + if _, err := stageinput.Bind(t.Context(), []stageinput.Attachment{{Path: attachmentPath}}, [][]byte{[]byte(h.token)}); err != nil { + t.Fatalf("attachment fixture is not safely bindable: %v", err) + } + message := "# compound post\n\n- attachment\n- **Markdown**\n" + created := h.stageApply(t, message, "lifecycle-create", + []string{"send", "channel", target.ID, "--attachment", attachmentPath}, + map[string]int{"POST /api/v4/files": 1, "POST /api/v4/posts": 1}) + postID := createPostID(t, created) + fileID := uploadFileID(t, created) + post := h.post(postID) + if post.ChannelID != target.ID || post.UserID != self.ID || post.Message != message || !slices.Equal(post.FileIDs, []string{fileID}) { + t.Fatalf("created post mismatch: %+v", post) + } + if downloaded := h.apiBytes(http.MethodGet, "/files/"+fileID); !bytes.Equal(downloaded, attachment) { + t.Fatalf("downloaded attachment differs: got %d bytes, want %d", len(downloaded), len(attachment)) + } + + replyMessage := "> reply\n\nwith `code`\n" + replied := h.stageApply(t, replyMessage, "lifecycle-reply", []string{"reply", postID}, map[string]int{"POST /api/v4/posts": 1}) + replyID := createPostID(t, replied) + reply := h.post(replyID) + if reply.ChannelID != target.ID || reply.UserID != self.ID || reply.RootID != postID || reply.Message != replyMessage { + t.Fatalf("reply mismatch: %+v", reply) + } + + editedMessage := "# edited\n\nattachment remains\n" + h.stageApply(t, editedMessage, "lifecycle-edit", []string{"post-edit", postID}, map[string]int{"PUT /api/v4/posts/" + postID + "/patch": 1}) + edited := h.post(postID) + if edited.Message != editedMessage || !slices.Equal(edited.FileIDs, []string{fileID}) { + t.Fatalf("edit did not preserve exact content and files: %+v", edited) + } + + h.stageApply(t, "", "lifecycle-react", []string{"react", postID, "eyes"}, map[string]int{"POST /api/v4/reactions": 1}) + if !h.hasReaction(postID, self.ID, "eyes") { + t.Fatal("reaction was not present after apply") + } + h.stageApply(t, "", "lifecycle-unreact", []string{"unreact", postID, "eyes"}, map[string]int{"DELETE /api/v4/users/" + self.ID + "/posts/" + postID + "/reactions/eyes": 1}) + if h.hasReaction(postID, self.ID, "eyes") { + t.Fatal("reaction remained after unreact apply") + } + + h.stageApply(t, "", "lifecycle-delete", []string{"post-delete", replyID}, map[string]int{"DELETE /api/v4/posts/" + replyID: 1}) + if !h.postDeleted(replyID) { + t.Fatal("reply remained live after delete apply") + } +} + +type livePost struct { + ID string `json:"id"` + ChannelID string `json:"channel_id"` + UserID string `json:"user_id"` + RootID string `json:"root_id"` + Message string `json:"message"` + FileIDs []string `json:"file_ids"` + DeleteAt int64 `json:"delete_at"` +} + +func (h *liveHarness) stageApply(t *testing.T, stdin, requestPrefix string, stageArgs []string, expectedWrites map[string]int) applyReceipt { + t.Helper() + args := append([]string{"--json", "stage"}, stageArgs...) + args = append(args, "--request-id", requestPrefix+"-stage") + writesBeforeStage := h.mutationSnapshot() + var staged stageReceipt + decodeCLI(t, h.cli(stdin, args...), &staged) + h.assertMutationDelta(writesBeforeStage, nil) + if staged.Schema != "mm/v2/stage-receipt" || staged.Stage.StageRef == "" { + t.Fatalf("invalid stage receipt: %+v", staged) + } + writesBeforeApply := h.mutationSnapshot() + var applied applyReceipt + decodeCLI(t, h.cli("", "--json", "apply", staged.Stage.StageRef, "--request-id", requestPrefix+"-apply"), &applied) + h.assertMutationDelta(writesBeforeApply, expectedWrites) + if applied.Schema != "mm/v2/apply-receipt" || applied.Outcome != "succeeded" { + t.Fatalf("invalid apply receipt: %+v", applied) + } + return applied +} + +func uploadFileID(t *testing.T, receipt applyReceipt) string { + t.Helper() + for _, step := range receipt.Steps { + if step.Kind != "upload_attachment" { + continue + } + var result struct { + FileID string `json:"fileId"` + } + if err := json.Unmarshal(step.Result, &result); err == nil && result.FileID != "" { + return result.FileID + } + } + t.Fatal("apply receipt has no validated upload result") + return "" +} + +func (h *liveHarness) post(postID string) livePost { + h.t.Helper() + var result livePost + h.api(http.MethodGet, "/posts/"+postID, nil, &result) + if result.ID != postID || result.DeleteAt != 0 { + h.t.Fatalf("invalid live post: %+v", result) + } + return result +} + +func (h *liveHarness) apiBytes(method, path string) []byte { + h.t.Helper() + request, err := http.NewRequest(method, h.url+"/api/v4"+path, nil) + if err != nil { + h.t.Fatal(err) + } + request.Header.Set("Authorization", "Bearer "+h.token) + response, err := h.client.Do(request) + if err != nil { + h.t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode < 200 || response.StatusCode >= 300 { + h.t.Fatalf("Mattermost byte response returned status %d", response.StatusCode) + } + result, err := io.ReadAll(io.LimitReader(response.Body, 4<<20)) + if err != nil { + h.t.Fatal(err) + } + return result +} + +func (h *liveHarness) hasReaction(postID, userID, emoji string) bool { + h.t.Helper() + var reactions []struct { + UserID string `json:"user_id"` + PostID string `json:"post_id"` + EmojiName string `json:"emoji_name"` + } + h.api(http.MethodGet, "/posts/"+postID+"/reactions", nil, &reactions) + for _, reaction := range reactions { + if reaction.UserID == userID && reaction.PostID == postID && reaction.EmojiName == emoji { + return true + } + } + return false +} + +func (h *liveHarness) postDeleted(postID string) bool { + h.t.Helper() + request, err := http.NewRequest(http.MethodGet, h.url+"/api/v4/posts/"+postID, nil) + if err != nil { + h.t.Fatal(err) + } + request.Header.Set("Authorization", "Bearer "+h.token) + response, err := h.client.Do(request) + if err != nil { + h.t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode == http.StatusNotFound { + return true + } + if response.StatusCode != http.StatusOK { + h.t.Fatalf("deleted-post probe returned status %d", response.StatusCode) + } + var result livePost + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&result); err != nil { + h.t.Fatal(err) + } + return result.ID == postID && result.DeleteAt > 0 +} From fa6cfa1ef8c02c15608afc87041fa55df55044a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 10:45:23 +0300 Subject: [PATCH 095/119] test: prove apply journal crash recovery --- internal/stagestore/apply_test.go | 140 ++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/internal/stagestore/apply_test.go b/internal/stagestore/apply_test.go index 6ad501c..d98c118 100644 --- a/internal/stagestore/apply_test.go +++ b/internal/stagestore/apply_test.go @@ -5,9 +5,13 @@ package stagestore import ( "context" "crypto/sha256" + "database/sql" "encoding/hex" "encoding/json" "errors" + "os" + "os/exec" + "path/filepath" "strings" "sync" "testing" @@ -913,4 +917,140 @@ func TestWritableOpenRecoversInterruptedApplyFromJournalEvidence(t *testing.T) { } } +func TestApplyJournalSurvivesProcessDeath(t *testing.T) { + const ( + helperActionEnv = "MM_TEST_CRASH_APPLY_ACTION" + helperPathEnv = "MM_TEST_CRASH_APPLY_PATH" + helperStageEnv = "MM_TEST_CRASH_APPLY_STAGE" + helperExitCode = 91 + requestID = "process-death-request" + ) + if action := os.Getenv(helperActionEnv); action != "" { + path, stageID := os.Getenv(helperPathEnv), os.Getenv(helperStageEnv) + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + detail, err := s.Show(context.Background(), stageID) + if err != nil { + t.Fatal(err) + } + attempt, err := s.ClaimApply(context.Background(), claimInput(detail.StageSummary, requestID, RecoveryModeOrdinary)) + if err != nil { + t.Fatal(err) + } + switch action { + case "claimed": + case "dispatch_intent": + err = s.BeginDispatch(context.Background(), attempt.ID, 1) + case "response_validated": + if err = s.BeginDispatch(context.Background(), attempt.ID, 1); err == nil { + err = s.MarkStepValidated(context.Background(), attempt.ID, 1, createPostResult(t, attempt, "post-after-crash")) + } + default: + t.Fatalf("unknown crash helper action %q", action) + } + if err != nil { + t.Fatal(err) + } + os.Exit(helperExitCode) + } + + for _, tc := range []struct { + action string + persistedStep StepState + wantRecovery Recovery + wantLifecycle Lifecycle + wantOutcome *AttemptOutcome + wantFound bool + wantBody bool + }{ + {action: "claimed", persistedStep: StepPending, wantRecovery: RecoveryNone, wantLifecycle: LifecycleOpen, wantFound: false, wantBody: true}, + {action: "dispatch_intent", persistedStep: StepDispatch, wantRecovery: RecoveryUnknown, wantLifecycle: LifecycleOpen, wantOutcome: outcomePointer(OutcomeUnknown), wantFound: true, wantBody: true}, + {action: "response_validated", persistedStep: StepValidated, wantRecovery: RecoveryForbidden, wantLifecycle: LifecycleCompleted, wantOutcome: outcomePointer(OutcomeSucceeded), wantFound: true, wantBody: false}, + } { + t.Run(tc.action, func(t *testing.T) { + path := testPath(t) + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + created := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) + input := claimInput(created.Stage, requestID, RecoveryModeOrdinary) + if err = s.Close(); err != nil { + t.Fatal(err) + } + + command := exec.Command(os.Args[0], "-test.run=^TestApplyJournalSurvivesProcessDeath$") + command.Env = append(os.Environ(), helperActionEnv+"="+tc.action, helperPathEnv+"="+path, helperStageEnv+"="+created.Stage.ID) + output, runErr := command.CombinedOutput() + var exitErr *exec.ExitError + if !errors.As(runErr, &exitErr) || exitErr.ExitCode() != helperExitCode { + t.Fatalf("crash helper exit=%v, output=%q", runErr, output) + } + assertInterruptedApplyEvidence(t, path, created.Stage.ID, tc.persistedStep) + + s, err = Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + detail, err := s.Show(context.Background(), created.Stage.ID) + if err != nil || detail.Recovery != tc.wantRecovery || detail.Lifecycle != tc.wantLifecycle || (detail.Body != nil) != tc.wantBody { + t.Fatalf("detail=%+v err=%v", detail, err) + } + recovered, found, err := s.FindApply(context.Background(), created.Stage.ServerURL, created.Stage.UserID, requestID, input.RequestDigest) + if err != nil || found != tc.wantFound { + t.Fatalf("found=%v attempt=%+v err=%v", found, recovered, err) + } + if tc.wantOutcome != nil && (recovered.Outcome == nil || *recovered.Outcome != *tc.wantOutcome) { + t.Fatalf("outcome=%v want=%v", recovered.Outcome, *tc.wantOutcome) + } + }) + } +} + +func assertInterruptedApplyEvidence(t *testing.T, path, stageID string, wantStep StepState) { + t.Helper() + probePath := cloneCrashedStore(t, path) + db, err := sql.Open("sqlite", sqliteURI(probePath, false)) + if err != nil { + t.Fatal(err) + } + db.SetMaxOpenConns(1) + defer db.Close() + var lifecycle, recovery, step string + var outcomePending, claimedEvents, requests int + err = db.QueryRow(`SELECT s.lifecycle,s.recovery,a.outcome IS NULL,p.state, + (SELECT COUNT(*) FROM apply_events e WHERE e.attempt_id=a.id AND e.event='claimed'), + (SELECT COUNT(*) FROM apply_requests r WHERE r.attempt_id=a.id) + FROM stages s JOIN apply_attempts a ON a.id=s.claim_attempt_id + JOIN apply_steps p ON p.attempt_id=a.id AND p.ordinal=1 WHERE s.id=?`, stageID). + Scan(&lifecycle, &recovery, &outcomePending, &step, &claimedEvents, &requests) + if err != nil || lifecycle != string(LifecycleApplying) || recovery != string(RecoveryNone) || outcomePending != 1 || step != string(wantStep) || claimedEvents != 1 || requests != 1 { + t.Fatalf("persisted interrupted apply lifecycle=%q recovery=%q pending=%d step=%q events=%d requests=%d err=%v", lifecycle, recovery, outcomePending, step, claimedEvents, requests, err) + } +} + +func cloneCrashedStore(t *testing.T, path string) string { + t.Helper() + clone := filepath.Join(t.TempDir(), "state", "mattermost-cli", DatabaseFilename) + if err := os.MkdirAll(filepath.Dir(clone), 0o700); err != nil { + t.Fatal(err) + } + for _, suffix := range []string{"", "-wal", "-shm", "-journal"} { + data, err := os.ReadFile(path + suffix) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + t.Fatal(err) + } + if err = os.WriteFile(clone+suffix, data, 0o600); err != nil { + t.Fatal(err) + } + } + return clone +} + func outcomePointer(value AttemptOutcome) *AttemptOutcome { return &value } From fea6af9df93c5d57bf8ac7810d02cd26f837adbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 10:54:50 +0300 Subject: [PATCH 096/119] test: verify live unknown outcome recovery --- tests/e2e/go-live_test.go | 107 +++++++++++++++++++++++++++--- tests/e2e/go-unknown-live_test.go | 62 +++++++++++++++++ 2 files changed, 160 insertions(+), 9 deletions(-) create mode 100644 tests/e2e/go-unknown-live_test.go diff --git a/tests/e2e/go-live_test.go b/tests/e2e/go-live_test.go index 7f9ab39..5ed202a 100644 --- a/tests/e2e/go-live_test.go +++ b/tests/e2e/go-live_test.go @@ -5,6 +5,7 @@ package e2e_test import ( "bytes" "encoding/json" + "errors" "fmt" "io" "maps" @@ -33,6 +34,13 @@ type liveHarness struct { client *http.Client mu sync.Mutex writes map[string]int + crash *responseCrash +} + +type responseCrash struct { + key string + process <-chan *os.Process + result chan<- error } type stageReceipt struct { @@ -43,11 +51,15 @@ type stageReceipt struct { } type applyReceipt struct { - Schema string `json:"schema"` - AttemptID string `json:"attemptId"` - Outcome string `json:"outcome"` - Steps []struct { + Schema string `json:"schema"` + AttemptID string `json:"attemptId"` + RecoveryMode string `json:"recoveryMode"` + ForcedDuplicateRisk bool `json:"forcedDuplicateRisk"` + Outcome string `json:"outcome"` + Recovery string `json:"recovery"` + Steps []struct { Kind string `json:"kind"` + State string `json:"state"` Result json.RawMessage `json:"result"` } `json:"steps"` } @@ -102,6 +114,26 @@ func newLiveHarness(t *testing.T) *liveHarness { proxyTransport := &http.Transport{Proxy: nil} proxy := httputil.NewSingleHostReverseProxy(parsed) proxy.Transport = proxyTransport + proxy.ModifyResponse = func(response *http.Response) error { + key := response.Request.Method + " " + response.Request.URL.Path + harness.mu.Lock() + crash := harness.crash + if crash != nil && crash.key == key { + harness.crash = nil + } else { + crash = nil + } + harness.mu.Unlock() + if crash != nil { + select { + case process := <-crash.process: + crash.result <- process.Kill() + case <-time.After(5 * time.Second): + crash.result <- errors.New("timed out waiting for the CLI process") + } + } + return nil + } server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { if request.Method != http.MethodGet && request.Method != http.MethodHead && request.Method != http.MethodOptions { harness.mu.Lock() @@ -290,6 +322,32 @@ func (h *liveHarness) api(method, path string, body []byte, target any) { } func (h *liveHarness) cli(stdin string, args ...string) []byte { + h.t.Helper() + stdout, stderr, code := h.cliResult(stdin, args...) + if code != 0 { + h.t.Fatalf("mm %s exited %d, stderr=%s", strings.Join(args, " "), code, fmt.Sprintf("%q", stderr)) + } + if len(stderr) != 0 { + h.t.Fatalf("mm %s emitted stderr on success: %q", strings.Join(args, " "), stderr) + } + return stdout +} + +func (h *liveHarness) cliResult(stdin string, args ...string) ([]byte, []byte, int) { + h.t.Helper() + command, stdout, stderr := h.cliCommand(stdin, args...) + err := command.Run() + if err == nil { + return stdout.Bytes(), stderr.Bytes(), 0 + } + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + h.t.Fatalf("mm %s failed to run: %v", strings.Join(args, " "), err) + } + return stdout.Bytes(), stderr.Bytes(), exitErr.ExitCode() +} + +func (h *liveHarness) cliCommand(stdin string, args ...string) (*exec.Cmd, *bytes.Buffer, *bytes.Buffer) { h.t.Helper() command := exec.Command(h.binary, args...) command.Stdin = strings.NewReader(stdin) @@ -309,11 +367,42 @@ func (h *liveHarness) cli(stdin string, args ...string) []byte { } var stdout, stderr bytes.Buffer command.Stdout, command.Stderr = &stdout, &stderr - if err := command.Run(); err != nil { - h.t.Fatalf("mm %s failed: %v, stderr=%s", strings.Join(args, " "), err, fmt.Sprintf("%q", stderr.String())) + return command, &stdout, &stderr +} + +func (h *liveHarness) armCrashAfterResponse(key string) (chan<- *os.Process, <-chan error) { + h.t.Helper() + process := make(chan *os.Process, 1) + result := make(chan error, 1) + h.mu.Lock() + defer h.mu.Unlock() + if h.crash != nil { + h.t.Fatal("response crash is already armed") + } + h.crash = &responseCrash{key: key, process: process, result: result} + return process, result +} + +func (h *liveHarness) cliKilledAfterResponse(stdin, key string, args ...string) ([]byte, []byte) { + h.t.Helper() + process, crashResult := h.armCrashAfterResponse(key) + command, stdout, stderr := h.cliCommand(stdin, args...) + if err := command.Start(); err != nil { + h.t.Fatal(err) + } + process <- command.Process + runErr := command.Wait() + select { + case crashErr := <-crashResult: + if crashErr != nil { + h.t.Fatalf("could not terminate CLI after accepted response: %v", crashErr) + } + case <-time.After(5 * time.Second): + h.t.Fatal("accepted response did not trigger the armed CLI crash") } - if stderr.Len() != 0 { - h.t.Fatalf("mm %s emitted stderr on success: %q", strings.Join(args, " "), stderr.String()) + var exitErr *exec.ExitError + if !errors.As(runErr, &exitErr) || exitErr.ExitCode() != -1 { + h.t.Fatalf("CLI was not killed after accepted response: %v", runErr) } - return stdout.Bytes() + return stdout.Bytes(), stderr.Bytes() } diff --git a/tests/e2e/go-unknown-live_test.go b/tests/e2e/go-unknown-live_test.go new file mode 100644 index 0000000..e04dc88 --- /dev/null +++ b/tests/e2e/go-unknown-live_test.go @@ -0,0 +1,62 @@ +//go:build e2e + +package e2e_test + +import "testing" + +func TestGoApplyCrashAfterServerAcceptanceRequiresExplicitForce(t *testing.T) { + h := newLiveHarness(t) + self := h.user("me") + alice := h.user("username/alice") + channel := h.createChannel("direct", []string{self.ID, alice.ID}) + message := "# accepted before crash\n\nthis may already exist exactly once\n" + before := h.posts(channel.ID) + + writesBeforeStage := h.mutationSnapshot() + var staged stageReceipt + decodeCLI(t, h.cli(message, "--json", "stage", "send", "dm", "alice", "--request-id", "crash-stage"), &staged) + h.assertMutationDelta(writesBeforeStage, nil) + if staged.Schema != "mm/v2/stage-receipt" || staged.Stage.StageRef == "" { + t.Fatalf("invalid stage receipt: %+v", staged) + } + + writesBeforeCrash := h.mutationSnapshot() + stdout, stderr := h.cliKilledAfterResponse("", "POST /api/v4/posts", "--json", "apply", staged.Stage.StageRef, "--request-id", "crash-apply") + h.assertMutationDelta(writesBeforeCrash, map[string]int{"POST /api/v4/posts": 1}) + if len(stdout) != 0 || len(stderr) != 0 { + t.Fatalf("killed CLI emitted output: stdout=%q stderr=%q", stdout, stderr) + } + afterCrash := h.posts(channel.ID) + if countMessage(afterCrash, message) != 1 || len(afterCrash.Order) != len(before.Order)+1 { + t.Fatalf("accepted crash mutation mismatch: before=%d after=%d matches=%d", len(before.Order), len(afterCrash.Order), countMessage(afterCrash, message)) + } + + writesBeforeReplay := h.mutationSnapshot() + replayRaw, replayStderr, replayCode := h.cliResult("", "--json", "apply", staged.Stage.StageRef, "--request-id", "crash-apply") + h.assertMutationDelta(writesBeforeReplay, nil) + var replay applyReceipt + decodeCLI(t, replayRaw, &replay) + if replayCode != 5 || len(replayStderr) != 0 || replay.Schema != "mm/v2/apply-receipt" || replay.AttemptID == "" || replay.Outcome != "unknown" || replay.Recovery != "force_unknown" || replay.RecoveryMode != "ordinary" || replay.ForcedDuplicateRisk || + len(replay.Steps) != 1 || replay.Steps[0].Kind != "create_post" || replay.Steps[0].State != "outcome_unknown" || string(replay.Steps[0].Result) != "null" { + t.Fatalf("invalid recovered replay: code=%d stderr=%q receipt=%+v", replayCode, replayStderr, replay) + } + if afterReplay := h.posts(channel.ID); countMessage(afterReplay, message) != 1 || len(afterReplay.Order) != len(afterCrash.Order) { + t.Fatalf("unknown replay dispatched again: after-crash=%d after-replay=%d matches=%d", len(afterCrash.Order), len(afterReplay.Order), countMessage(afterReplay, message)) + } + + writesBeforeForce := h.mutationSnapshot() + forcedRaw, forcedStderr, forcedCode := h.cliResult("", "--json", "apply", staged.Stage.StageRef, "--force-unknown", "--request-id", "crash-force") + h.assertMutationDelta(writesBeforeForce, map[string]int{"POST /api/v4/posts": 1}) + var forced applyReceipt + decodeCLI(t, forcedRaw, &forced) + const forceWarning = "warning: forcing an unknown stage may duplicate a real Mattermost side effect; inspect the destination first\n" + if forcedCode != 0 || string(forcedStderr) != forceWarning || forced.Schema != "mm/v2/apply-receipt" || forced.AttemptID == "" || forced.AttemptID == replay.AttemptID || forced.Outcome != "succeeded" || forced.Recovery != "forbidden" || forced.RecoveryMode != "force_unknown" || !forced.ForcedDuplicateRisk || + len(forced.Steps) != 1 || forced.Steps[0].Kind != "create_post" || forced.Steps[0].State != "response_validated" { + t.Fatalf("invalid forced apply: code=%d stderr=%q receipt=%+v", forcedCode, forcedStderr, forced) + } + forcedPostID := createPostID(t, forced) + afterForce := h.posts(channel.ID) + if _, existed := afterCrash.Posts[forcedPostID]; existed || afterForce.Posts[forcedPostID].Message != message || countMessage(afterForce, message) != 2 || len(afterForce.Order) != len(before.Order)+2 { + t.Fatalf("explicit force did not expose duplicate risk: before=%d after=%d matches=%d", len(before.Order), len(afterForce.Order), countMessage(afterForce, message)) + } +} From a39cdca529aed308c0a0f434748edc817aef924f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 11:06:20 +0300 Subject: [PATCH 097/119] test: verify concurrent apply exclusion --- internal/stagestore/lock_hook_e2e.go | 16 +++++ internal/stagestore/lock_hook_unix.go | 5 ++ internal/stagestore/secure_unix.go | 1 + scripts/test-e2e.mjs | 2 +- tests/e2e/go-concurrent-live_test.go | 88 +++++++++++++++++++++++++++ tests/e2e/go-live_test.go | 38 ++++++++++++ 6 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 internal/stagestore/lock_hook_e2e.go create mode 100644 internal/stagestore/lock_hook_unix.go create mode 100644 tests/e2e/go-concurrent-live_test.go diff --git a/internal/stagestore/lock_hook_e2e.go b/internal/stagestore/lock_hook_e2e.go new file mode 100644 index 0000000..78b3bc3 --- /dev/null +++ b/internal/stagestore/lock_hook_e2e.go @@ -0,0 +1,16 @@ +//go:build (darwin || linux) && e2e + +package stagestore + +import "os" + +func notifyFlockContention() { + path := os.Getenv("MM_E2E_LOCK_CONTENTION_MARKER") + if path == "" { + return + } + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err == nil { + _ = file.Close() + } +} diff --git a/internal/stagestore/lock_hook_unix.go b/internal/stagestore/lock_hook_unix.go new file mode 100644 index 0000000..70e061a --- /dev/null +++ b/internal/stagestore/lock_hook_unix.go @@ -0,0 +1,5 @@ +//go:build (darwin || linux) && !e2e + +package stagestore + +func notifyFlockContention() {} diff --git a/internal/stagestore/secure_unix.go b/internal/stagestore/secure_unix.go index 980ed53..3850bca 100644 --- a/internal/stagestore/secure_unix.go +++ b/internal/stagestore/secure_unix.go @@ -384,6 +384,7 @@ func flockContext(ctx context.Context, fd int) error { if !errors.Is(err, unix.EWOULDBLOCK) || !time.Now().Before(deadline) { return ErrBusy } + notifyFlockContention() timer := time.NewTimer(10 * time.Millisecond) select { case <-ctx.Done(): diff --git a/scripts/test-e2e.mjs b/scripts/test-e2e.mjs index 2f3a3a5..5ff4ad2 100644 --- a/scripts/test-e2e.mjs +++ b/scripts/test-e2e.mjs @@ -186,7 +186,7 @@ try { await run('bunx', ['vitest', 'run', '--config', 'vitest.e2e.config.ts'], { env: { MM_E2E_URL: url, MM_E2E_TOKEN: token }, }) - await run('go', ['build', '-o', goBinary, './cmd/mm']) + await run('go', ['build', '-tags=e2e', '-o', goBinary, './cmd/mm']) await run('go', ['test', '-tags=e2e', '-count=1', './tests/e2e'], { env: { MM_E2E_URL: url, diff --git a/tests/e2e/go-concurrent-live_test.go b/tests/e2e/go-concurrent-live_test.go new file mode 100644 index 0000000..20f129c --- /dev/null +++ b/tests/e2e/go-concurrent-live_test.go @@ -0,0 +1,88 @@ +//go:build e2e + +package e2e_test + +import ( + "errors" + "os" + "path/filepath" + "testing" + "time" +) + +func TestGoConcurrentApplyProcessesDispatchExactlyOnce(t *testing.T) { + h := newLiveHarness(t) + self := h.user("me") + alice := h.user("username/alice") + channel := h.createChannel("direct", []string{self.ID, alice.ID}) + message := "# concurrent apply\n\nonly one process may dispatch\n" + before := h.posts(channel.ID) + + writesBeforeStage := h.mutationSnapshot() + var staged stageReceipt + decodeCLI(t, h.cli(message, "--json", "stage", "send", "dm", "alice", "--request-id", "concurrent-stage"), &staged) + h.assertMutationDelta(writesBeforeStage, nil) + if staged.Schema != "mm/v2/stage-receipt" || staged.Stage.StageRef == "" { + t.Fatalf("invalid stage receipt: %+v", staged) + } + + arrived, release := h.armRequestGate("POST /api/v4/posts") + first, firstStdout, firstStderr := h.cliCommand("", "--json", "apply", staged.Stage.StageRef, "--request-id", "concurrent-apply") + contentionMarker := filepath.Join(h.home, "second-apply-contended") + second, secondStdout, secondStderr := h.cliCommandWithEnv("", []string{"MM_E2E_LOCK_CONTENTION_MARKER=" + contentionMarker}, "--json", "apply", staged.Stage.StageRef, "--request-id", "concurrent-apply") + writesBeforeApply := h.mutationSnapshot() + if err := first.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = first.Process.Kill() }) + select { + case <-arrived: + case <-time.After(5 * time.Second): + t.Fatal("first apply did not reach the gated mutation dispatch") + } + if err := second.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = second.Process.Kill() }) + secondDone := make(chan error, 1) + go func() { secondDone <- second.Wait() }() + deadline := time.Now().Add(5 * time.Second) + for { + if _, err := os.Stat(contentionMarker); err == nil { + break + } else if !errors.Is(err, os.ErrNotExist) { + t.Fatal(err) + } + if !time.Now().Before(deadline) { + t.Fatal("second apply never contended on the winner's live store lock") + } + select { + case err := <-secondDone: + t.Fatalf("second apply exited before lock contention: %v", err) + case <-time.After(10 * time.Millisecond): + } + } + close(release) + firstErr, secondErr := first.Wait(), <-secondDone + if firstErr != nil || secondErr != nil || firstStderr.Len() != 0 || secondStderr.Len() != 0 { + t.Fatalf("concurrent apply failed: first=%v/%q second=%v/%q", firstErr, firstStderr.String(), secondErr, secondStderr.String()) + } + h.assertMutationDelta(writesBeforeApply, map[string]int{"POST /api/v4/posts": 1}) + + var firstReceipt, secondReceipt applyReceipt + decodeCLI(t, firstStdout.Bytes(), &firstReceipt) + decodeCLI(t, secondStdout.Bytes(), &secondReceipt) + if firstStdout.String() != secondStdout.String() || firstReceipt.Schema != "mm/v2/apply-receipt" || firstReceipt.AttemptID == "" || firstReceipt.AttemptID != secondReceipt.AttemptID || secondReceipt.Schema != firstReceipt.Schema || + firstReceipt.Outcome != "succeeded" || secondReceipt.Outcome != "succeeded" || firstReceipt.RecoveryMode != "ordinary" || secondReceipt.RecoveryMode != "ordinary" || firstReceipt.ForcedDuplicateRisk || secondReceipt.ForcedDuplicateRisk || + firstReceipt.Recovery != "forbidden" || secondReceipt.Recovery != "forbidden" || len(firstReceipt.Steps) != 1 || len(secondReceipt.Steps) != 1 || firstReceipt.Steps[0].Kind != "create_post" || secondReceipt.Steps[0].Kind != "create_post" || firstReceipt.Steps[0].State != "response_validated" || secondReceipt.Steps[0].State != "response_validated" { + t.Fatalf("concurrent receipts diverged: first=%+v second=%+v", firstReceipt, secondReceipt) + } + firstPostID, secondPostID := createPostID(t, firstReceipt), createPostID(t, secondReceipt) + if firstPostID != secondPostID { + t.Fatalf("concurrent receipts bind different posts: %q != %q", firstPostID, secondPostID) + } + after := h.posts(channel.ID) + if after.Posts[firstPostID].Message != message || countMessage(after, message) != 1 || len(after.Order) != len(before.Order)+1 { + t.Fatalf("concurrent apply server state mismatch: before=%d after=%d matches=%d", len(before.Order), len(after.Order), countMessage(after, message)) + } +} diff --git a/tests/e2e/go-live_test.go b/tests/e2e/go-live_test.go index 5ed202a..cb9f172 100644 --- a/tests/e2e/go-live_test.go +++ b/tests/e2e/go-live_test.go @@ -35,6 +35,7 @@ type liveHarness struct { mu sync.Mutex writes map[string]int crash *responseCrash + gate *requestGate } type responseCrash struct { @@ -43,6 +44,12 @@ type responseCrash struct { result chan<- error } +type requestGate struct { + key string + arrived chan<- struct{} + release <-chan struct{} +} + type stageReceipt struct { Schema string `json:"schema"` Stage struct { @@ -135,11 +142,24 @@ func newLiveHarness(t *testing.T) *liveHarness { return nil } server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + var gate *requestGate if request.Method != http.MethodGet && request.Method != http.MethodHead && request.Method != http.MethodOptions { harness.mu.Lock() harness.writes[request.Method+" "+request.URL.Path]++ + if harness.gate != nil && harness.gate.key == request.Method+" "+request.URL.Path { + gate, harness.gate = harness.gate, nil + } harness.mu.Unlock() } + if gate != nil { + gate.arrived <- struct{}{} + select { + case <-gate.release: + case <-time.After(5 * time.Second): + http.Error(response, "E2E mutation gate timed out", http.StatusGatewayTimeout) + return + } + } proxy.ServeHTTP(response, request) })) harness.cliURL = server.URL @@ -348,6 +368,10 @@ func (h *liveHarness) cliResult(stdin string, args ...string) ([]byte, []byte, i } func (h *liveHarness) cliCommand(stdin string, args ...string) (*exec.Cmd, *bytes.Buffer, *bytes.Buffer) { + return h.cliCommandWithEnv(stdin, nil, args...) +} + +func (h *liveHarness) cliCommandWithEnv(stdin string, extraEnv []string, args ...string) (*exec.Cmd, *bytes.Buffer, *bytes.Buffer) { h.t.Helper() command := exec.Command(h.binary, args...) command.Stdin = strings.NewReader(stdin) @@ -365,6 +389,7 @@ func (h *liveHarness) cliCommand(stdin string, args ...string) (*exec.Cmd, *byte "NO_COLOR=1", "TERM=dumb", } + command.Env = append(command.Env, extraEnv...) var stdout, stderr bytes.Buffer command.Stdout, command.Stderr = &stdout, &stderr return command, &stdout, &stderr @@ -383,6 +408,19 @@ func (h *liveHarness) armCrashAfterResponse(key string) (chan<- *os.Process, <-c return process, result } +func (h *liveHarness) armRequestGate(key string) (<-chan struct{}, chan<- struct{}) { + h.t.Helper() + arrived := make(chan struct{}, 1) + release := make(chan struct{}) + h.mu.Lock() + defer h.mu.Unlock() + if h.gate != nil { + h.t.Fatal("mutation request gate is already armed") + } + h.gate = &requestGate{key: key, arrived: arrived, release: release} + return arrived, release +} + func (h *liveHarness) cliKilledAfterResponse(stdin, key string, args ...string) ([]byte, []byte) { h.t.Helper() process, crashResult := h.armCrashAfterResponse(key) From 09fe69c919d4c2e38704d5e4fc79b8df5d952cac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 11:17:24 +0300 Subject: [PATCH 098/119] test: verify live read workflows --- tests/e2e/go-read-live_test.go | 174 +++++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 tests/e2e/go-read-live_test.go diff --git a/tests/e2e/go-read-live_test.go b/tests/e2e/go-read-live_test.go new file mode 100644 index 0000000..ad0eda7 --- /dev/null +++ b/tests/e2e/go-read-live_test.go @@ -0,0 +1,174 @@ +//go:build e2e + +package e2e_test + +import ( + "bytes" + "encoding/json" + "net/http" + "testing" + + "github.com/ardasevinc/mattermost-cli/internal/output" + "github.com/ardasevinc/mattermost-cli/internal/schema" +) + +func TestGoRealServerChannelCursorSearchAndThreadReads(t *testing.T) { + h := newLiveHarness(t) + self := h.user("me") + var team struct { + ID string `json:"id"` + } + h.api(http.MethodGet, "/teams/name/e2e", nil, &team) + channelBody, _ := json.Marshal(map[string]string{ + "team_id": team.ID, "name": "go-read-acceptance", "display_name": "Go Read Acceptance", "type": "O", + }) + var target channel + h.api(http.MethodPost, "/channels", channelBody, &target) + baseline := h.posts(target.ID) + expected := make(map[string]struct{ text, rootID, userID string }, len(baseline.Order)+4) + for _, postID := range baseline.Order { + post := h.post(postID) + expected[post.ID] = struct{ text, rootID, userID string }{post.Message, post.RootID, post.UserID} + } + + first := h.createLivePost(target.ID, "", "read acceptance first") + second := h.createLivePost(target.ID, "", "read acceptance second") + root := h.createLivePost(target.ID, "", "needle-cursor-thread-root") + reply := h.createLivePost(target.ID, root.ID, "> exact reply\n\nwith `code`\n") + expected[first.ID] = struct{ text, rootID, userID string }{"read acceptance first", "", self.ID} + expected[second.ID] = struct{ text, rootID, userID string }{"read acceptance second", "", self.ID} + expected[root.ID] = struct{ text, rootID, userID string }{"needle-cursor-thread-root", "", self.ID} + expected[reply.ID] = struct{ text, rootID, userID string }{"> exact reply\n\nwith `code`\n", root.ID, self.ID} + + firstPageRaw := h.cli("", "--json", "--no-threads", "channel", "go-read-acceptance", "--team", "e2e", "--limit", "2") + validateLiveDocument(t, "mm/v2/channel", firstPageRaw) + var firstPage output.ChannelEnvelope + decodeCLI(t, firstPageRaw, &firstPage) + if firstPage.Schema != "mm/v2/channel" || firstPage.Data.Channel.ID != target.ID || firstPage.Data.Metadata.Completeness != output.MachineUnknown || firstPage.Data.Metadata.Selection.QueryTruncated != nil || firstPage.Data.Metadata.Selection.NextCursor == nil || len(firstPage.Data.Messages) != 2 { + t.Fatalf("invalid first channel page: %+v", firstPage) + } + + pages := []output.ChannelEnvelope{firstPage} + cursor := *firstPage.Data.Metadata.Selection.NextCursor + for len(pages) < 5 { + pageRaw := h.cli("", "--json", "--no-threads", "channel", "go-read-acceptance", "--team", "e2e", "--limit", "2", "--cursor", cursor) + validateLiveDocument(t, "mm/v2/channel", pageRaw) + var page output.ChannelEnvelope + decodeCLI(t, pageRaw, &page) + if page.Data.Metadata.Selection.InputCursor == nil || *page.Data.Metadata.Selection.InputCursor != cursor || page.Data.Channel.ID != target.ID { + t.Fatalf("invalid resumed channel page: %+v", page) + } + pages = append(pages, page) + if page.Data.Metadata.Selection.NextCursor == nil { + break + } + if *page.Data.Metadata.Selection.NextCursor == cursor { + t.Fatal("live channel cursor did not advance") + } + cursor = *page.Data.Metadata.Selection.NextCursor + } + lastPage := pages[len(pages)-1] + if lastPage.Data.Metadata.Completeness != output.MachineComplete || lastPage.Data.Metadata.Selection.NextCursor != nil { + t.Fatalf("cursor pagination did not reach proven completion: %+v", lastPage.Data.Metadata) + } + seen := make(map[string]bool) + for _, page := range pages { + assertLiveReadChannel(t, page.Data.Channel, target.ID) + if page.Data.Metadata.Selection.SelectedCount != len(page.Data.Messages) || page.Data.Metadata.VisiblePostCount != len(page.Data.Messages) || page.Data.Metadata.VisibleThreads.Status != "not_requested" { + t.Fatalf("channel page metadata does not bind emitted messages: %+v", page.Data.Metadata) + } + for _, message := range page.Data.Messages { + if seen[message.ID] { + t.Fatalf("cursor pages duplicated post %q", message.ID) + } + want, ok := expected[message.ID] + if !ok || len(message.Replies) != 0 { + t.Fatalf("cursor pages emitted an unexpected or nested post: %+v", message) + } + assertLiveReadMessage(t, message, message.ID, want.text, want.rootID, want.userID) + seen[message.ID] = true + } + } + if len(seen) != len(expected) { + t.Fatalf("cursor pages did not emit the exact fixture set: seen=%v expected=%v", seen, expected) + } + + threadRaw := h.cli("", "--json", "thread", reply.ID) + validateLiveDocument(t, "mm/v2/thread", threadRaw) + var thread output.ThreadEnvelope + decodeCLI(t, threadRaw, &thread) + if thread.Data.Root == nil { + t.Fatalf("live thread omitted its root: %+v", thread) + } + assertLiveReadChannel(t, thread.Data.Channel, target.ID) + assertLiveReadMessage(t, *thread.Data.Root, root.ID, "needle-cursor-thread-root", "", self.ID) + if len(thread.Data.Root.Replies) != 1 { + t.Fatalf("live thread has the wrong reply shape: %+v", thread.Data.Root.Replies) + } + assertLiveReadMessage(t, thread.Data.Root.Replies[0], reply.ID, "> exact reply\n\nwith `code`\n", root.ID, self.ID) + if len(thread.Data.UnboundPosts) != 0 || thread.Data.Metadata.Completeness != output.MachineComplete || thread.Data.Metadata.Selection.SelectedCount != 2 || thread.Data.Metadata.VisiblePostCount != 2 || thread.Data.Metadata.VisibleThreads.Status != "complete" || thread.Data.Metadata.VisibleThreads.HydratedRootCount != 1 || len(thread.Data.Metadata.VisibleThreads.FailedRootIDs) != 0 { + t.Fatalf("invalid live thread: %+v", thread) + } + + searchRaw := h.cli("", "--json", "search", "needle-cursor-thread-root", "--team", "e2e", "--limit", "5") + validateLiveDocument(t, "mm/v2/search", searchRaw) + var search output.SearchEnvelope + decodeCLI(t, searchRaw, &search) + if len(search.Results) != 1 { + t.Fatalf("invalid live search result: %+v", search) + } + result := search.Results[0] + assertLiveReadChannel(t, result.Channel, target.ID) + if len(result.Messages) != 1 || len(result.Messages[0].Replies) != 1 { + t.Fatalf("search did not return exactly one hydrated thread: %+v", result.Messages) + } + assertLiveReadMessage(t, result.Messages[0], root.ID, "needle-cursor-thread-root", "", self.ID) + assertLiveReadMessage(t, result.Messages[0].Replies[0], reply.ID, "> exact reply\n\nwith `code`\n", root.ID, self.ID) + if result.Metadata.Completeness != output.MachineComplete || result.Metadata.Selection.SelectedCount != 1 || result.Metadata.VisiblePostCount != 2 || result.Metadata.VisibleThreads.Status != "complete" || result.Metadata.VisibleThreads.HydratedRootCount != 1 || len(result.Metadata.VisibleThreads.FailedRootIDs) != 0 { + t.Fatalf("search metadata does not prove exact complete hydration: %+v", result.Metadata) + } +} + +func (h *liveHarness) createLivePost(channelID, rootID, message string) livePost { + h.t.Helper() + body := map[string]string{"channel_id": channelID, "message": message} + if rootID != "" { + body["root_id"] = rootID + } + raw, err := json.Marshal(body) + if err != nil { + h.t.Fatal(err) + } + var post livePost + h.api(http.MethodPost, "/posts", raw, &post) + if post.ID == "" || post.ChannelID != channelID || post.RootID != rootID || post.Message != message { + h.t.Fatalf("invalid created fixture post: %+v", post) + } + return post +} + +func validateLiveDocument(t *testing.T, schemaID string, raw []byte) { + t.Helper() + registry, err := schema.Load() + if err != nil { + t.Fatal(err) + } + if err = registry.Validate(schemaID, bytes.NewReader(raw)); err != nil { + t.Fatalf("%s output failed schema validation: %v", schemaID, err) + } +} + +func assertLiveReadChannel(t *testing.T, channel output.MachineChannel, id string) { + t.Helper() + if channel.ID != id || channel.Type != "public" || channel.Name != "go-read-acceptance" || channel.DisplayName != "Go Read Acceptance" || channel.MetadataStatus != "resolved" { + t.Fatalf("live read channel identity mismatch: %+v", channel) + } +} + +func assertLiveReadMessage(t *testing.T, message output.MachineMessage, id, text, rootID, userID string) { + t.Helper() + rootMatches := rootID == "" && message.RootID == nil || rootID != "" && message.RootID != nil && *message.RootID == rootID + if message.ID != id || message.Text != text || message.UserID != userID || !rootMatches { + t.Fatalf("live read message mismatch: %+v", message) + } +} From 45df9718a459e47dde84f1dbac19c78ccc7a06c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 11:33:54 +0300 Subject: [PATCH 099/119] test: verify live watch events --- internal/mattermost/watch.go | 11 +- internal/mattermost/watch_ready.go | 5 + internal/mattermost/watch_ready_e2e.go | 16 +++ internal/mattermost/watch_test.go | 36 ++++++ tests/e2e/go-lifecycle-live_test.go | 1 + tests/e2e/go-watch-live_test.go | 162 +++++++++++++++++++++++++ 6 files changed, 228 insertions(+), 3 deletions(-) create mode 100644 internal/mattermost/watch_ready.go create mode 100644 internal/mattermost/watch_ready_e2e.go create mode 100644 tests/e2e/go-watch-live_test.go diff --git a/internal/mattermost/watch.go b/internal/mattermost/watch.go index 9233af0..25fd97f 100644 --- a/internal/mattermost/watch.go +++ b/internal/mattermost/watch.go @@ -274,6 +274,12 @@ func runConnection(ctx context.Context, options WatchOptions, state *watchState, pongDeadline := time.Time{} var pendingPing int64 = -1 stable := false + markReady := func() { + if state.authenticated && state.hello && heartbeatAt.IsZero() { + heartbeatAt = options.Now().Add(options.HeartbeatInterval) + notifyWatchReady() + } + } for { deadline := handshakeDeadline if state.authenticated && state.hello { @@ -350,6 +356,7 @@ func runConnection(ctx context.Context, options WatchOptions, state *watchState, heartbeatAt = options.Now().Add(options.HeartbeatInterval) } } + markReady() if frame.Event == "" { continue } @@ -395,9 +402,7 @@ func runConnection(ctx context.Context, options WatchOptions, state *watchState, } state.nextServer++ } - if state.authenticated && state.hello && heartbeatAt.IsZero() { - heartbeatAt = options.Now().Add(options.HeartbeatInterval) - } + markReady() if !state.authenticated || !state.hello || frame.Event != "posted" { continue } diff --git a/internal/mattermost/watch_ready.go b/internal/mattermost/watch_ready.go new file mode 100644 index 0000000..93831d1 --- /dev/null +++ b/internal/mattermost/watch_ready.go @@ -0,0 +1,5 @@ +//go:build !e2e + +package mattermost + +func notifyWatchReady() {} diff --git a/internal/mattermost/watch_ready_e2e.go b/internal/mattermost/watch_ready_e2e.go new file mode 100644 index 0000000..d27d017 --- /dev/null +++ b/internal/mattermost/watch_ready_e2e.go @@ -0,0 +1,16 @@ +//go:build e2e + +package mattermost + +import "os" + +func notifyWatchReady() { + path := os.Getenv("MM_E2E_WATCH_READY_MARKER") + if path == "" { + return + } + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err == nil { + _ = file.Close() + } +} diff --git a/internal/mattermost/watch_test.go b/internal/mattermost/watch_test.go index 397c235..9032208 100644 --- a/internal/mattermost/watch_test.go +++ b/internal/mattermost/watch_test.go @@ -143,6 +143,42 @@ func TestWatchAuthWireInterleavingDuplicateSequenceAndCancellation(t *testing.T) t.Fatalf("auth=%s", wire) } } +func TestWatchHelloBeforeAuthSchedulesFullHeartbeatInterval(t *testing.T) { + socket := newFakeSocket() + socket.reads <- readResult{type_: transport.MessageText, data: []byte(`{"event":"hello","seq":0,"data":{"connection_id":"one"}}`)} + socket.reads <- readResult{type_: transport.MessageText, data: []byte(`{"status":"OK","seq_reply":1}`)} + now := time.Unix(100, 0).UTC() + durations := make(chan time.Duration, 4) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- Watch(ctx, WatchOptions{ + URL: "https://mm.example.com", Token: "secret", Sink: &recordingSink{}, + HandshakeTimeout: 30 * time.Second, HeartbeatInterval: 10 * time.Second, + Now: func() time.Time { return now }, + NewTimer: func(duration time.Duration) WatchTimer { + durations <- duration + return instantTimer{make(chan time.Time)} + }, + Dial: func(context.Context, string) (transport.WebSocket, error) { return socket, nil }, + }) + }() + want := []time.Duration{30 * time.Second, 30 * time.Second, 10 * time.Second} + for i, expected := range want { + select { + case got := <-durations: + if got != expected { + t.Fatalf("timer %d duration=%s, want %s", i, got, expected) + } + case <-time.After(time.Second): + t.Fatalf("timer %d was not scheduled", i) + } + } + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatal(err) + } +} func TestWatchSinkErrorIsTerminalWithoutReconnect(t *testing.T) { socket := newFakeSocket() sink := &recordingSink{err: errors.New("hostile")} diff --git a/tests/e2e/go-lifecycle-live_test.go b/tests/e2e/go-lifecycle-live_test.go index eadb02a..6d738c9 100644 --- a/tests/e2e/go-lifecycle-live_test.go +++ b/tests/e2e/go-lifecycle-live_test.go @@ -102,6 +102,7 @@ type livePost struct { RootID string `json:"root_id"` Message string `json:"message"` FileIDs []string `json:"file_ids"` + CreateAt int64 `json:"create_at"` DeleteAt int64 `json:"delete_at"` } diff --git a/tests/e2e/go-watch-live_test.go b/tests/e2e/go-watch-live_test.go new file mode 100644 index 0000000..91fa84e --- /dev/null +++ b/tests/e2e/go-watch-live_test.go @@ -0,0 +1,162 @@ +//go:build e2e + +package e2e_test + +import ( + "bufio" + "encoding/json" + "io" + "net/http" + "os" + "path/filepath" + "testing" + "time" +) + +type liveWatchEvent struct { + Schema string `json:"schema"` + Type string `json:"type"` + Sequence struct { + ConnectionID string `json:"connectionId"` + Number int64 `json:"number"` + } `json:"sequence"` + PostID string `json:"postId"` + ChannelID string `json:"channelId"` + ChannelName string `json:"channelName"` + SenderID string `json:"senderId"` + Sender string `json:"sender"` + Message string `json:"message"` + Timestamp string `json:"timestamp"` + RootID *string `json:"rootId"` + FileIDs []string `json:"fileIds"` + Redactions []json.RawMessage `json:"redactions"` +} + +func TestGoRealServerWatchEmitsExactPostedEventAndStopsCleanly(t *testing.T) { + h := newLiveHarness(t) + var self struct { + ID string `json:"id"` + Username string `json:"username"` + } + h.api(http.MethodGet, "/users/me", nil, &self) + var team struct { + ID string `json:"id"` + } + h.api(http.MethodGet, "/teams/name/e2e", nil, &team) + channelBody, _ := json.Marshal(map[string]string{ + "team_id": team.ID, "name": "go-watch-acceptance", "display_name": "Go Watch Acceptance", "type": "O", + }) + var target channel + h.api(http.MethodPost, "/channels", channelBody, &target) + + writesBefore := h.mutationSnapshot() + readyMarker := filepath.Join(h.home, "watch-ready") + command, _, stderr := h.cliCommandWithEnv("", []string{"MM_E2E_WATCH_READY_MARKER=" + readyMarker}, "--json", "watch", "go-watch-acceptance", "--team", "e2e") + command.Stdout = nil + stdout, err := command.StdoutPipe() + if err != nil { + t.Fatal(err) + } + if err = command.Start(); err != nil { + t.Fatal(err) + } + finished := false + t.Cleanup(func() { + if !finished && command.Process != nil { + _ = command.Process.Kill() + _ = command.Wait() + } + }) + + deadline := time.Now().Add(5 * time.Second) + for { + if info, statErr := os.Stat(readyMarker); statErr == nil { + if info.Mode().Perm() != 0o600 || info.Size() != 0 { + t.Fatalf("invalid watch readiness marker: mode=%o size=%d", info.Mode().Perm(), info.Size()) + } + break + } else if !os.IsNotExist(statErr) { + t.Fatal(statErr) + } + if time.Now().After(deadline) { + t.Fatal("watch did not authenticate and receive Mattermost hello") + } + time.Sleep(10 * time.Millisecond) + } + + message := "# live watch\n\n- **exact** markdown\n- `code`\n" + created := h.createLivePost(target.ID, "", message) + lineResult := make(chan []byte, 1) + lineError := make(chan error, 1) + reader := bufio.NewReader(stdout) + go func() { + line, readErr := reader.ReadBytes('\n') + if readErr != nil { + lineError <- readErr + return + } + lineResult <- line + }() + + var line []byte + select { + case line = <-lineResult: + case readErr := <-lineError: + t.Fatalf("watch event read failed: %v", readErr) + case <-time.After(5 * time.Second): + t.Fatal("watch did not emit the live post") + } + validateLiveDocument(t, "mm/v2/watch-event", line) + var event liveWatchEvent + if err = json.Unmarshal(line, &event); err != nil { + t.Fatal(err) + } + expectedTimestamp := time.UnixMilli(created.CreateAt).UTC().Format("2006-01-02T15:04:05.000Z") + if event.Schema != "mm/v2/watch-event" || event.Type != "posted" || event.Sequence.ConnectionID == "" || event.Sequence.Number < 0 || event.PostID != created.ID || event.ChannelID != target.ID || event.ChannelName != "go-watch-acceptance" || event.SenderID != self.ID || event.Sender != self.Username || event.Message != message || event.Timestamp != expectedTimestamp || event.RootID != nil || len(event.FileIDs) != 0 || len(event.Redactions) != 0 { + t.Fatalf("live watch event mismatch: %+v", event) + } + + if err = command.Process.Signal(os.Interrupt); err != nil { + t.Fatal(err) + } + type drainResult struct { + remaining []byte + err error + } + drained := make(chan drainResult, 1) + go func() { + remaining, readErr := io.ReadAll(reader) + drained <- drainResult{remaining, readErr} + }() + var remaining []byte + select { + case result := <-drained: + remaining, err = result.remaining, result.err + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + _ = command.Process.Kill() + _ = command.Wait() + finished = true + t.Fatal("watch did not close stdout within 5s after SIGINT") + } + waited := make(chan error, 1) + go func() { waited <- command.Wait() }() + select { + case err = <-waited: + finished = true + if err != nil { + t.Fatalf("watch did not stop cleanly after SIGINT: %v, stderr=%q", err, stderr.Bytes()) + } + case <-time.After(5 * time.Second): + _ = command.Process.Kill() + err = <-waited + finished = true + t.Fatalf("watch process did not exit within 5s after SIGINT: %v", err) + } + if len(remaining) != 0 || stderr.Len() != 0 { + t.Fatalf("watch emitted unexpected trailing output: stdout=%q stderr=%q", remaining, stderr.Bytes()) + } + h.assertMutationDelta(writesBefore, nil) +} From 804514e77c25859d540e2e0f7f14f40f0c1496e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 11:46:05 +0300 Subject: [PATCH 100/119] docs: lock retention lifecycle contract --- docs/V2_CONTRACT.md | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/docs/V2_CONTRACT.md b/docs/V2_CONTRACT.md index eece071..b12878e 100644 --- a/docs/V2_CONTRACT.md +++ b/docs/V2_CONTRACT.md @@ -97,6 +97,15 @@ When the selected XDG path differs from the v1 path, migration behavior is manda v2 never silently moves, rewrites, merges, or deletes the legacy file. `config --init` and all explicit writes target the selected v2 path. +Retention policy uses two TOML keys with integer-second values: + +```toml +stage_ttl_seconds = 0 +stage_prune_after_seconds = 0 +``` + +Both values must be non-negative integers. Zero disables the corresponding policy. A wrong type, negative value, or value that cannot be represented safely is a configuration error. TTL has no daemon: a positive policy is enforced opportunistically after store recovery whenever a writable stage operation opens the database. Read-only `stage list` and `stage show` never mutate local state. A bulk prune requires either a positive configured prune age or an explicit positive `--older-than` duration; zero never means prune everything. + State path: - `$XDG_STATE_HOME/mattermost-cli` when `XDG_STATE_HOME` is set and absolute; @@ -158,7 +167,8 @@ mm stage show mm stage revise mm stage revise --revive mm stage cancel -mm stage prune +mm stage prune [--older-than ] +mm stage prune @ [--abandon-recovery] [--request-id ] mm apply @ mm apply @ --resume-partial @@ -354,6 +364,8 @@ Every stage also has an aggregate recovery requirement derived monotonically fro `force_unknown` dominates `resume_partial`, which dominates `none`. A later rejected attempt never erases uncertainty from an earlier attempt. Only confirmed completion or explicit lifecycle closure changes recovery to `forbidden`. +The store separately tracks retained recovery material as `none`, `resume_partial`, or `force_unknown`. Public recovery answers whether and how a stage may be applied; retained recovery material answers whether pruning would deliberately destroy evidence or inputs needed for partial/unknown recovery. Cancel changes public recovery to `forbidden` but does not erase retained recovery material. Confirmed completion clears it. Explicit recovery abandonment clears it only after an append-only audit tombstone is durable. + Normal transitions are: | Event | Lifecycle after event | Attempt outcome | Recovery requirement | @@ -399,9 +411,15 @@ After confirmed success: Rejected, partial, unknown, and unapplied stages retain the minimum content required for inspection and deliberate recovery until eligible cleanup. -TTL is configurable. Its default is `0`, meaning stages never expire automatically. Enabling TTL is explicit. Age is measured from the latest revision or attempt activity. Automatic expiry applies only to inactive open stages with recovery `none`; it never races an applying claim or expires a recovery-eligible stage. Expiry changes lifecycle eligibility but preserves prior attempt outcomes and does not pretend secure erasure. +TTL is configurable and disabled by default. Age is measured from `stages.updated_at`, which advances on revisions and attempt activity. Opportunistic expiry applies only to inactive open stages whose public recovery and retained recovery material are both `none`; it never races an applying claim or expires a recovery-eligible stage. Expiry changes lifecycle eligibility, retains staged content, records an append-only retention event, preserves prior attempt outcomes, and does not pretend secure erasure. Reviving an expired stage records a corresponding retention event. + +Cancel is refused while applying. On an open stage it revokes future apply without rewriting history or discarding retained recovery material. + +Bulk `stage prune` is an explicitly human maintenance action. It atomically selects only completed, canceled, and expired stages older than one fixed explicit or configured cutoff, with no claim and no retained recovery material. It has no caller request ID, emits a bounded count-only result, and fails closed if no positive age exists. Structured machine prune is always exact and replayable. + +Exact `stage prune @` bypasses the age threshold because the exact reference is deliberate intent, but still refuses applying or claimed stages and requires the current revision and semantic digest to remain unchanged. Ordinary exact prune accepts only completed, canceled, or expired stages with no retained recovery material. Removing retained staged content and source bindings from a stage with `resume_partial` or `force_unknown` material requires the exact reference plus `--abandon-recovery`; this makes the stage `pruned`, public recovery `forbidden`, clears the retained-material marker, and preserves an append-only audit tombstone stating that recovery was deliberately abandoned. No durable attachment spool exists outside SQLite source bindings. -Cancel is refused while applying. On an open stage it revokes future apply without rewriting history. `stage prune` defaults to completed, canceled, and expired stages older than an explicit or configured age. It refuses applying and recovery-eligible stages. Removing content or retained spools from a `resume_partial` or `force_unknown` stage requires an exact stage reference plus `--abandon-recovery`; this makes the stage `pruned`, recovery `forbidden`, and preserves an audit tombstone stating that recovery material was deliberately destroyed. +Every exact structured prune uses `mm/v2/stage-prune-request`, includes caller request ID, stage ID, expected revision, expected digest, and the abandonment choice, and reuses the stored stage's server/user replay scope. Its narrow result reuses `mm/v2/stage-receipt` with action `pruned`. Bulk human prune emits `mm/v2/stage-prune-result` with the fixed cutoff, pruned count, and recorded timestamp; it never emits an unbounded stage list. ## 15. Receipts, errors, and output failure @@ -480,7 +498,7 @@ Required layers: - transition and race tests for show-revise-apply, revise/cancel/prune versus applying, known-partial resume, uncertainty-bearing expiry/cancel, and stale revisions; - recovery-history tests for unknown then forced rejection, revise after partial/unknown, definitively rejected suffix resume, and the impossibility of clearing aggregate uncertainty through a later outcome; - adversarial attachment tests for in-place writes during spool creation, inode/path replacement, truncation, symlink swaps, and active-credential bytes; -- spool recovery tests for pre-dispatch crash cleanup, stale applying retention, successful cleanup, and explicit recovery abandonment; +- execution-spool tests for pre-dispatch crash cleanup, stale applying retention, and successful cleanup, plus SQLite source-binding tests for explicit recovery abandonment; - lost-output idempotency tests for stage creation and revision as well as apply receipts; - security fixture parity for every v1 secret pattern and hostile presentation input; - artifact tests against the exact released archives and npm platform packages. From 47978161cb102c87341f8962cf0867d227757711 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 12:12:59 +0300 Subject: [PATCH 101/119] feat: add audited stage retention lifecycle --- internal/cli/apply.go | 2 +- internal/cli/config.go | 11 +- internal/cli/root_test.go | 2 +- internal/cli/runtime.go | 2 + internal/cli/stage_create.go | 12 + internal/cli/stage_inspect.go | 4 +- internal/cli/stage_manage.go | 201 ++++++++++++- internal/cli/stage_manage_test.go | 95 ++++++ internal/cli/store.go | 27 ++ internal/cli/store_test.go | 6 +- internal/config/config.go | 71 +++-- internal/config/config_test.go | 30 +- internal/output/machine.go | 30 +- internal/stageoutput/output.go | 17 +- internal/stageoutput/output_test.go | 13 + internal/stagerequest/request.go | 22 ++ internal/stagerequest/request_test.go | 30 ++ internal/stagestore/apply.go | 8 +- internal/stagestore/domain.go | 14 +- internal/stagestore/domain_test.go | 3 +- internal/stagestore/retention.go | 286 +++++++++++++++++++ internal/stagestore/retention_test.go | 262 +++++++++++++++++ internal/stagestore/schema.go | 109 +++++++ schemas/v2/config.schema.json | 10 +- schemas/v2/examples/config.json | 2 +- schemas/v2/examples/stage-prune-request.json | 1 + schemas/v2/examples/stage-prune-result.json | 1 + schemas/v2/examples/store-doctor.json | 2 +- schemas/v2/examples/store-migrations.json | 5 +- schemas/v2/stage-prune-request.schema.json | 24 ++ schemas/v2/stage-prune-result.schema.json | 21 ++ schemas/v2/stage-receipt.schema.json | 22 +- schemas/v2/store-doctor.schema.json | 6 +- schemas/v2/store-migrations.schema.json | 6 +- 34 files changed, 1292 insertions(+), 65 deletions(-) create mode 100644 internal/stagestore/retention.go create mode 100644 internal/stagestore/retention_test.go create mode 100644 schemas/v2/examples/stage-prune-request.json create mode 100644 schemas/v2/examples/stage-prune-result.json create mode 100644 schemas/v2/stage-prune-request.schema.json create mode 100644 schemas/v2/stage-prune-result.schema.json diff --git a/internal/cli/apply.go b/internal/cli/apply.go index 9ca1903..fa503df 100644 --- a/internal/cli/apply.go +++ b/internal/cli/apply.go @@ -63,7 +63,7 @@ func newApplyCommand(state *rootState) *cobra.Command { } else if flagChanged(cmd, "request-id") && !applyRequestIDPattern.MatchString(requestID) { return invalidFailure("invalid --request-id") } - return nil + return resolveStageOptions(state, cmd) }, } command.Flags().BoolVar(&fromJSON, "from-json", false, "read one versioned apply request from stdin") diff --git a/internal/cli/config.go b/internal/cli/config.go index 9ccd101..250365c 100644 --- a/internal/cli/config.go +++ b/internal/cli/config.go @@ -1,6 +1,7 @@ package cli import ( + "strconv" "strings" "github.com/spf13/cobra" @@ -92,6 +93,11 @@ func configMachineStatus(action string, file config.FileState, created *bool) ou if file.InsecurePermissions { permissions = "insecure" } + var stageTTLSeconds, stagePruneAfterSeconds *int64 + if file.Error != config.FileErrorParse && file.Error != config.FileErrorRead { + stageTTLSeconds = pointer(file.Config.StageTTLSeconds) + stagePruneAfterSeconds = pointer(file.Config.StagePruneAfterSeconds) + } var readPath *string if file.Exists { readPathValue := file.ReadPath @@ -109,7 +115,8 @@ func configMachineStatus(action string, file config.FileState, created *bool) ou return output.ConfigEnvelope{ Schema: "mm/v2/config", Action: action, SelectedPath: file.SelectedPath, ReadPath: readPath, Migration: pointer(migration), Exists: pointer(file.Exists), URLConfigured: pointer(file.Config.URL != ""), - TokenConfigured: pointer(file.Config.Token != ""), Permissions: pointer(permissions), ReadStatus: pointer(readStatus), + TokenConfigured: pointer(file.Config.Token != ""), StageTTLSeconds: stageTTLSeconds, + StagePruneAfterSeconds: stagePruneAfterSeconds, Permissions: pointer(permissions), ReadStatus: pointer(readStatus), ParseStatus: pointer(parseStatus), UnsafeReason: unsafeReason, Created: created, Warning: warning, } } @@ -175,6 +182,8 @@ func writeConfigHuman(state *rootState, status output.ConfigEnvelope) error { "Exists: " + yesNo(valueOr(status.Exists, false)), "URL configured: " + yesNo(valueOr(status.URLConfigured, false)), "Token configured: " + yesNo(valueOr(status.TokenConfigured, false)), + "Stage TTL seconds: " + strconv.FormatInt(valueOr(status.StageTTLSeconds, int64(0)), 10), + "Stage prune after seconds: " + strconv.FormatInt(valueOr(status.StagePruneAfterSeconds, int64(0)), 10), "Permissions: " + valueOr(status.Permissions, "not_applicable"), "Read status: " + valueOr(status.ReadStatus, "not_attempted"), "Parse status: " + valueOr(status.ParseStatus, "not_attempted"), diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index 0e2a44f..f8bdc9d 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -57,7 +57,7 @@ func TestSchemaList(t *testing.T) { if code != 0 { t.Fatalf("exit code = %d, want 0; stderr = %q", code, stderr.String()) } - if got, want := stdout.String(), "mm/v2/apply-receipt\nmm/v2/apply-request\nmm/v2/channel\nmm/v2/channels\nmm/v2/config\nmm/v2/dms\nmm/v2/doctor\nmm/v2/error\nmm/v2/group-dms\nmm/v2/mentions\nmm/v2/search\nmm/v2/stage\nmm/v2/stage-cancel-request\nmm/v2/stage-preview\nmm/v2/stage-receipt\nmm/v2/stage-request\nmm/v2/stage-revise-request\nmm/v2/stages\nmm/v2/store-doctor\nmm/v2/store-migrations\nmm/v2/teams\nmm/v2/thread\nmm/v2/unread\nmm/v2/users\nmm/v2/watch-diagnostic\nmm/v2/watch-event\nmm/v2/whoami\n"; got != want { + if got, want := stdout.String(), "mm/v2/apply-receipt\nmm/v2/apply-request\nmm/v2/channel\nmm/v2/channels\nmm/v2/config\nmm/v2/dms\nmm/v2/doctor\nmm/v2/error\nmm/v2/group-dms\nmm/v2/mentions\nmm/v2/search\nmm/v2/stage\nmm/v2/stage-cancel-request\nmm/v2/stage-preview\nmm/v2/stage-prune-request\nmm/v2/stage-prune-result\nmm/v2/stage-receipt\nmm/v2/stage-request\nmm/v2/stage-revise-request\nmm/v2/stages\nmm/v2/store-doctor\nmm/v2/store-migrations\nmm/v2/teams\nmm/v2/thread\nmm/v2/unread\nmm/v2/users\nmm/v2/watch-diagnostic\nmm/v2/watch-event\nmm/v2/whoami\n"; got != want { t.Fatalf("stdout = %q, want %q", got, want) } } diff --git a/internal/cli/runtime.go b/internal/cli/runtime.go index f20dd8c..6f91fe0 100644 --- a/internal/cli/runtime.go +++ b/internal/cli/runtime.go @@ -74,6 +74,8 @@ type rootState struct { pendingWarnings []machineWarning semanticExit int disableHeuristics bool + stageTTLSeconds int64 + stagePruneSeconds int64 } type runtimeFlags struct { diff --git a/internal/cli/stage_create.go b/internal/cli/stage_create.go index 761875c..ae22417 100644 --- a/internal/cli/stage_create.go +++ b/internal/cli/stage_create.go @@ -301,6 +301,10 @@ func openStagingService(cmd *cobra.Command, state *rootState, persist bool) (*st if err != nil { return nil, func() error { return nil }, localStateFailure{fmt.Errorf("could not safely open stage store")} } + if err = expireConfiguredStages(cmd, state, store); err != nil { + _ = store.Close() + return nil, func() error { return nil }, err + } storeDependency = store } closeStore := func() error { @@ -552,6 +556,14 @@ func classifyStageError(err error) error { return readFailure(errors.New("stage operation canceled before persistence")) } switch { + case errors.Is(err, stagestore.ErrInvalid): + return invalidFailure("invalid stage request") + case errors.Is(err, stagestore.ErrConflict): + return localStateFailure{errors.New("stage request conflicts with durable local state")} + case errors.Is(err, stagestore.ErrNotFound): + return localStateFailure{errors.New("stage not found")} + case errors.Is(err, stagestore.ErrNotEligible): + return localStateFailure{errors.New("stage lifecycle transition is not allowed")} case errors.Is(err, staging.ErrInvalid): return invalidFailure("invalid stage request") case errors.Is(err, staging.ErrInput): diff --git a/internal/cli/stage_inspect.go b/internal/cli/stage_inspect.go index ebb0462..99f3c8b 100644 --- a/internal/cli/stage_inspect.go +++ b/internal/cli/stage_inspect.go @@ -42,11 +42,11 @@ func newStageCommand(state *rootState) *cobra.Command { command.PersistentPreRunE = func(cmd *cobra.Command, _ []string) error { if fromJSON { state.flags.json = true - if cmd != command && cmd.Name() != "revise" && cmd.Name() != "cancel" { + if cmd != command && cmd.Name() != "revise" && cmd.Name() != "cancel" && cmd.Name() != "prune" { return invalidFailure("--from-json cannot be combined with a stage subcommand") } } - return resolveStoreRedaction(state, cmd) + return resolveStageOptions(state, cmd) } command.PersistentFlags().BoolVar(&fromJSON, "from-json", false, "read one versioned stage request from stdin") command.AddCommand(newStageListCommand(state), newStageShowCommand(state)) diff --git a/internal/cli/stage_manage.go b/internal/cli/stage_manage.go index e32496e..7b9ea02 100644 --- a/internal/cli/stage_manage.go +++ b/internal/cli/stage_manage.go @@ -2,10 +2,14 @@ package cli import ( "bytes" + "context" + "encoding/hex" "errors" "fmt" "io/fs" "os" + "strings" + "time" "github.com/spf13/cobra" @@ -19,7 +23,7 @@ import ( ) func newStageManagementCommands(state *rootState, fromJSON *bool) []*cobra.Command { - return []*cobra.Command{newStageReviseCommand(state, fromJSON), newStageCancelCommand(state, fromJSON)} + return []*cobra.Command{newStageReviseCommand(state, fromJSON), newStageCancelCommand(state, fromJSON), newStagePruneCommand(state, fromJSON)} } func newStageReviseCommand(state *rootState, fromJSON *bool) *cobra.Command { @@ -103,6 +107,186 @@ func newStageCancelCommand(state *rootState, fromJSON *bool) *cobra.Command { return command } +func newStagePruneCommand(state *rootState, fromJSON *bool) *cobra.Command { + var olderThan, requestID string + var abandonRecovery bool + command := &cobra.Command{Use: "prune [@]", Short: "Erase eligible retained stage content"} + command.Args = func(_ *cobra.Command, args []string) error { + if *fromJSON { + if len(args) != 0 { + return invalidFailure("--from-json cannot be combined with a stage reference") + } + return nil + } + if len(args) > 1 { + return invalidFailure("stage prune accepts at most one exact stage reference") + } + return nil + } + command.Flags().StringVar(&olderThan, "older-than", "", "prune terminal stages older than a Go duration such as 720h") + command.Flags().StringVar(&requestID, "request-id", "", "caller-generated replay key for exact prune") + command.Flags().BoolVar(&abandonRecovery, "abandon-recovery", false, "deliberately destroy partial or unknown recovery material") + command.RunE = func(cmd *cobra.Command, args []string) error { + if *fromJSON { + if anyFlagChanged(cmd, "older-than", "request-id", "abandon-recovery") { + return invalidFailure("structured prune cannot be combined with human prune flags") + } + decoder, err := stagerequest.NewDecoder() + if err != nil { + return internalFailure(err) + } + request, err := decoder.DecodePrune(state.streams.in) + if err != nil { + return classifyStageRequestDecode(err, "prune") + } + input, err := request.PruneInput() + if err != nil || pruneInputContainsCredential(input, state.credentials) { + return invalidFailure("invalid stage prune request") + } + return executeExactPrune(cmd, state, input) + } + if len(args) == 0 { + if flagChanged(cmd, "request-id") || abandonRecovery { + return invalidFailure("--request-id and --abandon-recovery require an exact stage reference") + } + age := time.Duration(state.stagePruneSeconds) * time.Second + if flagChanged(cmd, "older-than") { + var err error + age, err = time.ParseDuration(olderThan) + if err != nil || age < time.Second || age%time.Second != 0 { + return invalidFailure("--older-than must be a positive whole-second Go duration") + } + } + if age <= 0 { + return invalidFailure("bulk prune requires --older-than or positive stage_prune_after_seconds") + } + return executeBulkPrune(cmd, state, age) + } + if flagChanged(cmd, "older-than") { + return invalidFailure("--older-than cannot be combined with an exact stage reference") + } + if requestID != "" && !applyRequestIDPattern.MatchString(requestID) { + return invalidFailure("invalid --request-id") + } + stageID, revision, err := parseStageReference(args[0]) + if err != nil { + return err + } + store, err := openExistingStageStore(cmd, state) + if err != nil { + return err + } + detail, err := store.Show(cmd.Context(), stageID) + if err != nil { + _ = store.Close() + return classifyStageError(err) + } + if detail.Revision != revision { + _ = store.Close() + return localStateFailure{errors.New("stage revision changed")} + } + input := stagestore.PruneInput{StageID: stageID, RequestID: requestID, ExpectedRevision: revision, ExpectedDigest: detail.SemanticDigest, AbandonRecovery: abandonRecovery} + if pruneInputContainsCredential(input, state.credentials) { + _ = store.Close() + return invalidFailure("invalid stage prune request") + } + return executeExactPruneWithStore(cmd, state, store, detail.Destination, input) + } + return command +} + +func executeExactPrune(cmd *cobra.Command, state *rootState, input stagestore.PruneInput) error { + store, err := openExistingStageStore(cmd, state) + if err != nil { + return err + } + detail, err := store.Show(cmd.Context(), input.StageID) + if err != nil { + _ = store.Close() + return classifyStageError(err) + } + return executeExactPruneWithStore(cmd, state, store, detail.Destination, input) +} + +func executeExactPruneWithStore(cmd *cobra.Command, state *rootState, store *stagestore.Store, destination []byte, input stagestore.PruneInput) error { + result, operationErr := store.Prune(cmd.Context(), input) + if closeErr := store.Close(); closeErr != nil { + return localStateFailure{errors.New("could not close stage store safely")} + } + if operationErr != nil { + return classifyRetentionError(operationErr) + } + document, err := stageoutput.NewReceipt(result, destination, state.credentials) + if err != nil { + return localStateFailure{errors.New("stored stage receipt is invalid")} + } + if state.flags.json { + return writeStageJSON(state, document) + } + return writeAll(state.streams.out, []byte(document.Action+": "+safeStoreValue(state, document.Stage.StageRef)+"\n")) +} + +func executeBulkPrune(cmd *cobra.Command, state *rootState, age time.Duration) error { + paths, err := storePaths(state) + if err != nil { + return err + } + now := time.Now().UTC() + result := stagestore.BulkPruneResult{Schema: "mm/v2/stage-prune-result", Action: "pruned", Cutoff: now.Add(-age), RecordedAt: now} + if _, err = os.Stat(paths.DBPath); errors.Is(err, fs.ErrNotExist) { + return writeBulkPruneResult(state, result) + } else if err != nil { + return localStateFailure{errors.New("could not inspect stage store")} + } + store, err := openExistingStageStore(cmd, state) + if err != nil { + return err + } + result, err = store.PruneEligible(cmd.Context(), result.Cutoff, result.RecordedAt) + if closeErr := store.Close(); closeErr != nil { + return localStateFailure{errors.New("could not close stage store safely")} + } + if err != nil { + return classifyRetentionError(err) + } + return writeBulkPruneResult(state, result) +} + +func classifyRetentionError(err error) error { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, stagestore.ErrInvalid) || errors.Is(err, stagestore.ErrConflict) || + errors.Is(err, stagestore.ErrNotFound) || errors.Is(err, stagestore.ErrNotEligible) { + return classifyStageError(err) + } + return localStateFailure{errors.New("could not persist stage retention state")} +} + +func writeBulkPruneResult(state *rootState, result stagestore.BulkPruneResult) error { + document, err := stageoutput.NewPruneResult(result, state.credentials) + if err != nil { + return localStateFailure{errors.New("stored prune result is invalid")} + } + if state.flags.json { + return writeStageJSON(state, document) + } + return writeAll(state.streams.out, []byte(fmt.Sprintf("pruned: %d stages older than %s\n", document.PrunedCount, safeStoreValue(state, document.Cutoff)))) +} + +func pruneInputContainsCredential(input stagestore.PruneInput, credentials []string) bool { + values := []string{input.StageID, input.RequestID, hex.EncodeToString(input.ExpectedDigest[:])} + for _, credential := range credentials { + if credential == "" { + continue + } + for _, value := range values { + if strings.Contains(value, credential) { + return true + } + } + } + return false +} + func anyFlagChanged(command *cobra.Command, names ...string) bool { for _, name := range names { if command.Flags().Changed(name) { @@ -151,9 +335,24 @@ func openExistingStageStore(cmd *cobra.Command, state *rootState) (*stagestore.S if err != nil { return nil, localStateFailure{errors.New("could not safely open stage store")} } + if err = expireConfiguredStages(cmd, state, store); err != nil { + _ = store.Close() + return nil, err + } return store, nil } +func expireConfiguredStages(cmd *cobra.Command, state *rootState, store *stagestore.Store) error { + if state.stageTTLSeconds == 0 { + return nil + } + now := time.Now().UTC() + if _, err := store.ExpireEligible(cmd.Context(), now.Add(-time.Duration(state.stageTTLSeconds)*time.Second), now); err != nil { + return localStateFailure{errors.New("could not apply stage retention policy")} + } + return nil +} + func executeStageRevise(cmd *cobra.Command, state *rootState, input staging.ReviseInput) error { store, err := openExistingStageStore(cmd, state) if err != nil { diff --git a/internal/cli/stage_manage_test.go b/internal/cli/stage_manage_test.go index 07fd816..9aa7e06 100644 --- a/internal/cli/stage_manage_test.go +++ b/internal/cli/stage_manage_test.go @@ -6,9 +6,11 @@ import ( "bytes" "encoding/hex" "encoding/json" + "os" "path/filepath" "strings" "testing" + "time" mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" "github.com/ardasevinc/mattermost-cli/internal/stagestore" @@ -127,6 +129,7 @@ func TestStageMutationFromJSONRejectsHumanFlagMixingAsMachineError(t *testing.T) for _, args := range [][]string{ {"stage", "revise", "--from-json", "--message", "nope"}, {"stage", "cancel", "--from-json", "--request-id", "nope"}, + {"stage", "prune", "--from-json", "--older-than", "720h"}, } { var stdout, stderr bytes.Buffer code := Execute(t.Context(), args, strings.NewReader(`{}`), &stdout, &stderr) @@ -135,3 +138,95 @@ func TestStageMutationFromJSONRejectsHumanFlagMixingAsMachineError(t *testing.T) } } } + +func TestStructuredExactPruneIsReplayableAndErasesRetainedContent(t *testing.T) { + home, stateRoot := t.TempDir(), filepath.Join(t.TempDir(), "state") + stageID := createInspectionStage(t, home, stateRoot, "# retained\n\n**markdown**") + setOfflineStageEnvironment(t, stateRoot) + var stdout, stderr bytes.Buffer + if code := Execute(t.Context(), []string{"stage", "cancel", stageID}, nil, &stdout, &stderr); code != 0 { + t.Fatalf("cancel exit=%d stderr=%q", code, stderr.String()) + } + canceled := loadStoredStage(t, stateRoot, stageID) + request := map[string]any{ + "schema": "mm/v2/stage-prune-request", "requestId": "prune-structured-1", "stageId": stageID, + "expectedRevision": canceled.Revision, "expectedDigest": hex.EncodeToString(canceled.SemanticDigest[:]), "abandonRecovery": false, + } + raw, err := json.Marshal(request) + if err != nil { + t.Fatal(err) + } + for attempt := 0; attempt < 2; attempt++ { + stdout.Reset() + stderr.Reset() + code := Execute(t.Context(), []string{"stage", "prune", "--from-json"}, bytes.NewReader(raw), &stdout, &stderr) + if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), `"action":"pruned"`) || !strings.Contains(stdout.String(), `"replayed":`+[]string{"false", "true"}[attempt]) { + t.Fatalf("attempt=%d exit=%d stdout=%q stderr=%q", attempt, code, stdout.String(), stderr.String()) + } + } + pruned := loadStoredStage(t, stateRoot, stageID) + if pruned.Lifecycle != stagestore.LifecyclePruned || pruned.Body != nil || len(pruned.Attachments) != 0 { + t.Fatalf("pruned=%+v body=%q attachments=%d", pruned.StageSummary, pruned.Body, len(pruned.Attachments)) + } +} + +func TestBulkPruneFailsClosedWithoutAgeAndUsesConfiguredAge(t *testing.T) { + home, stateRoot, configRoot := t.TempDir(), filepath.Join(t.TempDir(), "state"), filepath.Join(t.TempDir(), "config") + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", configRoot) + t.Setenv("XDG_STATE_HOME", stateRoot) + t.Setenv("MM_URL", "") + t.Setenv("MM_TOKEN", "") + var stdout, stderr bytes.Buffer + code := Execute(t.Context(), []string{"stage", "prune"}, nil, &stdout, &stderr) + if code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "bulk prune requires") { + t.Fatalf("disabled exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + configPath := filepath.Join(configRoot, "mattermost-cli", "config.toml") + if err := os.MkdirAll(filepath.Dir(configPath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(configPath, []byte("stage_prune_after_seconds = 3600\n"), 0o600); err != nil { + t.Fatal(err) + } + stdout.Reset() + stderr.Reset() + code = Execute(t.Context(), []string{"--json", "stage", "prune"}, nil, &stdout, &stderr) + if code != 0 || stderr.Len() != 0 || !strings.Contains(stdout.String(), `"schema":"mm/v2/stage-prune-result"`) || !strings.Contains(stdout.String(), `"prunedCount":0`) { + t.Fatalf("configured exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } +} + +func TestConfiguredTTLIsOpportunisticAndReadOnlyInspectionDoesNotMutate(t *testing.T) { + home, stateRoot, configRoot := t.TempDir(), filepath.Join(t.TempDir(), "state"), filepath.Join(t.TempDir(), "config") + stageID := createInspectionStage(t, home, stateRoot, "still reviewable") + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", configRoot) + t.Setenv("XDG_STATE_HOME", stateRoot) + t.Setenv("MM_URL", "") + t.Setenv("MM_TOKEN", "") + configPath := filepath.Join(configRoot, "mattermost-cli", "config.toml") + if err := os.MkdirAll(filepath.Dir(configPath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(configPath, []byte("stage_ttl_seconds = 1\n"), 0o600); err != nil { + t.Fatal(err) + } + time.Sleep(1100 * time.Millisecond) + var stdout, stderr bytes.Buffer + if code := Execute(t.Context(), []string{"stage", "list"}, nil, &stdout, &stderr); code != 0 || stderr.Len() != 0 { + t.Fatalf("list exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + if detail := loadStoredStage(t, stateRoot, stageID); detail.Lifecycle != stagestore.LifecycleOpen { + t.Fatalf("read-only list expired stage: %+v", detail.StageSummary) + } + stdout.Reset() + stderr.Reset() + code := Execute(t.Context(), []string{"stage", "cancel", stageID}, nil, &stdout, &stderr) + if code != 6 || stdout.Len() != 0 { + t.Fatalf("cancel exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) + } + if detail := loadStoredStage(t, stateRoot, stageID); detail.Lifecycle != stagestore.LifecycleExpired || detail.Body == nil { + t.Fatalf("writable operation did not expire safely: %+v body=%q", detail.StageSummary, detail.Body) + } +} diff --git a/internal/cli/store.go b/internal/cli/store.go index 5af1335..13ce430 100644 --- a/internal/cli/store.go +++ b/internal/cli/store.go @@ -115,6 +115,33 @@ func resolveStoreRedaction(state *rootState, cmd *cobra.Command) error { return nil } +func resolveStageOptions(state *rootState, cmd *cobra.Command) error { + if err := resolveStoreRedaction(state, cmd); err != nil { + return err + } + home, err := state.deps.homeDir() + if err != nil { + return configFailure("could not resolve the home directory") + } + paths, err := config.ResolvePaths(home, state.deps.lookupEnv) + if err != nil { + return configFailure(err.Error()) + } + file := config.Load(paths) + if file.Error == config.FileErrorRead || file.Unsafe != "" { + return configFailure("could not safely read the Mattermost configuration") + } + if file.Error == config.FileErrorParse { + return configFailure("could not parse the Mattermost configuration") + } + if file.InsecurePermissions && file.Config.Token != "" { + return configFailure("Mattermost configuration containing a token must not be accessible by other users") + } + state.stageTTLSeconds = file.Config.StageTTLSeconds + state.stagePruneSeconds = file.Config.StagePruneAfterSeconds + return nil +} + func storePaths(state *rootState) (stagestore.Paths, error) { home, err := state.deps.homeDir() if err != nil { diff --git a/internal/cli/store_test.go b/internal/cli/store_test.go index 28b25a2..122d128 100644 --- a/internal/cli/store_test.go +++ b/internal/cli/store_test.go @@ -36,7 +36,7 @@ func TestStoreDoctorAbsentIsReadOnlyAndSchemaValid(t *testing.T) { if err := registry.Validate("mm/v2/store-doctor", bytes.NewReader(stdout.Bytes())); err != nil { t.Fatalf("schema: %v\n%s", err, stdout.String()) } - for _, fact := range []string{`"filesystemSafe":null`, `"applicationId":null`, `"integrity":null`, `"applied":null`, `"latest":10`, `"valid":null`, `"journalMode":null`} { + for _, fact := range []string{`"filesystemSafe":null`, `"applicationId":null`, `"integrity":null`, `"applied":null`, `"latest":11`, `"valid":null`, `"journalMode":null`} { if !strings.Contains(stdout.String(), fact) { t.Fatalf("absent report omitted %s: %s", fact, stdout.String()) } @@ -110,7 +110,7 @@ func TestStoreMigrationsIsOfflineAndSchemaValid(t *testing.T) { if err := registry.Validate("mm/v2/store-migrations", bytes.NewReader(stdout.Bytes())); err != nil { t.Fatalf("schema: %v\n%s", err, stdout.String()) } - want := "{\"schema\":\"mm/v2/store-migrations\",\"latest\":10,\"migrations\":[{\"version\":1,\"name\":\"core-stage-state\",\"checksum\":\"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f\"},{\"version\":2,\"name\":\"immutable-local-request-receipts\",\"checksum\":\"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c\"},{\"version\":3,\"name\":\"caller-intent-stage-create-replay\",\"checksum\":\"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5\"},{\"version\":4,\"name\":\"caller-intent-stage-revise-replay\",\"checksum\":\"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4\"},{\"version\":5,\"name\":\"revision-plan-follows-composition\",\"checksum\":\"fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0\"},{\"version\":6,\"name\":\"durable-apply-journal\",\"checksum\":\"4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56\"},{\"version\":7,\"name\":\"status-confirmed-delete-results\",\"checksum\":\"bf980f24ff24fdf9d6a3a28b9ded97b51336fa1c8ef290d1bdcf91f1b593c5ce\"},{\"version\":8,\"name\":\"already-satisfied-edit-apply\",\"checksum\":\"cff30dd4fcc987876e092e2de1f5afd21ebeadbcb792fd550a56eb4c221edd10\"},{\"version\":9,\"name\":\"attachment-identity-binding\",\"checksum\":\"d0782e01014d010e5978b39bf6f04ef26508bec9318c7ace3be8bfbbcab4999d\"},{\"version\":10,\"name\":\"validated-upload-reuse\",\"checksum\":\"75088d75d41eab9d7d1b1c7b1fa6fc4ce604849321b88feeb110d7ada3447947\"}]}\n" + want := "{\"schema\":\"mm/v2/store-migrations\",\"latest\":11,\"migrations\":[{\"version\":1,\"name\":\"core-stage-state\",\"checksum\":\"e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f\"},{\"version\":2,\"name\":\"immutable-local-request-receipts\",\"checksum\":\"ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c\"},{\"version\":3,\"name\":\"caller-intent-stage-create-replay\",\"checksum\":\"237315b734e034951a7394ef708273af5f91ea6a44abe9927537571fbcee78d5\"},{\"version\":4,\"name\":\"caller-intent-stage-revise-replay\",\"checksum\":\"c4076ad9dba0494142f0a9cddd94e1ddcfb9c6384729f5a5f54098c8eb24f9e4\"},{\"version\":5,\"name\":\"revision-plan-follows-composition\",\"checksum\":\"fd14281b59cc1887375e125b6b782a1e4b1eff200b9e78286ec067d0286851e0\"},{\"version\":6,\"name\":\"durable-apply-journal\",\"checksum\":\"4a87ce0c406f1145b6f03a18d931681afa40dbe94ea69a89f655aa15de6bbd56\"},{\"version\":7,\"name\":\"status-confirmed-delete-results\",\"checksum\":\"bf980f24ff24fdf9d6a3a28b9ded97b51336fa1c8ef290d1bdcf91f1b593c5ce\"},{\"version\":8,\"name\":\"already-satisfied-edit-apply\",\"checksum\":\"cff30dd4fcc987876e092e2de1f5afd21ebeadbcb792fd550a56eb4c221edd10\"},{\"version\":9,\"name\":\"attachment-identity-binding\",\"checksum\":\"d0782e01014d010e5978b39bf6f04ef26508bec9318c7ace3be8bfbbcab4999d\"},{\"version\":10,\"name\":\"validated-upload-reuse\",\"checksum\":\"75088d75d41eab9d7d1b1c7b1fa6fc4ce604849321b88feeb110d7ada3447947\"},{\"version\":11,\"name\":\"retention-lifecycle-audit\",\"checksum\":\"f57092f61a66e5b10e6c748fcb85f2dc6ca2fe965d1c4ad819e399e56602f941\"}]}\n" if stdout.String() != want { t.Fatalf("stdout does not match golden migration contract: %q", stdout.String()) } @@ -310,7 +310,7 @@ func TestStoreHumanOutputStatesBounds(t *testing.T) { t.Fatalf("stdout=%q", stdout.String()) } stdout.Reset() - if code := Execute(context.Background(), []string{"store", "migrations"}, strings.NewReader(""), &stdout, &stderr); code != 0 || !strings.Contains(stdout.String(), "latest: 10\n1 core-stage-state ") { + if code := Execute(context.Background(), []string{"store", "migrations"}, strings.NewReader(""), &stdout, &stderr); code != 0 || !strings.Contains(stdout.String(), "latest: 11\n1 core-stage-state ") { t.Fatalf("exit=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String()) } } diff --git a/internal/config/config.go b/internal/config/config.go index 5f4c9c7..6523a4c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -4,8 +4,10 @@ import ( "errors" "fmt" "io" + "math" "os" "strings" + "time" "github.com/pelletier/go-toml/v2" ) @@ -18,8 +20,12 @@ const Template = `# Mattermost CLI Configuration url = "https://mattermost.example.com" token = "your-personal-access-token" # mention_names = ["Arda", "arda.sevinc"] +# stage_ttl_seconds = 0 +# stage_prune_after_seconds = 0 ` +const MaxRetentionSeconds = int64(math.MaxInt64) / int64(time.Second) + type Source string const ( @@ -55,10 +61,12 @@ const ( ) type File struct { - URL string - Token string - Redact *bool - MentionNames []string + URL string + Token string + Redact *bool + MentionNames []string + StageTTLSeconds int64 + StagePruneAfterSeconds int64 } type FileState struct { @@ -80,14 +88,16 @@ type Options struct { } type Resolved struct { - URL string - Token string - Redact bool - URLSource Source - TokenSource Source - RedactSource Source - MentionNames []string - File FileState + URL string + Token string + Redact bool + URLSource Source + TokenSource Source + RedactSource Source + MentionNames []string + StageTTLSeconds int64 + StagePruneAfterSeconds int64 + File FileState } func Load(paths Paths) FileState { @@ -165,14 +175,16 @@ func Resolve(options Options, lookup LookupEnv, file FileState) Resolved { token, tokenSource := first(options.Token, envNonempty(lookup, "MM_TOKEN"), file.Config.Token) redact, redactSource := resolveRedact(options.Redact, lookup, file.Config.Redact) return Resolved{ - URL: url, - Token: token, - Redact: redact, - URLSource: urlSource, - TokenSource: tokenSource, - RedactSource: redactSource, - MentionNames: append([]string(nil), file.Config.MentionNames...), - File: file, + URL: url, + Token: token, + Redact: redact, + URLSource: urlSource, + TokenSource: tokenSource, + RedactSource: redactSource, + MentionNames: append([]string(nil), file.Config.MentionNames...), + StageTTLSeconds: file.Config.StageTTLSeconds, + StagePruneAfterSeconds: file.Config.StagePruneAfterSeconds, + File: file, } } @@ -228,9 +240,28 @@ func parse(data []byte) (File, error) { } } } + var err error + if result.StageTTLSeconds, err = retentionSeconds(raw, "stage_ttl_seconds"); err != nil { + return File{}, err + } + if result.StagePruneAfterSeconds, err = retentionSeconds(raw, "stage_prune_after_seconds"); err != nil { + return File{}, err + } return result, nil } +func retentionSeconds(raw map[string]any, key string) (int64, error) { + value, exists := raw[key] + if !exists { + return 0, nil + } + seconds, ok := value.(int64) + if !ok || seconds < 0 || seconds > MaxRetentionSeconds { + return 0, fmt.Errorf("invalid %s", key) + } + return seconds, nil +} + func first(cli string, envValue string, file string) (string, Source) { if cli != "" { return cli, SourceCLI diff --git a/internal/config/config_test.go b/internal/config/config_test.go index c79cbe5..5e038ff 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -156,6 +156,8 @@ token = " fixture-token " redact = false mention_names = [" Arda ", "", 42, "arda.sevinc"] unknown = "ignored" +stage_ttl_seconds = 3600 +stage_prune_after_seconds = 86400 `, 0o600) state := Load(Paths{ConfigPath: path, LegacyPath: path}) @@ -172,6 +174,29 @@ unknown = "ignored" if !slices.Equal(state.Config.MentionNames, []string{"Arda", "arda.sevinc"}) { t.Fatalf("MentionNames = %q", state.Config.MentionNames) } + if state.Config.StageTTLSeconds != 3600 || state.Config.StagePruneAfterSeconds != 86400 { + t.Fatalf("retention = %d/%d", state.Config.StageTTLSeconds, state.Config.StagePruneAfterSeconds) + } +} + +func TestLoadRejectsUnsafeRetentionPolicyValues(t *testing.T) { + for _, test := range []struct { + name string + value string + }{ + {"wrong type", `"3600"`}, + {"negative", `-1`}, + {"duration overflow", `9223372037`}, + } { + t.Run(test.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + writeConfig(t, path, "stage_ttl_seconds = "+test.value, 0o600) + state := Load(Paths{ConfigPath: path, LegacyPath: path}) + if state.Error != FileErrorParse || state.Config.StageTTLSeconds != 0 { + t.Fatalf("state=%+v", state) + } + }) + } } func TestLoadCharacterizesMissingReadParseAndPermissions(t *testing.T) { @@ -231,7 +256,7 @@ func TestResolvePreservesPrecedenceAndRedactSemantics(t *testing.T) { fileRedact := false file := FileState{Config: File{ URL: "https://file.example", Token: "file-token", Redact: &fileRedact, - MentionNames: []string{"Arda"}, + MentionNames: []string{"Arda"}, StageTTLSeconds: 7, StagePruneAfterSeconds: 11, }} env := map[string]string{ "MM_URL": "https://env.example", "MM_TOKEN": "env-token", "MM_REDACT": "", @@ -245,6 +270,9 @@ func TestResolvePreservesPrecedenceAndRedactSemantics(t *testing.T) { if resolved.URL != "https://cli.example" || resolved.Token != "env-token" || resolved.Redact { t.Fatalf("Resolve() = %+v", resolved) } + if resolved.StageTTLSeconds != 7 || resolved.StagePruneAfterSeconds != 11 { + t.Fatalf("retention policy lost: %+v", resolved) + } resolved = Resolve(Options{}, mapLookup(env), file) if !resolved.Redact || resolved.RedactSource != SourceEnv { diff --git a/internal/output/machine.go b/internal/output/machine.go index 564caf9..2338ce2 100644 --- a/internal/output/machine.go +++ b/internal/output/machine.go @@ -125,20 +125,22 @@ type ErrorEnvelope struct { } type ConfigEnvelope struct { - Schema string `json:"schema"` - Action string `json:"action"` - SelectedPath string `json:"selectedPath"` - ReadPath *string `json:"readPath"` - Migration *string `json:"migration"` - Exists *bool `json:"exists"` - URLConfigured *bool `json:"urlConfigured"` - TokenConfigured *bool `json:"tokenConfigured"` - Permissions *string `json:"permissions"` - ReadStatus *string `json:"readStatus"` - ParseStatus *string `json:"parseStatus"` - UnsafeReason *string `json:"unsafeReason"` - Created *bool `json:"created"` - Warning *string `json:"warning"` + Schema string `json:"schema"` + Action string `json:"action"` + SelectedPath string `json:"selectedPath"` + ReadPath *string `json:"readPath"` + Migration *string `json:"migration"` + Exists *bool `json:"exists"` + URLConfigured *bool `json:"urlConfigured"` + TokenConfigured *bool `json:"tokenConfigured"` + StageTTLSeconds *int64 `json:"stageTtlSeconds"` + StagePruneAfterSeconds *int64 `json:"stagePruneAfterSeconds"` + Permissions *string `json:"permissions"` + ReadStatus *string `json:"readStatus"` + ParseStatus *string `json:"parseStatus"` + UnsafeReason *string `json:"unsafeReason"` + Created *bool `json:"created"` + Warning *string `json:"warning"` } type DoctorCheck struct { diff --git a/internal/stageoutput/output.go b/internal/stageoutput/output.go index b232ab3..a59be4f 100644 --- a/internal/stageoutput/output.go +++ b/internal/stageoutput/output.go @@ -85,6 +85,21 @@ type Stages struct { Stages []Summary `json:"stages"` NextCursor *string `json:"nextCursor"` } +type PruneResult struct { + Schema string `json:"schema"` + Action string `json:"action"` + Cutoff string `json:"cutoff"` + PrunedCount int64 `json:"prunedCount"` + RecordedAt string `json:"recordedAt"` +} + +func NewPruneResult(in stagestore.BulkPruneResult, credentials []string) (PruneResult, error) { + if in.Schema != "mm/v2/stage-prune-result" || in.Action != "pruned" || in.PrunedCount < 0 || in.PrunedCount > 9_007_199_254_740_991 || in.Cutoff.IsZero() || in.RecordedAt.IsZero() || !in.Cutoff.Before(in.RecordedAt) { + return PruneResult{}, ErrInvalid + } + out := PruneResult{in.Schema, in.Action, stamp(in.Cutoff), in.PrunedCount, stamp(in.RecordedAt)} + return out, validate("mm/v2/stage-prune-result", out, credentials) +} func NewPreview(operation stagestore.Operation, in staging.Preview, credentials []string) (Preview, error) { out := Preview{"mm/v2/stage-preview", false, string(operation), binding(in.ServerURL, in.ServerID, in.UserID), cloneDestination(in.Destination), clonePlan(in.Plan), false} @@ -111,7 +126,7 @@ func NewReceipt(in stagestore.MutationResult, destination json.RawMessage, crede } func newReceipt(in stagestore.MutationResult, d staging.Destination, credentials []string) (Receipt, error) { - actions := map[string]string{"create": "created", "revise": "revised", "cancel": "canceled"} + actions := map[string]string{"create": "created", "revise": "revised", "cancel": "canceled", "prune": "pruned"} if !validSummaryTimes(in.Stage) || in.RecordedAt.IsZero() || emittedTime(in.RecordedAt).Before(emittedTime(in.Stage.UpdatedAt)) { return Receipt{}, ErrInvalid } diff --git a/internal/stageoutput/output_test.go b/internal/stageoutput/output_test.go index b972b88..d3228f8 100644 --- a/internal/stageoutput/output_test.go +++ b/internal/stageoutput/output_test.go @@ -163,6 +163,19 @@ func TestReceiptsAndListsEnforceEmittedTimestampOrderAndUniqueIDs(t *testing.T) } } +func TestNewPruneResultUsesBoundedMillisecondContract(t *testing.T) { + recorded := time.Date(2026, 7, 17, 12, 0, 0, 999999999, time.UTC) + document, err := NewPruneResult(stagestore.BulkPruneResult{ + Schema: "mm/v2/stage-prune-result", Action: "pruned", Cutoff: recorded.Add(-720 * time.Hour), PrunedCount: 3, RecordedAt: recorded, + }, nil) + if err != nil || document.PrunedCount != 3 || document.RecordedAt != "2026-07-17T12:00:00.999Z" { + t.Fatalf("document=%+v err=%v", document, err) + } + if _, err = NewPruneResult(stagestore.BulkPruneResult{Schema: "mm/v2/stage-prune-result", Action: "pruned", Cutoff: recorded, RecordedAt: recorded}, nil); !errors.Is(err, ErrInvalid) { + t.Fatalf("nonpositive age accepted: %v", err) + } +} + func conversationDestination() staging.Destination { team := "team-1" return staging.Destination{Kind: "conversation", ChannelID: "channel-1", ChannelType: "public", TeamID: &team, ParticipantIDs: []string{}} diff --git a/internal/stagerequest/request.go b/internal/stagerequest/request.go index 53fd833..86a106b 100644 --- a/internal/stagerequest/request.go +++ b/internal/stagerequest/request.go @@ -26,6 +26,7 @@ const ( StageSchema = "mm/v2/stage-request" ReviseSchema = "mm/v2/stage-revise-request" CancelSchema = "mm/v2/stage-cancel-request" + PruneSchema = "mm/v2/stage-prune-request" ApplySchema = "mm/v2/apply-request" ) @@ -135,6 +136,15 @@ type CancelRequest struct { ExpectedDigest string `json:"expectedDigest"` } +type PruneRequest struct { + Schema string `json:"schema"` + RequestID string `json:"requestId"` + StageID string `json:"stageId"` + ExpectedRevision ExactInt64 `json:"expectedRevision"` + ExpectedDigest string `json:"expectedDigest"` + AbandonRecovery bool `json:"abandonRecovery"` +} + type ApplyRequest struct { Schema string `json:"schema"` RequestID string `json:"requestId"` @@ -189,6 +199,10 @@ func (d *Decoder) DecodeCancel(input io.Reader) (CancelRequest, error) { return decode[CancelRequest](d, CancelSchema, input) } +func (d *Decoder) DecodePrune(input io.Reader) (PruneRequest, error) { + return decode[PruneRequest](d, PruneSchema, input) +} + func (d *Decoder) DecodeApply(input io.Reader) (ApplyRequest, error) { return decode[ApplyRequest](d, ApplySchema, input) } @@ -470,6 +484,14 @@ func (r CancelRequest) CancelInput() (staging.CancelInput, error) { return staging.CancelInput{StageID: r.StageID, RequestID: r.RequestID, ExpectedRevision: int64(r.ExpectedRevision), ExpectedDigest: digest}, nil } +func (r PruneRequest) PruneInput() (stagestore.PruneInput, error) { + digest, err := decodeDigest(r.ExpectedDigest) + if err != nil || digest == ([32]byte{}) || validateConversion(PruneSchema, r) != nil { + return stagestore.PruneInput{}, ErrInvalid + } + return stagestore.PruneInput{StageID: r.StageID, RequestID: r.RequestID, ExpectedRevision: int64(r.ExpectedRevision), ExpectedDigest: digest, AbandonRecovery: r.AbandonRecovery}, nil +} + // ApplyClaimInput converts the public request and derives caller-intent replay // identity. The request ID is deliberately excluded from the digest. func (r ApplyRequest) ApplyClaimInput() (stagestore.ApplyClaimInput, error) { diff --git a/internal/stagerequest/request_test.go b/internal/stagerequest/request_test.go index 1c1cd11..2d5ed80 100644 --- a/internal/stagerequest/request_test.go +++ b/internal/stagerequest/request_test.go @@ -164,6 +164,15 @@ func TestReviseAndCancelDecodeAndStoreConversions(t *testing.T) { if err != nil || cancelInput.ExpectedDigest != reviseInput.ExpectedDigest || cancelInput.ExpectedRevision != 3 { t.Fatalf("cancel conversion: %#v %v", cancelInput, err) } + pruneJSON := `{"schema":"mm/v2/stage-prune-request","requestId":"p","stageId":"stg_abcdefghijklmnopqrstuvwxyzABCDEF","expectedRevision":4,"expectedDigest":"` + digestText + `","abandonRecovery":true}` + prune, err := decoder(t).DecodePrune(strings.NewReader(pruneJSON)) + if err != nil { + t.Fatal(err) + } + pruneInput, err := prune.PruneInput() + if err != nil || pruneInput.ExpectedRevision != 4 || pruneInput.ExpectedDigest != reviseInput.ExpectedDigest || !pruneInput.AbandonRecovery { + t.Fatalf("prune conversion: %#v %v", pruneInput, err) + } cancel.ExpectedDigest = strings.ToUpper(digestText) if _, err := cancel.CancelInput(); !errors.Is(err, ErrInvalid) { @@ -175,6 +184,27 @@ func TestReviseAndCancelDecodeAndStoreConversions(t *testing.T) { } } +func TestPruneDecodeRejectsUnknownFieldsDuplicateMembersAndZeroDigest(t *testing.T) { + base := `{"schema":"mm/v2/stage-prune-request","requestId":"p","stageId":"stg_abcdefghijklmnopqrstuvwxyzABCDEF","expectedRevision":1,"expectedDigest":"` + strings.Repeat("01", 32) + `","abandonRecovery":false}` + for name, raw := range map[string]string{ + "unknown": strings.Replace(base, `"abandonRecovery":false`, `"abandonRecovery":false,"extra":true`, 1), + "duplicate": strings.Replace(base, `"requestId":"p"`, `"requestId":"p","requestId":"q"`, 1), + } { + t.Run(name, func(t *testing.T) { + if _, err := decoder(t).DecodePrune(strings.NewReader(raw)); !errors.Is(err, ErrInvalid) { + t.Fatalf("accepted: %v", err) + } + }) + } + request, err := decoder(t).DecodePrune(strings.NewReader(strings.Replace(base, strings.Repeat("01", 32), strings.Repeat("0", 64), 1))) + if err != nil { + t.Fatal(err) + } + if _, err = request.PruneInput(); !errors.Is(err, ErrInvalid) { + t.Fatalf("zero digest accepted: %v", err) + } +} + func TestApplyDecodeConversionAndCallerIntentReplayDigest(t *testing.T) { digestText := strings.Repeat("ab", 32) raw := `{"schema":"mm/v2/apply-request","requestId":"apply-1","stageId":"stg_abcdefghijklmnopqrstuvwxyzABCDEF","revision":2,"expectedDigest":"` + digestText + `","recoveryMode":"resume_partial"}` diff --git a/internal/stagestore/apply.go b/internal/stagestore/apply.go index 3743f3a..e13c226 100644 --- a/internal/stagestore/apply.go +++ b/internal/stagestore/apply.go @@ -450,10 +450,16 @@ func (s *Store) FinalizeApply(ctx context.Context, attemptID string) (ApplyRecei } now := time.Now().UTC() stamp := formatTime(now) + material := recoveryMaterialFor(lifecycle, recovery) if _, err = tx.ExecContext(ctx, `UPDATE apply_attempts SET outcome=?,ended_at=? WHERE id=? AND outcome IS NULL`, outcome, stamp, attemptID); err != nil { return ApplyReceipt{}, localError(err) } - res, err := tx.ExecContext(ctx, `UPDATE stages SET lifecycle=?,recovery=?,claim_attempt_id=NULL,updated_at=? WHERE id=? AND lifecycle='applying' AND claim_attempt_id=?`, lifecycle, recovery, stamp, attempt.StageID, attemptID) + var res sql.Result + if retentionLifecycleAvailable() { + res, err = tx.ExecContext(ctx, `UPDATE stages SET lifecycle=?,recovery=?,recovery_material=?,claim_attempt_id=NULL,updated_at=? WHERE id=? AND lifecycle='applying' AND claim_attempt_id=?`, lifecycle, recovery, material, stamp, attempt.StageID, attemptID) + } else { + res, err = tx.ExecContext(ctx, `UPDATE stages SET lifecycle=?,recovery=?,claim_attempt_id=NULL,updated_at=? WHERE id=? AND lifecycle='applying' AND claim_attempt_id=?`, lifecycle, recovery, stamp, attempt.StageID, attemptID) + } if err != nil { return ApplyReceipt{}, localError(err) } diff --git a/internal/stagestore/domain.go b/internal/stagestore/domain.go index 2286596..7462ebe 100644 --- a/internal/stagestore/domain.go +++ b/internal/stagestore/domain.go @@ -266,7 +266,7 @@ func (s *Store) Revise(ctx context.Context, in ReviseInput) (MutationResult, err return MutationResult{}, ErrConflict } recovery := base.Recovery - if in.Revive { + if in.Revive && retentionLifecycleAvailable() { if base.Lifecycle != LifecycleExpired || base.Recovery != RecoveryForbidden { return MutationResult{}, ErrNotEligible } @@ -291,6 +291,12 @@ func (s *Store) Revise(ctx context.Context, in ReviseInput) (MutationResult, err if err = insertAttachments(ctx, tx, in.StageID, next, content.Attachments); err != nil { return MutationResult{}, err } + if in.Revive { + if _, err = tx.ExecContext(ctx, `INSERT INTO stage_retention_events(stage_id,revision,semantic_digest,event,from_lifecycle,from_recovery,recovery_material,policy_seconds,request_id,recorded_at) + VALUES(?,?,?,'revived',?,?, 'none',NULL,NULL,?)`, base.ID, base.Revision, base.SemanticDigest[:], base.Lifecycle, base.Recovery, stamp); err != nil { + return MutationResult{}, localError(err) + } + } resultSQL, err = tx.ExecContext(ctx, `UPDATE stages SET updated_at=?,lifecycle='open',recovery=?,current_revision=? WHERE id=? AND current_revision=? AND lifecycle=? AND recovery=?`, stamp, recovery, next, in.StageID, in.ExpectedRevision, base.Lifecycle, base.Recovery) if err != nil { return MutationResult{}, localError(err) @@ -908,7 +914,7 @@ func findCreate(ctx context.Context, q queryer, server, user, id string) (Create if err != nil { return CreateRecord{}, false, localError(err) } - if schemaName == "mm/v2/legacy-stage-request-conflict" || schemaName == "mm/v2/legacy-stage-revise-conflict" || schemaName == "mm/v2/legacy-request-conflict" || schemaName == "mm/v2/stage-revise-request" || schemaName == "mm/v2/stage-cancel-request" { + if schemaName == "mm/v2/legacy-stage-request-conflict" || schemaName == "mm/v2/legacy-stage-revise-conflict" || schemaName == "mm/v2/legacy-request-conflict" || schemaName == "mm/v2/stage-revise-request" || schemaName == "mm/v2/stage-cancel-request" || schemaName == "mm/v2/stage-prune-request" { return CreateRecord{}, false, ErrConflict } if schemaName != "mm/v2/stage-request" || len(digest) != 32 { @@ -959,7 +965,7 @@ func findCreate(ctx context.Context, q queryer, server, user, id string) (Create // apply, plaintext and attachment paths are intentionally erased, so the // immutable historical digest becomes the only possible content binding. func replayContentProjection(operation Operation, lifecycle Lifecycle, recovery Recovery, body []byte, destination, plan json.RawMessage, attachments []Attachment) (RevisionContent, bool, error) { - if lifecycle != LifecycleCompleted || recovery != RecoveryForbidden { + if lifecycle != LifecycleCompleted && lifecycle != LifecyclePruned || recovery != RecoveryForbidden { content, err := normalizeContent(operation, RevisionContent{body, destination, plan, attachments}) return content, false, err } @@ -1017,7 +1023,7 @@ func persistCreate(ctx context.Context, tx *sql.Tx, server, user, id string, rec return localError(err) } func validReplayResult(result MutationResult, requestSchema, server, user string) bool { - action := map[string]string{"mm/v2/stage-request": "create", "mm/v2/stage-revise-request": "revise", "mm/v2/stage-cancel-request": "cancel"}[requestSchema] + action := map[string]string{"mm/v2/stage-request": "create", "mm/v2/stage-revise-request": "revise", "mm/v2/stage-cancel-request": "cancel", "mm/v2/stage-prune-request": "prune"}[requestSchema] stage := result.Stage return action != "" && result.Schema == "mm/v2/stage-mutation-receipt" && result.Action == action && stage.ServerURL == server && stage.UserID == user && bounded(stage.ID, maxIdentityBytes) && validOperation(stage.Operation) && stage.Revision > 0 && stage.SemanticDigest != ([32]byte{}) && validLifecycle(stage.Lifecycle) && validRecovery(stage.Recovery) && diff --git a/internal/stagestore/domain_test.go b/internal/stagestore/domain_test.go index 5a803e9..2746763 100644 --- a/internal/stagestore/domain_test.go +++ b/internal/stagestore/domain_test.go @@ -656,7 +656,8 @@ func TestSimultaneousMutationCAS(t *testing.T) { func TestReviveOnlyLegalExpiredForbidden(t *testing.T) { s := openDomainStore(t) created, _ := s.Create(context.Background(), createInput("", "one")) - if _, err := s.db.Exec(`UPDATE stages SET lifecycle='expired',recovery='forbidden' WHERE id=?`, created.Stage.ID); err != nil { + recorded := time.Now().UTC().Add(2 * time.Second) + if count, err := s.ExpireEligible(context.Background(), recorded.Add(-time.Second), recorded); err != nil || count != 1 { t.Fatal(err) } input := reviseInput(created.Stage, "revive", "two") diff --git a/internal/stagestore/retention.go b/internal/stagestore/retention.go new file mode 100644 index 0000000..4c16657 --- /dev/null +++ b/internal/stagestore/retention.go @@ -0,0 +1,286 @@ +package stagestore + +import ( + "context" + "database/sql" + "errors" + "time" +) + +type RecoveryMaterial string + +const ( + MaterialNone RecoveryMaterial = "none" + MaterialPartial RecoveryMaterial = "resume_partial" + MaterialUnknown RecoveryMaterial = "force_unknown" +) + +type PruneInput struct { + StageID, RequestID string + ExpectedRevision int64 + ExpectedDigest [32]byte + AbandonRecovery bool +} + +type BulkPruneResult struct { + Schema string `json:"schema"` + Action string `json:"action"` + Cutoff time.Time `json:"cutoff"` + PrunedCount int64 `json:"prunedCount"` + RecordedAt time.Time `json:"recordedAt"` +} + +type RetentionEvent struct { + Sequence int64 + StageID string + Revision int64 + SemanticDigest [32]byte + Event string + FromLifecycle Lifecycle + FromRecovery Recovery + RecoveryMaterial RecoveryMaterial + PolicySeconds *int64 + RequestID string + RecordedAt time.Time +} + +func recoveryMaterialFor(lifecycle Lifecycle, recovery Recovery) RecoveryMaterial { + if lifecycle == LifecycleCompleted || recovery == RecoveryForbidden { + return MaterialNone + } + switch recovery { + case RecoveryPartial: + return MaterialPartial + case RecoveryUnknown: + return MaterialUnknown + default: + return MaterialNone + } +} + +// ExpireEligible durably revokes only inactive ordinary-apply stages at or +// before cutoff. The caller supplies one recorded time for the whole sweep. +func (s *Store) ExpireEligible(ctx context.Context, cutoff, recordedAt time.Time) (int64, error) { + if ctx == nil || cutoff.IsZero() || recordedAt.IsZero() || !cutoff.Before(recordedAt) || !retentionLifecycleAvailable() { + return 0, ErrInvalid + } + seconds := int64(recordedAt.Sub(cutoff) / time.Second) + if seconds < 1 { + return 0, ErrInvalid + } + cutoffStamp, stamp := formatTime(cutoff.UTC()), formatTime(recordedAt.UTC()) + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return 0, localError(err) + } + defer tx.Rollback() + result, err := tx.ExecContext(ctx, `INSERT INTO stage_retention_events(stage_id,revision,semantic_digest,event,from_lifecycle,from_recovery,recovery_material,policy_seconds,request_id,recorded_at) + SELECT s.id,s.current_revision,r.semantic_digest,'expired',s.lifecycle,s.recovery,s.recovery_material,?,NULL,? + FROM stages s JOIN stage_revisions r ON r.stage_id=s.id AND r.revision=s.current_revision + WHERE s.lifecycle='open' AND s.recovery='none' AND s.recovery_material='none' AND s.claim_attempt_id IS NULL AND s.updated_at<=?`, seconds, stamp, cutoffStamp) + if err != nil { + return 0, localError(err) + } + count, err := result.RowsAffected() + if err != nil { + return 0, localError(err) + } + result, err = tx.ExecContext(ctx, `UPDATE stages SET lifecycle='expired',recovery='forbidden',updated_at=? + WHERE lifecycle='open' AND recovery='none' AND recovery_material='none' AND claim_attempt_id IS NULL AND updated_at<=?`, stamp, cutoffStamp) + if err != nil { + return 0, localError(err) + } + updated, err := result.RowsAffected() + if err != nil || updated != count { + return 0, localError(errors.New("expiry selection changed")) + } + if err = tx.Commit(); err != nil { + return 0, localError(err) + } + runCommitHook() + return count, nil +} + +// PruneEligible atomically erases sensitive content for all eligible terminal +// stages at or before cutoff without returning an unbounded stage list. +func (s *Store) PruneEligible(ctx context.Context, cutoff, recordedAt time.Time) (BulkPruneResult, error) { + result := BulkPruneResult{"mm/v2/stage-prune-result", "pruned", cutoff.UTC(), 0, recordedAt.UTC()} + if ctx == nil || cutoff.IsZero() || recordedAt.IsZero() || !cutoff.Before(recordedAt) || !retentionLifecycleAvailable() { + return BulkPruneResult{}, ErrInvalid + } + seconds := int64(recordedAt.Sub(cutoff) / time.Second) + if seconds < 1 { + return BulkPruneResult{}, ErrInvalid + } + cutoffStamp, stamp := formatTime(cutoff.UTC()), formatTime(recordedAt.UTC()) + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return BulkPruneResult{}, localError(err) + } + defer tx.Rollback() + inserted, err := tx.ExecContext(ctx, `INSERT INTO stage_retention_events(stage_id,revision,semantic_digest,event,from_lifecycle,from_recovery,recovery_material,policy_seconds,request_id,recorded_at) + SELECT s.id,s.current_revision,r.semantic_digest,'pruned',s.lifecycle,s.recovery,s.recovery_material,?,NULL,? + FROM stages s JOIN stage_revisions r ON r.stage_id=s.id AND r.revision=s.current_revision + WHERE s.lifecycle IN ('completed','canceled','expired') AND s.recovery='forbidden' AND s.recovery_material='none' + AND s.claim_attempt_id IS NULL AND s.updated_at<=?`, seconds, stamp, cutoffStamp) + if err != nil { + return BulkPruneResult{}, localError(err) + } + result.PrunedCount, err = inserted.RowsAffected() + if err != nil { + return BulkPruneResult{}, localError(err) + } + updated, err := tx.ExecContext(ctx, `UPDATE stages SET lifecycle='pruned',recovery='forbidden',recovery_material='none',updated_at=? + WHERE lifecycle IN ('completed','canceled','expired') AND recovery='forbidden' AND recovery_material='none' + AND claim_attempt_id IS NULL AND updated_at<=?`, stamp, cutoffStamp) + if err != nil { + return BulkPruneResult{}, localError(err) + } + count, err := updated.RowsAffected() + if err != nil || count != result.PrunedCount { + return BulkPruneResult{}, localError(errors.New("prune selection changed")) + } + if err = erasePrunedContent(ctx, tx, stamp); err != nil { + return BulkPruneResult{}, err + } + if err = tx.Commit(); err != nil { + return BulkPruneResult{}, localError(err) + } + runCommitHook() + return result, nil +} + +func (s *Store) Prune(ctx context.Context, in PruneInput) (MutationResult, error) { + if ctx == nil || !bounded(in.StageID, maxIdentityBytes) || in.ExpectedRevision < 1 || !validRequestID(in.RequestID) || in.ExpectedDigest == ([32]byte{}) || !retentionLifecycleAvailable() { + return MutationResult{}, ErrInvalid + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return MutationResult{}, localError(err) + } + defer tx.Rollback() + base, err := scanCurrent(ctx, tx, in.StageID) + if err != nil { + return MutationResult{}, err + } + digest := pruneRequestDigest(in) + if in.RequestID != "" { + if replay, found, replayErr := loadReplay(ctx, tx, base.ServerURL, base.UserID, in.RequestID, "mm/v2/stage-prune-request", digest); replayErr != nil { + return MutationResult{}, replayErr + } else if found { + return replay, nil + } + } + if base.Revision != in.ExpectedRevision || base.SemanticDigest != in.ExpectedDigest { + return MutationResult{}, ErrConflict + } + var material RecoveryMaterial + var claimed sql.NullString + if err = tx.QueryRowContext(ctx, `SELECT recovery_material,claim_attempt_id FROM stages WHERE id=?`, in.StageID).Scan(&material, &claimed); err != nil { + return MutationResult{}, localError(err) + } + if claimed.Valid || base.Lifecycle == LifecycleApplying || base.Lifecycle == LifecyclePruned { + return MutationResult{}, ErrNotEligible + } + event := "pruned" + if in.AbandonRecovery { + if material != MaterialPartial && material != MaterialUnknown { + return MutationResult{}, ErrNotEligible + } + event = "abandoned_recovery" + } else if material != MaterialNone || base.Lifecycle != LifecycleCompleted && base.Lifecycle != LifecycleCanceled && base.Lifecycle != LifecycleExpired { + return MutationResult{}, ErrNotEligible + } + now := time.Now().UTC() + stamp := formatTime(now) + if _, err = tx.ExecContext(ctx, `INSERT INTO stage_retention_events(stage_id,revision,semantic_digest,event,from_lifecycle,from_recovery,recovery_material,policy_seconds,request_id,recorded_at) + VALUES(?,?,?,?,?,?,?,NULL,?,?)`, base.ID, base.Revision, base.SemanticDigest[:], event, base.Lifecycle, base.Recovery, material, nullable(in.RequestID), stamp); err != nil { + return MutationResult{}, localError(err) + } + updated, err := tx.ExecContext(ctx, `UPDATE stages SET lifecycle='pruned',recovery='forbidden',recovery_material='none',updated_at=? + WHERE id=? AND current_revision=? AND lifecycle=? AND recovery=? AND recovery_material=? AND claim_attempt_id IS NULL + AND EXISTS(SELECT 1 FROM stage_revisions WHERE stage_id=? AND revision=? AND semantic_digest=?)`, stamp, base.ID, base.Revision, base.Lifecycle, base.Recovery, material, base.ID, base.Revision, base.SemanticDigest[:]) + if err != nil { + return MutationResult{}, localError(err) + } + if !oneRow(updated) { + return MutationResult{}, ErrConflict + } + if _, err = tx.ExecContext(ctx, `UPDATE stage_revisions SET body=NULL WHERE stage_id=?`, base.ID); err != nil { + return MutationResult{}, localError(err) + } + if _, err = tx.ExecContext(ctx, `DELETE FROM stage_attachments WHERE stage_id=?`, base.ID); err != nil { + return MutationResult{}, localError(err) + } + summary := base.StageSummary + summary.Lifecycle, summary.Recovery, summary.UpdatedAt = LifecyclePruned, RecoveryForbidden, now + result := MutationResult{"mm/v2/stage-mutation-receipt", "prune", summary, false, now, false} + if err = persistReplay(ctx, tx, base.ServerURL, base.UserID, in.RequestID, "mm/v2/stage-prune-request", digest, result, stamp); err != nil { + return MutationResult{}, err + } + if err = tx.Commit(); err != nil { + return MutationResult{}, localError(err) + } + runCommitHook() + return result, nil +} + +func pruneRequestDigest(in PruneInput) [32]byte { + return digestValue(struct { + Domain string + StageID string + ExpectedRevision int64 + ExpectedDigest [32]byte + AbandonRecovery bool + }{"mm/v2/stage-prune-request/caller-intent/v1", in.StageID, in.ExpectedRevision, in.ExpectedDigest, in.AbandonRecovery}) +} + +func erasePrunedContent(ctx context.Context, tx *sql.Tx, stamp string) error { + selection := `SELECT stage_id FROM stage_retention_events WHERE recorded_at=? AND event='pruned'` + if _, err := tx.ExecContext(ctx, `UPDATE stage_revisions SET body=NULL WHERE stage_id IN (`+selection+`)`, stamp); err != nil { + return localError(err) + } + if _, err := tx.ExecContext(ctx, `DELETE FROM stage_attachments WHERE stage_id IN (`+selection+`)`, stamp); err != nil { + return localError(err) + } + return nil +} + +func (s *Store) RetentionEvents(ctx context.Context, stageID string) ([]RetentionEvent, error) { + if ctx == nil || !bounded(stageID, maxIdentityBytes) { + return nil, ErrInvalid + } + rows, err := s.db.QueryContext(ctx, `SELECT sequence,stage_id,revision,semantic_digest,event,from_lifecycle,from_recovery,recovery_material,policy_seconds,coalesce(request_id,''),recorded_at + FROM stage_retention_events WHERE stage_id=? ORDER BY sequence`, stageID) + if err != nil { + return nil, localError(err) + } + defer rows.Close() + var events []RetentionEvent + for rows.Next() { + var event RetentionEvent + var digest []byte + var seconds sql.NullInt64 + var stamp string + if err = rows.Scan(&event.Sequence, &event.StageID, &event.Revision, &digest, &event.Event, &event.FromLifecycle, &event.FromRecovery, &event.RecoveryMaterial, &seconds, &event.RequestID, &stamp); err != nil { + return nil, localError(err) + } + if len(digest) != 32 { + return nil, localError(errors.New("retention event digest")) + } + copy(event.SemanticDigest[:], digest) + if seconds.Valid { + value := seconds.Int64 + event.PolicySeconds = &value + } + event.RecordedAt, err = parseTime(stamp) + if err != nil { + return nil, err + } + events = append(events, event) + } + if err = rows.Err(); err != nil { + return nil, localError(err) + } + return events, nil +} diff --git a/internal/stagestore/retention_test.go b/internal/stagestore/retention_test.go new file mode 100644 index 0000000..7441a4b --- /dev/null +++ b/internal/stagestore/retention_test.go @@ -0,0 +1,262 @@ +//go:build darwin || linux + +package stagestore + +import ( + "context" + "errors" + "testing" + "time" +) + +func TestExpireEligibleUsesInclusiveActivityBoundaryAndWritesAuditEvents(t *testing.T) { + s := openDomainStore(t) + before, _ := s.Create(context.Background(), createInput("", "before")) + at, _ := s.Create(context.Background(), createInput("", "at")) + after, _ := s.Create(context.Background(), createInput("", "after")) + cutoff := time.Now().UTC().Add(time.Hour).Truncate(time.Millisecond) + for _, item := range []struct { + id string + stamp time.Time + }{{before.Stage.ID, cutoff.Add(-time.Millisecond)}, {at.Stage.ID, cutoff}, {after.Stage.ID, cutoff.Add(time.Millisecond)}} { + if _, err := s.db.Exec(`UPDATE stages SET updated_at=? WHERE id=?`, formatTime(item.stamp), item.id); err != nil { + t.Fatal(err) + } + } + recorded := cutoff.Add(time.Hour) + count, err := s.ExpireEligible(context.Background(), cutoff, recorded) + if err != nil || count != 2 { + t.Fatalf("expired=%d err=%v", count, err) + } + for _, item := range []struct { + stage StageSummary + want Lifecycle + }{{before.Stage, LifecycleExpired}, {at.Stage, LifecycleExpired}, {after.Stage, LifecycleOpen}} { + detail, showErr := s.Show(context.Background(), item.stage.ID) + if showErr != nil || detail.Lifecycle != item.want || detail.Body == nil { + t.Fatalf("stage=%s lifecycle=%s body=%q err=%v", item.stage.ID, detail.Lifecycle, detail.Body, showErr) + } + events, eventErr := s.RetentionEvents(context.Background(), item.stage.ID) + wantEvents := 0 + if item.want == LifecycleExpired { + wantEvents = 1 + } + if eventErr != nil || len(events) != wantEvents { + t.Fatalf("stage=%s events=%+v err=%v", item.stage.ID, events, eventErr) + } + if wantEvents == 1 && (events[0].Event != "expired" || events[0].PolicySeconds == nil || *events[0].PolicySeconds != 3600) { + t.Fatalf("expiry event=%+v", events[0]) + } + } +} + +func TestExactPruneErasesContentAndPreservesRequestReplay(t *testing.T) { + s := openDomainStore(t) + in := createInput("create-before-prune", "sensitive markdown") + created, err := s.Create(context.Background(), in) + if err != nil { + t.Fatal(err) + } + if _, err = s.Cancel(context.Background(), CancelInput{created.Stage.ID, "cancel-before-prune", created.Stage.Revision, created.Stage.SemanticDigest}); err != nil { + t.Fatal(err) + } + input := PruneInput{StageID: created.Stage.ID, RequestID: "prune-exact", ExpectedRevision: created.Stage.Revision, ExpectedDigest: created.Stage.SemanticDigest} + pruned, err := s.Prune(context.Background(), input) + if err != nil || pruned.Stage.Lifecycle != LifecyclePruned || pruned.Stage.Recovery != RecoveryForbidden { + t.Fatalf("pruned=%+v err=%v", pruned, err) + } + detail, err := s.Show(context.Background(), created.Stage.ID) + if err != nil || detail.Body != nil || len(detail.Attachments) != 0 || detail.Lifecycle != LifecyclePruned { + t.Fatalf("detail=%+v err=%v", detail, err) + } + events, err := s.RetentionEvents(context.Background(), created.Stage.ID) + if err != nil || len(events) != 1 || events[0].Event != "pruned" || events[0].RequestID != input.RequestID { + t.Fatalf("events=%+v err=%v", events, err) + } + replay, err := s.Prune(context.Background(), input) + if err != nil || !replay.Replay || replay.RecordedAt != pruned.RecordedAt { + t.Fatalf("replay=%+v err=%v", replay, err) + } + conflict := input + conflict.AbandonRecovery = true + if _, err = s.Prune(context.Background(), conflict); !errors.Is(err, ErrConflict) { + t.Fatalf("conflicting replay=%v", err) + } + createdReplay, found, err := s.FindCreate(context.Background(), in.ServerURL, in.UserID, in.RequestID) + if err != nil || !found || !createdReplay.Replay || createdReplay.Stage.ID != created.Stage.ID { + t.Fatalf("create replay=%+v found=%v err=%v", createdReplay, found, err) + } +} + +func TestUnknownThenCancelRequiresExplicitRecoveryAbandonment(t *testing.T) { + s := openDomainStore(t) + created := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) + attempt, err := s.ClaimApply(context.Background(), claimInput(created.Stage, "unknown-for-prune", RecoveryModeOrdinary)) + if err != nil { + t.Fatal(err) + } + if err = s.BeginDispatch(context.Background(), attempt.ID, 1); err != nil { + t.Fatal(err) + } + if err = s.MarkStepUnknown(context.Background(), attempt.ID, 1); err != nil { + t.Fatal(err) + } + if _, err = s.FinalizeApply(context.Background(), attempt.ID); err != nil { + t.Fatal(err) + } + detail, err := s.Show(context.Background(), created.Stage.ID) + if err != nil { + t.Fatal(err) + } + if _, err = s.Cancel(context.Background(), CancelInput{detail.ID, "cancel-unknown", detail.Revision, detail.SemanticDigest}); err != nil { + t.Fatal(err) + } + ordinary := PruneInput{StageID: detail.ID, RequestID: "ordinary-unknown", ExpectedRevision: detail.Revision, ExpectedDigest: detail.SemanticDigest} + if _, err = s.Prune(context.Background(), ordinary); !errors.Is(err, ErrNotEligible) { + t.Fatalf("ordinary prune=%v", err) + } + ordinary.RequestID = "abandon-unknown" + ordinary.AbandonRecovery = true + pruned, err := s.Prune(context.Background(), ordinary) + if err != nil || pruned.Stage.Lifecycle != LifecyclePruned { + t.Fatalf("abandon=%+v err=%v", pruned, err) + } + events, err := s.RetentionEvents(context.Background(), detail.ID) + if err != nil || len(events) != 1 || events[0].Event != "abandoned_recovery" || events[0].RecoveryMaterial != MaterialUnknown { + t.Fatalf("events=%+v err=%v", events, err) + } + var material RecoveryMaterial + if err = s.db.QueryRow(`SELECT recovery_material FROM stages WHERE id=?`, detail.ID).Scan(&material); err != nil || material != MaterialNone { + t.Fatalf("material=%s err=%v", material, err) + } +} + +func TestExactPruneRollsBackEventLifecycleAndErasureTogether(t *testing.T) { + s := openDomainStore(t) + created, _ := s.Create(context.Background(), createInput("", "rollback me")) + if _, err := s.Cancel(context.Background(), CancelInput{created.Stage.ID, "", created.Stage.Revision, created.Stage.SemanticDigest}); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`CREATE TRIGGER fail_prune_erasure BEFORE UPDATE OF body ON stage_revisions WHEN OLD.stage_id='` + created.Stage.ID + `' BEGIN SELECT RAISE(ABORT,'injected prune failure'); END;`); err != nil { + t.Fatal(err) + } + _, err := s.Prune(context.Background(), PruneInput{StageID: created.Stage.ID, ExpectedRevision: created.Stage.Revision, ExpectedDigest: created.Stage.SemanticDigest}) + if err == nil { + t.Fatal("injected prune failure succeeded") + } + detail, showErr := s.Show(context.Background(), created.Stage.ID) + events, eventErr := s.RetentionEvents(context.Background(), created.Stage.ID) + if showErr != nil || eventErr != nil || detail.Lifecycle != LifecycleCanceled || detail.Body == nil || len(detail.Attachments) == 0 || len(events) != 0 { + t.Fatalf("detail=%+v events=%+v showErr=%v eventErr=%v", detail, events, showErr, eventErr) + } +} + +func TestBulkPruneSkipsCanceledStageWithRetainedUnknownMaterial(t *testing.T) { + s := openDomainStore(t) + ordinary, _ := s.Create(context.Background(), createInput("", "ordinary canceled")) + if _, err := s.Cancel(context.Background(), CancelInput{ordinary.Stage.ID, "", ordinary.Stage.Revision, ordinary.Stage.SemanticDigest}); err != nil { + t.Fatal(err) + } + unknown := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) + attempt, err := s.ClaimApply(context.Background(), claimInput(unknown.Stage, "bulk-unknown", RecoveryModeOrdinary)) + if err != nil { + t.Fatal(err) + } + if err = s.BeginDispatch(context.Background(), attempt.ID, 1); err != nil { + t.Fatal(err) + } + if err = s.MarkStepUnknown(context.Background(), attempt.ID, 1); err != nil { + t.Fatal(err) + } + if _, err = s.FinalizeApply(context.Background(), attempt.ID); err != nil { + t.Fatal(err) + } + unknownDetail, err := s.Show(context.Background(), unknown.Stage.ID) + if err != nil { + t.Fatal(err) + } + if _, err = s.Cancel(context.Background(), CancelInput{unknownDetail.ID, "", unknownDetail.Revision, unknownDetail.SemanticDigest}); err != nil { + t.Fatal(err) + } + cutoff := time.Now().UTC().Add(time.Hour) + result, err := s.PruneEligible(context.Background(), cutoff, cutoff.Add(time.Hour)) + if err != nil || result.PrunedCount != 1 { + t.Fatalf("result=%+v err=%v", result, err) + } + ordinaryDetail, ordinaryErr := s.Show(context.Background(), ordinary.Stage.ID) + unknownDetail, unknownErr := s.Show(context.Background(), unknown.Stage.ID) + if ordinaryErr != nil || ordinaryDetail.Lifecycle != LifecyclePruned || unknownErr != nil || unknownDetail.Lifecycle != LifecycleCanceled || unknownDetail.Body == nil { + t.Fatalf("ordinary=%+v/%v unknown=%+v/%v", ordinaryDetail, ordinaryErr, unknownDetail, unknownErr) + } +} + +func TestMigrationBackfillsCanceledRecoveryMaterialFromAttemptHistory(t *testing.T) { + path := testPath(t) + original := migrations + migrations = append([]migration(nil), original[:10]...) + t.Cleanup(func() { migrations = original }) + s, err := Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + unknown := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"create_post","condition":"always"}]}`) + unknownAttempt, err := s.ClaimApply(context.Background(), claimInput(unknown.Stage, "migration-unknown", RecoveryModeOrdinary)) + if err != nil { + t.Fatal(err) + } + if err = s.BeginDispatch(context.Background(), unknownAttempt.ID, 1); err != nil { + t.Fatal(err) + } + if err = s.MarkStepUnknown(context.Background(), unknownAttempt.ID, 1); err != nil { + t.Fatal(err) + } + if _, err = s.FinalizeApply(context.Background(), unknownAttempt.ID); err != nil { + t.Fatal(err) + } + unknownCurrent, _ := s.Show(context.Background(), unknown.Stage.ID) + if _, err = s.Cancel(context.Background(), CancelInput{unknownCurrent.ID, "", unknownCurrent.Revision, unknownCurrent.SemanticDigest}); err != nil { + t.Fatal(err) + } + partial := createApplyStage(t, s, `{"steps":[{"ordinal":1,"type":"upload_attachment","condition":"always"},{"ordinal":2,"type":"create_post","condition":"always"}]}`) + partialAttempt, err := s.ClaimApply(context.Background(), claimInput(partial.Stage, "migration-partial", RecoveryModeOrdinary)) + if err != nil { + t.Fatal(err) + } + if err = s.BeginDispatch(context.Background(), partialAttempt.ID, 1); err != nil { + t.Fatal(err) + } + if err = s.MarkStepValidated(context.Background(), partialAttempt.ID, 1, []byte(`{"fileId":"file-1"}`)); err != nil { + t.Fatal(err) + } + if err = s.BeginDispatch(context.Background(), partialAttempt.ID, 2); err != nil { + t.Fatal(err) + } + if err = s.MarkStepRejected(context.Background(), partialAttempt.ID, 2, []byte(`{"status":403}`)); err != nil { + t.Fatal(err) + } + if _, err = s.FinalizeApply(context.Background(), partialAttempt.ID); err != nil { + t.Fatal(err) + } + partialCurrent, _ := s.Show(context.Background(), partial.Stage.ID) + if _, err = s.Cancel(context.Background(), CancelInput{partialCurrent.ID, "", partialCurrent.Revision, partialCurrent.SemanticDigest}); err != nil { + t.Fatal(err) + } + if err = s.Close(); err != nil { + t.Fatal(err) + } + migrations = original + s, err = Open(context.Background(), path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + for _, want := range []struct { + id string + material RecoveryMaterial + }{{unknown.Stage.ID, MaterialUnknown}, {partial.Stage.ID, MaterialPartial}} { + var got RecoveryMaterial + if err = s.db.QueryRow(`SELECT recovery_material FROM stages WHERE id=?`, want.id).Scan(&got); err != nil || got != want.material { + t.Fatalf("stage=%s material=%s want=%s err=%v", want.id, got, want.material, err) + } + } +} diff --git a/internal/stagestore/schema.go b/internal/stagestore/schema.go index 32567ca..9ead213 100644 --- a/internal/stagestore/schema.go +++ b/internal/stagestore/schema.go @@ -541,6 +541,111 @@ WHEN NOT ( OR OLD.state='dispatch_intent' AND NEW.state IN ('response_validated','rejected','outcome_unknown') ) BEGIN SELECT RAISE(ABORT, 'invalid apply step transition'); END; +`}, {version: 11, name: "retention-lifecycle-audit", sql: ` +ALTER TABLE stages ADD COLUMN recovery_material TEXT NOT NULL DEFAULT 'none' + CHECK (recovery_material IN ('none','resume_partial','force_unknown')); +UPDATE stages SET recovery_material=CASE + WHEN lifecycle IN ('completed','expired','pruned') THEN 'none' + WHEN recovery='force_unknown' THEN 'force_unknown' + WHEN recovery='resume_partial' THEN 'resume_partial' + WHEN lifecycle='canceled' AND EXISTS( + SELECT 1 FROM apply_attempts a WHERE a.stage_id=stages.id + AND (a.outcome='unknown' OR a.prior_recovery='force_unknown') + ) THEN 'force_unknown' + WHEN lifecycle='canceled' AND EXISTS( + SELECT 1 FROM apply_attempts a WHERE a.stage_id=stages.id AND a.outcome='partial' + ) THEN 'resume_partial' + ELSE 'none' +END; +CREATE TABLE stage_retention_events ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + stage_id TEXT NOT NULL REFERENCES stages(id), + revision INTEGER NOT NULL CHECK (revision > 0), + semantic_digest BLOB NOT NULL CHECK (length(semantic_digest) = 32), + event TEXT NOT NULL CHECK (event IN ('expired','revived','pruned','abandoned_recovery')), + from_lifecycle TEXT NOT NULL CHECK (from_lifecycle IN ('open','applying','completed','canceled','expired','pruned')), + from_recovery TEXT NOT NULL CHECK (from_recovery IN ('none','resume_partial','force_unknown','forbidden')), + recovery_material TEXT NOT NULL CHECK (recovery_material IN ('none','resume_partial','force_unknown')), + policy_seconds INTEGER CHECK (policy_seconds IS NULL OR policy_seconds > 0), + request_id TEXT, + recorded_at TEXT NOT NULL +) STRICT; +CREATE INDEX stages_expiry_eligible ON stages(updated_at,id) + WHERE lifecycle='open' AND recovery='none' AND recovery_material='none' AND claim_attempt_id IS NULL; +CREATE INDEX stage_retention_events_stage ON stage_retention_events(stage_id,sequence); +CREATE TRIGGER stage_retention_events_insert_valid BEFORE INSERT ON stage_retention_events +WHEN NOT EXISTS( + SELECT 1 FROM stages s JOIN stage_revisions r ON r.stage_id=s.id AND r.revision=s.current_revision + WHERE s.id=NEW.stage_id AND s.current_revision=NEW.revision AND r.semantic_digest=NEW.semantic_digest + AND s.lifecycle=NEW.from_lifecycle AND s.recovery=NEW.from_recovery AND s.recovery_material=NEW.recovery_material + AND ( + NEW.event='expired' AND s.lifecycle='open' AND s.recovery='none' AND s.recovery_material='none' + AND s.claim_attempt_id IS NULL AND NEW.policy_seconds IS NOT NULL AND NEW.request_id IS NULL + OR NEW.event='revived' AND s.lifecycle='expired' AND s.recovery='forbidden' AND s.recovery_material='none' + AND s.claim_attempt_id IS NULL AND NEW.policy_seconds IS NULL AND NEW.request_id IS NULL + OR NEW.event='pruned' AND s.lifecycle IN ('completed','canceled','expired') AND s.recovery='forbidden' + AND s.recovery_material='none' AND s.claim_attempt_id IS NULL + OR NEW.event='abandoned_recovery' AND s.lifecycle!='applying' AND s.recovery_material IN ('resume_partial','force_unknown') + AND s.claim_attempt_id IS NULL AND NEW.policy_seconds IS NULL + ) +) +BEGIN SELECT RAISE(ABORT, 'invalid stage retention event'); END; +CREATE TRIGGER stage_retention_events_immutable_update BEFORE UPDATE ON stage_retention_events +BEGIN SELECT RAISE(ABORT, 'stage retention events are immutable'); END; +CREATE TRIGGER stage_retention_events_immutable_delete BEFORE DELETE ON stage_retention_events +BEGIN SELECT RAISE(ABORT, 'stage retention events are immutable'); END; +DROP TRIGGER stage_lifecycle_recovery_transition_valid; +CREATE TRIGGER stage_lifecycle_recovery_transition_valid BEFORE UPDATE OF lifecycle,recovery,recovery_material ON stages +WHEN (NEW.lifecycle IS NOT OLD.lifecycle OR NEW.recovery IS NOT OLD.recovery OR NEW.recovery_material IS NOT OLD.recovery_material) AND NOT ( + OLD.lifecycle='open' AND NEW.lifecycle='applying' AND NEW.recovery=OLD.recovery AND NEW.recovery_material=OLD.recovery_material AND NEW.claim_attempt_id IS NOT NULL + AND EXISTS(SELECT 1 FROM apply_attempts a WHERE a.id=NEW.claim_attempt_id AND a.stage_id=OLD.id AND a.outcome IS NULL) + OR OLD.lifecycle='applying' AND NEW.lifecycle='open' AND NEW.recovery=OLD.recovery AND NEW.recovery_material=OLD.recovery_material AND NEW.claim_attempt_id IS NULL + AND EXISTS(SELECT 1 FROM apply_attempts a WHERE a.id=OLD.claim_attempt_id AND a.outcome IS NULL) + AND NOT EXISTS(SELECT 1 FROM apply_steps p WHERE p.attempt_id=OLD.claim_attempt_id AND p.state!='pending') + OR OLD.lifecycle='applying' AND NEW.claim_attempt_id IS NULL AND EXISTS( + SELECT 1 FROM apply_attempts a WHERE a.id=OLD.claim_attempt_id AND a.outcome IS NOT NULL AND ( + a.outcome IN ('succeeded','already_satisfied') AND NEW.lifecycle='completed' AND NEW.recovery='forbidden' AND NEW.recovery_material='none' + OR a.outcome='rejected' AND NEW.lifecycle='open' AND NEW.recovery=a.prior_recovery + AND NEW.recovery_material=CASE WHEN a.prior_recovery='force_unknown' THEN 'force_unknown' WHEN a.prior_recovery='resume_partial' THEN 'resume_partial' ELSE 'none' END + OR a.outcome='partial' AND NEW.lifecycle='open' AND NEW.recovery=CASE WHEN a.prior_recovery='force_unknown' THEN 'force_unknown' ELSE 'resume_partial' END + AND NEW.recovery_material=CASE WHEN a.prior_recovery='force_unknown' THEN 'force_unknown' ELSE 'resume_partial' END + OR a.outcome='unknown' AND NEW.lifecycle='open' AND NEW.recovery='force_unknown' AND NEW.recovery_material='force_unknown' + ) + ) + OR OLD.lifecycle='open' AND NEW.lifecycle='canceled' AND NEW.recovery='forbidden' AND NEW.recovery_material=OLD.recovery_material + AND OLD.claim_attempt_id IS NULL AND NEW.claim_attempt_id IS NULL + OR OLD.lifecycle='open' AND OLD.recovery='none' AND OLD.recovery_material='none' + AND NEW.lifecycle='expired' AND NEW.recovery='forbidden' AND NEW.recovery_material='none' + AND OLD.claim_attempt_id IS NULL AND NEW.claim_attempt_id IS NULL + AND EXISTS(SELECT 1 FROM stage_retention_events e WHERE e.stage_id=OLD.id AND e.revision=OLD.current_revision + AND e.semantic_digest=(SELECT semantic_digest FROM stage_revisions WHERE stage_id=OLD.id AND revision=OLD.current_revision) + AND e.event='expired' AND e.from_lifecycle=OLD.lifecycle AND e.from_recovery=OLD.recovery + AND e.recovery_material=OLD.recovery_material) + OR OLD.lifecycle='expired' AND OLD.recovery='forbidden' AND OLD.recovery_material='none' + AND NEW.lifecycle='open' AND NEW.recovery='none' AND NEW.recovery_material='none' + AND OLD.claim_attempt_id IS NULL AND NEW.claim_attempt_id IS NULL AND NEW.current_revision=OLD.current_revision+1 + AND EXISTS(SELECT 1 FROM stage_retention_events e WHERE e.stage_id=OLD.id AND e.revision=OLD.current_revision + AND e.event='revived' AND e.from_lifecycle=OLD.lifecycle AND e.from_recovery=OLD.recovery AND e.recovery_material=OLD.recovery_material) + AND EXISTS(SELECT 1 FROM stage_revisions r WHERE r.stage_id=OLD.id AND r.revision=OLD.current_revision AND r.state='superseded') + AND EXISTS(SELECT 1 FROM stage_revisions r WHERE r.stage_id=OLD.id AND r.revision=NEW.current_revision AND r.state='current') + OR OLD.lifecycle!='applying' AND NEW.lifecycle='pruned' AND NEW.recovery='forbidden' AND NEW.recovery_material='none' + AND OLD.claim_attempt_id IS NULL AND NEW.claim_attempt_id IS NULL + AND EXISTS(SELECT 1 FROM stage_retention_events e WHERE e.stage_id=OLD.id AND e.revision=OLD.current_revision + AND e.semantic_digest=(SELECT semantic_digest FROM stage_revisions WHERE stage_id=OLD.id AND revision=OLD.current_revision) + AND e.event=CASE WHEN OLD.recovery_material='none' THEN 'pruned' ELSE 'abandoned_recovery' END + AND e.from_lifecycle=OLD.lifecycle AND e.from_recovery=OLD.recovery AND e.recovery_material=OLD.recovery_material) +) +BEGIN SELECT RAISE(ABORT, 'invalid stage lifecycle, recovery, or retained material transition'); END; +DROP TRIGGER stage_revision_body_erasure_only; +CREATE TRIGGER stage_revision_body_erasure_only BEFORE UPDATE OF body ON stage_revisions +WHEN NEW.body IS NOT OLD.body AND NOT (OLD.body IS NOT NULL AND NEW.body IS NULL AND EXISTS( + SELECT 1 FROM stages s WHERE s.id=OLD.stage_id AND s.lifecycle IN ('completed','pruned') AND s.recovery='forbidden' +)) +BEGIN SELECT RAISE(ABORT, 'stage revision body is immutable'); END; +DROP TRIGGER stage_attachment_delete_after_completion; +CREATE TRIGGER stage_attachment_delete_after_completion BEFORE DELETE ON stage_attachments +WHEN NOT EXISTS(SELECT 1 FROM stages s WHERE s.id=OLD.stage_id AND s.lifecycle IN ('completed','pruned') AND s.recovery='forbidden') +BEGIN SELECT RAISE(ABORT, 'stage attachment bindings are immutable'); END; `}} func attachmentIdentityAvailable() bool { @@ -550,3 +655,7 @@ func attachmentIdentityAvailable() bool { func validatedUploadReuseAvailable() bool { return len(migrations) >= 10 && migrations[9].version == 10 && migrations[9].name == "validated-upload-reuse" } + +func retentionLifecycleAvailable() bool { + return len(migrations) >= 11 && migrations[10].version == 11 && migrations[10].name == "retention-lifecycle-audit" +} diff --git a/schemas/v2/config.schema.json b/schemas/v2/config.schema.json index 9b3ea9c..f5de2e6 100644 --- a/schemas/v2/config.schema.json +++ b/schemas/v2/config.schema.json @@ -4,7 +4,7 @@ "title": "mm v2 configuration status", "type": "object", "additionalProperties": false, - "required": ["schema", "action", "selectedPath", "readPath", "migration", "exists", "urlConfigured", "tokenConfigured", "permissions", "readStatus", "parseStatus", "unsafeReason", "created", "warning"], + "required": ["schema", "action", "selectedPath", "readPath", "migration", "exists", "urlConfigured", "tokenConfigured", "stageTtlSeconds", "stagePruneAfterSeconds", "permissions", "readStatus", "parseStatus", "unsafeReason", "created", "warning"], "properties": { "schema": { "const": "mm/v2/config" }, "action": { "enum": ["status", "path", "init"] }, @@ -14,6 +14,8 @@ "exists": { "type": ["boolean", "null"] }, "urlConfigured": { "type": ["boolean", "null"] }, "tokenConfigured": { "type": ["boolean", "null"] }, + "stageTtlSeconds": { "type": ["integer", "null"], "minimum": 0, "maximum": 9223372036 }, + "stagePruneAfterSeconds": { "type": ["integer", "null"], "minimum": 0, "maximum": 9223372036 }, "permissions": { "type": ["string", "null"], "enum": [null, "secure", "insecure", "unknown", "not_applicable"] }, "readStatus": { "type": ["string", "null"], "enum": [null, "ok", "missing", "error"] }, "parseStatus": { "type": ["string", "null"], "enum": [null, "ok", "not_attempted", "error"] }, @@ -24,7 +26,7 @@ "allOf": [ { "if": { "properties": { "action": { "const": "path" } }, "required": ["action"] }, - "then": { "properties": { "readPath": { "const": null }, "migration": { "const": null }, "exists": { "const": null }, "urlConfigured": { "const": null }, "tokenConfigured": { "const": null }, "permissions": { "const": null }, "readStatus": { "const": null }, "parseStatus": { "const": null }, "unsafeReason": { "const": null }, "created": { "const": null }, "warning": { "const": null } } } + "then": { "properties": { "readPath": { "const": null }, "migration": { "const": null }, "exists": { "const": null }, "urlConfigured": { "const": null }, "tokenConfigured": { "const": null }, "stageTtlSeconds": { "const": null }, "stagePruneAfterSeconds": { "const": null }, "permissions": { "const": null }, "readStatus": { "const": null }, "parseStatus": { "const": null }, "unsafeReason": { "const": null }, "created": { "const": null }, "warning": { "const": null } } } }, { "if": { "properties": { "action": { "const": "status" } }, "required": ["action"] }, @@ -49,7 +51,7 @@ }, { "if": { "properties": { "readStatus": { "const": "error" } }, "required": ["readStatus"] }, - "then": { "properties": { "exists": { "const": true }, "urlConfigured": { "const": false }, "tokenConfigured": { "const": false }, "permissions": { "const": "unknown" }, "parseStatus": { "const": "not_attempted" } } } + "then": { "properties": { "exists": { "const": true }, "urlConfigured": { "const": false }, "tokenConfigured": { "const": false }, "stageTtlSeconds": { "const": null }, "stagePruneAfterSeconds": { "const": null }, "permissions": { "const": "unknown" }, "parseStatus": { "const": "not_attempted" } } } }, { "if": { "properties": { "readStatus": { "const": "missing" } }, "required": ["readStatus"] }, @@ -81,7 +83,7 @@ }, { "if": { "properties": { "parseStatus": { "const": "error" } }, "required": ["parseStatus"] }, - "then": { "properties": { "exists": { "const": true }, "urlConfigured": { "const": false }, "tokenConfigured": { "const": false }, "readStatus": { "const": "ok" }, "permissions": { "enum": ["secure", "insecure"] }, "unsafeReason": { "const": null } } } + "then": { "properties": { "exists": { "const": true }, "urlConfigured": { "const": false }, "tokenConfigured": { "const": false }, "stageTtlSeconds": { "const": null }, "stagePruneAfterSeconds": { "const": null }, "readStatus": { "const": "ok" }, "permissions": { "enum": ["secure", "insecure"] }, "unsafeReason": { "const": null } } } }, { "if": { "properties": { "parseStatus": { "const": "ok" } }, "required": ["parseStatus"] }, diff --git a/schemas/v2/examples/config.json b/schemas/v2/examples/config.json index a3c4428..59b57f2 100644 --- a/schemas/v2/examples/config.json +++ b/schemas/v2/examples/config.json @@ -1 +1 @@ -{"schema":"mm/v2/config","action":"status","selectedPath":"/home/agent/.config/mattermost-cli/config.toml","readPath":null,"migration":"none","exists":false,"urlConfigured":false,"tokenConfigured":false,"permissions":"not_applicable","readStatus":"missing","parseStatus":"not_attempted","unsafeReason":null,"created":null,"warning":null} +{"schema":"mm/v2/config","action":"status","selectedPath":"/home/agent/.config/mattermost-cli/config.toml","readPath":null,"migration":"none","exists":false,"urlConfigured":false,"tokenConfigured":false,"stageTtlSeconds":0,"stagePruneAfterSeconds":0,"permissions":"not_applicable","readStatus":"missing","parseStatus":"not_attempted","unsafeReason":null,"created":null,"warning":null} diff --git a/schemas/v2/examples/stage-prune-request.json b/schemas/v2/examples/stage-prune-request.json new file mode 100644 index 0000000..9d567a6 --- /dev/null +++ b/schemas/v2/examples/stage-prune-request.json @@ -0,0 +1 @@ +{"schema":"mm/v2/stage-prune-request","requestId":"agent-20260717-004","stageId":"stg_0123456789abcdefghijklmnopqrstuv","expectedRevision":3,"expectedDigest":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","abandonRecovery":false} diff --git a/schemas/v2/examples/stage-prune-result.json b/schemas/v2/examples/stage-prune-result.json new file mode 100644 index 0000000..ca80cc8 --- /dev/null +++ b/schemas/v2/examples/stage-prune-result.json @@ -0,0 +1 @@ +{"schema":"mm/v2/stage-prune-result","action":"pruned","cutoff":"2026-07-01T00:00:00.000Z","prunedCount":3,"recordedAt":"2026-07-17T12:00:00.000Z"} diff --git a/schemas/v2/examples/store-doctor.json b/schemas/v2/examples/store-doctor.json index 8473a3c..8c050c6 100644 --- a/schemas/v2/examples/store-doctor.json +++ b/schemas/v2/examples/store-doctor.json @@ -1 +1 @@ -{"schema":"mm/v2/store-doctor","path":"/home/arda/.local/state/mattermost-cli/stages.sqlite3","report":{"exists":false,"filesystemSafe":null,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":10,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}} +{"schema":"mm/v2/store-doctor","path":"/home/arda/.local/state/mattermost-cli/stages.sqlite3","report":{"exists":false,"filesystemSafe":null,"applicationId":null,"integrity":null,"integrityTruncated":null,"foreignKeyIssues":null,"foreignKeyRows":null,"foreignKeyTruncated":null,"migrations":{"applied":null,"latest":11,"valid":null},"journalMode":null,"synchronous":null,"secureDelete":null,"foreignKeys":null,"trustedSchema":null,"queryOnly":null,"walFallback":null,"permissionModelLimitations":[]}} diff --git a/schemas/v2/examples/store-migrations.json b/schemas/v2/examples/store-migrations.json index a93d1ec..c4e9f92 100644 --- a/schemas/v2/examples/store-migrations.json +++ b/schemas/v2/examples/store-migrations.json @@ -1,6 +1,6 @@ { "schema": "mm/v2/store-migrations", - "latest": 10, + "latest": 11, "migrations": [ {"version": 1, "name": "core-stage-state", "checksum": "e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f"}, {"version": 2, "name": "immutable-local-request-receipts", "checksum": "ac1c291e7201786935cce68dd459f61150048c505e4f29ad813304c309009b8c"}, @@ -11,6 +11,7 @@ {"version": 7, "name": "status-confirmed-delete-results", "checksum": "bf980f24ff24fdf9d6a3a28b9ded97b51336fa1c8ef290d1bdcf91f1b593c5ce"}, {"version": 8, "name": "already-satisfied-edit-apply", "checksum": "cff30dd4fcc987876e092e2de1f5afd21ebeadbcb792fd550a56eb4c221edd10"}, {"version": 9, "name": "attachment-identity-binding", "checksum": "d0782e01014d010e5978b39bf6f04ef26508bec9318c7ace3be8bfbbcab4999d"}, - {"version": 10, "name": "validated-upload-reuse", "checksum": "75088d75d41eab9d7d1b1c7b1fa6fc4ce604849321b88feeb110d7ada3447947"} + {"version": 10, "name": "validated-upload-reuse", "checksum": "75088d75d41eab9d7d1b1c7b1fa6fc4ce604849321b88feeb110d7ada3447947"}, + {"version": 11, "name": "retention-lifecycle-audit", "checksum": "f57092f61a66e5b10e6c748fcb85f2dc6ca2fe965d1c4ad819e399e56602f941"} ] } diff --git a/schemas/v2/stage-prune-request.schema.json b/schemas/v2/stage-prune-request.schema.json new file mode 100644 index 0000000..f1c2ef8 --- /dev/null +++ b/schemas/v2/stage-prune-request.schema.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:stage-prune-request", + "type": "object", + "additionalProperties": false, + "required": ["schema", "requestId", "stageId", "expectedRevision", "expectedDigest", "abandonRecovery"], + "properties": { + "schema": { "const": "mm/v2/stage-prune-request" }, + "requestId": { "$ref": "#/$defs/requestId" }, + "stageId": { "$ref": "#/$defs/stageId" }, + "expectedRevision": { "$ref": "#/$defs/revision" }, + "expectedDigest": { "$ref": "#/$defs/digest" }, + "abandonRecovery": { "type": "boolean" } + }, + "$defs": { + "requestId": { + "type": "string", "minLength": 1, "maxLength": 256, + "pattern": "^[A-Za-z0-9][A-Za-z0-9._~:-]*$" + }, + "stageId": { "type": "string", "pattern": "^stg_[A-Za-z0-9_-]{32}$" }, + "revision": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, + "digest": { "type": "string", "pattern": "^[0-9a-f]{64}$" } + } +} diff --git a/schemas/v2/stage-prune-result.schema.json b/schemas/v2/stage-prune-result.schema.json new file mode 100644 index 0000000..d11a97e --- /dev/null +++ b/schemas/v2/stage-prune-result.schema.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:mm:schema:v2:stage-prune-result", + "type": "object", + "additionalProperties": false, + "required": ["schema", "action", "cutoff", "prunedCount", "recordedAt"], + "properties": { + "schema": { "const": "mm/v2/stage-prune-result" }, + "action": { "const": "pruned" }, + "cutoff": { "$ref": "#/$defs/timestamp" }, + "prunedCount": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, + "recordedAt": { "$ref": "#/$defs/timestamp" } + }, + "$defs": { + "timestamp": { + "type": "string", + "format": "date-time", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]{3}Z$" + } + } +} diff --git a/schemas/v2/stage-receipt.schema.json b/schemas/v2/stage-receipt.schema.json index cb3fccc..01f6b08 100644 --- a/schemas/v2/stage-receipt.schema.json +++ b/schemas/v2/stage-receipt.schema.json @@ -19,7 +19,8 @@ "enum": [ "created", "revised", - "canceled" + "canceled", + "pruned" ] }, "revived": { @@ -118,6 +119,25 @@ } } }, + { + "if": { + "properties": { + "action": { + "const": "pruned" + } + } + }, + "then": { + "properties": { + "stage": { + "properties": { + "lifecycle": { "const": "pruned" }, + "recovery": { "const": "forbidden" } + } + } + } + } + }, { "if": { "properties": { diff --git a/schemas/v2/store-doctor.schema.json b/schemas/v2/store-doctor.schema.json index 255a5e6..d127d5f 100644 --- a/schemas/v2/store-doctor.schema.json +++ b/schemas/v2/store-doctor.schema.json @@ -15,7 +15,7 @@ "exists": { "type": "boolean" }, "filesystemSafe": {}, "applicationId": {}, "integrity": {}, "integrityTruncated": {}, "foreignKeyIssues": {}, "foreignKeyRows": {}, "foreignKeyTruncated": {}, "journalMode": {}, "synchronous": {}, "secureDelete": {}, "foreignKeys": {}, "trustedSchema": {}, "queryOnly": {}, "walFallback": {}, - "migrations": { "type": "object", "additionalProperties": false, "required": ["applied", "latest", "valid"], "properties": { "applied": {}, "latest": { "const": 10 }, "valid": {} } }, + "migrations": { "type": "object", "additionalProperties": false, "required": ["applied", "latest", "valid"], "properties": { "applied": {}, "latest": { "const": 11 }, "valid": {} } }, "permissionModelLimitations": { "type": "array", "items": { "$ref": "#/$defs/safeString" } } }, "allOf": [ @@ -23,14 +23,14 @@ "filesystemSafe": { "const": null }, "applicationId": { "const": null }, "integrity": { "const": null }, "integrityTruncated": { "const": null }, "foreignKeyIssues": { "const": null }, "foreignKeyRows": { "const": null }, "foreignKeyTruncated": { "const": null }, "journalMode": { "const": null }, "synchronous": { "const": null }, "secureDelete": { "const": null }, "foreignKeys": { "const": null }, "trustedSchema": { "const": null }, "queryOnly": { "const": null }, "walFallback": { "const": null }, - "migrations": { "properties": { "applied": { "const": null }, "latest": { "const": 10 }, "valid": { "const": null } } } + "migrations": { "properties": { "applied": { "const": null }, "latest": { "const": 11 }, "valid": { "const": null } } } } } }, { "if": { "properties": { "exists": { "const": true } }, "required": ["exists"] }, "then": { "properties": { "filesystemSafe": { "const": true }, "applicationId": { "const": 1296913970 }, "integrity": { "$ref": "#/$defs/nonemptyBoundedStrings" }, "integrityTruncated": { "type": "boolean" }, "foreignKeyIssues": { "type": "integer", "minimum": 0, "maximum": 21 }, "foreignKeyRows": { "$ref": "#/$defs/boundedStrings" }, "foreignKeyTruncated": { "type": "boolean" }, "journalMode": { "enum": ["wal", "delete", "truncate", "persist"] }, "synchronous": { "type": "integer" }, "secureDelete": { "type": "integer" }, "foreignKeys": { "const": true }, "trustedSchema": { "const": false }, "queryOnly": { "const": true }, "walFallback": { "type": "boolean" }, - "migrations": { "properties": { "applied": { "const": 10 }, "latest": { "const": 10 }, "valid": { "const": true } } } + "migrations": { "properties": { "applied": { "const": 11 }, "latest": { "const": 11 }, "valid": { "const": true } } } }, "allOf": [ { "oneOf": [ { "properties": { "integrity": { "const": ["ok"] }, "integrityTruncated": { "const": false } } }, diff --git a/schemas/v2/store-migrations.schema.json b/schemas/v2/store-migrations.schema.json index 68f7147..7667dde 100644 --- a/schemas/v2/store-migrations.schema.json +++ b/schemas/v2/store-migrations.schema.json @@ -6,9 +6,9 @@ "required": ["schema", "latest", "migrations"], "properties": { "schema": { "const": "mm/v2/store-migrations" }, - "latest": { "const": 10 }, + "latest": { "const": 11 }, "migrations": { - "type": "array", "minItems": 10, "maxItems": 10, + "type": "array", "minItems": 11, "maxItems": 11, "prefixItems": [{ "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { "version": { "const": 1 }, "name": { "const": "core-stage-state" }, "checksum": { "const": "e69a3e2524903dbdb4ef5a9691ce8674fac8fe284d09e95bc5983ec9e9fef92f" } } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { @@ -29,6 +29,8 @@ "version": { "const": 9 }, "name": { "const": "attachment-identity-binding" }, "checksum": { "const": "d0782e01014d010e5978b39bf6f04ef26508bec9318c7ace3be8bfbbcab4999d" } } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { "version": { "const": 10 }, "name": { "const": "validated-upload-reuse" }, "checksum": { "const": "75088d75d41eab9d7d1b1c7b1fa6fc4ce604849321b88feeb110d7ada3447947" } + } }, { "type": "object", "additionalProperties": false, "required": ["version", "name", "checksum"], "properties": { + "version": { "const": 11 }, "name": { "const": "retention-lifecycle-audit" }, "checksum": { "const": "f57092f61a66e5b10e6c748fcb85f2dc6ca2fe965d1c4ad819e399e56602f941" } } }], "items": false } From 944c9405f1ad77efd8130ac3f17162e1c02fce44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 12:25:36 +0300 Subject: [PATCH 102/119] test: add paired differential conformance corpus --- cmd/conformance/main.go | 59 +++++++++--- conformance/README.md | 2 +- conformance/pair.schema.json | 80 +++++++++++++++++ .../scenarios/pairs/invalid-limit.json | 22 +++++ conformance/scenarios/pairs/teams.json | 62 +++++++++++++ conformance/scenarios/pairs/whoami.json | 52 +++++++++++ docs/V1_PARITY_MATRIX.md | 18 +++- internal/conformance/runner.go | 10 +++ internal/conformance/runner_test.go | 16 ++++ internal/conformance/scenario.go | 90 ++++++++++++++----- internal/conformance/scenario_test.go | 29 ++++++ justfile | 5 +- 12 files changed, 409 insertions(+), 36 deletions(-) create mode 100644 conformance/pair.schema.json create mode 100644 conformance/scenarios/pairs/invalid-limit.json create mode 100644 conformance/scenarios/pairs/teams.json create mode 100644 conformance/scenarios/pairs/whoami.json diff --git a/cmd/conformance/main.go b/cmd/conformance/main.go index 77969bd..8856279 100644 --- a/cmd/conformance/main.go +++ b/cmd/conformance/main.go @@ -11,32 +11,69 @@ import ( "github.com/ardasevinc/mattermost-cli/internal/conformance" ) +type stringList []string + +func (values *stringList) String() string { return fmt.Sprint([]string(*values)) } +func (values *stringList) Set(value string) error { + *values = append(*values, value) + return nil +} + func main() { flags := flag.NewFlagSet("conformance", flag.ContinueOnError) flags.SetOutput(os.Stderr) scenarioPath := flags.String("scenario", "", "path to a conformance scenario") + pairPath := flags.String("pair", "", "path to a paired oracle/candidate scenario") cwd := flags.String("cwd", "", "working directory for the command under test") + oraclePath := flags.String("oracle", "", "oracle executable for a paired scenario") + candidatePath := flags.String("candidate", "", "candidate executable for a paired scenario") + var oraclePrefix, candidatePrefix stringList + flags.Var(&oraclePrefix, "oracle-prefix", "repeatable oracle prefix argument") + flags.Var(&candidatePrefix, "candidate-prefix", "repeatable candidate prefix argument") if err := flags.Parse(os.Args[1:]); err != nil { os.Exit(2) } command := flags.Args() - if *scenarioPath == "" || len(command) == 0 { - _, _ = fmt.Fprintln(os.Stderr, "usage: conformance --scenario FILE [--cwd DIR] -- COMMAND [PREFIX_ARGS...]") + if (*scenarioPath == "") == (*pairPath == "") { + _, _ = fmt.Fprintln(os.Stderr, "exactly one of --scenario or --pair is required") + os.Exit(2) + } + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + if *scenarioPath != "" { + if len(command) == 0 || *oraclePath != "" || *candidatePath != "" || len(oraclePrefix) != 0 || len(candidatePrefix) != 0 { + _, _ = fmt.Fprintln(os.Stderr, "usage: conformance --scenario FILE [--cwd DIR] -- COMMAND [PREFIX_ARGS...]") + os.Exit(2) + } + scenario, err := conformance.Load(*scenarioPath) + if err == nil { + err = conformance.Run(ctx, conformance.Command{ + Path: command[0], + PrefixArgs: command[1:], + Dir: *cwd, + }, scenario) + } + finish(scenario.Name, err) + } + if len(command) != 0 || *oraclePath == "" || *candidatePath == "" { + _, _ = fmt.Fprintln(os.Stderr, "usage: conformance --pair FILE [--cwd DIR] --oracle PATH [--oracle-prefix ARG...] --candidate PATH [--candidate-prefix ARG...]") os.Exit(2) } - scenario, err := conformance.Load(*scenarioPath) + pair, err := conformance.LoadPair(*pairPath) if err == nil { - ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer cancel() - err = conformance.Run(ctx, conformance.Command{ - Path: command[0], - PrefixArgs: command[1:], - Dir: *cwd, - }, scenario) + err = conformance.RunPair(ctx, + conformance.Command{Path: *oraclePath, PrefixArgs: oraclePrefix, Dir: *cwd}, + conformance.Command{Path: *candidatePath, PrefixArgs: candidatePrefix, Dir: *cwd}, + pair, + ) } + finish(pair.Name, err) +} + +func finish(name string, err error) { if err != nil { _, _ = fmt.Fprintln(os.Stderr, "conformance:", err) os.Exit(1) } - _, _ = fmt.Fprintln(os.Stdout, "passed:", scenario.Name) + _, _ = fmt.Fprintln(os.Stdout, "passed:", name) } diff --git a/conformance/README.md b/conformance/README.md index c19cca7..ce08028 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -2,6 +2,6 @@ This directory owns language-neutral Mattermost scenarios used to compare the frozen TypeScript v1.6.0 oracle with Go v2. -Scenarios describe fake-server behavior, expected request order and bytes, subprocess stdout/stderr, and exit classes. `scenario.schema.json` is the language-neutral manifest contract. The Go runner executes scenarios against a selected binary. Fixtures remain after the TypeScript implementation is removed. +Scenarios describe fake-server behavior, expected request order and bytes, subprocess stdout/stderr, and exit classes. `scenario.schema.json` covers one executable. `pair.schema.json` locks independent oracle and candidate expectations in one semantic case, so intentional v2 output changes are explicit instead of hidden by a false byte-equality claim. The Go runner executes both sides in isolated homes against independent fake servers. Fixtures remain after the TypeScript implementation is removed. The scenario schema and runner land before feature ports. See `docs/V2_CONTRACT.md` and `docs/V1_PARITY_MATRIX.md` for the acceptance boundary. diff --git a/conformance/pair.schema.json b/conformance/pair.schema.json new file mode 100644 index 0000000..df16bab --- /dev/null +++ b/conformance/pair.schema.json @@ -0,0 +1,80 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "mm/conformance-pair/v1", + "title": "mattermost-cli paired conformance scenario", + "type": "object", + "additionalProperties": false, + "required": ["schema", "name", "oracle", "candidate"], + "properties": { + "schema": { "const": "mm/conformance-pair/v1" }, + "name": { "type": "string", "minLength": 1 }, + "oracle": { "$ref": "#/$defs/case" }, + "candidate": { "$ref": "#/$defs/case" } + }, + "$defs": { + "case": { + "type": "object", + "additionalProperties": false, + "required": ["args", "expected"], + "properties": { + "args": { "type": "array", "items": { "type": "string" } }, + "stdin": { "type": "string" }, + "timeoutMs": { "type": "integer", "minimum": 1, "maximum": 300000 }, + "env": { "$ref": "#/$defs/stringMap" }, + "http": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["request", "response"], + "properties": { + "request": { "$ref": "#/$defs/request" }, + "response": { "$ref": "#/$defs/response" } + } + } + }, + "expected": { "$ref": "#/$defs/expected" } + } + }, + "stringMap": { + "type": "object", + "additionalProperties": { "type": "string" } + }, + "expected": { + "type": "object", + "additionalProperties": false, + "required": ["exitCode", "stdout", "stderr"], + "properties": { + "exitCode": { "type": "integer", "minimum": 0, "maximum": 255 }, + "stdout": { "type": "string" }, + "stderr": { "type": "string" } + } + }, + "request": { + "type": "object", + "additionalProperties": false, + "required": ["method", "uri"], + "properties": { + "method": { "type": "string", "minLength": 1 }, + "uri": { "type": "string", "pattern": "^/" }, + "headers": { "$ref": "#/$defs/stringMap" }, + "ignoreHeaders": { + "type": "array", + "uniqueItems": true, + "items": { "type": "string", "minLength": 1 } + }, + "body": { "type": "string" } + } + }, + "response": { + "type": "object", + "additionalProperties": false, + "required": ["status"], + "properties": { + "status": { "type": "integer", "minimum": 100, "maximum": 599 }, + "headers": { "$ref": "#/$defs/stringMap" }, + "body": { "type": "string" } + } + } + } +} diff --git a/conformance/scenarios/pairs/invalid-limit.json b/conformance/scenarios/pairs/invalid-limit.json new file mode 100644 index 0000000..c1b14bc --- /dev/null +++ b/conformance/scenarios/pairs/invalid-limit.json @@ -0,0 +1,22 @@ +{ + "schema": "mm/conformance-pair/v1", + "name": "invalid numeric input fails before network on both versions", + "oracle": { + "args": ["--json", "users", "--limit", "0"], + "env": { "MM_TOKEN": "fixture-mattermost-token" }, + "expected": { + "exitCode": 1, + "stdout": "", + "stderr": "Error: --limit must be a positive number.\n" + } + }, + "candidate": { + "args": ["--json", "users", "--limit", "0"], + "env": { "MM_TOKEN": "fixture-mattermost-token" }, + "expected": { + "exitCode": 2, + "stdout": "", + "stderr": "{\"schema\":\"mm/v2/error\",\"code\":\"invalid_input\",\"message\":\"--limit must be a positive number\",\"exitCode\":2,\"recovery\":\"none\"}\n" + } + } +} diff --git a/conformance/scenarios/pairs/teams.json b/conformance/scenarios/pairs/teams.json new file mode 100644 index 0000000..a5ecee9 --- /dev/null +++ b/conformance/scenarios/pairs/teams.json @@ -0,0 +1,62 @@ +{ + "schema": "mm/conformance-pair/v1", + "name": "teams preserves sorting redaction and narrow fields across v1 and v2", + "oracle": { + "args": ["--json", "teams"], + "env": { "MM_TOKEN": "fixture-mattermost-token" }, + "http": [ + { + "request": { + "method": "GET", + "uri": "/api/v4/users/me", + "headers": { "Authorization": "Bearer fixture-mattermost-token" }, + "ignoreHeaders": ["Accept", "Accept-Encoding", "Accept-Language", "Connection", "Content-Type", "Sec-Fetch-Mode", "User-Agent"] + }, + "response": { "status": 200, "headers": { "Content-Type": "application/json" }, "body": "{\"id\":\"user-1\",\"username\":\"arda\"}" } + }, + { + "request": { + "method": "GET", + "uri": "/api/v4/users/user-1/teams", + "headers": { "Authorization": "Bearer fixture-mattermost-token" }, + "ignoreHeaders": ["Accept", "Accept-Encoding", "Accept-Language", "Connection", "Content-Type", "Sec-Fetch-Mode", "User-Agent"] + }, + "response": { "status": 200, "headers": { "Content-Type": "application/json" }, "body": "[{\"id\":\"z\",\"name\":\"beta\",\"display_name\":\"Beta\",\"type\":\"I\",\"email\":\"hidden\"},{\"id\":\"b\",\"name\":\"alpha\",\"display_name\":null,\"type\":\"O\"},{\"id\":\"a\",\"name\":\"alpha\",\"display_name\":\"AKIAIOSFODNN7EXAMPLE\",\"type\":\"O\"}]" } + } + ], + "expected": { + "exitCode": 0, + "stdout": "[\n {\n \"id\": \"a\",\n \"name\": \"alpha\",\n \"displayName\": \"AK...LE\",\n \"type\": \"open\"\n },\n {\n \"id\": \"b\",\n \"name\": \"alpha\",\n \"type\": \"open\"\n },\n {\n \"id\": \"z\",\n \"name\": \"beta\",\n \"displayName\": \"Beta\",\n \"type\": \"invite_only\"\n }\n]\n", + "stderr": "" + } + }, + "candidate": { + "args": ["--json", "teams"], + "env": { "MM_TOKEN": "fixture-mattermost-token" }, + "http": [ + { + "request": { + "method": "GET", + "uri": "/api/v4/users/me", + "headers": { "Authorization": "Bearer fixture-mattermost-token" }, + "ignoreHeaders": ["Accept", "Accept-Encoding", "Accept-Language", "Connection", "Content-Type", "Sec-Fetch-Mode", "User-Agent"] + }, + "response": { "status": 200, "headers": { "Content-Type": "application/json" }, "body": "{\"id\":\"user-1\",\"username\":\"arda\"}" } + }, + { + "request": { + "method": "GET", + "uri": "/api/v4/users/user-1/teams", + "headers": { "Authorization": "Bearer fixture-mattermost-token" }, + "ignoreHeaders": ["Accept", "Accept-Encoding", "Accept-Language", "Connection", "Content-Type", "Sec-Fetch-Mode", "User-Agent"] + }, + "response": { "status": 200, "headers": { "Content-Type": "application/json" }, "body": "[{\"id\":\"z\",\"name\":\"beta\",\"display_name\":\"Beta\",\"type\":\"I\",\"email\":\"hidden\"},{\"id\":\"b\",\"name\":\"alpha\",\"display_name\":null,\"type\":\"O\"},{\"id\":\"a\",\"name\":\"alpha\",\"display_name\":\"AKIAIOSFODNN7EXAMPLE\",\"type\":\"O\"}]" } + } + ], + "expected": { + "exitCode": 0, + "stdout": "{\"schema\":\"mm/v2/teams\",\"teams\":[{\"id\":\"a\",\"name\":\"alpha\",\"displayName\":\"AK...LE\",\"type\":\"open\"},{\"id\":\"b\",\"name\":\"alpha\",\"displayName\":null,\"type\":\"open\"},{\"id\":\"z\",\"name\":\"beta\",\"displayName\":\"Beta\",\"type\":\"invite_only\"}]}\n", + "stderr": "" + } + } +} diff --git a/conformance/scenarios/pairs/whoami.json b/conformance/scenarios/pairs/whoami.json new file mode 100644 index 0000000..ad77fd7 --- /dev/null +++ b/conformance/scenarios/pairs/whoami.json @@ -0,0 +1,52 @@ +{ + "schema": "mm/conformance-pair/v1", + "name": "whoami preserves narrow identity semantics across v1 and v2", + "oracle": { + "args": ["--json", "whoami"], + "env": { "MM_TOKEN": "fixture-mattermost-token" }, + "http": [ + { + "request": { + "method": "GET", + "uri": "/api/v4/users/me", + "headers": { "Authorization": "Bearer fixture-mattermost-token" }, + "ignoreHeaders": ["Accept", "Accept-Encoding", "Accept-Language", "Connection", "Content-Type", "Sec-Fetch-Mode", "User-Agent"] + }, + "response": { + "status": 200, + "headers": { "Content-Type": "application/json" }, + "body": "{\"id\":\"user-1\",\"username\":\"alice\",\"first_name\":\"Alice\",\"last_name\":\"Agent\",\"nickname\":\"ali\",\"roles\":\"system_user team_user\",\"email\":\"must-not-leak@example.test\"}" + } + } + ], + "expected": { + "exitCode": 0, + "stdout": "{\n \"id\": \"user-1\",\n \"username\": \"alice\",\n \"displayName\": \"Alice Agent\",\n \"nickname\": \"ali\",\n \"roles\": [\n \"system_user\",\n \"team_user\"\n ]\n}\n", + "stderr": "" + } + }, + "candidate": { + "args": ["--json", "whoami"], + "env": { "MM_TOKEN": "fixture-mattermost-token" }, + "http": [ + { + "request": { + "method": "GET", + "uri": "/api/v4/users/me", + "headers": { "Authorization": "Bearer fixture-mattermost-token" }, + "ignoreHeaders": ["Accept", "Accept-Encoding", "Accept-Language", "Connection", "Content-Type", "Sec-Fetch-Mode", "User-Agent"] + }, + "response": { + "status": 200, + "headers": { "Content-Type": "application/json" }, + "body": "{\"id\":\"user-1\",\"username\":\"alice\",\"first_name\":\"Alice\",\"last_name\":\"Agent\",\"nickname\":\"ali\",\"roles\":\"system_user team_user\",\"email\":\"must-not-leak@example.test\"}" + } + } + ], + "expected": { + "exitCode": 0, + "stdout": "{\"schema\":\"mm/v2/whoami\",\"data\":{\"id\":\"user-1\",\"username\":\"alice\",\"displayName\":\"Alice Agent\",\"nickname\":\"ali\",\"roles\":[\"system_user\",\"team_user\"]}}\n", + "stderr": "" + } + } +} diff --git a/docs/V1_PARITY_MATRIX.md b/docs/V1_PARITY_MATRIX.md index e32c96e..01ff7c2 100644 --- a/docs/V1_PARITY_MATRIX.md +++ b/docs/V1_PARITY_MATRIX.md @@ -31,7 +31,7 @@ This is the removal gate for the TypeScript implementation. A row may become `ve | `--no-redact` | disable heuristic redaction, never active-token masking | preserve | scaffolded | | `--threads` | hydrate complete visible threads | preserve | scaffolded | | `--no-threads` | selected seeds only except `thread` | preserve | scaffolded | -| numeric validation | canonical positive safe integer only | preserve bounded canonical integer validation | scaffolded | +| numeric validation | canonical positive safe integer only | preserve bounded canonical integer validation | verified (`conformance/scenarios/pairs/invalid-limit.json`, Go validation tests) | | duration validation | `^\d+[hdwm]$` | preserve | scaffolded | | URL normalization | WHATWG normalization plus custom loopback test | preserve safe canonicalization; reject transport-ambiguous IPv4/backslash forms | intentionally_changed | @@ -59,8 +59,8 @@ This is the removal gate for the TypeScript implementation. A row may become `ve | --- | --- | --- | --- | | `doctor` | global flags | same read-only readiness checks; `mm/v2/doctor` | oracle | | `config` | `--path`, `--init` | preserve plus deterministic XDG migration warning | scaffolded | -| `whoami` | global flags | narrow validated identity | scaffolded | -| `teams` | global flags | validated deterministic teams | scaffolded | +| `whoami` | global flags | narrow validated identity | verified (`conformance/scenarios/pairs/whoami.json`, `internal/cli/identity_test.go`) | +| `teams` | global flags | validated deterministic teams | verified (`conformance/scenarios/pairs/teams.json`, `internal/cli/identity_test.go`) | | `users [query]` | `--team`, `-l`/`--limit 20` | preserve exact directory semantics | scaffolded | | `channels` | `--type all`; `dm/public/private/group/all` | preserve account-wide dedupe/team identity | scaffolded | | `dms` | repeated `-u`/`--user`; `-l`/`--limit 50`; `-s`/`--since 7d`; `-c`/`--channel`; `--cursor` | preserve exact D-channel and aggregate semantics | scaffolded | @@ -180,6 +180,18 @@ The E2E disposition specifically preserves verbatim short Markdown DM and near-l Removal requires every v1 test file to link to one or more verified Go tests/scenarios in this table or a finer generated inventory. +## Differential corpus + +Paired fixtures use `mm/conformance-pair/v1`. Oracle and candidate run in separate isolated homes against separate sequential fake servers. Each side locks its own exact args, stdin, environment, HTTP method/URI/headers/body, stdout, stderr, and exit code. This proves preserved semantics while making contract-authorized v2 schema and exit-code changes explicit. + +Current executable pairs: + +- `invalid-limit.json`: canonical numeric rejection and zero network activity +- `teams.json`: authenticated request order, narrow field projection, deterministic sorting, and secret redaction +- `whoami.json`: authenticated request shape, narrow identity projection, and private-field exclusion + +The sequential request transcript is the fake server's resulting state for these read-only cases. Mutation state is proved by Go fault tests and the disposable Mattermost E2E suite rather than inferred from stdout. + ## Build, CI, and release disposition | v1 gate | Go v2 replacement | Status | diff --git a/internal/conformance/runner.go b/internal/conformance/runner.go index 21854be..ad745fa 100644 --- a/internal/conformance/runner.go +++ b/internal/conformance/runner.go @@ -23,6 +23,16 @@ type Command struct { Dir string } +func RunPair(ctx context.Context, oracle, candidate Command, pair PairScenario) error { + if err := Run(ctx, oracle, pair.Oracle); err != nil { + return fmt.Errorf("oracle: %w", err) + } + if err := Run(ctx, candidate, pair.Candidate); err != nil { + return fmt.Errorf("candidate: %w", err) + } + return nil +} + func Run(ctx context.Context, command Command, scenario Scenario) error { if command.Path == "" { return fmt.Errorf("command path is required") diff --git a/internal/conformance/runner_test.go b/internal/conformance/runner_test.go index 1af668a..39e9eb8 100644 --- a/internal/conformance/runner_test.go +++ b/internal/conformance/runner_test.go @@ -1,6 +1,7 @@ package conformance import ( + "context" "net/http" "net/http/httptest" "slices" @@ -8,6 +9,21 @@ import ( "testing" ) +func expected(exit int, stdout, stderr string) *ProcessExpected { + return &ProcessExpected{ExitCode: &exit, Stdout: &stdout, Stderr: &stderr} +} + +func TestRunPairIdentifiesFailingSide(t *testing.T) { + pair := PairScenario{ + Oracle: Scenario{Args: []string{"-c", "printf oracle"}, Expected: expected(0, "oracle", "")}, + Candidate: Scenario{Args: []string{"-c", "printf candidate"}, Expected: expected(0, "wrong", "")}, + } + err := RunPair(context.Background(), Command{Path: "sh"}, Command{Path: "sh"}, pair) + if err == nil || !strings.Contains(err.Error(), "candidate: stdout mismatch") { + t.Fatalf("RunPair() error = %v", err) + } +} + func TestLimitedBufferBoundsStoredOutput(t *testing.T) { buffer := newLimitedBuffer(4) input := []byte("abcdefgh") diff --git a/internal/conformance/scenario.go b/internal/conformance/scenario.go index a0c85c9..a817b33 100644 --- a/internal/conformance/scenario.go +++ b/internal/conformance/scenario.go @@ -10,7 +10,10 @@ import ( "regexp" ) -const SchemaV1 = "mm/conformance/v1" +const ( + SchemaV1 = "mm/conformance/v1" + PairSchemaV1 = "mm/conformance-pair/v1" +) var envNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) @@ -25,6 +28,13 @@ type Scenario struct { Expected *ProcessExpected `json:"expected"` } +type PairScenario struct { + Schema string `json:"schema"` + Name string `json:"name"` + Oracle Scenario `json:"oracle"` + Candidate Scenario `json:"candidate"` +} + type HTTPExchange struct { Request HTTPRequestExpected `json:"request"` Response HTTPResponse `json:"response"` @@ -51,61 +61,101 @@ type ProcessExpected struct { } func Load(path string) (Scenario, error) { + var scenario Scenario + if err := decodeFile(path, &scenario); err != nil { + return Scenario{}, fmt.Errorf("decode scenario: %w", err) + } + if err := validateScenario(scenario, true); err != nil { + return Scenario{}, err + } + return scenario, nil +} + +func LoadPair(path string) (PairScenario, error) { + var pair PairScenario + if err := decodeFile(path, &pair); err != nil { + return PairScenario{}, fmt.Errorf("decode pair scenario: %w", err) + } + if pair.Schema != PairSchemaV1 { + return PairScenario{}, fmt.Errorf("unsupported pair scenario schema %q", pair.Schema) + } + if pair.Name == "" { + return PairScenario{}, fmt.Errorf("pair scenario name is required") + } + if err := validateScenario(pair.Oracle, false); err != nil { + return PairScenario{}, fmt.Errorf("oracle: %w", err) + } + if err := validateScenario(pair.Candidate, false); err != nil { + return PairScenario{}, fmt.Errorf("candidate: %w", err) + } + return pair, nil +} + +func decodeFile(path string, destination any) error { f, err := os.Open(path) // #nosec G304 -- operator-selected local scenario fixture. if err != nil { - return Scenario{}, err + return err } defer func() { _ = f.Close() }() decoder := json.NewDecoder(f) decoder.DisallowUnknownFields() - var scenario Scenario - if err := decoder.Decode(&scenario); err != nil { - return Scenario{}, fmt.Errorf("decode scenario: %w", err) + if err := decoder.Decode(destination); err != nil { + return err } var trailing any if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { if err == nil { - return Scenario{}, fmt.Errorf("decode scenario: trailing JSON value") + return fmt.Errorf("trailing JSON value") } - return Scenario{}, fmt.Errorf("decode scenario trailer: %w", err) + return fmt.Errorf("decode trailer: %w", err) + } + return nil +} + +func validateScenario(scenario Scenario, requireHeader bool) error { + if requireHeader && scenario.Schema != SchemaV1 { + return fmt.Errorf("unsupported scenario schema %q", scenario.Schema) + } + if !requireHeader && scenario.Schema != "" { + return fmt.Errorf("nested scenario must not declare schema") } - if scenario.Schema != SchemaV1 { - return Scenario{}, fmt.Errorf("unsupported scenario schema %q", scenario.Schema) + if requireHeader && scenario.Name == "" { + return fmt.Errorf("scenario name is required") } - if scenario.Name == "" { - return Scenario{}, fmt.Errorf("scenario name is required") + if !requireHeader && scenario.Name != "" { + return fmt.Errorf("nested scenario must not declare name") } if scenario.Args == nil { - return Scenario{}, fmt.Errorf("scenario args are required") + return fmt.Errorf("scenario args are required") } if scenario.Expected == nil || scenario.Expected.ExitCode == nil || scenario.Expected.Stdout == nil || scenario.Expected.Stderr == nil { - return Scenario{}, fmt.Errorf("scenario expected exitCode, stdout, and stderr are required") + return fmt.Errorf("scenario expected exitCode, stdout, and stderr are required") } if *scenario.Expected.ExitCode < 0 || *scenario.Expected.ExitCode > 255 { - return Scenario{}, fmt.Errorf("scenario expected exitCode must be between 0 and 255") + return fmt.Errorf("scenario expected exitCode must be between 0 and 255") } if scenario.Timeout != nil && (*scenario.Timeout < 1 || *scenario.Timeout > 300_000) { - return Scenario{}, fmt.Errorf("scenario timeoutMs must be between 1 and 300000 when set") + return fmt.Errorf("scenario timeoutMs must be between 1 and 300000 when set") } for name := range scenario.Env { if !envNamePattern.MatchString(name) { - return Scenario{}, fmt.Errorf("invalid scenario environment name %q", name) + return fmt.Errorf("invalid scenario environment name %q", name) } } for i, exchange := range scenario.HTTP { if exchange.Request.Method == "" || exchange.Request.URI == "" { - return Scenario{}, fmt.Errorf("http exchange %d requires request method and uri", i) + return fmt.Errorf("http exchange %d requires request method and uri", i) } if exchange.Response.Status < 100 || exchange.Response.Status > 599 { - return Scenario{}, fmt.Errorf("http exchange %d has invalid response status", i) + return fmt.Errorf("http exchange %d has invalid response status", i) } for _, name := range exchange.Request.IgnoreHeaders { switch http.CanonicalHeaderKey(name) { case "Authorization", "Cookie", "Proxy-Authorization": - return Scenario{}, fmt.Errorf("http exchange %d cannot ignore security-sensitive header %q", i, name) + return fmt.Errorf("http exchange %d cannot ignore security-sensitive header %q", i, name) } } } - return scenario, nil + return nil } diff --git a/internal/conformance/scenario_test.go b/internal/conformance/scenario_test.go index 3f8bdb8..6b8f906 100644 --- a/internal/conformance/scenario_test.go +++ b/internal/conformance/scenario_test.go @@ -46,6 +46,35 @@ func TestLoadRejectsIgnoredAuthorizationHeader(t *testing.T) { } } +func TestLoadPairRequiresStrictCompleteCases(t *testing.T) { + path := filepath.Join(t.TempDir(), "pair.json") + content := `{"schema":"mm/conformance-pair/v1","name":"pair","oracle":{"args":[],"expected":{"exitCode":0,"stdout":"","stderr":""}},"candidate":{"schema":"mm/conformance/v1","args":[],"expected":{"exitCode":0,"stdout":"","stderr":""}}}` + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + _, err := LoadPair(path) + if err == nil || !strings.Contains(err.Error(), "nested scenario must not declare schema") { + t.Fatalf("LoadPair() error = %v", err) + } +} + +func TestPairFixturesLoad(t *testing.T) { + paths, err := filepath.Glob(filepath.Join("..", "..", "conformance", "scenarios", "pairs", "*.json")) + if err != nil { + t.Fatal(err) + } + if len(paths) == 0 { + t.Fatal("no paired conformance fixtures found") + } + for _, path := range paths { + t.Run(filepath.Base(path), func(t *testing.T) { + if _, err := LoadPair(path); err != nil { + t.Fatal(err) + } + }) + } +} + func TestSequentialServerReportsMissingRequests(t *testing.T) { server := newSequentialServer([]HTTPExchange{{ Request: HTTPRequestExpected{Method: "GET", URI: "/api/v4/users/me"}, diff --git a/justfile b/justfile index d22278f..fdb73f6 100644 --- a/justfile +++ b/justfile @@ -35,10 +35,13 @@ oracle-smoke: git diff --quiet v1.6.0 -- src package.json bun.lock tsconfig.json go run ./cmd/conformance --scenario conformance/scenarios/v1/whoami.json --cwd . -- bun src/index.ts +parity-smoke: + @tmp="$(mktemp -d "${TMPDIR:-/tmp}/mattermost-cli-parity.XXXXXX")"; trap 'find "$tmp" -type f -delete; rmdir "$tmp"' EXIT; go build -o "$tmp/mm" ./cmd/mm; for scenario in conformance/scenarios/pairs/*.json; do go run ./cmd/conformance --pair "$scenario" --cwd . --oracle bun --oracle-prefix src/index.ts --candidate "$tmp/mm"; done + go-gate: go-format-check go-test go-race go-vet go-modules go-build go-e2e-compile go-cross-build git diff --check legacy-gate: bun run verify -gate: go-gate legacy-gate oracle-smoke +gate: go-gate legacy-gate oracle-smoke parity-smoke From b274fd45e046da9493307ed6455faee690169472 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 12:30:55 +0300 Subject: [PATCH 103/119] fix: reject writable retention configuration --- internal/cli/config.go | 13 ++++++++++--- internal/cli/config_test.go | 20 ++++++++++++++++++++ internal/cli/runtime.go | 4 ++++ internal/cli/runtime_test.go | 17 +++++++++++++++++ internal/cli/stage_inspect_test.go | 23 +++++++++++++++++++++++ internal/cli/store.go | 3 +++ internal/config/config.go | 4 +++- internal/config/config_test.go | 19 +++++++++++++++++++ 8 files changed, 99 insertions(+), 4 deletions(-) diff --git a/internal/cli/config.go b/internal/cli/config.go index 250365c..a803b20 100644 --- a/internal/cli/config.go +++ b/internal/cli/config.go @@ -53,7 +53,7 @@ func newConfigCommand(state *rootState) *cobra.Command { } file := config.Load(paths) status := state.presentConfigStatus(configMachineStatus(action, file, created)) - if file.Error != "" || file.Unsafe != "" || (file.InsecurePermissions && file.Config.Token != "") { + if file.Error != "" || file.Unsafe != "" || file.WritableByOthers || (file.InsecurePermissions && file.Config.Token != "") { state.setSemanticExit(3) } return writeConfigStatus(state, status) @@ -182,8 +182,8 @@ func writeConfigHuman(state *rootState, status output.ConfigEnvelope) error { "Exists: " + yesNo(valueOr(status.Exists, false)), "URL configured: " + yesNo(valueOr(status.URLConfigured, false)), "Token configured: " + yesNo(valueOr(status.TokenConfigured, false)), - "Stage TTL seconds: " + strconv.FormatInt(valueOr(status.StageTTLSeconds, int64(0)), 10), - "Stage prune after seconds: " + strconv.FormatInt(valueOr(status.StagePruneAfterSeconds, int64(0)), 10), + "Stage TTL seconds: " + optionalInt64(status.StageTTLSeconds), + "Stage prune after seconds: " + optionalInt64(status.StagePruneAfterSeconds), "Permissions: " + valueOr(status.Permissions, "not_applicable"), "Read status: " + valueOr(status.ReadStatus, "not_attempted"), "Parse status: " + valueOr(status.ParseStatus, "not_attempted"), @@ -207,3 +207,10 @@ func yesNo(value bool) string { } return "no" } + +func optionalInt64(value *int64) string { + if value == nil { + return "unknown" + } + return strconv.FormatInt(*value, 10) +} diff --git a/internal/cli/config_test.go b/internal/cli/config_test.go index ea5a4ca..fb9ed06 100644 --- a/internal/cli/config_test.go +++ b/internal/cli/config_test.go @@ -260,6 +260,26 @@ func TestConfigDiagnosticDocumentsUseHandledExitThree(t *testing.T) { } } +func TestHumanConfigReportsUnknownRetentionOnReadOrParseFailure(t *testing.T) { + for _, body := range []string{"broken = [", ""} { + home := t.TempDir() + path := filepath.Join(home, ".config", "mattermost-cli", "config.toml") + if body == "" { + if err := os.MkdirAll(path, 0o700); err != nil { + t.Fatal(err) + } + } else { + writeFile(t, path, body, 0o600) + } + t.Setenv("HOME", home) + var stdout, stderr bytes.Buffer + code := Execute(context.Background(), []string{"config"}, strings.NewReader(""), &stdout, &stderr) + if code != 3 || stderr.Len() != 0 || !strings.Contains(stdout.String(), "Stage TTL seconds: unknown") || !strings.Contains(stdout.String(), "Stage prune after seconds: unknown") { + t.Fatalf("body=%q exit=%d stdout=%q stderr=%q", body, code, stdout.String(), stderr.String()) + } + } +} + func TestConfigPathDoesNotInspectSelectedFile(t *testing.T) { home := t.TempDir() path := filepath.Join(home, ".config", "mattermost-cli", "config.toml") diff --git a/internal/cli/runtime.go b/internal/cli/runtime.go index 6f91fe0..2010c76 100644 --- a/internal/cli/runtime.go +++ b/internal/cli/runtime.go @@ -141,6 +141,10 @@ func (s *rootState) runtimeFor(cmd *cobra.Command) (*Runtime, error) { s.runtimeErr = configFailure("could not parse the Mattermost configuration") return nil, s.runtimeErr } + if file.WritableByOthers { + s.runtimeErr = configFailure("Mattermost configuration must not be writable by other users") + return nil, s.runtimeErr + } if file.InsecurePermissions && file.Config.Token != "" { s.runtimeErr = configFailure("Mattermost configuration containing a token must not be accessible by other users") return nil, s.runtimeErr diff --git a/internal/cli/runtime_test.go b/internal/cli/runtime_test.go index 96a5d44..1905b51 100644 --- a/internal/cli/runtime_test.go +++ b/internal/cli/runtime_test.go @@ -59,6 +59,23 @@ func TestRuntimeUsesMacIndependentXDGPath(t *testing.T) { } } +func TestRuntimeRejectsWritableTokenlessConfigBeforeUsingEnvironmentCredential(t *testing.T) { + home := t.TempDir() + path := filepath.Join(home, ".config", "mattermost-cli", "config.toml") + writeFile(t, path, `url = "https://attacker-controlled.example"`, 0o620) + if err := os.Chmod(path, 0o620); err != nil { + t.Fatal(err) + } + state, command, captured := runtimeProbe(t, home, map[string]string{"MM_TOKEN": "environment-token"}, false) + defer state.close() + defer state.releaseCredentials() + command.SetArgs([]string{"probe"}) + err := command.Execute() + if err == nil || exitCode(err) != 3 || !strings.Contains(err.Error(), "must not be writable by other users") || captured.runtime != nil { + t.Fatalf("runtime=%v err=%v", captured.runtime, err) + } +} + func TestReadDisplayAcceptsNegativeBooleanFlags(t *testing.T) { state, command, _ := runtimeProbe(t, t.TempDir(), map[string]string{"MM_URL": "https://example.com", "MM_TOKEN": "token"}, false) defer state.close() diff --git a/internal/cli/stage_inspect_test.go b/internal/cli/stage_inspect_test.go index 0a7514e..416652f 100644 --- a/internal/cli/stage_inspect_test.go +++ b/internal/cli/stage_inspect_test.go @@ -75,6 +75,29 @@ func TestStageListAbsentIsOfflineReadOnlyAndSchemaValid(t *testing.T) { } } +func TestStageCommandsRejectConfigWritableByOtherUsers(t *testing.T) { + home, stateRoot := t.TempDir(), filepath.Join(t.TempDir(), "state") + path := filepath.Join(home, ".config", "mattermost-cli", "config.toml") + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("stage_ttl_seconds = 1\n"), 0o620); err != nil { + t.Fatal(err) + } + if err := os.Chmod(path, 0o620); err != nil { + t.Fatal(err) + } + var stdout, stderr bytes.Buffer + _, command := stageInspectCommand(t, home, stateRoot, false, &stdout, &stderr) + err := command.execute(t.Context(), "list") + if err == nil || exitCode(err) != 3 || !strings.Contains(err.Error(), "must not be writable by other users") || stdout.Len() != 0 { + t.Fatalf("err=%v stdout=%q stderr=%q", err, stdout.String(), stderr.String()) + } + if _, statErr := os.Stat(stateRoot); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("unsafe policy touched store: %v", statErr) + } +} + func TestStageListValidatesBoundsAndCursorBeforeStoreAccess(t *testing.T) { for _, args := range [][]string{{"list", "--limit", "0"}, {"list", "--limit", "101"}, {"list", "--cursor="}, {"list", "--cursor", "not-a-cursor"}} { t.Run(strings.Join(args, "_"), func(t *testing.T) { diff --git a/internal/cli/store.go b/internal/cli/store.go index 13ce430..0575be7 100644 --- a/internal/cli/store.go +++ b/internal/cli/store.go @@ -137,6 +137,9 @@ func resolveStageOptions(state *rootState, cmd *cobra.Command) error { if file.InsecurePermissions && file.Config.Token != "" { return configFailure("Mattermost configuration containing a token must not be accessible by other users") } + if file.WritableByOthers { + return configFailure("Mattermost configuration governing stage retention must not be writable by other users") + } state.stageTTLSeconds = file.Config.StageTTLSeconds state.stagePruneSeconds = file.Config.StagePruneAfterSeconds return nil diff --git a/internal/config/config.go b/internal/config/config.go index 6523a4c..28cc01b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -76,6 +76,7 @@ type FileState struct { LegacyPath string Exists bool InsecurePermissions bool + WritableByOthers bool Error FileError Unsafe UnsafeReason Migration Migration @@ -135,7 +136,7 @@ func Init(path string) (bool, error) { file, err := createConfigFile(path) if errors.Is(err, os.ErrExist) { existing := inspect(path) - if existing.Unsafe != "" || existing.Error == FileErrorRead || (existing.InsecurePermissions && existing.Config.Token != "") { + if existing.Unsafe != "" || existing.Error == FileErrorRead || existing.WritableByOthers || (existing.InsecurePermissions && existing.Config.Token != "") { return false, fmt.Errorf("existing config path is unsafe") } return false, nil @@ -203,6 +204,7 @@ func inspect(path string) FileState { state.Exists = true defer func() { _ = file.Close() }() state.InsecurePermissions = info.Mode().Perm()&0o077 != 0 + state.WritableByOthers = info.Mode().Perm()&0o022 != 0 data, err := io.ReadAll(io.LimitReader(file, maxConfigBytes+1)) if err != nil || len(data) > maxConfigBytes { state.Error = FileErrorRead diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 5e038ff..79462f0 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -111,6 +111,20 @@ func TestInitRejectsExistingTokenWithInsecurePermissions(t *testing.T) { } } +func TestWritableTokenlessConfigIsUnsafeForPolicyAndInit(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + writeConfig(t, path, `stage_ttl_seconds = 0`, 0o620) + + state := Load(Paths{ConfigPath: path, LegacyPath: path}) + if !state.InsecurePermissions || !state.WritableByOthers { + t.Fatalf("Load() state = %+v, want writable-by-others classification", state) + } + created, err := Init(path) + if err == nil || created { + t.Fatalf("Init() = (%v, %v), want unsafe-policy failure", created, err) + } +} + func TestLoadRejectsSymlinkedAncestorInsideUserDirectory(t *testing.T) { root := t.TempDir() realDirectory := filepath.Join(root, "real") @@ -218,6 +232,11 @@ func TestLoadCharacterizesMissingReadParseAndPermissions(t *testing.T) { if state := Load(Paths{ConfigPath: insecure, LegacyPath: insecure}); !state.InsecurePermissions { t.Fatalf("insecure state = %+v", state) } + writable := filepath.Join(root, "writable.toml") + writeConfig(t, writable, `stage_ttl_seconds = 1`, 0o602) + if state := Load(Paths{ConfigPath: writable, LegacyPath: writable}); !state.WritableByOthers { + t.Fatalf("writable state = %+v", state) + } } func TestLoadIgnoresUnsupportedTopLevelTypes(t *testing.T) { From 53b3a1995ca960be4fef1445d4ebf6c4f9e2f0e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 12:39:06 +0300 Subject: [PATCH 104/119] build: add reproducible native distribution --- justfile | 5 +- scripts/install.sh | 101 ++++++++++++++++++ scripts/install_test.go | 145 +++++++++++++++++++++++++ scripts/release/main.go | 200 +++++++++++++++++++++++++++++++++++ scripts/release/main_test.go | 97 +++++++++++++++++ 5 files changed, 547 insertions(+), 1 deletion(-) create mode 100755 scripts/install.sh create mode 100644 scripts/install_test.go create mode 100644 scripts/release/main.go create mode 100644 scripts/release/main_test.go diff --git a/justfile b/justfile index fdb73f6..e6e0532 100644 --- a/justfile +++ b/justfile @@ -4,7 +4,7 @@ default: @just --list go-format-check: - @unformatted="$(gofmt -l cmd internal tests/e2e)"; if [[ -n "$unformatted" ]]; then print -r -- "$unformatted"; exit 1; fi + @unformatted="$(gofmt -l cmd internal scripts tests/e2e)"; if [[ -n "$unformatted" ]]; then print -r -- "$unformatted"; exit 1; fi go-test: go test ./... @@ -28,6 +28,9 @@ go-e2e-compile: go-cross-build: @tmp="$(mktemp -d "${TMPDIR:-/tmp}/mattermost-cli-cross.XXXXXX")"; trap 'find "$tmp" -type f -delete; rmdir "$tmp"' EXIT; for target in darwin/arm64 darwin/amd64 linux/arm64 linux/amd64; do os="${target%/*}"; arch="${target#*/}"; echo "building $target"; CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" go build -trimpath -o "$tmp/mm-$os-$arch" ./cmd/mm; done +release-artifacts version commit output="dist": + go run ./scripts/release --version "{{version}}" --commit "{{commit}}" --output "{{output}}" + docker-e2e: bun run test:e2e diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 0000000..161efde --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,101 @@ +#!/bin/sh +set -eu + +repo="ardasevinc/mattermost-cli" +version="${MATTERMOST_CLI_VERSION:-latest}" +install_dir="${MATTERMOST_CLI_INSTALL_DIR:-}" + +log() { printf '%s\n' "$*" >&2; } +fail() { log "error: $*"; exit 1; } +need() { command -v "$1" >/dev/null 2>&1 || fail "missing required command: $1"; } + +need curl +need tar + +os="$(uname -s | tr '[:upper:]' '[:lower:]')" +case "$os" in + darwin|linux) ;; + *) fail "unsupported OS: $os" ;; +esac + +arch="$(uname -m)" +case "$arch" in + arm64|aarch64) arch="arm64" ;; + x86_64|amd64) arch="amd64" ;; + *) fail "unsupported architecture: $arch" ;; +esac + +if [ "$version" = "latest" ]; then + latest_url="https://api.github.com/repos/${repo}/releases/latest" + version="$(curl -fsSL "$latest_url" | sed -n 's/.*"tag_name":[[:space:]]*"\([^"]*\)".*/\1/p' | head -n 1)" + [ -n "$version" ] || fail "could not resolve latest release" +fi +case "$version" in + v[0-9]*.[0-9]*.[0-9]*|[0-9]*.[0-9]*.[0-9]*) ;; + *) fail "release version must be a v-prefixed semantic version" ;; +esac +tag="$version" +case "$tag" in v*) ;; *) tag="v$tag" ;; esac +version="${tag#v}" +case "$version" in *[!0-9A-Za-z.-]*|.*|*..*|*.) fail "invalid release version" ;; esac + +asset="mattermost-cli_${version}_${os}_${arch}.tar.gz" +base_url="https://github.com/${repo}/releases/download/${tag}" + +if [ -z "$install_dir" ]; then + if [ -n "${GOBIN:-}" ]; then + install_dir="$GOBIN" + elif [ -n "${GOPATH:-}" ]; then + install_dir="$GOPATH/bin" + else + install_dir="$HOME/.local/bin" + fi +fi + +tmp="" +install_tmp="" +cleanup() { + [ -z "$install_tmp" ] || rm -f -- "$install_tmp" + [ -z "$tmp" ] || rm -rf -- "$tmp" +} +trap cleanup EXIT INT TERM +tmp="$(mktemp -d "${TMPDIR:-/tmp}/mattermost-cli-install.XXXXXX")" +chmod 700 "$tmp" + +log "downloading mattermost-cli ${tag} for ${os}/${arch}" +curl -fsSL "${base_url}/${asset}" -o "$tmp/$asset" +curl -fsSL "${base_url}/checksums.txt" -o "$tmp/checksums.txt" + +expected="$(awk -v asset="$asset" '$2 == asset { print $1 }' "$tmp/checksums.txt")" +case "$expected" in + *' '*|*'\n'*|'') fail "checksum for ${asset} must appear exactly once" ;; + *[!0-9a-f]* ) fail "checksum for ${asset} is invalid" ;; +esac +[ "${#expected}" -eq 64 ] || fail "checksum for ${asset} is invalid" + +if command -v sha256sum >/dev/null 2>&1; then + actual="$(sha256sum "$tmp/$asset" | awk '{print $1}')" +elif command -v shasum >/dev/null 2>&1; then + actual="$(shasum -a 256 "$tmp/$asset" | awk '{print $1}')" +else + fail "missing sha256sum or shasum" +fi +[ "$actual" = "$expected" ] || fail "checksum mismatch for ${asset}" + +[ "$(tar -tzf "$tmp/$asset")" = "mm" ] || fail "archive member list is invalid" +tar -xzf "$tmp/$asset" -C "$tmp" +[ -f "$tmp/mm" ] && [ ! -L "$tmp/mm" ] || fail "archive did not contain a regular mm binary" +chmod 755 "$tmp/mm" +mkdir -p "$install_dir" +install_tmp="$(mktemp "$install_dir/.mattermost-cli-install.XXXXXX")" +cp "$tmp/mm" "$install_tmp" +chmod 755 "$install_tmp" +mv -f "$install_tmp" "$install_dir/mm" +install_tmp="" + +log "installed $install_dir/mm" +"$install_dir/mm" --version +case ":$PATH:" in + *":$install_dir:"*) ;; + *) log "note: $install_dir is not on PATH" ;; +esac diff --git a/scripts/install_test.go b/scripts/install_test.go new file mode 100644 index 0000000..278c3bc --- /dev/null +++ b/scripts/install_test.go @@ -0,0 +1,145 @@ +package scripts_test + +import ( + "archive/tar" + "compress/gzip" + "crypto/sha256" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" +) + +func TestInstallerTargetsPinnedArtifactsAndCleansPrivateTemps(t *testing.T) { + for _, target := range []struct{ os, arch, assetOS, assetArch string }{ + {"Darwin", "arm64", "darwin", "arm64"}, + {"Darwin", "x86_64", "darwin", "amd64"}, + {"Linux", "aarch64", "linux", "arm64"}, + {"Linux", "x86_64", "linux", "amd64"}, + } { + t.Run(target.assetOS+"-"+target.assetArch, func(t *testing.T) { + runInstaller(t, target.os, target.arch, 1) + }) + } +} + +func TestInstallerConcurrentReplacementRemainsAtomic(t *testing.T) { + runInstaller(t, "Linux", "x86_64", 2) +} + +func runInstaller(t *testing.T, goos, arch string, concurrency int) { + t.Helper() + root := t.TempDir() + fixtures, fakeBin := filepath.Join(root, "fixtures"), filepath.Join(root, "bin") + installDir, tempDir := filepath.Join(root, "install"), filepath.Join(root, "tmp") + for _, path := range []string{fixtures, fakeBin, installDir, tempDir} { + if err := os.MkdirAll(path, 0o700); err != nil { + t.Fatal(err) + } + } + assetOS := strings.ToLower(goos) + assetArch := map[string]string{"arm64": "arm64", "aarch64": "arm64", "x86_64": "amd64"}[arch] + asset := fmt.Sprintf("mattermost-cli_9.8.7_%s_%s.tar.gz", assetOS, assetArch) + payload := []byte("#!/bin/sh\nprintf 'mm version 9.8.7 (fixture)\\n'\n") + writeInstallerArchive(t, filepath.Join(fixtures, asset), payload) + archive, err := os.ReadFile(filepath.Join(fixtures, asset)) + if err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(archive) + if err := os.WriteFile(filepath.Join(fixtures, "checksums.txt"), []byte(fmt.Sprintf("%x %s\n", digest, asset)), 0o600); err != nil { + t.Fatal(err) + } + writeExecutable(t, filepath.Join(fakeBin, "uname"), "#!/bin/sh\nif [ \"$1\" = -s ]; then printf '%s\\n' \"$FAKE_OS\"; else printf '%s\\n' \"$FAKE_ARCH\"; fi\n") + writeExecutable(t, filepath.Join(fakeBin, "curl"), `#!/bin/sh +dest= +url= +while [ "$#" -gt 0 ]; do + case "$1" in + -o) dest="$2"; shift 2 ;; + http*) url="$1"; shift ;; + *) shift ;; + esac +done +[ -n "$dest" ] && [ -n "$url" ] +cp "$FIXTURE_DIR/${url##*/}" "$dest" +`) + environment := append(os.Environ(), + "PATH="+fakeBin+":"+os.Getenv("PATH"), + "FAKE_OS="+goos, + "FAKE_ARCH="+arch, + "FIXTURE_DIR="+fixtures, + "MATTERMOST_CLI_VERSION=v9.8.7", + "MATTERMOST_CLI_INSTALL_DIR="+installDir, + "TMPDIR="+tempDir, + ) + var wait sync.WaitGroup + errors := make(chan error, concurrency) + for range concurrency { + wait.Add(1) + go func() { + defer wait.Done() + command := exec.Command("/bin/sh", "install.sh") // #nosec G204 -- fixed test command. + command.Env = environment + output, err := command.CombinedOutput() + if err != nil { + errors <- fmt.Errorf("installer: %w: %s", err, output) + } + }() + } + wait.Wait() + close(errors) + for err := range errors { + t.Fatal(err) + } + installed, err := exec.Command(filepath.Join(installDir, "mm"), "--version").Output() // #nosec G204 -- test-owned path. + if err != nil || string(installed) != "mm version 9.8.7 (fixture)\n" { + t.Fatalf("installed=%q err=%v", installed, err) + } + for _, directory := range []string{tempDir, installDir} { + entries, err := os.ReadDir(directory) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), "mattermost-cli-install.") || strings.HasPrefix(entry.Name(), ".mattermost-cli-install.") { + t.Fatalf("temporary path survived: %s", filepath.Join(directory, entry.Name())) + } + } + } +} + +func writeInstallerArchive(t *testing.T, path string, payload []byte) { + t.Helper() + file, err := os.Create(path) // #nosec G304 -- test-owned path. + if err != nil { + t.Fatal(err) + } + zipper := gzip.NewWriter(file) + archive := tar.NewWriter(zipper) + if err := archive.WriteHeader(&tar.Header{Name: "mm", Mode: 0o755, Size: int64(len(payload)), Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + if _, err := archive.Write(payload); err != nil { + t.Fatal(err) + } + if err := archive.Close(); err != nil { + t.Fatal(err) + } + if err := zipper.Close(); err != nil { + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } +} + +func writeExecutable(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o700); err != nil { + t.Fatal(err) + } +} diff --git a/scripts/release/main.go b/scripts/release/main.go new file mode 100644 index 0000000..d791803 --- /dev/null +++ b/scripts/release/main.go @@ -0,0 +1,200 @@ +package main + +import ( + "archive/tar" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "errors" + "flag" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strings" + "time" +) + +var ( + versionPattern = regexp.MustCompile(`^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$`) + commitPattern = regexp.MustCompile(`^[0-9a-f]{7,40}$`) +) + +type target struct{ os, arch string } + +func main() { + var version, commit, output string + flag.StringVar(&version, "version", "", "release version without the v prefix") + flag.StringVar(&commit, "commit", "", "source commit SHA") + flag.StringVar(&output, "output", "dist", "artifact output directory") + flag.Parse() + if err := run(version, commit, output); err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(version, commit, output string) error { + version = strings.TrimPrefix(strings.TrimSpace(version), "v") + commit = strings.TrimSpace(commit) + if !versionPattern.MatchString(version) { + return errors.New("release version must be semantic and omit the v prefix") + } + if !commitPattern.MatchString(commit) { + return errors.New("release commit must be a 7-40 character lowercase hex SHA") + } + if output == "" { + return errors.New("release output directory is required") + } + if err := prepareOutput(output); err != nil { + return err + } + tmp, err := os.MkdirTemp("", "mattermost-cli-release-") + if err != nil { + return err + } + defer func() { _ = os.RemoveAll(tmp) }() + + checksums := make(map[target]string, 4) + for _, target := range []target{{"darwin", "amd64"}, {"darwin", "arm64"}, {"linux", "amd64"}, {"linux", "arm64"}} { + binary := filepath.Join(tmp, target.os+"-"+target.arch, "mm") + if err := os.MkdirAll(filepath.Dir(binary), 0o755); err != nil { // #nosec G301 -- temporary release inputs contain no secrets. + return err + } + ldflags := fmt.Sprintf("-s -w -buildid= -X github.com/ardasevinc/mattermost-cli/internal/buildinfo.Version=%s -X github.com/ardasevinc/mattermost-cli/internal/buildinfo.Commit=%s", version, commit) + command := exec.Command("go", "build", "-trimpath", "-buildvcs=false", "-ldflags", ldflags, "-o", binary, "./cmd/mm") // #nosec G204 -- values are validated or from the closed target list. + command.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS="+target.os, "GOARCH="+target.arch) + command.Stdout, command.Stderr = os.Stdout, os.Stderr + if err := command.Run(); err != nil { + return fmt.Errorf("build %s/%s: %w", target.os, target.arch, err) + } + name := fmt.Sprintf("mattermost-cli_%s_%s_%s.tar.gz", version, target.os, target.arch) + archivePath := filepath.Join(output, name) + if err := writeArchive(archivePath, binary); err != nil { + return fmt.Errorf("package %s/%s: %w", target.os, target.arch, err) + } + digest, err := fileDigest(archivePath) + if err != nil { + return err + } + checksums[target] = digest + } + formulaPath := filepath.Join(output, "mattermost-cli.rb") + if err := os.WriteFile(formulaPath, []byte(formula(version, checksums)), 0o644); err != nil { // #nosec G306 -- public Homebrew formula. + return err + } + formulaDigest, err := fileDigest(formulaPath) + if err != nil { + return err + } + lines := []string{formulaDigest + " mattermost-cli.rb"} + for target, digest := range checksums { + lines = append(lines, digest+" "+fmt.Sprintf("mattermost-cli_%s_%s_%s.tar.gz", version, target.os, target.arch)) + } + sort.Strings(lines) + return os.WriteFile(filepath.Join(output, "checksums.txt"), []byte(strings.Join(lines, "\n")+"\n"), 0o644) // #nosec G306 -- public release checksums. +} + +func formula(version string, checksums map[target]string) string { + digest := func(goos, arch string) string { return checksums[target{goos, arch}] } + return fmt.Sprintf(`class MattermostCli < Formula + desc "Mattermost CLI for agents and humans" + homepage "https://github.com/ardasevinc/mattermost-cli" + version %q + license "MIT" + + on_macos do + if Hardware::CPU.arm? + url "https://github.com/ardasevinc/mattermost-cli/releases/download/v#{version}/mattermost-cli_#{version}_darwin_arm64.tar.gz" + sha256 %q + else + url "https://github.com/ardasevinc/mattermost-cli/releases/download/v#{version}/mattermost-cli_#{version}_darwin_amd64.tar.gz" + sha256 %q + end + end + + on_linux do + if Hardware::CPU.arm? + url "https://github.com/ardasevinc/mattermost-cli/releases/download/v#{version}/mattermost-cli_#{version}_linux_arm64.tar.gz" + sha256 %q + else + url "https://github.com/ardasevinc/mattermost-cli/releases/download/v#{version}/mattermost-cli_#{version}_linux_amd64.tar.gz" + sha256 %q + end + end + + def install + bin.install "mm" + end + + test do + assert_match version.to_s, shell_output("#{bin}/mm --version") + assert_match "Mattermost CLI", shell_output("#{bin}/mm --help") + end +end +`, version, digest("darwin", "arm64"), digest("darwin", "amd64"), digest("linux", "arm64"), digest("linux", "amd64")) +} + +func prepareOutput(path string) error { + if err := os.MkdirAll(path, 0o755); err != nil { // #nosec G301 -- release artifacts are public-readable. + return err + } + entries, err := os.ReadDir(path) + if err != nil { + return err + } + if len(entries) != 0 { + return errors.New("release output directory must be empty") + } + return nil +} + +func writeArchive(path, binary string) (returnErr error) { + data, err := os.ReadFile(binary) // #nosec G304 -- internally generated binary. + if err != nil { + return err + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644) // #nosec G302,G304 -- public release artifact. + if err != nil { + return err + } + defer func() { + if err := file.Close(); returnErr == nil && err != nil { + returnErr = err + } + }() + zipper, err := gzip.NewWriterLevel(file, gzip.BestCompression) + if err != nil { + return err + } + zipper.ModTime = time.Unix(0, 0).UTC() + zipper.OS = 255 + archive := tar.NewWriter(zipper) + header := &tar.Header{Name: "mm", Mode: 0o755, Size: int64(len(data)), ModTime: time.Unix(0, 0).UTC(), Typeflag: tar.TypeReg, Format: tar.FormatUSTAR} + if err := archive.WriteHeader(header); err != nil { + return err + } + if _, err := archive.Write(data); err != nil { + return err + } + if err := archive.Close(); err != nil { + return err + } + return zipper.Close() +} + +func fileDigest(path string) (string, error) { + file, err := os.Open(path) // #nosec G304 -- internally generated release artifact. + if err != nil { + return "", err + } + defer func() { _ = file.Close() }() + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +} diff --git a/scripts/release/main_test.go b/scripts/release/main_test.go new file mode 100644 index 0000000..16e8531 --- /dev/null +++ b/scripts/release/main_test.go @@ -0,0 +1,97 @@ +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestWriteArchiveIsCanonicalAndDeterministic(t *testing.T) { + directory := t.TempDir() + binary := filepath.Join(directory, "input") + if err := os.WriteFile(binary, []byte("mm-test"), 0o755); err != nil { + t.Fatal(err) + } + first, second := filepath.Join(directory, "first.tar.gz"), filepath.Join(directory, "second.tar.gz") + if err := writeArchive(first, binary); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(binary, time.Now(), time.Now()); err != nil { + t.Fatal(err) + } + if err := writeArchive(second, binary); err != nil { + t.Fatal(err) + } + one, err := os.ReadFile(first) + if err != nil { + t.Fatal(err) + } + two, err := os.ReadFile(second) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(one, two) { + t.Fatal("archive bytes changed with source metadata") + } + zipper, err := gzip.NewReader(bytes.NewReader(one)) + if err != nil { + t.Fatal(err) + } + archive := tar.NewReader(zipper) + header, err := archive.Next() + if err != nil { + t.Fatal(err) + } + if header.Name != "mm" || header.Mode != 0o755 || !header.ModTime.Equal(time.Unix(0, 0)) || header.Uid != 0 || header.Gid != 0 { + t.Fatalf("noncanonical header: %+v", header) + } + payload, err := io.ReadAll(archive) + if err != nil || string(payload) != "mm-test" { + t.Fatalf("payload=%q err=%v", payload, err) + } + if _, err := archive.Next(); err != io.EOF { + t.Fatalf("archive has unexpected second member: %v", err) + } +} + +func TestReleaseArgumentsAndOutputAreClosed(t *testing.T) { + for _, test := range []struct{ version, commit, want string }{ + {"", "abcdef0", "release version"}, + {"latest", "abcdef0", "release version"}, + {"1.2.3", "dev", "release commit"}, + {"1.2.3", "ABCDEF0", "release commit"}, + } { + err := run(test.version, test.commit, t.TempDir()) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("version=%q commit=%q err=%v", test.version, test.commit, err) + } + } + nonempty := t.TempDir() + if err := os.WriteFile(filepath.Join(nonempty, "stale"), []byte("stale"), 0o600); err != nil { + t.Fatal(err) + } + if err := run("1.2.3", "abcdef0", nonempty); err == nil || !strings.Contains(err.Error(), "must be empty") { + t.Fatalf("nonempty output err=%v", err) + } +} + +func TestFormulaBindsEveryExactArchiveDigest(t *testing.T) { + checksums := map[target]string{ + {"darwin", "arm64"}: strings.Repeat("a", 64), + {"darwin", "amd64"}: strings.Repeat("b", 64), + {"linux", "arm64"}: strings.Repeat("c", 64), + {"linux", "amd64"}: strings.Repeat("d", 64), + } + got := formula("2.0.0", checksums) + for _, value := range []string{"version \"2.0.0\"", "mattermost-cli_#{version}_darwin_arm64.tar.gz", "mattermost-cli_#{version}_linux_amd64.tar.gz", strings.Repeat("a", 64), strings.Repeat("d", 64), `bin.install "mm"`} { + if !strings.Contains(got, value) { + t.Fatalf("formula missing %q:\n%s", value, got) + } + } +} From 8bab0338d6f6ff603ce2a4851d231c40f2f62db2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 12:43:07 +0300 Subject: [PATCH 105/119] build: add native npm package launcher --- .gitignore | 3 + justfile | 3 + npm/README.md | 7 + npm/bin/mm.js | 43 +++++ scripts/npm-package/main.go | 262 +++++++++++++++++++++++++++++++ scripts/npm-package/main_test.go | 110 +++++++++++++ 6 files changed, 428 insertions(+) create mode 100644 npm/README.md create mode 100644 npm/bin/mm.js create mode 100644 scripts/npm-package/main.go create mode 100644 scripts/npm-package/main_test.go diff --git a/.gitignore b/.gitignore index 84222e9..ef6ccff 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,10 @@ node_modules out dist bin +!npm/bin/ +!npm/bin/mm.js *.tgz +npm-dist # code coverage coverage diff --git a/justfile b/justfile index e6e0532..8fb0d4d 100644 --- a/justfile +++ b/justfile @@ -31,6 +31,9 @@ go-cross-build: release-artifacts version commit output="dist": go run ./scripts/release --version "{{version}}" --commit "{{commit}}" --output "{{output}}" +npm-packages version release_dir="dist" output="npm-dist": + go run ./scripts/npm-package --version "{{version}}" --release-dir "{{release_dir}}" --output "{{output}}" + docker-e2e: bun run test:e2e diff --git a/npm/README.md b/npm/README.md new file mode 100644 index 0000000..8bc37a4 --- /dev/null +++ b/npm/README.md @@ -0,0 +1,7 @@ +# mattermost-cli + +Native Mattermost CLI for agents and humans. The npm package is a small launcher that selects a checksum-bound platform package produced from the same Go release build. + +No install script downloads or executes code. npm resolves the matching optional dependency for darwin or linux on arm64 or amd64. + +See https://github.com/ardasevinc/mattermost-cli for documentation and release verification. diff --git a/npm/bin/mm.js b/npm/bin/mm.js new file mode 100644 index 0000000..2e2b19c --- /dev/null +++ b/npm/bin/mm.js @@ -0,0 +1,43 @@ +#!/usr/bin/env node +'use strict' + +const { spawnSync } = require('node:child_process') + +const packages = { + 'darwin-arm64': '@ardasevinc/mattermost-cli-darwin-arm64', + 'darwin-x64': '@ardasevinc/mattermost-cli-darwin-amd64', + 'linux-arm64': '@ardasevinc/mattermost-cli-linux-arm64', + 'linux-x64': '@ardasevinc/mattermost-cli-linux-amd64', +} + +const target = `${process.platform}-${process.arch}` +const packageName = packages[target] +if (!packageName) { + process.stderr.write(`mattermost-cli: unsupported platform ${target}\n`) + process.exit(1) +} + +let binary +try { + binary = require.resolve(`${packageName}/bin/mm`) +} catch { + process.stderr.write( + `mattermost-cli: native package ${packageName} is missing; reinstall mattermost-cli without omitting optional dependencies\n`, + ) + process.exit(1) +} + +const result = spawnSync(binary, process.argv.slice(2), { stdio: 'inherit' }) +if (result.error) { + process.stderr.write('mattermost-cli: could not start the native mm binary\n') + process.exit(1) +} +if (result.signal) { + try { + process.kill(process.pid, result.signal) + } catch { + process.exit(1) + } +} else { + process.exit(result.status ?? 1) +} diff --git a/scripts/npm-package/main.go b/scripts/npm-package/main.go new file mode 100644 index 0000000..f6c3307 --- /dev/null +++ b/scripts/npm-package/main.go @@ -0,0 +1,262 @@ +package main + +import ( + "archive/tar" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "strings" +) + +var versionPattern = regexp.MustCompile(`^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$`) + +type target struct { + goos, goarch string + npmOS, npmCPU string +} + +type packageJSON struct { + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Author string `json:"author"` + License string `json:"license"` + Repository repository `json:"repository"` + Keywords []string `json:"keywords,omitempty"` + Bin map[string]string `json:"bin,omitempty"` + Files []string `json:"files"` + Engines map[string]string `json:"engines,omitempty"` + OS []string `json:"os,omitempty"` + CPU []string `json:"cpu,omitempty"` + OptionalDependencies map[string]string `json:"optionalDependencies,omitempty"` + PublishConfig map[string]string `json:"publishConfig"` +} + +type repository struct { + Type string `json:"type"` + URL string `json:"url"` +} + +func main() { + var version, releaseDirectory, output string + flag.StringVar(&version, "version", "", "release version without the v prefix") + flag.StringVar(&releaseDirectory, "release-dir", "dist", "directory containing native release archives") + flag.StringVar(&output, "output", "npm-dist", "generated npm package directory") + flag.Parse() + if err := run(version, releaseDirectory, output); err != nil { + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(version, releaseDirectory, output string) error { + version = strings.TrimPrefix(strings.TrimSpace(version), "v") + if !versionPattern.MatchString(version) { + return errors.New("npm package version must be semantic and omit the v prefix") + } + if releaseDirectory == "" || output == "" { + return errors.New("release and output directories are required") + } + if err := prepareOutput(output); err != nil { + return err + } + repositoryRoot, err := findRepositoryRoot() + if err != nil { + return err + } + checksums, err := loadChecksums(filepath.Join(releaseDirectory, "checksums.txt")) + if err != nil { + return err + } + targets := []target{ + {"darwin", "amd64", "darwin", "x64"}, {"darwin", "arm64", "darwin", "arm64"}, + {"linux", "amd64", "linux", "x64"}, {"linux", "arm64", "linux", "arm64"}, + } + dependencies := make(map[string]string, len(targets)) + for _, target := range targets { + name := platformPackageName(target) + dependencies[name] = version + archiveName := fmt.Sprintf("mattermost-cli_%s_%s_%s.tar.gz", version, target.goos, target.goarch) + archivePath := filepath.Join(releaseDirectory, archiveName) + if err := requireDigest(archivePath, checksums[archiveName]); err != nil { + return fmt.Errorf("verify %s: %w", archiveName, err) + } + binary, err := extractBinary(archivePath) + if err != nil { + return fmt.Errorf("extract %s: %w", archiveName, err) + } + directory := filepath.Join(output, strings.TrimPrefix(name, "@ardasevinc/")) + if err := os.MkdirAll(filepath.Join(directory, "bin"), 0o755); err != nil { // #nosec G301 -- public package tree. + return err + } + if err := os.WriteFile(filepath.Join(directory, "bin", "mm"), binary, 0o755); err != nil { // #nosec G306 -- public executable. + return err + } + manifest := baseManifest(name, version) + manifest.Description = "Native binary for mattermost-cli on " + target.goos + "/" + target.goarch + manifest.OS, manifest.CPU = []string{target.npmOS}, []string{target.npmCPU} + manifest.Files = []string{"bin/mm"} + if err := writeManifest(filepath.Join(directory, "package.json"), manifest); err != nil { + return err + } + } + launcher := filepath.Join(output, "mattermost-cli") + if err := os.MkdirAll(filepath.Join(launcher, "bin"), 0o755); err != nil { // #nosec G301 -- public package tree. + return err + } + for _, source := range []struct{ from, to string }{ + {filepath.Join(repositoryRoot, "npm", "bin", "mm.js"), filepath.Join(launcher, "bin", "mm.js")}, + {filepath.Join(repositoryRoot, "npm", "README.md"), filepath.Join(launcher, "README.md")}, + {filepath.Join(repositoryRoot, "LICENSE"), filepath.Join(launcher, "LICENSE")}, + } { + data, err := os.ReadFile(source.from) // #nosec G304 -- fixed repository sources. + if err != nil { + return err + } + mode := os.FileMode(0o644) + if strings.HasSuffix(source.to, ".js") { + mode = 0o755 + } + if err := os.WriteFile(source.to, data, mode); err != nil { // #nosec G306 -- public package files. + return err + } + } + manifest := baseManifest("mattermost-cli", version) + manifest.Description = "Mattermost CLI for agents and humans" + manifest.Keywords = []string{"mattermost", "cli", "agents", "messages", "go"} + manifest.Bin = map[string]string{"mm": "bin/mm.js"} + manifest.Files = []string{"bin/mm.js", "README.md", "LICENSE"} + manifest.Engines = map[string]string{"node": ">=18"} + manifest.OptionalDependencies = dependencies + return writeManifest(filepath.Join(launcher, "package.json"), manifest) +} + +func findRepositoryRoot() (string, error) { + directory, err := os.Getwd() + if err != nil { + return "", err + } + for { + if info, err := os.Stat(filepath.Join(directory, "go.mod")); err == nil && info.Mode().IsRegular() { + return directory, nil + } + parent := filepath.Dir(directory) + if parent == directory { + return "", errors.New("could not locate repository root") + } + directory = parent + } +} + +func baseManifest(name, version string) packageJSON { + return packageJSON{ + Name: name, Version: version, Author: "Arda Sevinc ", License: "MIT", + Repository: repository{Type: "git", URL: "https://github.com/ardasevinc/mattermost-cli"}, + PublishConfig: map[string]string{"access": "public"}, + } +} + +func platformPackageName(target target) string { + return "@ardasevinc/mattermost-cli-" + target.goos + "-" + target.goarch +} + +func prepareOutput(path string) error { + if err := os.MkdirAll(path, 0o755); err != nil { // #nosec G301 -- generated public package tree. + return err + } + entries, err := os.ReadDir(path) + if err != nil { + return err + } + if len(entries) != 0 { + return errors.New("npm package output directory must be empty") + } + return nil +} + +func loadChecksums(path string) (map[string]string, error) { + data, err := os.ReadFile(path) // #nosec G304 -- selected local release directory. + if err != nil { + return nil, err + } + result := make(map[string]string) + for _, line := range strings.Split(strings.TrimSuffix(string(data), "\n"), "\n") { + parts := strings.Split(line, " ") + if len(parts) != 2 || len(parts[0]) != 64 || strings.ContainsAny(parts[1], `/\\`) { + return nil, errors.New("invalid checksums manifest") + } + if _, err := hex.DecodeString(parts[0]); err != nil { + return nil, errors.New("invalid checksums manifest") + } + if _, duplicate := result[parts[1]]; duplicate { + return nil, errors.New("duplicate checksum entry") + } + result[parts[1]] = parts[0] + } + return result, nil +} + +func requireDigest(path, expected string) error { + if expected == "" { + return errors.New("archive checksum is missing") + } + file, err := os.Open(path) // #nosec G304 -- selected release artifact. + if err != nil { + return err + } + defer func() { _ = file.Close() }() + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + return err + } + if got := hex.EncodeToString(hash.Sum(nil)); got != expected { + return errors.New("archive checksum mismatch") + } + return nil +} + +func extractBinary(path string) ([]byte, error) { + file, err := os.Open(path) // #nosec G304 -- checksum-verified release artifact. + if err != nil { + return nil, err + } + defer func() { _ = file.Close() }() + zipper, err := gzip.NewReader(file) + if err != nil { + return nil, err + } + defer func() { _ = zipper.Close() }() + archive := tar.NewReader(zipper) + header, err := archive.Next() + if err != nil { + return nil, err + } + if header.Name != "mm" || header.Typeflag != tar.TypeReg || header.Mode != 0o755 || header.Size < 1 || header.Size > 100<<20 { + return nil, errors.New("archive has invalid mm member") + } + binary, err := io.ReadAll(io.LimitReader(archive, header.Size+1)) + if err != nil || int64(len(binary)) != header.Size { + return nil, errors.New("archive has invalid mm payload") + } + if _, err := archive.Next(); err != io.EOF { + return nil, errors.New("archive contains extra members") + } + return binary, nil +} + +func writeManifest(path string, manifest packageJSON) error { + data, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return err + } + data = append(data, '\n') + return os.WriteFile(path, data, 0o644) // #nosec G306 -- public npm manifest. +} diff --git a/scripts/npm-package/main_test.go b/scripts/npm-package/main_test.go new file mode 100644 index 0000000..f15b563 --- /dev/null +++ b/scripts/npm-package/main_test.go @@ -0,0 +1,110 @@ +package main + +import ( + "archive/tar" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRunBuildsLauncherAndChecksumBoundPlatformPackages(t *testing.T) { + releaseDirectory := t.TempDir() + checksums := make([]string, 0, 4) + for _, target := range []target{ + {"darwin", "amd64", "darwin", "x64"}, {"darwin", "arm64", "darwin", "arm64"}, + {"linux", "amd64", "linux", "x64"}, {"linux", "arm64", "linux", "arm64"}, + } { + name := "mattermost-cli_2.0.0_" + target.goos + "_" + target.goarch + ".tar.gz" + path := filepath.Join(releaseDirectory, name) + writeArchive(t, path, []byte(target.goos+"/"+target.goarch)) + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(data) + checksums = append(checksums, hex.EncodeToString(digest[:])+" "+name) + } + if err := os.WriteFile(filepath.Join(releaseDirectory, "checksums.txt"), []byte(strings.Join(checksums, "\n")+"\n"), 0o600); err != nil { + t.Fatal(err) + } + output := filepath.Join(t.TempDir(), "packages") + if err := run("2.0.0", releaseDirectory, output); err != nil { + t.Fatal(err) + } + launcher := readManifest(t, filepath.Join(output, "mattermost-cli", "package.json")) + if launcher.Name != "mattermost-cli" || launcher.Version != "2.0.0" || launcher.Bin["mm"] != "bin/mm.js" || len(launcher.OptionalDependencies) != 4 { + t.Fatalf("launcher manifest = %+v", launcher) + } + for _, target := range []target{ + {"darwin", "amd64", "darwin", "x64"}, {"darwin", "arm64", "darwin", "arm64"}, + {"linux", "amd64", "linux", "x64"}, {"linux", "arm64", "linux", "arm64"}, + } { + shortName := strings.TrimPrefix(platformPackageName(target), "@ardasevinc/") + directory := filepath.Join(output, shortName) + manifest := readManifest(t, filepath.Join(directory, "package.json")) + if manifest.Name != platformPackageName(target) || manifest.Version != "2.0.0" || len(manifest.OS) != 1 || manifest.OS[0] != target.npmOS || len(manifest.CPU) != 1 || manifest.CPU[0] != target.npmCPU { + t.Fatalf("%s manifest = %+v", shortName, manifest) + } + binary, err := os.ReadFile(filepath.Join(directory, "bin", "mm")) + if err != nil || string(binary) != target.goos+"/"+target.goarch { + t.Fatalf("%s binary=%q err=%v", shortName, binary, err) + } + } +} + +func TestRunRejectsMissingOrChangedArchive(t *testing.T) { + releaseDirectory := t.TempDir() + if err := os.WriteFile(filepath.Join(releaseDirectory, "checksums.txt"), []byte(strings.Repeat("0", 64)+" mattermost-cli_2.0.0_darwin_amd64.tar.gz\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(releaseDirectory, "mattermost-cli_2.0.0_darwin_amd64.tar.gz"), []byte("changed"), 0o600); err != nil { + t.Fatal(err) + } + err := run("2.0.0", releaseDirectory, filepath.Join(t.TempDir(), "out")) + if err == nil || !strings.Contains(err.Error(), "checksum mismatch") { + t.Fatalf("run() error = %v", err) + } +} + +func readManifest(t *testing.T, path string) packageJSON { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var manifest packageJSON + if err := json.Unmarshal(data, &manifest); err != nil { + t.Fatal(err) + } + return manifest +} + +func writeArchive(t *testing.T, path string, payload []byte) { + t.Helper() + file, err := os.Create(path) // #nosec G304 -- test-owned path. + if err != nil { + t.Fatal(err) + } + zipper := gzip.NewWriter(file) + archive := tar.NewWriter(zipper) + if err := archive.WriteHeader(&tar.Header{Name: "mm", Mode: 0o755, Size: int64(len(payload)), Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + if _, err := archive.Write(payload); err != nil { + t.Fatal(err) + } + if err := archive.Close(); err != nil { + t.Fatal(err) + } + if err := zipper.Close(); err != nil { + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } +} From 6499d034c31747c0eeffd6e9bc6785ede0103fcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 12:45:21 +0300 Subject: [PATCH 106/119] ci: add native release provenance pipeline --- .github/workflows/ci.yml | 41 +++++++++++ .github/workflows/release.yml | 131 ++++++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd1dbe9..ab22d67 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,3 +58,44 @@ jobs: npm install --global --prefix "$RUNNER_TEMP/npm-global" "./${{ steps.pack.outputs.tarball }}" test "$("$RUNNER_TEMP/npm-global/bin/mm" --version)" = "$(node -p "require('./package.json').version")" "$RUNNER_TEMP/npm-global/bin/mm" --help + + distribution: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version-file: go.mod + cache: true + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24 + - name: Build release artifacts reproducibly + run: | + go run ./scripts/release --version 2.0.0-dev --commit "$GITHUB_SHA" --output dist + GOCACHE="$RUNNER_TEMP/repro-cache" go run ./scripts/release --version 2.0.0-dev --commit "$GITHUB_SHA" --output repro + diff -r dist repro + (cd dist && sha256sum --check checksums.txt) + ruby -c dist/mattermost-cli.rb + - name: Generate and pack exact npm packages + shell: bash + run: | + go run ./scripts/npm-package --version 2.0.0-dev --release-dir dist --output npm-dist + mkdir npm-packs + for package_dir in npm-dist/*; do + npm pack --json --ignore-scripts --pack-destination npm-packs "$package_dir" >/dev/null + done + test "$(find npm-packs -maxdepth 1 -name '*.tgz' | wc -l)" -eq 5 + - name: Smoke native archive and npm launcher + run: | + mkdir "$RUNNER_TEMP/native-smoke" + tar -xzf dist/mattermost-cli_2.0.0-dev_linux_amd64.tar.gz -C "$RUNNER_TEMP/native-smoke" + test "$("$RUNNER_TEMP/native-smoke/mm" --version)" = "mm version 2.0.0-dev ($GITHUB_SHA)" + mkdir "$RUNNER_TEMP/npm-smoke" + cd "$RUNNER_TEMP/npm-smoke" + npm init -y >/dev/null + npm install --ignore-scripts \ + "$GITHUB_WORKSPACE/npm-packs/ardasevinc-mattermost-cli-linux-amd64-2.0.0-dev.tgz" \ + "$GITHUB_WORKSPACE/npm-packs/mattermost-cli-2.0.0-dev.tgz" >/dev/null + test "$(./node_modules/.bin/mm --version)" = "mm version 2.0.0-dev ($GITHUB_SHA)" + ./node_modules/.bin/mm --help diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..2d748cb --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,131 @@ +name: Release + +on: + push: + tags: ["v*"] + +permissions: + actions: read + contents: write + id-token: write + attestations: write + artifact-metadata: write + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + +jobs: + release: + name: Reproducible native release + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + with: + go-version-file: go.mod + cache: true + - name: Validate tag and source version + id: version + shell: bash + run: | + version="${GITHUB_REF_NAME#v}" + test "$GITHUB_REF_NAME" = "v$version" + source_version="$(awk -F'"' '/^[[:space:]]*Version = / {print $2}' internal/buildinfo/buildinfo.go)" + test "$version" = "$source_version" + if [[ "$version" == *-* ]]; then prerelease=true; else prerelease=false; fi + printf 'version=%s\n' "$version" >> "$GITHUB_OUTPUT" + printf 'prerelease=%s\n' "$prerelease" >> "$GITHUB_OUTPUT" + - name: Require tagged commit on main + run: | + git fetch --no-tags origin main + git merge-base --is-ancestor "$GITHUB_SHA" origin/main + - name: Require successful CI for tagged commit + env: + GH_TOKEN: ${{ github.token }} + run: | + conclusion="$(gh api "repos/$GITHUB_REPOSITORY/actions/workflows/ci.yml/runs?head_sha=$GITHUB_SHA&status=completed" --jq '.workflow_runs | sort_by(.created_at) | last | .conclusion')" + test "$conclusion" = success + - name: Reject existing release + env: + GH_TOKEN: ${{ github.token }} + run: | + if gh release view "$GITHUB_REF_NAME" >/dev/null 2>&1; then + echo "release $GITHUB_REF_NAME already exists" >&2 + exit 1 + fi + - name: Test release source + run: | + go mod download + GOPROXY=off go test -race -p 1 ./... + GOPROXY=off go vet ./... + go install golang.org/x/vuln/cmd/govulncheck@v1.6.0 + govulncheck ./... + - name: Build deterministic archives twice + env: + GOPROXY: "off" + run: | + GOCACHE="$RUNNER_TEMP/gocache-a" go run ./scripts/release --version "${{ steps.version.outputs.version }}" --commit "$GITHUB_SHA" --output dist + GOCACHE="$RUNNER_TEMP/gocache-b" go run ./scripts/release --version "${{ steps.version.outputs.version }}" --commit "$GITHUB_SHA" --output repro + diff -r dist repro + - name: Verify archives, checksums, formula, and Linux binary + run: | + cd dist + sha256sum --check checksums.txt + ruby -c mattermost-cli.rb + mkdir "$RUNNER_TEMP/mm-smoke" + tar -xzf "mattermost-cli_${{ steps.version.outputs.version }}_linux_amd64.tar.gz" -C "$RUNNER_TEMP/mm-smoke" + test "$("$RUNNER_TEMP/mm-smoke/mm" --version)" = "mm version ${{ steps.version.outputs.version }} ($GITHUB_SHA)" + "$RUNNER_TEMP/mm-smoke/mm" --help + "$RUNNER_TEMP/mm-smoke/mm" config --path + - name: Attest release artifacts + uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 + with: + subject-path: "dist/*" + - name: Verify release attestations + env: + GH_TOKEN: ${{ github.token }} + run: | + for artifact in dist/*; do + gh attestation verify "$artifact" --repo "$GITHUB_REPOSITORY" + done + - name: Create complete draft release + id: create_release + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 + with: + files: dist/* + fail_on_unmatched_files: true + generate_release_notes: true + draft: true + overwrite_files: false + prerelease: ${{ steps.version.outputs.prerelease }} + - name: Verify assets and publish release + env: + GH_TOKEN: ${{ github.token }} + RELEASE_ID: ${{ steps.create_release.outputs.id }} + run: | + find dist -maxdepth 1 -type f -exec basename {} \; | sort > "$RUNNER_TEMP/expected-assets" + test -n "$RELEASE_ID" + gh api "repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID" --jq '.assets[].name' | sort > "$RUNNER_TEMP/actual-assets" + diff -u "$RUNNER_TEMP/expected-assets" "$RUNNER_TEMP/actual-assets" + if [[ "${{ steps.version.outputs.prerelease }}" == true ]]; then latest=false; else latest=true; fi + gh api --method PATCH "repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID" \ + -F draft=false \ + -F prerelease="${{ steps.version.outputs.prerelease }}" \ + -f make_latest="$latest" >/dev/null + - name: Update Homebrew tap formula + env: + GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }} + VERSION: ${{ steps.version.outputs.version }} + run: | + test -n "$GH_TOKEN" + path="Formula/mattermost-cli.rb" + content="$(base64 -w 0 < dist/mattermost-cli.rb)" + sha="$(gh api "repos/ardasevinc/homebrew-tap/contents/$path" --jq .sha 2>/dev/null || true)" + args=(--method PUT "repos/ardasevinc/homebrew-tap/contents/$path" -f message="mattermost-cli $VERSION" -f content="$content" -f branch=main) + if [[ -n "$sha" ]]; then args+=(-f sha="$sha"); fi + gh api "${args[@]}" >/dev/null From 02100f1e4db6655da1fbb4861a908e1cdb24eeda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 12:46:29 +0300 Subject: [PATCH 107/119] ci: publish native npm packages with oidc --- .github/workflows/publish.yml | 118 ++++++++++++++++++++++++++-------- 1 file changed, 90 insertions(+), 28 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 15f66c9..1e5a0e8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,4 +1,4 @@ -name: Publish +name: Publish npm packages on: release: @@ -7,49 +7,111 @@ on: permissions: contents: read id-token: write + attestations: read + +concurrency: + group: npm-${{ github.event.release.tag_name }} + cancel-in-progress: false jobs: publish: runs-on: ubuntu-latest + timeout-minutes: 20 environment: npm steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ github.event.release.tag_name }} - - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + persist-credentials: false + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: - bun-version: 1.3.14 + go-version-file: go.mod + cache: true - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 24 registry-url: https://registry.npmjs.org - - run: bun install --frozen-lockfile - - run: bun audit - - run: bun run verify - env: - RELEASE_TAG: ${{ github.event.release.tag_name }} - - name: Require an unpublished npm version + - name: Validate release identity + id: version shell: bash run: | - name=$(node -p "require('./package.json').name") - version=$(node -p "require('./package.json').version") - versions=$(npm view "$name" versions --json) - node -e 'const versions = JSON.parse(process.argv[1]); if (versions.includes(process.argv[2])) { throw new Error("npm version " + process.argv[2] + " is already published") }' "$versions" "$version" - - name: Pack and smoke exact tarball - id: pack + version="${{ github.event.release.tag_name }}" + version="${version#v}" + test "${{ github.event.release.tag_name }}" = "v$version" + source_version="$(awk -F'"' '/^[[:space:]]*Version = / {print $2}' internal/buildinfo/buildinfo.go)" + test "$version" = "$source_version" + commit="$(git rev-parse HEAD)" + test "$commit" = "$(git rev-parse '${{ github.event.release.tag_name }}^{commit}')" + printf 'version=%s\n' "$version" >> "$GITHUB_OUTPUT" + printf 'commit=%s\n' "$commit" >> "$GITHUB_OUTPUT" + - name: Download and verify native release assets + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir dist + gh release download "${{ github.event.release.tag_name }}" --dir dist + (cd dist && sha256sum --check checksums.txt) + for artifact in dist/*; do + gh attestation verify "$artifact" --repo "$GITHUB_REPOSITORY" + done + - name: Generate and pack npm packages shell: bash run: | - tarball=$(npm pack --json --ignore-scripts | node -e 'let input=""; process.stdin.on("data", c => input += c); process.stdin.on("end", () => process.stdout.write(JSON.parse(input)[0].filename))') - printf 'tarball=%s\n' "$tarball" >> "$GITHUB_OUTPUT" - actual=$(tar -tzf "$tarball" | LC_ALL=C sort) - expected=$(printf '%s\n' package/LICENSE package/README.md package/dist/index.js package/package.json | LC_ALL=C sort) + go run ./scripts/npm-package --version "${{ steps.version.outputs.version }}" --release-dir dist --output npm-dist + mkdir npm-packs + for package_dir in npm-dist/*; do + npm pack --json --ignore-scripts --pack-destination npm-packs "$package_dir" >/dev/null + done + test "$(find npm-packs -maxdepth 1 -name '*.tgz' | wc -l)" -eq 5 + for tarball in npm-packs/ardasevinc-*.tgz; do + actual="$(tar -tzf "$tarball" | LC_ALL=C sort)" + expected="$(printf '%s\n' package/bin/mm package/package.json | LC_ALL=C sort)" + diff -u <(printf '%s\n' "$expected") <(printf '%s\n' "$actual") + done + main="npm-packs/mattermost-cli-${{ steps.version.outputs.version }}.tgz" + actual="$(tar -tzf "$main" | LC_ALL=C sort)" + expected="$(printf '%s\n' package/LICENSE package/README.md package/bin/mm.js package/package.json | LC_ALL=C sort)" diff -u <(printf '%s\n' "$expected") <(printf '%s\n' "$actual") - mkdir package-smoke - tar -xzf "$tarball" -C package-smoke - test "$(node package-smoke/package/dist/index.js --version)" = "$(node -p "require('./package.json').version")" - node package-smoke/package/dist/index.js --help - node package-smoke/package/dist/index.js config --path - npm install --global --prefix "$RUNNER_TEMP/npm-global" "./$tarball" - test "$("$RUNNER_TEMP/npm-global/bin/mm" --version)" = "$(node -p "require('./package.json').version")" - "$RUNNER_TEMP/npm-global/bin/mm" --help - - run: npm publish "./${{ steps.pack.outputs.tarball }}" --access public --provenance + - name: Smoke exact Linux npm install + run: | + mkdir "$RUNNER_TEMP/npm-smoke" + cd "$RUNNER_TEMP/npm-smoke" + npm init -y >/dev/null + npm install --ignore-scripts \ + "$GITHUB_WORKSPACE/npm-packs/ardasevinc-mattermost-cli-linux-amd64-${{ steps.version.outputs.version }}.tgz" \ + "$GITHUB_WORKSPACE/npm-packs/mattermost-cli-${{ steps.version.outputs.version }}.tgz" >/dev/null + test "$(./node_modules/.bin/mm --version)" = "mm version ${{ steps.version.outputs.version }} (${{ steps.version.outputs.commit }})" + ./node_modules/.bin/mm --help + - name: Preflight immutable npm versions + shell: bash + run: | + for tarball in npm-packs/*.tgz; do + name="$(tar -xOzf "$tarball" package/package.json | node -p 'JSON.parse(require("fs").readFileSync(0, "utf8")).name')" + version="${{ steps.version.outputs.version }}" + remote="$(npm view "$name@$version" dist.integrity --json 2>/dev/null || true)" + if [[ -n "$remote" ]]; then + remote="$(node -p 'JSON.parse(process.argv[1])' "$remote")" + local_integrity="sha512-$(openssl dgst -sha512 -binary "$tarball" | base64 -w 0)" + test "$remote" = "$local_integrity" + fi + done + - name: Publish native platform packages + shell: bash + run: | + for tarball in npm-packs/ardasevinc-*.tgz; do + name="$(tar -xOzf "$tarball" package/package.json | node -p 'JSON.parse(require("fs").readFileSync(0, "utf8")).name')" + if npm view "$name@${{ steps.version.outputs.version }}" version >/dev/null 2>&1; then + echo "already published identically: $name@${{ steps.version.outputs.version }}" + else + npm publish "$tarball" --access public --provenance + fi + done + - name: Publish launcher package last + shell: bash + run: | + tarball="npm-packs/mattermost-cli-${{ steps.version.outputs.version }}.tgz" + if npm view "mattermost-cli@${{ steps.version.outputs.version }}" version >/dev/null 2>&1; then + echo "already published identically: mattermost-cli@${{ steps.version.outputs.version }}" + else + npm publish "$tarball" --access public --provenance + fi From e9afe360d4bca62214c8e8da65adf0f9fce0c04e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 12:48:48 +0300 Subject: [PATCH 108/119] build: adopt Go v2 module path --- cmd/conformance/main.go | 2 +- cmd/mm/main.go | 2 +- cmd/mm/signal_other.go | 2 +- cmd/mm/signal_unix.go | 2 +- cmd/mm/signal_unix_test.go | 2 +- docs/V2_CONTRACT.md | 2 +- go.mod | 2 +- internal/api/client.go | 4 ++-- internal/api/client_test.go | 2 +- internal/apply/post.go | 8 ++++---- internal/apply/post_test.go | 10 +++++----- internal/apply/reaction.go | 8 ++++---- internal/apply/service.go | 10 +++++----- internal/apply/service_test.go | 6 +++--- internal/cli/apply.go | 12 ++++++------ internal/cli/apply_test.go | 8 ++++---- internal/cli/channel.go | 6 +++--- internal/cli/channel_test.go | 14 +++++++------- internal/cli/config.go | 6 +++--- internal/cli/config_test.go | 2 +- internal/cli/dms.go | 12 ++++++------ internal/cli/dms_test.go | 2 +- internal/cli/doctor.go | 8 ++++---- internal/cli/doctor_test.go | 2 +- internal/cli/group_dms.go | 8 ++++---- internal/cli/group_dms_test.go | 4 ++-- internal/cli/identity.go | 6 +++--- internal/cli/identity_test.go | 4 ++-- internal/cli/mentions.go | 6 +++--- internal/cli/mentions_test.go | 2 +- internal/cli/read.go | 12 ++++++------ internal/cli/root.go | 8 ++++---- internal/cli/root_test.go | 2 +- internal/cli/runtime.go | 10 +++++----- internal/cli/runtime_test.go | 8 ++++---- internal/cli/search.go | 6 +++--- internal/cli/search_test.go | 2 +- internal/cli/stage_create.go | 12 ++++++------ internal/cli/stage_create_test.go | 4 ++-- internal/cli/stage_inspect.go | 10 +++++----- internal/cli/stage_inspect_test.go | 4 ++-- internal/cli/stage_manage.go | 14 +++++++------- internal/cli/stage_manage_test.go | 4 ++-- internal/cli/store.go | 8 ++++---- internal/cli/store_test.go | 4 ++-- internal/cli/thread.go | 4 ++-- internal/cli/thread_test.go | 8 ++++---- internal/cli/unread.go | 6 +++--- internal/cli/unread_test.go | 2 +- internal/cli/watch.go | 6 +++--- internal/cli/watch_test.go | 6 +++--- internal/doctor/doctor.go | 8 ++++---- internal/doctor/doctor_test.go | 4 ++-- internal/mattermost/conversation_mutations.go | 2 +- internal/mattermost/conversation_mutations_test.go | 2 +- internal/mattermost/file_mutations.go | 2 +- internal/mattermost/file_mutations_test.go | 2 +- internal/mattermost/mutations.go | 2 +- internal/mattermost/mutations_test.go | 2 +- internal/mattermost/reaction_mutations.go | 2 +- internal/mattermost/watch.go | 6 +++--- internal/mattermost/watch_test.go | 4 ++-- internal/normalization/post.go | 8 ++++---- internal/normalization/post_test.go | 4 ++-- internal/output/identity_machine.go | 2 +- internal/output/identity_machine_test.go | 6 +++--- internal/output/machine_convert_test.go | 4 ++-- internal/output/model.go | 2 +- internal/output/unread.go | 2 +- internal/output/unread_test.go | 6 +++--- internal/output/watch.go | 4 ++-- internal/output/watch_test.go | 4 ++-- internal/retrieval/channel.go | 2 +- internal/retrieval/channel_test.go | 2 +- internal/retrieval/dms.go | 2 +- internal/retrieval/dms_test.go | 2 +- internal/retrieval/group_dms_test.go | 2 +- internal/retrieval/hydration.go | 2 +- internal/retrieval/mentions.go | 2 +- internal/retrieval/mentions_test.go | 2 +- internal/retrieval/search.go | 2 +- internal/retrieval/search_test.go | 2 +- internal/retrieval/thread.go | 2 +- internal/retrieval/thread_test.go | 2 +- internal/retrieval/unread.go | 2 +- internal/retrieval/unread_test.go | 2 +- internal/schema/apply_test.go | 2 +- internal/schema/read_test.go | 2 +- internal/schema/registry.go | 2 +- internal/schema/registry_test.go | 2 +- internal/schema/stage_test.go | 2 +- internal/schema/unread_test.go | 2 +- internal/stagecontent/content.go | 2 +- internal/stagecontent/content_test.go | 2 +- internal/stageinput/input.go | 2 +- internal/stageinput/input_test.go | 2 +- internal/stageinput/spool.go | 2 +- internal/stageoutput/output.go | 8 ++++---- internal/stageoutput/output_test.go | 6 +++--- internal/stagerequest/request.go | 8 ++++---- internal/stagerequest/request_test.go | 6 +++--- internal/stagestore/domain.go | 6 +++--- internal/stagestore/domain_test.go | 2 +- internal/staging/conversation.go | 2 +- internal/staging/conversation_stage.go | 4 ++-- internal/staging/conversation_stage_test.go | 4 ++-- internal/staging/intent.go | 4 ++-- internal/staging/intent_test.go | 4 ++-- internal/staging/post.go | 8 ++++---- internal/staging/post_test.go | 6 +++--- internal/staging/revision.go | 6 +++--- internal/staging/revision_test.go | 4 ++-- internal/staging/service.go | 10 +++++----- internal/staging/service_test.go | 6 +++--- internal/staging/types.go | 6 +++--- internal/staging/validation.go | 4 ++-- scripts/release/main.go | 2 +- tests/e2e/go-lifecycle-live_test.go | 2 +- tests/e2e/go-read-live_test.go | 4 ++-- 119 files changed, 272 insertions(+), 272 deletions(-) diff --git a/cmd/conformance/main.go b/cmd/conformance/main.go index 8856279..23243ba 100644 --- a/cmd/conformance/main.go +++ b/cmd/conformance/main.go @@ -8,7 +8,7 @@ import ( "os/signal" "syscall" - "github.com/ardasevinc/mattermost-cli/internal/conformance" + "github.com/ardasevinc/mattermost-cli/v2/internal/conformance" ) type stringList []string diff --git a/cmd/mm/main.go b/cmd/mm/main.go index 2c07ccd..d5b0250 100644 --- a/cmd/mm/main.go +++ b/cmd/mm/main.go @@ -4,7 +4,7 @@ import ( "context" "os" - "github.com/ardasevinc/mattermost-cli/internal/cli" + "github.com/ardasevinc/mattermost-cli/v2/internal/cli" ) func main() { diff --git a/cmd/mm/signal_other.go b/cmd/mm/signal_other.go index eb32ef4..d6cfaf5 100644 --- a/cmd/mm/signal_other.go +++ b/cmd/mm/signal_other.go @@ -8,7 +8,7 @@ import ( "os/signal" "sync" - "github.com/ardasevinc/mattermost-cli/internal/cli" + "github.com/ardasevinc/mattermost-cli/v2/internal/cli" ) func handleBrokenPipe() {} diff --git a/cmd/mm/signal_unix.go b/cmd/mm/signal_unix.go index 9e40a4b..e6fba41 100644 --- a/cmd/mm/signal_unix.go +++ b/cmd/mm/signal_unix.go @@ -9,7 +9,7 @@ import ( "sync" "syscall" - "github.com/ardasevinc/mattermost-cli/internal/cli" + "github.com/ardasevinc/mattermost-cli/v2/internal/cli" ) var brokenPipeSignals = make(chan os.Signal, 1) diff --git a/cmd/mm/signal_unix_test.go b/cmd/mm/signal_unix_test.go index 2088cf6..caa7352 100644 --- a/cmd/mm/signal_unix_test.go +++ b/cmd/mm/signal_unix_test.go @@ -14,7 +14,7 @@ import ( "testing" "time" - "github.com/ardasevinc/mattermost-cli/internal/cli" + "github.com/ardasevinc/mattermost-cli/v2/internal/cli" ) func TestClosedStdoutUsesStableExitClass(t *testing.T) { diff --git a/docs/V2_CONTRACT.md b/docs/V2_CONTRACT.md index b12878e..6493576 100644 --- a/docs/V2_CONTRACT.md +++ b/docs/V2_CONTRACT.md @@ -563,7 +563,7 @@ Primary Go v2 distribution: - checksums and verifiable build provenance; - Homebrew using the established patterns in Arda's current Go projects; - install script with checksum verification; -- `go install` for source-based installation. +- `go install github.com/ardasevinc/mattermost-cli/v2/cmd/mm@v2.0.0` for source-based installation. The existing unscoped npm package remains an upgrade path for prior npm users. Its v2 package becomes a small launcher backed by platform-specific optional packages containing the exact Go release binaries. It must not fetch and execute an unverified binary during `postinstall`. npm installation continuity does not imply v1 command or schema compatibility. diff --git a/go.mod b/go.mod index f3a561a..8124330 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/ardasevinc/mattermost-cli +module github.com/ardasevinc/mattermost-cli/v2 go 1.26.5 diff --git a/internal/api/client.go b/internal/api/client.go index e26d655..c196518 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -16,8 +16,8 @@ import ( "sync" "time" - "github.com/ardasevinc/mattermost-cli/internal/presentation" - "github.com/ardasevinc/mattermost-cli/internal/serverurl" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/serverurl" ) const ( diff --git a/internal/api/client_test.go b/internal/api/client_test.go index c0fe8cb..ee39fc2 100644 --- a/internal/api/client_test.go +++ b/internal/api/client_test.go @@ -13,7 +13,7 @@ import ( "testing" "time" - "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" ) func TestPreparedMutationFreezesRequestBeforeDispatch(t *testing.T) { diff --git a/internal/apply/post.go b/internal/apply/post.go index 4a14fbe..38f8b98 100644 --- a/internal/apply/post.go +++ b/internal/apply/post.go @@ -7,10 +7,10 @@ import ( "errors" "strings" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/stageinput" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" - "github.com/ardasevinc/mattermost-cli/internal/staging" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/staging" ) func (s *Service) applyPost(ctx context.Context, attempt stagestore.ApplyAttempt, operation stagestore.Operation, currentUserID string, destination staging.Destination, body []byte, attachments []stagestore.Attachment) (stagestore.ApplyReceipt, error) { diff --git a/internal/apply/post_test.go b/internal/apply/post_test.go index 4e63229..ee6fc4a 100644 --- a/internal/apply/post_test.go +++ b/internal/apply/post_test.go @@ -17,11 +17,11 @@ import ( "sync/atomic" "testing" - "github.com/ardasevinc/mattermost-cli/internal/api" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/stageinput" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" - "github.com/ardasevinc/mattermost-cli/internal/staging" + "github.com/ardasevinc/mattermost-cli/v2/internal/api" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/staging" ) func TestApplyCreatePostPreservesStagedMarkdownExactly(t *testing.T) { diff --git a/internal/apply/reaction.go b/internal/apply/reaction.go index aea2a0f..4d7aa8f 100644 --- a/internal/apply/reaction.go +++ b/internal/apply/reaction.go @@ -7,10 +7,10 @@ import ( "slices" "strings" - "github.com/ardasevinc/mattermost-cli/internal/api" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" - "github.com/ardasevinc/mattermost-cli/internal/staging" + "github.com/ardasevinc/mattermost-cli/v2/internal/api" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/staging" ) type preparedReaction interface { diff --git a/internal/apply/service.go b/internal/apply/service.go index 4191ceb..bbc40fe 100644 --- a/internal/apply/service.go +++ b/internal/apply/service.go @@ -11,11 +11,11 @@ import ( "path/filepath" "slices" - "github.com/ardasevinc/mattermost-cli/internal/api" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/stageinput" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" - "github.com/ardasevinc/mattermost-cli/internal/staging" + "github.com/ardasevinc/mattermost-cli/v2/internal/api" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/staging" ) var ( diff --git a/internal/apply/service_test.go b/internal/apply/service_test.go index ba7bc04..e00b3a0 100644 --- a/internal/apply/service_test.go +++ b/internal/apply/service_test.go @@ -16,9 +16,9 @@ import ( "sync/atomic" "testing" - "github.com/ardasevinc/mattermost-cli/internal/api" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/api" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) func TestApplyResolveDMCreatesOnceAndReplaysDurableReceipt(t *testing.T) { diff --git a/internal/cli/apply.go b/internal/cli/apply.go index fa503df..531b261 100644 --- a/internal/cli/apply.go +++ b/internal/cli/apply.go @@ -13,12 +13,12 @@ import ( "github.com/spf13/cobra" - "github.com/ardasevinc/mattermost-cli/internal/api" - applyservice "github.com/ardasevinc/mattermost-cli/internal/apply" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/schema" - "github.com/ardasevinc/mattermost-cli/internal/stagerequest" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/api" + applyservice "github.com/ardasevinc/mattermost-cli/v2/internal/apply" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagerequest" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) var stageReferencePattern = regexp.MustCompile(`^(stg_[A-Za-z0-9_-]{32})@([1-9][0-9]{0,15})$`) diff --git a/internal/cli/apply_test.go b/internal/cli/apply_test.go index 13b8f59..5f3c68d 100644 --- a/internal/cli/apply_test.go +++ b/internal/cli/apply_test.go @@ -16,10 +16,10 @@ import ( "sync/atomic" "testing" - mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" - "github.com/ardasevinc/mattermost-cli/internal/stagerequest" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" - "github.com/ardasevinc/mattermost-cli/internal/staging" + mmSchema "github.com/ardasevinc/mattermost-cli/v2/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagerequest" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/staging" ) func createCLIApplyStage(t *testing.T, stateRoot, serverURL, body string) stagestore.MutationResult { diff --git a/internal/cli/channel.go b/internal/cli/channel.go index 0329374..897eb68 100644 --- a/internal/cli/channel.go +++ b/internal/cli/channel.go @@ -7,9 +7,9 @@ import ( "github.com/spf13/cobra" - "github.com/ardasevinc/mattermost-cli/internal/cursor" - "github.com/ardasevinc/mattermost-cli/internal/output" - "github.com/ardasevinc/mattermost-cli/internal/retrieval" + "github.com/ardasevinc/mattermost-cli/v2/internal/cursor" + "github.com/ardasevinc/mattermost-cli/v2/internal/output" + "github.com/ardasevinc/mattermost-cli/v2/internal/retrieval" ) type channelFlags struct { diff --git a/internal/cli/channel_test.go b/internal/cli/channel_test.go index 2ef0291..39fd21d 100644 --- a/internal/cli/channel_test.go +++ b/internal/cli/channel_test.go @@ -14,13 +14,13 @@ import ( "testing" "time" - "github.com/ardasevinc/mattermost-cli/internal/api" - "github.com/ardasevinc/mattermost-cli/internal/config" - "github.com/ardasevinc/mattermost-cli/internal/cursor" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/presentation" - "github.com/ardasevinc/mattermost-cli/internal/retrieval" - mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/api" + "github.com/ardasevinc/mattermost-cli/v2/internal/config" + "github.com/ardasevinc/mattermost-cli/v2/internal/cursor" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/retrieval" + mmSchema "github.com/ardasevinc/mattermost-cli/v2/internal/schema" ) func TestChannelJSONRunsValidatedReadPipeline(t *testing.T) { diff --git a/internal/cli/config.go b/internal/cli/config.go index a803b20..5b8b073 100644 --- a/internal/cli/config.go +++ b/internal/cli/config.go @@ -6,9 +6,9 @@ import ( "github.com/spf13/cobra" - "github.com/ardasevinc/mattermost-cli/internal/config" - "github.com/ardasevinc/mattermost-cli/internal/output" - "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/config" + "github.com/ardasevinc/mattermost-cli/v2/internal/output" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" ) type configFlags struct { diff --git a/internal/cli/config_test.go b/internal/cli/config_test.go index fb9ed06..2448819 100644 --- a/internal/cli/config_test.go +++ b/internal/cli/config_test.go @@ -9,7 +9,7 @@ import ( "strings" "testing" - mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" + mmSchema "github.com/ardasevinc/mattermost-cli/v2/internal/schema" ) func TestConfigStatusIsOfflineAndNeverEmitsValues(t *testing.T) { diff --git a/internal/cli/dms.go b/internal/cli/dms.go index 4a4bcc8..998b668 100644 --- a/internal/cli/dms.go +++ b/internal/cli/dms.go @@ -9,12 +9,12 @@ import ( "github.com/spf13/cobra" - "github.com/ardasevinc/mattermost-cli/internal/api" - "github.com/ardasevinc/mattermost-cli/internal/cursor" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/normalization" - "github.com/ardasevinc/mattermost-cli/internal/output" - "github.com/ardasevinc/mattermost-cli/internal/retrieval" + "github.com/ardasevinc/mattermost-cli/v2/internal/api" + "github.com/ardasevinc/mattermost-cli/v2/internal/cursor" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/normalization" + "github.com/ardasevinc/mattermost-cli/v2/internal/output" + "github.com/ardasevinc/mattermost-cli/v2/internal/retrieval" ) type dmsFlags struct { diff --git a/internal/cli/dms_test.go b/internal/cli/dms_test.go index f3e3991..36fd01c 100644 --- a/internal/cli/dms_test.go +++ b/internal/cli/dms_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" + mmSchema "github.com/ardasevinc/mattermost-cli/v2/internal/schema" ) func TestDMsJSONDiscoversAccountWideAndAppliesGlobalLimit(t *testing.T) { diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index 9233a3a..29a2afb 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -6,10 +6,10 @@ import ( "github.com/spf13/cobra" - "github.com/ardasevinc/mattermost-cli/internal/config" - "github.com/ardasevinc/mattermost-cli/internal/doctor" - "github.com/ardasevinc/mattermost-cli/internal/output" - "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/config" + "github.com/ardasevinc/mattermost-cli/v2/internal/doctor" + "github.com/ardasevinc/mattermost-cli/v2/internal/output" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" ) func newDoctorCommand(state *rootState) *cobra.Command { diff --git a/internal/cli/doctor_test.go b/internal/cli/doctor_test.go index c3b9ee2..38429c7 100644 --- a/internal/cli/doctor_test.go +++ b/internal/cli/doctor_test.go @@ -12,7 +12,7 @@ import ( "sync/atomic" "testing" - mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" + mmSchema "github.com/ardasevinc/mattermost-cli/v2/internal/schema" ) func TestDoctorMachineReportUsesPublicPingAndAuthenticatedIdentity(t *testing.T) { diff --git a/internal/cli/group_dms.go b/internal/cli/group_dms.go index 0d9d1b3..aabb25b 100644 --- a/internal/cli/group_dms.go +++ b/internal/cli/group_dms.go @@ -7,10 +7,10 @@ import ( "github.com/spf13/cobra" - "github.com/ardasevinc/mattermost-cli/internal/cursor" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/output" - "github.com/ardasevinc/mattermost-cli/internal/retrieval" + "github.com/ardasevinc/mattermost-cli/v2/internal/cursor" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/output" + "github.com/ardasevinc/mattermost-cli/v2/internal/retrieval" ) type groupDMsFlags struct{ limit, since, channel, cursor string } diff --git a/internal/cli/group_dms_test.go b/internal/cli/group_dms_test.go index 36c573f..62f3292 100644 --- a/internal/cli/group_dms_test.go +++ b/internal/cli/group_dms_test.go @@ -10,8 +10,8 @@ import ( "testing" "time" - "github.com/ardasevinc/mattermost-cli/internal/cursor" - mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/cursor" + mmSchema "github.com/ardasevinc/mattermost-cli/v2/internal/schema" ) func TestGroupDMsJSONDiscoversFocusedChannelsAppliesGlobalLimitAndSanitizesLabel(t *testing.T) { diff --git a/internal/cli/identity.go b/internal/cli/identity.go index c1d99da..67bc21c 100644 --- a/internal/cli/identity.go +++ b/internal/cli/identity.go @@ -9,9 +9,9 @@ import ( "github.com/spf13/cobra" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/output" - "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/output" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" ) func newWhoAmICommand(state *rootState) *cobra.Command { diff --git a/internal/cli/identity_test.go b/internal/cli/identity_test.go index b8552b6..9170d3a 100644 --- a/internal/cli/identity_test.go +++ b/internal/cli/identity_test.go @@ -8,8 +8,8 @@ import ( "sync/atomic" "testing" - "github.com/ardasevinc/mattermost-cli/internal/output" - "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/output" + "github.com/ardasevinc/mattermost-cli/v2/internal/schema" ) func TestIdentityCommandsEmitStrictMachineSchemasAndHumanSemantics(t *testing.T) { diff --git a/internal/cli/mentions.go b/internal/cli/mentions.go index 4ef3db2..cfe32f8 100644 --- a/internal/cli/mentions.go +++ b/internal/cli/mentions.go @@ -6,9 +6,9 @@ import ( "github.com/spf13/cobra" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/output" - "github.com/ardasevinc/mattermost-cli/internal/retrieval" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/output" + "github.com/ardasevinc/mattermost-cli/v2/internal/retrieval" ) type mentionsFlags struct { diff --git a/internal/cli/mentions_test.go b/internal/cli/mentions_test.go index 12433b8..9375228 100644 --- a/internal/cli/mentions_test.go +++ b/internal/cli/mentions_test.go @@ -12,7 +12,7 @@ import ( "testing" "time" - mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" + mmSchema "github.com/ardasevinc/mattermost-cli/v2/internal/schema" ) func TestMentionsJSONUsesAliasesScopesGlobalSelectionAndHydration(t *testing.T) { diff --git a/internal/cli/read.go b/internal/cli/read.go index 0e2a4d6..4b90839 100644 --- a/internal/cli/read.go +++ b/internal/cli/read.go @@ -8,12 +8,12 @@ import ( "time" "unicode/utf16" - "github.com/ardasevinc/mattermost-cli/internal/api" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/normalization" - "github.com/ardasevinc/mattermost-cli/internal/output" - "github.com/ardasevinc/mattermost-cli/internal/presentation" - "github.com/ardasevinc/mattermost-cli/internal/retrieval" + "github.com/ardasevinc/mattermost-cli/v2/internal/api" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/normalization" + "github.com/ardasevinc/mattermost-cli/v2/internal/output" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/retrieval" "github.com/spf13/cobra" ) diff --git a/internal/cli/root.go b/internal/cli/root.go index ae5dd87..c8fb55f 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -10,10 +10,10 @@ import ( "github.com/spf13/cobra" - "github.com/ardasevinc/mattermost-cli/internal/buildinfo" - "github.com/ardasevinc/mattermost-cli/internal/output" - "github.com/ardasevinc/mattermost-cli/internal/presentation" - mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/buildinfo" + "github.com/ardasevinc/mattermost-cli/v2/internal/output" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" + mmSchema "github.com/ardasevinc/mattermost-cli/v2/internal/schema" ) type streams struct { diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index f8bdc9d..be0e4c0 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - "github.com/ardasevinc/mattermost-cli/internal/api" + "github.com/ardasevinc/mattermost-cli/v2/internal/api" ) func TestExecuteVersion(t *testing.T) { diff --git a/internal/cli/runtime.go b/internal/cli/runtime.go index 2010c76..4434a28 100644 --- a/internal/cli/runtime.go +++ b/internal/cli/runtime.go @@ -9,11 +9,11 @@ import ( "github.com/spf13/cobra" - "github.com/ardasevinc/mattermost-cli/internal/api" - "github.com/ardasevinc/mattermost-cli/internal/config" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/presentation" - "github.com/ardasevinc/mattermost-cli/internal/serverurl" + "github.com/ardasevinc/mattermost-cli/v2/internal/api" + "github.com/ardasevinc/mattermost-cli/v2/internal/config" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/serverurl" ) type clientFactory func(baseURL, token string) (*api.Client, error) diff --git a/internal/cli/runtime_test.go b/internal/cli/runtime_test.go index 1905b51..13928d8 100644 --- a/internal/cli/runtime_test.go +++ b/internal/cli/runtime_test.go @@ -11,10 +11,10 @@ import ( "github.com/spf13/cobra" - "github.com/ardasevinc/mattermost-cli/internal/api" - "github.com/ardasevinc/mattermost-cli/internal/config" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/api" + "github.com/ardasevinc/mattermost-cli/v2/internal/config" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" ) func TestRuntimeResolvesCLIEnvFilePrecedenceAndNormalizesURL(t *testing.T) { diff --git a/internal/cli/search.go b/internal/cli/search.go index 2f43119..dd0260a 100644 --- a/internal/cli/search.go +++ b/internal/cli/search.go @@ -6,9 +6,9 @@ import ( "github.com/spf13/cobra" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/output" - "github.com/ardasevinc/mattermost-cli/internal/retrieval" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/output" + "github.com/ardasevinc/mattermost-cli/v2/internal/retrieval" ) type searchFlags struct { diff --git a/internal/cli/search_test.go b/internal/cli/search_test.go index 71389b3..7eb81e6 100644 --- a/internal/cli/search_test.go +++ b/internal/cli/search_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" + mmSchema "github.com/ardasevinc/mattermost-cli/v2/internal/schema" ) func TestSearchJSONRunsSelectedTeamReadPipeline(t *testing.T) { diff --git a/internal/cli/stage_create.go b/internal/cli/stage_create.go index ae22417..a7dcf69 100644 --- a/internal/cli/stage_create.go +++ b/internal/cli/stage_create.go @@ -10,12 +10,12 @@ import ( "github.com/spf13/cobra" - "github.com/ardasevinc/mattermost-cli/internal/schema" - "github.com/ardasevinc/mattermost-cli/internal/stagecontent" - "github.com/ardasevinc/mattermost-cli/internal/stageoutput" - "github.com/ardasevinc/mattermost-cli/internal/stagerequest" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" - "github.com/ardasevinc/mattermost-cli/internal/staging" + "github.com/ardasevinc/mattermost-cli/v2/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagecontent" + "github.com/ardasevinc/mattermost-cli/v2/internal/stageoutput" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagerequest" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/staging" ) type stageCompositionFlags struct { diff --git a/internal/cli/stage_create_test.go b/internal/cli/stage_create_test.go index 8c44773..28bd77e 100644 --- a/internal/cli/stage_create_test.go +++ b/internal/cli/stage_create_test.go @@ -14,8 +14,8 @@ import ( "sync/atomic" "testing" - mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + mmSchema "github.com/ardasevinc/mattermost-cli/v2/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) type failOnRead struct{ reads atomic.Int32 } diff --git a/internal/cli/stage_inspect.go b/internal/cli/stage_inspect.go index 99f3c8b..740bfca 100644 --- a/internal/cli/stage_inspect.go +++ b/internal/cli/stage_inspect.go @@ -11,11 +11,11 @@ import ( "github.com/spf13/cobra" - "github.com/ardasevinc/mattermost-cli/internal/output" - "github.com/ardasevinc/mattermost-cli/internal/presentation" - "github.com/ardasevinc/mattermost-cli/internal/stagecursor" - "github.com/ardasevinc/mattermost-cli/internal/stageoutput" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/output" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagecursor" + "github.com/ardasevinc/mattermost-cli/v2/internal/stageoutput" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) type localStateFailure struct{ err error } diff --git a/internal/cli/stage_inspect_test.go b/internal/cli/stage_inspect_test.go index 416652f..c6ae2be 100644 --- a/internal/cli/stage_inspect_test.go +++ b/internal/cli/stage_inspect_test.go @@ -14,8 +14,8 @@ import ( "sync" "testing" - mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + mmSchema "github.com/ardasevinc/mattermost-cli/v2/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) func stageInspectCommand(t *testing.T, home, stateRoot string, jsonOutput bool, stdout, stderr *bytes.Buffer) (*rootState, *cobraCommandShim) { diff --git a/internal/cli/stage_manage.go b/internal/cli/stage_manage.go index 7b9ea02..e5f16df 100644 --- a/internal/cli/stage_manage.go +++ b/internal/cli/stage_manage.go @@ -13,13 +13,13 @@ import ( "github.com/spf13/cobra" - "github.com/ardasevinc/mattermost-cli/internal/schema" - "github.com/ardasevinc/mattermost-cli/internal/stagecontent" - "github.com/ardasevinc/mattermost-cli/internal/stageinput" - "github.com/ardasevinc/mattermost-cli/internal/stageoutput" - "github.com/ardasevinc/mattermost-cli/internal/stagerequest" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" - "github.com/ardasevinc/mattermost-cli/internal/staging" + "github.com/ardasevinc/mattermost-cli/v2/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagecontent" + "github.com/ardasevinc/mattermost-cli/v2/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/v2/internal/stageoutput" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagerequest" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/staging" ) func newStageManagementCommands(state *rootState, fromJSON *bool) []*cobra.Command { diff --git a/internal/cli/stage_manage_test.go b/internal/cli/stage_manage_test.go index 9aa7e06..e7dc489 100644 --- a/internal/cli/stage_manage_test.go +++ b/internal/cli/stage_manage_test.go @@ -12,8 +12,8 @@ import ( "testing" "time" - mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + mmSchema "github.com/ardasevinc/mattermost-cli/v2/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) func setOfflineStageEnvironment(t *testing.T, stateRoot string) { diff --git a/internal/cli/store.go b/internal/cli/store.go index 0575be7..77f8590 100644 --- a/internal/cli/store.go +++ b/internal/cli/store.go @@ -9,10 +9,10 @@ import ( "github.com/spf13/cobra" - "github.com/ardasevinc/mattermost-cli/internal/config" - "github.com/ardasevinc/mattermost-cli/internal/output" - "github.com/ardasevinc/mattermost-cli/internal/presentation" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/config" + "github.com/ardasevinc/mattermost-cli/v2/internal/output" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) type storeDoctorEnvelope struct { diff --git a/internal/cli/store_test.go b/internal/cli/store_test.go index 122d128..dbd524b 100644 --- a/internal/cli/store_test.go +++ b/internal/cli/store_test.go @@ -11,8 +11,8 @@ import ( "strings" "testing" - mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + mmSchema "github.com/ardasevinc/mattermost-cli/v2/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) func TestStoreDoctorAbsentIsReadOnlyAndSchemaValid(t *testing.T) { diff --git a/internal/cli/thread.go b/internal/cli/thread.go index 5652a78..5939e76 100644 --- a/internal/cli/thread.go +++ b/internal/cli/thread.go @@ -6,8 +6,8 @@ import ( "github.com/spf13/cobra" - "github.com/ardasevinc/mattermost-cli/internal/output" - "github.com/ardasevinc/mattermost-cli/internal/retrieval" + "github.com/ardasevinc/mattermost-cli/v2/internal/output" + "github.com/ardasevinc/mattermost-cli/v2/internal/retrieval" ) var safePostIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,128}$`) diff --git a/internal/cli/thread_test.go b/internal/cli/thread_test.go index fe0328f..944c228 100644 --- a/internal/cli/thread_test.go +++ b/internal/cli/thread_test.go @@ -11,10 +11,10 @@ import ( "testing" "time" - "github.com/ardasevinc/mattermost-cli/internal/api" - "github.com/ardasevinc/mattermost-cli/internal/config" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/api" + "github.com/ardasevinc/mattermost-cli/v2/internal/config" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + mmSchema "github.com/ardasevinc/mattermost-cli/v2/internal/schema" ) func TestThreadJSONRunsRootBoundReadPipeline(t *testing.T) { diff --git a/internal/cli/unread.go b/internal/cli/unread.go index 1840d73..4848104 100644 --- a/internal/cli/unread.go +++ b/internal/cli/unread.go @@ -8,9 +8,9 @@ import ( "github.com/spf13/cobra" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/output" - "github.com/ardasevinc/mattermost-cli/internal/retrieval" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/output" + "github.com/ardasevinc/mattermost-cli/v2/internal/retrieval" ) type unreadFlags struct { diff --git a/internal/cli/unread_test.go b/internal/cli/unread_test.go index bffc52a..40bce98 100644 --- a/internal/cli/unread_test.go +++ b/internal/cli/unread_test.go @@ -11,7 +11,7 @@ import ( "sync/atomic" "testing" - mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" + mmSchema "github.com/ardasevinc/mattermost-cli/v2/internal/schema" ) func TestUnreadValidatesFlagsBeforeNetwork(t *testing.T) { diff --git a/internal/cli/watch.go b/internal/cli/watch.go index 0b6904c..ea9c157 100644 --- a/internal/cli/watch.go +++ b/internal/cli/watch.go @@ -10,9 +10,9 @@ import ( "github.com/spf13/cobra" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/output" - "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/output" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" ) type watchFlags struct{ team, dm string } diff --git a/internal/cli/watch_test.go b/internal/cli/watch_test.go index 53f3226..654c920 100644 --- a/internal/cli/watch_test.go +++ b/internal/cli/watch_test.go @@ -11,9 +11,9 @@ import ( "testing" "time" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/presentation" - mmSchema "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" + mmSchema "github.com/ardasevinc/mattermost-cli/v2/internal/schema" ) func TestWatchSelectorsFailBeforeNetwork(t *testing.T) { diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index 859f078..bfeedc6 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -8,10 +8,10 @@ import ( "strings" "time" - "github.com/ardasevinc/mattermost-cli/internal/api" - "github.com/ardasevinc/mattermost-cli/internal/config" - "github.com/ardasevinc/mattermost-cli/internal/presentation" - "github.com/ardasevinc/mattermost-cli/internal/serverurl" + "github.com/ardasevinc/mattermost-cli/v2/internal/api" + "github.com/ardasevinc/mattermost-cli/v2/internal/config" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/serverurl" ) type Status string diff --git a/internal/doctor/doctor_test.go b/internal/doctor/doctor_test.go index 6f65c49..8f82be4 100644 --- a/internal/doctor/doctor_test.go +++ b/internal/doctor/doctor_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" - "github.com/ardasevinc/mattermost-cli/internal/api" - "github.com/ardasevinc/mattermost-cli/internal/config" + "github.com/ardasevinc/mattermost-cli/v2/internal/api" + "github.com/ardasevinc/mattermost-cli/v2/internal/config" ) type fakeCall struct { diff --git a/internal/mattermost/conversation_mutations.go b/internal/mattermost/conversation_mutations.go index b23807c..16bed6d 100644 --- a/internal/mattermost/conversation_mutations.go +++ b/internal/mattermost/conversation_mutations.go @@ -9,7 +9,7 @@ import ( "slices" "strings" - "github.com/ardasevinc/mattermost-cli/internal/api" + "github.com/ardasevinc/mattermost-cli/v2/internal/api" ) type ConversationMutations struct{ client *api.Client } diff --git a/internal/mattermost/conversation_mutations_test.go b/internal/mattermost/conversation_mutations_test.go index fe7557e..822554c 100644 --- a/internal/mattermost/conversation_mutations_test.go +++ b/internal/mattermost/conversation_mutations_test.go @@ -14,7 +14,7 @@ import ( "sync/atomic" "testing" - "github.com/ardasevinc/mattermost-cli/internal/api" + "github.com/ardasevinc/mattermost-cli/v2/internal/api" ) func conversationMutationJSON(id, channelType, name, displayName string) string { diff --git a/internal/mattermost/file_mutations.go b/internal/mattermost/file_mutations.go index c1a7cf5..4599c5f 100644 --- a/internal/mattermost/file_mutations.go +++ b/internal/mattermost/file_mutations.go @@ -10,7 +10,7 @@ import ( "strconv" "unicode/utf8" - "github.com/ardasevinc/mattermost-cli/internal/api" + "github.com/ardasevinc/mattermost-cli/v2/internal/api" ) var ErrUploadBinding = errors.New("Mattermost file no longer matches the validated upload") diff --git a/internal/mattermost/file_mutations_test.go b/internal/mattermost/file_mutations_test.go index 3a95275..233099a 100644 --- a/internal/mattermost/file_mutations_test.go +++ b/internal/mattermost/file_mutations_test.go @@ -10,7 +10,7 @@ import ( "sync/atomic" "testing" - "github.com/ardasevinc/mattermost-cli/internal/api" + "github.com/ardasevinc/mattermost-cli/v2/internal/api" ) func TestUploadMutationSendsExactRawFileAndValidatesIdentity(t *testing.T) { diff --git a/internal/mattermost/mutations.go b/internal/mattermost/mutations.go index 3a4a794..930b6cc 100644 --- a/internal/mattermost/mutations.go +++ b/internal/mattermost/mutations.go @@ -9,7 +9,7 @@ import ( "slices" "unicode/utf8" - "github.com/ardasevinc/mattermost-cli/internal/api" + "github.com/ardasevinc/mattermost-cli/v2/internal/api" ) const maxMutationMessageBytes = 65_535 diff --git a/internal/mattermost/mutations_test.go b/internal/mattermost/mutations_test.go index 8d74c7a..0f1cac9 100644 --- a/internal/mattermost/mutations_test.go +++ b/internal/mattermost/mutations_test.go @@ -11,7 +11,7 @@ import ( "sync/atomic" "testing" - "github.com/ardasevinc/mattermost-cli/internal/api" + "github.com/ardasevinc/mattermost-cli/v2/internal/api" ) type mutationRoundTripFunc func(*http.Request) (*http.Response, error) diff --git a/internal/mattermost/reaction_mutations.go b/internal/mattermost/reaction_mutations.go index 7aea6bc..c8c5c5f 100644 --- a/internal/mattermost/reaction_mutations.go +++ b/internal/mattermost/reaction_mutations.go @@ -6,7 +6,7 @@ import ( "net/url" "strings" - "github.com/ardasevinc/mattermost-cli/internal/api" + "github.com/ardasevinc/mattermost-cli/v2/internal/api" ) type ReactionMutationInput struct { diff --git a/internal/mattermost/watch.go b/internal/mattermost/watch.go index 25fd97f..ff42163 100644 --- a/internal/mattermost/watch.go +++ b/internal/mattermost/watch.go @@ -13,9 +13,9 @@ import ( "strings" "time" - "github.com/ardasevinc/mattermost-cli/internal/presentation" - "github.com/ardasevinc/mattermost-cli/internal/serverurl" - "github.com/ardasevinc/mattermost-cli/internal/transport" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/serverurl" + "github.com/ardasevinc/mattermost-cli/v2/internal/transport" ) const MaxWatchFrameBytes = 1 << 20 diff --git a/internal/mattermost/watch_test.go b/internal/mattermost/watch_test.go index 9032208..4201836 100644 --- a/internal/mattermost/watch_test.go +++ b/internal/mattermost/watch_test.go @@ -10,8 +10,8 @@ import ( "testing" "time" - "github.com/ardasevinc/mattermost-cli/internal/presentation" - "github.com/ardasevinc/mattermost-cli/internal/transport" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/transport" ) type recordingSink struct { diff --git a/internal/normalization/post.go b/internal/normalization/post.go index b3c5772..efac4d5 100644 --- a/internal/normalization/post.go +++ b/internal/normalization/post.go @@ -7,10 +7,10 @@ import ( "time" "unicode/utf16" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/output" - "github.com/ardasevinc/mattermost-cli/internal/presentation" - "github.com/ardasevinc/mattermost-cli/internal/serverurl" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/output" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/serverurl" ) const deletedPostText = "[deleted post]" diff --git a/internal/normalization/post_test.go b/internal/normalization/post_test.go index c63b7ec..ae3d8a1 100644 --- a/internal/normalization/post_test.go +++ b/internal/normalization/post_test.go @@ -7,8 +7,8 @@ import ( "testing" "time" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" ) const token = "ghp_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" diff --git a/internal/output/identity_machine.go b/internal/output/identity_machine.go index d236198..f902925 100644 --- a/internal/output/identity_machine.go +++ b/internal/output/identity_machine.go @@ -8,7 +8,7 @@ import ( "time" "unicode" - "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" ) const MaxSafeMachineInteger int64 = 9007199254740991 diff --git a/internal/output/identity_machine_test.go b/internal/output/identity_machine_test.go index 0681179..0766ca3 100644 --- a/internal/output/identity_machine_test.go +++ b/internal/output/identity_machine_test.go @@ -6,9 +6,9 @@ import ( "strings" "testing" - "github.com/ardasevinc/mattermost-cli/internal/output" - "github.com/ardasevinc/mattermost-cli/internal/presentation" - "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/output" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/schema" ) func TestIdentityDocumentsGoldenSchemaAndCredentialPresentation(t *testing.T) { diff --git a/internal/output/machine_convert_test.go b/internal/output/machine_convert_test.go index c4b7f73..a7d2957 100644 --- a/internal/output/machine_convert_test.go +++ b/internal/output/machine_convert_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" - "github.com/ardasevinc/mattermost-cli/internal/output" - "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/output" + "github.com/ardasevinc/mattermost-cli/v2/internal/schema" ) func TestMachineMessageFromMessagePreservesRichRecursiveDataWithoutAliasing(t *testing.T) { diff --git a/internal/output/model.go b/internal/output/model.go index cb95328..f87640c 100644 --- a/internal/output/model.go +++ b/internal/output/model.go @@ -3,7 +3,7 @@ package output import ( "time" - "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" ) // Redaction is presentation-owned because its Position is a UTF-16 code-unit diff --git a/internal/output/unread.go b/internal/output/unread.go index 6b4b41a..f122533 100644 --- a/internal/output/unread.go +++ b/internal/output/unread.go @@ -7,7 +7,7 @@ import ( "sort" "strings" - "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" ) type RawUnreadItem struct { diff --git a/internal/output/unread_test.go b/internal/output/unread_test.go index 999fe0b..49122c1 100644 --- a/internal/output/unread_test.go +++ b/internal/output/unread_test.go @@ -6,9 +6,9 @@ import ( "testing" "time" - "github.com/ardasevinc/mattermost-cli/internal/output" - "github.com/ardasevinc/mattermost-cli/internal/presentation" - "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/output" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/schema" ) func TestUnreadMachineGoldenSchemaOrderingAndActiveCredential(t *testing.T) { diff --git a/internal/output/watch.go b/internal/output/watch.go index 2c33a1e..bfee5bd 100644 --- a/internal/output/watch.go +++ b/internal/output/watch.go @@ -9,8 +9,8 @@ import ( "strings" "time" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" ) const MaxWatchLineBytes = 1 << 20 diff --git a/internal/output/watch_test.go b/internal/output/watch_test.go index ee32958..2c215e7 100644 --- a/internal/output/watch_test.go +++ b/internal/output/watch_test.go @@ -4,8 +4,8 @@ import ( "bytes" "encoding/json" "errors" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/presentation" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/presentation" "io" "strings" "testing" diff --git a/internal/retrieval/channel.go b/internal/retrieval/channel.go index b25f10f..a45491d 100644 --- a/internal/retrieval/channel.go +++ b/internal/retrieval/channel.go @@ -7,7 +7,7 @@ import ( "sort" "strings" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" ) var ErrInvalidChannelHistoryRequest = errors.New("invalid channel history request") diff --git a/internal/retrieval/channel_test.go b/internal/retrieval/channel_test.go index 45817bd..30fba71 100644 --- a/internal/retrieval/channel_test.go +++ b/internal/retrieval/channel_test.go @@ -6,7 +6,7 @@ import ( "fmt" "testing" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" ) type pageSourceFunc func(context.Context, string, mattermost.ChannelPostsOptions) (mattermost.OrderedPostsPage, error) diff --git a/internal/retrieval/dms.go b/internal/retrieval/dms.go index c8cbbb6..779f001 100644 --- a/internal/retrieval/dms.go +++ b/internal/retrieval/dms.go @@ -5,7 +5,7 @@ import ( "errors" "strings" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" ) var ErrInvalidDMHistoryRequest = errors.New("invalid direct-message history request") diff --git a/internal/retrieval/dms_test.go b/internal/retrieval/dms_test.go index 878a492..96abc11 100644 --- a/internal/retrieval/dms_test.go +++ b/internal/retrieval/dms_test.go @@ -6,7 +6,7 @@ import ( "fmt" "testing" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" ) func TestDMHistoryAppliesOneGlobalDeterministicLimit(t *testing.T) { diff --git a/internal/retrieval/group_dms_test.go b/internal/retrieval/group_dms_test.go index 854c1d5..7e8fba3 100644 --- a/internal/retrieval/group_dms_test.go +++ b/internal/retrieval/group_dms_test.go @@ -5,7 +5,7 @@ import ( "fmt" "testing" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" ) func TestGroupDMHistoryUsesGlobalDeterministicCapAndUnknownDominance(t *testing.T) { diff --git a/internal/retrieval/hydration.go b/internal/retrieval/hydration.go index 80c1e5b..4ec39f9 100644 --- a/internal/retrieval/hydration.go +++ b/internal/retrieval/hydration.go @@ -3,7 +3,7 @@ package retrieval import ( "context" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" ) type VisibleThreadsStatus uint8 diff --git a/internal/retrieval/mentions.go b/internal/retrieval/mentions.go index c48a6bc..298d38f 100644 --- a/internal/retrieval/mentions.go +++ b/internal/retrieval/mentions.go @@ -9,7 +9,7 @@ import ( "unicode/utf16" "unicode/utf8" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" "golang.org/x/text/cases" "golang.org/x/text/language" ) diff --git a/internal/retrieval/mentions_test.go b/internal/retrieval/mentions_test.go index df077e3..e1070d8 100644 --- a/internal/retrieval/mentions_test.go +++ b/internal/retrieval/mentions_test.go @@ -7,7 +7,7 @@ import ( "reflect" "testing" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" ) func TestMentionTermsTrimQuoteDedupeAndScope(t *testing.T) { diff --git a/internal/retrieval/search.go b/internal/retrieval/search.go index 7cffbbc..1ce3667 100644 --- a/internal/retrieval/search.go +++ b/internal/retrieval/search.go @@ -5,7 +5,7 @@ import ( "errors" "strings" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" ) const MaxSearchPages = 100 diff --git a/internal/retrieval/search_test.go b/internal/retrieval/search_test.go index 52bd112..04c8066 100644 --- a/internal/retrieval/search_test.go +++ b/internal/retrieval/search_test.go @@ -6,7 +6,7 @@ import ( "fmt" "testing" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" ) type searchSourceFunc func(context.Context, string, mattermost.SearchPageOptions) (mattermost.SearchPage, error) diff --git a/internal/retrieval/thread.go b/internal/retrieval/thread.go index adcd616..9c14fa9 100644 --- a/internal/retrieval/thread.go +++ b/internal/retrieval/thread.go @@ -5,7 +5,7 @@ import ( "errors" "strings" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" ) const MaxThreadPages = 100 diff --git a/internal/retrieval/thread_test.go b/internal/retrieval/thread_test.go index 27a5f34..f374082 100644 --- a/internal/retrieval/thread_test.go +++ b/internal/retrieval/thread_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" ) type threadSourceFunc func(context.Context, string, mattermost.ThreadPageOptions) (mattermost.OrderedPostsPage, error) diff --git a/internal/retrieval/unread.go b/internal/retrieval/unread.go index 72267f7..1040d6d 100644 --- a/internal/retrieval/unread.go +++ b/internal/retrieval/unread.go @@ -7,7 +7,7 @@ import ( "strings" "sync" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" ) const ( diff --git a/internal/retrieval/unread_test.go b/internal/retrieval/unread_test.go index bfb815d..d97e101 100644 --- a/internal/retrieval/unread_test.go +++ b/internal/retrieval/unread_test.go @@ -8,7 +8,7 @@ import ( "sync/atomic" "testing" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" ) type unreadUsersFake struct { diff --git a/internal/schema/apply_test.go b/internal/schema/apply_test.go index 6f609c5..7551b6a 100644 --- a/internal/schema/apply_test.go +++ b/internal/schema/apply_test.go @@ -6,7 +6,7 @@ import ( "io/fs" "testing" - publicschemas "github.com/ardasevinc/mattermost-cli/schemas" + publicschemas "github.com/ardasevinc/mattermost-cli/v2/schemas" ) func TestApplySchemasAndExamples(t *testing.T) { diff --git a/internal/schema/read_test.go b/internal/schema/read_test.go index eaf8301..7a83729 100644 --- a/internal/schema/read_test.go +++ b/internal/schema/read_test.go @@ -8,7 +8,7 @@ import ( jsonschema "github.com/santhosh-tekuri/jsonschema/v6" - publicschemas "github.com/ardasevinc/mattermost-cli/schemas" + publicschemas "github.com/ardasevinc/mattermost-cli/v2/schemas" ) func TestReadSchemasAreRegisteredAndStrict(t *testing.T) { diff --git a/internal/schema/registry.go b/internal/schema/registry.go index 3cd3d24..378ceed 100644 --- a/internal/schema/registry.go +++ b/internal/schema/registry.go @@ -13,7 +13,7 @@ import ( jsonschema "github.com/santhosh-tekuri/jsonschema/v6" - publicschemas "github.com/ardasevinc/mattermost-cli/schemas" + publicschemas "github.com/ardasevinc/mattermost-cli/v2/schemas" ) const maxDocumentBytes = 4 << 20 diff --git a/internal/schema/registry_test.go b/internal/schema/registry_test.go index e4a7e11..7244946 100644 --- a/internal/schema/registry_test.go +++ b/internal/schema/registry_test.go @@ -9,7 +9,7 @@ import ( "strings" "testing" - publicschemas "github.com/ardasevinc/mattermost-cli/schemas" + publicschemas "github.com/ardasevinc/mattermost-cli/v2/schemas" ) func TestReadAndValidateReturnsExactDocument(t *testing.T) { diff --git a/internal/schema/stage_test.go b/internal/schema/stage_test.go index ba9564b..4ccca91 100644 --- a/internal/schema/stage_test.go +++ b/internal/schema/stage_test.go @@ -9,7 +9,7 @@ import ( "strings" "testing" - publicschemas "github.com/ardasevinc/mattermost-cli/schemas" + publicschemas "github.com/ardasevinc/mattermost-cli/v2/schemas" ) // These tests deliberately stop at document-local invariants. The runtime must diff --git a/internal/schema/unread_test.go b/internal/schema/unread_test.go index 820b8b8..f589063 100644 --- a/internal/schema/unread_test.go +++ b/internal/schema/unread_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - publicschemas "github.com/ardasevinc/mattermost-cli/schemas" + publicschemas "github.com/ardasevinc/mattermost-cli/v2/schemas" ) func TestUnreadSchemaRequiresNonNullArraysAndHonestHistory(t *testing.T) { diff --git a/internal/stagecontent/content.go b/internal/stagecontent/content.go index 735a2c4..77bfb1b 100644 --- a/internal/stagecontent/content.go +++ b/internal/stagecontent/content.go @@ -12,7 +12,7 @@ import ( "path/filepath" "strings" - "github.com/ardasevinc/mattermost-cli/internal/messageinput" + "github.com/ardasevinc/mattermost-cli/v2/internal/messageinput" ) var ( diff --git a/internal/stagecontent/content_test.go b/internal/stagecontent/content_test.go index f7dbe81..472f8a3 100644 --- a/internal/stagecontent/content_test.go +++ b/internal/stagecontent/content_test.go @@ -11,7 +11,7 @@ import ( "strings" "testing" - "github.com/ardasevinc/mattermost-cli/internal/messageinput" + "github.com/ardasevinc/mattermost-cli/v2/internal/messageinput" ) func TestAcquireExplicitMessagePreservesBytes(t *testing.T) { diff --git a/internal/stageinput/input.go b/internal/stageinput/input.go index 034f0e6..00068e7 100644 --- a/internal/stageinput/input.go +++ b/internal/stageinput/input.go @@ -15,7 +15,7 @@ import ( "unicode" "unicode/utf8" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" "golang.org/x/text/unicode/norm" ) diff --git a/internal/stageinput/input_test.go b/internal/stageinput/input_test.go index 5a7e1c6..7b2cc5e 100644 --- a/internal/stageinput/input_test.go +++ b/internal/stageinput/input_test.go @@ -13,7 +13,7 @@ import ( "testing" "time" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) func TestTokenScannerEveryChunkBoundary(t *testing.T) { diff --git a/internal/stageinput/spool.go b/internal/stageinput/spool.go index 02ef924..3e16bc2 100644 --- a/internal/stageinput/spool.go +++ b/internal/stageinput/spool.go @@ -8,7 +8,7 @@ import ( "os" "path/filepath" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) // Spool is an unlinked private snapshot. It survives only while its descriptor diff --git a/internal/stageoutput/output.go b/internal/stageoutput/output.go index a59be4f..2f82c86 100644 --- a/internal/stageoutput/output.go +++ b/internal/stageoutput/output.go @@ -16,10 +16,10 @@ import ( "sync" "time" - "github.com/ardasevinc/mattermost-cli/internal/schema" - "github.com/ardasevinc/mattermost-cli/internal/stagecursor" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" - "github.com/ardasevinc/mattermost-cli/internal/staging" + "github.com/ardasevinc/mattermost-cli/v2/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagecursor" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/staging" ) var ErrInvalid = errors.New("stage output: invalid or unsafe state") diff --git a/internal/stageoutput/output_test.go b/internal/stageoutput/output_test.go index d3228f8..9c3c928 100644 --- a/internal/stageoutput/output_test.go +++ b/internal/stageoutput/output_test.go @@ -6,9 +6,9 @@ import ( "testing" "time" - "github.com/ardasevinc/mattermost-cli/internal/stagecursor" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" - "github.com/ardasevinc/mattermost-cli/internal/staging" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagecursor" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/staging" ) func TestConstructorsProduceStrictDocumentsAndSealInputs(t *testing.T) { diff --git a/internal/stagerequest/request.go b/internal/stagerequest/request.go index 86a106b..5a6f4a5 100644 --- a/internal/stagerequest/request.go +++ b/internal/stagerequest/request.go @@ -16,10 +16,10 @@ import ( "sync" "unicode/utf8" - "github.com/ardasevinc/mattermost-cli/internal/schema" - "github.com/ardasevinc/mattermost-cli/internal/stageinput" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" - "github.com/ardasevinc/mattermost-cli/internal/staging" + "github.com/ardasevinc/mattermost-cli/v2/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/staging" ) const ( diff --git a/internal/stagerequest/request_test.go b/internal/stagerequest/request_test.go index 2d5ed80..e8081a1 100644 --- a/internal/stagerequest/request_test.go +++ b/internal/stagerequest/request_test.go @@ -7,9 +7,9 @@ import ( "strings" "testing" - "github.com/ardasevinc/mattermost-cli/internal/schema" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" - "github.com/ardasevinc/mattermost-cli/internal/staging" + "github.com/ardasevinc/mattermost-cli/v2/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/staging" ) func decoder(t *testing.T) *Decoder { diff --git a/internal/stagestore/domain.go b/internal/stagestore/domain.go index 7462ebe..b83b1c3 100644 --- a/internal/stagestore/domain.go +++ b/internal/stagestore/domain.go @@ -16,9 +16,9 @@ import ( "unicode" "unicode/utf8" - "github.com/ardasevinc/mattermost-cli/internal/messageinput" - "github.com/ardasevinc/mattermost-cli/internal/serverurl" - "github.com/ardasevinc/mattermost-cli/internal/stagecursor" + "github.com/ardasevinc/mattermost-cli/v2/internal/messageinput" + "github.com/ardasevinc/mattermost-cli/v2/internal/serverurl" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagecursor" ) const ( diff --git a/internal/stagestore/domain_test.go b/internal/stagestore/domain_test.go index 2746763..d7f968f 100644 --- a/internal/stagestore/domain_test.go +++ b/internal/stagestore/domain_test.go @@ -13,7 +13,7 @@ import ( "testing" "time" - "github.com/ardasevinc/mattermost-cli/internal/stagecursor" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagecursor" ) func openDomainStore(t *testing.T) *Store { diff --git a/internal/staging/conversation.go b/internal/staging/conversation.go index b118df2..651dff2 100644 --- a/internal/staging/conversation.go +++ b/internal/staging/conversation.go @@ -4,7 +4,7 @@ import ( "context" "strings" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" ) func validTargetSyntax(target Target) bool { diff --git a/internal/staging/conversation_stage.go b/internal/staging/conversation_stage.go index 4600f08..b871b72 100644 --- a/internal/staging/conversation_stage.go +++ b/internal/staging/conversation_stage.go @@ -5,8 +5,8 @@ import ( "sort" "strings" - "github.com/ardasevinc/mattermost-cli/internal/stageinput" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) type resolveDMIntent struct { diff --git a/internal/staging/conversation_stage_test.go b/internal/staging/conversation_stage_test.go index 6877b9e..55dd4ec 100644 --- a/internal/staging/conversation_stage_test.go +++ b/internal/staging/conversation_stage_test.go @@ -11,8 +11,8 @@ import ( "sync/atomic" "testing" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) type directoryUsers struct { diff --git a/internal/staging/intent.go b/internal/staging/intent.go index a44d7f0..fcb1c20 100644 --- a/internal/staging/intent.go +++ b/internal/staging/intent.go @@ -5,8 +5,8 @@ import ( "crypto/sha256" "encoding/json" - "github.com/ardasevinc/mattermost-cli/internal/stageinput" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) type callerIntent struct { diff --git a/internal/staging/intent_test.go b/internal/staging/intent_test.go index b997372..5d7e249 100644 --- a/internal/staging/intent_test.go +++ b/internal/staging/intent_test.go @@ -4,8 +4,8 @@ import ( "encoding/hex" "testing" - "github.com/ardasevinc/mattermost-cli/internal/stageinput" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) func TestCallerIntentDigestGoldenAndEscapeAwareness(t *testing.T) { diff --git a/internal/staging/post.go b/internal/staging/post.go index 1403890..a1a29cc 100644 --- a/internal/staging/post.go +++ b/internal/staging/post.go @@ -9,10 +9,10 @@ import ( "errors" "strings" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/messageinput" - "github.com/ardasevinc/mattermost-cli/internal/stageinput" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/messageinput" + "github.com/ardasevinc/mattermost-cli/v2/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) type postOperation struct { diff --git a/internal/staging/post_test.go b/internal/staging/post_test.go index f071fa3..45103a5 100644 --- a/internal/staging/post_test.go +++ b/internal/staging/post_test.go @@ -13,9 +13,9 @@ import ( "strings" "testing" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/schema" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) type fakePosts struct { diff --git a/internal/staging/revision.go b/internal/staging/revision.go index 7067e95..5404ea7 100644 --- a/internal/staging/revision.go +++ b/internal/staging/revision.go @@ -7,9 +7,9 @@ import ( "encoding/json" "errors" - "github.com/ardasevinc/mattermost-cli/internal/messageinput" - "github.com/ardasevinc/mattermost-cli/internal/stageinput" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/messageinput" + "github.com/ardasevinc/mattermost-cli/v2/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) var ErrNotEligible = errors.New("staging: lifecycle transition not allowed") diff --git a/internal/staging/revision_test.go b/internal/staging/revision_test.go index 5fc5346..c07950b 100644 --- a/internal/staging/revision_test.go +++ b/internal/staging/revision_test.go @@ -8,8 +8,8 @@ import ( "strings" "testing" - "github.com/ardasevinc/mattermost-cli/internal/stageinput" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) type revisionStoreStub struct { diff --git a/internal/staging/service.go b/internal/staging/service.go index 88ac56b..06d4fbd 100644 --- a/internal/staging/service.go +++ b/internal/staging/service.go @@ -10,11 +10,11 @@ import ( "io" "reflect" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/messageinput" - "github.com/ardasevinc/mattermost-cli/internal/serverurl" - "github.com/ardasevinc/mattermost-cli/internal/stageinput" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/messageinput" + "github.com/ardasevinc/mattermost-cli/v2/internal/serverurl" + "github.com/ardasevinc/mattermost-cli/v2/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) var ( diff --git a/internal/staging/service_test.go b/internal/staging/service_test.go index 9faea80..a6fe638 100644 --- a/internal/staging/service_test.go +++ b/internal/staging/service_test.go @@ -14,9 +14,9 @@ import ( "sync/atomic" "testing" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/schema" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) type fakeUsers struct { diff --git a/internal/staging/types.go b/internal/staging/types.go index bf81d97..670f385 100644 --- a/internal/staging/types.go +++ b/internal/staging/types.go @@ -7,9 +7,9 @@ import ( "errors" "io" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/stageinput" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) type ConversationType uint8 diff --git a/internal/staging/validation.go b/internal/staging/validation.go index 250c47d..a66eff9 100644 --- a/internal/staging/validation.go +++ b/internal/staging/validation.go @@ -7,8 +7,8 @@ import ( "unicode" "unicode/utf8" - "github.com/ardasevinc/mattermost-cli/internal/mattermost" - "github.com/ardasevinc/mattermost-cli/internal/stagestore" + "github.com/ardasevinc/mattermost-cli/v2/internal/mattermost" + "github.com/ardasevinc/mattermost-cli/v2/internal/stagestore" ) func targetStrings(t Target) []string { diff --git a/scripts/release/main.go b/scripts/release/main.go index d791803..8e4a391 100644 --- a/scripts/release/main.go +++ b/scripts/release/main.go @@ -64,7 +64,7 @@ func run(version, commit, output string) error { if err := os.MkdirAll(filepath.Dir(binary), 0o755); err != nil { // #nosec G301 -- temporary release inputs contain no secrets. return err } - ldflags := fmt.Sprintf("-s -w -buildid= -X github.com/ardasevinc/mattermost-cli/internal/buildinfo.Version=%s -X github.com/ardasevinc/mattermost-cli/internal/buildinfo.Commit=%s", version, commit) + ldflags := fmt.Sprintf("-s -w -buildid= -X github.com/ardasevinc/mattermost-cli/v2/internal/buildinfo.Version=%s -X github.com/ardasevinc/mattermost-cli/v2/internal/buildinfo.Commit=%s", version, commit) command := exec.Command("go", "build", "-trimpath", "-buildvcs=false", "-ldflags", ldflags, "-o", binary, "./cmd/mm") // #nosec G204 -- values are validated or from the closed target list. command.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS="+target.os, "GOARCH="+target.arch) command.Stdout, command.Stderr = os.Stdout, os.Stderr diff --git a/tests/e2e/go-lifecycle-live_test.go b/tests/e2e/go-lifecycle-live_test.go index 6d738c9..00b0a3a 100644 --- a/tests/e2e/go-lifecycle-live_test.go +++ b/tests/e2e/go-lifecycle-live_test.go @@ -12,7 +12,7 @@ import ( "slices" "testing" - "github.com/ardasevinc/mattermost-cli/internal/stageinput" + "github.com/ardasevinc/mattermost-cli/v2/internal/stageinput" ) func TestGoApplyFullPostLifecycleWithAttachment(t *testing.T) { diff --git a/tests/e2e/go-read-live_test.go b/tests/e2e/go-read-live_test.go index ad0eda7..870211b 100644 --- a/tests/e2e/go-read-live_test.go +++ b/tests/e2e/go-read-live_test.go @@ -8,8 +8,8 @@ import ( "net/http" "testing" - "github.com/ardasevinc/mattermost-cli/internal/output" - "github.com/ardasevinc/mattermost-cli/internal/schema" + "github.com/ardasevinc/mattermost-cli/v2/internal/output" + "github.com/ardasevinc/mattermost-cli/v2/internal/schema" ) func TestGoRealServerChannelCursorSearchAndThreadReads(t *testing.T) { From ac440d80b922ce083988e49ca4c7ee494e3d2ca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 12:50:58 +0300 Subject: [PATCH 109/119] fix: stop single conformance runs after success --- cmd/conformance/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/conformance/main.go b/cmd/conformance/main.go index 23243ba..4cc99a1 100644 --- a/cmd/conformance/main.go +++ b/cmd/conformance/main.go @@ -54,6 +54,7 @@ func main() { }, scenario) } finish(scenario.Name, err) + return } if len(command) != 0 || *oraclePath == "" || *candidatePath == "" { _, _ = fmt.Fprintln(os.Stderr, "usage: conformance --pair FILE [--cwd DIR] --oracle PATH [--oracle-prefix ARG...] --candidate PATH [--candidate-prefix ARG...]") From efd432a2f7d6a43bc88accfb81b81cc149657c5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 13:04:30 +0300 Subject: [PATCH 110/119] ci: enforce Go static and dependency audits --- .github/workflows/ci.yml | 3 + .github/workflows/release.yml | 7 +- internal/cli/root.go | 4 - internal/mattermost/channels.go | 11 -- internal/mattermost/post_presentation_test.go | 3 +- internal/output/machine_convert.go | 2 +- internal/stagestore/domain.go | 23 --- justfile | 11 +- scripts/licenses/main.go | 173 ++++++++++++++++++ scripts/licenses/main_test.go | 60 ++++++ staticcheck.conf | 1 + 11 files changed, 255 insertions(+), 43 deletions(-) create mode 100644 scripts/licenses/main.go create mode 100644 scripts/licenses/main_test.go create mode 100644 staticcheck.conf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab22d67..d166057 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,9 @@ jobs: - run: go test ./... - run: go test -race ./... - run: go vet ./... + - run: go run honnef.co/go/tools/cmd/staticcheck@2026.1 ./... + - run: go run golang.org/x/vuln/cmd/govulncheck@v1.6.0 ./... + - run: go run ./scripts/licenses - run: go mod verify - run: go build -o "$RUNNER_TEMP/mm" ./cmd/mm - run: git diff --check diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2d748cb..aa1ff35 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,10 +61,13 @@ jobs: - name: Test release source run: | go mod download + go install honnef.co/go/tools/cmd/staticcheck@2026.1 + go install golang.org/x/vuln/cmd/govulncheck@v1.6.0 GOPROXY=off go test -race -p 1 ./... GOPROXY=off go vet ./... - go install golang.org/x/vuln/cmd/govulncheck@v1.6.0 - govulncheck ./... + GOPROXY=off staticcheck ./... + GOPROXY=off govulncheck ./... + GOPROXY=off go run ./scripts/licenses - name: Build deterministic archives twice env: GOPROXY: "off" diff --git a/internal/cli/root.go b/internal/cli/root.go index c8fb55f..37d21b9 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -165,10 +165,6 @@ func machineStageRef(err error) string { return "" } -func newRoot(s streams) *cobra.Command { - return newRootWithState(&rootState{streams: s, deps: defaultDependencies(s.out)}) -} - func newRootWithState(state *rootState) *cobra.Command { s := state.streams cmd := &cobra.Command{ diff --git a/internal/mattermost/channels.go b/internal/mattermost/channels.go index 38ac111..d0beb55 100644 --- a/internal/mattermost/channels.go +++ b/internal/mattermost/channels.go @@ -180,17 +180,6 @@ type Channels struct{ client channelTransport } func NewChannels(client channelTransport) *Channels { return &Channels{client: client} } -type channelList []Channel - -func (l *channelList) UnmarshalJSON(data []byte) error { - var channels []Channel - if err := json.Unmarshal(data, &channels); err != nil || channels == nil { - return ErrInvalidChannelsResponse - } - *l = channels - return nil -} - type selectedChannelList struct { wanted map[string]bool channels []Channel diff --git a/internal/mattermost/post_presentation_test.go b/internal/mattermost/post_presentation_test.go index 051a9f6..6382baf 100644 --- a/internal/mattermost/post_presentation_test.go +++ b/internal/mattermost/post_presentation_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "errors" + "math" "reflect" "testing" ) @@ -117,7 +118,7 @@ func TestECMAScriptNumberString(t *testing.T) { {1e-6, "0.000001"}, {1e20, "100000000000000000000"}, {1e-7, "1e-7"}, - {-0.0, "0"}, + {math.Copysign(0, -1), "0"}, {1e21, "1e+21"}, {-1e-7, "-1e-7"}, } { diff --git a/internal/output/machine_convert.go b/internal/output/machine_convert.go index 8a84aa6..8eb9f57 100644 --- a/internal/output/machine_convert.go +++ b/internal/output/machine_convert.go @@ -296,7 +296,7 @@ func machineChannel(channel Channel) (MachineChannel, error) { if (channel.Type == "unknown") != (channel.MetadataStatus == "unavailable") { return MachineChannel{}, fmt.Errorf("machine channel type %q and metadata status %q are inconsistent", channel.Type, channel.MetadataStatus) } - return MachineChannel{ID: channel.ID, Type: channel.Type, Name: channel.Name, DisplayName: channel.DisplayName, MetadataStatus: channel.MetadataStatus}, nil + return MachineChannel(channel), nil } func validMachineCompleteness(value MachineCompleteness) bool { diff --git a/internal/stagestore/domain.go b/internal/stagestore/domain.go index b83b1c3..03e71e8 100644 --- a/internal/stagestore/domain.go +++ b/internal/stagestore/domain.go @@ -472,29 +472,6 @@ const currentDetailSQL = `SELECT s.id,s.server_url,coalesce(s.server_id,''),s.us type rowScanner interface{ Scan(...any) error } -func scanSummary(row rowScanner) (StageSummary, error) { - var v StageSummary - var digest []byte - var created, updated string - err := row.Scan(&v.ID, &v.ServerURL, &v.ServerID, &v.UserID, &v.Operation, &v.Lifecycle, &v.Recovery, &v.Revision, &digest, &created, &updated) - if errors.Is(err, sql.ErrNoRows) { - return v, ErrNotFound - } - if err != nil { - return v, localError(err) - } - if len(digest) != 32 { - return v, localError(errors.New("digest")) - } - copy(v.SemanticDigest[:], digest) - if v.CreatedAt, err = parseTime(created); err != nil { - return v, err - } - if v.UpdatedAt, err = parseTime(updated); err != nil { - return v, err - } - return v, nil -} func scanDetail(row rowScanner) (StageDetail, error) { var v StageDetail var digest, body []byte diff --git a/justfile b/justfile index 8fb0d4d..7bda440 100644 --- a/justfile +++ b/justfile @@ -15,6 +15,15 @@ go-race: go-vet: go vet ./... +go-staticcheck: + go run honnef.co/go/tools/cmd/staticcheck@2026.1 ./... + +go-vuln: + go run golang.org/x/vuln/cmd/govulncheck@v1.6.0 ./... + +go-licenses: + go run ./scripts/licenses + go-modules: go mod verify @@ -44,7 +53,7 @@ oracle-smoke: parity-smoke: @tmp="$(mktemp -d "${TMPDIR:-/tmp}/mattermost-cli-parity.XXXXXX")"; trap 'find "$tmp" -type f -delete; rmdir "$tmp"' EXIT; go build -o "$tmp/mm" ./cmd/mm; for scenario in conformance/scenarios/pairs/*.json; do go run ./cmd/conformance --pair "$scenario" --cwd . --oracle bun --oracle-prefix src/index.ts --candidate "$tmp/mm"; done -go-gate: go-format-check go-test go-race go-vet go-modules go-build go-e2e-compile go-cross-build +go-gate: go-format-check go-test go-race go-vet go-staticcheck go-vuln go-licenses go-modules go-build go-e2e-compile go-cross-build git diff --check legacy-gate: diff --git a/scripts/licenses/main.go b/scripts/licenses/main.go new file mode 100644 index 0000000..22c0da9 --- /dev/null +++ b/scripts/licenses/main.go @@ -0,0 +1,173 @@ +// Command licenses enforces the reviewed license set for every module used by +// production code or tests. A new, removed, replaced, or relicensed module +// requires an explicit review and allowlist change. +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" +) + +type approval struct { + license string + marker string +} + +var approved = map[string]approval{ + "github.com/coder/websocket": {"ISC", "Permission to use, copy, modify, and distribute this software"}, + "github.com/dustin/go-humanize": {"MIT", "Permission is hereby granted, free of charge"}, + "github.com/google/uuid": {"BSD-3-Clause", "Redistribution and use in source and binary forms"}, + "github.com/mattn/go-isatty": {"MIT", "Permission is hereby granted, free of charge"}, + "github.com/ncruces/go-strftime": {"MIT", "Permission is hereby granted, free of charge"}, + "github.com/pelletier/go-toml/v2": {"MIT", "Permission is hereby granted, free of charge"}, + "github.com/remyoudompheng/bigfft": {"BSD-3-Clause", "Redistribution and use in source and binary forms"}, + "github.com/santhosh-tekuri/jsonschema/v6": {"Apache-2.0", "Apache License"}, + "github.com/spf13/cobra": {"Apache-2.0", "Apache License"}, + "github.com/spf13/pflag": {"BSD-3-Clause", "Redistribution and use in source and binary forms"}, + "golang.org/x/net": {"BSD-3-Clause", "Redistribution and use in source and binary forms"}, + "golang.org/x/sys": {"BSD-3-Clause", "Redistribution and use in source and binary forms"}, + "golang.org/x/text": {"BSD-3-Clause", "Redistribution and use in source and binary forms"}, + "modernc.org/libc": {"BSD-3-Clause", "Redistribution and use in source and binary forms"}, + "modernc.org/mathutil": {"BSD-3-Clause", "Redistribution and use in source and binary forms"}, + "modernc.org/memory": {"BSD-3-Clause", "Redistribution and use in source and binary forms"}, + "modernc.org/sqlite": {"BSD-3-Clause", "Redistribution and use in source and binary forms"}, +} + +type module struct { + Path string + Dir string + Main bool + Replace *module +} + +type listedPackage struct { + Module *module +} + +func main() { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + cmd := exec.CommandContext(ctx, "go", "list", "-deps", "-test", "-json", "./...") + cmd.Stderr = os.Stderr + out, err := cmd.StdoutPipe() + if err != nil { + fatal(err) + } + if err = cmd.Start(); err != nil { + fatal(err) + } + modules, decodeErr := decodeModules(out) + waitErr := cmd.Wait() + if decodeErr != nil { + fatal(decodeErr) + } + if waitErr != nil { + fatal(waitErr) + } + if err = verify(modules, approved); err != nil { + fatal(err) + } + fmt.Printf("license check passed: %d reviewed modules\n", len(modules)) +} + +func decodeModules(r io.Reader) (map[string]module, error) { + modules := make(map[string]module) + decoder := json.NewDecoder(r) + for { + var pkg listedPackage + err := decoder.Decode(&pkg) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("decode go list output: %w", err) + } + if pkg.Module == nil || pkg.Module.Main { + continue + } + if pkg.Module.Path == "" || pkg.Module.Dir == "" { + return nil, errors.New("dependency module is missing its path or directory") + } + modules[pkg.Module.Path] = *pkg.Module + } + return modules, nil +} + +func verify(modules map[string]module, allow map[string]approval) error { + paths := make([]string, 0, len(modules)) + for path := range modules { + paths = append(paths, path) + } + sort.Strings(paths) + for _, path := range paths { + mod := modules[path] + approval, ok := allow[path] + if !ok { + return fmt.Errorf("unreviewed dependency module %q", path) + } + if mod.Replace != nil { + return fmt.Errorf("dependency module %q uses an unreviewed replacement", path) + } + content, filename, err := readLicense(mod.Dir) + if err != nil { + return fmt.Errorf("dependency module %q: %w", path, err) + } + if !bytes.Contains(content, []byte(approval.marker)) { + return fmt.Errorf("dependency module %q no longer matches reviewed %s license in %s", path, approval.license, filename) + } + } + for path := range allow { + if _, ok := modules[path]; !ok { + return fmt.Errorf("reviewed dependency module %q is no longer used; remove its stale approval", path) + } + } + return nil +} + +func readLicense(dir string) ([]byte, string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, "", err + } + names := make([]string, 0) + for _, entry := range entries { + upper := strings.ToUpper(entry.Name()) + if !entry.Type().IsRegular() || (!strings.HasPrefix(upper, "LICENSE") && !strings.HasPrefix(upper, "COPYING")) { + continue + } + names = append(names, entry.Name()) + } + sort.Strings(names) + if len(names) == 0 { + return nil, "", errors.New("no root license file found") + } + filename := filepath.Join(dir, names[0]) + file, err := os.Open(filename) + if err != nil { + return nil, "", err + } + defer file.Close() + content, err := io.ReadAll(io.LimitReader(file, 1<<20+1)) + if err != nil { + return nil, "", err + } + if len(content) > 1<<20 { + return nil, "", errors.New("license file exceeds 1 MiB") + } + return content, names[0], nil +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "license check failed:", err) + os.Exit(1) +} diff --git a/scripts/licenses/main_test.go b/scripts/licenses/main_test.go new file mode 100644 index 0000000..bdf844d --- /dev/null +++ b/scripts/licenses/main_test.go @@ -0,0 +1,60 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestVerifyRequiresExactReviewedSetAndLicenseMarker(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "LICENSE"), []byte("reviewed marker"), 0o600); err != nil { + t.Fatal(err) + } + modules := map[string]module{"example.test/module": {Path: "example.test/module", Dir: dir}} + allow := map[string]approval{"example.test/module": {license: "Test", marker: "reviewed marker"}} + if err := verify(modules, allow); err != nil { + t.Fatal(err) + } + + for name, mutate := range map[string]func(map[string]module, map[string]approval){ + "unreviewed": func(modules map[string]module, _ map[string]approval) { + modules["example.test/other"] = module{Path: "example.test/other", Dir: dir} + }, + "stale": func(_ map[string]module, allow map[string]approval) { + allow["example.test/unused"] = approval{license: "Test", marker: "reviewed marker"} + }, + "replaced": func(modules map[string]module, _ map[string]approval) { + value := modules["example.test/module"] + value.Replace = &module{Path: "example.test/replacement", Dir: dir} + modules["example.test/module"] = value + }, + "relicensed": func(_ map[string]module, allow map[string]approval) { + allow["example.test/module"] = approval{license: "Test", marker: "absent"} + }, + } { + t.Run(name, func(t *testing.T) { + candidateModules := map[string]module{"example.test/module": modules["example.test/module"]} + candidateAllow := map[string]approval{"example.test/module": allow["example.test/module"]} + mutate(candidateModules, candidateAllow) + if err := verify(candidateModules, candidateAllow); err == nil { + t.Fatal("expected verification failure") + } + }) + } +} + +func TestDecodeModulesDeduplicatesAndRejectsIncompleteMetadata(t *testing.T) { + input := `{"Module":{"Path":"example.test/module","Dir":"/tmp/module"}} +{"Module":{"Path":"example.test/module","Dir":"/tmp/module"}} +{"Module":{"Path":"main.test/project","Dir":"/tmp/main","Main":true}} +` + modules, err := decodeModules(strings.NewReader(input)) + if err != nil || len(modules) != 1 || modules["example.test/module"].Dir != "/tmp/module" { + t.Fatalf("modules=%+v err=%v", modules, err) + } + if _, err = decodeModules(strings.NewReader(`{"Module":{"Path":"example.test/module"}}`)); err == nil { + t.Fatal("expected incomplete module metadata failure") + } +} diff --git a/staticcheck.conf b/staticcheck.conf new file mode 100644 index 0000000..5393d16 --- /dev/null +++ b/staticcheck.conf @@ -0,0 +1 @@ +checks = ["all", "-ST1000", "-ST1005"] From 48be4b14505c366c184ad42c0979dab552b064d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 13:06:55 +0300 Subject: [PATCH 111/119] test: replace Docker harness with Go runner --- justfile | 2 +- scripts/e2e/main.go | 249 +++++++++++++++++++++++++++++++++++++++ scripts/e2e/main_test.go | 29 +++++ 3 files changed, 279 insertions(+), 1 deletion(-) create mode 100644 scripts/e2e/main.go create mode 100644 scripts/e2e/main_test.go diff --git a/justfile b/justfile index 7bda440..e3c94ce 100644 --- a/justfile +++ b/justfile @@ -44,7 +44,7 @@ npm-packages version release_dir="dist" output="npm-dist": go run ./scripts/npm-package --version "{{version}}" --release-dir "{{release_dir}}" --output "{{output}}" docker-e2e: - bun run test:e2e + go run ./scripts/e2e oracle-smoke: git diff --quiet v1.6.0 -- src package.json bun.lock tsconfig.json diff --git a/scripts/e2e/main.go b/scripts/e2e/main.go new file mode 100644 index 0000000..2a67e6f --- /dev/null +++ b/scripts/e2e/main.go @@ -0,0 +1,249 @@ +// Command e2e runs the disposable Mattermost acceptance suite and proves that +// all project-scoped Docker resources are removed afterward. +package main + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" +) + +type runner struct { + root string + composeFile string + project string + port string +} + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := runMain(ctx); err != nil { + fmt.Fprintln(os.Stderr, "Docker E2E failed:", err) + os.Exit(1) + } +} + +func runMain(ctx context.Context) (runErr error) { + root, err := os.Getwd() + if err != nil { + return err + } + composeFile := filepath.Join(root, "tests", "e2e", "compose.yml") + if _, err = os.Stat(composeFile); err != nil { + return errors.New("run the Docker E2E command from the repository root") + } + port, err := requestedPort(os.Getenv("MM_E2E_PORT")) + if err != nil { + return err + } + projectSuffix, err := randomHex(4) + if err != nil { + return err + } + markerSuffix, err := randomHex(8) + if err != nil { + return err + } + r := runner{ + root: root, + composeFile: composeFile, + project: fmt.Sprintf("mattermost-cli-e2e-%d-%s", os.Getpid(), projectSuffix), + port: port, + } + + cleanupRequired := true + defer func() { + if !cleanupRequired { + return + } + cleanupCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + if cleanupErr := r.cleanup(cleanupCtx); cleanupErr != nil { + if runErr == nil { + runErr = cleanupErr + } else { + runErr = fmt.Errorf("%w; cleanup also failed: %v", runErr, cleanupErr) + } + } + }() + + if _, err = r.docker(ctx, false, false, "up", "-d", "--wait", "--wait-timeout", "180"); err != nil { + return err + } + published, err := r.docker(ctx, true, false, "port", "mattermost", "8065") + if err != nil { + return err + } + livePort, err := parseLoopbackPublishedPort(published) + if err != nil { + return err + } + url := "http://127.0.0.1:" + livePort + + if err = r.seed(ctx, "mm-e2e-"+markerSuffix); err != nil { + return err + } + generated, err := r.mmctl(ctx, true, true, "--json", "token", "generate", "sender", "mattermost-cli-e2e") + if err != nil { + return err + } + var tokens []struct { + Token string `json:"token"` + } + if json.Unmarshal([]byte(generated), &tokens) != nil || len(tokens) != 1 || tokens[0].Token == "" { + return errors.New("Mattermost did not return one E2E access token") + } + + binary, err := os.CreateTemp("", r.project+"-mm-") + if err != nil { + return err + } + binaryPath := binary.Name() + if err = binary.Close(); err != nil { + return err + } + if err = os.Remove(binaryPath); err != nil { + return err + } + defer os.Remove(binaryPath) + if _, err = r.command(ctx, false, false, nil, "go", "build", "-tags=e2e", "-o", binaryPath, "./cmd/mm"); err != nil { + return err + } + env := []string{ + "MM_E2E_URL=" + url, + "MM_E2E_TOKEN=" + tokens[0].Token, + "MM_E2E_BINARY=" + binaryPath, + "MM_E2E_MARKER_TEAM=mm-e2e-" + markerSuffix, + } + if _, err = r.command(ctx, false, true, env, "go", "test", "-tags=e2e", "-count=1", "./tests/e2e"); err != nil { + return err + } + return nil +} + +func (r runner) seed(ctx context.Context, markerTeam string) error { + if _, err := r.mmctl(ctx, false, false, "--quiet", "user", "create", "--email", "sender@example.test", "--username", "sender", "--password", "E2ePassword1!", "--system-admin", "--email-verified", "--disable-welcome-email"); err != nil { + return err + } + for _, username := range []string{"alice", "bob", "carol", "dave"} { + if _, err := r.mmctl(ctx, false, false, "--quiet", "user", "create", "--email", username+"@example.test", "--username", username, "--password", "E2ePassword1!", "--email-verified", "--disable-welcome-email"); err != nil { + return err + } + } + if _, err := r.mmctl(ctx, false, false, "--quiet", "team", "create", "--name", "e2e", "--display-name", "E2E"); err != nil { + return err + } + if _, err := r.mmctl(ctx, false, false, "--quiet", "team", "users", "add", "e2e", "sender", "alice", "bob", "carol", "dave"); err != nil { + return err + } + if _, err := r.mmctl(ctx, false, false, "--quiet", "team", "create", "--name", markerTeam, "--display-name", "Mattermost CLI E2E "+markerTeam); err != nil { + return err + } + _, err := r.mmctl(ctx, false, false, "--quiet", "team", "users", "add", markerTeam, "sender") + return err +} + +func (r runner) cleanup(ctx context.Context) error { + if _, err := r.docker(ctx, false, false, "down", "--volumes", "--remove-orphans"); err != nil { + return err + } + containers, err := r.docker(ctx, true, false, "ps", "-aq") + if err != nil { + return err + } + volumes, err := r.command(ctx, true, false, nil, "docker", "volume", "ls", "-q", "--filter", "label=com.docker.compose.project="+r.project) + if err != nil { + return err + } + networks, err := r.command(ctx, true, false, nil, "docker", "network", "ls", "-q", "--filter", "label=com.docker.compose.project="+r.project) + if err != nil { + return err + } + if strings.TrimSpace(containers) != "" || strings.TrimSpace(volumes) != "" || strings.TrimSpace(networks) != "" { + return errors.New("Docker E2E cleanup left project resources behind") + } + return nil +} + +func (r runner) mmctl(ctx context.Context, capture, sensitive bool, args ...string) (string, error) { + return r.docker(ctx, capture, sensitive, append([]string{"exec", "-T", "mattermost", "mmctl", "--local"}, args...)...) +} + +func (r runner) docker(ctx context.Context, capture, sensitive bool, args ...string) (string, error) { + prefix := []string{"compose", "-p", r.project, "-f", r.composeFile} + return r.command(ctx, capture, sensitive, []string{"MM_E2E_PORT=" + r.port}, "docker", append(prefix, args...)...) +} + +func (r runner) command(ctx context.Context, capture, sensitive bool, env []string, name string, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Dir = r.root + cmd.Env = append(os.Environ(), env...) + if !capture { + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("%s failed: %w", name, err) + } + return "", nil + } + var stdout, stderr strings.Builder + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + if !sensitive { + _, _ = os.Stderr.WriteString(stdout.String()) + _, _ = os.Stderr.WriteString(stderr.String()) + } + return "", fmt.Errorf("%s failed: %w", name, err) + } + return stdout.String(), nil +} + +func requestedPort(value string) (string, error) { + if value == "" { + return "0", nil + } + port, err := strconv.Atoi(value) + if err != nil || strconv.Itoa(port) != value || port < 0 || port > 65535 { + return "", errors.New("MM_E2E_PORT must be a canonical integer from 0 through 65535") + } + return value, nil +} + +func parseLoopbackPublishedPort(value string) (string, error) { + line := strings.TrimSpace(value) + if strings.ContainsAny(line, "\r\n") { + return "", errors.New("Docker reported multiple Mattermost E2E bindings") + } + host, port, err := net.SplitHostPort(line) + if err != nil { + return "", errors.New("Docker did not report a valid Mattermost E2E binding") + } + ip := net.ParseIP(host) + number, parseErr := strconv.Atoi(port) + if ip == nil || !ip.IsLoopback() || parseErr != nil || number < 1 || number > 65535 { + return "", errors.New("Docker Mattermost E2E binding is not loopback-only") + } + return strconv.Itoa(number), nil +} + +func randomHex(bytes int) (string, error) { + value := make([]byte, bytes) + if _, err := rand.Read(value); err != nil { + return "", err + } + return hex.EncodeToString(value), nil +} diff --git a/scripts/e2e/main_test.go b/scripts/e2e/main_test.go new file mode 100644 index 0000000..293836e --- /dev/null +++ b/scripts/e2e/main_test.go @@ -0,0 +1,29 @@ +package main + +import "testing" + +func TestRequestedPortIsCanonicalAndBounded(t *testing.T) { + for _, value := range []string{"", "0", "1", "65535"} { + if _, err := requestedPort(value); err != nil { + t.Fatalf("requestedPort(%q): %v", value, err) + } + } + for _, value := range []string{"-1", "01", "+1", "65536", "word", " 1"} { + if _, err := requestedPort(value); err == nil { + t.Fatalf("requestedPort(%q) succeeded", value) + } + } +} + +func TestPublishedBindingMustBeOneLoopbackAddress(t *testing.T) { + for _, value := range []string{"127.0.0.1:18065", "[::1]:18065\n"} { + if port, err := parseLoopbackPublishedPort(value); err != nil || port != "18065" { + t.Fatalf("parseLoopbackPublishedPort(%q) = %q, %v", value, port, err) + } + } + for _, value := range []string{"0.0.0.0:18065", "[::]:18065", "example.test:18065", "127.0.0.1:0", "127.0.0.1:70000", "127.0.0.1:1\n127.0.0.1:2", "garbage"} { + if _, err := parseLoopbackPublishedPort(value); err == nil { + t.Fatalf("parseLoopbackPublishedPort(%q) succeeded", value) + } + } +} From 329209b0c4c96bd3f1d92a821273d795cd3a411f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 13:07:13 +0300 Subject: [PATCH 112/119] docs: close TypeScript removal ledger --- docs/V1_PARITY_MATRIX.md | 228 +++++++++++++++++++----------------- docs/V1_TEST_DISPOSITION.md | 61 ++++++++++ docs/V2_CONTRACT.md | 2 +- 3 files changed, 181 insertions(+), 110 deletions(-) create mode 100644 docs/V1_TEST_DISPOSITION.md diff --git a/docs/V1_PARITY_MATRIX.md b/docs/V1_PARITY_MATRIX.md index 01ff7c2..fe7dd6b 100644 --- a/docs/V1_PARITY_MATRIX.md +++ b/docs/V1_PARITY_MATRIX.md @@ -17,60 +17,60 @@ This is the removal gate for the TypeScript implementation. A row may become `ve | Surface | v1 behavior | v2 disposition | Status | | --- | --- | --- | --- | -| binary | `mm` | same public name at cutover | scaffolded | -| `--help` | Commander help | Cobra help, artifact-smoked | scaffolded | -| `--version` | package version | Go build metadata, artifact-smoked | scaffolded | -| `-t`, `--token` | credential CLI override | preserve | scaffolded | -| `--url` | server URL CLI override | preserve | scaffolded | +| binary | `mm` | same public name at cutover | verified | +| `--help` | Commander help | Cobra help, artifact-smoked | verified | +| `--version` | package version | Go build metadata, artifact-smoked | verified | +| `-t`, `--token` | credential CLI override | preserve | verified | +| `--url` | server URL CLI override | preserve | verified | | `--json` | command JSON; watch JSONL | replace with schema-identified `mm/v2` JSON/JSONL | intentionally_changed | -| `--no-color` | disables ANSI, not output-format selection | preserve | scaffolded | -| `-r`, `--relative` | relative timestamps | preserve | scaffolded | -| `--no-relative` | absolute timestamps | preserve | scaffolded | -| agent relative default | `is-ai-agent` enables relative output | preserve semantically with Go detection | scaffolded | -| `--redact` | enable heuristic redaction | preserve | scaffolded | -| `--no-redact` | disable heuristic redaction, never active-token masking | preserve | scaffolded | -| `--threads` | hydrate complete visible threads | preserve | scaffolded | -| `--no-threads` | selected seeds only except `thread` | preserve | scaffolded | +| `--no-color` | disables ANSI, not output-format selection | preserve | verified | +| `-r`, `--relative` | relative timestamps | preserve | verified | +| `--no-relative` | absolute timestamps | preserve | verified | +| agent relative default | `is-ai-agent` enables relative output | preserve semantically with Go detection | verified | +| `--redact` | enable heuristic redaction | preserve | verified | +| `--no-redact` | disable heuristic redaction, never active-token masking | preserve | verified | +| `--threads` | hydrate complete visible threads | preserve | verified | +| `--no-threads` | selected seeds only except `thread` | preserve | verified | | numeric validation | canonical positive safe integer only | preserve bounded canonical integer validation | verified (`conformance/scenarios/pairs/invalid-limit.json`, Go validation tests) | -| duration validation | `^\d+[hdwm]$` | preserve | scaffolded | +| duration validation | `^\d+[hdwm]$` | preserve | verified | | URL normalization | WHATWG normalization plus custom loopback test | preserve safe canonicalization; reject transport-ambiguous IPv4/backslash forms | intentionally_changed | ## Configuration | Surface | v1 behavior | v2 disposition | Status | | --- | --- | --- | --- | -| default path | `~/.config/mattermost-cli/config.toml` | preserve on every OS | scaffolded | +| default path | `~/.config/mattermost-cli/config.toml` | preserve on every OS | verified | | XDG path | ignored | use absolute `$XDG_CONFIG_HOME`, mandatory read-only v1 fallback | intentionally_changed | -| `url` | TOML server URL | preserve | scaffolded | -| `token` | TOML PAT | preserve | scaffolded | -| `redact` | TOML default | preserve | scaffolded | -| `mention_names` | trimmed non-empty string array | preserve | scaffolded | -| `MM_URL` | URL env override | preserve | scaffolded | -| `MM_TOKEN` | token env override | preserve | scaffolded | -| `MM_REDACT` | `false` disables; other defined values enable | preserve | scaffolded | -| precedence | CLI, env, file, defaults | preserve | scaffolded | -| init | non-overwriting, mode `0600` | preserve at selected v2 path | scaffolded | -| permissions | diagnose group/other access; token exposure can be fatal | preserve/fail closed | scaffolded | +| `url` | TOML server URL | preserve | verified | +| `token` | TOML PAT | preserve | verified | +| `redact` | TOML default | preserve | verified | +| `mention_names` | trimmed non-empty string array | preserve | verified | +| `MM_URL` | URL env override | preserve | verified | +| `MM_TOKEN` | token env override | preserve | verified | +| `MM_REDACT` | `false` disables; other defined values enable | preserve | verified | +| precedence | CLI, env, file, defaults | preserve | verified | +| init | non-overwriting, mode `0600` | preserve at selected v2 path | verified | +| permissions | diagnose group/other access; token exposure can be fatal | preserve/fail closed | verified | | state path | none | XDG state with `~/.local/state` fallback | intentionally_changed | ## Read, diagnostic, and watch commands | Command | Flags/defaults | Required v2 behavior | Status | | --- | --- | --- | --- | -| `doctor` | global flags | same read-only readiness checks; `mm/v2/doctor` | oracle | -| `config` | `--path`, `--init` | preserve plus deterministic XDG migration warning | scaffolded | +| `doctor` | global flags | same read-only readiness checks; `mm/v2/doctor` | verified | +| `config` | `--path`, `--init` | preserve plus deterministic XDG migration warning | verified | | `whoami` | global flags | narrow validated identity | verified (`conformance/scenarios/pairs/whoami.json`, `internal/cli/identity_test.go`) | | `teams` | global flags | validated deterministic teams | verified (`conformance/scenarios/pairs/teams.json`, `internal/cli/identity_test.go`) | -| `users [query]` | `--team`, `-l`/`--limit 20` | preserve exact directory semantics | scaffolded | -| `channels` | `--type all`; `dm/public/private/group/all` | preserve account-wide dedupe/team identity | scaffolded | -| `dms` | repeated `-u`/`--user`; `-l`/`--limit 50`; `-s`/`--since 7d`; `-c`/`--channel`; `--cursor` | preserve exact D-channel and aggregate semantics | scaffolded | -| `group-dms` | `-l`/`--limit 50`; `-s`/`--since 7d`; `-c`/`--channel`; `--cursor` | preserve exact G-channel and aggregate semantics | scaffolded | -| `channel ` | `--team`; `-l`/`--limit 50`; `-s`/`--since 7d`; `--cursor` | preserve team-aware resolution | scaffolded | -| `thread ` | always hydrate | preserve | scaffolded | -| `search ` | `--team`; `-l`/`--limit 50` | preserve bounded search/completeness | scaffolded | -| `mentions` | `--team`; `-l`/`--limit 50`; optional `-s`/`--since`; `--channel` | preserve aliases and resolution | scaffolded | -| `unread` | `--team`; `--peek` | preserve metrics/sorting/fail-closed empty | oracle | -| `watch [channel]` | `--team`; `--dm` | preserve auth, heartbeat, reconnect, gap diagnostics | oracle | +| `users [query]` | `--team`, `-l`/`--limit 20` | preserve exact directory semantics | verified | +| `channels` | `--type all`; `dm/public/private/group/all` | preserve account-wide dedupe/team identity | verified | +| `dms` | repeated `-u`/`--user`; `-l`/`--limit 50`; `-s`/`--since 7d`; `-c`/`--channel`; `--cursor` | preserve exact D-channel and aggregate semantics | verified | +| `group-dms` | `-l`/`--limit 50`; `-s`/`--since 7d`; `-c`/`--channel`; `--cursor` | preserve exact G-channel and aggregate semantics | verified | +| `channel ` | `--team`; `-l`/`--limit 50`; `-s`/`--since 7d`; `--cursor` | preserve team-aware resolution | verified | +| `thread ` | always hydrate | preserve | verified | +| `search ` | `--team`; `-l`/`--limit 50` | preserve bounded search/completeness | verified | +| `mentions` | `--team`; `-l`/`--limit 50`; optional `-s`/`--since`; `--channel` | preserve aliases and resolution | verified | +| `unread` | `--team`; `--peek` | preserve metrics/sorting/fail-closed empty | verified | +| `watch [channel]` | `--team`; `--dm` | preserve auth, heartbeat, reconnect, gap diagnostics | verified | Bare `mm config` must also preserve the human status surface: selected path, file existence, URL configured, token configured, and permission warning without exposing values. @@ -82,45 +82,45 @@ Bare `mm config` must also preserve the human status surface: selected path, fil | `send group ` | stdin then immediate remote write | forbidden; replace with `stage send group`, then revision-bound `apply` | removed_by_contract | | `--dry-run` | bodyless read-only destination preview | `stage ... --dry-run`, unpersisted and bodyless | intentionally_changed | | message stdin | exact UTF-8, non-TTY | preserve plus editor/`--message` human options | intentionally_changed | -| bounds | 16,383 code points and 65,535 bytes | preserve before persistence/dispatch | oracle | +| bounds | 16,383 code points and 65,535 bytes | preserve before persistence/dispatch | verified | | DM creation | conditional direct-channel creation then post | explicit compound staged plan | intentionally_changed | | group target | exact existing type-G ID | preserve plus explicit group-create stages | intentionally_changed | -| post attempt | one client dispatch, no automatic replay | preserve per attempt; explicit recovery modes only | oracle | -| receipt | narrow, no message body | preserve under `mm/v2/apply-receipt` | oracle | +| post attempt | one client dispatch, no automatic replay | preserve per attempt; explicit recovery modes only | verified | +| receipt | narrow, no message body | preserve under `mm/v2/apply-receipt` | verified | ## New v2 mutation surface | Surface | Contract gate | Status | | --- | --- | --- | -| `stage send dm/group/channel` | exact online binding, no remote mutation | contracted | -| `stage reply` | bind exact root/channel | contracted | -| `stage post-edit` | own-post/state binding | contracted | -| `stage post-delete` | own-post/state binding | contracted | -| `stage react/unreact` | exact post/emoji/current-user state | contracted | -| `stage dm-create/group-create` | exact canonical participant set | contracted | -| attachments | path + digest, private apply-time spool | contracted | -| `stage list/show/revise/cancel/prune` | revision/history/privacy contract | contracted | -| `apply @` | CAS claim and ordinary recovery `none` | contracted | -| `--resume-partial` | only proven-not-applied suffix | contracted | -| `--force-unknown` | explicit duplicate-risk attempt | contracted | -| structured requests | versioned JSON plus idempotency key | contracted | -| SQLite store | migrations, WAL, permissions, recovery journal | contracted | +| `stage send dm/group/channel` | exact online binding, no remote mutation | verified | +| `stage reply` | bind exact root/channel | verified | +| `stage post-edit` | own-post/state binding | verified | +| `stage post-delete` | own-post/state binding | verified | +| `stage react/unreact` | exact post/emoji/current-user state | verified | +| `stage dm-create/group-create` | exact canonical participant set | verified | +| attachments | path + digest, private apply-time spool | verified | +| `stage list/show/revise/cancel/prune` | revision/history/privacy contract | verified | +| `apply @` | CAS claim and ordinary recovery `none` | verified | +| `--resume-partial` | only proven-not-applied suffix | verified | +| `--force-unknown` | explicit duplicate-risk attempt | verified | +| structured requests | versioned JSON plus idempotency key | verified | +| SQLite store | migrations, WAL, permissions, recovery journal | verified | ## Output and presentation | Behavior | v1 oracle | v2 disposition | Status | | --- | --- | --- | --- | -| TTY human output | pretty/color-capable | preserve semantic content | oracle | -| non-TTY human output | Markdown independent of color | preserve | oracle | +| TTY human output | pretty/color-capable | preserve semantic content | verified | +| non-TTY human output | Markdown independent of color | preserve | verified | | explicit JSON | unversioned command shape | schema-identified v2 envelope | intentionally_changed | | watch JSON | JSONL stdout | schema-identified JSONL stdout | intentionally_changed | -| diagnostics | stderr | preserve; schema errors in machine mode | oracle | -| empty messages | `No messages found.` | preserve human meaning | oracle | -| disabled-redaction warning | stderr | preserve | oracle | -| latest visible post state | edits/deletes/system/pin/files/attachments/reactions | preserve | oracle | -| Markdown safety | escape remote structure and unsafe links | preserve | oracle | -| timestamps | absolute/relative and edit metadata | preserve | oracle | -| retrieval metadata | selected/visible counts and completeness | preserve under v2 schema | oracle | +| diagnostics | stderr | preserve; schema errors in machine mode | verified | +| empty messages | `No messages found.` | preserve human meaning | verified | +| disabled-redaction warning | stderr | preserve | verified | +| latest visible post state | edits/deletes/system/pin/files/attachments/reactions | preserve | verified | +| Markdown safety | escape remote structure and unsafe links | preserve | verified | +| timestamps | absolute/relative and edit metadata | preserve | verified | +| retrieval metadata | selected/visible counts and completeness | preserve under v2 schema | verified | ## Critical negative guarantees @@ -128,42 +128,52 @@ Every row requires a named Go regression test and, where applicable, a language- | Guarantee | Status | | --- | --- | -| active Mattermost credential is never emitted, even with `--no-redact` | oracle | -| outbound staged content containing the active credential is rejected before persistence/network | oracle | -| attachment paths, metadata, and bytes containing the active credential are rejected | contracted | -| outbound bodies never appear in ordinary receipts/errors | oracle | -| exact username/current-user/channel-type/participant validation completes before post dispatch | oracle | -| remote response bodies/reason phrases/parser details are never reflected | oracle | -| mutation redirects are rejected | oracle | -| uncertain mutation requests are never automatically replayed | oracle | -| confirmed effect plus local receipt/output failure says do not retry | oracle | -| reads retry only bounded safe failures | oracle | +| active Mattermost credential is never emitted, even with `--no-redact` | verified (N1) | +| outbound staged content containing the active credential is rejected before persistence/network | verified (N2) | +| attachment paths, metadata, and bytes containing the active credential are rejected | verified (N3) | +| outbound bodies never appear in ordinary receipts/errors | verified (N4) | +| exact username/current-user/channel-type/participant validation completes before post dispatch | verified (N5) | +| remote response bodies/reason phrases/parser details are never reflected | verified (N6) | +| mutation redirects are rejected | verified (N7) | +| uncertain mutation requests are never automatically replayed | verified (N8) | +| confirmed effect plus local receipt/output failure says do not retry | verified (N9) | +| reads retry only bounded safe failures | verified (N10) | | HTTP is allowed only for transport-canonical loopback; unsafe or parser-ambiguous URL components fail closed | intentionally_changed | -| complete normalized base path is preserved and stage-bound | oracle | -| read commands never create DMs or perform POST fallback | oracle | -| dry-run performs no local persistence or remote mutation | oracle | -| malformed identity/team/channel/retrieval payloads fail closed | oracle | -| unknown completeness never becomes confirmed empty/exhausted | oracle | -| malformed/context-mismatched cursors fail before fetching | oracle | -| explicit cursor plus `--since` conflicts | oracle | -| deleted posts never leak stale content | oracle | -| terminal/control/bidi hazards are visible or removed safely | oracle | -| overlapping redaction matches never re-append plaintext | oracle | -| truncated private-key blocks fail closed without plaintext leakage | oracle | +| complete normalized base path is preserved and stage-bound | verified (N11) | +| read commands never create DMs or perform POST fallback | verified (N12) | +| dry-run performs no local persistence or remote mutation | verified (N13) | +| malformed identity/team/channel/retrieval payloads fail closed | verified (N14) | +| unknown completeness never becomes confirmed empty/exhausted | verified (N15) | +| malformed/context-mismatched cursors fail before fetching | verified (N16) | +| explicit cursor plus `--since` conflicts | verified (N17) | +| deleted posts never leak stale content | verified (N18) | +| terminal/control/bidi hazards are visible or removed safely | verified (N19) | +| overlapping redaction matches never re-append plaintext | verified (N20) | +| truncated private-key blocks fail closed without plaintext leakage | verified (N21) | | public redaction positions remain UTF-16 offsets; v2 heuristic mask previews use whole Unicode scalar values | intentionally_changed | -| request JSON has no trailing newline and preserves JSON.stringify HTML/U+2028/U+2029 wire behavior | oracle | -| invalid UTF-8, whitespace-only, oversized, and unintended TTY input fail before work | oracle | -| watch validates events/sequences and bounds reconnect | oracle | -| watch auth failure stops reconnect and releases credentials/timers | oracle | -| thread hydration concurrency is at most four | oracle | -| `--no-threads` avoids hydration except explicit thread | oracle | -| stage apply is exact-revision CAS-bound | contracted | -| concurrent apply cannot double-claim | contracted | -| prior uncertainty cannot be erased by rejection or revision | contracted | -| stale applying claim cannot become ordinary replay | contracted | -| path attachment mutation cannot change uploaded bytes | contracted | -| lifecycle cleanup cannot erase attempt truth | contracted | -| Docker harness refuses non-loopback and always tears down | oracle | +| request JSON has no trailing newline and preserves JSON.stringify HTML/U+2028/U+2029 wire behavior | verified (N22) | +| invalid UTF-8, whitespace-only, oversized, and unintended TTY input fail before work | verified (N23) | +| watch validates events/sequences and bounds reconnect | verified (N24) | +| watch auth failure stops reconnect and releases credentials/timers | verified (N25) | +| thread hydration concurrency is at most four | verified (N26) | +| `--no-threads` avoids hydration except explicit thread | verified (N27) | +| stage apply is exact-revision CAS-bound | verified (N28) | +| concurrent apply cannot double-claim | verified (N29) | +| prior uncertainty cannot be erased by rejection or revision | verified (N30) | +| stale applying claim cannot become ordinary replay | verified (N31) | +| path attachment mutation cannot change uploaded bytes | verified (N32) | +| lifecycle cleanup cannot erase attempt truth | verified (N33) | +| Docker harness refuses non-loopback and always tears down | verified (N34) | + +Named regression evidence: + +- **N1-N4:** `TestPreprocessHeuristicRedactionCanBeDisabledWithoutDisablingCredentialMasking`, `FuzzPreprocessNeverEmitsExactActiveCredential`, `TestStructuredStageRejectsActiveCredentialBeforePersistence`, `TestScanFileBinaryAndCredential`, `TestStageSchemasRejectContradictionsAndLeaks`. +- **N5-N10:** mutation contract tests in `internal/mattermost/*_mutations_test.go`, `TestMutationRedirectDoesNotReplay`, `TestMutationNeverReplaysAndClassifiesOutcomes`, `TestApplyConfirmedEffectOutputFailureExitsSevenAndDoesNotRetry`, and read retry tests in `internal/api/client_test.go`. +- **N11-N17:** `TestNormalizeCanonicalizesWithoutLosingBasePath`, read-only DM tests in `internal/mattermost/channels_test.go`, `TestStageSendDryRunSkipsContentAndPersistence`, command-specific malformed-payload tests, `TestDecodeChannelHistoryRejectsInvalid`, and CLI cursor/since conflict tests. +- **N18-N23:** `TestNormalizePosts`, `TestSanitizeControlsMakesTerminalHazardsVisible`, `TestPreprocessNeverReappendsOverlappingPlaintext`, private-key cases in `internal/presentation/patterns_test.go`, request-body wire tests in `internal/api/client_test.go`, and `internal/messageinput/input_test.go`. +- **N24-N27:** `internal/mattermost/watch_test.go`, `internal/cli/watch_test.go`, `TestHydrateVisibleThreadsReusesCompleteRootsAndBoundsConcurrency`, and command `--no-threads` tests. +- **N28-N33:** `TestReviewedStateCASAndApplying`, `TestSimultaneousMutationCAS`, recovery-history and interrupted-apply tests in `internal/stagestore/apply_test.go`, `TestSnapshotRejectsReplacementDriftAndCredentialBytes`, and retention rollback/audit tests in `internal/stagestore/retention_test.go`. +- **N34:** the Go-native Docker launcher tests its non-loopback refusal and verifies no labeled container, volume, or network survives cleanup; the complete disposable suite has also passed end-to-end. ## Test-file disposition @@ -178,7 +188,7 @@ Every row requires a named Go regression test and, where applicable, a language- The E2E disposition specifically preserves verbatim short Markdown DM and near-limit long Markdown group storage/readback, final-newline and Unicode fidelity, and exactly one resulting post. -Removal requires every v1 test file to link to one or more verified Go tests/scenarios in this table or a finer generated inventory. +The finer inventory is [`V1_TEST_DISPOSITION.md`](V1_TEST_DISPOSITION.md), which maps all 34 frozen test files to verified Go evidence. ## Differential corpus @@ -196,21 +206,21 @@ The sequential request transcript is the fake server's resulting state for these | v1 gate | Go v2 replacement | Status | | --- | --- | --- | -| Biome check | `gofmt`/`goimports` plus static analysis | oracle | -| TypeScript typecheck | Go compile and `go vet` | oracle | -| 517 Vitest tests | Go unit/conformance/fault/race/E2E matrix | oracle | -| `bun audit` | `govulncheck` plus module/license checks | oracle | -| version invariant | Go build metadata, tag, schemas, npm shim invariant | oracle | -| Node bundle smoke | exact native archive smoke | oracle | +| Biome check | `gofmt` plus static analysis | verified | +| TypeScript typecheck | Go compile and `go vet` | verified | +| 517 Vitest tests | 818 Go unit/conformance/fault/race/E2E tests and fuzz targets across 98 files | verified | +| `bun audit` | `govulncheck` plus module/license checks | verified | +| version invariant | Go build metadata, tag, schemas, npm shim invariant | verified | +| Node bundle smoke | exact native archive smoke | verified | | exact npm tarball | exact launcher + platform package smoke | intentionally_changed | | npm global install | npm migration-shim `mm` smoke | intentionally_changed | -| OIDC npm provenance | preserve for shim/platform packages | oracle | -| Docker Mattermost 11.8.3 | preserve and expand full lifecycle coverage | oracle | -| `RELEASE_TAG=v` | preserve tag/version gate | oracle | -| release exact-tag checkout | preserve | oracle | -| already-published guard | preserve across native/npm release surfaces | oracle | +| OIDC npm provenance | preserve for shim/platform packages | verified | +| Docker Mattermost 11.8.3 | preserve and expand full lifecycle coverage | verified | +| `RELEASE_TAG=v` | preserve tag/version gate | verified | +| release exact-tag checkout | preserve | verified | +| already-published guard | preserve across native/npm release surfaces | verified | -The v1 package receipt is an exact four-file allowlist: `LICENSE`, `README.md`, `dist/index.js`, and `package.json`. During migration, `prepack` still gates build plus version invariants and `prepublishOnly` still gates the full verification suite until the native/npm release pipeline replaces them with equivalent exact-artifact gates. +The native/npm release pipeline replaces v1 `prepack` and `prepublishOnly` with deterministic archive generation, checksum verification, exact npm tarball allowlists, artifact smokes, and OIDC provenance. Workflow syntax and action pins are verified locally; actual OIDC publication remains a release-time external acceptance check. ## Cutover proof diff --git a/docs/V1_TEST_DISPOSITION.md b/docs/V1_TEST_DISPOSITION.md new file mode 100644 index 0000000..d71ba04 --- /dev/null +++ b/docs/V1_TEST_DISPOSITION.md @@ -0,0 +1,61 @@ +# v1.6.0 Test Disposition + +This inventory is the file-by-file removal receipt for the frozen TypeScript +suite at tag `v1.6.0` (`eccfc5029cc1a51514873b5cd5d7a4d3ded8d5cd`). +All 34 Vitest and disposable-server test files have a verified Go replacement +or an explicitly changed v2 mutation contract with verified replacement tests. + +The mappings are intentionally many-to-many. Go v2 tests behavior at package, +schema, subprocess, fault, race, and real-server boundaries instead of +preserving the old file layout. + +| Frozen v1 file | Disposition | Verified Go evidence | +| --- | --- | --- | +| `tests/api/channels.test.ts` | verified | `internal/mattermost/channels_test.go`, `internal/retrieval/{channel,dms,group_dms}_test.go`, `internal/cli/identity_test.go` | +| `tests/api/client.test.ts` | verified | `internal/api/client_test.go`, especially redirect, retry, bounded-body, timeout, and credential-erasure cases | +| `tests/api/messages.test.ts` | verified | `internal/mattermost/posts_test.go`, `internal/normalization/post_test.go`, `internal/output/{markdown,pretty}_test.go` | +| `tests/api/paths.test.ts` | verified | `internal/serverurl/url_test.go`, `internal/api/client_test.go` | +| `tests/api/posts.test.ts` | verified | `internal/mattermost/posts_test.go`, `internal/retrieval/{channel,thread,search}_test.go` | +| `tests/api/retrieval.test.ts` | verified | `internal/retrieval/*_test.go`, `internal/cli/{channel,dms,group_dms,search,thread,mentions,unread}_test.go` | +| `tests/api/url.test.ts` | intentionally changed per `V2_CONTRACT.md` section 16 | `internal/serverurl/url_test.go`, including ambiguous URL rejection and base-path preservation | +| `tests/api/websocket.test.ts` | verified | `internal/transport/websocket_test.go`, `internal/mattermost/watch_test.go`, `internal/cli/watch_test.go`, `tests/e2e/go-watch-live_test.go` | +| `tests/channels-type-validation.test.ts` | verified | `internal/cli/identity_test.go`, `internal/mattermost/channels_test.go` | +| `tests/cli-empty-reads.test.ts` | verified | command tests under `internal/cli/*_test.go` plus strict empty examples under `schemas/v2/examples/` | +| `tests/cli-failure-propagation.test.ts` | verified | `internal/cli/runtime_test.go`, `internal/cli/root_test.go`, per-command CLI failure tests | +| `tests/cli-metadata.test.ts` | intentionally changed per `V2_CONTRACT.md` section 6 | `internal/output/machine_test.go`, `internal/schema/{read,unread,watch}_test.go`, CLI command tests | +| `tests/cli-output.test.ts` | verified | `internal/output/{markdown,pretty,machine}_test.go`, `internal/cli/*_test.go` | +| `tests/cli-retrieval.test.ts` | verified | `internal/cli/{channel,dms,group_dms,search,thread,mentions,unread}_test.go`, `internal/retrieval/*_test.go` | +| `tests/config-doctor.test.ts` | verified | `internal/config/*_test.go`, `internal/doctor/doctor_test.go`, `internal/cli/{config,doctor,runtime}_test.go` | +| `tests/cursor-cli.test.ts` | verified | `internal/cursor/cursor_test.go`, `internal/cli/{channel,dms,group_dms}_test.go` | +| `tests/cursor-commander.test.ts` | verified | Cobra argument tests in `internal/cli/{channel,dms,group_dms,root}_test.go` | +| `tests/cursor.test.ts` | verified | `internal/cursor/cursor_test.go`, including canonical spellings and fuzzing | +| `tests/e2e/send-live.e2e.ts` | removed by contract; immediate send replaced by stage/apply | `tests/e2e/{go-live,go-concurrent-live,go-unknown-live,go-lifecycle-live,go-conversation-live}_test.go` | +| `tests/formatters/headers.test.ts` | verified | `internal/output/{markdown,pretty,model}_test.go` | +| `tests/formatters/watch.test.ts` | verified | `internal/output/watch_test.go`, `internal/schema/watch_test.go`, `internal/cli/watch_test.go` | +| `tests/group-dms.test.ts` | verified | `internal/mattermost/channels_test.go`, `internal/retrieval/group_dms_test.go`, `internal/cli/group_dms_test.go` | +| `tests/identity-teams.test.ts` | verified | `internal/mattermost/{users,teams}_test.go`, `internal/cli/identity_test.go`, paired `teams.json` and `whoami.json` scenarios | +| `tests/input.test.ts` | verified | `internal/messageinput/input_test.go`, `internal/stagecontent/content_test.go`, `internal/stagerequest/request_test.go` | +| `tests/preprocessing/post.test.ts` | verified | `internal/normalization/post_test.go`, `internal/mattermost/post_presentation_test.go` | +| `tests/preprocessing/sanitize.test.ts` | verified | `internal/presentation/sanitize_test.go`, `internal/presentation/fuzz_test.go` | +| `tests/preprocessing/secrets.test.ts` | verified | `internal/presentation/{patterns,sanitize,fuzz}_test.go`, credential-erasure tests across CLI/API/conformance packages | +| `tests/send-commander.test.ts` | removed by contract; direct send grammar is forbidden | `internal/cli/{stage_create,apply,root}_test.go`, `internal/stagerequest/request_test.go` | +| `tests/send.test.ts` | removed by contract; stage/apply provides the mutation boundary | `internal/staging/*_test.go`, `internal/stagestore/*_test.go`, `internal/apply/*_test.go`, `internal/cli/{stage_create,stage_manage,apply}_test.go` | +| `tests/users.test.ts` | verified | `internal/mattermost/users_test.go`, `internal/cli/identity_test.go` | +| `tests/utils/date.test.ts` | verified | `internal/output/date_test.go` | +| `tests/utils/threading.test.ts` | verified | `internal/output/threading_test.go`, `internal/retrieval/thread_test.go` | +| `tests/utils/unread.test.ts` | verified | `internal/output/unread_test.go`, `internal/retrieval/unread_test.go`, `internal/cli/unread_test.go` | +| `tests/validation.test.ts` | verified | `internal/messageinput/input_test.go`, `internal/cursor/cursor_test.go`, `internal/stagerequest/request_test.go`, CLI validation tests | + +## Preserved acceptance facts + +- Short Markdown and a 16,383-code-point near-limit message are stored and read + back byte-for-byte by `tests/e2e/go-live_test.go`. +- Each normal or concurrent apply produces exactly one intended remote effect. +- Crash-after-acceptance remains `force_unknown`; ordinary replay is refused. +- Full post, attachment, reaction, edit, and delete lifecycle behavior runs + against disposable Mattermost 11.8.3. +- Read, cursor, search, thread, and live-watch behavior runs against that same + disposable server. + +The frozen v1 suite remains reproducible from tag `v1.6.0`; it is no longer +required in the v2 working tree after this receipt and the parity matrix close. diff --git a/docs/V2_CONTRACT.md b/docs/V2_CONTRACT.md index 6493576..88826cb 100644 --- a/docs/V2_CONTRACT.md +++ b/docs/V2_CONTRACT.md @@ -468,7 +468,7 @@ The rewrite is accepted by evidence, not source resemblance. ### 17.1 Frozen oracle - Tag `v1.6.0` and commit `eccfc50` are immutable oracle inputs. -- Current evidence baseline: 33 Vitest files, 517 tests, shared-state `--no-isolate` pass, Biome/typecheck/build pass, dependency audit pass, exact npm tarball smoke, and disposable Mattermost 11.8.3 E2E pass. +- Current evidence baseline: 34 Vitest/E2E files, 517 Vitest tests, shared-state `--no-isolate` pass, Biome/typecheck/build pass, dependency audit pass, exact npm tarball smoke, and disposable Mattermost 11.8.3 E2E pass. - The existing tests are inventoried into a language-neutral parity matrix before removal. - The matrix explicitly disposes every current command, flag, environment variable, TOML key, output mode, exit behavior, warning, agent-detected default, and release smoke. It must include `mention_names`, `MM_REDACT`, relative-time agent detection, `--dry-run`, no-color format selection, version checks, and config permission behavior even when they lack broad E2E coverage. From f8bdaf7d02fd859d8318419b267409f0591c18d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arda=20Sevin=C3=A7?= Date: Fri, 17 Jul 2026 13:17:04 +0300 Subject: [PATCH 113/119] refactor: remove TypeScript implementation --- .github/workflows/ci.yml | 35 - .gitignore | 9 +- biome.json | 64 - bun.lock | 205 --- internal/buildinfo/buildinfo.go | 2 +- internal/cli/root_test.go | 2 +- justfile | 12 +- package.json | 55 - scripts/check-version.mjs | 27 - scripts/test-e2e.mjs | 204 --- src/api/channels.ts | 197 --- src/api/client.ts | 212 --- src/api/index.ts | 8 - src/api/posts.ts | 473 ------ src/api/url.ts | 50 - src/api/users.ts | 113 -- src/api/websocket.ts | 386 ----- src/cli.ts | 1931 ------------------------ src/config.ts | 216 --- src/cursor.ts | 107 -- src/doctor.ts | 244 --- src/formatters/index.ts | 6 - src/formatters/json.ts | 11 - src/formatters/markdown.ts | 218 --- src/formatters/pretty.ts | 301 ---- src/formatters/watch.ts | 13 - src/index.ts | 567 ------- src/input.ts | 40 - src/preprocessing/credential.ts | 22 - src/preprocessing/index.ts | 8 - src/preprocessing/patterns.ts | 182 --- src/preprocessing/pipeline.ts | 26 - src/preprocessing/post.ts | 232 --- src/preprocessing/sanitize.ts | 37 - src/preprocessing/secrets.ts | 149 -- src/types.ts | 362 ----- src/utils/colors.ts | 57 - src/utils/date.ts | 91 -- src/utils/index.ts | 4 - src/utils/threading.ts | 60 - src/utils/unread.ts | 31 - src/validation.ts | 5 - tests/api/channels.test.ts | 134 -- tests/api/client.test.ts | 339 ----- tests/api/messages.test.ts | 134 -- tests/api/paths.test.ts | 56 - tests/api/posts.test.ts | 79 - tests/api/retrieval.test.ts | 762 ---------- tests/api/url.test.ts | 73 - tests/api/websocket.test.ts | 492 ------ tests/channels-type-validation.test.ts | 365 ----- tests/cli-empty-reads.test.ts | 340 ----- tests/cli-failure-propagation.test.ts | 132 -- tests/cli-metadata.test.ts | 601 -------- tests/cli-output.test.ts | 13 - tests/cli-retrieval.test.ts | 1049 ------------- tests/config-doctor.test.ts | 262 ---- tests/cursor-cli.test.ts | 389 ----- tests/cursor-commander.test.ts | 42 - tests/cursor.test.ts | 52 - tests/e2e/send-live.e2e.ts | 130 -- tests/fixtures/markdown.ts | 55 - tests/formatters/headers.test.ts | 281 ---- tests/formatters/watch.test.ts | 30 - tests/group-dms.test.ts | 285 ---- tests/helpers/fake-fetch.ts | 32 - tests/identity-teams.test.ts | 209 --- tests/input.test.ts | 76 - tests/preprocessing/post.test.ts | 275 ---- tests/preprocessing/sanitize.test.ts | 112 -- tests/preprocessing/secrets.test.ts | 237 --- tests/send-commander.test.ts | 53 - tests/send.test.ts | 389 ----- tests/setup.ts | 4 - tests/users.test.ts | 194 --- tests/utils/date.test.ts | 89 -- tests/utils/threading.test.ts | 75 - tests/utils/unread.test.ts | 60 - tests/validation.test.ts | 28 - tsconfig.json | 29 - vitest.config.ts | 13 - vitest.e2e.config.ts | 10 - 82 files changed, 4 insertions(+), 14950 deletions(-) delete mode 100644 biome.json delete mode 100644 bun.lock delete mode 100644 package.json delete mode 100644 scripts/check-version.mjs delete mode 100644 scripts/test-e2e.mjs delete mode 100644 src/api/channels.ts delete mode 100644 src/api/client.ts delete mode 100644 src/api/index.ts delete mode 100644 src/api/posts.ts delete mode 100644 src/api/url.ts delete mode 100644 src/api/users.ts delete mode 100644 src/api/websocket.ts delete mode 100644 src/cli.ts delete mode 100644 src/config.ts delete mode 100644 src/cursor.ts delete mode 100644 src/doctor.ts delete mode 100644 src/formatters/index.ts delete mode 100644 src/formatters/json.ts delete mode 100644 src/formatters/markdown.ts delete mode 100644 src/formatters/pretty.ts delete mode 100644 src/formatters/watch.ts delete mode 100755 src/index.ts delete mode 100644 src/input.ts delete mode 100644 src/preprocessing/credential.ts delete mode 100644 src/preprocessing/index.ts delete mode 100644 src/preprocessing/patterns.ts delete mode 100644 src/preprocessing/pipeline.ts delete mode 100644 src/preprocessing/post.ts delete mode 100644 src/preprocessing/sanitize.ts delete mode 100644 src/preprocessing/secrets.ts delete mode 100644 src/types.ts delete mode 100644 src/utils/colors.ts delete mode 100644 src/utils/date.ts delete mode 100644 src/utils/index.ts delete mode 100644 src/utils/threading.ts delete mode 100644 src/utils/unread.ts delete mode 100644 src/validation.ts delete mode 100644 tests/api/channels.test.ts delete mode 100644 tests/api/client.test.ts delete mode 100644 tests/api/messages.test.ts delete mode 100644 tests/api/paths.test.ts delete mode 100644 tests/api/posts.test.ts delete mode 100644 tests/api/retrieval.test.ts delete mode 100644 tests/api/url.test.ts delete mode 100644 tests/api/websocket.test.ts delete mode 100644 tests/channels-type-validation.test.ts delete mode 100644 tests/cli-empty-reads.test.ts delete mode 100644 tests/cli-failure-propagation.test.ts delete mode 100644 tests/cli-metadata.test.ts delete mode 100644 tests/cli-output.test.ts delete mode 100644 tests/cli-retrieval.test.ts delete mode 100644 tests/config-doctor.test.ts delete mode 100644 tests/cursor-cli.test.ts delete mode 100644 tests/cursor-commander.test.ts delete mode 100644 tests/cursor.test.ts delete mode 100644 tests/e2e/send-live.e2e.ts delete mode 100644 tests/fixtures/markdown.ts delete mode 100644 tests/formatters/headers.test.ts delete mode 100644 tests/formatters/watch.test.ts delete mode 100644 tests/group-dms.test.ts delete mode 100644 tests/helpers/fake-fetch.ts delete mode 100644 tests/identity-teams.test.ts delete mode 100644 tests/input.test.ts delete mode 100644 tests/preprocessing/post.test.ts delete mode 100644 tests/preprocessing/sanitize.test.ts delete mode 100644 tests/preprocessing/secrets.test.ts delete mode 100644 tests/send-commander.test.ts delete mode 100644 tests/send.test.ts delete mode 100644 tests/setup.ts delete mode 100644 tests/users.test.ts delete mode 100644 tests/utils/date.test.ts delete mode 100644 tests/utils/threading.test.ts delete mode 100644 tests/utils/unread.test.ts delete mode 100644 tests/validation.test.ts delete mode 100644 tsconfig.json delete mode 100644 vitest.config.ts delete mode 100644 vitest.e2e.config.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d166057..a9e97b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,41 +27,6 @@ jobs: - run: go build -o "$RUNNER_TEMP/mm" ./cmd/mm - run: git diff --check - verify: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 - with: - bun-version: 1.3.14 - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: 22 - - run: bun install --frozen-lockfile - - run: bun run verify - - run: bun audit - - name: Pack and verify contents - id: pack - shell: bash - run: | - tarball=$(npm pack --json --ignore-scripts | node -e 'let input=""; process.stdin.on("data", c => input += c); process.stdin.on("end", () => process.stdout.write(JSON.parse(input)[0].filename))') - printf 'tarball=%s\n' "$tarball" >> "$GITHUB_OUTPUT" - actual=$(tar -tzf "$tarball" | LC_ALL=C sort) - expected=$(printf '%s\n' package/LICENSE package/README.md package/dist/index.js package/package.json | LC_ALL=C sort) - diff -u <(printf '%s\n' "$expected") <(printf '%s\n' "$actual") - - name: Smoke packed CLI on Node 22 - run: | - mkdir package-smoke - tar -xzf "${{ steps.pack.outputs.tarball }}" -C package-smoke - test "$(node package-smoke/package/dist/index.js --version)" = "$(node -p "require('./package.json').version")" - node package-smoke/package/dist/index.js --help - node package-smoke/package/dist/index.js config --path - - name: Smoke global npm install - run: | - npm install --global --prefix "$RUNNER_TEMP/npm-global" "./${{ steps.pack.outputs.tarball }}" - test "$("$RUNNER_TEMP/npm-global/bin/mm" --version)" = "$(node -p "require('./package.json').version")" - "$RUNNER_TEMP/npm-global/bin/mm" --help - distribution: runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index ef6ccff..ebe8176 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,7 @@ -# dependencies (bun install) -node_modules - # output out dist +npm-dist bin !npm/bin/ !npm/bin/mm.js @@ -29,9 +27,7 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json .env.local # caches -.eslintcache .cache -*.tsbuildinfo # IntelliJ based IDEs .idea @@ -47,6 +43,3 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json # Claude Code .claude/ - -# Bun build artifacts -*.bun-build diff --git a/biome.json b/biome.json deleted file mode 100644 index 400028f..0000000 --- a/biome.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "$schema": "https://biomejs.dev/schemas/2.4.4/schema.json", - "root": true, - "vcs": { - "enabled": true, - "clientKind": "git", - "useIgnoreFile": true, - "defaultBranch": "main" - }, - "files": { - "includes": ["src/**/*.ts", "tests/**/*.ts"] - }, - "formatter": { - "enabled": true, - "indentStyle": "space", - "indentWidth": 2, - "lineWidth": 100, - "lineEnding": "lf" - }, - "linter": { - "enabled": true, - "rules": { - "recommended": true, - "correctness": { - "noUnusedVariables": "error", - "noUnusedImports": "error" - }, - "style": { - "useConst": "warn", - "useNodejsImportProtocol": "error", - "noNonNullAssertion": "warn", - "useImportType": "error" - }, - "suspicious": { - "noExplicitAny": "warn" - } - } - }, - "javascript": { - "formatter": { - "quoteStyle": "single", - "trailingCommas": "all", - "semicolons": "asNeeded" - } - }, - "json": { - "formatter": { - "indentStyle": "space", - "indentWidth": 2 - } - }, - "overrides": [ - { - "includes": ["**/*.test.ts"], - "linter": { - "rules": { - "suspicious": { - "noExplicitAny": "off" - } - } - } - } - ] -} diff --git a/bun.lock b/bun.lock deleted file mode 100644 index f42c994..0000000 --- a/bun.lock +++ /dev/null @@ -1,205 +0,0 @@ -{ - "lockfileVersion": 1, - "configVersion": 1, - "workspaces": { - "": { - "name": "mattermost-cli", - "dependencies": { - "commander": "^14.0.2", - "is-ai-agent": "^0.1.0", - "smol-toml": "1.7.0", - }, - "devDependencies": { - "@biomejs/biome": "^2.4.4", - "@types/bun": "1.3.14", - "typescript": "5.9.3", - "vitest": "4.1.10", - }, - }, - }, - "packages": { - "@biomejs/biome": ["@biomejs/biome@2.4.4", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.4.4", "@biomejs/cli-darwin-x64": "2.4.4", "@biomejs/cli-linux-arm64": "2.4.4", "@biomejs/cli-linux-arm64-musl": "2.4.4", "@biomejs/cli-linux-x64": "2.4.4", "@biomejs/cli-linux-x64-musl": "2.4.4", "@biomejs/cli-win32-arm64": "2.4.4", "@biomejs/cli-win32-x64": "2.4.4" }, "bin": { "biome": "bin/biome" } }, "sha512-tigwWS5KfJf0cABVd52NVaXyAVv4qpUXOWJ1rxFL8xF1RVoeS2q/LK+FHgYoKMclJCuRoCWAPy1IXaN9/mS61Q=="], - - "@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.4.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-jZ+Xc6qvD6tTH5jM6eKX44dcbyNqJHssfl2nnwT6vma6B1sj7ZLTGIk6N5QwVBs5xGN52r3trk5fgd3sQ9We9A=="], - - "@biomejs/cli-darwin-x64": ["@biomejs/cli-darwin-x64@2.4.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-Dh1a/+W+SUCXhEdL7TiX3ArPTFCQKJTI1mGncZNWfO+6suk+gYA4lNyJcBB+pwvF49uw0pEbUS49BgYOY4hzUg=="], - - "@biomejs/cli-linux-arm64": ["@biomejs/cli-linux-arm64@2.4.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-V/NFfbWhsUU6w+m5WYbBenlEAz8eYnSqRMDMAW3K+3v0tYVkNyZn8VU0XPxk/lOqNXLSCCrV7FmV/u3SjCBShg=="], - - "@biomejs/cli-linux-arm64-musl": ["@biomejs/cli-linux-arm64-musl@2.4.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-+sPAXq3bxmFwhVFJnSwkSF5Rw2ZAJMH3MF6C9IveAEOdSpgajPhoQhbbAK12SehN9j2QrHpk4J/cHsa/HqWaYQ=="], - - "@biomejs/cli-linux-x64": ["@biomejs/cli-linux-x64@2.4.4", "", { "os": "linux", "cpu": "x64" }, "sha512-R4+ZCDtG9kHArasyBO+UBD6jr/FcFCTH8QkNTOCu0pRJzCWyWC4EtZa2AmUZB5h3e0jD7bRV2KvrENcf8rndBg=="], - - "@biomejs/cli-linux-x64-musl": ["@biomejs/cli-linux-x64-musl@2.4.4", "", { "os": "linux", "cpu": "x64" }, "sha512-gGvFTGpOIQDb5CQ2VC0n9Z2UEqlP46c4aNgHmAMytYieTGEcfqhfCFnhs6xjt0S3igE6q5GLuIXtdQt3Izok+g=="], - - "@biomejs/cli-win32-arm64": ["@biomejs/cli-win32-arm64@2.4.4", "", { "os": "win32", "cpu": "arm64" }, "sha512-trzCqM7x+Gn832zZHgr28JoYagQNX4CZkUZhMUac2YxvvyDRLJDrb5m9IA7CaZLlX6lTQmADVfLEKP1et1Ma4Q=="], - - "@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.4.4", "", { "os": "win32", "cpu": "x64" }, "sha512-gnOHKVPFAAPrpoPt2t+Q6FZ7RPry/FDV3GcpU53P3PtLNnQjBmKyN2Vh/JtqXet+H4pme8CC76rScwdjDcT1/A=="], - - "@emnapi/core": ["@emnapi/core@1.11.1", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.2", "tslib": "^2.4.0" } }, "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ=="], - - "@emnapi/runtime": ["@emnapi/runtime@1.11.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw=="], - - "@emnapi/wasi-threads": ["@emnapi/wasi-threads@1.2.2", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA=="], - - "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.5", "", {}, "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="], - - "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.1.6", "", { "dependencies": { "@tybys/wasm-util": "^0.10.3" }, "peerDependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1" } }, "sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg=="], - - "@oxc-project/types": ["@oxc-project/types@0.139.0", "", {}, "sha512-r9gHphtCs+1M7J0pw6Sn/hh/Wpa/iQrOOkrNAlVLF/gHq+/CJmHIWKKUUhdWjcD6CIa8idarspCsASiXCXvFUw=="], - - "@rolldown/binding-android-arm64": ["@rolldown/binding-android-arm64@1.1.5", "", { "os": "android", "cpu": "arm64" }, "sha512-lZg8fqIv2v7FF237bwMgzGZEJvGL79/s5knJ/i6FmsGF4XXlzccZ4jb+TrFIxtSSxFtIpdsgrPZeMk1I9AFcyQ=="], - - "@rolldown/binding-darwin-arm64": ["@rolldown/binding-darwin-arm64@1.1.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-51Bnx9pNiMRKSUNtBfySkNJ9vMU9Hh3I1ozDd6gyPPYzaXCfnptUcEZxXGYFn+ul2dtcMUiqGR1Yai2K10uoTw=="], - - "@rolldown/binding-darwin-x64": ["@rolldown/binding-darwin-x64@1.1.5", "", { "os": "darwin", "cpu": "x64" }, "sha512-Tm+gbfC0aHu1tBA/JvKQh32S0K6YgCHkiAF4/W6xX0K0RmNuc94VeK419dJoE65R5aRxmo+noZQSWrAMF6yb6g=="], - - "@rolldown/binding-freebsd-x64": ["@rolldown/binding-freebsd-x64@1.1.5", "", { "os": "freebsd", "cpu": "x64" }, "sha512-JMzDKCCXq93YccG5gz3hvOs1oXRKAf0XYpfOS88e+wZrC8Iugj6j68867vrYZkvpDDpKn/KoKORThmchMpF6TA=="], - - "@rolldown/binding-linux-arm-gnueabihf": ["@rolldown/binding-linux-arm-gnueabihf@1.1.5", "", { "os": "linux", "cpu": "arm" }, "sha512-uML21j2K5TfPGutKxub+M+nLjZIrWjXQ5Grx4lCe/nimTj9B4L63zHpjXLl4y0L3mcm2htEQIb06oCG/szerNw=="], - - "@rolldown/binding-linux-arm64-gnu": ["@rolldown/binding-linux-arm64-gnu@1.1.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-navSiuTMogvnQoZoM/v+l3ZWo50/NTwSHSzheABx/RCnmUPaKwq9qSo4Br2OYRs21+Fz8uFqITZM3H4opOB0/Q=="], - - "@rolldown/binding-linux-arm64-musl": ["@rolldown/binding-linux-arm64-musl@1.1.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-lAryqH7IteztmCXQXk0etKj4wBQ7Gx5S6LjKhsgp9zb8I5bsuvU/2llH1hDQcjsFeqIsovMVN339/8pUDDBXxA=="], - - "@rolldown/binding-linux-ppc64-gnu": ["@rolldown/binding-linux-ppc64-gnu@1.1.5", "", { "os": "linux", "cpu": "ppc64" }, "sha512-fsK/sNBnxzBlL4O1JNrZakVQxPspqpED5dLtNsZS9oOKmtSpdNIzxH2kkol5HYTWJN47sE20ztMJPxfZ89qGOg=="], - - "@rolldown/binding-linux-s390x-gnu": ["@rolldown/binding-linux-s390x-gnu@1.1.5", "", { "os": "linux", "cpu": "s390x" }, "sha512-gLYb4BIadlfTOYT5gO503n8zQjXflgzpD0FcyKh0Mzx3rqCZKnHoJWV9xe1KXUJ5lx2JfcSHr/mhzS0PC/McAA=="], - - "@rolldown/binding-linux-x64-gnu": ["@rolldown/binding-linux-x64-gnu@1.1.5", "", { "os": "linux", "cpu": "x64" }, "sha512-FjcpEKUyJygHgs1o50VYNvkt5+7Le/VEdYt0AkRpkL33MnyQfwr8l5mXwMmfmTbyMPr5vJLC+8/Gd9gXnwU1QQ=="], - - "@rolldown/binding-linux-x64-musl": ["@rolldown/binding-linux-x64-musl@1.1.5", "", { "os": "linux", "cpu": "x64" }, "sha512-Me+PfPI2TMeOQk0gYWfLQZtTktrmzbr8cDboqX83XKc7UrgAi55gF+2dUkWdxd19n55Essp2yeca+O9N5rBxHg=="], - - "@rolldown/binding-openharmony-arm64": ["@rolldown/binding-openharmony-arm64@1.1.5", "", { "os": "none", "cpu": "arm64" }, "sha512-yc5WrLzXks6zCQfn9Oxr8pORKyl/pF+QjHmW/Qx3qu0oyrrNC+y2JLTU1E2rcWYAmzlnqngWXHQjy51VzW70Vw=="], - - "@rolldown/binding-wasm32-wasi": ["@rolldown/binding-wasm32-wasi@1.1.5", "", { "dependencies": { "@emnapi/core": "1.11.1", "@emnapi/runtime": "1.11.1", "@napi-rs/wasm-runtime": "^1.1.6" }, "cpu": "none" }, "sha512-VbQGPX2b4r48TAMIM2cjgluIM1HYutm4pcTEJsle7iEP7sB1dFqtPLBVbdLAZCxy1txCcPxf4QFf4v8uvltPqA=="], - - "@rolldown/binding-win32-arm64-msvc": ["@rolldown/binding-win32-arm64-msvc@1.1.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-gHv82k63z4qpV5+Q1y/12KrK0ltWBukVDI8nZcbT7Tt/ZlOIVwppazneq0F93oDxTo3IgAMEDIoQh3E2n6mVsw=="], - - "@rolldown/binding-win32-x64-msvc": ["@rolldown/binding-win32-x64-msvc@1.1.5", "", { "os": "win32", "cpu": "x64" }, "sha512-tTZuDBPw85tEN5PQi1pnEBzDy0Z49HtScLAbD5t6hyeU92A95pRWaSMw1GZZi/RwgSgUIl0xrSlXIT/9QzvYSA=="], - - "@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.1", "", {}, "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw=="], - - "@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="], - - "@tybys/wasm-util": ["@tybys/wasm-util@0.10.3", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg=="], - - "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], - - "@types/chai": ["@types/chai@5.2.3", "", { "dependencies": { "@types/deep-eql": "*", "assertion-error": "^2.0.1" } }, "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA=="], - - "@types/deep-eql": ["@types/deep-eql@4.0.2", "", {}, "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw=="], - - "@types/estree": ["@types/estree@1.0.9", "", {}, "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg=="], - - "@types/node": ["@types/node@25.1.0", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-t7frlewr6+cbx+9Ohpl0NOTKXZNV9xHRmNOvql47BFJKcEG1CxtxlPEEe+gR9uhVWM4DwhnvTF110mIL4yP9RA=="], - - "@vitest/expect": ["@vitest/expect@4.1.10", "", { "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", "@vitest/spy": "4.1.10", "@vitest/utils": "4.1.10", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" } }, "sha512-YsCn+qAk1GWjQOWFEsEcL2gNQ0zmVmQu3T03qP6UyjhtmdtwtbuI+DASn/7iQB3HGTXkdBwGddzxPlmiql5vlA=="], - - "@vitest/mocker": ["@vitest/mocker@4.1.10", "", { "dependencies": { "@vitest/spy": "4.1.10", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, "peerDependencies": { "msw": "^2.4.9", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" }, "optionalPeers": ["msw", "vite"] }, "sha512-v0xaezt+DKEmKfaxg133ldzADrwLGd7Ze1MfQQTYfvs8OqZIwbxyxaYURivwV7sWy5fqn3rH5uOrSp07bp44Ow=="], - - "@vitest/pretty-format": ["@vitest/pretty-format@4.1.10", "", { "dependencies": { "tinyrainbow": "^3.1.0" } }, "sha512-W1HsjSH4MXQ9YfmmhLAoIYf1HRfekQCGngeIgcei6MP5QQGWUe0gkopdZQaVCFO+JDJMrAJGwa5pRpNpvy4P8Q=="], - - "@vitest/runner": ["@vitest/runner@4.1.10", "", { "dependencies": { "@vitest/utils": "4.1.10", "pathe": "^2.0.3" } }, "sha512-IKI6kpIH+LmpROplyLwBBaCfMgOZOMsygVa6BARD6ahA04VRuJSa6OaVG7kRvSEMD870Vd91rSSw0eegtWyLGg=="], - - "@vitest/snapshot": ["@vitest/snapshot@4.1.10", "", { "dependencies": { "@vitest/pretty-format": "4.1.10", "@vitest/utils": "4.1.10", "magic-string": "^0.30.21", "pathe": "^2.0.3" } }, "sha512-xRkfOT1qpTAi/Ti4Y1LtfRc3kEuqxGw59eN2jN9pRWMtS/XDevekhcFSqvQqjUNGksfjMJu3Y+oJ+4Ypn2OaJw=="], - - "@vitest/spy": ["@vitest/spy@4.1.10", "", {}, "sha512-PLf/Ugvoq5wO/b4rwYCR1h2PSIdXz7wnkQFMiUpLdtM7l6pqVFcQIBEHyT1+l+cj7mNwAfZHzqXqDyjvOuwbDw=="], - - "@vitest/utils": ["@vitest/utils@4.1.10", "", { "dependencies": { "@vitest/pretty-format": "4.1.10", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" } }, "sha512-fy9am/HWxbaGt/Sawrp90vt6Y6jQwf1RX77cz3uwoJwJVMli/e1IEwRPnMNJ7vKfPTwo0diXifkpPvwH9v7nGA=="], - - "assertion-error": ["assertion-error@2.0.1", "", {}, "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA=="], - - "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], - - "chai": ["chai@6.2.2", "", {}, "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg=="], - - "commander": ["commander@14.0.2", "", {}, "sha512-TywoWNNRbhoD0BXs1P3ZEScW8W5iKrnbithIl0YH+uCmBd0QpPOA8yc82DS3BIE5Ma6FnBVUsJ7wVUDz4dvOWQ=="], - - "convert-source-map": ["convert-source-map@2.0.0", "", {}, "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="], - - "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], - - "es-module-lexer": ["es-module-lexer@2.3.1", "", {}, "sha512-shc1dbU90Yl/xq1QrC7QRtfcwURZuVRfPhZbDoldJ1cn1gzDvBaBWlv0eFolj5+0znnPJz5TXLxsN77X/12KTA=="], - - "estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], - - "expect-type": ["expect-type@1.4.0", "", {}, "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA=="], - - "fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], - - "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], - - "is-ai-agent": ["is-ai-agent@0.1.0", "", { "peerDependencies": { "typescript": "^5" } }, "sha512-Qo2fUHEm3TF3z95mcLDyei/vdmyxgsuWmn5vt5gw2C8ClintP8NmHuSg0ujcnxvo6l9SELi10xR4X2yEWJzkZQ=="], - - "lightningcss": ["lightningcss@1.32.0", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.32.0", "lightningcss-darwin-arm64": "1.32.0", "lightningcss-darwin-x64": "1.32.0", "lightningcss-freebsd-x64": "1.32.0", "lightningcss-linux-arm-gnueabihf": "1.32.0", "lightningcss-linux-arm64-gnu": "1.32.0", "lightningcss-linux-arm64-musl": "1.32.0", "lightningcss-linux-x64-gnu": "1.32.0", "lightningcss-linux-x64-musl": "1.32.0", "lightningcss-win32-arm64-msvc": "1.32.0", "lightningcss-win32-x64-msvc": "1.32.0" } }, "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ=="], - - "lightningcss-android-arm64": ["lightningcss-android-arm64@1.32.0", "", { "os": "android", "cpu": "arm64" }, "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg=="], - - "lightningcss-darwin-arm64": ["lightningcss-darwin-arm64@1.32.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ=="], - - "lightningcss-darwin-x64": ["lightningcss-darwin-x64@1.32.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w=="], - - "lightningcss-freebsd-x64": ["lightningcss-freebsd-x64@1.32.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig=="], - - "lightningcss-linux-arm-gnueabihf": ["lightningcss-linux-arm-gnueabihf@1.32.0", "", { "os": "linux", "cpu": "arm" }, "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw=="], - - "lightningcss-linux-arm64-gnu": ["lightningcss-linux-arm64-gnu@1.32.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ=="], - - "lightningcss-linux-arm64-musl": ["lightningcss-linux-arm64-musl@1.32.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg=="], - - "lightningcss-linux-x64-gnu": ["lightningcss-linux-x64-gnu@1.32.0", "", { "os": "linux", "cpu": "x64" }, "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA=="], - - "lightningcss-linux-x64-musl": ["lightningcss-linux-x64-musl@1.32.0", "", { "os": "linux", "cpu": "x64" }, "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg=="], - - "lightningcss-win32-arm64-msvc": ["lightningcss-win32-arm64-msvc@1.32.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw=="], - - "lightningcss-win32-x64-msvc": ["lightningcss-win32-x64-msvc@1.32.0", "", { "os": "win32", "cpu": "x64" }, "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q=="], - - "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], - - "nanoid": ["nanoid@3.3.16", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q=="], - - "obug": ["obug@2.1.3", "", {}, "sha512-9miFgM2OFba7hB+pRgvtV84pYTBaoTHohvmIgiRt6dRIzbwEOIaNaP+dIlGs2fNFoB0SeISs0Jz5WFVRid6Xyg=="], - - "pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], - - "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], - - "picomatch": ["picomatch@4.0.5", "", {}, "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A=="], - - "postcss": ["postcss@8.5.19", "", { "dependencies": { "nanoid": "^3.3.12", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-Mz8SaolMd8nB+G13WkORcxQKHZ/NE4xXevtkJHVuG+guo9/wYKlIMTKAqGdEmYOXR2ijPjTYNHssizdaVSUNdQ=="], - - "rolldown": ["rolldown@1.1.5", "", { "dependencies": { "@oxc-project/types": "=0.139.0", "@rolldown/pluginutils": "^1.0.0" }, "optionalDependencies": { "@rolldown/binding-android-arm64": "1.1.5", "@rolldown/binding-darwin-arm64": "1.1.5", "@rolldown/binding-darwin-x64": "1.1.5", "@rolldown/binding-freebsd-x64": "1.1.5", "@rolldown/binding-linux-arm-gnueabihf": "1.1.5", "@rolldown/binding-linux-arm64-gnu": "1.1.5", "@rolldown/binding-linux-arm64-musl": "1.1.5", "@rolldown/binding-linux-ppc64-gnu": "1.1.5", "@rolldown/binding-linux-s390x-gnu": "1.1.5", "@rolldown/binding-linux-x64-gnu": "1.1.5", "@rolldown/binding-linux-x64-musl": "1.1.5", "@rolldown/binding-openharmony-arm64": "1.1.5", "@rolldown/binding-wasm32-wasi": "1.1.5", "@rolldown/binding-win32-arm64-msvc": "1.1.5", "@rolldown/binding-win32-x64-msvc": "1.1.5" }, "bin": { "rolldown": "./bin/cli.mjs" } }, "sha512-t9z29cJjXf/vxQ8dyhCSpt6H6aSwHTk8cT5I3iy6SMXuFpk5mB6PL6XfC8PCwrPTx93udwKUm9HRteAlTGBLiA=="], - - "siginfo": ["siginfo@2.0.0", "", {}, "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g=="], - - "smol-toml": ["smol-toml@1.7.0", "", {}, "sha512-aqVvWoyO21L23mb+drl4RmMXbf6N7FdHjAhTRA9ZBL7apWBgfWC16KjrASI+1p9GAroljyMHj6fK67i0UiTNvQ=="], - - "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], - - "stackback": ["stackback@0.0.2", "", {}, "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw=="], - - "std-env": ["std-env@4.2.0", "", {}, "sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw=="], - - "tinybench": ["tinybench@2.9.0", "", {}, "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg=="], - - "tinyexec": ["tinyexec@1.2.4", "", {}, "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg=="], - - "tinyglobby": ["tinyglobby@0.2.17", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.4" } }, "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g=="], - - "tinyrainbow": ["tinyrainbow@3.1.0", "", {}, "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw=="], - - "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], - - "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], - - "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], - - "vite": ["vite@8.1.4", "", { "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.5", "postcss": "^8.5.16", "rolldown": "~1.1.4", "tinyglobby": "^0.2.17" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "@vitejs/devtools": "^0.3.0", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "@vitejs/devtools", "esbuild", "jiti", "less", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-bTT9PsdWO+MQMNG9ZXIP/qM9wGh37DFxTV/sPq9cFpHr3w4jkgef032PkAL9jAqhk3Nz8NQw3O8n6/xFkqO4QQ=="], - - "vitest": ["vitest@4.1.10", "", { "dependencies": { "@vitest/expect": "4.1.10", "@vitest/mocker": "4.1.10", "@vitest/pretty-format": "4.1.10", "@vitest/runner": "4.1.10", "@vitest/snapshot": "4.1.10", "@vitest/spy": "4.1.10", "@vitest/utils": "4.1.10", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", "obug": "^2.1.1", "pathe": "^2.0.3", "picomatch": "^4.0.3", "std-env": "^4.0.0-rc.1", "tinybench": "^2.9.0", "tinyexec": "^1.0.2", "tinyglobby": "^0.2.15", "tinyrainbow": "^3.1.0", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", "why-is-node-running": "^2.3.0" }, "peerDependencies": { "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", "@vitest/browser-playwright": "4.1.10", "@vitest/browser-preview": "4.1.10", "@vitest/browser-webdriverio": "4.1.10", "@vitest/coverage-istanbul": "4.1.10", "@vitest/coverage-v8": "4.1.10", "@vitest/ui": "4.1.10", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@opentelemetry/api", "@types/node", "@vitest/browser-playwright", "@vitest/browser-preview", "@vitest/browser-webdriverio", "@vitest/coverage-istanbul", "@vitest/coverage-v8", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "./vitest.mjs" } }, "sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw=="], - - "why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="], - } -} diff --git a/internal/buildinfo/buildinfo.go b/internal/buildinfo/buildinfo.go index d1f2b01..381186a 100644 --- a/internal/buildinfo/buildinfo.go +++ b/internal/buildinfo/buildinfo.go @@ -1,6 +1,6 @@ package buildinfo var ( - Version = "2.0.0-dev" + Version = "2.0.0" Commit = "dev" ) diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index be0e4c0..b4982d9 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -20,7 +20,7 @@ func TestExecuteVersion(t *testing.T) { if code != 0 { t.Fatalf("exit code = %d, want 0", code) } - if got, want := stdout.String(), "mm version 2.0.0-dev (dev)\n"; got != want { + if got, want := stdout.String(), "mm version 2.0.0 (dev)\n"; got != want { t.Fatalf("stdout = %q, want %q", got, want) } if stderr.Len() != 0 { diff --git a/justfile b/justfile index e3c94ce..da117af 100644 --- a/justfile +++ b/justfile @@ -46,17 +46,7 @@ npm-packages version release_dir="dist" output="npm-dist": docker-e2e: go run ./scripts/e2e -oracle-smoke: - git diff --quiet v1.6.0 -- src package.json bun.lock tsconfig.json - go run ./cmd/conformance --scenario conformance/scenarios/v1/whoami.json --cwd . -- bun src/index.ts - -parity-smoke: - @tmp="$(mktemp -d "${TMPDIR:-/tmp}/mattermost-cli-parity.XXXXXX")"; trap 'find "$tmp" -type f -delete; rmdir "$tmp"' EXIT; go build -o "$tmp/mm" ./cmd/mm; for scenario in conformance/scenarios/pairs/*.json; do go run ./cmd/conformance --pair "$scenario" --cwd . --oracle bun --oracle-prefix src/index.ts --candidate "$tmp/mm"; done - go-gate: go-format-check go-test go-race go-vet go-staticcheck go-vuln go-licenses go-modules go-build go-e2e-compile go-cross-build git diff --check -legacy-gate: - bun run verify - -gate: go-gate legacy-gate oracle-smoke parity-smoke +gate: go-gate diff --git a/package.json b/package.json deleted file mode 100644 index f149d37..0000000 --- a/package.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "name": "mattermost-cli", - "version": "1.6.0", - "description": "Mattermost CLI for safe message retrieval, live watch, and deliberate DM/group sending", - "author": "Arda Sevinc ", - "license": "MIT", - "packageManager": "bun@1.3.14", - "repository": { - "type": "git", - "url": "https://github.com/ardasevinc/mattermost-cli" - }, - "keywords": ["mattermost", "cli", "dm", "messages", "bun", "secret-redaction"], - "main": "dist/index.js", - "type": "module", - "bin": { - "mm": "dist/index.js" - }, - "files": [ - "dist", - "LICENSE", - "README.md" - ], - "engines": { - "node": ">=22.0.0" - }, - "scripts": { - "start": "bun src/index.ts", - "mm": "bun src/index.ts", - "check": "biome check .", - "check:fix": "biome check --write .", - "format": "biome format --write .", - "lint": "biome lint .", - "lint:fix": "biome lint --write .", - "typecheck": "tsc --noEmit", - "check:version": "node scripts/check-version.mjs", - "test": "vitest run", - "test:e2e": "bun run build && node scripts/test-e2e.mjs", - "test:watch": "vitest", - "build": "bun build src/index.ts --target node --outfile dist/index.js --minify && node -e \"const fs=require('fs');const f='dist/index.js';const c=fs.readFileSync(f,'utf8').replace(/^#!.*\\n/,'');fs.writeFileSync(f,'#!/usr/bin/env node\\n'+c)\" && (chmod +x dist/index.js || true)", - "verify": "bun run check && bun run typecheck && bun run test && bun run check:version && bun run build", - "prepack": "bun run build && bun run check:version", - "prepublishOnly": "bun run verify" - }, - "devDependencies": { - "@biomejs/biome": "^2.4.4", - "@types/bun": "1.3.14", - "typescript": "5.9.3", - "vitest": "4.1.10" - }, - "dependencies": { - "commander": "^14.0.2", - "is-ai-agent": "^0.1.0", - "smol-toml": "1.7.0" - } -} diff --git a/scripts/check-version.mjs b/scripts/check-version.mjs deleted file mode 100644 index d17c493..0000000 --- a/scripts/check-version.mjs +++ /dev/null @@ -1,27 +0,0 @@ -import { readFile } from "node:fs/promises"; - -const packageJson = JSON.parse(await readFile(new URL("../package.json", import.meta.url), "utf8")); -const skill = await readFile(new URL("../skills/mattermost-cli/SKILL.md", import.meta.url), "utf8"); -const skillVersion = skill.match(/^version:\s*([^\s]+)\s*$/m)?.[1]; - -if (!skillVersion) { - throw new Error("Could not find a version in skills/mattermost-cli/SKILL.md"); -} - -if (packageJson.version !== skillVersion) { - throw new Error( - `Version mismatch: package.json is ${packageJson.version}, skill is ${skillVersion}`, - ); -} - -const releaseTag = process.env.RELEASE_TAG; -if (releaseTag) { - const expectedTag = `v${packageJson.version}`; - if (releaseTag !== expectedTag) { - throw new Error( - `Version mismatch: expected release tag ${expectedTag}, got ${releaseTag}`, - ); - } -} - -console.log(`Version check passed: ${packageJson.version}`); diff --git a/scripts/test-e2e.mjs b/scripts/test-e2e.mjs deleted file mode 100644 index 5ff4ad2..0000000 --- a/scripts/test-e2e.mjs +++ /dev/null @@ -1,204 +0,0 @@ -import { spawn } from 'node:child_process' -import { randomUUID } from 'node:crypto' -import { rmSync } from 'node:fs' -import { fileURLToPath } from 'node:url' -import os from 'node:os' -import path from 'node:path' - -const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') -const composeFile = path.join(root, 'tests/e2e/compose.yml') -const project = `mattermost-cli-e2e-${process.pid}-${randomUUID().slice(0, 8)}` -const markerTeam = `mm-e2e-${randomUUID().replaceAll('-', '').slice(0, 16)}` -const requestedPort = process.env.MM_E2E_PORT || '0' -const compose = ['compose', '-p', project, '-f', composeFile] -const goBinary = path.join(os.tmpdir(), `${project}-mm`) -const children = new Set() - -function run(command, args, options = {}) { - return new Promise((resolve, reject) => { - const child = spawn(command, args, { - cwd: root, - stdio: options.capture ? ['ignore', 'pipe', 'pipe'] : 'inherit', - env: { ...process.env, MM_E2E_PORT: requestedPort, ...options.env }, - }) - children.add(child) - let stdout = '' - let stderr = '' - if (options.capture) { - child.stdout.setEncoding('utf8') - child.stderr.setEncoding('utf8') - child.stdout.on('data', (chunk) => { - stdout += chunk - }) - child.stderr.on('data', (chunk) => { - stderr += chunk - }) - } - child.once('error', (error) => { - children.delete(child) - reject(error) - }) - child.once('close', (code, signal) => { - children.delete(child) - if (code === 0) { - resolve(stdout) - return - } - if (options.capture && !options.sensitive) { - if (stdout) process.stderr.write(stdout) - if (stderr) process.stderr.write(stderr) - } - reject(new Error(`${command} exited with status ${code ?? signal ?? 'unknown'}`)) - }) - }) -} - -async function mmctl(args, capture = false, sensitive = false) { - return run( - 'docker', - [...compose, 'exec', '-T', 'mattermost', 'mmctl', '--local', ...args], - { capture, sensitive }, - ) -} - -let cleanupPromise - -function cleanup() { - cleanupPromise ??= cleanupOnce() - return cleanupPromise -} - -async function cleanupOnce() { - await run('docker', [...compose, 'down', '--volumes', '--remove-orphans']) - const containers = (await run('docker', [...compose, 'ps', '-aq'], { capture: true })).trim() - const volumes = ( - await run( - 'docker', - ['volume', 'ls', '-q', '--filter', `label=com.docker.compose.project=${project}`], - { capture: true }, - ) - ).trim() - const networks = ( - await run( - 'docker', - ['network', 'ls', '-q', '--filter', `label=com.docker.compose.project=${project}`], - { capture: true }, - ) - ).trim() - if (containers || volumes || networks) { - throw new Error('Docker E2E cleanup left project resources behind') - } -} - -let terminating = false -for (const [signal, exitCode] of [ - ['SIGINT', 130], - ['SIGTERM', 143], -]) { - process.once(signal, () => { - void terminate(exitCode) - }) -} - -async function terminate(exitCode) { - if (terminating) return - terminating = true - for (const child of children) child.kill('SIGTERM') - try { - await cleanup() - } finally { - rmSync(goBinary, { force: true }) - process.exit(exitCode) - } -} - -try { - await run('docker', [...compose, 'up', '-d', '--wait', '--wait-timeout', '180']) - const published = ( - await run('docker', [...compose, 'port', 'mattermost', '8065'], { - capture: true, - }) - ).trim() - const port = published.match(/:(\d+)$/)?.[1] - if (!port) throw new Error('Docker did not report the Mattermost E2E port') - const url = `http://127.0.0.1:${port}` - - await mmctl([ - '--quiet', - 'user', - 'create', - '--email', - 'sender@example.test', - '--username', - 'sender', - '--password', - 'E2ePassword1!', - '--system-admin', - '--email-verified', - '--disable-welcome-email', - ]) - for (const username of ['alice', 'bob', 'carol', 'dave']) { - await mmctl([ - '--quiet', - 'user', - 'create', - '--email', - `${username}@example.test`, - '--username', - username, - '--password', - 'E2ePassword1!', - '--email-verified', - '--disable-welcome-email', - ]) - } - await mmctl(['--quiet', 'team', 'create', '--name', 'e2e', '--display-name', 'E2E']) - await mmctl([ - '--quiet', - 'team', - 'users', - 'add', - 'e2e', - 'sender', - 'alice', - 'bob', - 'carol', - 'dave', - ]) - await mmctl([ - '--quiet', - 'team', - 'create', - '--name', - markerTeam, - '--display-name', - `Mattermost CLI E2E ${markerTeam}`, - ]) - await mmctl(['--quiet', 'team', 'users', 'add', markerTeam, 'sender']) - const generated = JSON.parse( - await mmctl(['--json', 'token', 'generate', 'sender', 'mattermost-cli-e2e'], true, true), - ) - const token = generated?.[0]?.token - if (typeof token !== 'string' || token.length === 0) { - throw new Error('Mattermost did not return an E2E access token') - } - - await run('bunx', ['vitest', 'run', '--config', 'vitest.e2e.config.ts'], { - env: { MM_E2E_URL: url, MM_E2E_TOKEN: token }, - }) - await run('go', ['build', '-tags=e2e', '-o', goBinary, './cmd/mm']) - await run('go', ['test', '-tags=e2e', '-count=1', './tests/e2e'], { - env: { - MM_E2E_URL: url, - MM_E2E_TOKEN: token, - MM_E2E_BINARY: goBinary, - MM_E2E_MARKER_TEAM: markerTeam, - }, - }) -} finally { - try { - await cleanup() - } finally { - rmSync(goBinary, { force: true }) - } -} diff --git a/src/api/channels.ts b/src/api/channels.ts deleted file mode 100644 index b1a0af3..0000000 --- a/src/api/channels.ts +++ /dev/null @@ -1,197 +0,0 @@ -// Channel fetching with DM filtering - -import { preprocess, sanitizeTerminalLabel } from '../preprocessing' -import type { Channel, ChannelMember, Team } from '../types' -import { getClient, MattermostMutationOutcomeUnknownError } from './client' -import { getMe, getUserByUsername } from './users' - -export interface CanonicalTeam { - id: string - name: string - displayName: string -} - -export function normalizeCanonicalTeams(teams: unknown): CanonicalTeam[] { - if (!Array.isArray(teams)) throw new Error('Invalid teams response.') - return teams.map((team) => { - if (typeof team !== 'object' || team === null || Array.isArray(team)) { - throw new Error('Invalid teams response.') - } - const raw = team as Record - if ( - typeof raw.id !== 'string' || - raw.id.length === 0 || - typeof raw.name !== 'string' || - raw.name.length === 0 || - typeof raw.display_name !== 'string' || - (raw.type !== 'O' && raw.type !== 'I') - ) { - throw new Error('Invalid teams response.') - } - return { id: raw.id, name: raw.name, displayName: raw.display_name } - }) -} - -export async function getMyChannels(userId?: string): Promise { - const resolvedUserId = userId ?? (await getMe()).id - const client = getClient() - - // Get all channels for the user's teams first, then their direct channels - // For DMs, we use the user's direct channels endpoint - const channels = await client.get( - `/users/${encodeURIComponent(resolvedUserId)}/channels`, - ) - - return channels -} - -export async function getMyDMChannels(userId?: string): Promise { - const channels = await getMyChannels(userId) - // Filter for direct messages only (type 'D') - return [...new Map(channels.filter((ch) => ch.type === 'D').map((ch) => [ch.id, ch])).values()] -} - -export async function getMyGroupDMChannels(userId?: string): Promise { - const channels = await getMyChannels(userId) - // Filter for group DMs (type 'G') - return channels.filter((ch) => ch.type === 'G') -} - -export async function getDMChannelWithUser(userId: string): Promise { - const me = await getMe() - const channels = await getMyDMChannels() - - // Find the DM channel with this user by checking the channel name - // DM channel names are "{userId1}__{userId2}" sorted alphabetically - return ( - channels.find((ch) => { - const otherId = getOtherUserIdFromDMChannel(ch, me.id) - return otherId === userId - }) ?? null - ) -} - -export async function getDMChannelByUsername(username: string): Promise { - const user = await getUserByUsername(username) - return getDMChannelWithUser(user.id) -} - -export async function createDirectChannel(myUserId: string, otherUserId: string): Promise { - if (!myUserId.trim() || !otherUserId.trim()) { - throw new Error('Invalid direct-message participants.') - } - const client = getClient() - const channel = await client.post('/channels/direct', [myUserId, otherUserId]) - if ( - typeof channel !== 'object' || - channel === null || - Array.isArray(channel) || - typeof (channel as Record).id !== 'string' || - ((channel as Record).id as string).trim() === '' || - (channel as Record).type !== 'D' || - typeof (channel as Record).name !== 'string' || - typeof (channel as Record).display_name !== 'string' || - (channel as Record).team_id !== '' - ) { - throw new MattermostMutationOutcomeUnknownError() - } - - const directChannel = channel as Channel - if (getOtherUserIdFromDMChannel(directChannel, myUserId) !== otherUserId) { - throw new MattermostMutationOutcomeUnknownError() - } - return directChannel -} - -export async function getChannel(channelId: string): Promise { - const client = getClient() - return client.get(`/channels/${encodeURIComponent(channelId)}`) -} - -export function normalizeChannelName(channelName: string): string { - return channelName.replace(/^#/, '') -} - -export async function getMyTeams(userId?: string): Promise { - const resolvedUserId = userId ?? (await getMe()).id - const client = getClient() - return client.get(`/users/${encodeURIComponent(resolvedUserId)}/teams`) -} - -export async function getChannelByName(teamId: string, channelName: string): Promise { - const client = getClient() - const name = normalizeChannelName(channelName) - return client.get( - `/teams/${encodeURIComponent(teamId)}/channels/name/${encodeURIComponent(name)}`, - ) -} - -export function resolveTeamIdFromList(teams: unknown, teamName?: string): string { - const safeLabel = (value: string) => - sanitizeTerminalLabel(preprocess(value, { redact: false }).text) - const canonical = normalizeCanonicalTeams(teams) - if (canonical.length === 0) { - throw new Error('You are not a member of any teams.') - } - - if (teamName) { - const team = canonical.find((t) => t.name === teamName || t.displayName === teamName) - if (!team) { - throw new Error( - `Team "${safeLabel(teamName)}" not found. Your teams: ${canonical - .map((team) => safeLabel(team.name)) - .join(', ')}`, - ) - } - return team.id - } - - if (canonical.length === 1) { - const [team] = canonical - if (!team) throw new Error('You are not a member of any teams.') - return team.id - } - - throw new Error( - `You belong to multiple teams. Use --team to specify:\n` + - canonical - .map((team) => ` ${safeLabel(team.name)} (${safeLabel(team.displayName)})`) - .join('\n'), - ) -} - -export async function resolveTeamId(teamName?: string): Promise { - return resolveTeamIdFromList(await getMyTeams(), teamName) -} - -export async function getTeamChannelMembers(teamId: string): Promise { - const me = await getMe() - const client = getClient() - return client.get( - `/users/${encodeURIComponent(me.id)}/teams/${encodeURIComponent(teamId)}/channels/members`, - ) -} - -export async function getChannelMember(channelId: string): Promise { - const me = await getMe() - const client = getClient() - return client.get( - `/channels/${encodeURIComponent(channelId)}/members/${encodeURIComponent(me.id)}`, - ) -} - -// Extract the other user's ID from a DM channel name -// DM channel names are formatted as "{userId1}__{userId2}" sorted alphabetically -export function getOtherUserIdFromDMChannel(channel: Channel, myUserId: string): string | null { - if (channel.type !== 'D') return null - - const parts = channel.name.split('__') - if (parts.length !== 2) return null - const [left, right] = parts - if (!left || !right) return null - - if (left === myUserId && right === myUserId) return myUserId - if (left === myUserId) return right - if (right === myUserId) return left - return null -} diff --git a/src/api/client.ts b/src/api/client.ts deleted file mode 100644 index 284d7c0..0000000 --- a/src/api/client.ts +++ /dev/null @@ -1,212 +0,0 @@ -// Base Mattermost API client with auth and error handling - -import { registerActiveMattermostCredential } from '../preprocessing' -import { normalizeServerUrl } from './url' - -export const REQUEST_TIMEOUT_MS = 15_000 -const MAX_RETRY_DELAY_MS = 30_000 -const MAX_RETRIES = 2 - -export function rateLimitDelay(response: Pick, retryCount: number): number { - const retryAfter = response.headers.get('Retry-After') - if (retryAfter) { - const seconds = Number(retryAfter) - const delay = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(retryAfter) - Date.now() - if (Number.isFinite(delay)) return Math.min(Math.max(delay, 0), MAX_RETRY_DELAY_MS) - } - - const resetSeconds = Number(response.headers.get('X-RateLimit-Reset')) - if (Number.isFinite(resetSeconds) && resetSeconds > 0) { - return Math.min(resetSeconds * 1000, MAX_RETRY_DELAY_MS) - } - return Math.min(2 ** retryCount * 1000, MAX_RETRY_DELAY_MS) -} - -function retryDelay(retryCount: number): number { - return Math.min(2 ** retryCount * 1000, MAX_RETRY_DELAY_MS) -} - -function waitForRetry(delay: number, reason: string, attempt: number): Promise { - console.error(`Mattermost request ${reason}; retrying in ${delay}ms (attempt ${attempt + 1}).`) - return new Promise((resolve) => setTimeout(resolve, delay)) -} - -class RequestTimeoutError extends Error {} -class RequestTransportError extends Error {} -class InvalidJSONResponseError extends Error {} - -export class MattermostMutationOutcomeUnknownError extends Error { - constructor() { - super( - 'Mattermost did not confirm the write. Its outcome is unknown; check the destination before retrying.', - ) - this.name = 'MattermostMutationOutcomeUnknownError' - } -} - -interface RequestAttempt { - response: Response - data?: T -} - -export class MattermostClient { - private baseUrl: string - private token: string - constructor(baseUrl: string, token: string) { - this.baseUrl = normalizeServerUrl(baseUrl) - this.token = token - } - - private async attempt( - method: string, - url: string, - body?: unknown, - followRedirects = true, - ): Promise> { - const controller = new AbortController() - const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS) - - try { - let response: Response - try { - response = await fetch(url, { - method, - headers: { - Authorization: `Bearer ${this.token}`, - 'Content-Type': 'application/json', - }, - body: body ? JSON.stringify(body) : undefined, - signal: controller.signal, - redirect: followRedirects ? 'follow' : 'manual', - }) - } catch { - if (controller.signal.aborted) throw new RequestTimeoutError() - throw new RequestTransportError() - } - - if (!response.ok) return { response } - - try { - return { response, data: (await response.json()) as T } - } catch (error) { - if (controller.signal.aborted) throw new RequestTimeoutError() - if (error instanceof TypeError) throw new RequestTransportError() - throw new InvalidJSONResponseError() - } - } finally { - clearTimeout(timeout) - } - } - - async request( - method: string, - path: string, - body?: unknown, - retrySafe = method === 'GET', - retryCount = 0, - ): Promise { - const url = `${this.baseUrl}/api/v4${path}` - let attempt: RequestAttempt - try { - attempt = await this.attempt(method, url, body, retrySafe || method === 'GET') - } catch (error) { - if (error instanceof InvalidJSONResponseError) { - if (!retrySafe && method !== 'GET') throw new MattermostMutationOutcomeUnknownError() - throw new Error('Mattermost returned an invalid JSON response.') - } - if (!(error instanceof RequestTimeoutError || error instanceof RequestTransportError)) { - throw error - } - if (!retrySafe || retryCount >= MAX_RETRIES) { - if (!retrySafe && method !== 'GET') { - throw new MattermostMutationOutcomeUnknownError() - } - if (error instanceof RequestTimeoutError) { - throw new Error(`Mattermost request timed out after ${REQUEST_TIMEOUT_MS}ms.`) - } - throw new Error('Unable to connect to Mattermost due to a network error.') - } - const delay = retryDelay(retryCount) - await waitForRetry( - delay, - error instanceof RequestTimeoutError ? 'timed out' : 'hit a network error', - retryCount, - ) - return this.request(method, path, body, retrySafe, retryCount + 1) - } - const { response } = attempt - - if ( - !retrySafe && - method !== 'GET' && - ((response.status >= 300 && response.status < 400) || response.status >= 500) - ) { - throw new MattermostMutationOutcomeUnknownError() - } - - if (retrySafe && response.status === 429 && retryCount < MAX_RETRIES) { - const delay = rateLimitDelay(response, retryCount) - await waitForRetry(delay, 'was rate limited', retryCount) - return this.request(method, path, body, retrySafe, retryCount + 1) - } - - if (retrySafe && [502, 503, 504].includes(response.status) && retryCount < MAX_RETRIES) { - const delay = retryDelay(retryCount) - await waitForRetry(delay, `received HTTP ${response.status}`, retryCount) - return this.request(method, path, body, retrySafe, retryCount + 1) - } - - if (!response.ok) { - throw new MattermostAPIError(`API request failed: ${response.status}.`, response.status) - } - - return attempt.data as T - } - - get(path: string): Promise { - return this.request('GET', path) - } - - post(path: string, body?: unknown, retrySafe = false): Promise { - return this.request('POST', path, body, retrySafe) - } - - put(path: string, body?: unknown): Promise { - return this.request('PUT', path, body) - } - - delete(path: string): Promise { - return this.request('DELETE', path) - } -} - -export class MattermostAPIError extends Error { - constructor( - message: string, - public status: number, - ) { - super(message) - this.name = 'MattermostAPIError' - } -} - -// Singleton instance -let client: MattermostClient | null = null -let releaseClientCredential: (() => void) | undefined - -export function initClient(baseUrl: string, token: string): MattermostClient { - const candidate = new MattermostClient(baseUrl, token) - const releaseCandidateCredential = registerActiveMattermostCredential(token) - const releasePreviousCredential = releaseClientCredential - client = candidate - releaseClientCredential = releaseCandidateCredential - releasePreviousCredential?.() - return candidate -} - -export function getClient(): MattermostClient { - if (!client) { - throw new Error('Mattermost client not initialized. Call initClient() first.') - } - return client -} diff --git a/src/api/index.ts b/src/api/index.ts deleted file mode 100644 index 6dc04c6..0000000 --- a/src/api/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -// API module exports - -export * from './channels' -export * from './client' -export * from './posts' -export * from './url' -export * from './users' -export * from './websocket' diff --git a/src/api/posts.ts b/src/api/posts.ts deleted file mode 100644 index 845e382..0000000 --- a/src/api/posts.ts +++ /dev/null @@ -1,473 +0,0 @@ -// Post/message fetching with pagination - -import { comparePostIds } from '../cursor' -import { validateMessageContent } from '../input' -import type { Post, PostRetrievalResult, SearchResponse } from '../types' -import { getClient, MattermostMutationOutcomeUnknownError } from './client' - -interface GetPostsOptions { - limit?: number - page?: number - skipFetchThreads?: boolean - before?: string -} - -const MAX_SEARCH_SCAN_PAGES = 100 -const MATTERMOST_ID_PATTERN = /^[a-z0-9]{26}$/ - -function isUrlEncodable(value: string): boolean { - try { - encodeURIComponent(value) - return true - } catch { - return false - } -} - -export interface CreatedPostReceipt { - id: string - channelId: string - userId: string - createAt: number - pendingPostId: string -} - -export async function createPost( - channelId: string, - message: string, - pendingPostId: string = crypto.randomUUID(), -): Promise { - if (!channelId.trim()) throw new Error('A destination channel is required.') - validateMessageContent(message) - if (!pendingPostId.trim()) throw new Error('A pending post ID is required.') - - const post = await getClient().post('/posts', { - channel_id: channelId, - message, - pending_post_id: pendingPostId, - }) - if (typeof post !== 'object' || post === null || Array.isArray(post)) { - throw new MattermostMutationOutcomeUnknownError() - } - const raw = post as Record - if ( - typeof raw.id !== 'string' || - !MATTERMOST_ID_PATTERN.test(raw.id) || - !isUrlEncodable(raw.id) || - raw.channel_id !== channelId || - typeof raw.user_id !== 'string' || - raw.user_id.trim().length === 0 || - typeof raw.create_at !== 'number' || - !Number.isFinite(raw.create_at) || - raw.create_at <= 0 || - Number.isNaN(new Date(raw.create_at).getTime()) - ) { - throw new MattermostMutationOutcomeUnknownError() - } - - return { - id: raw.id, - channelId, - userId: raw.user_id, - createAt: raw.create_at, - pendingPostId, - } -} - -export async function getChannelPosts( - channelId: string, - options: GetPostsOptions = {}, -): Promise<{ - posts: Post[] - rawCount: number - firstInaccessiblePostTime: number | null - hasNext: boolean | null - incompletePayload: boolean -}> { - const { limit = 50, page = 0, skipFetchThreads = true, before } = options - const client = getClient() - - const params = new URLSearchParams() - params.set('per_page', String(Math.min(limit, 200))) // API max is 200 - params.set('page', String(page)) - params.set('skipFetchThreads', String(skipFetchThreads)) - if (before !== undefined) params.set('before', before) - - const rawResponse = await client.get( - `/channels/${encodeURIComponent(channelId)}/posts?${params}`, - ) - - const response = normalizeOrderedPostsPage(rawResponse) - - // Convert posts object to array, sorted by order - const posts = response.order - .map((id) => response.posts[id]) - .filter((post): post is Post => !!post && post.delete_at === 0) // Exclude deleted - - return { - posts, - rawCount: response.order.length, - firstInaccessiblePostTime: response.firstInaccessiblePostTime, - hasNext: response.hasNext, - incompletePayload: response.incompletePayload, - } -} - -function byMostRecentPost>(a: T, b: T): number { - const diff = b.create_at - a.create_at - if (diff !== 0) return diff - return comparePostIds(a.id, b.id) -} - -function objectRecord(value: unknown): Record | null { - return typeof value === 'object' && value !== null && !Array.isArray(value) - ? (value as Record) - : null -} - -function normalizeOrderedPostsPage(value: unknown): { - order: string[] - posts: Record - hasNext: boolean | null - firstInaccessiblePostTime: number | null - incompletePayload: boolean -} { - const response = objectRecord(value) - if (!response || !Array.isArray(response.order)) { - throw new Error('Mattermost returned an invalid posts response.') - } - - const order = response.order.filter((id): id is string => typeof id === 'string') - const rawPosts = objectRecord(response.posts) - const posts: Record = {} - let incompletePayload = order.length !== response.order.length || (!rawPosts && order.length > 0) - for (const id of order) { - const candidate = rawPosts?.[id] - if (!candidate || typeof candidate !== 'object' || Array.isArray(candidate)) { - incompletePayload = true - continue - } - posts[id] = candidate as Post - } - - const hasNext = typeof response.has_next === 'boolean' ? response.has_next : null - if (Object.hasOwn(response, 'has_next') && response.has_next !== undefined && hasNext === null) { - incompletePayload = true - } - const firstInaccessiblePostTime = - typeof response.first_inaccessible_post_time === 'number' && - response.first_inaccessible_post_time > 0 - ? response.first_inaccessible_post_time - : null - - return { order, posts, hasNext, firstInaccessiblePostTime, incompletePayload } -} - -// Fetch all posts with pagination, respecting limit and since -export async function getAllChannelPosts( - channelId: string, - options: { - limit?: number - since?: number - boundary?: { createAt: number; id: string } - safeBeforePostId?: string - } = {}, -): Promise { - const { limit = 50, since, boundary, safeBeforePostId } = options - const postsById = new Map() - const seenIds = new Set() - const target = limit + 1 - const pageSize = Math.min(target, 200) - let page = 0 - let stagnantPages = 0 - let exhausted = false - let uncertain = false - let activeBeforePostId = safeBeforePostId - let retriedWithoutMissingAnchor = false - - while (true) { - const pageResult = await getChannelPosts(channelId, { - limit: pageSize, - page, - before: activeBeforePostId, - }) - const posts = pageResult.posts - if (pageResult.firstInaccessiblePostTime !== null) uncertain = true - if (pageResult.incompletePayload) uncertain = true - - if (pageResult.rawCount === 0) { - if (page === 0 && activeBeforePostId !== undefined && !retriedWithoutMissingAnchor) { - activeBeforePostId = undefined - retriedWithoutMissingAnchor = true - stagnantPages = 0 - continue - } - if (pageResult.hasNext === true) uncertain = true - exhausted = !uncertain - break - } - - let madeProgress = false - for (const post of posts) { - if (seenIds.has(post.id)) continue - seenIds.add(post.id) - madeProgress = true - const afterBoundary = - boundary === undefined || - post.create_at < boundary.createAt || - (post.create_at === boundary.createAt && comparePostIds(post.id, boundary.id) > 0) - if ((since === undefined || post.create_at >= since) && afterBoundary) { - postsById.set(post.id, post) - } - } - - page += 1 - - if (madeProgress) stagnantPages = 0 - else stagnantPages += 1 - - if (since !== undefined && posts.length > 0 && posts.every((post) => post.create_at < since)) { - exhausted = !uncertain - break - } - if (postsById.size >= target) { - const selected = takeMostRecentPosts([...postsById.values()], limit) - const cutoff = selected[selected.length - 1]?.create_at - if (cutoff !== undefined && posts.some((post) => post.create_at < cutoff)) break - } - if (pageResult.rawCount < pageSize && pageResult.hasNext !== true) { - exhausted = !uncertain - break - } - if (stagnantPages >= 2) { - uncertain = true - break - } - } - - const selected = takeMostRecentPosts([...postsById.values()], limit) - return { - posts: selected, - truncated: postsById.size > limit ? true : uncertain ? null : exhausted ? false : null, - safeBeforeValid: safeBeforePostId === undefined || !retriedWithoutMissingAnchor, - } -} - -export function takeMostRecentPosts>( - posts: T[], - limit: number, -): T[] { - return [...new Map(posts.map((post) => [post.id, post])).values()] - .sort(byMostRecentPost) - .slice(0, limit) -} - -// Fetch a full thread (root + all replies) -export async function getPostThread(postId: string): Promise { - const client = getClient() - const posts = new Map() - let fromPost: string | undefined - let fromCreateAt: number | undefined - let uncertain = false - let stagnantPages = 0 - let successfulPages = 0 - - while (true) { - const params = new URLSearchParams({ perPage: '200', direction: 'down' }) - if (fromPost !== undefined) params.set('fromPost', fromPost) - if (fromCreateAt !== undefined) params.set('fromCreateAt', String(fromCreateAt)) - let response: ReturnType - try { - response = normalizeOrderedPostsPage( - await client.get(`/posts/${encodeURIComponent(postId)}/thread?${params}`), - ) - } catch (error) { - if (successfulPages === 0) throw error - return { posts: [...posts.values()], truncated: null } - } - successfulPages += 1 - - if (response.firstInaccessiblePostTime !== null || response.incompletePayload) uncertain = true - let added = 0 - for (const id of response.order) { - const post = response.posts[id] - if (!post || post.delete_at !== 0 || posts.has(id)) continue - posts.set(id, post) - added += 1 - } - - if (response.hasNext !== true) { - const visible = [...posts.values()] - const root = visible.find((post) => !post.root_id) - const visibleReplyCount = root ? visible.filter((post) => post.root_id === root.id).length : 0 - const legacyComplete = - response.hasNext === null && - !response.incompletePayload && - root !== undefined && - visibleReplyCount >= root.reply_count - return { - posts: visible, - truncated: uncertain ? null : response.hasNext === false || legacyComplete ? false : null, - } - } - - const cursorPost = [...response.order] - .reverse() - .map((id) => response.posts[id]) - .find((post): post is Post => !!post) - if (!cursorPost) { - return { posts: [...posts.values()], truncated: uncertain ? null : true } - } - - const nextFromPost = cursorPost.id - const nextFromCreateAt = cursorPost.create_at - const cursorAdvanced = nextFromPost !== fromPost || nextFromCreateAt !== fromCreateAt - stagnantPages = added > 0 && cursorAdvanced ? 0 : stagnantPages + 1 - if (stagnantPages >= 2) { - return { posts: [...posts.values()], truncated: uncertain ? null : true } - } - fromPost = nextFromPost - fromCreateAt = nextFromCreateAt - } -} - -export async function searchPosts( - teamId: string, - terms: string, - limit = 50, - accept: (post: Post) => boolean = () => true, -): Promise { - const client = getClient() - const posts = new Map() - const seenIds = new Set() - const order: string[] = [] - const matches: Record = {} - const target = limit + 1 - const perPage = Math.min(target, 100) - let page = 0 - let stagnantPages = 0 - let exhausted = false - let uncertain = false - - while (true) { - const rawResponse = await client.post( - `/teams/${encodeURIComponent(teamId)}/posts/search`, - { - terms, - is_or_search: false, - page, - per_page: perPage, - }, - true, - ) - const response = objectRecord(rawResponse) - if (!response || !Array.isArray(response.order)) { - throw new Error('Mattermost returned an invalid search response.') - } - const responseOrder = response.order.filter((id): id is string => typeof id === 'string') - if (responseOrder.length !== response.order.length) uncertain = true - const responsePosts = objectRecord(response.posts) - if (!responsePosts && responseOrder.length > 0) uncertain = true - const responseMatches = objectRecord(response.matches) - const hasNext = typeof response.has_next === 'boolean' ? response.has_next : undefined - if ( - typeof response.first_inaccessible_post_time === 'number' && - response.first_inaccessible_post_time > 0 - ) { - uncertain = true - } - if (hasNext === true && responseOrder.length === 0) uncertain = true - let madeProgress = false - const acceptedThisPage: Post[] = [] - for (const id of responseOrder) { - if (seenIds.has(id)) continue - seenIds.add(id) - madeProgress = true - const candidate = responsePosts?.[id] - if (!candidate || typeof candidate !== 'object' || Array.isArray(candidate)) { - uncertain = true - continue - } - const post = candidate as Post - if (post.delete_at !== 0 || !accept(post)) continue - posts.set(id, post) - acceptedThisPage.push(post) - order.push(id) - const rawMatches = responseMatches?.[id] - if (Array.isArray(rawMatches)) { - matches[id] = rawMatches.filter((match): match is string => typeof match === 'string') - } - } - page += 1 - if (madeProgress) stagnantPages = 0 - else stagnantPages += 1 - - // Search paging is only fully honored by Elasticsearch-backed servers. A short page is not - // proof of exhaustion, so rely on an empty response or bounded stagnation instead. - if (responseOrder.length === 0) { - exhausted = !uncertain - break - } - if (hasNext === false) { - exhausted = !uncertain - break - } - if (page >= MAX_SEARCH_SCAN_PAGES) { - uncertain = true - break - } - if (stagnantPages >= 2) { - uncertain = true - break - } - if (posts.size < target) continue - - const selected = takeMostRecentPosts([...posts.values()], limit) - const cutoff = selected[selected.length - 1]?.create_at - if (cutoff !== undefined && acceptedThisPage.some((post) => post.create_at < cutoff)) break - } - - const selected = takeMostRecentPosts([...posts.values()], limit) - const selectedIds = new Set(selected.map(({ id }) => id)) - return { - order: selected.map(({ id }) => id), - posts: Object.fromEntries([...posts].filter(([id]) => selectedIds.has(id))), - matches: Object.fromEntries(Object.entries(matches).filter(([id]) => selectedIds.has(id))), - truncated: posts.size > limit ? true : uncertain ? null : exhausted ? false : null, - } -} - -// Parse duration string to milliseconds -// Supports: "24h", "7d", "30d", "1w", "2m" (months) -export function parseDuration(duration: string): number { - const match = duration.match(/^(\d+)([hdwm])$/i) - if (!match) { - throw new Error('Invalid duration format. Use formats like "24h", "7d", "1w", "2m".') - } - - const valueText = match[1] - const unitText = match[2] - if (!valueText || !unitText) { - throw new Error('Invalid duration format.') - } - - const value = parseInt(valueText, 10) - const unit = unitText.toLowerCase() - - const now = Date.now() - const msPerHour = 60 * 60 * 1000 - const msPerDay = 24 * msPerHour - - switch (unit) { - case 'h': - return now - value * msPerHour - case 'd': - return now - value * msPerDay - case 'w': - return now - value * 7 * msPerDay - case 'm': - return now - value * 30 * msPerDay - default: - throw new Error(`Unknown duration unit: ${unit}`) - } -} diff --git a/src/api/url.ts b/src/api/url.ts deleted file mode 100644 index df0d1cd..0000000 --- a/src/api/url.ts +++ /dev/null @@ -1,50 +0,0 @@ -function isLoopbackHostname(hostname: string): boolean { - if (hostname === 'localhost' || hostname === '[::1]') return true - - const octets = hostname.split('.') - return ( - octets.length === 4 && - octets[0] === '127' && - octets.every((octet) => /^\d{1,3}$/.test(octet) && Number(octet) <= 255) - ) -} - -export function normalizeServerUrl(serverUrl: string): string { - let url: URL - - try { - url = new URL(serverUrl) - } catch { - throw new Error('Invalid Mattermost URL') - } - - if (url.username || url.password || url.search || url.hash) { - throw new Error( - 'Invalid Mattermost URL: credentials, query strings, and fragments are not allowed', - ) - } - - if (url.protocol !== 'https:' && url.protocol !== 'http:') { - throw new Error('Mattermost URL must use HTTPS, or HTTP on a loopback host.') - } - - if (url.protocol === 'http:' && !isLoopbackHostname(url.hostname)) { - throw new Error( - 'Refusing to send a Mattermost token over plaintext HTTP. Use HTTPS or a loopback URL.', - ) - } - - return url.toString().replace(/\/+$/, '') -} - -export function assertSecureServerUrl(serverUrl: string): void { - normalizeServerUrl(serverUrl) -} - -export function buildPostPermalink(serverUrl: string, postId: string): string { - const encodedPostId = encodeURIComponent(postId).replace( - /[!'()*]/g, - (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`, - ) - return `${normalizeServerUrl(serverUrl)}/_redirect/pl/${encodedPostId}` -} diff --git a/src/api/users.ts b/src/api/users.ts deleted file mode 100644 index 5233d04..0000000 --- a/src/api/users.ts +++ /dev/null @@ -1,113 +0,0 @@ -// User fetching with in-memory caching - -import type { User } from '../types' -import { getClient } from './client' - -// Cache users during the session -const userCache = new Map() -const usernameToId = new Map() -const MAX_USERS_LIST_PAGE = 200 -const MAX_USERS_SEARCH_PAGE = 1000 - -export interface UserDirectoryResult { - users: unknown - truncated: boolean | null -} - -export async function fetchUsers(options: { - query?: string - teamId?: string - limit: number -}): Promise { - const client = getClient() - const query = options.query?.trim() - const endpointLimit = query ? MAX_USERS_SEARCH_PAGE : MAX_USERS_LIST_PAGE - const probeLimit = Math.min(options.limit + 1, endpointLimit) - const users = query - ? await client.post( - '/users/search', - { - term: query, - ...(options.teamId ? { team_id: options.teamId } : {}), - limit: probeLimit, - allow_inactive: false, - }, - true, - ) - : await client.get( - `/users?page=0&per_page=${probeLimit}&active=true${options.teamId ? `&in_team=${encodeURIComponent(options.teamId)}` : ''}`, - ) - - const count = Array.isArray(users) ? users.length : 0 - return { - users, - truncated: - count > options.limit - ? true - : probeLimit === endpointLimit && count === endpointLimit - ? null - : false, - } -} - -export async function getMe(): Promise { - const client = getClient() - const user = await client.get('/users/me') - cacheUser(user) - return user -} - -export async function getUser(userId: string): Promise { - // Check cache first - const cached = userCache.get(userId) - if (cached) return cached - - const client = getClient() - const user = await client.get(`/users/${encodeURIComponent(userId)}`) - cacheUser(user) - return user -} - -export async function getUserByUsername(username: string): Promise { - // Check cache first - const cachedId = usernameToId.get(username.toLowerCase()) - if (cachedId) { - const cachedUser = userCache.get(cachedId) - if (cachedUser) return cachedUser - usernameToId.delete(username.toLowerCase()) - } - - const client = getClient() - const user = await client.get(`/users/username/${encodeURIComponent(username)}`) - cacheUser(user) - return user -} - -export async function getUsersByIds(userIds: string[]): Promise { - // Filter out already cached - const uncachedIds = userIds.filter((id) => !userCache.has(id)) - - if (uncachedIds.length > 0) { - const client = getClient() - const users = await client.post('/users/ids', uncachedIds, true) - users.forEach(cacheUser) - } - - // Return all from cache (now populated) - return userIds.map((id) => userCache.get(id)).filter(Boolean) as User[] -} - -function cacheUser(user: User): void { - if (typeof user?.id !== 'string' || typeof user?.username !== 'string') return - userCache.set(user.id, user) - usernameToId.set(user.username.toLowerCase(), user.id) -} - -export function getCachedUser(userId: string): User | undefined { - return userCache.get(userId) -} - -export function clearUserCache(): void { - userCache.clear() - usernameToId.clear() -} diff --git a/src/api/websocket.ts b/src/api/websocket.ts deleted file mode 100644 index e18e287..0000000 --- a/src/api/websocket.ts +++ /dev/null @@ -1,386 +0,0 @@ -import { - preprocess, - registerActiveMattermostCredential, - sanitizeTerminalLabel, -} from '../preprocessing' -import type { Post, WSPostEvent } from '../types' -import { normalizeServerUrl } from './url' - -interface SocketMessage { - event?: string - status?: string - error?: unknown - data?: Record - seq?: number - seq_reply?: number -} - -export interface WebSocketGap { - expected: number - received: number - reason: 'sequence_mismatch' | 'connection_changed' -} - -export interface WebSocketDiagnostics { - reconnect?: (attempt: number, delayMs: number) => void - gap?: (gap: WebSocketGap) => void - connected?: () => void - malformed?: (message: string) => void -} - -export interface WebSocketOptions { - channelId?: string - diagnostics?: WebSocketDiagnostics - WebSocket?: typeof globalThis.WebSocket - random?: () => number - handshakeTimeoutMs?: number - heartbeatIntervalMs?: number - heartbeatTimeoutMs?: number - backoffBaseMs?: number - backoffMaxMs?: number -} - -export function getSocketErrorMessage(error: unknown): string { - if (typeof error === 'string') - return sanitizeTerminalLabel(preprocess(error, { redact: false }).text) - if (error && typeof error === 'object' && 'message' in error) { - const message = (error as { message?: unknown }).message - if (typeof message === 'string') { - return sanitizeTerminalLabel(preprocess(message, { redact: false }).text) - } - } - return 'WebSocket request failed.' -} - -function toWebSocketUrl(serverUrl: string): URL { - const url = new URL(normalizeServerUrl(serverUrl)) - url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:' - url.pathname = `${url.pathname.replace(/\/+$/, '')}/api/v4/websocket` - return url -} - -function parseSocketMessage(raw: unknown): SocketMessage | null { - if (typeof raw !== 'string') return null - try { - return JSON.parse(raw) as SocketMessage - } catch { - return null - } -} - -function validTimestamp(value: unknown, fallback: number): number { - return typeof value === 'number' && - Number.isFinite(value) && - Number.isFinite(new Date(value).getTime()) - ? value - : fallback -} - -function parsePostEvent(payload: SocketMessage): { - post: Post - channelName: string - senderName: string -} | null { - if (payload.event !== 'posted' || !payload.data) return null - const event = payload as unknown as WSPostEvent - if (typeof event.data.post !== 'string') return null - try { - const value: unknown = JSON.parse(event.data.post) - if (!value || typeof value !== 'object' || Array.isArray(value)) return null - const raw = value as Record - if ( - typeof raw.id !== 'string' || - typeof raw.channel_id !== 'string' || - typeof raw.user_id !== 'string' || - typeof raw.message !== 'string' || - typeof raw.root_id !== 'string' || - typeof raw.create_at !== 'number' || - !Number.isFinite(raw.create_at) || - !Number.isFinite(new Date(raw.create_at).getTime()) || - !Array.isArray(raw.file_ids) || - !raw.file_ids.every((id) => typeof id === 'string') - ) { - return null - } - const post: Post = { - id: raw.id, - channel_id: raw.channel_id, - user_id: raw.user_id, - message: raw.message, - root_id: raw.root_id, - create_at: raw.create_at, - file_ids: raw.file_ids as string[], - update_at: validTimestamp(raw.update_at, raw.create_at), - delete_at: validTimestamp(raw.delete_at, 0), - edit_at: validTimestamp(raw.edit_at, 0), - type: typeof raw.type === 'string' ? raw.type : '', - props: - raw.props && typeof raw.props === 'object' && !Array.isArray(raw.props) - ? (raw.props as Record) - : {}, - hashtags: typeof raw.hashtags === 'string' ? raw.hashtags : '', - reply_count: - typeof raw.reply_count === 'number' && - Number.isSafeInteger(raw.reply_count) && - raw.reply_count >= 0 - ? raw.reply_count - : 0, - ...(typeof raw.is_pinned === 'boolean' ? { is_pinned: raw.is_pinned } : {}), - pending_post_id: typeof raw.pending_post_id === 'string' ? raw.pending_post_id : '', - } - return { - post, - channelName: typeof event.data.channel_name === 'string' ? event.data.channel_name : '', - senderName: typeof event.data.sender_name === 'string' ? event.data.sender_name : '', - } - } catch { - return null - } -} - -/** A resumable Mattermost WebSocket. Reconnection is scheduled only from onclose. */ -export function connectWebSocket( - serverUrl: string, - token: string, - onPost: (post: Post, channelName: string, senderName: string) => void, - onError: (error: Error) => void, - channelIdOrOptions?: string | WebSocketOptions, -): { close: () => void; done: Promise } { - const releaseCredential = registerActiveMattermostCredential(token) - const options: WebSocketOptions = - typeof channelIdOrOptions === 'string' - ? { channelId: channelIdOrOptions } - : (channelIdOrOptions ?? {}) - const Socket = options.WebSocket ?? globalThis.WebSocket - const random = options.random ?? Math.random - const handshakeTimeoutMs = options.handshakeTimeoutMs ?? 15_000 - const heartbeatIntervalMs = options.heartbeatIntervalMs ?? 30_000 - const heartbeatTimeoutMs = options.heartbeatTimeoutMs ?? 10_000 - const backoffBaseMs = options.backoffBaseMs ?? 1_000 - const backoffMaxMs = options.backoffMaxMs ?? 30_000 - - let socket: WebSocket | undefined - let generation = 0 - let stopped = false - let fatal = false - let reconnectAttempt = 0 - let connectionId: string | undefined - let nextServerSequence = 0 - let actionSequence = 1 - let handshakeTimer: ReturnType | undefined - let heartbeatTimer: ReturnType | undefined - let pongTimer: ReturnType | undefined - let reconnectTimer: ReturnType | undefined - let pendingPingSeq: number | undefined - let resolveDone!: () => void - const done = new Promise((resolve) => { - resolveDone = resolve - }) - - const clearConnectionTimers = (): void => { - if (handshakeTimer) clearTimeout(handshakeTimer) - if (heartbeatTimer) clearTimeout(heartbeatTimer) - if (pongTimer) clearTimeout(pongTimer) - handshakeTimer = heartbeatTimer = pongTimer = undefined - pendingPingSeq = undefined - } - - const close = (): void => { - if (stopped) return - stopped = true - generation++ - clearConnectionTimers() - if (reconnectTimer) clearTimeout(reconnectTimer) - reconnectTimer = undefined - const active = socket - socket = undefined - try { - if (active && active.readyState < 2) active.close(1000, 'client shutdown') - } catch { - // Cleanup and credential release must not depend on a host WebSocket implementation. - } - releaseCredential() - resolveDone() - } - - const failFatal = (error: Error): void => { - if (fatal || stopped) return - fatal = true - close() - try { - onError(error) - } catch { - // User callbacks cannot be allowed to bypass terminal cleanup or escape event dispatch. - } - } - - const scheduleHeartbeat = (currentGeneration: number): void => { - if (stopped || currentGeneration !== generation) return - heartbeatTimer = setTimeout(() => { - if (stopped || currentGeneration !== generation || !socket || socket.readyState !== 1) return - pendingPingSeq = actionSequence++ - socket.send(JSON.stringify({ seq: pendingPingSeq, action: 'ping', data: {} })) - pongTimer = setTimeout(() => { - if (stopped || currentGeneration !== generation) return - socket?.close(4000, 'heartbeat timeout') - }, heartbeatTimeoutMs) - }, heartbeatIntervalMs) - } - - const scheduleReconnect = (): void => { - if (stopped || fatal || reconnectTimer) return - const exponential = Math.min(backoffMaxMs, backoffBaseMs * 2 ** reconnectAttempt) - const delay = Math.round(exponential * (0.8 + random() * 0.4)) - reconnectAttempt++ - options.diagnostics?.reconnect?.(reconnectAttempt, delay) - reconnectTimer = setTimeout(() => { - reconnectTimer = undefined - try { - connect() - } catch { - failFatal(new Error('WebSocket connection failed.')) - } - }, delay) - } - - const connect = (): void => { - if (stopped || fatal) return - const currentGeneration = ++generation - const url = toWebSocketUrl(serverUrl) - if (connectionId) { - url.searchParams.set('connection_id', connectionId) - url.searchParams.set('sequence_number', String(nextServerSequence)) - } - const current = new Socket(url.toString()) - socket = current - let authenticated = false - let helloReceived = false - let authSeq = -1 - - const completeHandshake = (): void => { - if (!authenticated || !helloReceived || currentGeneration !== generation) return - if (handshakeTimer) clearTimeout(handshakeTimer) - handshakeTimer = undefined - reconnectAttempt = 0 - options.diagnostics?.connected?.() - scheduleHeartbeat(currentGeneration) - } - - const rejectSequence = (received: number): void => { - options.diagnostics?.gap?.({ - expected: nextServerSequence, - received, - reason: 'sequence_mismatch', - }) - current.close(4000, 'sequence mismatch') - } - - handshakeTimer = setTimeout(() => { - if (currentGeneration === generation) current.close(4000, 'handshake timeout') - }, handshakeTimeoutMs) - - current.onopen = () => { - if (currentGeneration !== generation || stopped) return - authSeq = actionSequence++ - current.send( - JSON.stringify({ - seq: authSeq, - action: 'authentication_challenge', - data: { token }, - }), - ) - } - - current.onmessage = (event) => { - if (currentGeneration !== generation || stopped) return - const payload = parseSocketMessage(event.data) - if (!payload) { - options.diagnostics?.malformed?.('Malformed WebSocket message skipped.') - return - } - - if (payload.status === 'FAIL') { - const message = getSocketErrorMessage(payload.error) - const normalized = message.toLowerCase() - const isAuthenticationError = - /\b(?:authentication|authenticate|unauthorized|not authorized|invalid token)\b/.test( - normalized, - ) - if (payload.seq_reply === authSeq || isAuthenticationError) { - failFatal(new Error('Authentication failed. Check your token.')) - } else { - current.close(4000, 'request failed') - } - return - } - - if (payload.status === 'OK' && payload.seq_reply === authSeq) { - authenticated = true - completeHandshake() - } else if (payload.status === 'OK' && payload.seq_reply === pendingPingSeq && pongTimer) { - clearTimeout(pongTimer) - pongTimer = undefined - pendingPingSeq = undefined - scheduleHeartbeat(currentGeneration) - } - - if (payload.event) { - if (typeof payload.seq !== 'number') { - options.diagnostics?.malformed?.('WebSocket event without a sequence skipped.') - return - } - if (payload.event === 'hello' && typeof payload.data?.connection_id === 'string') { - const receivedConnectionId = payload.data.connection_id - if (connectionId && receivedConnectionId !== connectionId) { - options.diagnostics?.gap?.({ - expected: nextServerSequence, - received: payload.seq, - reason: 'connection_changed', - }) - nextServerSequence = 0 - } - connectionId = receivedConnectionId - } - if (payload.seq !== nextServerSequence) { - rejectSequence(payload.seq) - return - } - nextServerSequence = payload.seq + 1 - if (payload.event === 'hello') { - helloReceived = true - completeHandshake() - } - } - - if (!authenticated || !helloReceived || payload.event !== 'posted') return - const parsed = parsePostEvent(payload) - if (!parsed) { - options.diagnostics?.malformed?.('Malformed WebSocket post payload skipped.') - return - } - if (options.channelId && parsed.post.channel_id !== options.channelId) return - onPost(parsed.post, parsed.channelName, parsed.senderName) - } - - current.onerror = () => { - if (currentGeneration !== generation || stopped) return - if (current.readyState < 2) current.close(4000, 'connection failure') - } - - current.onclose = () => { - if (currentGeneration !== generation || stopped) return - generation++ - clearConnectionTimers() - if (socket === current) socket = undefined - scheduleReconnect() - } - } - - try { - connect() - } catch (error) { - close() - throw error - } - return { close, done } -} diff --git a/src/cli.ts b/src/cli.ts deleted file mode 100644 index 9341c06..0000000 --- a/src/cli.ts +++ /dev/null @@ -1,1931 +0,0 @@ -// CLI command handlers - -import type { CanonicalTeam } from './api' -import { - buildPostPermalink, - connectWebSocket, - createDirectChannel, - createPost, - fetchUsers, - getAllChannelPosts, - getCachedUser, - getChannel, - getChannelByName, - getChannelMember, - getDMChannelByUsername, - getMe, - getMyChannels, - getMyDMChannels, - getMyGroupDMChannels, - getMyTeams, - getOtherUserIdFromDMChannel, - getPostThread, - getTeamChannelMembers, - getUser, - getUserByUsername, - getUsersByIds, - initClient, - MattermostAPIError, - MattermostMutationOutcomeUnknownError, - normalizeCanonicalTeams, - normalizeChannelName, - parseDuration, - resolveTeamId, - searchPosts, - takeMostRecentPosts, -} from './api' -import { decodeChannelHistoryCursor, encodeChannelHistoryCursor } from './cursor' -import { - formatJSON, - formatMarkdown, - formatPretty, - formatWatchEvent, - formatWatchJSON, -} from './formatters' -import { normalizePosts, postUserIds, preprocess, sanitizeTerminalLabel } from './preprocessing' -import type { - Channel, - ChannelMember, - ChannelOptions, - ChannelTypeFilter, - CLIOptions, - DMsOptions, - GroupDMsOptions, - IdentityOptions, - MentionOptions, - MessageOutput, - Post, - ProcessedChannel, - Redaction, - RetrievalMetadata, - SearchOptions, - SendDirectMessageOptions, - SendGroupMessageOptions, - UnreadOptions, - UsersOptions, - WatchEvent, -} from './types' -import { - calculateUnreadMetrics, - formatDate, - formatRelativeTime, - groupIntoThreads, - sortUnreadEntries, -} from './utils' - -interface ChannelListItem { - id: string - type: ProcessedChannel['type'] - name: string - displayName?: string - team: (Pick & { displayName?: string }) | null - lastPost: string | null - messageCount: number -} - -const INCOMPLETE_EMPTY_RETRIEVAL_ERROR = - 'Message retrieval was incomplete, so an empty result cannot be confirmed.' - -function requireConfirmedEmpty(selectedCount: number, queryTruncated: boolean | null): void { - if (selectedCount === 0 && queryTruncated === null) { - throw new Error(INCOMPLETE_EMPTY_RETRIEVAL_ERROR) - } -} - -interface UnreadSummaryItem { - channel: Channel - processedChannel: ProcessedChannel - unreadCount: number - mentionCount: number - lastViewedAt: number -} - -export interface SendReceipt { - status: 'dry_run' | 'sent' - destination: { - type: 'dm' | 'group' - label: string - channelId: string | null - willCreate: boolean - } - post?: { - id: string - createAt: string - pendingPostId: string - permalink: string - } -} - -export class MattermostDeliveryConfirmedError extends Error { - constructor() { - super( - 'Mattermost confirmed delivery, but the local receipt could not be written. Do not retry.', - ) - this.name = 'MattermostDeliveryConfirmedError' - } -} - -export interface WhoAmIOutput { - id: string - username: string - displayName?: string - nickname?: string - roles: string[] -} - -export interface TeamOutput { - id: string - name: string - displayName?: string - type: 'open' | 'invite_only' -} - -export interface UserDirectoryOutput { - id: string - username: string - displayName?: string - nickname?: string -} - -export function resolveUsersTeamId(teams: unknown, requested: string, redact = true): string { - const canonical = normalizeCanonicalTeams(teams) - const match = canonical.find((team) => team.name === requested || team.displayName === requested) - if (match) return match.id - - const safeRequested = safeString(requested, redact) - const available = canonical.map((team) => safeString(team.name, redact)).join(', ') - throw new Error( - `Team "${safeRequested}" not found.${available ? ` Your teams: ${available}` : ''}`, - ) -} - -export function normalizeUsers(users: unknown, redact = true): UserDirectoryOutput[] { - if (!Array.isArray(users)) throw new Error('Invalid users response.') - return users - .map((user) => { - if (!isRecord(user)) throw new Error('Invalid users response.') - const id = requiredString(user.id, 'Invalid users response.') - const username = requiredString(user.username, 'Invalid users response.') - const firstName = safeString(user.first_name, redact) - const lastName = safeString(user.last_name, redact) - const displayName = [firstName, lastName].filter(Boolean).join(' ') - const nickname = safeString(user.nickname, redact) - return { - id: safeString(id, redact), - username: safeString(username, redact), - ...(displayName ? { displayName } : {}), - ...(nickname ? { nickname } : {}), - } - }) - .sort((a, b) => { - if (a.username !== b.username) return a.username < b.username ? -1 : 1 - if (a.id === b.id) return 0 - return a.id < b.id ? -1 : 1 - }) -} - -function safeString(value: unknown, redact: boolean): string { - if (typeof value !== 'string') return '' - return sanitizeTerminalLabel(preprocess(value, { redact }).text) -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value) -} - -function requiredString(value: unknown, errorMessage: string): string { - if (typeof value !== 'string' || value.length === 0) throw new Error(errorMessage) - return value -} - -export function normalizeWhoAmI(user: unknown, redact = true): WhoAmIOutput { - if (!isRecord(user)) throw new Error('Invalid identity response.') - const raw = user - const id = requiredString(raw.id, 'Invalid identity response.') - const username = requiredString(raw.username, 'Invalid identity response.') - const firstName = safeString(raw.first_name, redact) - const lastName = safeString(raw.last_name, redact) - const displayName = [firstName, lastName].filter(Boolean).join(' ') - const nickname = safeString(raw.nickname, redact) - const roles = - typeof raw.roles === 'string' - ? raw.roles - .split(/\s+/) - .map((role) => safeString(role, redact)) - .filter(Boolean) - : [] - - return { - id: safeString(id, redact), - username: safeString(username, redact), - ...(displayName ? { displayName } : {}), - ...(nickname ? { nickname } : {}), - roles, - } -} - -export function normalizeTeams(teams: unknown, redact = true): TeamOutput[] { - if (!Array.isArray(teams)) throw new Error('Invalid teams response.') - - return teams - .map((team) => { - if (!isRecord(team)) throw new Error('Invalid teams response.') - const id = requiredString(team.id, 'Invalid teams response.') - const name = requiredString(team.name, 'Invalid teams response.') - if (team.type !== 'O' && team.type !== 'I') throw new Error('Invalid teams response.') - - const raw = team - const displayName = safeString(raw.display_name, redact) - return { - id: safeString(id, redact), - name: safeString(name, redact), - ...(displayName ? { displayName } : {}), - type: raw.type === 'O' ? ('open' as const) : ('invite_only' as const), - } - }) - .sort((a, b) => { - if (a.name !== b.name) return a.name < b.name ? -1 : 1 - if (a.id === b.id) return 0 - return a.id < b.id ? -1 : 1 - }) -} - -export async function showWhoAmI(options: IdentityOptions): Promise { - initClient(options.url, options.token) - const output = normalizeWhoAmI(await getMe(), options.redact) - - if (options.json) { - console.log(JSON.stringify(output, null, 2)) - return - } - - const labels = [output.displayName, output.nickname ? `aka ${output.nickname}` : undefined] - .filter(Boolean) - .join(', ') - console.log(`@${output.username}${labels ? ` (${labels})` : ''} [${output.id}]`) - console.log(`Roles: ${output.roles.length > 0 ? output.roles.join(', ') : 'none'}`) -} - -export async function listTeams(options: IdentityOptions): Promise { - initClient(options.url, options.token) - const me = await getMe() - normalizeWhoAmI(me, options.redact) - if (!isRecord(me)) throw new Error('Invalid identity response.') - const userId = requiredString(me.id, 'Invalid identity response.') - const output = normalizeTeams(await getMyTeams(userId), options.redact) - - if (options.json) { - console.log(JSON.stringify(output, null, 2)) - return - } - - if (output.length === 0) { - console.log('No teams found.') - return - } - for (const team of output) { - const display = - team.displayName && team.displayName !== team.name ? ` (${team.displayName})` : '' - console.log(`${team.name}${display} [${team.id}] ${team.type}`) - } -} - -export async function listUsers(options: UsersOptions): Promise { - initClient(options.url, options.token) - let teamId: string | undefined - if (options.team) { - const me = await getMe() - if (!isRecord(me)) throw new Error('Invalid identity response.') - const userId = requiredString(me.id, 'Invalid identity response.') - teamId = resolveUsersTeamId(await getMyTeams(userId), options.team, options.redact) - } - - const query = options.query?.trim() || undefined - const result = await fetchUsers({ query, teamId, limit: options.limit }) - const users = normalizeUsers(result.users, options.redact).slice(0, options.limit) - const retrieval = { - selectedCount: users.length, - requestedLimit: options.limit, - query: query ? safeString(query, options.redact) : null, - teamId: teamId ? safeString(teamId, options.redact) : null, - truncated: result.truncated, - } - - if (options.json) { - console.log(JSON.stringify({ users, retrieval }, null, 2)) - return - } - if (users.length === 0) { - console.log('No users found.') - return - } - for (const user of users) { - const labels = [user.displayName, user.nickname ? `aka ${user.nickname}` : undefined] - .filter(Boolean) - .join(', ') - console.log(`@${user.username}${labels ? ` (${labels})` : ''} [${user.id}]`) - } - const coverage = - result.truncated === true ? 'truncated' : result.truncated === false ? 'complete' : 'unknown' - console.log(`Showing ${users.length} of up to ${options.limit} users (coverage: ${coverage}).`) -} - -export class BoundedPostIdSet { - private readonly ids = new Set() - private readonly order: string[] = [] - - constructor(private readonly limit = 1000) {} - - add(id: string): boolean { - if (this.ids.has(id)) return false - this.ids.add(id) - this.order.push(id) - if (this.order.length > this.limit) { - const expired = this.order.shift() - if (expired) this.ids.delete(expired) - } - return true - } -} - -export function mergeTruncation( - states: Array, - candidateCount: number, - limit: number, -): boolean | null { - if (candidateCount > limit || states.includes(true)) return true - if (states.includes(null)) return null - return false -} - -export function createWatchPostHandler( - options: Pick, - write: (line: string) => void = console.log, -): (post: Post, channelName: string, senderName: string) => void { - const seenPostIds = new BoundedPostIdSet() - - return (post, channelName, senderName) => { - if (!seenPostIds.add(post.id)) return - const redactions: Redaction[] = [] - const clean = (value: string, field: string, oneLine = true): string => { - const result = preprocess(value, { redact: options.redact }) - redactions.push(...result.redactions.map((item) => ({ ...item, field }))) - return oneLine ? result.text.replace(/\n/g, '\\n').replace(/\t/g, '\\t') : result.text - } - const username = clean( - getCachedUser(post.user_id)?.username || senderName || 'unknown', - 'watch.sender', - ) - const text = clean(post.message, 'watch.message', false) - const event: WatchEvent = { - type: 'posted', - postId: clean(post.id, 'watch.postId'), - channelId: clean(post.channel_id, 'watch.channelId'), - channelName: clean(channelName, 'watch.channelName'), - sender: username, - senderId: clean(post.user_id, 'watch.senderId'), - message: text, - timestamp: new Date(post.create_at).toISOString(), - rootId: post.root_id ? clean(post.root_id, 'watch.rootId') : undefined, - fileIds: arrayStringValues(post.file_ids).map((id) => clean(id, 'watch.fileId')), - redactions, - } - write( - options.json - ? formatWatchJSON(event) - : formatWatchEvent(event, options.color && Boolean(process.stdout.isTTY)), - ) - } -} - -function arrayStringValues(value: unknown): string[] { - return Array.isArray(value) - ? value.filter((item): item is string => typeof item === 'string') - : [] -} - -function nonNegativeInteger(value: unknown): number { - return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : 0 -} - -function channelTypeLabel(type: Channel['type']): Exclude { - switch (type) { - case 'O': - return 'public' - case 'P': - return 'private' - case 'D': - return 'dm' - case 'G': - return 'group' - default: - throw new Error('Mattermost returned an unknown channel type.') - } -} - -function channelLabel(channel: ProcessedChannel): string { - if (channel.type === 'unknown') return `Unknown channel (${channel.id})` - if (channel.type === 'dm' || channel.type === 'group') return channel.name - const display = channel.displayName ? ` (${channel.displayName})` : '' - return `#${channel.name}${display}` -} - -function presentOneLine(value: string, redact: boolean): string { - return sanitizeTerminalLabel(preprocess(value, { redact }).text) -} - -export function hasLiteralMention(message: string, terms: string[]): boolean { - const lowerMessage = message.toLowerCase() - return terms.some((term) => { - const literal = term.toLowerCase() - if (literal.length === 0) return false - const isAlias = !literal.startsWith('@') - - let index = lowerMessage.indexOf(literal) - while (index !== -1) { - const previous = lowerMessage[index - 1] - const next = lowerMessage[index + literal.length] - const isBoundaryCharacter = (character: string | undefined) => - character !== undefined && - (isAlias ? /[\p{L}\p{M}\p{N}]/u.test(character) : /[a-z0-9._-]/i.test(character)) - if (!isBoundaryCharacter(previous) && !isBoundaryCharacter(next)) return true - index = lowerMessage.indexOf(literal, index + 1) - } - return false - }) -} - -export function isExactMentionPost(post: Post, term: string, since?: number): boolean { - return ( - post.delete_at === 0 && - (since === undefined || post.create_at >= since) && - hasLiteralMention(post.message, [term.startsWith('@') ? term : term.slice(1, -1)]) - ) -} - -export function mentionSearchAfterDate(since: number): string { - return new Date(since - 24 * 60 * 60 * 1000).toISOString().slice(0, 10) -} - -function groupPostsByChannel(posts: Post[]): Map { - const grouped = new Map() - - for (const post of posts) { - const list = grouped.get(post.channel_id) - if (list) { - list.push(post) - } else { - grouped.set(post.channel_id, [post]) - } - } - - return grouped -} - -function presentVisibleThreads( - value: RetrievalMetadata['visibleThreads'], - redact: boolean, - redactions: Redaction[], -): RetrievalMetadata['visibleThreads'] { - return { - ...value, - failedRootIds: value.failedRootIds.map((id) => { - const result = preprocess(id, { redact }) - redactions.push( - ...result.redactions.map((item) => ({ ...item, field: 'retrieval.failedRootId' })), - ) - return result.text.replace(/\n/g, '\\n').replace(/\t/g, '\\t') - }), - } -} - -async function buildProcessedChannel( - channel: Channel, - myUserId: string, - redact = true, - redactions: Redaction[] = [], -): Promise { - requireRawChannelShape(channel, channel.id) - const type = channelTypeLabel(channel.type) - const clean = (value: string, field: string): string => { - const result = preprocess(value, { redact }) - redactions.push(...result.redactions.map((item) => ({ ...item, field }))) - return result.text.replace(/\n/g, '\\n').replace(/\t/g, '\\t') - } - - if (type === 'dm') { - const otherUserId = getOtherUserIdFromDMChannel(channel, myUserId) - if (!otherUserId) { - throw new Error('Mattermost returned an invalid channel response.') - } - - const otherUser = await getUser(otherUserId) - return { - id: clean(channel.id, 'channel.id'), - type: 'dm', - name: `@${clean(otherUser.username, 'channel.dmUsername')}`, - metadataStatus: 'resolved', - } - } - - if (type === 'group') { - return { - id: clean(channel.id, 'channel.id'), - type, - name: clean(channel.display_name || channel.name, 'channel.displayName'), - metadataStatus: 'resolved', - } - } - - return { - id: clean(channel.id, 'channel.id'), - type, - name: clean(channel.name, 'channel.name'), - displayName: channel.display_name - ? clean(channel.display_name, 'channel.displayName') - : undefined, - metadataStatus: 'resolved', - } -} - -function isRequiredRawChannelShape(value: unknown, expectedId?: string): value is Channel { - if (typeof value !== 'object' || value === null) return false - const channel = value as Record - if ( - typeof channel.id !== 'string' || - channel.id.trim().length === 0 || - (expectedId !== undefined && channel.id !== expectedId) || - (channel.type !== 'O' && - channel.type !== 'P' && - channel.type !== 'D' && - channel.type !== 'G') || - typeof channel.name !== 'string' || - typeof channel.display_name !== 'string' || - typeof channel.team_id !== 'string' - ) { - return false - } - - if (channel.type === 'O' || channel.type === 'P') { - return channel.name.trim().length > 0 && channel.team_id.trim().length > 0 - } - if (channel.team_id !== '') return false - if (channel.type === 'G') return channel.name.trim().length > 0 - - const userIds = channel.name.split('__') - return userIds.length === 2 && userIds.every((id) => id.trim().length > 0) -} - -function requireRawChannelShape(value: unknown, expectedId?: string): Channel { - if (!isRequiredRawChannelShape(value, expectedId)) { - throw new Error('Mattermost returned an invalid channel response.') - } - return value -} - -function requireConversationChannelContext(channel: Channel, myUserId: string): Channel { - requireRawChannelShape(channel, channel.id) - if (channel.type === 'D' && !getOtherUserIdFromDMChannel(channel, myUserId)) { - throw new Error('Mattermost returned an invalid channel response.') - } - return channel -} - -function validateAndDedupeConversationChannels(channels: Channel[], myUserId: string): Channel[] { - const validated = channels.map((channel) => requireConversationChannelContext(channel, myUserId)) - return [...new Map(validated.map((channel) => [channel.id, channel])).values()] -} - -function unavailableProcessedChannel( - channelId: string, - redact: boolean, - redactions: Redaction[], -): ProcessedChannel { - const result = preprocess(channelId, { redact }) - redactions.push(...result.redactions.map((item) => ({ ...item, field: 'channel.id' }))) - const safeId = sanitizeTerminalLabel(result.text) - console.error(`Warning: Channel metadata is unavailable for ${safeId || 'an unknown channel'}.`) - return { - id: safeId, - type: 'unknown', - name: 'unknown', - metadataStatus: 'unavailable', - } -} - -async function resolveProcessedChannel( - channelId: string, - myUserId: string, - redact: boolean, - knownChannel?: Channel, -): Promise<{ channel: ProcessedChannel; redactions: Redaction[] }> { - const redactions: Redaction[] = [] - let channel: Channel - - if (knownChannel) { - channel = requireRawChannelShape(knownChannel, channelId) - } else { - if (!channelId) { - return { - channel: unavailableProcessedChannel(channelId, redact, redactions), - redactions, - } - } - try { - const candidate: unknown = await getChannel(channelId) - if (!isRequiredRawChannelShape(candidate, channelId)) { - return { - channel: unavailableProcessedChannel(channelId, redact, redactions), - redactions, - } - } - channel = candidate - } catch (error) { - if (error instanceof MattermostAPIError && error.status === 401) throw error - return { - channel: unavailableProcessedChannel(channelId, redact, redactions), - redactions, - } - } - } - - if (!knownChannel && channel.type === 'D' && !getOtherUserIdFromDMChannel(channel, myUserId)) { - return { - channel: unavailableProcessedChannel(channelId, redact, redactions), - redactions, - } - } - - return { - channel: await buildProcessedChannel(channel, myUserId, redact, redactions), - redactions, - } -} - -async function buildOutputsFromPosts( - posts: Post[], - myUserId: string, - options: CLIOptions, - retrieval: Pick< - RetrievalMetadata['selection'], - 'source' | 'requestedLimit' | 'since' | 'queryTruncated' - > & - Partial>, - knownChannels: Map = new Map(), -): Promise { - const grouped = groupPostsByChannel(posts) - const outputs: MessageOutput[] = [] - const hydratedGroups: Array<{ - channelId: string - seedPosts: Post[] - channelPosts: Post[] - visibleThreads: RetrievalMetadata['visibleThreads'] - }> = [] - - for (const [channelId, seedPosts] of grouped) { - const { posts: channelPosts, visibleThreads } = await hydrateVisibleThreads( - seedPosts, - options.threads, - options.redact, - ) - hydratedGroups.push({ channelId, seedPosts, channelPosts, visibleThreads }) - } - - const userIds = [ - ...new Set(hydratedGroups.flatMap(({ channelPosts }) => postUserIds(channelPosts))), - ] - if (userIds.length > 0) await getUsersByIds(userIds) - const usersById = new Map( - userIds.flatMap((id) => { - const user = getCachedUser(id) - return user ? ([[id, user]] as const) : [] - }), - ) - - for (const { channelId, seedPosts, channelPosts, visibleThreads } of hydratedGroups) { - const { channel: processedChannel, redactions: channelRedactions } = - await resolveProcessedChannel( - channelId, - myUserId, - options.redact, - knownChannels.get(channelId), - ) - const { messages, redactions } = normalizePosts( - channelPosts, - usersById, - myUserId, - options.url, - buildPostPermalink, - options.redact, - ) - const presentedVisibleThreads = presentVisibleThreads( - visibleThreads, - options.redact, - redactions, - ) - - outputs.push({ - channel: processedChannel, - messages: options.threads ? groupIntoThreads(messages) : messages, - redactions: [...channelRedactions, ...redactions], - retrieval: retrievalMetadata( - retrieval, - seedPosts.length, - presentedVisibleThreads, - channelPosts.length, - ), - }) - } - - return outputs -} - -export async function hydrateVisibleThreads( - seedPosts: Post[], - requested: boolean, - redact = true, -): Promise<{ posts: Post[]; visibleThreads: RetrievalMetadata['visibleThreads'] }> { - if (!requested) { - return { - posts: seedPosts, - visibleThreads: { status: 'not_requested', hydratedRootCount: 0, failedRootIds: [] }, - } - } - - const postsById = new Map(seedPosts.map((post) => [post.id, post])) - const rootIds = [ - ...new Set( - seedPosts.flatMap((post) => - post.root_id ? [post.root_id] : post.reply_count > 0 ? [post.id] : [], - ), - ), - ] - if (rootIds.length === 0) { - return { - posts: seedPosts, - visibleThreads: { status: 'complete', hydratedRootCount: 0, failedRootIds: [] }, - } - } - - const failedRootIdSet = new Set() - let hydratedRootCount = 0 - let nextIndex = 0 - const hydrateRoot = async (rootId: string) => { - const existingRoot = postsById.get(rootId) - const loadedReplyCount = seedPosts.filter((post) => post.root_id === rootId).length - if (existingRoot && loadedReplyCount >= existingRoot.reply_count) { - hydratedRootCount += 1 - return - } - - try { - const result = await getPostThread(rootId) - for (const post of result.posts) postsById.set(post.id, post) - const root = result.posts.find((post) => post.id === rootId && !post.root_id) - if (result.truncated === false && root) { - hydratedRootCount += 1 - } else { - failedRootIdSet.add(rootId) - console.error( - `Warning: Thread ${preprocess(rootId, { redact }).text.replace(/\n/g, '\\n').replace(/\t/g, '\\t')} could only be partially hydrated.`, - ) - } - } catch { - failedRootIdSet.add(rootId) - console.error( - `Warning: Could not hydrate thread ${preprocess(rootId, { redact }).text.replace(/\n/g, '\\n').replace(/\t/g, '\\t')}.`, - ) - } - } - const worker = async () => { - while (true) { - const index = nextIndex - nextIndex += 1 - const rootId = rootIds[index] - if (rootId === undefined) return - await hydrateRoot(rootId) - } - } - await Promise.all(Array.from({ length: Math.min(4, rootIds.length) }, () => worker())) - const failedRootIds = rootIds.filter((rootId) => failedRootIdSet.has(rootId)) - - return { - posts: [...postsById.values()], - visibleThreads: { - status: failedRootIds.length === 0 ? 'complete' : 'partial', - hydratedRootCount, - failedRootIds, - }, - } -} - -function retrievalMetadata( - selection: Pick< - RetrievalMetadata['selection'], - 'source' | 'requestedLimit' | 'since' | 'queryTruncated' - > & - Partial>, - selectedCount: number, - visibleThreads: RetrievalMetadata['visibleThreads'] = { - status: 'not_requested', - hydratedRootCount: 0, - failedRootIds: [], - }, - visiblePostCount = selectedCount, -): RetrievalMetadata { - return { - selection: { - ...selection, - selectedCount, - inputCursor: selection.inputCursor ?? null, - nextCursor: selection.nextCursor ?? null, - }, - visibleThreads, - visiblePostCount, - deletedPostsIncluded: false, - } -} - -export type OutputMode = 'json' | 'pretty' | 'markdown' - -export function selectOutputMode(json: boolean, isTTY: boolean): OutputMode { - if (json) return 'json' - return isTTY ? 'pretty' : 'markdown' -} - -function formatOutput(outputs: MessageOutput[], options: CLIOptions): void { - if (outputs.length === 0 && !options.json) { - console.log('No messages found.') - return - } - - const mode = selectOutputMode(options.json, Boolean(process.stdout.isTTY)) - - if (mode === 'json') { - console.log(formatJSON(outputs)) - } else if (mode === 'pretty') { - console.log(formatPretty(outputs, { color: options.color, relative: options.relative })) - } else { - console.log(formatMarkdown(outputs, { relative: options.relative })) - } -} - -function printRedactionWarning(enabled: boolean): void { - if (!enabled) { - console.error('Warning: Secret redaction is disabled. Output may contain secrets.') - } -} - -function buildChannelListItem( - channel: ProcessedChannel, - rawChannel: Channel, - team: CanonicalTeam | null, - redact: boolean, -): ChannelListItem { - const lastPostAt = finiteTimestamp(rawChannel.last_post_at) - const presentedTeam = team - ? { - id: safeString(team.id, redact), - name: safeString(team.name, redact), - ...(team.displayName ? { displayName: safeString(team.displayName, redact) } : {}), - } - : null - return { - id: channel.id, - type: channel.type, - name: channel.name, - displayName: channel.displayName, - team: presentedTeam, - lastPost: lastPostAt > 0 ? new Date(lastPostAt).toISOString() : null, - messageCount: nonNegativeInteger(rawChannel.total_msg_count), - } -} - -function finiteTimestamp(value: unknown): number { - return typeof value === 'number' && - Number.isFinite(value) && - Number.isFinite(new Date(value).getTime()) - ? value - : 0 -} - -export async function listChannels(options: { - url: string - token: string - json: boolean - color: boolean - relative: boolean - redact: boolean - typeFilter: ChannelTypeFilter -}): Promise { - initClient(options.url, options.token) - const validTypeFilters: readonly string[] = ['all', 'dm', 'public', 'private', 'group'] - if (!validTypeFilters.includes(options.typeFilter)) { - throw new Error( - `Invalid channel type "${presentOneLine(String(options.typeFilter), false)}". Expected one of: ${validTypeFilters.join(', ')}.`, - ) - } - - const me = await getMe() - let channels = [ - ...new Map((await getMyChannels(me.id)).map((channel) => [channel.id, channel])).values(), - ] - - if (options.typeFilter !== 'all') { - const typeMap: Record, Channel['type']> = { - dm: 'D', - public: 'O', - private: 'P', - group: 'G', - } - const filterType = typeMap[options.typeFilter] - channels = channels.filter((ch) => ch.type === filterType) - } - - let teamById = new Map() - if (channels.some((channel) => channel.type === 'O' || channel.type === 'P')) { - teamById = new Map( - normalizeCanonicalTeams(await getMyTeams(me.id)).map((team) => [team.id, team]), - ) - for (const channel of channels) { - if ( - (channel.type === 'O' || channel.type === 'P') && - (typeof channel.team_id !== 'string' || - channel.team_id.length === 0 || - !teamById.has(channel.team_id)) - ) { - throw new Error('Invalid channels response.') - } - } - } - - const dmChannels = channels.filter((ch) => ch.type === 'D') - const otherUserIds = dmChannels - .map((ch) => getOtherUserIdFromDMChannel(ch, me.id)) - .filter((id): id is string => !!id) - - if (otherUserIds.length > 0) { - await getUsersByIds(otherUserIds) - } - - const output = await Promise.all( - channels.map(async (channel) => { - const processed = await buildProcessedChannel(channel, me.id, options.redact) - const team = - channel.type === 'O' || channel.type === 'P' ? teamById.get(channel.team_id) : null - return buildChannelListItem(processed, channel, team ?? null, options.redact) - }), - ) - - output.sort((a, b) => { - if (!a.lastPost) return 1 - if (!b.lastPost) return -1 - return new Date(b.lastPost).getTime() - new Date(a.lastPost).getTime() - }) - - if (options.json) { - console.log(JSON.stringify(output, null, 2)) - return - } - - const grouped = Map.groupBy(output, (channel) => channel.type) - const typeLabels: Record = { - public: 'Public Channels', - private: 'Private Channels', - dm: 'Direct Messages', - group: 'Group Messages', - } - const typeOrder = ['public', 'private', 'group', 'dm'] as const - - for (const type of typeOrder) { - const items = grouped.get(type) - if (!items || items.length === 0) continue - - console.log(`\n${typeLabels[type]}:\n`) - for (const channel of items) { - let lastPost = 'never' - if (channel.lastPost) { - const date = new Date(channel.lastPost) - lastPost = options.relative - ? formatRelativeTime(date) - : formatDate(date, { includeYear: true }) - } - - const label = - channel.type === 'dm' || channel.type === 'group' - ? channel.name - : `${channel.team?.name}/#${channel.name}` - const display = channel.displayName ? ` (${channel.displayName})` : '' - console.log( - ` ${label.padEnd(25)}${display ? display.padEnd(25) : ''.padEnd(25)} [${channel.id}] ${channel.messageCount} msgs, last: ${lastPost}`, - ) - } - } - - console.log(`\nTotal: ${output.length} channels`) -} - -function renderSendReceipt(receipt: SendReceipt, json: boolean): string { - if (json) { - return JSON.stringify(receipt, null, 2) - } - const destination = receipt.destination.channelId - ? `${receipt.destination.label} [${receipt.destination.channelId}]` - : receipt.destination.label - if (receipt.status === 'dry_run') { - return receipt.destination.willCreate - ? `Would create a conversation with ${destination}, then send one message.` - : `Would send one message to ${destination}.` - } - return `Sent one message to ${destination}. Post ${receipt.post?.id}.` -} - -function emitSendReceipt(receipt: SendReceipt, json: boolean): void { - console.log(renderSendReceipt(receipt, json)) -} - -async function writeConfirmedSendReceipt(receipt: SendReceipt, json: boolean): Promise { - try { - const output = `${renderSendReceipt(receipt, json)}\n` - await new Promise((resolve, reject) => { - const onError = (error: Error) => { - cleanup() - reject(error) - } - const cleanup = () => process.stdout.off('error', onError) - process.stdout.once('error', onError) - try { - process.stdout.write(output, (error) => { - cleanup() - if (error) reject(error) - else resolve() - }) - } catch (error) { - cleanup() - reject(error) - } - }) - } catch { - throw new MattermostDeliveryConfirmedError() - } -} - -function safeSendLabel(value: string, redact: boolean): string { - return safeString(value, redact).replace(/\r?\n/g, '\\n').replace(/\t/g, '\\t') -} - -export async function sendDirectMessage(options: SendDirectMessageOptions): Promise { - if (options.message?.includes(options.token)) { - throw new Error('Refusing to send the active Mattermost credential.') - } - const username = options.username.replace(/^@/, '').trim() - if (!username) throw new Error('A direct-message username is required.') - initClient(options.url, options.token) - - const me = await getMe() - if (typeof me?.id !== 'string' || me.id.trim().length === 0) { - throw new Error('Mattermost returned an invalid identity response.') - } - const recipient = await getUserByUsername(username) - if ( - typeof recipient?.id !== 'string' || - recipient.id.trim().length === 0 || - typeof recipient.username !== 'string' || - recipient.username.length === 0 || - recipient.username.toLowerCase() !== username.toLowerCase() - ) { - throw new Error('Mattermost returned an invalid user response.') - } - - const channels = validateAndDedupeConversationChannels(await getMyDMChannels(me.id), me.id) - let channel = - channels.find((candidate) => getOtherUserIdFromDMChannel(candidate, me.id) === recipient.id) ?? - null - const label = `@${safeSendLabel(recipient.username, options.redact)}` - - if (options.dryRun) { - emitSendReceipt( - { - status: 'dry_run', - destination: { - type: 'dm', - label, - channelId: channel ? safeSendLabel(channel.id, options.redact) : null, - willCreate: channel === null, - }, - }, - options.json, - ) - return - } - - if (options.message === undefined) throw new Error('Message content is required.') - if (!channel) { - try { - channel = requireConversationChannelContext( - await createDirectChannel(me.id, recipient.id), - me.id, - ) - } catch (error) { - if (error instanceof MattermostMutationOutcomeUnknownError) { - throw new Error( - 'Mattermost did not confirm DM setup. The message was not attempted; run a dry-run before retrying.', - ) - } - throw error - } - } - const post = await createPost(channel.id, options.message) - if (post.userId !== me.id) throw new MattermostMutationOutcomeUnknownError() - const safePostId = safeSendLabel(post.id, options.redact) - await writeConfirmedSendReceipt( - { - status: 'sent', - destination: { - type: 'dm', - label, - channelId: safeSendLabel(channel.id, options.redact), - willCreate: false, - }, - post: { - id: safePostId, - createAt: new Date(post.createAt).toISOString(), - pendingPostId: safeSendLabel(post.pendingPostId, options.redact), - permalink: safeSendLabel(buildPostPermalink(options.url, safePostId), options.redact), - }, - }, - options.json, - ) -} - -export async function sendGroupMessage(options: SendGroupMessageOptions): Promise { - if (options.message?.includes(options.token)) { - throw new Error('Refusing to send the active Mattermost credential.') - } - if (!options.channelId.trim()) throw new Error('A group-DM channel ID is required.') - initClient(options.url, options.token) - const channel = requireRawChannelShape(await getChannel(options.channelId), options.channelId) - if (channel.type !== 'G') { - throw new Error(`Channel "${presentOneLine(channel.id, options.redact)}" is not a group DM.`) - } - const label = safeSendLabel(channel.display_name || channel.name, options.redact) - const channelId = safeSendLabel(channel.id, options.redact) - - if (options.dryRun) { - emitSendReceipt( - { - status: 'dry_run', - destination: { type: 'group', label, channelId, willCreate: false }, - }, - options.json, - ) - return - } - - if (options.message === undefined) throw new Error('Message content is required.') - const me = await getMe() - if (typeof me?.id !== 'string' || me.id.trim().length === 0) { - throw new Error('Mattermost returned an invalid identity response.') - } - const post = await createPost(channel.id, options.message) - if (post.userId !== me.id) throw new MattermostMutationOutcomeUnknownError() - const safePostId = safeSendLabel(post.id, options.redact) - await writeConfirmedSendReceipt( - { - status: 'sent', - destination: { type: 'group', label, channelId, willCreate: false }, - post: { - id: safePostId, - createAt: new Date(post.createAt).toISOString(), - pendingPostId: safeSendLabel(post.pendingPostId, options.redact), - permalink: safeSendLabel(buildPostPermalink(options.url, safePostId), options.redact), - }, - }, - options.json, - ) -} - -export async function fetchDMs(options: DMsOptions): Promise { - if (options.cursor !== undefined) decodeChannelHistoryCursor(options.cursor) - if (options.cursor !== undefined && !options.channel) { - throw new Error('A cursor requires --channel for direct-message history.') - } - if (options.cursor !== undefined && options.user.length > 0) { - throw new Error('A cursor cannot be combined with --user.') - } - if (options.cursor !== undefined && options.sinceExplicit) { - throw new Error('A cursor cannot be combined with --since.') - } - initClient(options.url, options.token) - - let myUserId: string - let channels: Channel[] = [] - - if (options.channel) { - const channel = requireRawChannelShape(await getChannel(options.channel), options.channel) - if (channel.type !== 'D') { - throw new Error( - `Channel "${presentOneLine(channel.id, options.redact)}" is not a direct-message channel.`, - ) - } - channels = [channel] - myUserId = (await getMe()).id - } else if (options.user.length > 0) { - const me = await getMe() - myUserId = me.id - const discoveredDMChannels = (await getMyChannels(me.id)).filter( - (channel) => channel.type === 'D', - ) - const dmChannels = validateAndDedupeConversationChannels(discoveredDMChannels, me.id) - for (const username of options.user) { - try { - const user = await getUserByUsername(username) - if (typeof user?.id !== 'string' || user.id.length === 0) { - throw new Error('Mattermost returned an invalid user response.') - } - const channel = dmChannels.find( - (candidate) => getOtherUserIdFromDMChannel(candidate, me.id) === user.id, - ) - if (channel) { - channels.push(channel) - } else { - console.error( - `Warning: No direct-message channel exists with @${presentOneLine(username, false)}.`, - ) - } - } catch (error) { - if (!(error instanceof MattermostAPIError) || error.status !== 404) throw error - console.error(`Warning: User @${presentOneLine(username, false)} was not found.`) - } - } - } else { - const me = await getMe() - myUserId = me.id - channels = await getMyDMChannels(me.id) - } - - channels = [...new Map(channels.map((channel) => [channel.id, channel])).values()] - - await fetchConversationChannels(channels, myUserId, options) -} - -export async function fetchGroupDMs(options: GroupDMsOptions): Promise { - if (options.cursor !== undefined) decodeChannelHistoryCursor(options.cursor) - if (options.cursor !== undefined && !options.channel) { - throw new Error('A cursor requires --channel for group-DM history.') - } - if (options.cursor !== undefined && options.sinceExplicit) { - throw new Error('A cursor cannot be combined with --since.') - } - initClient(options.url, options.token) - - if (options.channel) { - const channel = requireRawChannelShape(await getChannel(options.channel), options.channel) - if (channel.type !== 'G') { - throw new Error(`Channel "${presentOneLine(channel.id, options.redact)}" is not a group DM.`) - } - const me = await getMe() - await fetchConversationChannels([channel], me.id, options) - return - } - - const me = await getMe() - const channels = await getMyGroupDMChannels(me.id) - await fetchConversationChannels( - [...new Map(channels.map((channel) => [channel.id, channel])).values()], - me.id, - options, - ) -} - -async function fetchConversationChannels( - channels: Channel[], - myUserId: string, - options: GroupDMsOptions | DMsOptions, -): Promise { - channels = validateAndDedupeConversationChannels(channels, myUserId) - if (channels.length === 0) { - formatOutput([], options) - return - } - - printRedactionWarning(options.redact) - - const cursor = - options.cursor !== undefined ? decodeChannelHistoryCursor(options.cursor) : undefined - if (cursor && (channels.length !== 1 || cursor.channelId !== channels[0]?.id)) { - throw new Error('Cursor does not match the selected channel.') - } - const since = cursor - ? (cursor.since ?? undefined) - : options.since - ? parseDuration(options.since) - : undefined - const channelPosts = new Map() - const truncationStates: Array = [] - const allPosts: Post[] = [] - let inputSafeBeforeValid = true - - for (const channel of channels) { - const result = await getAllChannelPosts(channel.id, { - limit: options.limit, - since, - boundary: cursor?.boundary, - safeBeforePostId: cursor?.safeBeforePostId, - }) - const posts = result.posts - truncationStates.push(result.truncated) - if (cursor && result.safeBeforeValid === false) inputSafeBeforeValid = false - if (posts.length === 0) continue - - channelPosts.set(channel.id, posts) - allPosts.push(...posts) - } - - if (allPosts.length === 0 && !(cursor && truncationStates.some((state) => state === null))) { - requireConfirmedEmpty(0, mergeTruncation(truncationStates, 0, options.limit)) - formatOutput([], options) - return - } - - // `--limit` is a total output budget across all matched conversation channels. - const selectedPostIds = new Set( - takeMostRecentPosts(allPosts, options.limit).map((post) => post.id), - ) - const queryTruncated = mergeTruncation(truncationStates, allPosts.length, options.limit) - const selectedPosts = allPosts.filter((post) => selectedPostIds.has(post.id)) - const lastSelected = takeMostRecentPosts(selectedPosts, options.limit).at(-1) - const safeBeforePostId = lastSelected - ? ([...selectedPosts].reverse().find((post) => post.create_at > lastSelected.create_at)?.id ?? - (inputSafeBeforeValid ? cursor?.safeBeforePostId : undefined)) - : undefined - const nextCursor = - options.channel && lastSelected && queryTruncated !== false - ? encodeChannelHistoryCursor({ - v: 1, - scope: 'channel', - channelId: channels[0]?.id ?? '', - boundary: { createAt: lastSelected.create_at, id: lastSelected.id }, - since: since ?? null, - ...(safeBeforePostId === undefined ? {} : { safeBeforePostId }), - }) - : cursor && selectedPosts.length === 0 && queryTruncated === null - ? (options.cursor ?? null) - : null - if (selectedPosts.length === 0 && cursor && queryTruncated === null) { - const channel = channels[0] - if (!channel) throw new Error('Cursor channel is unavailable.') - const channelRedactions: Redaction[] = [] - const processedChannel = await buildProcessedChannel( - channel, - myUserId, - options.redact, - channelRedactions, - ) - formatOutput( - [ - { - channel: processedChannel, - messages: [], - redactions: channelRedactions, - retrieval: retrievalMetadata( - { - source: 'recent', - requestedLimit: options.limit, - since: since === undefined ? null : new Date(since).toISOString(), - queryTruncated, - inputCursor: options.cursor ?? null, - nextCursor, - }, - 0, - { - status: options.threads ? 'complete' : 'not_requested', - hydratedRootCount: 0, - failedRootIds: [], - }, - 0, - ), - }, - ], - options, - ) - return - } - const outputs = await buildOutputsFromPosts( - selectedPosts, - myUserId, - options, - { - source: 'recent', - requestedLimit: options.limit, - since: since === undefined ? null : new Date(since).toISOString(), - queryTruncated, - inputCursor: options.cursor ?? null, - nextCursor, - }, - new Map(channels.map((channel) => [channel.id, channel])), - ) - - formatOutput(outputs, options) -} - -export async function fetchChannel(options: ChannelOptions): Promise { - const cursor = - options.cursor !== undefined ? decodeChannelHistoryCursor(options.cursor) : undefined - if (options.cursor !== undefined && options.sinceExplicit) { - throw new Error('A cursor cannot be combined with --since.') - } - initClient(options.url, options.token) - - const me = await getMe() - const teamId = await resolveTeamId(options.team) - const channel = requireRawChannelShape(await getChannelByName(teamId, options.channel)) - if ( - (channel.type !== 'O' && channel.type !== 'P') || - channel.team_id !== teamId || - channel.name !== normalizeChannelName(options.channel) - ) { - throw new Error('Mattermost returned an invalid channel response.') - } - if (cursor && cursor.channelId !== channel.id) { - throw new Error('Cursor does not match the selected channel.') - } - - printRedactionWarning(options.redact) - - const since = cursor - ? (cursor.since ?? undefined) - : options.since - ? parseDuration(options.since) - : undefined - const result = await getAllChannelPosts(channel.id, { - limit: options.limit, - since, - boundary: cursor?.boundary, - safeBeforePostId: cursor?.safeBeforePostId, - }) - const posts = result.posts - - if (posts.length === 0 && !(cursor && result.truncated === null)) { - requireConfirmedEmpty(0, result.truncated) - formatOutput([], options) - return - } - - if (posts.length === 0 && cursor && result.truncated === null) { - const channelRedactions: Redaction[] = [] - const processedChannel = await buildProcessedChannel( - channel, - me.id, - options.redact, - channelRedactions, - ) - formatOutput( - [ - { - channel: processedChannel, - messages: [], - redactions: channelRedactions, - retrieval: retrievalMetadata( - { - source: 'recent', - requestedLimit: options.limit, - since: since === undefined ? null : new Date(since).toISOString(), - queryTruncated: null, - inputCursor: options.cursor ?? null, - nextCursor: options.cursor ?? null, - }, - 0, - { - status: options.threads ? 'complete' : 'not_requested', - hydratedRootCount: 0, - failedRootIds: [], - }, - 0, - ), - }, - ], - options, - ) - return - } - - const boundaryPost = posts.at(-1) - const safeBeforePostId = boundaryPost - ? ([...posts].reverse().find((post) => post.create_at > boundaryPost.create_at)?.id ?? - (result.safeBeforeValid === false ? undefined : cursor?.safeBeforePostId)) - : undefined - - const outputs = await buildOutputsFromPosts( - posts, - me.id, - options, - { - source: 'recent', - requestedLimit: options.limit, - since: since === undefined ? null : new Date(since).toISOString(), - queryTruncated: result.truncated, - inputCursor: options.cursor ?? null, - nextCursor: - posts.length > 0 && result.truncated !== false - ? encodeChannelHistoryCursor({ - v: 1, - scope: 'channel', - channelId: channel.id, - boundary: { createAt: posts.at(-1)?.create_at ?? 0, id: posts.at(-1)?.id ?? '' }, - since: since ?? null, - ...(safeBeforePostId === undefined ? {} : { safeBeforePostId }), - }) - : null, - }, - new Map([[channel.id, channel]]), - ) - formatOutput(outputs, options) -} - -export async function fetchThread(options: CLIOptions & { postId: string }): Promise { - initClient(options.url, options.token) - - printRedactionWarning(options.redact) - - const me = await getMe() - const result = await getPostThread(options.postId) - const posts = result.posts - - if (posts.length === 0) { - console.error('Thread not found or empty') - process.exit(1) - } - - const rootPost = posts.find((post) => !post.root_id) - const channelPost = rootPost ?? posts[0] - const missingRootId = rootPost ? undefined : posts.find((post) => post.root_id)?.root_id - const requestedRootId = rootPost?.id ?? missingRootId ?? options.postId - const threadComplete = result.truncated === false && rootPost !== undefined - if (result.truncated !== false || !rootPost) { - const safeRequestedRootId = preprocess(requestedRootId, { redact: options.redact }) - .text.replace(/\n/g, '\\n') - .replace(/\t/g, '\\t') - console.error(`Warning: Thread ${safeRequestedRootId} could only be partially hydrated.`) - } - - const userIds = postUserIds(posts) - if (userIds.length > 0) { - await getUsersByIds(userIds) - } - - const usersById = new Map( - userIds.flatMap((id) => { - const user = getCachedUser(id) - return user ? ([[id, user]] as const) : [] - }), - ) - const { messages, redactions } = normalizePosts( - posts, - usersById, - me.id, - options.url, - buildPostPermalink, - options.redact, - ) - const threadState: RetrievalMetadata['visibleThreads'] = { - status: threadComplete ? 'complete' : 'partial', - hydratedRootCount: threadComplete ? 1 : 0, - failedRootIds: threadComplete ? [] : [requestedRootId], - } - const presentedThreadState = presentVisibleThreads(threadState, options.redact, redactions) - - const { channel: processedChannel, redactions: channelRedactions } = - await resolveProcessedChannel(channelPost?.channel_id ?? '', me.id, options.redact) - - formatOutput( - [ - { - channel: processedChannel, - messages: groupIntoThreads(messages), - redactions: [...channelRedactions, ...redactions], - retrieval: retrievalMetadata( - { - source: 'thread', - requestedLimit: null, - since: null, - queryTruncated: result.truncated, - }, - posts.length, - presentedThreadState, - ), - }, - ], - options, - ) -} - -export async function searchMessages(options: SearchOptions): Promise { - initClient(options.url, options.token) - - const query = options.query.trim() - if (!query) { - console.error('Error: Search query cannot be empty') - process.exit(1) - } - - printRedactionWarning(options.redact) - - const me = await getMe() - const teamId = await resolveTeamId(options.team) - const response = await searchPosts(teamId, query, options.limit) - - const posts = response.order - .map((id) => response.posts[id]) - .filter((post): post is Post => !!post && post.delete_at === 0) - .slice(0, options.limit) - - if (posts.length === 0) { - requireConfirmedEmpty(0, response.truncated) - formatOutput([], options) - return - } - - const outputs = await buildOutputsFromPosts(posts, me.id, options, { - source: 'search', - requestedLimit: options.limit, - since: null, - queryTruncated: response.truncated, - }) - formatOutput(outputs, options) -} - -export async function fetchMentions(options: MentionOptions): Promise { - initClient(options.url, options.token) - - printRedactionWarning(options.redact) - - const me = await getMe() - const teamId = await resolveTeamId(options.team) - - const baseTerms = [`@${me.username}`] - for (const mentionName of options.mentionNames) { - if (mentionName.trim().length > 0) { - baseTerms.push(`"${mentionName.trim()}"`) - } - } - - const modifiers: string[] = [] - const since = options.since ? parseDuration(options.since) : undefined - if (since !== undefined) { - modifiers.push(`after:${mentionSearchAfterDate(since)}`) - } - if (options.channel) { - modifiers.push(`in:${normalizeChannelName(options.channel)}`) - } - - const dedupedPosts = new Map() - const truncationStates: Array = [] - - const searchTerms = [...new Set(baseTerms)] - - for (const term of searchTerms) { - const searchTerm = [term, ...modifiers].join(' ') - const response = await searchPosts(teamId, searchTerm, options.limit, (post) => - isExactMentionPost(post, term, since), - ) - truncationStates.push(response.truncated) - - for (const id of response.order) { - const post = response.posts[id] - if (post) { - dedupedPosts.set(post.id, post) - } - } - } - - const posts = takeMostRecentPosts([...dedupedPosts.values()], options.limit) - const queryTruncated = mergeTruncation(truncationStates, dedupedPosts.size, options.limit) - - if (posts.length === 0) { - requireConfirmedEmpty(0, queryTruncated) - formatOutput([], options) - return - } - - const outputs = await buildOutputsFromPosts(posts, me.id, options, { - source: 'mentions', - requestedLimit: options.limit, - since: since === undefined ? null : new Date(since).toISOString(), - queryTruncated, - }) - formatOutput(outputs, options) -} - -export async function showUnread(options: UnreadOptions): Promise { - initClient(options.url, options.token) - - const me = await getMe() - const teamId = await resolveTeamId(options.team) - const channels = [ - ...new Map((await getMyChannels()).map((channel) => [channel.id, channel])).values(), - ].filter( - (channel) => - channel.type === 'D' || - channel.type === 'G' || - ((channel.type === 'O' || channel.type === 'P') && channel.team_id === teamId), - ) - const teamMembers = await getTeamChannelMembers(teamId) - - const memberByChannelId = new Map(teamMembers.map((member) => [member.channel_id, member])) - const unreadEntries: UnreadSummaryItem[] = [] - - for (const channel of channels) { - let member: ChannelMember | undefined = memberByChannelId.get(channel.id) - - if (!member && (channel.type === 'D' || channel.type === 'G')) { - member = await getChannelMember(channel.id) - } - - if (!member) continue - - const { unreadCount, mentionCount } = calculateUnreadMetrics(channel, member) - if (unreadCount <= 0) continue - - const processedChannel = await buildProcessedChannel(channel, me.id, options.redact) - - unreadEntries.push({ - channel, - processedChannel, - unreadCount, - mentionCount, - lastViewedAt: nonNegativeInteger(member.last_viewed_at), - }) - } - - const sortedEntries = sortUnreadEntries(unreadEntries) - - if (sortedEntries.length === 0) { - console.log(options.json ? JSON.stringify({ unread: [] }, null, 2) : 'All caught up!') - return - } - - const buildPeekOutput = async (entry: UnreadSummaryItem): Promise => { - const result = await getAllChannelPosts(entry.channel.id, { - limit: options.peek, - since: entry.lastViewedAt || undefined, - }) - if (result.posts.length === 0) { - requireConfirmedEmpty(0, result.truncated) - return undefined - } - const outputs = await buildOutputsFromPosts( - result.posts, - me.id, - options, - { - source: 'unread', - requestedLimit: options.peek ?? null, - since: entry.lastViewedAt ? new Date(entry.lastViewedAt).toISOString() : null, - queryTruncated: result.truncated, - }, - new Map([[entry.channel.id, entry.channel]]), - ) - return outputs[0] - } - - if (options.json) { - const result: { - unread: Array<{ - channel: ProcessedChannel - unreadCount: number - mentionCount: number - lastViewedAt: number - }> - peek?: MessageOutput[] - } = { - unread: sortedEntries.map((entry) => ({ - channel: entry.processedChannel, - unreadCount: entry.unreadCount, - mentionCount: entry.mentionCount, - lastViewedAt: entry.lastViewedAt, - })), - } - - if (options.peek && options.peek > 0) { - const peekOutputs: MessageOutput[] = [] - - for (const entry of sortedEntries) { - const output = await buildPeekOutput(entry) - if (output) peekOutputs.push(output) - } - - result.peek = peekOutputs - } - - console.log(JSON.stringify(result, null, 2)) - return - } - - console.log('Unread Channels:\n') - for (const entry of sortedEntries) { - const summary = `${entry.unreadCount} unread${ - entry.mentionCount > 0 ? `, ${entry.mentionCount} mentions` : '' - }` - console.log(` ${channelLabel(entry.processedChannel).padEnd(32)} ${summary}`) - } - - console.log(`\nTotal: ${sortedEntries.length} channels with unread messages`) - - if (!options.peek || options.peek <= 0) return - - const peekOutputs: MessageOutput[] = [] - - for (const entry of sortedEntries) { - const output = await buildPeekOutput(entry) - if (output) peekOutputs.push(output) - } - - if (peekOutputs.length > 0) { - console.log('') - formatOutput(peekOutputs, options) - } -} - -export async function watchChannel( - options: CLIOptions & { channel?: string; team?: string; dm?: string }, -): Promise { - initClient(options.url, options.token) - const presentLabel = (value: string): string => - preprocess(value, { redact: options.redact }).text.replace(/\n/g, '\\n').replace(/\t/g, '\\t') - - printRedactionWarning(options.redact) - - if (options.channel && options.dm) { - console.error('Error: Use either a channel target or --dm , not both.') - process.exit(1) - } - if (!options.channel && !options.dm) { - console.error('Error: Provide a channel name or use --dm .') - process.exit(1) - } - - const channel = options.dm - ? await getDMChannelByUsername(options.dm) - : await getChannelByName(await resolveTeamId(options.team), options.channel as string) - if (!channel) { - const target = options.dm - ? `DM channel with @${presentLabel(options.dm)}` - : `channel #${presentLabel(options.channel ?? '')}` - console.error(`Error: ${target} not found.`) - process.exit(1) - } - - const watchTarget = - channel.type === 'D' - ? `DMs with @${presentLabel(options.dm ?? '')}` - : `#${presentLabel(channel.name)}` - console.error(`Watching ${watchTarget} (Ctrl+C to stop)`) - - await new Promise((resolve, reject) => { - let closed = false - const handlePost = createWatchPostHandler(options) - - const closeAndCleanup = (closeSocket: () => void): void => { - if (closed) return - closed = true - process.off('SIGINT', handleSigint) - process.off('SIGTERM', handleSigterm) - closeSocket() - } - - const handleSignal = (): void => { - closeAndCleanup(connection.close) - resolve() - } - const handleSigint = handleSignal - const handleSigterm = handleSignal - - const connection = connectWebSocket( - options.url, - options.token, - handlePost, - (error) => { - closeAndCleanup(connection.close) - reject(error) - }, - { - channelId: channel.id, - diagnostics: { - reconnect: (attempt, delayMs) => - console.error( - `WebSocket disconnected; reconnecting in ${delayMs}ms (attempt ${attempt}).`, - ), - gap: ({ expected, received }) => - console.error( - `Warning: WebSocket sequence gap detected (expected ${expected}, received ${received}); live events may be missing.`, - ), - malformed: (message) => console.error(`Warning: ${message}`), - }, - }, - ) - - process.on('SIGINT', handleSigint) - process.on('SIGTERM', handleSigterm) - }) -} diff --git a/src/config.ts b/src/config.ts deleted file mode 100644 index 47e861f..0000000 --- a/src/config.ts +++ /dev/null @@ -1,216 +0,0 @@ -// Config file handling for ~/.config/mattermost-cli/config.toml - -import { access, mkdir, readFile, stat, writeFile } from 'node:fs/promises' -import { homedir } from 'node:os' -import { join } from 'node:path' -import { parse as parseTOML } from 'smol-toml' -import { setActiveMattermostCredential } from './preprocessing' - -export interface FileConfig { - url?: string - token?: string - redact?: boolean - mention_names?: string[] -} - -export type ConfigSource = 'cli' | 'env' | 'file' | 'missing' - -export interface ResolvedConfigState { - url?: string - token?: string - urlSource: ConfigSource - tokenSource: ConfigSource - fileConfig: FileConfig - configPath: string - fileExists: boolean - insecurePermissions: boolean - fileError?: 'read' | 'parse' -} - -const CONFIG_PATH = join(homedir(), '.config', 'mattermost-cli', 'config.toml') - -/** - * Check if config file has insecure permissions (group/other readable). - * Returns true if permissions are too open. - */ -async function hasInsecurePermissions(): Promise { - try { - const stats = await stat(CONFIG_PATH) - // Check if group or other have any permissions (mode & 0o077) - return (stats.mode & 0o077) !== 0 - } catch { - return false - } -} - -async function inspectConfigFile(configPath: string): Promise<{ - config: FileConfig - exists: boolean - insecurePermissions: boolean - error?: 'read' | 'parse' -}> { - let insecurePermissions = false - try { - const stats = await stat(configPath) - insecurePermissions = (stats.mode & 0o077) !== 0 - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') { - return { config: {}, exists: false, insecurePermissions: false } - } - return { config: {}, exists: true, insecurePermissions: false, error: 'read' } - } - - let content: string - try { - content = await readFile(configPath, 'utf-8') - } catch { - return { config: {}, exists: true, insecurePermissions, error: 'read' } - } - - try { - const parsed = parseTOML(content) - const url = typeof parsed.url === 'string' ? parsed.url.trim() : undefined - const token = typeof parsed.token === 'string' ? parsed.token.trim() : undefined - const redact = typeof parsed.redact === 'boolean' ? parsed.redact : undefined - const mentionNames = Array.isArray(parsed.mention_names) - ? parsed.mention_names - .filter((value): value is string => typeof value === 'string') - .map((value) => value.trim()) - .filter((value) => value.length > 0) - : undefined - - return { - config: { - url: url || undefined, - token: token || undefined, - redact, - mention_names: mentionNames, - }, - exists: true, - insecurePermissions, - } - } catch { - return { config: {}, exists: true, insecurePermissions, error: 'parse' } - } -} - -export async function resolveConfigState( - options: { url?: string; token?: string }, - environment: NodeJS.ProcessEnv = process.env, - configPath = CONFIG_PATH, -): Promise { - const file = await inspectConfigFile(configPath) - const url = options.url || environment.MM_URL || file.config.url - const token = options.token || environment.MM_TOKEN || file.config.token - if (token) setActiveMattermostCredential(token) - - return { - url, - token, - urlSource: options.url - ? 'cli' - : environment.MM_URL - ? 'env' - : file.config.url - ? 'file' - : 'missing', - tokenSource: options.token - ? 'cli' - : environment.MM_TOKEN - ? 'env' - : file.config.token - ? 'file' - : 'missing', - fileConfig: file.config, - configPath, - fileExists: file.exists, - insecurePermissions: file.insecurePermissions, - fileError: file.error, - } -} - -async function fileExists(path: string): Promise { - try { - await access(path) - return true - } catch { - return false - } -} - -export async function loadConfigFile(): Promise { - const state = await inspectConfigFile(CONFIG_PATH) - if (state.insecurePermissions) { - console.warn( - `Warning: ${CONFIG_PATH} has insecure permissions.\n` + ` Run: chmod 600 "${CONFIG_PATH}"`, - ) - } - if (state.error) { - console.warn( - `Warning: Could not ${state.error === 'parse' ? 'parse' : 'read'} config at ${CONFIG_PATH}`, - ) - } - return state.config -} - -export function getConfigPath(): string { - return CONFIG_PATH -} - -const CONFIG_TEMPLATE = `# Mattermost CLI Configuration -# https://github.com/ardasevinc/mattermost-cli - -url = "https://mattermost.example.com" -token = "your-personal-access-token" -# mention_names = ["Arda", "arda.sevinc"] -` - -export async function initConfigFile(): Promise<{ created: boolean; path: string }> { - const { dirname } = await import('node:path') - - const dir = dirname(CONFIG_PATH) - - // Create directory if it doesn't exist - await mkdir(dir, { recursive: true }) - - if (await fileExists(CONFIG_PATH)) { - return { created: false, path: CONFIG_PATH } - } - - // Write template atomically with secure permissions (0o600) - // Using 'wx' flag ensures we don't overwrite if file was created between check and write - await writeFile(CONFIG_PATH, CONFIG_TEMPLATE, { mode: 0o600, flag: 'wx' }) - - return { created: true, path: CONFIG_PATH } -} - -export async function getConfigStatus(): Promise<{ - exists: boolean - path: string - hasUrl: boolean - hasToken: boolean - insecurePerms: boolean -}> { - const exists = await fileExists(CONFIG_PATH) - - if (!exists) { - return { - exists: false, - path: CONFIG_PATH, - hasUrl: false, - hasToken: false, - insecurePerms: false, - } - } - - const insecurePerms = await hasInsecurePermissions() - const config = await loadConfigFile() - - return { - exists: true, - path: CONFIG_PATH, - hasUrl: !!config.url, - hasToken: !!config.token, - insecurePerms, - } -} diff --git a/src/cursor.ts b/src/cursor.ts deleted file mode 100644 index e56db2c..0000000 --- a/src/cursor.ts +++ /dev/null @@ -1,107 +0,0 @@ -export interface ChannelHistoryCursor { - v: 1 - scope: 'channel' - channelId: string - boundary: { createAt: number; id: string } - since: number | null - safeBeforePostId?: string -} - -const MAX_ENCODED_CURSOR_LENGTH = 2048 -const MAX_DECODED_CURSOR_LENGTH = 1536 -const MAX_DATE_MILLISECONDS = 8_640_000_000_000_000 -const MAX_ID_LENGTH = 128 -const SAFE_ID = /^[A-Za-z0-9_-]+$/ - -function invalidCursor(): never { - throw new Error('Invalid cursor.') -} - -function exactKeys(value: Record, required: string[], optional: string[] = []) { - const allowed = new Set([...required, ...optional]) - return ( - required.every((key) => Object.hasOwn(value, key)) && - Object.keys(value).every((key) => allowed.has(key)) - ) -} - -function isSafeId(value: unknown): value is string { - return ( - typeof value === 'string' && - value.length > 0 && - value.length <= MAX_ID_LENGTH && - SAFE_ID.test(value) - ) -} - -function validateCursor(value: unknown): ChannelHistoryCursor { - if (!value || typeof value !== 'object' || Array.isArray(value)) return invalidCursor() - const candidate = value as Record - if ( - !exactKeys(candidate, ['v', 'scope', 'channelId', 'boundary', 'since'], ['safeBeforePostId']) - ) { - return invalidCursor() - } - const boundary = candidate.boundary - if (!boundary || typeof boundary !== 'object' || Array.isArray(boundary)) return invalidCursor() - const boundaryValue = boundary as Record - if (!exactKeys(boundaryValue, ['createAt', 'id'])) return invalidCursor() - if ( - candidate.v !== 1 || - candidate.scope !== 'channel' || - !isSafeId(candidate.channelId) || - !isSafeId(boundaryValue.id) || - typeof boundaryValue.createAt !== 'number' || - !Number.isSafeInteger(boundaryValue.createAt) || - boundaryValue.createAt < 0 || - boundaryValue.createAt > MAX_DATE_MILLISECONDS || - !( - candidate.since === null || - (typeof candidate.since === 'number' && - Number.isSafeInteger(candidate.since) && - candidate.since >= 0 && - candidate.since <= boundaryValue.createAt) - ) || - (Object.hasOwn(candidate, 'safeBeforePostId') && !isSafeId(candidate.safeBeforePostId)) - ) { - return invalidCursor() - } - return value as ChannelHistoryCursor -} - -export function encodeChannelHistoryCursor(cursor: ChannelHistoryCursor): string { - const validated = validateCursor(cursor) - const encoded = Buffer.from(JSON.stringify(validated), 'utf8').toString('base64url') - if (encoded.length > MAX_ENCODED_CURSOR_LENGTH) return invalidCursor() - return encoded -} - -export function decodeChannelHistoryCursor(encoded: string): ChannelHistoryCursor { - if ( - encoded.length === 0 || - encoded.length > MAX_ENCODED_CURSOR_LENGTH || - !/^[A-Za-z0-9_-]+$/.test(encoded) - ) { - return invalidCursor() - } - let text: string - try { - const bytes = Buffer.from(encoded, 'base64url') - if (bytes.length === 0 || bytes.length > MAX_DECODED_CURSOR_LENGTH) return invalidCursor() - text = bytes.toString('utf8') - if (Buffer.from(text, 'utf8').toString('base64url') !== encoded) return invalidCursor() - } catch { - return invalidCursor() - } - let value: unknown - try { - value = JSON.parse(text) - } catch { - return invalidCursor() - } - return validateCursor(value) -} - -export function comparePostIds(a: string, b: string): number { - return a < b ? -1 : a > b ? 1 : 0 -} diff --git a/src/doctor.ts b/src/doctor.ts deleted file mode 100644 index ea65165..0000000 --- a/src/doctor.ts +++ /dev/null @@ -1,244 +0,0 @@ -import { normalizeServerUrl } from './api/url' -import type { ResolvedConfigState } from './config' -import { preprocess, sanitizeTerminalLabel } from './preprocessing' - -export type DoctorStatus = 'pass' | 'warn' | 'fail' | 'skipped' - -export interface DoctorCheck { - name: 'configuration' | 'server' | 'authentication' - status: DoctorStatus - message: string - details?: Record -} - -export interface DoctorReport { - ok: boolean - checks: DoctorCheck[] -} - -type Fetcher = typeof fetch - -export interface DoctorOptions { - fetcher?: Fetcher - redact?: boolean - timeoutMs?: number -} - -const DEFAULT_TIMEOUT_MS = 10_000 - -function configurationCheck(config: ResolvedConfigState): DoctorCheck { - const details = { urlSource: config.urlSource, tokenSource: config.tokenSource } - if (config.fileError) { - return { - name: 'configuration', - status: 'fail', - message: 'config file could not be loaded', - details, - } - } - if (config.insecurePermissions && config.fileConfig.token) { - return { - name: 'configuration', - status: 'fail', - message: 'config file permissions expose a stored token; run chmod 600', - details, - } - } - if (!config.url || !config.token) { - return { - name: 'configuration', - status: 'fail', - message: 'configuration is incomplete', - details, - } - } - if (config.insecurePermissions) { - return { - name: 'configuration', - status: 'warn', - message: 'config file permissions are broader than recommended; run chmod 600', - details, - } - } - return { name: 'configuration', status: 'pass', message: 'credentials resolved', details } -} - -async function requestJson( - fetcher: Fetcher, - url: string, - headers?: RequestInit['headers'], - timeoutMs = DEFAULT_TIMEOUT_MS, -): Promise<{ ok: true; value: unknown } | { ok: false; status?: number }> { - const controller = new AbortController() - let timer: ReturnType | undefined - try { - const request = (async (): Promise< - { ok: true; value: unknown } | { ok: false; status?: number } - > => { - const response = await fetcher(url, { - method: 'GET', - headers, - signal: controller.signal, - }) - if (!response.ok) return { ok: false, status: response.status } - try { - return { ok: true, value: await response.json() } - } catch { - return { ok: false, status: response.status } - } - })() - const timeout = new Promise<{ ok: false }>((resolve) => { - timer = setTimeout(() => { - controller.abort() - resolve({ ok: false }) - }, timeoutMs) - }) - return await Promise.race([request, timeout]) - } catch { - return { ok: false } - } finally { - if (timer !== undefined) clearTimeout(timer) - } -} - -function safeRemoteString(value: unknown, config: ResolvedConfigState, redact: boolean): string { - if (typeof value !== 'string' || value.length === 0) return 'unknown' - const withoutConfiguredToken = config.token - ? value.split(config.token).join('[REDACTED:token]') - : value - return sanitizeTerminalLabel(preprocess(withoutConfiguredToken, { redact }).text) -} - -export async function runDoctor( - config: ResolvedConfigState, - options: DoctorOptions = {}, -): Promise { - const fetcher = options.fetcher ?? fetch - const redact = options.redact ?? true - const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS - const checks: DoctorCheck[] = [configurationCheck(config)] - let baseUrl: string | undefined - - if (!config.url) { - checks.push({ name: 'server', status: 'skipped', message: 'Mattermost URL is missing' }) - } else { - try { - baseUrl = normalizeServerUrl(config.url) - const result = await requestJson( - fetcher, - `${baseUrl}/api/v4/system/ping?get_server_status=true`, - undefined, - timeoutMs, - ) - if (!result.ok) { - checks.push({ - name: 'server', - status: 'fail', - message: 'server health request failed', - details: result.status ? { httpStatus: result.status } : undefined, - }) - } else { - const payload = - typeof result.value === 'object' && result.value !== null ? result.value : {} - const record = payload as Record - const status = safeRemoteString(record.status, config, redact) - const databaseStatus = safeRemoteString(record.database_status, config, redact) - const filestoreStatus = safeRemoteString(record.filestore_status, config, redact) - const healthValues = [status, databaseStatus, filestoreStatus] - const checkStatus: DoctorStatus = healthValues.some( - (value) => value !== 'OK' && value !== 'unknown', - ) - ? 'fail' - : healthValues.includes('unknown') - ? 'warn' - : 'pass' - checks.push({ - name: 'server', - status: checkStatus, - message: - checkStatus === 'pass' - ? 'server is healthy' - : checkStatus === 'warn' - ? 'server responded with incomplete health data' - : 'server reported an unhealthy component', - details: { - status, - databaseStatus, - filestoreStatus, - }, - }) - } - } catch { - checks.push({ - name: 'server', - status: 'fail', - message: 'Mattermost URL is invalid or unsafe', - }) - } - } - - if (!baseUrl || !config.token) { - checks.push({ - name: 'authentication', - status: 'skipped', - message: !config.token ? 'Mattermost token is missing' : 'valid Mattermost URL is missing', - }) - } else { - const result = await requestJson( - fetcher, - `${baseUrl}/api/v4/users/me`, - { - Authorization: `Bearer ${config.token}`, - }, - timeoutMs, - ) - if (!result.ok) { - checks.push({ - name: 'authentication', - status: 'fail', - message: 'authentication request failed', - details: result.status ? { httpStatus: result.status } : undefined, - }) - } else { - const payload = typeof result.value === 'object' && result.value !== null ? result.value : {} - const record = payload as Record - if ( - typeof record.id !== 'string' || - record.id.trim().length === 0 || - typeof record.username !== 'string' || - record.username.trim().length === 0 - ) { - checks.push({ - name: 'authentication', - status: 'fail', - message: 'authentication response was invalid', - }) - } else { - checks.push({ - name: 'authentication', - status: 'pass', - message: 'authenticated', - details: { - id: safeRemoteString(record.id, config, redact), - username: safeRemoteString(record.username, config, redact), - }, - }) - } - } - } - - return { ok: checks.every((check) => check.status !== 'fail'), checks } -} - -export function formatDoctorReport(report: DoctorReport): string { - return report.checks - .map((check) => { - const details = check.details - ? ` (${Object.entries(check.details) - .map(([key, value]) => `${key}=${String(value)}`) - .join(', ')})` - : '' - return `${check.status.padEnd(7)} ${check.name}: ${check.message}${details}` - }) - .join('\n') -} diff --git a/src/formatters/index.ts b/src/formatters/index.ts deleted file mode 100644 index a0b6afe..0000000 --- a/src/formatters/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Formatter exports - -export { formatJSON, formatJSONCompact } from './json' -export { formatMarkdown } from './markdown' -export { formatPretty } from './pretty' -export { formatWatchEvent, formatWatchJSON } from './watch' diff --git a/src/formatters/json.ts b/src/formatters/json.ts deleted file mode 100644 index 7b24a40..0000000 --- a/src/formatters/json.ts +++ /dev/null @@ -1,11 +0,0 @@ -// JSON output formatter - -import type { MessageOutput } from '../types' - -export function formatJSON(outputs: MessageOutput[]): string { - return JSON.stringify(outputs, null, 2) -} - -export function formatJSONCompact(outputs: MessageOutput[]): string { - return JSON.stringify(outputs) -} diff --git a/src/formatters/markdown.ts b/src/formatters/markdown.ts deleted file mode 100644 index 450458e..0000000 --- a/src/formatters/markdown.ts +++ /dev/null @@ -1,218 +0,0 @@ -// Markdown output formatter - -import type { MessageOutput, ProcessedChannel, ProcessedMessage } from '../types' -import { formatDateLong, formatRelativeTime, formatTime } from '../utils/date' - -export interface MarkdownOptions { - relative?: boolean -} - -export function formatMarkdown(outputs: MessageOutput[], options: MarkdownOptions = {}): string { - const sections: string[] = [] - - for (const output of outputs) { - sections.push(formatChannelMarkdown(output, options.relative ?? false)) - } - - return sections.join('\n\n---\n\n') -} - -function channelHeaderMarkdown(channel: ProcessedChannel): string { - if (channel.type === 'unknown') { - return `## Unknown channel (${escapeMarkdown(channel.id)})` - } - if (channel.type === 'dm') { - return `## DMs with ${escapeMarkdown(channel.name)}` - } - if (channel.type === 'group') { - return `## Group DM: ${escapeMarkdown(channel.name)}` - } - const display = channel.displayName ? ` (${escapeMarkdown(channel.displayName)})` : '' - return `## #${escapeMarkdown(channel.name)}${display}` -} - -function formatChannelMarkdown(output: MessageOutput, relative: boolean): string { - const { channel, messages } = output - const lines: string[] = [] - - lines.push(channelHeaderMarkdown(channel)) - lines.push('') - - // Group messages by date - const messagesByDate = groupByDate(messages) - - for (const [date, msgs] of messagesByDate) { - lines.push(`### ${date}`) - lines.push('') - - for (const msg of msgs) { - lines.push(formatMessage(msg, relative)) - lines.push('') - } - } - - // Add redaction summary if any - if (output.redactions.length > 0) { - lines.push('') - lines.push(`_${output.redactions.length} secret(s) redacted_`) - } - - const state = output.retrieval.selection.queryTruncated - lines.push('') - lines.push( - `_Coverage: ${output.retrieval.selection.selectedCount} selected, ${output.retrieval.visiblePostCount} visible; query ${state === true ? 'truncated' : state === false ? 'complete' : 'completeness unknown'}_`, - ) - if (output.retrieval.selection.nextCursor) { - lines.push(`Next cursor: \`${output.retrieval.selection.nextCursor}\``) - } - - return lines.join('\n') -} - -function formatMessage(msg: ProcessedMessage, relative: boolean, depth: number = 0): string { - const timeStr = relative ? formatRelativeTime(msg.timestamp) : formatTime(msg.timestamp) - const lines: string[] = [] - const prefix = depth > 0 ? '> '.repeat(depth) : '' - - const safePermalink = safeHttpUrl(msg.permalink) - const postId = escapeMarkdown(msg.id) - const postRef = safePermalink ? `[${postId}](<${safePermalink}>)` : postId - const markers = messageMarkers(msg) - lines.push( - `${prefix}**${escapeMarkdown(msg.user)}**${markers ? ` ${markers}` : ''} (${timeStr}, ${postRef}):`, - ) - - // Quote the message content - const quotePrefix = `${prefix}> ` - lines.push(quoteLines(escapeMarkdown(msg.text), quotePrefix)) - const stateParts = [`Updated ${msg.updatedAt.toISOString()}`] - if (msg.editedAt) stateParts.push(`edited ${msg.editedAt.toISOString()}`) - if (msg.deletedAt) stateParts.push(`deleted ${msg.deletedAt.toISOString()}`) - lines.push(`${quotePrefix}_${stateParts.join('; ')}_`) - - // Add file attachments if any - if (msg.fileDetails.length > 0) { - const files = msg.fileDetails - .map((file) => { - const label = file.name ? `${file.name} (${file.id})` : file.id - const details = [ - file.mime, - file.extension, - file.size === undefined ? undefined : `${file.size} B`, - ] - .filter(Boolean) - .join(', ') - return escapeMarkdown(`${label}${details ? `, ${details}` : ''}`) - }) - .join(', ') - lines.push(`${quotePrefix}_Files: ${files}_`) - } - - for (const attachment of msg.attachments) { - const title = attachment.title ?? 'Attachment' - const safeTitleLink = attachment.titleLink ? safeHttpUrl(attachment.titleLink) : undefined - if (attachment.pretext) lines.push(quoteLines(escapeMarkdown(attachment.pretext), quotePrefix)) - lines.push( - safeTitleLink - ? `${quotePrefix}**[${escapeMarkdown(title)}](<${safeTitleLink}>)**` - : `${quotePrefix}**${escapeMarkdown(title)}**`, - ) - if (attachment.authorName) - lines.push(quoteLines(`By: ${escapeMarkdown(attachment.authorName)}`, quotePrefix)) - if (attachment.text) lines.push(quoteLines(escapeMarkdown(attachment.text), quotePrefix)) - for (const field of attachment.fields ?? []) { - lines.push( - quoteLines( - `${field.title ? `**${escapeMarkdown(field.title)}:** ` : ''}${escapeMarkdown(field.value ?? '')}`, - quotePrefix, - ), - ) - } - if (attachment.fallback) - lines.push(quoteLines(`Fallback: ${escapeMarkdown(attachment.fallback)}`, quotePrefix)) - if (attachment.footer) - lines.push(quoteLines(`_${escapeMarkdown(attachment.footer)}_`, quotePrefix)) - if (attachment.color) lines.push(`${quotePrefix}Color: ${escapeMarkdown(attachment.color)}`) - if (attachment.timestamp) - lines.push(`${quotePrefix}Timestamp: ${escapeMarkdown(attachment.timestamp)}`) - for (const url of [ - attachment.authorLink, - attachment.authorIcon, - attachment.footerIcon, - attachment.imageUrl, - attachment.thumbUrl, - ]) { - const safeUrl = url ? safeHttpUrl(url) : undefined - if (safeUrl) lines.push(`${quotePrefix}<${safeUrl}>`) - } - } - - if (msg.reactions.length > 0) { - const reactions = msg.reactions - .map(({ emoji, count, actors }) => { - const names = actors.map((actor) => escapeMarkdown(actor.username ?? actor.id)).join(', ') - return `:${escapeMarkdown(emoji)}: ${count}${names ? ` (${names})` : ''}` - }) - .join(' · ') - lines.push(`${quotePrefix}_Reactions: ${reactions}_`) - } - - // Render replies - if (msg.replies && msg.replies.length > 0) { - lines.push('') - for (const reply of msg.replies) { - lines.push(formatMessage(reply, relative, depth + 1)) - lines.push('') - } - } - - return lines.join('\n') -} - -function messageMarkers(msg: ProcessedMessage): string { - const markers: string[] = [] - if (msg.isDeleted) markers.push('[deleted]') - else if (msg.editedAt) markers.push('[edited]') - if (msg.isSystem) - markers.push(msg.postType ? `[system:${escapeMarkdown(msg.postType)}]` : '[system]') - if (msg.isPinned) markers.push('[pinned]') - return markers.join(' ') -} - -function escapeMarkdown(value: string): string { - return value.replace(/([\\`*_[\]{}()<>#+\-.!|~])/g, '\\$1') -} - -function quoteLines(value: string, prefix: string): string { - return value - .split('\n') - .map((line) => `${prefix}${line}`) - .join('\n') -} - -function safeHttpUrl(url: string): string | undefined { - try { - const parsed = new URL(url) - if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return undefined - return url.replace(//g, '%3E').replace(/\\/g, '%5C') - } catch { - return undefined - } -} - -function groupByDate(messages: ProcessedMessage[]): Map { - const groups = new Map() - - // Sort messages oldest first for display - const sorted = [...messages].sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime()) - - for (const msg of sorted) { - const date = formatDateLong(msg.timestamp) - if (!groups.has(date)) { - groups.set(date, []) - } - groups.get(date)?.push(msg) - } - - return groups -} diff --git a/src/formatters/pretty.ts b/src/formatters/pretty.ts deleted file mode 100644 index a6fd8d9..0000000 --- a/src/formatters/pretty.ts +++ /dev/null @@ -1,301 +0,0 @@ -// Pretty terminal output with ANSI colors - -import type { MessageOutput, ProcessedChannel, ProcessedMessage } from '../types' -import { bold, cyan, dim, userColor } from '../utils/colors' -import { formatRelativeTime, formatTime, getDateGroupLabel } from '../utils/date' - -function channelHeader(channel: ProcessedChannel): string { - if (channel.type === 'unknown') { - return `⚠ Unknown channel (${cyan(channel.id)})` - } - if (channel.type === 'dm') { - return `💬 DMs with ${cyan(channel.name)}` - } - if (channel.type === 'group') { - return `💬 Group DM: ${cyan(channel.name)}` - } - const name = `#${channel.name}` - const display = channel.displayName ? ` (${channel.displayName})` : '' - return `📢 ${cyan(name)}${display}` -} - -function channelHeaderPlain(channel: ProcessedChannel): string { - if (channel.type === 'unknown') { - return `Unknown channel (${channel.id})` - } - if (channel.type === 'dm') { - return `DMs with ${channel.name}` - } - if (channel.type === 'group') { - return `Group DM: ${channel.name}` - } - const display = channel.displayName ? ` (${channel.displayName})` : '' - return `#${channel.name}${display}` -} - -export interface PrettyOptions { - color?: boolean - relative?: boolean -} - -export function formatPretty( - outputs: MessageOutput[], - options: PrettyOptions | boolean = true, -): string { - // Handle legacy boolean parameter - const opts: PrettyOptions = typeof options === 'boolean' ? { color: options } : options - const useColor = opts.color ?? true - const relative = opts.relative ?? false - - if (!useColor) { - return formatPrettyNoColor(outputs, relative) - } - - const sections: string[] = [] - - for (const output of outputs) { - sections.push(formatChannelPretty(output, relative)) - } - - return sections.join(`\n${dim('─'.repeat(60))}\n\n`) -} - -function formatChannelPretty(output: MessageOutput, relative: boolean): string { - const { channel, messages } = output - const lines: string[] = [] - - // Header - lines.push(bold(channelHeader(channel))) - lines.push('') - - // Group messages by date - const messagesByDate = groupByDate(messages) - - for (const [date, msgs] of messagesByDate) { - lines.push(dim(` ── ${date} ──`)) - lines.push('') - - for (const msg of msgs) { - lines.push(formatMessagePretty(msg, relative)) - } - } - - // Redaction notice - if (output.redactions.length > 0) { - lines.push('') - lines.push(dim(` ⚠ ${output.redactions.length} secret(s) redacted`)) - } - appendCoverage(lines, output) - - return lines.join('\n') -} - -function formatMessagePretty( - msg: ProcessedMessage, - relative: boolean, - indent: string = ' ', -): string { - const timeStr = relative ? formatRelativeTime(msg.timestamp) : formatTime(msg.timestamp) - const time = dim(timeStr) - const user = userColor(msg.user) - const markers = messageMarkers(msg) - - const lines: string[] = [] - lines.push( - `${indent}${time} ${bold(user)}${markers ? ` ${dim(markers)}` : ''} ${dim(compactPostRef(msg))}`, - ) - - // Indent message content - const textIndent = `${indent} ` - const indentedText = msg.text - .split('\n') - .map((line) => `${textIndent}${line}`) - .join('\n') - lines.push(indentedText) - lines.push(dim(`${textIndent}${formatStateTimes(msg)}`)) - - // File attachments - if (msg.fileDetails.length > 0) { - lines.push(dim(`${textIndent}📎 ${formatFiles(msg)}`)) - } - appendRichContent(lines, msg, textIndent, (value) => dim(value)) - - lines.push('') - - // Render replies - if (msg.replies && msg.replies.length > 0) { - for (const reply of msg.replies) { - lines.push(formatMessagePretty(reply, relative, `${indent} ↳ `)) - } - } - - return lines.join('\n') -} - -function formatPrettyNoColor(outputs: MessageOutput[], relative: boolean): string { - const sections: string[] = [] - - for (const output of outputs) { - const { channel, messages } = output - const lines: string[] = [] - - lines.push(channelHeaderPlain(channel)) - lines.push('─'.repeat(40)) - - const messagesByDate = groupByDate(messages) - - for (const [date, msgs] of messagesByDate) { - lines.push(` -- ${date} --`) - lines.push('') - - for (const msg of msgs) { - formatMessageNoColor(msg, relative, lines, ' ') - } - } - - if (output.redactions.length > 0) { - lines.push(` [${output.redactions.length} secret(s) redacted]`) - } - appendCoverage(lines, output) - - sections.push(lines.join('\n')) - } - - return sections.join(`\n${'='.repeat(60)}\n\n`) -} - -function formatMessageNoColor( - msg: ProcessedMessage, - relative: boolean, - lines: string[], - indent: string, -): void { - const timeStr = relative ? formatRelativeTime(msg.timestamp) : formatTime(msg.timestamp) - const markers = messageMarkers(msg) - lines.push( - `${indent}[${timeStr}] ${msg.user}${markers ? ` ${markers}` : ''} ${compactPostRef(msg)}`, - ) - const textIndent = `${indent} ` - const indentedText = msg.text - .split('\n') - .map((line) => `${textIndent}${line}`) - .join('\n') - lines.push(indentedText) - lines.push(`${textIndent}${formatStateTimes(msg)}`) - if (msg.fileDetails.length > 0) { - lines.push(`${textIndent}Files: ${formatFiles(msg)}`) - } - appendRichContent(lines, msg, textIndent, (value) => value) - lines.push('') - - if (msg.replies && msg.replies.length > 0) { - for (const reply of msg.replies) { - formatMessageNoColor(reply, relative, lines, `${indent} > `) - } - } -} - -function compactPostRef(msg: ProcessedMessage): string { - const id = msg.id.length > 8 ? msg.id.slice(0, 8) : msg.id - return `${id} ${msg.permalink}` -} - -function messageMarkers(msg: ProcessedMessage): string { - const markers: string[] = [] - if (msg.isDeleted) markers.push('[deleted]') - else if (msg.editedAt) markers.push('[edited]') - if (msg.isSystem) markers.push(msg.postType ? `[system:${msg.postType}]` : '[system]') - if (msg.isPinned) markers.push('[pinned]') - return markers.join(' ') -} - -function formatFiles(msg: ProcessedMessage): string { - return msg.fileDetails - .map((file) => { - const label = file.name ? `${file.name} (${file.id})` : file.id - const details = [ - file.mime, - file.extension, - file.size === undefined ? undefined : `${file.size} B`, - ] - .filter(Boolean) - .join(', ') - return `${label}${details ? `, ${details}` : ''}` - }) - .join(', ') -} - -function formatStateTimes(msg: ProcessedMessage): string { - const values = [`Updated ${msg.updatedAt.toISOString()}`] - if (msg.editedAt) values.push(`edited ${msg.editedAt.toISOString()}`) - if (msg.deletedAt) values.push(`deleted ${msg.deletedAt.toISOString()}`) - return values.join('; ') -} - -function appendRichContent( - lines: string[], - msg: ProcessedMessage, - indent: string, - decorate: (value: string) => string, -): void { - for (const attachment of msg.attachments) { - if (attachment.pretext) lines.push(decorate(`${indent}${attachment.pretext}`)) - if (attachment.title) lines.push(decorate(`${indent}Attachment: ${attachment.title}`)) - if (attachment.titleLink) lines.push(decorate(`${indent} Link: ${attachment.titleLink}`)) - if (attachment.authorName) lines.push(decorate(`${indent} By: ${attachment.authorName}`)) - if (attachment.text) lines.push(decorate(`${indent} ${attachment.text}`)) - for (const field of attachment.fields ?? []) { - lines.push( - decorate(`${indent} ${field.title ? `${field.title}: ` : ''}${field.value ?? ''}`), - ) - } - if (attachment.footer) lines.push(decorate(`${indent} ${attachment.footer}`)) - if (attachment.fallback) lines.push(decorate(`${indent} Fallback: ${attachment.fallback}`)) - if (attachment.color) lines.push(decorate(`${indent} Color: ${attachment.color}`)) - if (attachment.timestamp) lines.push(decorate(`${indent} Timestamp: ${attachment.timestamp}`)) - for (const url of [ - attachment.authorLink, - attachment.authorIcon, - attachment.footerIcon, - attachment.imageUrl, - attachment.thumbUrl, - ]) { - if (url) lines.push(decorate(`${indent} ${url}`)) - } - } - if (msg.reactions.length > 0) { - const reactions = msg.reactions - .map(({ emoji, count, actors }) => { - const names = actors.map((actor) => actor.username ?? actor.id).join(', ') - return `:${emoji}: ${count}${names ? ` (${names})` : ''}` - }) - .join(' ') - lines.push(decorate(`${indent}Reactions: ${reactions}`)) - } -} - -function appendCoverage(lines: string[], output: MessageOutput): void { - const state = output.retrieval.selection.queryTruncated - lines.push( - ` Coverage: ${output.retrieval.selection.selectedCount} selected, ${output.retrieval.visiblePostCount} visible; query ${state === true ? 'truncated' : state === false ? 'complete' : 'completeness unknown'}`, - ) - if (output.retrieval.selection.nextCursor) { - lines.push(` Next cursor: ${output.retrieval.selection.nextCursor}`) - } -} - -function groupByDate(messages: ProcessedMessage[]): Map { - const groups = new Map() - - const sorted = [...messages].sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime()) - - for (const msg of sorted) { - const date = getDateGroupLabel(msg.timestamp) - if (!groups.has(date)) { - groups.set(date, []) - } - groups.get(date)?.push(msg) - } - - return groups -} diff --git a/src/formatters/watch.ts b/src/formatters/watch.ts deleted file mode 100644 index f31ae44..0000000 --- a/src/formatters/watch.ts +++ /dev/null @@ -1,13 +0,0 @@ -import type { WatchEvent } from '../types' -import { dim, formatTime, userColor } from '../utils' - -export function formatWatchJSON(event: WatchEvent): string { - return JSON.stringify(event) -} - -export function formatWatchEvent(event: WatchEvent, color: boolean): string { - const message = event.message.replace(/\s+/g, ' ').trim() || '[empty message]' - const time = formatTime(new Date(event.timestamp)) - if (color) return `${dim(`[${time}]`)} ${userColor(event.sender)}: ${message}` - return `[${time}] ${event.sender}: ${message}` -} diff --git a/src/index.ts b/src/index.ts deleted file mode 100755 index 5a2e752..0000000 --- a/src/index.ts +++ /dev/null @@ -1,567 +0,0 @@ -#!/usr/bin/env bun -// Mattermost CLI - Entry point - -import { Command } from 'commander' -import { isAgent } from 'is-ai-agent' -import pkg from '../package.json' -import { - fetchChannel, - fetchDMs, - fetchGroupDMs, - fetchMentions, - fetchThread, - listChannels, - listTeams, - listUsers, - searchMessages, - sendDirectMessage, - sendGroupMessage, - showUnread, - showWhoAmI, - watchChannel, -} from './cli' -import { getConfigPath, getConfigStatus, initConfigFile, resolveConfigState } from './config' -import { formatDoctorReport, runDoctor } from './doctor' -import { readMessageInput } from './input' -import { setActiveMattermostCredential } from './preprocessing' -import { parsePositiveSafeInteger } from './validation' - -const isRunningUnderAgent = isAgent() !== null - -function resolveRelative(opts: { relative?: boolean }): boolean { - if (opts.relative !== undefined) return opts.relative - return isRunningUnderAgent -} - -function resolveRedact(opts: { redact?: boolean }, fileConfig: { redact?: boolean }): boolean { - if (opts.redact !== undefined) return opts.redact - if (process.env.MM_REDACT !== undefined) return process.env.MM_REDACT !== 'false' - if (fileConfig.redact !== undefined) return fileConfig.redact - return true -} - -function validateLimit(value: string): number { - const n = parsePositiveSafeInteger(value) - if (n === null) { - console.error('Error: --limit must be a positive number.') - process.exit(1) - } - return n -} - -function validatePeek(value?: string): number | undefined { - if (value === undefined) return undefined - const n = parsePositiveSafeInteger(value) - if (n === null) { - console.error('Error: --peek must be a positive number.') - process.exit(1) - } - return n -} - -function validateDuration(value: string | undefined): string | undefined { - if (value !== undefined && !/^\d+[hdwm]$/i.test(value)) { - console.error('Error: --since must use a duration such as "24h", "7d", "1w", or "2m".') - process.exit(1) - } - return value -} - -const program = new Command() - -program - .name('mm') - .description('Mattermost CLI - Read, watch, and deliberately send messages') - .version(pkg.version) - -program - .option('-t, --token ', 'Mattermost personal access token (or MM_TOKEN env)') - .option('--url ', 'Mattermost server URL (or MM_URL env)') - .option('--json', 'Output as JSON (JSON Lines for watch)', false) - .option('--no-color', 'Disable colored output') - .option( - '-r, --relative', - 'Show times as relative (e.g., "2 days ago"); auto-enabled under AI agents', - ) - .option('--no-relative', 'Show absolute dates/times') - .option('--redact', 'Enable secret redaction (default)') - .option('--no-redact', 'Disable secret redaction') - .option('--threads', 'Show thread structure (default)') - .option('--no-threads', 'Return selected seed posts only (except thread command)') - -async function resolveConfig(options: { url?: string; token?: string }): Promise<{ - url: string - token: string - fileConfig: { redact?: boolean; mentionNames: string[] } -}> { - const state = await resolveConfigState(options) - const { url, token, fileConfig, configPath } = state - if (token) setActiveMattermostCredential(token) - if (state.insecurePermissions) { - console.warn( - `Warning: ${configPath} has insecure permissions.\n` + ` Run: chmod 600 "${configPath}"`, - ) - } - if (state.fileError) { - console.warn( - `Warning: Could not ${state.fileError === 'parse' ? 'parse' : 'read'} config at ${configPath}`, - ) - } - - if (!url) { - console.error( - 'Error: Mattermost URL required.\n' + - ' 1. Use --url flag\n' + - ' 2. Set MM_URL env var\n' + - ` 3. Add to ${configPath}`, - ) - process.exit(1) - } - if (!token) { - console.error( - 'Error: Mattermost token required.\n' + - ' 1. Use --token flag\n' + - ' 2. Set MM_TOKEN env var\n' + - ` 3. Add to ${configPath}`, - ) - process.exit(1) - } - - return { - url, - token, - fileConfig: { - redact: fileConfig.redact, - mentionNames: fileConfig.mention_names ?? [], - }, - } -} - -program - .command('doctor') - .description('Check configuration, server health, and authentication') - .action(async () => { - const opts = program.opts() - const config = await resolveConfigState(opts) - const report = await runDoctor(config, { - redact: resolveRedact(opts, config.fileConfig), - }) - console.log(opts.json ? JSON.stringify(report) : formatDoctorReport(report)) - if (!report.ok) process.exitCode = 1 - }) - -program - .command('config') - .description('Manage config file') - .option('--path', 'Print config file path') - .option('--init', 'Create config file with template') - .action(async (opts) => { - try { - if (opts.path) { - console.log(getConfigPath()) - return - } - - if (opts.init) { - const result = await initConfigFile() - if (result.created) { - console.log(`Created config file: ${result.path}`) - console.log('Edit the file to add your Mattermost URL and token.') - } else { - console.log(`Config file already exists: ${result.path}`) - } - return - } - - const status = await getConfigStatus() - console.log(`Config path: ${status.path}`) - console.log(`Exists: ${status.exists ? 'yes' : 'no'}`) - if (status.exists) { - console.log(`URL configured: ${status.hasUrl ? 'yes' : 'no'}`) - console.log(`Token configured: ${status.hasToken ? 'yes' : 'no'}`) - if (status.insecurePerms) { - console.log('\nWarning: Config file has insecure permissions.') - console.log(` Run: chmod 600 "${status.path}"`) - } - } else { - console.log('\nRun `mm config --init` to create a config file.') - } - } catch (err) { - console.error('Error:', err instanceof Error ? err.message : err) - process.exit(1) - } - }) - -program - .command('whoami') - .description('Show the authenticated user') - .action(async () => { - const opts = program.opts() - const config = await resolveConfig(opts) - try { - await showWhoAmI({ - url: config.url, - token: config.token, - json: opts.json, - redact: resolveRedact(opts, config.fileConfig), - }) - } catch (err) { - console.error('Error:', err instanceof Error ? err.message : err) - process.exit(1) - } - }) - -program - .command('teams') - .description('List teams for the authenticated user') - .action(async () => { - const opts = program.opts() - const config = await resolveConfig(opts) - try { - await listTeams({ - url: config.url, - token: config.token, - json: opts.json, - redact: resolveRedact(opts, config.fileConfig), - }) - } catch (err) { - console.error('Error:', err instanceof Error ? err.message : err) - process.exit(1) - } - }) - -program - .command('users [query]') - .description('List or search active users') - .option('--team ', 'Restrict users to a team') - .option('-l, --limit ', 'Max users to show', '20') - .action(async (query, cmdOpts) => { - const opts = program.opts() - const config = await resolveConfig(opts) - try { - await listUsers({ - url: config.url, - token: config.token, - json: opts.json, - redact: resolveRedact(opts, config.fileConfig), - query, - team: cmdOpts.team, - limit: validateLimit(cmdOpts.limit), - }) - } catch (err) { - console.error('Error:', err instanceof Error ? err.message : err) - process.exit(1) - } - }) - -program - .command('channels') - .description('List account-wide channels with team identity for public/private channels') - .option('--type ', 'Filter by channel type: dm, public, private, group, all', 'all') - .action(async (cmdOpts) => { - const opts = program.opts() - const config = await resolveConfig(opts) - - try { - await listChannels({ - url: config.url, - token: config.token, - json: opts.json, - color: opts.color, - relative: resolveRelative(opts), - redact: resolveRedact(opts, config.fileConfig), - typeFilter: cmdOpts.type, - }) - } catch (err) { - console.error('Error:', err instanceof Error ? err.message : err) - process.exit(1) - } - }) - -program - .command('dms') - .description('Fetch direct messages') - .option('-u, --user ', 'Filter by username (repeatable)') - .option('-l, --limit ', 'Max total seed messages across matched DMs', '50') - .option('-s, --since ', 'Time range: "24h", "7d", "30d"', '7d') - .option('-c, --channel ', 'Specific direct-message channel ID (type D only)') - .option('--cursor ', 'Resume deterministic channel history') - .action(async (cmdOpts, command) => { - const globalOpts = program.opts() - const config = await resolveConfig(globalOpts) - - try { - await fetchDMs({ - url: config.url, - token: config.token, - json: globalOpts.json, - color: globalOpts.color, - relative: resolveRelative(globalOpts), - redact: resolveRedact(globalOpts, config.fileConfig), - threads: globalOpts.threads ?? true, - user: cmdOpts.user || [], - limit: validateLimit(cmdOpts.limit), - since: validateDuration(cmdOpts.since) as string, - channel: cmdOpts.channel, - cursor: cmdOpts.cursor, - sinceExplicit: command.getOptionValueSource('since') === 'cli', - }) - } catch (err) { - console.error('Error:', err instanceof Error ? err.message : err) - process.exit(1) - } - }) - -program - .command('group-dms') - .description('Fetch group direct messages') - .option('-l, --limit ', 'Max total seed messages across matched group DMs', '50') - .option('-s, --since ', 'Time range: "24h", "7d", "30d"', '7d') - .option('-c, --channel ', 'Specific group DM channel ID (type G only)') - .option('--cursor ', 'Resume deterministic channel history') - .action(async (cmdOpts, command) => { - const globalOpts = program.opts() - const config = await resolveConfig(globalOpts) - - try { - await fetchGroupDMs({ - url: config.url, - token: config.token, - json: globalOpts.json, - color: globalOpts.color, - relative: resolveRelative(globalOpts), - redact: resolveRedact(globalOpts, config.fileConfig), - threads: globalOpts.threads ?? true, - limit: validateLimit(cmdOpts.limit), - since: validateDuration(cmdOpts.since) as string, - channel: cmdOpts.channel, - cursor: cmdOpts.cursor, - sinceExplicit: command.getOptionValueSource('since') === 'cli', - }) - } catch (err) { - console.error('Error:', err instanceof Error ? err.message : err) - process.exit(1) - } - }) - -const send = program.command('send').description('Send a message to a direct or group conversation') - -send - .command('dm ') - .description('Send stdin to a direct-message user, creating the DM if needed') - .option('--dry-run', 'Resolve the exact destination without writing', false) - .action(async (username, cmdOpts) => { - const globalOpts = program.opts() - const config = await resolveConfig(globalOpts) - try { - await sendDirectMessage({ - url: config.url, - token: config.token, - json: globalOpts.json, - redact: resolveRedact(globalOpts, config.fileConfig), - username, - dryRun: cmdOpts.dryRun, - message: cmdOpts.dryRun ? undefined : await readMessageInput(process.stdin), - }) - } catch (err) { - console.error('Error:', err instanceof Error ? err.message : err) - process.exit(1) - } - }) - -send - .command('group ') - .description('Send stdin to an existing group-DM channel') - .option('--dry-run', 'Resolve the exact destination without writing', false) - .action(async (channelId, cmdOpts) => { - const globalOpts = program.opts() - const config = await resolveConfig(globalOpts) - try { - await sendGroupMessage({ - url: config.url, - token: config.token, - json: globalOpts.json, - redact: resolveRedact(globalOpts, config.fileConfig), - channelId, - dryRun: cmdOpts.dryRun, - message: cmdOpts.dryRun ? undefined : await readMessageInput(process.stdin), - }) - } catch (err) { - console.error('Error:', err instanceof Error ? err.message : err) - process.exit(1) - } - }) - -program - .command('channel ') - .description('Fetch messages from a channel by name') - .option('--team ', 'Team name (auto-detected if you belong to one team)') - .option('-l, --limit ', 'Max seed messages (thread context may exceed)', '50') - .option('-s, --since ', 'Time range: "24h", "7d", "30d"', '7d') - .option('--cursor ', 'Resume deterministic channel history') - .action(async (name, cmdOpts, command) => { - const globalOpts = program.opts() - const config = await resolveConfig(globalOpts) - - try { - await fetchChannel({ - url: config.url, - token: config.token, - json: globalOpts.json, - color: globalOpts.color, - relative: resolveRelative(globalOpts), - redact: resolveRedact(globalOpts, config.fileConfig), - threads: globalOpts.threads ?? true, - channel: name, - team: cmdOpts.team, - limit: validateLimit(cmdOpts.limit), - since: validateDuration(cmdOpts.since) as string, - cursor: cmdOpts.cursor, - sinceExplicit: command.getOptionValueSource('since') === 'cli', - }) - } catch (err) { - console.error('Error:', err instanceof Error ? err.message : err) - process.exit(1) - } - }) - -program - .command('search ') - .description('Search messages within one selected team') - .option('--team ', 'Team name (auto-detected if you belong to one team)') - .option('-l, --limit ', 'Max seed results (thread context may exceed)', '50') - .action(async (query, cmdOpts) => { - const globalOpts = program.opts() - const config = await resolveConfig(globalOpts) - - try { - await searchMessages({ - url: config.url, - token: config.token, - json: globalOpts.json, - color: globalOpts.color, - relative: resolveRelative(globalOpts), - redact: resolveRedact(globalOpts, config.fileConfig), - threads: globalOpts.threads ?? true, - query, - team: cmdOpts.team, - limit: validateLimit(cmdOpts.limit), - }) - } catch (err) { - console.error('Error:', err instanceof Error ? err.message : err) - process.exit(1) - } - }) - -program - .command('mentions') - .description('Find mentions within one selected team') - .option('--team ', 'Team name (auto-detected if you belong to one team)') - .option('-l, --limit ', 'Max seed results (thread context may exceed)', '50') - .option('-s, --since ', 'Time range: "24h", "7d", "30d"') - .option('--channel ', 'Scope mentions to a channel name') - .action(async (cmdOpts) => { - const globalOpts = program.opts() - const config = await resolveConfig(globalOpts) - - try { - await fetchMentions({ - url: config.url, - token: config.token, - json: globalOpts.json, - color: globalOpts.color, - relative: resolveRelative(globalOpts), - redact: resolveRedact(globalOpts, config.fileConfig), - threads: globalOpts.threads ?? true, - team: cmdOpts.team, - limit: validateLimit(cmdOpts.limit), - since: validateDuration(cmdOpts.since), - channel: cmdOpts.channel, - mentionNames: config.fileConfig.mentionNames, - }) - } catch (err) { - console.error('Error:', err instanceof Error ? err.message : err) - process.exit(1) - } - }) - -program - .command('unread') - .description('Show channels with unread messages') - .option('--team ', 'Team name (auto-detected if you belong to one team)') - .option('--peek ', 'Fetch N messages from each unread channel') - .action(async (cmdOpts) => { - const globalOpts = program.opts() - const config = await resolveConfig(globalOpts) - - try { - await showUnread({ - url: config.url, - token: config.token, - json: globalOpts.json, - color: globalOpts.color, - relative: resolveRelative(globalOpts), - redact: resolveRedact(globalOpts, config.fileConfig), - threads: globalOpts.threads ?? true, - team: cmdOpts.team, - peek: validatePeek(cmdOpts.peek), - }) - } catch (err) { - console.error('Error:', err instanceof Error ? err.message : err) - process.exit(1) - } - }) - -program - .command('watch [channel]') - .description('Watch posted events in a channel or DM with automatic reconnect') - .option('--team ', 'Team name (auto-detected if you belong to one team)') - .option('--dm ', 'Watch direct messages with a username') - .action(async (channel, cmdOpts) => { - const globalOpts = program.opts() - const config = await resolveConfig(globalOpts) - - try { - await watchChannel({ - url: config.url, - token: config.token, - json: globalOpts.json, - color: globalOpts.color, - relative: resolveRelative(globalOpts), - redact: resolveRedact(globalOpts, config.fileConfig), - threads: globalOpts.threads ?? true, - team: cmdOpts.team, - channel, - dm: cmdOpts.dm, - }) - } catch (err) { - console.error('Error:', err instanceof Error ? err.message : err) - process.exit(1) - } - }) - -program - .command('thread ') - .description('Fetch and display a specific thread') - .action(async (postId) => { - const globalOpts = program.opts() - const config = await resolveConfig(globalOpts) - - try { - await fetchThread({ - url: config.url, - token: config.token, - json: globalOpts.json, - color: globalOpts.color, - relative: resolveRelative(globalOpts), - redact: resolveRedact(globalOpts, config.fileConfig), - threads: true, - postId, - }) - } catch (err) { - console.error('Error:', err instanceof Error ? err.message : err) - process.exit(1) - } - }) - -await program.parseAsync() diff --git a/src/input.ts b/src/input.ts deleted file mode 100644 index f7012fe..0000000 --- a/src/input.ts +++ /dev/null @@ -1,40 +0,0 @@ -export const MAX_MESSAGE_BYTES = 65_535 -export const MAX_MESSAGE_CHARACTERS = 16_383 - -export function validateMessageContent(message: string): void { - if (!message.trim()) throw new Error('Message cannot be empty.') - if ([...message].length > MAX_MESSAGE_CHARACTERS) { - throw new Error(`Message exceeds ${MAX_MESSAGE_CHARACTERS} Unicode characters.`) - } - if (Buffer.byteLength(message) > MAX_MESSAGE_BYTES) { - throw new Error(`Message exceeds ${MAX_MESSAGE_BYTES} UTF-8 bytes.`) - } -} - -export interface MessageInputStream extends AsyncIterable { - isTTY?: boolean -} - -export async function readMessageInput(stream: MessageInputStream): Promise { - if (stream.isTTY) throw new Error('Message content must be piped on stdin.') - - const chunks: Uint8Array[] = [] - let bytes = 0 - for await (const chunk of stream) { - const encoded = typeof chunk === 'string' ? Buffer.from(chunk) : chunk - bytes += encoded.byteLength - if (bytes > MAX_MESSAGE_BYTES) { - throw new Error(`Message exceeds ${MAX_MESSAGE_BYTES} UTF-8 bytes.`) - } - chunks.push(encoded) - } - - let message: string - try { - message = new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)) - } catch { - throw new Error('Message must be valid UTF-8.') - } - validateMessageContent(message) - return message -} diff --git a/src/preprocessing/credential.ts b/src/preprocessing/credential.ts deleted file mode 100644 index f70ccb0..0000000 --- a/src/preprocessing/credential.ts +++ /dev/null @@ -1,22 +0,0 @@ -const DEFAULT_OWNER = Symbol('resolved-config') -const credentialByOwner = new Map() - -export function setActiveMattermostCredential(token: string | undefined): void { - if (token === undefined) { - credentialByOwner.clear() - } else if (token) { - credentialByOwner.set(DEFAULT_OWNER, token) - } -} - -export function registerActiveMattermostCredential(token: string): () => void { - const owner = Symbol('mattermost-credential-owner') - if (token) credentialByOwner.set(owner, token) - return () => { - if (credentialByOwner.get(owner) === token) credentialByOwner.delete(owner) - } -} - -export function getActiveMattermostCredentials(): readonly string[] { - return [...new Set(credentialByOwner.values())] -} diff --git a/src/preprocessing/index.ts b/src/preprocessing/index.ts deleted file mode 100644 index 371213d..0000000 --- a/src/preprocessing/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -// Preprocessing pipeline - -export { registerActiveMattermostCredential, setActiveMattermostCredential } from './credential' -export { SECRET_PATTERNS } from './patterns' -export { preprocess } from './pipeline' -export { normalizePosts, postUserIds } from './post' -export { sanitizeControlCharacters, sanitizeTerminalLabel } from './sanitize' -export { detectSecrets, maskSecret, redactSecrets } from './secrets' diff --git a/src/preprocessing/patterns.ts b/src/preprocessing/patterns.ts deleted file mode 100644 index bf60e55..0000000 --- a/src/preprocessing/patterns.ts +++ /dev/null @@ -1,182 +0,0 @@ -// Secret detection patterns - -export interface SecretPattern { - name: string - pattern: RegExp - // Some patterns need context awareness (e.g., AWS secret keys near access keys) - contextRequired?: boolean -} - -export const SECRET_PATTERNS: SecretPattern[] = [ - // AWS - { - name: 'aws_access_key', - pattern: /\b((?:AKIA|ASIA)[0-9A-Z]{16})\b/g, - }, - { - name: 'aws_secret_key', - pattern: - /(?:aws[_-]?secret[_-]?(?:access[_-]?)?key|secret[_-]?key)["\s:=]+["']?([A-Za-z0-9/+=]{40})["']?/gi, - }, - - // GitHub - { - name: 'github_stateless_token', - pattern: /(? ({ type: 'mattermost_credential', value })), - }), - ) -} - -function sanitizeResult(result: PreprocessResult): PreprocessResult { - const { text: redactedText, redactions } = result - return { - text: sanitizeControlCharacters(redactedText), - redactions: redactions.map((redaction) => ({ - ...redaction, - masked: sanitizeControlCharacters(redaction.masked), - position: sanitizeControlCharacters(redactedText.slice(0, redaction.position)).length, - })), - } -} diff --git a/src/preprocessing/post.ts b/src/preprocessing/post.ts deleted file mode 100644 index 116ba6b..0000000 --- a/src/preprocessing/post.ts +++ /dev/null @@ -1,232 +0,0 @@ -import type { - Post, - ProcessedAttachment, - ProcessedFile, - ProcessedMessage, - ReactionActor, - ReactionSummary, - Redaction, - User, -} from '../types' -import { setCanonicalPostIdentity } from '../utils/threading' -import { preprocess } from './pipeline' - -const DELETED_POST_TEXT = '[deleted post]' -type UserLookup = ReadonlyMap - -function record(value: unknown): Record | undefined { - return typeof value === 'object' && value !== null && !Array.isArray(value) - ? (value as Record) - : undefined -} - -function stringValue(value: unknown, allowNumber = false): string | undefined { - if (typeof value === 'string') return value - if (allowNumber && typeof value === 'number' && Number.isFinite(value)) return String(value) - return undefined -} - -function arrayValue(value: unknown): unknown[] { - return Array.isArray(value) ? value : [] -} - -function validTimestamp(value: unknown, fallback = 0): number { - return typeof value === 'number' && - Number.isFinite(value) && - Number.isFinite(new Date(value).getTime()) - ? value - : fallback -} - -function nonNegativeInteger(value: unknown): number { - return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : 0 -} - -export function postUserIds(posts: Post[]): string[] { - const ids = new Set() - for (const post of posts) { - if (post.user_id) ids.add(post.user_id) - for (const candidate of arrayValue(post.metadata?.reactions)) { - const id = stringValue(record(candidate)?.user_id) - if (id) ids.add(id) - } - } - return [...ids] -} - -export function normalizePosts( - posts: Post[], - users: UserLookup, - myUserId: string, - serverUrl: string, - buildPermalink: (serverUrl: string, postId: string) => string, - redact: boolean, -): { messages: ProcessedMessage[]; redactions: Redaction[] } { - const allRedactions: Redaction[] = [] - const clean = (value: string, field: string, oneLine = false): string => { - const result = preprocess(value, { redact }) - allRedactions.push(...result.redactions.map((item) => ({ ...item, field }))) - return oneLine ? result.text.replace(/\n/g, '\\n').replace(/\t/g, '\\t') : result.text - } - - const messages = posts.map((post): ProcessedMessage => { - const rawId = stringValue(post.id) ?? '' - const rawUserId = stringValue(post.user_id) ?? '' - const rawMessage = stringValue(post.message) ?? '' - const rawType = stringValue(post.type) ?? '' - const rawRootId = stringValue(post.root_id) ?? '' - const createAt = validTimestamp(post.create_at) - const updateAt = validTimestamp(post.update_at, createAt) - const editAt = validTimestamp(post.edit_at) - const deleteAt = validTimestamp(post.delete_at) - const isDeleted = deleteAt > 0 - const props = record(post.props) ?? {} - const rawOverride = stringValue(post.override_username) ?? stringValue(props.override_username) - const rawUsername = users.get(rawUserId)?.username ?? rawUserId - const displayUser = rawOverride - ? clean(rawOverride, 'user', true) - : !rawUserId || rawType.startsWith('system_') - ? 'system' - : rawUserId === myUserId - ? 'you' - : clean(rawUsername, 'user', true) - - const rawFiles = arrayValue(post.metadata?.files).flatMap((candidate) => { - const value = record(candidate) - const id = stringValue(value?.id) - return value && id ? [{ value, id }] : [] - }) - const metadataFiles = new Map(rawFiles.map(({ id, value }) => [id, value])) - const rawFileIds = [ - ...new Set([ - ...arrayValue(post.file_ids).flatMap((value) => { - const id = stringValue(value) - return id ? [id] : [] - }), - ...metadataFiles.keys(), - ]), - ] - const fileDetails: ProcessedFile[] = isDeleted - ? [] - : rawFileIds.map((rawId) => { - const file = metadataFiles.get(rawId) - const name = stringValue(file?.name) - const mime = stringValue(file?.mime_type) - const extension = stringValue(file?.extension) - return { - id: clean(rawId, 'file.id', true), - ...(name ? { name: clean(name, 'file.name', true) } : {}), - ...(mime ? { mime: clean(mime, 'file.mime', true) } : {}), - ...(typeof file?.size === 'number' && Number.isFinite(file.size) - ? { size: file.size } - : {}), - ...(extension ? { extension: clean(extension, 'file.extension', true) } : {}), - } - }) - - const attachments: ProcessedAttachment[] = isDeleted - ? [] - : arrayValue(props.attachments).flatMap((candidate, index) => { - const source = record(candidate) - if (!source) return [] - const cleanedValues = new Map() - const take = (key: string, oneLine = false, allowNumber = false) => { - const cacheKey = `${key}:${oneLine}:${allowNumber}` - if (cleanedValues.has(cacheKey)) return cleanedValues.get(cacheKey) - const value = stringValue(source[key], allowNumber) - const cleaned = value ? clean(value, `attachment.${index}.${key}`, oneLine) : undefined - cleanedValues.set(cacheKey, cleaned) - return cleaned - } - const fields = arrayValue(source.fields).flatMap((candidateField, fieldIndex) => { - const field = record(candidateField) - if (!field) return [] - const title = stringValue(field.title, true) - const value = stringValue(field.value, true) - if (!title && !value && typeof field.short !== 'boolean') return [] - return [ - { - ...(title - ? { title: clean(title, `attachment.${index}.fields.${fieldIndex}.title`) } - : {}), - ...(value - ? { value: clean(value, `attachment.${index}.fields.${fieldIndex}.value`) } - : {}), - ...(typeof field.short === 'boolean' ? { short: field.short } : {}), - }, - ] - }) - const attachment: ProcessedAttachment = { - ...(take('fallback') ? { fallback: take('fallback') } : {}), - ...(take('pretext') ? { pretext: take('pretext') } : {}), - ...(take('title') ? { title: take('title') } : {}), - ...(take('title_link', true) ? { titleLink: take('title_link', true) } : {}), - ...(take('text') ? { text: take('text') } : {}), - ...(fields.length > 0 ? { fields } : {}), - ...(take('footer') ? { footer: take('footer') } : {}), - ...(take('footer_icon', true) ? { footerIcon: take('footer_icon', true) } : {}), - ...(take('author_name') ? { authorName: take('author_name') } : {}), - ...(take('author_link', true) ? { authorLink: take('author_link', true) } : {}), - ...(take('author_icon', true) ? { authorIcon: take('author_icon', true) } : {}), - ...(take('color', true) ? { color: take('color', true) } : {}), - ...(take('image_url', true) ? { imageUrl: take('image_url', true) } : {}), - ...(take('thumb_url', true) ? { thumbUrl: take('thumb_url', true) } : {}), - ...(take('ts', true, true) ? { timestamp: take('ts', true, true) } : {}), - } - return Object.keys(attachment).length > 0 ? [attachment] : [] - }) - - const reactionGroups = new Map>() - if (!isDeleted) { - for (const candidate of arrayValue(post.metadata?.reactions)) { - const reaction = record(candidate) - const rawEmoji = stringValue(reaction?.emoji_name) - const rawId = stringValue(reaction?.user_id) - if (!rawEmoji || !rawId) continue - const username = users.get(rawId)?.username - const actor: ReactionActor = { - id: clean(rawId, 'reaction.actor.id', true), - ...(username ? { username: clean(username, 'reaction.actor.username', true) } : {}), - } - const group = reactionGroups.get(rawEmoji) - if (group) group.push({ rawId, actor }) - else reactionGroups.set(rawEmoji, [{ rawId, actor }]) - } - } - const reactions: ReactionSummary[] = [...reactionGroups] - .sort(([a], [b]) => a.localeCompare(b)) - .map(([rawEmoji, entries]) => { - entries.sort((a, b) => a.rawId.localeCompare(b.rawId)) - return { - emoji: clean(rawEmoji, 'reaction.emoji', true), - count: entries.length, - actors: entries.map(({ actor }) => actor), - } - }) - - const message: ProcessedMessage = { - id: clean(rawId, 'post.id', true), - permalink: clean(buildPermalink(serverUrl, rawId), 'post.permalink', true), - user: displayUser, - userId: clean(rawUserId, 'post.userId', true), - text: isDeleted ? DELETED_POST_TEXT : clean(rawMessage, 'post.message'), - timestamp: new Date(createAt), - updatedAt: new Date(updateAt), - ...(editAt > 0 ? { editedAt: new Date(editAt) } : {}), - ...(deleteAt > 0 ? { deletedAt: new Date(deleteAt) } : {}), - isDeleted, - postType: clean(rawType, 'post.type', true), - isSystem: rawType.startsWith('system_') || !rawUserId, - isPinned: typeof post.is_pinned === 'boolean' ? post.is_pinned : false, - files: isDeleted ? [] : rawFileIds.map((id) => clean(id, 'file.id', true)), - fileDetails, - attachments, - reactions, - rootId: rawRootId ? clean(rawRootId, 'post.rootId', true) : undefined, - replyCount: nonNegativeInteger(post.reply_count) || undefined, - } - setCanonicalPostIdentity(message, rawId, rawRootId || undefined) - return message - }) - return { messages, redactions: allRedactions } -} diff --git a/src/preprocessing/sanitize.ts b/src/preprocessing/sanitize.ts deleted file mode 100644 index 2889e72..0000000 --- a/src/preprocessing/sanitize.ts +++ /dev/null @@ -1,37 +0,0 @@ -function isUnsafeControlCode(code: number): boolean { - return ( - (code >= 0x00 && code <= 0x08) || - (code >= 0x0b && code <= 0x1f) || - (code >= 0x7f && code <= 0x9f) || - code === 0x061c || - code === 0x200e || - code === 0x200f || - (code >= 0x202a && code <= 0x202e) || - (code >= 0x2066 && code <= 0x2069) - ) -} - -function visibleUnicodeEscape(code: number): string { - return `\\u${code.toString(16).padStart(4, '0')}` -} - -/** - * Make terminal control bytes visible instead of executable while preserving ordinary message - * whitespace. This protects pretty, plain, markdown, JSON, and live watch output through one - * preprocessing boundary. - */ -export function sanitizeControlCharacters(text: string): string { - let result = '' - - for (const character of text.replace(/\r\n/g, '\n')) { - const code = character.charCodeAt(0) - result += isUnsafeControlCode(code) ? visibleUnicodeEscape(code) : character - } - - return result -} - -/** Keep remotely controlled labels and errors on one visible terminal line. */ -export function sanitizeTerminalLabel(text: string): string { - return sanitizeControlCharacters(text).replace(/\n/g, '\\n').replace(/\t/g, '\\t') -} diff --git a/src/preprocessing/secrets.ts b/src/preprocessing/secrets.ts deleted file mode 100644 index 96a6782..0000000 --- a/src/preprocessing/secrets.ts +++ /dev/null @@ -1,149 +0,0 @@ -// Secret detection and masking - -import type { Redaction } from '../types' -import { SECRET_PATTERNS } from './patterns' - -interface DetectedSecret { - type: string - value: string - start: number - end: number -} - -interface SecretGroup { - start: number - end: number - secrets: DetectedSecret[] -} - -function groupOverlappingSecrets(secrets: DetectedSecret[]): SecretGroup[] { - const sorted = [...secrets].sort((a, b) => a.start - b.start || b.end - a.end) - const groups: SecretGroup[] = [] - - for (const secret of sorted) { - const current = groups.at(-1) - - if (!current || secret.start >= current.end) { - groups.push({ start: secret.start, end: secret.end, secrets: [secret] }) - continue - } - - current.end = Math.max(current.end, secret.end) - current.secrets.push(secret) - } - - return groups -} - -// Detect all secrets in text -export function detectSecrets(text: string): DetectedSecret[] { - const secrets: DetectedSecret[] = [] - const seen = new Set() // Avoid duplicates at same position - - for (const { name, pattern } of SECRET_PATTERNS) { - // Reset regex state (important for global patterns) - pattern.lastIndex = 0 - - let match: RegExpExecArray | null = pattern.exec(text) - while (match !== null) { - // Use captured group if available, otherwise full match - const value = match[1] || match[0] - const offset = match[0].indexOf(value) - const start = match.index + (offset >= 0 ? offset : 0) - const end = start + value.length - const key = `${start}:${end}` - - if (!seen.has(key)) { - seen.add(key) - secrets.push({ type: name, value, start, end }) - } - - match = pattern.exec(text) - } - } - - // Sort by position (start) - return secrets.sort((a, b) => a.start - b.start) -} - -// Mask a secret value, showing first and last few chars -export function maskSecret(value: string, type: string): string { - if (type === 'mattermost_credential') return '[REDACTED:mattermost_credential]' - // Very short secrets get fully redacted - if (value.length <= 8) { - return `[REDACTED:${type}]` - } - - // Show ~10% of chars on each end, min 2, max 4 - const visibleCount = Math.max(2, Math.min(4, Math.floor(value.length * 0.1))) - - const prefix = value.slice(0, visibleCount) - const suffix = value.slice(-visibleCount) - - return `${prefix}...${suffix}` -} - -// Redact all secrets in text, returning new text and redaction log -export function redactSecrets( - text: string, - options: { detectPatterns?: boolean; exact?: Array<{ type: string; value: string }> } = {}, -): { - text: string - redactions: Redaction[] -} { - const secrets = options.detectPatterns === false ? [] : detectSecrets(text) - for (const exact of options.exact ?? []) { - if (!exact.value) continue - let start = text.indexOf(exact.value) - while (start !== -1) { - secrets.push({ - type: exact.type, - value: exact.value, - start, - end: start + exact.value.length, - }) - start = text.indexOf(exact.value, start + exact.value.length) - } - } - - if (secrets.length === 0) { - return { text, redactions: [] } - } - - const redactions: Redaction[] = [] - let result = '' - let lastEnd = 0 - - for (const group of groupOverlappingSecrets(secrets)) { - // Add text before this secret - result += text.slice(lastEnd, group.start) - - // Mask the full union once so nested/overlapping matches cannot move the cursor backwards - // and re-append part of a previously redacted value. - const groupValue = text.slice(group.start, group.end) - const dominantType = group.secrets.some(({ type }) => type === 'mattermost_credential') - ? 'mattermost_credential' - : (group.secrets[0]?.type ?? 'secret') - const masked = maskSecret(groupValue, dominantType) - const emittedPosition = result.length - result += masked - - const types = [...new Set(group.secrets.map((secret) => secret.type))] - if (dominantType === 'mattermost_credential') { - types.splice(types.indexOf(dominantType), 1) - types.unshift(dominantType) - } - redactions.push({ - type: types.join('+'), - masked, - position: emittedPosition, - }) - - lastEnd = group.end - } - - // Add remaining text - result += text.slice(lastEnd) - - return { text: result, redactions } -} diff --git a/src/types.ts b/src/types.ts deleted file mode 100644 index de10c10..0000000 --- a/src/types.ts +++ /dev/null @@ -1,362 +0,0 @@ -// Mattermost API types - -export interface User { - id: string - username: string - nickname: string - first_name: string - last_name: string - email: string - roles?: string -} - -export interface Team { - id: string - name: string - display_name: string - type: 'O' | 'I' // Open, Invite-only -} - -export interface Channel { - id: string - team_id: string - type: 'O' | 'P' | 'D' | 'G' // Open, Private, Direct, Group - display_name: string - name: string - header: string - purpose: string - last_post_at: number - total_msg_count: number - creator_id: string -} - -export interface Post { - id: string - create_at: number - update_at: number - delete_at: number - edit_at: number - user_id: string - channel_id: string - message: string - type: string - props: Record - hashtags: string - root_id: string // empty = root post, non-empty = reply to this post - reply_count: number // number of replies (root posts only) - file_ids: string[] - pending_post_id: string - is_pinned?: boolean - override_username?: string - metadata?: PostMetadata -} - -export interface PostMetadata { - files?: FileInfo[] - reactions?: Reaction[] - embeds?: PostEmbed[] -} - -export interface PostEmbed { - type?: string - url?: string - data?: Record -} - -export interface FileInfo { - id: string - name: string - extension: string - size: number - mime_type: string -} - -export interface Reaction { - user_id: string - post_id: string - emoji_name: string - create_at: number -} - -export interface PostAttachmentField { - title?: unknown - value?: unknown - short?: boolean -} - -export interface PostAttachment { - fallback?: unknown - pretext?: unknown - title?: unknown - title_link?: unknown - text?: unknown - fields?: unknown - footer?: unknown - footer_icon?: unknown - author_name?: unknown - author_link?: unknown - author_icon?: unknown - color?: unknown - image_url?: unknown - thumb_url?: unknown - ts?: unknown -} - -export interface PostsResponse { - order: string[] - posts: Record - next_post_id: string - prev_post_id: string - has_next?: boolean - first_inaccessible_post_time?: number -} - -export interface SearchResponse { - order: string[] - posts: Record - matches: Record - first_inaccessible_post_time?: number - has_next?: boolean -} - -export interface PostRetrievalResult { - posts: Post[] - truncated: boolean | null - safeBeforeValid?: boolean -} - -export interface ChannelMember { - channel_id: string - user_id: string - msg_count: number - mention_count: number - last_viewed_at: number -} - -// CLI types - -export type ChannelTypeFilter = 'all' | 'dm' | 'public' | 'private' | 'group' - -export interface CLIOptions { - url: string - token: string - json: boolean - color: boolean - relative: boolean - redact: boolean - threads: boolean -} - -export interface DMsOptions extends CLIOptions { - user: string[] - limit: number - since: string - channel?: string - cursor?: string - sinceExplicit?: boolean -} - -export interface GroupDMsOptions extends CLIOptions { - limit: number - since: string - channel?: string - cursor?: string - sinceExplicit?: boolean -} - -export interface ChannelOptions extends CLIOptions { - channel: string // channel name - team?: string // team name (required if multi-team) - limit: number - since: string - cursor?: string - sinceExplicit?: boolean -} - -export interface SearchOptions extends CLIOptions { - query: string - team?: string - limit: number -} - -export interface MentionOptions extends CLIOptions { - team?: string - limit: number - since?: string - channel?: string - mentionNames: string[] -} - -export interface UnreadOptions extends CLIOptions { - team?: string - peek?: number -} - -export interface IdentityOptions { - url: string - token: string - json: boolean - redact: boolean -} - -export interface UsersOptions extends IdentityOptions { - query?: string - team?: string - limit: number -} - -export interface SendDirectMessageOptions extends IdentityOptions { - username: string - message?: string - dryRun: boolean -} - -export interface SendGroupMessageOptions extends IdentityOptions { - channelId: string - message?: string - dryRun: boolean -} - -// Processed message for output - -export interface ProcessedMessage { - id: string - permalink: string - user: string - userId: string - text: string - timestamp: Date - updatedAt: Date - editedAt?: Date - deletedAt?: Date - isDeleted: boolean - postType: string - isSystem: boolean - isPinned: boolean - files: string[] - fileDetails: ProcessedFile[] - attachments: ProcessedAttachment[] - reactions: ReactionSummary[] - rootId?: string - replyCount?: number - replies?: ProcessedMessage[] -} - -export interface ProcessedFile { - id: string - name?: string - mime?: string - size?: number - extension?: string -} - -export interface ProcessedAttachmentField { - title?: string - value?: string - short?: boolean -} - -export interface ProcessedAttachment { - fallback?: string - pretext?: string - title?: string - titleLink?: string - text?: string - fields?: ProcessedAttachmentField[] - footer?: string - footerIcon?: string - authorName?: string - authorLink?: string - authorIcon?: string - color?: string - imageUrl?: string - thumbUrl?: string - timestamp?: string -} - -export interface ReactionActor { - id: string - username?: string -} - -export interface ReactionSummary { - emoji: string - count: number - actors: ReactionActor[] -} - -export interface ProcessedChannel { - id: string - type: 'dm' | 'public' | 'private' | 'group' | 'unknown' - name: string // "@username" for DMs, "channel-name" for channels - displayName?: string // Channel display name (channels only) - metadataStatus: 'resolved' | 'unavailable' -} - -export interface MessageOutput { - channel: ProcessedChannel - messages: ProcessedMessage[] - redactions: Redaction[] - retrieval: RetrievalMetadata -} - -export interface RetrievalMetadata { - selection: { - source: 'recent' | 'search' | 'mentions' | 'unread' | 'thread' - selectedCount: number - requestedLimit: number | null - since: string | null - queryTruncated: boolean | null - inputCursor: string | null - nextCursor: string | null - } - visibleThreads: { - status: 'not_requested' | 'complete' | 'partial' - hydratedRootCount: number - failedRootIds: string[] - } - visiblePostCount: number - deletedPostsIncluded: false -} - -export interface Redaction { - type: string - masked: string - position: number - field?: string -} - -export interface PreprocessResult { - text: string - redactions: Redaction[] -} - -export interface WSPostEvent { - event: 'posted' - data: { - post: string - channel_type: string - channel_name: string - channel_display_name: string - sender_name: string - mentions?: string - } - broadcast: { - channel_id: string - team_id: string - } -} - -export interface WatchEvent { - type: 'posted' - postId: string - channelId: string - channelName: string - sender: string - senderId: string - message: string - timestamp: string - rootId?: string - fileIds: string[] - redactions: Redaction[] -} diff --git a/src/utils/colors.ts b/src/utils/colors.ts deleted file mode 100644 index 1a05834..0000000 --- a/src/utils/colors.ts +++ /dev/null @@ -1,57 +0,0 @@ -const ansi = { - reset: '\x1b[0m', - bold: '\x1b[1m', - dim: '\x1b[2m', - italic: '\x1b[3m', - black: '\x1b[30m', - red: '\x1b[31m', - green: '\x1b[32m', - yellow: '\x1b[33m', - blue: '\x1b[34m', - magenta: '\x1b[35m', - cyan: '\x1b[36m', - white: '\x1b[37m', -} as const - -function colorize(text: string, ...codes: string[]): string { - return `${codes.join('')}${text}${ansi.reset}` -} - -export function bold(text: string): string { - return colorize(text, ansi.bold) -} - -export function dim(text: string): string { - return colorize(text, ansi.dim) -} - -export function cyan(text: string): string { - return colorize(text, ansi.cyan) -} - -export function yellow(text: string): string { - return colorize(text, ansi.yellow) -} - -export function green(text: string): string { - return colorize(text, ansi.green) -} - -export function magenta(text: string): string { - return colorize(text, ansi.magenta) -} - -export function blue(text: string): string { - return colorize(text, ansi.blue) -} - -export function userColor(username: string): string { - const userColors = [cyan, yellow, green, magenta, blue] - let hash = 0 - for (const char of username) { - hash = (hash << 5) - hash + char.charCodeAt(0) - hash &= hash - } - const colorFn = userColors[Math.abs(hash) % userColors.length] ?? cyan - return colorFn(username) -} diff --git a/src/utils/date.ts b/src/utils/date.ts deleted file mode 100644 index a4001e7..0000000 --- a/src/utils/date.ts +++ /dev/null @@ -1,91 +0,0 @@ -// Centralized date formatting utilities - -/** - * Format date as "D Mon" or "D Mon YYYY" (European style) - */ -export function formatDate(date: Date, options?: { includeYear?: boolean }): string { - return date.toLocaleDateString('en-GB', { - day: 'numeric', - month: 'short', - year: options?.includeYear ? 'numeric' : undefined, - }) -} - -/** - * Format date with full weekday and month names (for markdown headings) - */ -export function formatDateLong(date: Date): string { - return date.toLocaleDateString('en-GB', { - weekday: 'long', - day: 'numeric', - month: 'long', - year: 'numeric', - }) -} - -/** - * Format time as HH:MM (24-hour) - */ -export function formatTime(date: Date): string { - return date.toLocaleTimeString('en-GB', { - hour: '2-digit', - minute: '2-digit', - hour12: false, - }) -} - -/** - * Format as relative time using Intl.RelativeTimeFormat - * e.g., "2 days ago", "5 minutes ago", "just now" - */ -export function formatRelativeTime(date: Date): string { - const now = Date.now() - const diff = date.getTime() - now // negative for past - const absDiff = Math.abs(diff) - - const rtf = new Intl.RelativeTimeFormat('en', { numeric: 'auto' }) - - const minute = 60 * 1000 - const hour = 60 * minute - const day = 24 * hour - const week = 7 * day - const month = 30 * day - const year = 365 * day - - if (absDiff < minute) { - return 'just now' - } else if (absDiff < hour) { - return rtf.format(Math.round(diff / minute), 'minute') - } else if (absDiff < day) { - return rtf.format(Math.round(diff / hour), 'hour') - } else if (absDiff < week) { - return rtf.format(Math.round(diff / day), 'day') - } else if (absDiff < month) { - return rtf.format(Math.round(diff / week), 'week') - } else if (absDiff < year) { - return rtf.format(Math.round(diff / month), 'month') - } else { - return rtf.format(Math.round(diff / year), 'year') - } -} - -/** - * Get date group label for message grouping. - * Returns "Today", "Yesterday", or formatted date. - */ -export function getDateGroupLabel(date: Date): string { - const today = new Date() - const yesterday = new Date(today) - yesterday.setDate(yesterday.getDate() - 1) - - if (date.toDateString() === today.toDateString()) { - return 'Today' - } - if (date.toDateString() === yesterday.toDateString()) { - return 'Yesterday' - } - - return formatDate(date, { - includeYear: date.getFullYear() !== today.getFullYear(), - }) -} diff --git a/src/utils/index.ts b/src/utils/index.ts deleted file mode 100644 index a9167c1..0000000 --- a/src/utils/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './colors' -export * from './date' -export * from './threading' -export * from './unread' diff --git a/src/utils/threading.ts b/src/utils/threading.ts deleted file mode 100644 index ef27da8..0000000 --- a/src/utils/threading.ts +++ /dev/null @@ -1,60 +0,0 @@ -import type { ProcessedMessage } from '../types' - -const canonicalIdentities = new WeakMap() - -export function setCanonicalPostIdentity( - message: ProcessedMessage, - id: string, - rootId?: string, -): void { - canonicalIdentities.set(message, { id, rootId }) -} - -function canonicalIdentity(message: ProcessedMessage): { id: string; rootId?: string } { - return canonicalIdentities.get(message) ?? { id: message.id, rootId: message.rootId } -} - -function byTimestamp(a: ProcessedMessage, b: ProcessedMessage): number { - const diff = a.timestamp.getTime() - b.timestamp.getTime() - if (diff !== 0) return diff - return a.id.localeCompare(b.id) -} - -// Group flat messages into a threaded structure. -// Replies with missing roots are kept as standalone messages. -export function groupIntoThreads(messages: ProcessedMessage[]): ProcessedMessage[] { - const sorted = [...messages].sort(byTimestamp) - const rootMap = new Map() - const roots: ProcessedMessage[] = [] - const standaloneReplies: ProcessedMessage[] = [] - - for (const msg of sorted) { - const identity = canonicalIdentity(msg) - if (!identity.rootId) { - const root: ProcessedMessage = { ...msg, replies: [] } - setCanonicalPostIdentity(root, identity.id) - rootMap.set(identity.id, root) - roots.push(root) - } - } - - for (const msg of sorted) { - const identity = canonicalIdentity(msg) - if (!identity.rootId) continue - - const root = rootMap.get(identity.rootId) - if (root) { - root.replies?.push(msg) - } else { - standaloneReplies.push(msg) - } - } - - for (const root of roots) { - if (root.replies && root.replies.length > 1) { - root.replies.sort(byTimestamp) - } - } - - return [...roots, ...standaloneReplies].sort(byTimestamp) -} diff --git a/src/utils/unread.ts b/src/utils/unread.ts deleted file mode 100644 index 6791509..0000000 --- a/src/utils/unread.ts +++ /dev/null @@ -1,31 +0,0 @@ -import type { Channel, ChannelMember } from '../types' - -export interface UnreadMetrics { - unreadCount: number - mentionCount: number -} - -export interface UnreadSortable { - unreadCount: number - mentionCount: number -} - -export function calculateUnreadMetrics(channel: Channel, member: ChannelMember): UnreadMetrics { - const total = nonNegativeInteger(channel.total_msg_count) - const read = nonNegativeInteger(member.msg_count) - return { - unreadCount: Math.max(0, total - read), - mentionCount: nonNegativeInteger(member.mention_count), - } -} - -function nonNegativeInteger(value: unknown): number { - return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 ? value : 0 -} - -export function sortUnreadEntries(entries: T[]): T[] { - return [...entries].sort((a, b) => { - if (b.mentionCount !== a.mentionCount) return b.mentionCount - a.mentionCount - return b.unreadCount - a.unreadCount - }) -} diff --git a/src/validation.ts b/src/validation.ts deleted file mode 100644 index 9398be2..0000000 --- a/src/validation.ts +++ /dev/null @@ -1,5 +0,0 @@ -export function parsePositiveSafeInteger(value: string): number | null { - if (!/^[1-9]\d*$/.test(value)) return null - const parsed = Number(value) - return Number.isSafeInteger(parsed) ? parsed : null -} diff --git a/tests/api/channels.test.ts b/tests/api/channels.test.ts deleted file mode 100644 index 56ca874..0000000 --- a/tests/api/channels.test.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { describe, expect, test } from 'vitest' -import { - getOtherUserIdFromDMChannel, - normalizeChannelName, - resolveTeamIdFromList, -} from '../../src/api/channels' -import { setActiveMattermostCredential } from '../../src/preprocessing' -import type { Channel, Team } from '../../src/types' - -function makeDMChannel(name: string): Channel { - return { - id: 'ch1', - team_id: '', - type: 'D', - display_name: '', - name, - header: '', - purpose: '', - last_post_at: 0, - total_msg_count: 0, - creator_id: '', - } -} - -function makeTeam(id: string, name: string, displayName?: string): Team { - return { - id, - name, - display_name: displayName ?? name, - type: 'O', - } -} - -describe('getOtherUserIdFromDMChannel', () => { - test('extracts other user ID from DM channel name', () => { - const channel = makeDMChannel('abc123__def456') - expect(getOtherUserIdFromDMChannel(channel, 'abc123')).toBe('def456') - expect(getOtherUserIdFromDMChannel(channel, 'def456')).toBe('abc123') - }) - - test('returns null for non-DM channels', () => { - const channel = { ...makeDMChannel('general'), type: 'O' as const } - expect(getOtherUserIdFromDMChannel(channel, 'abc123')).toBeNull() - }) - - test('returns null for malformed channel names', () => { - const channel = makeDMChannel('no-separator') - expect(getOtherUserIdFromDMChannel(channel, 'abc123')).toBeNull() - }) - - test('returns null when a name part is empty', () => { - const channel = makeDMChannel('__def456') - expect(getOtherUserIdFromDMChannel(channel, 'abc123')).toBeNull() - }) - - test('returns null when the current user is not a channel participant', () => { - const channel = makeDMChannel('abc123__def456') - expect(getOtherUserIdFromDMChannel(channel, 'someone-else')).toBeNull() - }) - - test('preserves self-DM semantics', () => { - const channel = makeDMChannel('abc123__abc123') - expect(getOtherUserIdFromDMChannel(channel, 'abc123')).toBe('abc123') - }) -}) - -describe('normalizeChannelName', () => { - test('strips leading # from channel names', () => { - expect(normalizeChannelName('#general')).toBe('general') - }) - - test('keeps names without # unchanged', () => { - expect(normalizeChannelName('dev')).toBe('dev') - }) -}) - -describe('resolveTeamIdFromList', () => { - test('returns only team id when user belongs to one team', () => { - const teams = [makeTeam('t1', 'core', 'Core')] - expect(resolveTeamIdFromList(teams)).toBe('t1') - }) - - test('selects team by slug name', () => { - const teams = [makeTeam('t1', 'core', 'Core'), makeTeam('t2', 'eng', 'Engineering')] - expect(resolveTeamIdFromList(teams, 'eng')).toBe('t2') - }) - - test('selects team by display name', () => { - const teams = [makeTeam('t1', 'core', 'Core Team'), makeTeam('t2', 'eng', 'Engineering')] - expect(resolveTeamIdFromList(teams, 'Core Team')).toBe('t1') - }) - - test('throws clear error when no teams', () => { - expect(() => resolveTeamIdFromList([])).toThrow('You are not a member of any teams.') - }) - - test.each([ - [{ name: 'core', display_name: 'Core', type: 'O' }], - [{ id: 't1', name: 'core', display_name: 'Core' }], - [{ id: 't1', name: 'core', display_name: 7, type: 'O' }], - ])('fails closed for malformed canonical team data: %j', (teams) => { - expect(() => resolveTeamIdFromList(teams)).toThrow('Invalid teams response.') - }) - - test('throws clear error when team is missing', () => { - const teams = [makeTeam('t1', 'core'), makeTeam('t2', 'eng')] - expect(() => resolveTeamIdFromList(teams, 'sales')).toThrow( - 'Team "sales" not found. Your teams: core, eng', - ) - }) - - test('protects registered credentials in requested and available team labels', () => { - const credential = 'active-team-token' - setActiveMattermostCredential(credential) - expect(() => - resolveTeamIdFromList([makeTeam('t1', credential)], `${credential}-missing`), - ).toThrow( - 'Team "[REDACTED:mattermost_credential]-missing" not found. Your teams: [REDACTED:mattermost_credential]', - ) - }) - - test('throws clear error on multi-team without --team', () => { - const teams = [makeTeam('t1', 'core', 'Core Team'), makeTeam('t2', 'eng', 'Engineering')] - expect(() => resolveTeamIdFromList(teams)).toThrow( - 'You belong to multiple teams. Use --team to specify:', - ) - }) - - test('neutralizes terminal controls in team names shown in errors', () => { - const teams = [makeTeam('t1', 'core\u001b[2J', 'Core\rSpoofed'), makeTeam('t2', 'eng')] - - expect(() => resolveTeamIdFromList(teams)).toThrow(' core\\u001b[2J (Core\\u000dSpoofed)') - }) -}) diff --git a/tests/api/client.test.ts b/tests/api/client.test.ts deleted file mode 100644 index 8cc6cb3..0000000 --- a/tests/api/client.test.ts +++ /dev/null @@ -1,339 +0,0 @@ -import { createServer } from 'node:http' -import { afterEach, describe, expect, test, vi } from 'vitest' -import { - getClient, - initClient, - MattermostClient, - MattermostMutationOutcomeUnknownError, - REQUEST_TIMEOUT_MS, - rateLimitDelay, -} from '../../src/api/client' -import { preprocess } from '../../src/preprocessing' - -describe('MattermostClient', () => { - afterEach(() => { - vi.useRealTimers() - vi.unstubAllGlobals() - vi.restoreAllMocks() - }) - - test('singleton replacement releases its prior owned credential', () => { - initClient('https://mattermost.example.com', 'old-client-token') - initClient('https://mattermost.example.com', 'new-client-token') - expect(preprocess('old-client-token new-client-token', { redact: false }).text).toBe( - 'old-client-token [REDACTED:mattermost_credential]', - ) - }) - - test('failed singleton replacement preserves the prior client and credential only', () => { - const previous = initClient('https://mattermost.example.com', 'old-client-token') - expect(() => initClient('not a URL', 'attempted-token')).toThrow('Invalid Mattermost URL') - expect(getClient()).toBe(previous) - expect(preprocess('old-client-token attempted-token', { redact: false }).text).toBe( - '[REDACTED:mattermost_credential] attempted-token', - ) - }) - - test('does not expose remote reason phrases or the configured token', async () => { - vi.stubGlobal( - 'fetch', - vi.fn().mockResolvedValue({ - headers: new Headers(), - ok: false, - status: 500, - statusText: 'Server Error fake-token\u001b[2Jspoofed', - text: async () => 'remote body', - }), - ) - - const client = new MattermostClient('https://mattermost.example.com', 'fake-token') - - await expect(client.get('/users/me')).rejects.toThrow('API request failed: 500.') - }) - - test('honors Mattermost relative reset seconds and bounds remote delays', () => { - expect(rateLimitDelay({ headers: new Headers({ 'X-RateLimit-Reset': '7' }) }, 0)).toBe(7000) - expect(rateLimitDelay({ headers: new Headers({ 'X-RateLimit-Reset': '3600' }) }, 0)).toBe( - 30 * 1000, - ) - expect(rateLimitDelay({ headers: new Headers({ 'X-RateLimit-Reset': 'invalid' }) }, 2)).toBe( - 4000, - ) - }) - - test('prefers Retry-After and accepts its HTTP-date form', () => { - vi.useFakeTimers() - vi.setSystemTime(new Date('2026-07-14T00:00:00.000Z')) - expect( - rateLimitDelay( - { - headers: new Headers({ - 'Retry-After': 'Tue, 14 Jul 2026 00:00:03 GMT', - 'X-RateLimit-Reset': String(Date.now() / 1000 + 30), - }), - }, - 0, - ), - ).toBe(3000) - vi.useRealTimers() - }) - - test('waits for X-RateLimit-Reset before retrying a 429', async () => { - vi.useFakeTimers() - const fetchMock = vi - .fn() - .mockResolvedValueOnce( - new Response('', { status: 429, headers: { 'X-RateLimit-Reset': '2' } }), - ) - .mockResolvedValueOnce(Response.json({ id: 'me' })) - vi.stubGlobal('fetch', fetchMock) - const request = new MattermostClient('https://mattermost.example.com', 'fake-token').get( - '/users/me', - ) - - await vi.advanceTimersByTimeAsync(1999) - expect(fetchMock).toHaveBeenCalledTimes(1) - await vi.advanceTimersByTimeAsync(1) - await expect(request).resolves.toEqual({ id: 'me' }) - expect(fetchMock).toHaveBeenCalledTimes(2) - }) - - test('aborts each stalled attempt and bounds read retries', async () => { - vi.useFakeTimers() - const fetchMock = vi.fn((_url: string, init?: RequestInit) => { - return new Promise((_resolve, reject) => { - init?.signal?.addEventListener('abort', () => - reject(new DOMException('aborted', 'AbortError')), - ) - }) - }) - vi.stubGlobal('fetch', fetchMock) - const request = new MattermostClient('https://mattermost.example.com', 'fake-token').get( - '/users/me', - ) - const rejection = expect(request).rejects.toThrow(`timed out after ${REQUEST_TIMEOUT_MS}ms`) - - await vi.advanceTimersByTimeAsync(REQUEST_TIMEOUT_MS + 1000) - await vi.advanceTimersByTimeAsync(REQUEST_TIMEOUT_MS + 2000) - await vi.advanceTimersByTimeAsync(REQUEST_TIMEOUT_MS) - - await rejection - expect(fetchMock).toHaveBeenCalledTimes(3) - }) - - test('retries read-only gateway failures but not ordinary posts', async () => { - vi.useFakeTimers() - const fetchMock = vi - .fn() - .mockResolvedValueOnce(new Response('', { status: 503 })) - .mockResolvedValueOnce(Response.json({ id: 'me' })) - .mockResolvedValueOnce(new Response('', { status: 503 })) - vi.stubGlobal('fetch', fetchMock) - const client = new MattermostClient('https://mattermost.example.com', 'fake-token') - const read = client.get('/users/me') - await vi.advanceTimersByTimeAsync(1000) - - await expect(read).resolves.toEqual({ id: 'me' }) - await expect(client.post('/posts', { message: 'hello' })).rejects.toBeInstanceOf( - MattermostMutationOutcomeUnknownError, - ) - expect(fetchMock).toHaveBeenCalledTimes(3) - }) - - test('keeps the timeout active while consuming the response body', async () => { - vi.useFakeTimers() - const fetchMock = vi - .fn() - .mockImplementationOnce((_url: string, init?: RequestInit) => - Promise.resolve({ - ok: true, - status: 200, - json: () => - new Promise((_resolve, reject) => { - init?.signal?.addEventListener('abort', () => reject(new DOMException('secret body'))) - }), - }), - ) - .mockResolvedValueOnce(Response.json({ id: 'me' })) - vi.stubGlobal('fetch', fetchMock) - const request = new MattermostClient('https://mattermost.example.com', 'fake-token').get( - '/users/me', - ) - - await vi.advanceTimersByTimeAsync(REQUEST_TIMEOUT_MS + 1000) - await expect(request).resolves.toEqual({ id: 'me' }) - expect(fetchMock).toHaveBeenCalledTimes(2) - }) - - test.each([ - ['POST', (client: MattermostClient) => client.post('/posts', { message: 'hello' })], - ['PUT', (client: MattermostClient) => client.put('/posts/id', { message: 'hello' })], - ['DELETE', (client: MattermostClient) => client.delete('/posts/id')], - ] as const)('does not replay a mutating %s request after a 429', async (_method, request) => { - const fetchMock = vi.fn().mockResolvedValue( - new Response('', { - status: 429, - headers: { 'Retry-After': '0' }, - }), - ) - vi.stubGlobal('fetch', fetchMock) - - await expect( - request(new MattermostClient('https://mattermost.example.com', 'fake-token')), - ).rejects.toMatchObject({ status: 429 }) - expect(fetchMock).toHaveBeenCalledTimes(1) - }) - - test('does not follow a redirect that would replay a mutation', async () => { - const hits: string[] = [] - const server = createServer((request, response) => { - hits.push(`${request.method} ${request.url}`) - response.writeHead(307, { Location: '/redirected' }) - response.end() - }) - await new Promise((resolve, reject) => { - server.once('error', reject) - server.listen(0, '127.0.0.1', resolve) - }) - try { - const address = server.address() - if (!address || typeof address === 'string') throw new Error('Test server did not bind') - const client = new MattermostClient(`http://127.0.0.1:${address.port}`, 'fake-token') - - await expect(client.post('/posts', { message: 'one attempt' })).rejects.toBeInstanceOf( - MattermostMutationOutcomeUnknownError, - ) - expect(hits).toEqual(['POST /api/v4/posts']) - } finally { - await new Promise((resolve, reject) => - server.close((error) => (error ? reject(error) : resolve())), - ) - } - }) - - test.each([ - 'timeout', - 'transport failure', - ] as const)('does not retry a mutation after a %s and reports an unknown outcome', async (kind) => { - vi.useFakeTimers() - const fetchMock = - kind === 'timeout' - ? vi.fn( - (_url: string, init?: RequestInit) => - new Promise((_resolve, reject) => { - init?.signal?.addEventListener('abort', () => - reject(new DOMException('secret mutation body', 'AbortError')), - ) - }), - ) - : vi.fn().mockRejectedValue(new TypeError('secret transport detail')) - vi.stubGlobal('fetch', fetchMock) - const request = new MattermostClient('https://mattermost.example.com', 'fake-token').post( - '/posts', - { message: 'secret message' }, - ) - const rejection = expect(request).rejects.toBeInstanceOf(MattermostMutationOutcomeUnknownError) - - if (kind === 'timeout') await vi.advanceTimersByTimeAsync(REQUEST_TIMEOUT_MS) - await rejection - expect(fetchMock).toHaveBeenCalledTimes(1) - }) - - test('reports malformed success JSON without reflecting parser or body details', async () => { - vi.stubGlobal( - 'fetch', - vi.fn().mockResolvedValue({ - ok: true, - status: 200, - json: () => Promise.reject(new SyntaxError('Unexpected token: secret-body-fragment')), - }), - ) - - await expect( - new MattermostClient('https://mattermost.example.com', 'fake-token').get('/users/me'), - ).rejects.toThrow('Mattermost returned an invalid JSON response.') - }) - - test('treats malformed mutation success JSON as an unknown outcome without replaying', async () => { - const fetchMock = vi.fn().mockResolvedValue({ - ok: true, - status: 201, - json: () => Promise.reject(new SyntaxError('secret response fragment')), - }) - vi.stubGlobal('fetch', fetchMock) - - await expect( - new MattermostClient('https://mattermost.example.com', 'fake-token').post('/posts', { - message: 'secret message', - }), - ).rejects.toBeInstanceOf(MattermostMutationOutcomeUnknownError) - expect(fetchMock).toHaveBeenCalledTimes(1) - }) - - test('retries a terminated response body for a read-only request', async () => { - vi.useFakeTimers() - const fetchMock = vi - .fn() - .mockResolvedValueOnce({ - ok: true, - status: 200, - json: () => Promise.reject(new TypeError('terminated secret-body-fragment')), - }) - .mockResolvedValueOnce(Response.json({ id: 'me' })) - vi.stubGlobal('fetch', fetchMock) - const request = new MattermostClient('https://mattermost.example.com', 'fake-token').get( - '/users/me', - ) - - await vi.advanceTimersByTimeAsync(1000) - await expect(request).resolves.toEqual({ id: 'me' }) - expect(fetchMock).toHaveBeenCalledTimes(2) - }) - - test('bounds repeated response-body terminations and hides their cause', async () => { - vi.useFakeTimers() - const fetchMock = vi.fn().mockResolvedValue({ - ok: true, - status: 200, - json: () => Promise.reject(new TypeError('terminated secret-body-fragment')), - }) - vi.stubGlobal('fetch', fetchMock) - const request = new MattermostClient('https://mattermost.example.com', 'fake-token').get( - '/users/me', - ) - const rejection = expect(request).rejects.toThrow( - 'Unable to connect to Mattermost due to a network error.', - ) - - await vi.advanceTimersByTimeAsync(3000) - await rejection - expect(fetchMock).toHaveBeenCalledTimes(3) - }) - - test('reports bounded rate-limit waits on stderr without consuming response bodies', async () => { - vi.useFakeTimers() - const text = vi.fn() - const fetchMock = vi - .fn() - .mockResolvedValueOnce({ - headers: new Headers({ 'Retry-After': '3600' }), - ok: false, - status: 429, - statusText: 'Too Many Requests', - text, - }) - .mockResolvedValueOnce(Response.json({ id: 'me' })) - vi.stubGlobal('fetch', fetchMock) - const stderr = vi.spyOn(console, 'error').mockImplementation(() => {}) - const request = new MattermostClient('https://mattermost.example.com', 'fake-token').get( - '/users/me', - ) - - await vi.advanceTimersByTimeAsync(30_000) - await expect(request).resolves.toEqual({ id: 'me' }) - expect(stderr).toHaveBeenCalledWith( - 'Mattermost request was rate limited; retrying in 30000ms (attempt 1).', - ) - expect(text).not.toHaveBeenCalled() - }) -}) diff --git a/tests/api/messages.test.ts b/tests/api/messages.test.ts deleted file mode 100644 index 89ecc5f..0000000 --- a/tests/api/messages.test.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { afterEach, describe, expect, test, vi } from 'vitest' -import { - createDirectChannel, - createPost, - initClient, - MattermostMutationOutcomeUnknownError, -} from '../../src/api' -import { MAX_MESSAGE_CHARACTERS } from '../../src/input' - -describe('message write API', () => { - const postId = 'p'.repeat(26) - - afterEach(() => { - vi.unstubAllGlobals() - vi.restoreAllMocks() - }) - - test('creates a direct channel for the exact two participants without retry opt-in', async () => { - const fetchMock = vi.fn().mockResolvedValue( - Response.json({ - id: 'dm-channel', - type: 'D', - name: 'me__recipient', - display_name: '', - team_id: '', - }), - ) - vi.stubGlobal('fetch', fetchMock) - initClient('https://mattermost.test', 'token') - - await expect(createDirectChannel('me', 'recipient')).resolves.toMatchObject({ - id: 'dm-channel', - type: 'D', - }) - expect(fetchMock).toHaveBeenCalledWith( - 'https://mattermost.test/api/v4/channels/direct', - expect.objectContaining({ method: 'POST', body: JSON.stringify(['me', 'recipient']) }), - ) - }) - - test.each([ - [{ id: '', type: 'D', name: 'me__recipient' }], - [{ id: ' ', type: 'D', name: 'me__recipient', display_name: '', team_id: '' }], - [{ id: 'channel', type: 'G', name: 'me__recipient' }], - [{ id: 'channel', type: 'D', name: 'me__someone-else' }], - [{ id: 'channel', type: 'D', name: 'me__recipient', display_name: '', team_id: 'team' }], - [null], - ])('rejects a malformed or mismatched direct-channel response', async (response) => { - vi.stubGlobal('fetch', vi.fn().mockResolvedValue(Response.json(response))) - initClient('https://mattermost.test', 'token') - - await expect(createDirectChannel('me', 'recipient')).rejects.toBeInstanceOf( - MattermostMutationOutcomeUnknownError, - ) - }) - - test('creates a post with a stable pending ID and returns only a narrow receipt', async () => { - const fetchMock = vi.fn().mockResolvedValue( - Response.json({ - id: postId, - channel_id: 'channel-id', - user_id: 'sender-id', - create_at: 1_784_023_427_000, - message: 'server-visible message', - hostile: 'ignored', - }), - ) - vi.stubGlobal('fetch', fetchMock) - initClient('https://mattermost.test', 'token') - - await expect(createPost('channel-id', 'hello', 'pending-id')).resolves.toEqual({ - id: postId, - channelId: 'channel-id', - userId: 'sender-id', - createAt: 1_784_023_427_000, - pendingPostId: 'pending-id', - }) - expect(fetchMock).toHaveBeenCalledWith( - 'https://mattermost.test/api/v4/posts', - expect.objectContaining({ - method: 'POST', - body: JSON.stringify({ - channel_id: 'channel-id', - message: 'hello', - pending_post_id: 'pending-id', - }), - }), - ) - }) - - test.each([ - [{ id: '', channel_id: 'channel-id', user_id: 'sender', create_at: 1 }], - [{ id: ' ', channel_id: 'channel-id', user_id: 'sender', create_at: 1 }], - [{ id: '\ud800', channel_id: 'channel-id', user_id: 'sender', create_at: 1 }], - [{ id: 'private launch detail', channel_id: 'channel-id', user_id: 'sender', create_at: 1 }], - [{ id: postId, channel_id: 'wrong', user_id: 'sender', create_at: 1 }], - [{ id: postId, channel_id: 'channel-id', user_id: '', create_at: 1 }], - [{ id: postId, channel_id: 'channel-id', user_id: ' ', create_at: 1 }], - [{ id: postId, channel_id: 'channel-id', user_id: 'sender', create_at: Number.NaN }], - [{ id: postId, channel_id: 'channel-id', user_id: 'sender', create_at: 1e100 }], - [null], - ])('rejects a malformed or mismatched create-post response', async (response) => { - vi.stubGlobal('fetch', vi.fn().mockResolvedValue(Response.json(response))) - initClient('https://mattermost.test', 'token') - - await expect(createPost('channel-id', 'hello', 'pending-id')).rejects.toBeInstanceOf( - MattermostMutationOutcomeUnknownError, - ) - }) - - test.each([ - '', - ' ', - '\n\t', - ])('rejects an empty message before network access', async (message) => { - const fetchMock = vi.fn() - vi.stubGlobal('fetch', fetchMock) - initClient('https://mattermost.test', 'token') - - await expect(createPost('channel-id', message)).rejects.toThrow('Message cannot be empty.') - expect(fetchMock).not.toHaveBeenCalled() - }) - - test('rejects a message above the Mattermost character limit before network access', async () => { - const fetchMock = vi.fn() - vi.stubGlobal('fetch', fetchMock) - initClient('https://mattermost.test', 'token') - - await expect(createPost('channel-id', 'a'.repeat(MAX_MESSAGE_CHARACTERS + 1))).rejects.toThrow( - `Message exceeds ${MAX_MESSAGE_CHARACTERS} Unicode characters.`, - ) - expect(fetchMock).not.toHaveBeenCalled() - }) -}) diff --git a/tests/api/paths.test.ts b/tests/api/paths.test.ts deleted file mode 100644 index 19ac800..0000000 --- a/tests/api/paths.test.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { afterEach, describe, expect, test, vi } from 'vitest' -import { getChannel, getChannelByName, getMyChannels } from '../../src/api/channels' -import { initClient } from '../../src/api/client' -import { getChannelPosts, getPostThread, searchPosts } from '../../src/api/posts' -import { clearUserCache, getUser, getUserByUsername } from '../../src/api/users' - -describe('Mattermost API paths', () => { - afterEach(() => { - clearUserCache() - vi.unstubAllGlobals() - }) - - test('encodes every dynamic path segment while leaving query parameters intact', async () => { - const urls: URL[] = [] - const responses: unknown[] = [ - { id: 'user/id', username: 'first' }, - { id: 'second', username: 'name/with space' }, - [], - { id: 'channel/id' }, - { id: 'named-channel' }, - { order: [], posts: {}, has_next: false }, - { order: [], posts: {}, has_next: false }, - { order: [], posts: {}, matches: {}, has_next: false }, - ] - vi.stubGlobal( - 'fetch', - vi.fn(async (input: string | URL | Request) => { - urls.push(new URL(String(input))) - return Response.json(responses.shift()) - }), - ) - initClient('https://mattermost.example.com', 'fake-token') - - await getUser('user/id') - await getUserByUsername('name/with space') - await getMyChannels('user/id') - await getChannel('channel/id') - await getChannelByName('team/id', '#release/name') - await getChannelPosts('channel/id', { before: 'post/id' }) - await getPostThread('post/id') - await searchPosts('team/id', 'term', 1) - - expect(urls.map(({ pathname }) => pathname)).toEqual([ - '/api/v4/users/user%2Fid', - '/api/v4/users/username/name%2Fwith%20space', - '/api/v4/users/user%2Fid/channels', - '/api/v4/channels/channel%2Fid', - '/api/v4/teams/team%2Fid/channels/name/release%2Fname', - '/api/v4/channels/channel%2Fid/posts', - '/api/v4/posts/post%2Fid/thread', - '/api/v4/teams/team%2Fid/posts/search', - ]) - expect(urls[5]?.searchParams.get('before')).toBe('post/id') - expect(urls[6]?.searchParams.get('direction')).toBe('down') - }) -}) diff --git a/tests/api/posts.test.ts b/tests/api/posts.test.ts deleted file mode 100644 index c2bd4c9..0000000 --- a/tests/api/posts.test.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { describe, expect, test } from 'vitest' -import { parseDuration, takeMostRecentPosts } from '../../src/api/posts' - -describe('parseDuration', () => { - test('parses hours', () => { - const now = Date.now() - const since = parseDuration('24h') - - // Should be roughly 24 hours ago (within 1 second tolerance) - const expected = now - 24 * 60 * 60 * 1000 - expect(Math.abs(since - expected)).toBeLessThan(1000) - }) - - test('parses days', () => { - const now = Date.now() - const since = parseDuration('7d') - - const expected = now - 7 * 24 * 60 * 60 * 1000 - expect(Math.abs(since - expected)).toBeLessThan(1000) - }) - - test('parses weeks', () => { - const now = Date.now() - const since = parseDuration('2w') - - const expected = now - 2 * 7 * 24 * 60 * 60 * 1000 - expect(Math.abs(since - expected)).toBeLessThan(1000) - }) - - test('parses months (30 days)', () => { - const now = Date.now() - const since = parseDuration('1m') - - const expected = now - 30 * 24 * 60 * 60 * 1000 - expect(Math.abs(since - expected)).toBeLessThan(1000) - }) - - test('throws on invalid format', () => { - expect(() => parseDuration('invalid')).toThrow() - expect(() => parseDuration('24')).toThrow() - expect(() => parseDuration('h')).toThrow() - }) -}) - -describe('takeMostRecentPosts', () => { - test('keeps the most recent posts across channels under a global limit', () => { - const posts = [ - { id: 'a1', channel_id: 'chan-a', create_at: 1000 }, - { id: 'b1', channel_id: 'chan-b', create_at: 5000 }, - { id: 'a2', channel_id: 'chan-a', create_at: 4000 }, - { id: 'b2', channel_id: 'chan-b', create_at: 3000 }, - ] - - const result = takeMostRecentPosts(posts, 2) - - expect(result.map((post) => post.id)).toEqual(['b1', 'a2']) - }) - - test('uses post id as a stable tiebreaker when timestamps match', () => { - const posts = [ - { id: 'post-b', channel_id: 'chan-a', create_at: 1000 }, - { id: 'post-a', channel_id: 'chan-b', create_at: 1000 }, - ] - - const result = takeMostRecentPosts(posts, 2) - - expect(result.map((post) => post.id)).toEqual(['post-a', 'post-b']) - }) - - test('does not let duplicate post ids consume the global limit', () => { - const posts = [ - { id: 'new', create_at: 3000 }, - { id: 'new', create_at: 3000 }, - { id: 'older', create_at: 2000 }, - ] - - expect(takeMostRecentPosts(posts, 2).map((post) => post.id)).toEqual(['new', 'older']) - }) -}) diff --git a/tests/api/retrieval.test.ts b/tests/api/retrieval.test.ts deleted file mode 100644 index 50e9214..0000000 --- a/tests/api/retrieval.test.ts +++ /dev/null @@ -1,762 +0,0 @@ -import { afterEach, describe, expect, test, vi } from 'vitest' -import { getMyDMChannels } from '../../src/api/channels' -import { initClient } from '../../src/api/client' -import { - getAllChannelPosts, - getChannelPosts, - getPostThread, - searchPosts, - takeMostRecentPosts, -} from '../../src/api/posts' -import type { Channel, Post, PostsResponse, SearchResponse } from '../../src/types' -import { installRouteFetch } from '../helpers/fake-fetch' - -function post(id: string, createAt: number): Post { - return { id, create_at: createAt, delete_at: 0, channel_id: 'channel', message: id } as Post -} - -function page(items: Post[]): PostsResponse { - return { - order: items.map(({ id }) => id), - posts: Object.fromEntries(items.map((item) => [item.id, item])), - } as PostsResponse -} - -afterEach(() => vi.unstubAllGlobals()) - -describe('route-aware retrieval integration', () => { - test('uses limit plus one and proves channel truncation locally', async () => { - const { requests } = installRouteFetch([ - { - method: 'GET', - path: '/api/v4/channels/channel/posts', - handle: () => page([post('new', 3), post('selected', 2), post('extra', 1)]), - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await getAllChannelPosts('channel', { limit: 2 }) - - expect(result.posts.map(({ id }) => id)).toEqual(['new', 'selected']) - expect(result.truncated).toBe(true) - expect(requests).toHaveLength(1) - expect(requests[0]?.url.searchParams.get('per_page')).toBe('3') - }) - - test('resumes locally across equal-millisecond peers without admitting newer posts', async () => { - installRouteFetch([ - { - method: 'GET', - path: '/api/v4/channels/channel/posts', - handle: ({ url }) => - url.searchParams.get('page') === '0' - ? page([ - post('arrived-later', 300), - post('anchor', 200), - post('peer-after-anchor', 200), - ]) - : page([post('older', 100)]), - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await getAllChannelPosts('channel', { - limit: 2, - boundary: { createAt: 200, id: 'anchor' }, - }) - - expect(result.posts.map(({ id }) => id)).toEqual(['peer-after-anchor', 'older']) - expect(result.truncated).toBe(false) - }) - - test('uses a safe before anchor while retaining the local boundary filter', async () => { - const { requests } = installRouteFetch([ - { - method: 'GET', - path: '/api/v4/channels/channel/posts', - handle: () => page([post('peer', 200), post('older', 100)]), - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await getAllChannelPosts('channel', { - limit: 2, - boundary: { createAt: 200, id: 'anchor' }, - safeBeforePostId: 'safe-newer', - }) - - expect(requests[0]?.url.searchParams.get('before')).toBe('safe-newer') - expect(result.posts.map(({ id }) => id)).toEqual(['peer', 'older']) - }) - - test('carries the safe anchor across bounded deep mixed-timestamp pages', async () => { - const { requests } = installRouteFetch([ - { - method: 'GET', - path: '/api/v4/channels/channel/posts', - handle: ({ url }) => { - const pageNumber = Number(url.searchParams.get('page')) - if (pageNumber === 0) { - return page([ - post('newest-than-boundary', 400), - post('newer-than-boundary', 300), - post('anchor', 200), - ]) - } - if (pageNumber === 1) return page([post('peer', 200), post('older', 100)]) - return page([]) - }, - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await getAllChannelPosts('channel', { - limit: 2, - boundary: { createAt: 200, id: 'anchor' }, - safeBeforePostId: 'safe-newer', - }) - - expect(result.posts.map(({ id }) => id)).toEqual(['peer', 'older']) - expect(requests).toHaveLength(2) - expect(requests.every(({ url }) => url.searchParams.get('before') === 'safe-newer')).toBe(true) - }) - - test('retries page zero once without a deleted safe anchor and recovers remaining history', async () => { - const { requests } = installRouteFetch([ - { - method: 'GET', - path: '/api/v4/channels/channel/posts', - handle: ({ url }) => - url.searchParams.has('before') ? page([]) : page([post('peer', 200), post('older', 100)]), - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await getAllChannelPosts('channel', { - limit: 2, - boundary: { createAt: 200, id: 'anchor' }, - safeBeforePostId: 'deleted-safe-anchor', - }) - - expect(result.posts.map(({ id }) => id)).toEqual(['peer', 'older']) - expect(result.truncated).toBe(false) - expect(result.safeBeforeValid).toBe(false) - expect(requests).toHaveLength(2) - expect(requests.map(({ url }) => url.searchParams.get('before'))).toEqual([ - 'deleted-safe-anchor', - null, - ]) - expect(requests.every(({ url }) => url.searchParams.get('page') === '0')).toBe(true) - }) - - test('does not claim exhaustion for an empty channel page that reports more data', async () => { - installRouteFetch([ - { - method: 'GET', - path: '/api/v4/channels/channel/posts', - handle: () => ({ ...page([]), has_next: true }), - }, - ]) - initClient('https://mattermost.test', 'token') - - expect((await getAllChannelPosts('channel', { limit: 2 })).truncated).toBeNull() - }) - - test('retries an empty has-next page zero without an unchanged safe anchor', async () => { - const { requests } = installRouteFetch([ - { - method: 'GET', - path: '/api/v4/channels/channel/posts', - handle: ({ url }) => - url.searchParams.has('before') - ? { ...page([]), has_next: true } - : { ...page([post('older', 100)]), has_next: false }, - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await getAllChannelPosts('channel', { - limit: 2, - safeBeforePostId: 'missing-anchor', - }) - - expect(result.posts.map(({ id }) => id)).toEqual(['older']) - expect(result.truncated).toBe(false) - expect(result.safeBeforeValid).toBe(false) - expect(requests.map(({ url }) => url.searchParams.get('page'))).toEqual(['0', '0']) - expect(requests.map(({ url }) => url.searchParams.get('before'))).toEqual([ - 'missing-anchor', - null, - ]) - }) - - test('reports unknown after two full stagnant channel pages', async () => { - const { requests } = installRouteFetch([ - { - method: 'GET', - path: '/api/v4/channels/channel/posts', - handle: () => page([post('a', 2), post('b', 1), post('a', 2), post('b', 1)]), - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await getAllChannelPosts('channel', { limit: 3 }) - - expect(result.truncated).toBeNull() - expect(requests).toHaveLength(3) - }) - - test('requests channel pages without server-side thread expansion', async () => { - const { requests } = installRouteFetch([ - { - method: 'GET', - path: '/api/v4/channels/channel/posts', - handle: () => page([]), - }, - ]) - initClient('https://mattermost.test', 'token') - - await getChannelPosts('channel') - - expect(requests[0]?.url.searchParams.get('skipFetchThreads')).toBe('true') - }) - - test('continues after a raw full channel page whose posts are missing or deleted', async () => { - const { requests } = installRouteFetch([ - { - method: 'GET', - path: '/api/v4/channels/channel/posts', - handle: ({ url }) => - url.searchParams.get('page') === '0' - ? { - order: ['missing-a', 'deleted', 'missing-b'], - posts: { deleted: { ...post('deleted', 3), delete_at: 1 } }, - } - : page([post('live', 2)]), - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await getAllChannelPosts('channel', { limit: 2 }) - - expect(result.posts.map(({ id }) => id)).toEqual(['live']) - expect(result.truncated).toBeNull() - expect(requests).toHaveLength(2) - }) - - test('channel history treats a null post map as unknown instead of empty', async () => { - installRouteFetch([ - { - method: 'GET', - path: '/api/v4/channels/channel/posts', - handle: () => ({ order: ['stale-hit'], posts: null, has_next: false }), - }, - ]) - initClient('https://mattermost.test', 'token') - - expect(await getAllChannelPosts('channel', { limit: 1 })).toMatchObject({ - posts: [], - truncated: null, - }) - }) - - test('does not claim channel exhaustion past an inaccessible post boundary', async () => { - installRouteFetch([ - { - method: 'GET', - path: '/api/v4/channels/channel/posts', - handle: () => ({ ...page([post('visible', 2)]), first_inaccessible_post_time: 1 }), - }, - ]) - initClient('https://mattermost.test', 'token') - - expect((await getAllChannelPosts('channel', { limit: 2 })).truncated).toBeNull() - }) - - test('only marks a thread complete when the API explicitly reports no next page', async () => { - let hasNext: boolean | undefined = false - let inaccessibleTime: number | undefined - installRouteFetch([ - { - method: 'GET', - path: '/api/v4/posts/root/thread', - handle: () => ({ - ...page([post('root', 1)]), - has_next: hasNext, - first_inaccessible_post_time: inaccessibleTime, - }), - }, - ]) - initClient('https://mattermost.test', 'token') - - expect((await getPostThread('root')).truncated).toBe(false) - hasNext = true - expect((await getPostThread('root')).truncated).toBe(true) - hasNext = undefined - expect((await getPostThread('root')).truncated).toBeNull() - hasNext = false - inaccessibleTime = 1 - expect((await getPostThread('root')).truncated).toBeNull() - }) - - test('proves legacy thread completeness only from root reply counts', async () => { - const completeRoot = { ...post('complete-root', 1), root_id: '', reply_count: 1 } - const completeReply = { ...post('complete-reply', 2), root_id: 'complete-root', reply_count: 0 } - const partialRoot = { ...post('partial-root', 1), root_id: '', reply_count: 2 } - const partialReply = { ...post('partial-reply', 2), root_id: 'partial-root', reply_count: 0 } - installRouteFetch([ - { - method: 'GET', - path: '/api/v4/posts/complete-root/thread', - handle: () => page([completeRoot, completeReply]), - }, - { - method: 'GET', - path: '/api/v4/posts/partial-root/thread', - handle: () => page([partialRoot, partialReply]), - }, - ]) - initClient('https://mattermost.test', 'token') - - expect((await getPostThread('complete-root')).truncated).toBe(false) - expect((await getPostThread('partial-root')).truncated).toBeNull() - }) - - test('thread retrieval preserves valid posts but reports missing ordered payloads as unknown', async () => { - const root = { ...post('root', 1), root_id: '', reply_count: 1 } - installRouteFetch([ - { - method: 'GET', - path: '/api/v4/posts/root/thread', - handle: () => ({ - order: ['root', 'missing-reply'], - posts: { root }, - has_next: false, - }), - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await getPostThread('root') - - expect(result.posts.map(({ id }) => id)).toEqual(['root']) - expect(result.truncated).toBeNull() - }) - - test('thread retrieval rejects a malformed first page generically', async () => { - installRouteFetch([ - { - method: 'GET', - path: '/api/v4/posts/root/thread', - handle: () => null, - }, - ]) - initClient('https://mattermost.test', 'token') - - await expect(getPostThread('root')).rejects.toThrow( - 'Mattermost returned an invalid posts response.', - ) - }) - - test('paginates threads with Mattermost cursor casing and dedupes posts', async () => { - const root = { ...post('root', 1), root_id: '', reply_count: 2 } - const firstReply = { ...post('reply-1', 2), root_id: 'root', reply_count: 0 } - const secondReply = { ...post('reply-2', 3), root_id: 'root', reply_count: 0 } - const { requests } = installRouteFetch([ - { - method: 'GET', - path: '/api/v4/posts/root/thread', - handle: ({ url }) => - url.searchParams.has('fromPost') - ? { ...page([root, firstReply, secondReply]), has_next: false } - : { ...page([root, firstReply]), has_next: true }, - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await getPostThread('root') - - expect(result.posts.map(({ id }) => id)).toEqual(['root', 'reply-1', 'reply-2']) - expect(result.truncated).toBe(false) - expect(requests).toHaveLength(2) - expect(requests[0]?.url.searchParams.get('perPage')).toBe('200') - expect(requests[0]?.url.searchParams.get('direction')).toBe('down') - expect(requests[1]?.url.searchParams.get('fromPost')).toBe('reply-1') - expect(requests[1]?.url.searchParams.get('fromCreateAt')).toBe('2') - }) - - test('recovers after an inclusive duplicate-only page advances the cursor', async () => { - const root = { ...post('root', 1), root_id: '', reply_count: 2 } - const firstReply = { ...post('reply-1', 2), root_id: 'root', reply_count: 0 } - const duplicateCursor = { ...root, id: 'cursor', create_at: 3 } - const secondReply = { ...post('reply-2', 4), root_id: 'root', reply_count: 0 } - let pageNumber = 0 - const { requests } = installRouteFetch([ - { - method: 'GET', - path: '/api/v4/posts/root/thread', - handle: () => { - pageNumber += 1 - if (pageNumber === 1) - return { ...page([root, firstReply, duplicateCursor]), has_next: true } - if (pageNumber === 2) return { ...page([duplicateCursor, firstReply]), has_next: true } - return { ...page([duplicateCursor, secondReply]), has_next: false } - }, - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await getPostThread('root') - - expect(result.posts.map(({ id }) => id)).toEqual(['root', 'reply-1', 'cursor', 'reply-2']) - expect(result.truncated).toBe(false) - expect(requests).toHaveLength(3) - }) - - test('terminates after bounded duplicate-only cursor stagnation', async () => { - const root = { ...post('root', 1), root_id: '', reply_count: 2 } - const { requests } = installRouteFetch([ - { - method: 'GET', - path: '/api/v4/posts/root/thread', - handle: () => ({ ...page([root]), has_next: true }), - }, - ]) - initClient('https://mattermost.test', 'token') - - expect((await getPostThread('root')).truncated).toBe(true) - expect(requests).toHaveLength(3) - }) - - test('search proves local truncation after accepted filtering and dedupe', async () => { - installRouteFetch([ - { - method: 'POST', - path: '/api/v4/teams/team/posts/search', - handle: () => ({ - ...page([post('new', 3), post('selected', 2), post('extra', 1)]), - matches: { new: ['n'], selected: ['s'], extra: ['e'] }, - }), - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await searchPosts('team', 'needle', 2) - - expect(result.truncated).toBe(true) - expect(result.order).toEqual(['new', 'selected']) - expect(Object.keys(result.matches)).toEqual(['new', 'selected']) - }) - - test('an empty search response proves exhaustion', async () => { - installRouteFetch([ - { - method: 'POST', - path: '/api/v4/teams/team/posts/search', - handle: () => ({ ...page([]), matches: {} }), - }, - ]) - initClient('https://mattermost.test', 'token') - - expect((await searchPosts('team', 'needle', 2)).truncated).toBe(false) - }) - - test('does not claim search exhaustion past an inaccessible post boundary', async () => { - installRouteFetch([ - { - method: 'POST', - path: '/api/v4/teams/team/posts/search', - handle: () => ({ - ...page([]), - matches: {}, - first_inaccessible_post_time: 1, - has_next: false, - }), - }, - ]) - initClient('https://mattermost.test', 'token') - - expect((await searchPosts('team', 'needle', 2)).truncated).toBeNull() - }) - - test('paginates without server since and enforces exact local time, dedupe, and hard limit', async () => { - let calls = 0 - const { requests } = installRouteFetch([ - { - method: 'GET', - path: '/api/v4/channels/channel/posts', - handle: () => { - calls += 1 - if (calls === 1) { - return page([ - post('p4', 130), - post('p3', 120), - post('p2', 110), - post('p1', 100), - post('old', 99), - ]) - } - return page([]) - }, - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await getAllChannelPosts('channel', { since: 100, limit: 4 }) - - expect(result.posts.map(({ id }) => id)).toEqual(['p4', 'p3', 'p2', 'p1']) - expect(result.truncated).toBe(false) - expect(requests).toHaveLength(2) - expect(requests.every(({ url }) => !url.searchParams.has('since'))).toBe(true) - expect(requests.map(({ url }) => url.searchParams.get('page'))).toEqual(['0', '1']) - expect(requests.every(({ url }) => url.searchParams.get('skipFetchThreads') === 'true')).toBe( - true, - ) - }) - - test('treats a short channel page as exhausted without an extra request', async () => { - const { requests } = installRouteFetch([ - { - method: 'GET', - path: '/api/v4/channels/channel/posts', - handle: () => page([post('p2', 2), post('p1', 1), post('p1', 1)]), - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await getAllChannelPosts('channel', { limit: 3 }) - expect(result.posts.map(({ id }) => id)).toEqual(['p2', 'p1']) - expect(requests).toHaveLength(1) - expect(result.truncated).toBe(false) - }) - - test('does not skip posts sharing the timestamp at a page boundary', async () => { - const { requests } = installRouteFetch([ - { - method: 'GET', - path: '/api/v4/channels/channel/posts', - handle: ({ url }) => { - const pageNumber = Number(url.searchParams.get('page')) - if (pageNumber === 0) return page([post('d', 100), post('c', 100), post('d', 100)]) - if (pageNumber === 1) return page([post('d', 100), post('c', 100), post('d', 100)]) - if (pageNumber === 2) return page([post('b', 100), post('a', 100), post('b', 100)]) - return page([post('old', 99)]) - }, - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await getAllChannelPosts('channel', { limit: 2 }) - - expect(result.posts.map(({ id }) => id)).toEqual(['a', 'b']) - expect(requests.map(({ url }) => url.searchParams.get('page'))).toEqual(['0', '1', '2', '3']) - expect(requests.every(({ url }) => !url.searchParams.has('before'))).toBe(true) - }) - - test('paginates search with a global limit and ID dedupe', async () => { - const { requests } = installRouteFetch([ - { - method: 'POST', - path: '/api/v4/teams/team/posts/search', - handle: ({ body }) => { - const pageNumber = (body as { page: number }).page - const items = - pageNumber === 0 - ? [post('a', 3), post('b', 2), post('a', 3)] - : [post('b', 2), post('c', 1)] - return { ...page(items), matches: {} } as SearchResponse - }, - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await searchPosts('team', 'needle', 3) - - expect(result.order).toEqual(['a', 'b', 'c']) - expect(result.truncated).toBeNull() - expect(requests.map(({ body }) => (body as { page: number }).page)).toEqual([0, 1, 2, 3]) - }) - - test('search tolerates a duplicate-only full page before later progress', async () => { - const { requests } = installRouteFetch([ - { - method: 'POST', - path: '/api/v4/teams/team/posts/search', - handle: ({ body }) => { - const pageNumber = (body as { page: number }).page - const items = - pageNumber === 0 - ? [post('d', 4), post('c', 3), post('d', 4), post('c', 3)] - : pageNumber === 1 - ? [post('d', 4), post('c', 3), post('d', 4), post('c', 3)] - : [post('b', 2), post('a', 1), post('b', 2), post('a', 1)] - return { ...page(items), matches: {} } - }, - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await searchPosts('team', 'needle', 4) - - expect(result.order).toEqual(['d', 'c', 'b', 'a']) - expect(result.truncated).toBeNull() - expect(requests.map(({ body }) => (body as { page: number }).page)).toEqual([0, 1, 2, 3, 4]) - }) - - test('mention-mode search completes equal-time cutoff ties', async () => { - const { requests } = installRouteFetch([ - { - method: 'POST', - path: '/api/v4/teams/team/posts/search', - handle: ({ body }) => { - const pageNumber = (body as { page: number }).page - const items = - pageNumber === 0 - ? [post('d', 100), post('c', 100)] - : pageNumber === 1 - ? [post('b', 100), post('a', 100)] - : [] - return { ...page(items), matches: {} } - }, - }, - ]) - initClient('https://mattermost.test', 'token') - - const response = await searchPosts('team', '@arda', 2, () => true) - - const selected = response.order - .map((id) => response.posts[id]) - .filter((candidate): candidate is Post => !!candidate) - expect(takeMostRecentPosts(selected, 2).map(({ id }) => id)).toEqual(['a', 'b']) - expect(response.truncated).toBe(true) - expect(requests.map(({ body }) => (body as { page: number }).page)).toEqual([0, 1, 2]) - }) - - test('search ignores missing and deleted hits and continues after a short page', async () => { - const { requests } = installRouteFetch([ - { - method: 'POST', - path: '/api/v4/teams/team/posts/search', - handle: ({ body }) => { - const pageNumber = (body as { page: number }).page - if (pageNumber === 0) { - return { - order: ['missing', 'deleted'], - posts: { deleted: { ...post('deleted', 3), delete_at: 1 } }, - matches: {}, - } - } - const items = - pageNumber === 1 ? [post('live-a', 2)] : pageNumber === 2 ? [post('live-b', 1)] : [] - return { ...page(items), matches: {} } - }, - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await searchPosts('team', 'needle', 2) - - expect(result.order).toEqual(['live-a', 'live-b']) - expect(result.truncated).toBeNull() - expect(requests.map(({ body }) => (body as { page: number }).page)).toEqual([0, 1, 2, 3]) - }) - - test('search fails safely when an ordered page has a null post map', async () => { - const { requests } = installRouteFetch([ - { - method: 'POST', - path: '/api/v4/teams/team/posts/search', - handle: ({ body }) => { - const pageNumber = (body as { page: number }).page - return pageNumber === 0 - ? { order: ['stale-hit'], posts: null, matches: null, has_next: false } - : { order: [], posts: {}, matches: {}, has_next: false } - }, - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await searchPosts('team', 'needle', 1) - - expect(result).toMatchObject({ order: [], posts: {}, matches: {}, truncated: null }) - expect(requests).toHaveLength(1) - }) - - test('search rejects a malformed top-level page without reflecting it', async () => { - installRouteFetch([ - { - method: 'POST', - path: '/api/v4/teams/team/posts/search', - handle: () => null, - }, - ]) - initClient('https://mattermost.test', 'token') - - await expect(searchPosts('team', 'needle', 1)).rejects.toThrow( - 'Mattermost returned an invalid search response.', - ) - }) - - test('search bounds unique stale-hit pages and reports unknown completeness', async () => { - const { requests } = installRouteFetch([ - { - method: 'POST', - path: '/api/v4/teams/team/posts/search', - handle: ({ body }) => { - const pageNumber = (body as { page: number }).page - return { order: [`stale-${pageNumber}`], posts: null, matches: null } - }, - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await searchPosts('team', 'needle', 1) - - expect(result).toMatchObject({ order: [], posts: {}, matches: {}, truncated: null }) - expect(requests).toHaveLength(100) - }) - - test('search continues to page two when exact local filtering rejects page one', async () => { - const { requests } = installRouteFetch([ - { - method: 'POST', - path: '/api/v4/teams/team/posts/search', - handle: ({ body }) => { - const pageNumber = (body as { page: number }).page - const item = - pageNumber === 0 - ? { ...post('too-old', 999), message: '@arda' } - : { ...post('exact-boundary', 1000), message: '@arda' } - return { ...page([item]), matches: {} } - }, - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await searchPosts( - 'team', - '@arda after:1970-01-01', - 1, - (candidate) => candidate.create_at >= 1000 && candidate.message === '@arda', - ) - - expect(result.order).toEqual(['exact-boundary']) - expect(result.truncated).toBeNull() - expect(requests.map(({ body }) => (body as { page: number }).page)).toEqual([0, 1, 2, 3]) - }) - - test('dedupes direct channels returned by Mattermost', async () => { - const dm = { id: 'dm', type: 'D', name: 'me__you' } as Channel - installRouteFetch([ - { - method: 'GET', - path: '/api/v4/users/me', - handle: () => ({ id: 'me', username: 'me' }), - }, - { method: 'GET', path: '/api/v4/users/me/channels', handle: () => [dm, dm] }, - ]) - initClient('https://mattermost.test', 'token') - - expect(await getMyDMChannels()).toEqual([dm]) - }) -}) diff --git a/tests/api/url.test.ts b/tests/api/url.test.ts deleted file mode 100644 index f04d552..0000000 --- a/tests/api/url.test.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { describe, expect, test } from 'vitest' -import { assertSecureServerUrl, buildPostPermalink, normalizeServerUrl } from '../../src/api/url' - -describe('assertSecureServerUrl', () => { - test.each([ - 'https://mattermost.example.com', - 'http://localhost:8065', - 'http://127.0.0.1:8065', - 'http://127.0.0.2:8065', - 'http://[::1]:8065', - ])('allows secure or loopback URL %s', (url) => { - expect(() => assertSecureServerUrl(url)).not.toThrow() - }) - - test.each([ - 'http://mattermost.example.com', - 'http://localhost.evil:8065', - 'http://127.evil:8065', - ])('rejects remote plaintext HTTP URL %s', (url) => { - expect(() => assertSecureServerUrl(url)).toThrow( - 'Refusing to send a Mattermost token over plaintext HTTP', - ) - }) - - test('rejects unsupported URL protocols', () => { - expect(() => assertSecureServerUrl('ftp://mattermost.example.com')).toThrow( - 'Mattermost URL must use HTTPS', - ) - }) - - test.each([ - 'https://user:password@mattermost.example.com', - 'https://mattermost.example.com?token=secret', - 'https://mattermost.example.com#fragment', - ])('rejects ambiguous or credential-bearing URL %s', (url) => { - expect(() => assertSecureServerUrl(url)).toThrow('Invalid Mattermost URL') - }) - - test('rejects malformed URLs without echoing them', () => { - const malformed = 'not a url with secret-token-value' - - expect(() => assertSecureServerUrl(malformed)).toThrow('Invalid Mattermost URL') - expect(() => assertSecureServerUrl(malformed)).not.toThrow(malformed) - }) -}) - -describe('normalizeServerUrl', () => { - test('canonicalizes casing, surrounding whitespace, and trailing slashes', () => { - expect(normalizeServerUrl(' HTTPS://Mattermost.Example.Com/// ')).toBe( - 'https://mattermost.example.com', - ) - }) - - test('preserves a Mattermost subpath', () => { - expect(normalizeServerUrl('https://mattermost.example.com/chat/')).toBe( - 'https://mattermost.example.com/chat', - ) - }) -}) - -describe('buildPostPermalink', () => { - test('preserves a server subpath and normalizes trailing slashes', () => { - expect(buildPostPermalink('https://mattermost.example.com/chat///', 'post-id')).toBe( - 'https://mattermost.example.com/chat/_redirect/pl/post-id', - ) - }) - - test('encodes a special post id without double-encoding the server subpath', () => { - expect(buildPostPermalink('https://mattermost.example.com/chat)', 'post(special)')).toBe( - 'https://mattermost.example.com/chat)/_redirect/pl/post%28special%29', - ) - }) -}) diff --git a/tests/api/websocket.test.ts b/tests/api/websocket.test.ts deleted file mode 100644 index 4687203..0000000 --- a/tests/api/websocket.test.ts +++ /dev/null @@ -1,492 +0,0 @@ -import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest' -import { connectWebSocket, getSocketErrorMessage } from '../../src/api/websocket' -import { preprocess } from '../../src/preprocessing' -import type { Post } from '../../src/types' - -class FakeWebSocket { - static instances: FakeWebSocket[] = [] - readonly url: string - readyState = 0 - sent: string[] = [] - closeCalls: Array<[number | undefined, string | undefined]> = [] - onopen: (() => void) | null = null - onmessage: ((event: MessageEvent) => void) | null = null - onerror: (() => void) | null = null - onclose: (() => void) | null = null - - constructor(url: string | URL) { - this.url = String(url) - FakeWebSocket.instances.push(this) - } - - open(): void { - this.readyState = 1 - this.onopen?.() - } - - raw(data: unknown): void { - this.onmessage?.({ data } as MessageEvent) - } - - message(payload: unknown): void { - this.raw(JSON.stringify(payload)) - } - - send(data: string): void { - this.sent.push(data) - } - - close(code?: number, reason?: string): void { - this.closeCalls.push([code, reason]) - this.readyState = 3 - this.onclose?.() - } - - drop(): void { - this.readyState = 3 - this.onclose?.() - } - - error(): void { - this.onerror?.() - } -} - -const post: Post = { - id: 'post-1', - create_at: 1, - update_at: 1, - delete_at: 0, - edit_at: 0, - user_id: 'user-1', - channel_id: 'channel-1', - message: 'hello', - type: '', - props: {}, - hashtags: '', - root_id: '', - reply_count: 0, - file_ids: [], - pending_post_id: '', -} - -function authenticate(socket: FakeWebSocket, connectionId = 'connection-1', sequence = 0): void { - socket.open() - const auth = JSON.parse(socket.sent[0] as string) as { seq: number } - socket.message({ event: 'hello', seq: sequence, data: { connection_id: connectionId } }) - socket.message({ status: 'OK', seq_reply: auth.seq }) -} - -function posted(socket: FakeWebSocket, sequence: number, value: Post = post): void { - socket.message({ - event: 'posted', - seq: sequence, - data: { post: JSON.stringify(value), channel_name: 'town', sender_name: 'arda' }, - broadcast: { channel_id: 'misleading-broadcast-id' }, - }) -} - -beforeEach(() => { - vi.useFakeTimers() - FakeWebSocket.instances = [] -}) - -afterEach(() => vi.useRealTimers()) - -describe('getSocketErrorMessage', () => { - test('reads and sanitizes Mattermost WebSocket errors', () => { - expect(getSocketErrorMessage({ message: 'Not authorized' })).toBe('Not authorized') - expect(getSocketErrorMessage({ message: 'failed\u001b[2Jspoofed' })).toBe( - 'failed\\u001b[2Jspoofed', - ) - expect(getSocketErrorMessage({ id: 'unknown' })).toBe('WebSocket request failed.') - }) -}) - -describe('connectWebSocket', () => { - test('releases credentials when URL validation fails synchronously', () => { - expect(() => connectWebSocket('not a URL', 'invalid-url-token', vi.fn(), vi.fn())).toThrow( - 'Invalid Mattermost URL', - ) - expect(preprocess('invalid-url-token', { redact: false }).text).toBe('invalid-url-token') - }) - - test('releases credentials when the initial socket constructor throws', () => { - class ThrowingWebSocket { - constructor() { - throw new Error('socket construction failed') - } - } - expect(() => - connectWebSocket('https://mm.example.com', 'constructor-token', vi.fn(), vi.fn(), { - WebSocket: ThrowingWebSocket as unknown as typeof WebSocket, - }), - ).toThrow('socket construction failed') - expect(preprocess('constructor-token', { redact: false }).text).toBe('constructor-token') - }) - - test('keeps concurrent socket credentials live and releases each on close', () => { - const first = connectWebSocket('https://mm.example.com', 'socket-one', vi.fn(), vi.fn(), { - WebSocket: FakeWebSocket as unknown as typeof WebSocket, - }) - const second = connectWebSocket('https://mm.example.com', 'socket-two', vi.fn(), vi.fn(), { - WebSocket: FakeWebSocket as unknown as typeof WebSocket, - }) - expect(preprocess('socket-one socket-two', { redact: false }).text).toBe( - '[REDACTED:mattermost_credential] [REDACTED:mattermost_credential]', - ) - first.close() - expect(preprocess('socket-one socket-two', { redact: false }).text).toBe( - 'socket-one [REDACTED:mattermost_credential]', - ) - second.close() - expect(preprocess('socket-one socket-two', { redact: false }).text).toBe( - 'socket-one socket-two', - ) - }) - - test('resumes from the next expected server sequence', () => { - const connection = connectWebSocket('https://mm.example.com/base', 'secret', vi.fn(), vi.fn(), { - WebSocket: FakeWebSocket as unknown as typeof WebSocket, - random: () => 0, - }) - const first = FakeWebSocket.instances[0] as FakeWebSocket - authenticate(first) - first.message({ event: 'typing', seq: 1, data: {} }) - first.drop() - vi.advanceTimersByTime(800) - - const resumed = FakeWebSocket.instances[1] as FakeWebSocket - expect(resumed.url).toContain('connection_id=connection-1') - expect(resumed.url).toContain('sequence_number=2') - connection.close() - }) - - test('rejects a sequence mismatch without emitting the event', () => { - const onPost = vi.fn() - const gap = vi.fn() - const connection = connectWebSocket('https://mm.example.com', 'secret', onPost, vi.fn(), { - WebSocket: FakeWebSocket as unknown as typeof WebSocket, - diagnostics: { gap }, - }) - const socket = FakeWebSocket.instances[0] as FakeWebSocket - authenticate(socket) - posted(socket, 2) - - expect(onPost).not.toHaveBeenCalled() - expect(gap).toHaveBeenCalledWith({ expected: 1, received: 2, reason: 'sequence_mismatch' }) - expect(socket.closeCalls[0]).toEqual([4000, 'sequence mismatch']) - connection.close() - }) - - test('diagnoses a changed connection id and resets the sequence', () => { - const gap = vi.fn() - const onPost = vi.fn() - const connection = connectWebSocket('https://mm.example.com', 'secret', onPost, vi.fn(), { - WebSocket: FakeWebSocket as unknown as typeof WebSocket, - random: () => 0, - diagnostics: { gap }, - }) - const first = FakeWebSocket.instances[0] as FakeWebSocket - authenticate(first) - first.drop() - vi.advanceTimersByTime(800) - const second = FakeWebSocket.instances[1] as FakeWebSocket - authenticate(second, 'connection-2', 0) - posted(second, 1) - - expect(gap).toHaveBeenCalledWith({ - expected: 1, - received: 0, - reason: 'connection_changed', - }) - expect(onPost).toHaveBeenCalledTimes(1) - connection.close() - }) - - test('advances sequence for unrelated events and filters by post channel id', () => { - const onPost = vi.fn() - const connection = connectWebSocket('https://mm.example.com', 'secret', onPost, vi.fn(), { - channelId: 'channel-1', - WebSocket: FakeWebSocket as unknown as typeof WebSocket, - }) - const socket = FakeWebSocket.instances[0] as FakeWebSocket - authenticate(socket) - socket.message({ event: 'typing', seq: 1, data: {} }) - posted(socket, 2, { ...post, channel_id: 'other-channel' }) - posted(socket, 3) - - expect(onPost).toHaveBeenCalledTimes(1) - expect(onPost).toHaveBeenCalledWith(post, 'town', 'arda') - connection.close() - }) - - test('uses bounded exponential backoff with plus/minus twenty percent jitter', () => { - const reconnect = vi.fn() - const connection = connectWebSocket('https://mm.example.com', 'secret', vi.fn(), vi.fn(), { - WebSocket: FakeWebSocket as unknown as typeof WebSocket, - random: () => 0, - backoffBaseMs: 100, - backoffMaxMs: 300, - diagnostics: { reconnect }, - }) - ;(FakeWebSocket.instances[0] as FakeWebSocket).drop() - vi.advanceTimersByTime(80) - ;(FakeWebSocket.instances[1] as FakeWebSocket).drop() - vi.advanceTimersByTime(160) - ;(FakeWebSocket.instances[2] as FakeWebSocket).drop() - vi.advanceTimersByTime(240) - ;(FakeWebSocket.instances[3] as FakeWebSocket).drop() - expect(reconnect.mock.calls.map((call) => call[1])).toEqual([80, 160, 240, 240]) - connection.close() - - const upper = vi.fn() - const second = connectWebSocket('https://mm.example.com', 'secret', vi.fn(), vi.fn(), { - WebSocket: FakeWebSocket as unknown as typeof WebSocket, - random: () => 1, - diagnostics: { reconnect: upper }, - }) - ;(FakeWebSocket.instances.at(-1) as FakeWebSocket).drop() - expect(upper).toHaveBeenCalledWith(1, 1200) - second.close() - }) - - test('turns a synchronous reconnect constructor failure into bounded fatal cleanup', async () => { - class ReconnectThrowingWebSocket extends FakeWebSocket { - constructor(url: string | URL) { - super(url) - if (FakeWebSocket.instances.length > 1) throw new Error('reconnect construction failed') - } - } - const credential = 'reconnect-constructor-token' - const onError = vi.fn() - const connection = connectWebSocket('https://mm.example.com', credential, vi.fn(), onError, { - WebSocket: ReconnectThrowingWebSocket as unknown as typeof WebSocket, - random: () => 0, - }) - ;(FakeWebSocket.instances[0] as FakeWebSocket).drop() - expect(() => vi.advanceTimersByTime(800)).not.toThrow() - await connection.done - expect(onError).toHaveBeenCalledTimes(1) - expect(onError).toHaveBeenCalledWith(new Error('WebSocket connection failed.')) - expect(preprocess(credential, { redact: false }).text).toBe(credential) - expect(vi.getTimerCount()).toBe(0) - }) - - test('error plus close schedules one retry and stale socket callbacks are ignored', () => { - const onPost = vi.fn() - const reconnect = vi.fn() - const connection = connectWebSocket('https://mm.example.com', 'secret', onPost, vi.fn(), { - WebSocket: FakeWebSocket as unknown as typeof WebSocket, - random: () => 0, - diagnostics: { reconnect }, - }) - const first = FakeWebSocket.instances[0] as FakeWebSocket - authenticate(first) - first.error() - first.drop() - expect(reconnect).toHaveBeenCalledTimes(1) - vi.advanceTimersByTime(800) - authenticate(FakeWebSocket.instances[1] as FakeWebSocket, 'connection-2') - posted(first, 1) - expect(onPost).not.toHaveBeenCalled() - connection.close() - }) - - test('re-arms heartbeat after its matching response and cancels timers on close', async () => { - const connection = connectWebSocket('https://mm.example.com', 'secret', vi.fn(), vi.fn(), { - WebSocket: FakeWebSocket as unknown as typeof WebSocket, - heartbeatIntervalMs: 20, - heartbeatTimeoutMs: 10, - }) - const socket = FakeWebSocket.instances[0] as FakeWebSocket - authenticate(socket) - vi.advanceTimersByTime(20) - const ping = JSON.parse(socket.sent[1] as string) as { seq: number } - socket.message({ status: 'OK', seq_reply: ping.seq }) - vi.advanceTimersByTime(20) - expect(socket.sent).toHaveLength(3) - - connection.close() - await connection.done - expect(vi.getTimerCount()).toBe(0) - }) - - test('reports malformed payloads without writing directly to console', () => { - const malformed = vi.fn() - const error = vi.spyOn(console, 'error').mockImplementation(() => undefined) - const connection = connectWebSocket('https://mm.example.com', 'secret', vi.fn(), vi.fn(), { - WebSocket: FakeWebSocket as unknown as typeof WebSocket, - diagnostics: { malformed }, - }) - const socket = FakeWebSocket.instances[0] as FakeWebSocket - socket.raw('{nope') - authenticate(socket) - socket.message({ event: 'posted', seq: 1, data: { post: '{bad' } }) - expect(malformed).toHaveBeenCalledTimes(2) - expect(error).not.toHaveBeenCalled() - connection.close() - }) - - test('rejects malformed post fields before watch formatting can throw', () => { - const malformed = vi.fn() - const onPost = vi.fn() - const connection = connectWebSocket('https://mm.example.com', 'secret', onPost, vi.fn(), { - WebSocket: FakeWebSocket as unknown as typeof WebSocket, - diagnostics: { malformed }, - }) - const socket = FakeWebSocket.instances[0] as FakeWebSocket - authenticate(socket) - const invalid = [ - { ...post, message: null }, - { ...post, user_id: 42 }, - { ...post, create_at: null }, - { ...post, create_at: 1e100 }, - { ...post, file_ids: [7] }, - ] - invalid.forEach((value, index) => { - socket.message({ - event: 'posted', - seq: index + 1, - data: { post: JSON.stringify(value), channel_name: 'town', sender_name: 'arda' }, - }) - }) - - expect(onPost).not.toHaveBeenCalled() - expect(malformed).toHaveBeenCalledTimes(invalid.length) - connection.close() - }) - - test('normalizes malformed optional post scalars before watch emission', () => { - const onPost = vi.fn() - const connection = connectWebSocket('https://mm.example.com', 'secret', onPost, vi.fn(), { - WebSocket: FakeWebSocket as unknown as typeof WebSocket, - }) - const socket = FakeWebSocket.instances[0] as FakeWebSocket - authenticate(socket) - posted(socket, 1, { - ...post, - reply_count: '7' as unknown as number, - is_pinned: 'true' as unknown as boolean, - }) - expect(onPost).toHaveBeenCalledWith(expect.objectContaining({ reply_count: 0 }), 'town', 'arda') - expect((onPost.mock.calls[0]?.[0] as Post).is_pinned).toBeUndefined() - connection.close() - }) - - test('uses fixed local close text for hostile FAIL reasons', () => { - const credential = 'mattermost-secret-token' - const connection = connectWebSocket('https://mm.example.com', credential, vi.fn(), vi.fn(), { - WebSocket: FakeWebSocket as unknown as typeof WebSocket, - }) - const socket = FakeWebSocket.instances[0] as FakeWebSocket - socket.open() - socket.message({ - status: 'FAIL', - seq_reply: 999, - error: { message: `${credential}\u202e${'x'.repeat(1000)}` }, - }) - expect(socket.closeCalls[0]).toEqual([4000, 'request failed']) - expect(JSON.stringify(socket.closeCalls)).not.toContain(credential) - connection.close() - }) - - test('does not mistake author-related FAIL text for authentication failure', () => { - const onError = vi.fn() - const connection = connectWebSocket('https://mm.example.com', 'secret', vi.fn(), onError, { - WebSocket: FakeWebSocket as unknown as typeof WebSocket, - }) - const socket = FakeWebSocket.instances[0] as FakeWebSocket - socket.open() - socket.message({ status: 'FAIL', seq_reply: 999, error: { message: 'Author was not found' } }) - expect(onError).not.toHaveBeenCalled() - expect(socket.closeCalls[0]).toEqual([4000, 'request failed']) - connection.close() - }) - - test('normalizes optional WebSocket timestamps outside the ECMAScript Date range', () => { - const onPost = vi.fn() - const connection = connectWebSocket('https://mm.example.com', 'secret', onPost, vi.fn(), { - WebSocket: FakeWebSocket as unknown as typeof WebSocket, - }) - const socket = FakeWebSocket.instances[0] as FakeWebSocket - authenticate(socket) - posted(socket, 1, { - ...post, - update_at: 8.64e15 + 1, - edit_at: -8.64e15 - 1, - delete_at: Number.POSITIVE_INFINITY, - }) - expect(onPost).toHaveBeenCalledWith( - expect.objectContaining({ update_at: 1, edit_at: 0, delete_at: 0 }), - 'town', - 'arda', - ) - connection.close() - }) - - test('accepts the maximum finite ECMAScript timestamp without toISOString failure', () => { - const onPost = vi.fn() - const connection = connectWebSocket('https://mm.example.com', 'secret', onPost, vi.fn(), { - WebSocket: FakeWebSocket as unknown as typeof WebSocket, - }) - const socket = FakeWebSocket.instances[0] as FakeWebSocket - authenticate(socket) - posted(socket, 1, { ...post, create_at: 8.64e15 }) - - expect(onPost).toHaveBeenCalledWith( - expect.objectContaining({ create_at: 8.64e15 }), - 'town', - 'arda', - ) - connection.close() - }) - - test('uses a fifteen second handshake timeout by default', () => { - const connection = connectWebSocket('https://mm.example.com', 'secret', vi.fn(), vi.fn(), { - WebSocket: FakeWebSocket as unknown as typeof WebSocket, - }) - const socket = FakeWebSocket.instances[0] as FakeWebSocket - vi.advanceTimersByTime(14_999) - expect(socket.closeCalls).toHaveLength(0) - vi.advanceTimersByTime(1) - expect(socket.closeCalls[0]).toEqual([4000, 'handshake timeout']) - connection.close() - }) - - test('treats authentication failure as fatal and never reconnects', async () => { - const onError = vi.fn() - const connection = connectWebSocket('https://mm.example.com', 'secret', vi.fn(), onError, { - WebSocket: FakeWebSocket as unknown as typeof WebSocket, - }) - const socket = FakeWebSocket.instances[0] as FakeWebSocket - socket.open() - const auth = JSON.parse(socket.sent[0] as string) as { seq: number } - socket.message({ status: 'FAIL', seq_reply: auth.seq, error: { message: 'Not authorized' } }) - await connection.done - vi.runAllTimers() - expect(onError).toHaveBeenCalledWith(new Error('Authentication failed. Check your token.')) - expect(FakeWebSocket.instances).toHaveLength(1) - expect(preprocess('secret', { redact: false }).text).toBe('secret') - }) - - test('cleans up before a throwing fatal-error callback and still settles done', async () => { - const credential = 'throwing-callback-token' - const onError = vi.fn(() => { - expect(preprocess(credential, { redact: false }).text).toBe(credential) - throw new Error('callback failure') - }) - const connection = connectWebSocket('https://mm.example.com', credential, vi.fn(), onError, { - WebSocket: FakeWebSocket as unknown as typeof WebSocket, - }) - const socket = FakeWebSocket.instances[0] as FakeWebSocket - socket.open() - const auth = JSON.parse(socket.sent[0] as string) as { seq: number } - expect(() => - socket.message({ status: 'FAIL', seq_reply: auth.seq, error: { message: 'Unauthorized' } }), - ).not.toThrow() - await connection.done - expect(onError).toHaveBeenCalledTimes(1) - expect(vi.getTimerCount()).toBe(0) - }) -}) diff --git a/tests/channels-type-validation.test.ts b/tests/channels-type-validation.test.ts deleted file mode 100644 index 543624f..0000000 --- a/tests/channels-type-validation.test.ts +++ /dev/null @@ -1,365 +0,0 @@ -import { spawnSync } from 'node:child_process' -import { afterEach, describe, expect, test, vi } from 'vitest' -import { clearUserCache } from '../src/api/users' -import { listChannels } from '../src/cli' -import type { Channel, ChannelTypeFilter } from '../src/types' -import { installRouteFetch } from './helpers/fake-fetch' - -const channelIdentities: Pick[] = [ - { id: 'public', type: 'O', name: 'public', display_name: 'Public' }, - { id: 'private', type: 'P', name: 'private', display_name: 'Private' }, - { id: 'dm', type: 'D', name: 'me__other', display_name: '' }, - { id: 'group', type: 'G', name: 'group', display_name: 'Group' }, -] -const channels: Channel[] = channelIdentities.map( - (channel) => - ({ - ...channel, - team_id: channel.type === 'O' || channel.type === 'P' ? 'team' : '', - header: '', - purpose: '', - last_post_at: 0, - total_msg_count: 0, - creator_id: 'me', - }) satisfies Channel, -) - -afterEach(() => { - clearUserCache() - vi.restoreAllMocks() - vi.unstubAllGlobals() -}) - -test('rejects an invalid CLI --type value after credential protection and before network', () => { - const result = spawnSync('bun', ['src/index.ts', 'channels', '--type', 'bogus'], { - cwd: process.cwd(), - encoding: 'utf8', - env: { ...process.env, MM_URL: 'https://mattermost.test', MM_TOKEN: 'token' }, - stdio: 'pipe', - }) - - expect(result.status).not.toBe(0) - expect(result.stderr).toContain('Invalid channel type "bogus"') -}) - -test.each([ - ['CLI', ['--token', 'cli-active-token'], { MM_TOKEN: undefined }, 'cli-active-token'], - ['env', [], { MM_TOKEN: 'env-active-token' }, 'env-active-token'], -])('protects the %s token in invalid pre-network --type output with --no-redact', (_, args, env, token) => { - const result = spawnSync( - 'bun', - ['src/index.ts', '--no-redact', ...args, 'channels', '--type', token], - { - cwd: process.cwd(), - encoding: 'utf8', - env: { ...process.env, MM_URL: 'https://mattermost.test', ...env }, - stdio: 'pipe', - }, - ) - expect(result.status).not.toBe(0) - expect(result.stderr).toContain('[REDACTED:mattermost_credential]') - expect(result.stderr).not.toContain(token) -}) - -test.each([ - ['users limit', ['--token', 'bad-limit', 'users', '--limit', 'bad-limit'], {}, 'bad-limit'], - ['unread peek', ['unread', '--peek', 'bad-peek'], { MM_TOKEN: 'bad-peek' }, 'bad-peek'], - ['DM since', ['--token', 'bad-since', 'dms', '--since', 'bad-since'], {}, 'bad-since'], -])('does not reflect invalid %s values that equal active credentials', (_, args, env, token) => { - const result = spawnSync('bun', ['src/index.ts', '--no-redact', ...args], { - cwd: process.cwd(), - encoding: 'utf8', - env: { ...process.env, MM_URL: 'https://mattermost.test', MM_TOKEN: undefined, ...env }, - stdio: 'pipe', - }) - expect(result.status).not.toBe(0) - expect(result.stderr).not.toContain(token) - expect(result.stderr).toMatch(/must be a positive number|must use a duration/) -}) - -test('rejects an invalid direct call before fetching', async () => { - const fetch = vi.fn() - vi.stubGlobal('fetch', fetch) - - await expect( - listChannels({ - url: 'https://mattermost.test', - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - typeFilter: 'bogus' as ChannelTypeFilter, - }), - ).rejects.toThrow('Invalid channel type "bogus"') - expect(fetch).not.toHaveBeenCalled() -}) - -describe.each([ - ['all', ['public', 'private', 'dm', 'group']], - ['dm', ['dm']], - ['public', ['public']], - ['private', ['private']], - ['group', ['group']], -] satisfies [ChannelTypeFilter, string[]][])('channels --type %s', (typeFilter, expectedIds) => { - test('returns channels of the requested type', async () => { - installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => ({ id: 'me', username: 'me' }) }, - { method: 'GET', path: '/api/v4/users/me/channels', handle: () => channels }, - { - method: 'GET', - path: '/api/v4/users/me/teams', - handle: () => [{ id: 'team', name: 'core', display_name: 'Core', type: 'O' }], - }, - { - method: 'POST', - path: '/api/v4/users/ids', - handle: () => [{ id: 'other', username: 'other' }], - }, - ]) - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) - - await listChannels({ - url: 'https://mattermost.test', - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - typeFilter, - }) - - const output = JSON.parse(String(log.mock.calls[0]?.[0])) as { id: string }[] - expect(output.map(({ id }) => id).sort()).toEqual(expectedIds.sort()) - }) -}) - -test('dedupes channel IDs and emits narrow team identity only for O/P channels', async () => { - const { requests } = installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => ({ id: 'me', username: 'me' }) }, - { - method: 'GET', - path: '/api/v4/users/me/channels', - handle: () => [channels[0], channels[0], channels[3]], - }, - { - method: 'GET', - path: '/api/v4/users/me/teams', - handle: () => [ - { id: 'team', name: 'co\nre', display_name: 'AKIA1234567890ABCDEF', type: 'O' }, - ], - }, - ]) - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) - - await listChannels({ - url: 'https://mattermost.test', - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - typeFilter: 'all', - }) - - const output = JSON.parse(String(log.mock.calls[0]?.[0])) - expect(output).toHaveLength(2) - expect(output.find(({ id }: { id: string }) => id === 'public')?.team).toEqual({ - id: 'team', - name: 'co\\nre', - displayName: 'AK...EF', - }) - expect(output.find(({ id }: { id: string }) => id === 'group')?.team).toBeNull() - expect(requests.filter(({ url }) => url.pathname.endsWith('/teams'))).toHaveLength(1) -}) - -test('fails closed when a discovered direct channel excludes the current user', async () => { - installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => ({ id: 'me', username: 'me' }) }, - { - method: 'GET', - path: '/api/v4/users/me/channels', - handle: () => [ - { - ...channels[2], - name: 'alice__bob', - }, - ], - }, - ]) - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) - - await expect( - listChannels({ - url: 'https://mattermost.test', - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - typeFilter: 'all', - }), - ).rejects.toThrow('Mattermost returned an invalid channel response.') - expect(log).not.toHaveBeenCalled() -}) - -test('does not fetch teams for D/G-only discovery', async () => { - const { requests } = installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => ({ id: 'me', username: 'me' }) }, - { - method: 'GET', - path: '/api/v4/users/me/channels', - handle: () => [channels[3]], - }, - ]) - vi.spyOn(console, 'log').mockImplementation(() => undefined) - await listChannels({ - url: 'https://mattermost.test', - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - typeFilter: 'group', - }) - expect(requests.some(({ url }) => url.pathname.endsWith('/teams'))).toBe(false) -}) - -test.each(['', 'other'])('fails closed for O/P with invalid team membership %j', async (teamId) => { - installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => ({ id: 'me', username: 'me' }) }, - { - method: 'GET', - path: '/api/v4/users/me/channels', - handle: () => [{ ...channels[0], team_id: teamId }], - }, - { - method: 'GET', - path: '/api/v4/users/me/teams', - handle: () => [{ id: 'team', name: 'core', display_name: 'Core', type: 'O' }], - }, - ]) - await expect( - listChannels({ - url: 'https://mattermost.test', - token: 'token', - json: true, - color: false, - relative: false, - redact: false, - typeFilter: 'all', - }), - ).rejects.toThrow('Invalid channels response.') -}) - -test('human channel listing labels group DMs without a hash', async () => { - installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => ({ id: 'me', username: 'me' }) }, - { - method: 'GET', - path: '/api/v4/users/me/channels', - handle: () => channels.filter((channel) => channel.type === 'G'), - }, - ]) - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) - - await listChannels({ - url: 'https://mattermost.test', - token: 'token', - json: false, - color: false, - relative: false, - redact: true, - typeFilter: 'group', - }) - - const output = log.mock.calls.map(([value]) => String(value)).join('\n') - expect(output).toContain('Group') - expect(output).toContain('[group]') - expect(output).not.toContain('#Group') -}) - -test('human O/P labels include team slug and channel ID', async () => { - installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => ({ id: 'me', username: 'me' }) }, - { method: 'GET', path: '/api/v4/users/me/channels', handle: () => [channels[0]] }, - { - method: 'GET', - path: '/api/v4/users/me/teams', - handle: () => [{ id: 'team', name: 'core', display_name: 'Core', type: 'O' }], - }, - ]) - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) - - await listChannels({ - url: 'https://mattermost.test', - token: 'token', - json: false, - color: false, - relative: false, - redact: true, - typeFilter: 'public', - }) - - expect(log.mock.calls.map(([value]) => String(value)).join('\n')).toContain('core/#public') - expect(log.mock.calls.map(([value]) => String(value)).join('\n')).toContain('[public]') -}) - -test('normalizes malformed channel count and timestamp scalars in JSON output', async () => { - installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => ({ id: 'me', username: 'me' }) }, - { - method: 'GET', - path: '/api/v4/users/me/channels', - handle: () => [ - { - ...channels[0], - total_msg_count: '9000', - last_post_at: 'tomorrow', - }, - ], - }, - { - method: 'GET', - path: '/api/v4/users/me/teams', - handle: () => [{ id: 'team', name: 'core', display_name: 'Core', type: 'O' }], - }, - ]) - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) - await listChannels({ - url: 'https://mattermost.test', - token: 'token', - json: true, - color: false, - relative: false, - redact: false, - typeFilter: 'all', - }) - expect(JSON.parse(String(log.mock.calls[0]?.[0]))[0]).toMatchObject({ - messageCount: 0, - lastPost: null, - }) -}) - -test('fails generically for a hostile unknown remote channel type', async () => { - const hostile = 'X\u202eactive-token' - installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => ({ id: 'me', username: 'me' }) }, - { - method: 'GET', - path: '/api/v4/users/me/channels', - handle: () => [{ ...channels[0], type: hostile }], - }, - ]) - await expect( - listChannels({ - url: 'https://mattermost.test', - token: 'active-token', - json: true, - color: false, - relative: false, - redact: false, - typeFilter: 'all', - }), - ).rejects.toThrow('Mattermost returned an invalid channel response.') -}) diff --git a/tests/cli-empty-reads.test.ts b/tests/cli-empty-reads.test.ts deleted file mode 100644 index de4ece7..0000000 --- a/tests/cli-empty-reads.test.ts +++ /dev/null @@ -1,340 +0,0 @@ -import { afterEach, describe, expect, test, vi } from 'vitest' -import { clearUserCache } from '../src/api/users' -import { - fetchChannel, - fetchDMs, - fetchGroupDMs, - fetchMentions, - searchMessages, - showUnread, -} from '../src/cli' -import type { - Channel, - ChannelOptions, - CLIOptions, - DMsOptions, - GroupDMsOptions, - MentionOptions, - SearchOptions, - UnreadOptions, -} from '../src/types' -import { installRouteFetch } from './helpers/fake-fetch' - -const serverUrl = 'https://mattermost.test' -const me = { id: 'me', username: 'me' } -const team = { id: 'team', name: 'team', display_name: 'Team', type: 'O' } -const general = { - id: 'general', - team_id: 'team', - type: 'O', - name: 'general', - display_name: 'General', - header: '', - purpose: '', - last_post_at: 0, - total_msg_count: 0, - creator_id: 'me', -} satisfies Channel -const direct = { - ...general, - id: 'dm', - team_id: '', - type: 'D', - name: 'me__other', - display_name: '', -} satisfies Channel - -const baseOptions: CLIOptions = { - url: serverUrl, - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: true, -} - -function commonRoutes() { - return [ - { method: 'GET', path: '/api/v4/users/me', handle: () => me }, - { method: 'GET', path: '/api/v4/users/me/teams', handle: () => [team] }, - ] -} - -function emptyPage() { - return { order: [], posts: {}, has_next: false } -} - -function uncertainEmptyPage() { - return { ...emptyPage(), first_inaccessible_post_time: 1 } -} - -function captureOutput() { - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) - const exit = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never) - return { - output: () => log.mock.calls.map(([value]) => String(value)).join('\n'), - exit, - } -} - -type EmptyCommand = { - name: string - install: () => void - run: (json: boolean) => Promise -} - -const emptyCommands: EmptyCommand[] = [ - { - name: 'channel', - install: () => { - installRouteFetch([ - ...commonRoutes(), - { method: 'GET', path: '/api/v4/teams/team/channels/name/general', handle: () => general }, - { method: 'GET', path: '/api/v4/channels/general/posts', handle: emptyPage }, - ]) - }, - run: (json) => - fetchChannel({ - ...baseOptions, - json, - channel: 'general', - limit: 50, - since: '', - } satisfies ChannelOptions), - }, - { - name: 'search', - install: () => { - installRouteFetch([ - ...commonRoutes(), - { method: 'POST', path: '/api/v4/teams/team/posts/search', handle: emptyPage }, - ]) - }, - run: (json) => - searchMessages({ ...baseOptions, json, query: 'needle', limit: 50 } satisfies SearchOptions), - }, - { - name: 'mentions', - install: () => { - installRouteFetch([ - ...commonRoutes(), - { method: 'POST', path: '/api/v4/teams/team/posts/search', handle: emptyPage }, - ]) - }, - run: (json) => - fetchMentions({ - ...baseOptions, - json, - limit: 50, - mentionNames: [], - } satisfies MentionOptions), - }, -] - -afterEach(() => { - clearUserCache() - vi.restoreAllMocks() - vi.unstubAllGlobals() -}) - -test.each([ - [ - 'search', - () => searchMessages({ ...baseOptions, query: 'needle', limit: 50 } satisfies SearchOptions), - ], - [ - 'mentions', - () => fetchMentions({ ...baseOptions, limit: 50, mentionNames: [] } satisfies MentionOptions), - ], - ['unread', () => showUnread({ ...baseOptions } satisfies UnreadOptions)], -] as const)('%s rejects a malformed single team before any scoped request', async (_name, run) => { - const { requests } = installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => me }, - { - method: 'GET', - path: '/api/v4/users/me/teams', - handle: () => [{ name: 'team', display_name: 'Team', type: 'O' }], - }, - ]) - - await expect(run()).rejects.toThrow('Invalid teams response.') - const paths = requests.map(({ method, url }) => `${method} ${url.pathname}`) - expect(paths.at(-1)).toBe('GET /api/v4/users/me/teams') - expect(paths.some((path) => path.includes('/posts/search'))).toBe(false) - expect(paths.some((path) => path.includes('/channels/members'))).toBe(false) - expect(paths.some((path) => path.endsWith('/channels'))).toBe(false) -}) - -describe.each(emptyCommands)('$name empty reads', ({ install, run }) => { - test('emits exact JSON and does not exit', async () => { - install() - const capture = captureOutput() - - await run(true) - - expect(capture.output()).toBe('[]') - expect(capture.exit).not.toHaveBeenCalled() - }) - - test('emits neutral human output and does not exit', async () => { - install() - const capture = captureOutput() - - await run(false) - - expect(capture.output()).toBe('No messages found.') - expect(capture.exit).not.toHaveBeenCalled() - }) -}) - -describe('unknown empty reads', () => { - test.each([ - ['channel', () => fetchChannel({ ...baseOptions, channel: 'general', limit: 50, since: '' })], - ['search', () => searchMessages({ ...baseOptions, query: 'needle', limit: 50 })], - ['mentions', () => fetchMentions({ ...baseOptions, limit: 50, mentionNames: [] })], - ] as const)('%s rejects instead of claiming an empty success', async (name, run) => { - installRouteFetch([ - ...commonRoutes(), - ...(name === 'channel' - ? [ - { - method: 'GET', - path: '/api/v4/teams/team/channels/name/general', - handle: () => general, - }, - { method: 'GET', path: '/api/v4/channels/general/posts', handle: uncertainEmptyPage }, - ] - : [ - { - method: 'POST', - path: '/api/v4/teams/team/posts/search', - handle: uncertainEmptyPage, - }, - ]), - ]) - - await expect(run()).rejects.toThrow( - 'Message retrieval was incomplete, so an empty result cannot be confirmed.', - ) - }) - - test('merged DM history rejects when its only empty channel has unknown completeness', async () => { - installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => me }, - { method: 'GET', path: '/api/v4/users/me/channels', handle: () => [direct] }, - { method: 'GET', path: '/api/v4/channels/dm/posts', handle: uncertainEmptyPage }, - ]) - - await expect(fetchDMs({ ...baseOptions, user: [], limit: 50, since: '' })).rejects.toThrow( - 'Message retrieval was incomplete', - ) - }) - - test.each([ - ['direct', { ...direct }, fetchDMs], - ['group', { ...direct, id: 'group', type: 'G' as const }, fetchGroupDMs], - ] as const)('explicit %s history rejects an unknown empty result', async (_name, channel, run) => { - installRouteFetch([ - { method: 'GET', path: `/api/v4/channels/${channel.id}`, handle: () => channel }, - { method: 'GET', path: '/api/v4/users/me', handle: () => me }, - { - method: 'GET', - path: `/api/v4/channels/${channel.id}/posts`, - handle: uncertainEmptyPage, - }, - ]) - - await expect( - run({ - ...baseOptions, - channel: channel.id, - limit: 50, - since: '', - ...(run === fetchDMs ? { user: [] } : {}), - } as DMsOptions & GroupDMsOptions), - ).rejects.toThrow('Message retrieval was incomplete') - }) - - test('unread peek rejects when unread posts cannot be confirmed empty', async () => { - installRouteFetch([ - ...commonRoutes(), - { - method: 'GET', - path: '/api/v4/users/me/channels', - handle: () => [{ ...general, total_msg_count: 1 }], - }, - { - method: 'GET', - path: '/api/v4/users/me/teams/team/channels/members', - handle: () => [ - { channel_id: 'general', msg_count: 0, mention_count: 0, last_viewed_at: 1 }, - ], - }, - { method: 'GET', path: '/api/v4/channels/general/posts', handle: uncertainEmptyPage }, - ]) - - await expect(showUnread({ ...baseOptions, peek: 5 } satisfies UnreadOptions)).rejects.toThrow( - 'Message retrieval was incomplete', - ) - }) -}) - -describe('DM empty reads', () => { - test.each([ - ['no matched channels', []], - ['matched channel with no posts', [direct]], - ] as const)('%s emits exact JSON and does not exit', async (_case, channels) => { - installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => me }, - { method: 'GET', path: '/api/v4/users/me/channels', handle: () => channels }, - ...(channels.length > 0 - ? [{ method: 'GET', path: '/api/v4/channels/dm/posts', handle: emptyPage }] - : []), - ]) - const capture = captureOutput() - - await fetchDMs({ - ...baseOptions, - user: [], - limit: 50, - since: '', - } satisfies DMsOptions) - - expect(capture.output()).toBe('[]') - expect(capture.exit).not.toHaveBeenCalled() - }) - - test('emits neutral human output and does not exit', async () => { - installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => me }, - { method: 'GET', path: '/api/v4/users/me/channels', handle: () => [] }, - ]) - const capture = captureOutput() - - await fetchDMs({ - ...baseOptions, - json: false, - user: [], - limit: 50, - since: '', - } satisfies DMsOptions) - - expect(capture.output()).toBe('No messages found.') - expect(capture.exit).not.toHaveBeenCalled() - }) -}) - -test('empty unread JSON keeps its command-specific structured shape', async () => { - installRouteFetch([ - ...commonRoutes(), - { method: 'GET', path: '/api/v4/users/me/channels', handle: () => [] }, - { method: 'GET', path: '/api/v4/users/me/teams/team/channels/members', handle: () => [] }, - ]) - const capture = captureOutput() - - await showUnread({ ...baseOptions } satisfies CLIOptions) - - expect(JSON.parse(capture.output())).toEqual({ unread: [] }) - expect(capture.exit).not.toHaveBeenCalled() -}) diff --git a/tests/cli-failure-propagation.test.ts b/tests/cli-failure-propagation.test.ts deleted file mode 100644 index cda8036..0000000 --- a/tests/cli-failure-propagation.test.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { afterEach, describe, expect, test, vi } from 'vitest' -import { clearUserCache } from '../src/api/users' -import { fetchDMs, showUnread } from '../src/cli' -import type { Channel, DMsOptions, UnreadOptions } from '../src/types' - -const base = { - url: 'https://mattermost.test', - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: false, -} -const me = { id: 'me', username: 'me' } -const dm = { - id: 'dm', - team_id: '', - type: 'D', - name: 'carol__me', - display_name: '', - total_msg_count: 0, -} as Channel - -function response(body: unknown, status = 200): Response { - return Response.json(body, { status, statusText: status === 404 ? 'Not Found' : 'Server Error' }) -} - -afterEach(() => { - clearUserCache() - vi.restoreAllMocks() - vi.unstubAllGlobals() -}) - -describe('dms --user failures', () => { - test('warns separately for a missing user and an existing user without a DM in a mixed request', async () => { - vi.stubGlobal( - 'fetch', - vi.fn(async (input: string | URL | Request) => { - const path = new URL(typeof input === 'string' || input instanceof URL ? input : input.url) - .pathname - if (path.endsWith('/users/me')) return response(me) - if (path.endsWith('/users/me/channels')) return response([dm]) - if (path.endsWith('/users/username/alice')) { - return response({ message: 'REMOTE_SECRET_BODY' }, 404) - } - if (path.endsWith('/users/username/bob')) return response({ id: 'bob', username: 'bob' }) - if (path.endsWith('/users/username/carol')) { - return response({ id: 'carol', username: 'carol' }) - } - if (path.endsWith('/channels/dm/posts')) { - return response({ order: [], posts: {}, has_next: false }) - } - throw new Error(`unexpected route ${path}`) - }), - ) - const warn = vi.spyOn(console, 'error').mockImplementation(() => undefined) - vi.spyOn(console, 'log').mockImplementation(() => undefined) - - await fetchDMs({ ...base, user: ['alice', 'bob', 'carol'], limit: 50, since: '' }) - - const warnings = warn.mock.calls.flat().join('\n') - expect(warnings).toContain('Warning: User @alice was not found.') - expect(warnings).toContain('Warning: No direct-message channel exists with @bob.') - expect(warnings).not.toContain('REMOTE_SECRET_BODY') - }) - - test.each([ - 500, 401, - ])('propagates %i lookup errors without leaking the remote body', async (status) => { - vi.stubGlobal( - 'fetch', - vi.fn(async (input: string | URL | Request) => { - const path = new URL(typeof input === 'string' || input instanceof URL ? input : input.url) - .pathname - if (path.endsWith('/users/me')) return response(me) - if (path.endsWith('/users/me/channels')) return response([]) - return response({ message: 'REMOTE_SECRET_BODY' }, status) - }), - ) - - await expect( - fetchDMs({ ...base, user: ['alice'], limit: 50, since: '' } satisfies DMsOptions), - ).rejects.toThrow(`API request failed: ${status}`) - await expect( - fetchDMs({ ...base, user: ['alice'], limit: 50, since: '' } satisfies DMsOptions), - ).rejects.not.toThrow('REMOTE_SECRET_BODY') - }) - - test('propagates a malformed successful user lookup', async () => { - vi.stubGlobal( - 'fetch', - vi.fn(async (input: string | URL | Request) => { - const path = new URL(typeof input === 'string' || input instanceof URL ? input : input.url) - .pathname - if (path.endsWith('/users/me')) return response(me) - if (path.endsWith('/users/me/channels')) return response([]) - return response({ username: 'alice' }) - }), - ) - - await expect( - fetchDMs({ ...base, user: ['alice'], limit: 50, since: '' } satisfies DMsOptions), - ).rejects.toThrow('Mattermost returned an invalid user response.') - }) -}) - -test('unread propagates a direct-channel membership request failure', async () => { - vi.stubGlobal( - 'fetch', - vi.fn(async (input: string | URL | Request) => { - const path = new URL(typeof input === 'string' || input instanceof URL ? input : input.url) - .pathname - if (path.endsWith('/users/me')) return response(me) - if (path.endsWith('/users/me/teams')) { - return response([{ id: 'team', name: 'team', display_name: 'Team', type: 'O' }]) - } - if (path.endsWith('/users/me/channels')) return response([{ ...dm, total_msg_count: 1 }]) - if (path.endsWith('/users/me/teams/team/channels/members')) return response([]) - if (path.endsWith('/channels/dm/members/me')) { - return response({ message: 'REMOTE_SECRET_BODY' }, 500) - } - throw new Error(`unexpected route ${path}`) - }), - ) - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) - - await expect(showUnread({ ...base } satisfies UnreadOptions)).rejects.toThrow( - 'API request failed: 500', - ) - expect(log).not.toHaveBeenCalledWith('All caught up!') -}) diff --git a/tests/cli-metadata.test.ts b/tests/cli-metadata.test.ts deleted file mode 100644 index a67a1c9..0000000 --- a/tests/cli-metadata.test.ts +++ /dev/null @@ -1,601 +0,0 @@ -import { afterEach, describe, expect, test, vi } from 'vitest' -import { clearUserCache } from '../src/api/users' -import { fetchChannel, fetchMentions, searchMessages, showUnread } from '../src/cli' -import type { Channel, MessageOutput, Post, PostsResponse, SearchResponse } from '../src/types' -import { installRouteFetch } from './helpers/fake-fetch' - -const serverUrl = 'https://mattermost.test' -const me = { id: 'me', username: 'me' } -const team = { id: 'team', name: 'team', display_name: 'Team', type: 'O' } - -function channel(id: string): Channel { - return { - id, - team_id: 'team', - type: 'O', - name: id, - display_name: id.toUpperCase(), - header: '', - purpose: '', - last_post_at: 0, - total_msg_count: 0, - creator_id: 'me', - } -} - -function post(id: string, channelId: string, createAt: number, message = id): Post { - return { - id, - channel_id: channelId, - user_id: 'me', - create_at: createAt, - update_at: createAt, - delete_at: 0, - edit_at: 0, - message, - type: '', - props: {}, - hashtags: '', - file_ids: [], - root_id: '', - reply_count: 0, - pending_post_id: '', - } -} - -function page(posts: Post[]): PostsResponse { - return { - order: posts.map(({ id }) => id), - posts: Object.fromEntries(posts.map((item) => [item.id, item])), - } as PostsResponse -} - -function searchPage(posts: Post[]): SearchResponse { - return { ...page(posts), matches: {} } -} - -function commonRoutes() { - return [ - { method: 'GET', path: '/api/v4/users/me', handle: () => me }, - { method: 'GET', path: '/api/v4/users/me/teams', handle: () => [team] }, - ] -} - -function outputLog() { - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) - return () => JSON.parse(String(log.mock.calls.at(-1)?.[0])) -} - -function expectMetadata( - output: MessageOutput, - expected: { - source: MessageOutput['retrieval']['selection']['source'] - count: number - limit: number | null - since: string | null - truncated: boolean | null - }, -) { - expect(output.retrieval.selection).toEqual({ - source: expected.source, - selectedCount: expected.count, - requestedLimit: expected.limit, - since: expected.since, - queryTruncated: expected.truncated, - inputCursor: null, - nextCursor: null, - }) - expect(output.retrieval.visiblePostCount).toBe(expected.count) - expect(output.channel.metadataStatus).toBe('resolved') - expect(output.retrieval.visibleThreads).toEqual({ - status: 'complete', - hydratedRootCount: 0, - failedRootIds: [], - }) - expect( - output.messages.every((message) => message.permalink.startsWith(`${serverUrl}/_redirect/pl/`)), - ).toBe(true) -} - -afterEach(() => { - clearUserCache() - vi.useRealTimers() - vi.restoreAllMocks() - vi.unstubAllGlobals() -}) - -describe('command-level JSON retrieval metadata', () => { - test.each([ - ['blank id', { ...channel('general'), id: '' }], - ['blank name', { ...channel('general'), name: ' ' }], - ['non-string display name', { ...channel('general'), display_name: null }], - ['blank team id', { ...channel('general'), team_id: '' }], - [ - 'wrong channel kind', - { - ...channel('general'), - team_id: '', - type: 'D', - name: 'me__other', - display_name: '', - }, - ], - ] as const)('fails closed for a primary channel with %s', async (_label, malformed) => { - const { requests } = installRouteFetch([ - ...commonRoutes(), - { - method: 'GET', - path: '/api/v4/teams/team/channels/name/general', - handle: () => malformed, - }, - ]) - - await expect( - fetchChannel({ - url: serverUrl, - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: false, - channel: 'general', - limit: 1, - since: '', - }), - ).rejects.toThrow('Mattermost returned an invalid channel response.') - expect(requests.some(({ url }) => url.pathname.endsWith('/posts'))).toBe(false) - }) - - test('fails closed when a channel-name lookup returns a different canonical channel', async () => { - const { requests } = installRouteFetch([ - ...commonRoutes(), - { - method: 'GET', - path: '/api/v4/teams/team/channels/name/general', - handle: () => ({ ...channel('other'), id: 'other' }), - }, - ]) - - await expect( - fetchChannel({ - url: serverUrl, - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: false, - channel: '#general', - limit: 1, - since: '', - }), - ).rejects.toThrow('Mattermost returned an invalid channel response.') - expect(requests.some(({ url }) => url.pathname.endsWith('/posts'))).toBe(false) - }) - - test('channel reports the exact local since boundary', async () => { - vi.useFakeTimers() - vi.setSystemTime(new Date('2026-07-14T12:00:00.000Z')) - const general = channel('general') - installRouteFetch([ - ...commonRoutes(), - { method: 'GET', path: '/api/v4/teams/team/channels/name/general', handle: () => general }, - { - method: 'GET', - path: '/api/v4/channels/general/posts', - handle: () => page([post('channel-post', 'general', Date.now())]), - }, - ]) - const readOutput = outputLog() - - await fetchChannel({ - url: serverUrl, - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: true, - channel: 'general', - limit: 2, - since: '24h', - }) - - expectMetadata(readOutput()[0], { - source: 'recent', - count: 1, - limit: 2, - since: '2026-07-13T12:00:00.000Z', - truncated: false, - }) - }) - - test('--no-threads returns only thread-shaped seeds without hydration calls', async () => { - const general = channel('general') - const seed = { ...post('root', 'general', Date.now()), reply_count: 2 } - const { requests } = installRouteFetch([ - ...commonRoutes(), - { method: 'GET', path: '/api/v4/teams/team/channels/name/general', handle: () => general }, - { - method: 'GET', - path: '/api/v4/channels/general/posts', - handle: () => page([seed]), - }, - ]) - const readOutput = outputLog() - - await fetchChannel({ - url: serverUrl, - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: false, - channel: 'general', - limit: 1, - since: '24h', - }) - - const [output] = readOutput() as MessageOutput[] - expect(output?.messages.map(({ id }) => id)).toEqual(['root']) - expect(output?.retrieval.visibleThreads).toEqual({ - status: 'not_requested', - hydratedRootCount: 0, - failedRootIds: [], - }) - expect(output?.retrieval.visiblePostCount).toBe(1) - expect(requests.some(({ url }) => url.pathname.includes('/thread'))).toBe(false) - }) - - test.each([ - ['search', 'needle'], - ['mentions', '@me'], - ] as const)('%s keeps per-channel counts with global limit/truncation', async (source, text) => { - const alpha = channel('alpha') - const beta = channel('beta') - const alphaPost = { - ...post('alpha-new', 'alpha', 3, text), - user_id: 'alpha-user', - metadata: { - reactions: [{ user_id: 'reactor', post_id: 'alpha-new', emoji_name: 'eyes', create_at: 1 }], - }, - } - const betaPost = { ...post('beta-new', 'beta', 2, text), user_id: 'beta-user' } - const { requests } = installRouteFetch([ - ...commonRoutes(), - { - method: 'POST', - path: '/api/v4/teams/team/posts/search', - handle: () => searchPage([alphaPost, betaPost, post('alpha-extra', 'alpha', 1, text)]), - }, - { - method: 'POST', - path: '/api/v4/users/ids', - handle: () => [ - { id: 'alpha-user', username: 'alpha-user' }, - { id: 'beta-user', username: 'beta-user' }, - { id: 'reactor', username: 'reactor' }, - ], - }, - { method: 'GET', path: '/api/v4/channels/alpha', handle: () => alpha }, - { method: 'GET', path: '/api/v4/channels/beta', handle: () => beta }, - ]) - const readOutput = outputLog() - - if (source === 'search') { - await searchMessages({ - url: serverUrl, - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: true, - query: 'needle', - limit: 2, - }) - } else { - await fetchMentions({ - url: serverUrl, - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: true, - limit: 2, - mentionNames: [], - }) - } - - const outputs = readOutput() as MessageOutput[] - expect(outputs).toHaveLength(2) - for (const output of outputs) { - expectMetadata(output, { - source, - count: 1, - limit: 2, - since: null, - truncated: true, - }) - } - expect(requests.filter(({ url }) => url.pathname.endsWith('/users/ids'))).toHaveLength(1) - expect(requests.find(({ url }) => url.pathname.endsWith('/users/ids'))?.body).toEqual([ - 'alpha-user', - 'reactor', - 'beta-user', - ]) - expect(requests.some(({ url }) => /\/(files|reactions)(\/|$)/.test(url.pathname))).toBe(false) - }) - - test.each([ - ['search', 'needle'], - ['mentions', '@me'], - ] as const)('%s preserves every selected post when one channel metadata lookup fails', async (source, text) => { - const remoteBody = `REMOTE_${'sk-abcdefghijklmnopqrstuvwxyz123456'}` - installRouteFetch([ - ...commonRoutes(), - { - method: 'POST', - path: '/api/v4/teams/team/posts/search', - handle: () => ({ - ...searchPage([post('alpha-post', 'alpha', 2, text), post('beta-post', 'beta', 1, text)]), - has_next: false, - }), - }, - { method: 'GET', path: '/api/v4/channels/alpha', handle: () => channel('alpha') }, - { - method: 'GET', - path: '/api/v4/channels/beta', - handle: () => Response.json({ message: remoteBody }, { status: 500 }), - }, - ]) - const readOutput = outputLog() - const error = vi.spyOn(console, 'error').mockImplementation(() => undefined) - - if (source === 'search') { - await searchMessages({ - url: serverUrl, - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: true, - query: text, - limit: 2, - }) - } else { - await fetchMentions({ - url: serverUrl, - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: true, - limit: 2, - mentionNames: [], - }) - } - - const outputs = readOutput() as MessageOutput[] - expect(outputs.flatMap(({ messages }) => messages.map(({ id }) => id))).toEqual([ - 'alpha-post', - 'beta-post', - ]) - expect(outputs.map(({ channel: outputChannel }) => outputChannel)).toEqual([ - expect.objectContaining({ id: 'alpha', type: 'public', metadataStatus: 'resolved' }), - { - id: 'beta', - type: 'unknown', - name: 'unknown', - metadataStatus: 'unavailable', - }, - ]) - const warnings = error.mock.calls - .flat() - .map(String) - .filter((message) => message.startsWith('Warning: Channel metadata is unavailable')) - expect(warnings).toEqual(['Warning: Channel metadata is unavailable for beta.']) - expect(`${JSON.stringify(outputs)}\n${error.mock.calls.flat().join('\n')}`).not.toContain( - remoteBody, - ) - }) - - test.each([ - ['search', 'needle'], - ['mentions', '@me'], - ] as const)('%s keeps matched seed counts separate from hydrated context', async (source, text) => { - const general = channel('general') - const root = { ...post('root', 'general', 1, 'older context'), reply_count: 2 } - const seed = { ...post('seed', 'general', 3, text), root_id: 'root' } - const sibling = { - ...post('sibling', 'general', 2, 'context sk-abcdefghijklmnopqrstuvwxyz123456'), - root_id: 'root', - user_id: 'other', - } - installRouteFetch([ - ...commonRoutes(), - { - method: 'POST', - path: '/api/v4/teams/team/posts/search', - handle: () => ({ ...searchPage([seed]), has_next: false }), - }, - { - method: 'GET', - path: '/api/v4/posts/root/thread', - handle: () => ({ ...page([root, sibling, seed]), has_next: false }), - }, - { method: 'GET', path: '/api/v4/channels/general', handle: () => general }, - { - method: 'POST', - path: '/api/v4/users/ids', - handle: () => [{ id: 'other', username: 'other' }], - }, - ]) - const readOutput = outputLog() - - if (source === 'search') { - await searchMessages({ - url: serverUrl, - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: true, - query: text, - limit: 1, - }) - } else { - await fetchMentions({ - url: serverUrl, - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: true, - limit: 1, - mentionNames: [], - }) - } - - const [output] = readOutput() as MessageOutput[] - expect(output?.retrieval.selection.selectedCount).toBe(1) - expect(output?.retrieval.visiblePostCount).toBe(3) - expect(output?.retrieval.visibleThreads.status).toBe('complete') - const visible = - output?.messages.flatMap((message) => [message, ...(message.replies ?? [])]) ?? [] - expect(visible.map(({ id }) => id)).toEqual(['root', 'sibling', 'seed']) - const hydratedSibling = visible.find(({ id }) => id === 'sibling') - expect(hydratedSibling?.user).toBe('other') - expect(hydratedSibling?.text).not.toContain('sk-abcdefghijklmnopqrstuvwxyz123456') - expect(hydratedSibling?.permalink).toBe(`${serverUrl}/_redirect/pl/sibling`) - }) - - test('unread peek reports each channel boundary and selected count', async () => { - const alpha = { ...channel('alpha'), last_post_at: 300, total_msg_count: 2 } - const beta = { ...channel('beta'), last_post_at: 200, total_msg_count: 2 } - installRouteFetch([ - ...commonRoutes(), - { method: 'GET', path: '/api/v4/users/me/channels', handle: () => [alpha, beta] }, - { - method: 'GET', - path: '/api/v4/users/me/teams/team/channels/members', - handle: () => [ - { - channel_id: 'alpha', - user_id: 'me', - msg_count: 1, - mention_count: 0, - last_viewed_at: 100, - }, - { channel_id: 'beta', user_id: 'me', msg_count: 1, mention_count: 0, last_viewed_at: 50 }, - ], - }, - { - method: 'GET', - path: '/api/v4/channels/alpha/posts', - handle: () => page([post('alpha-post', 'alpha', 300)]), - }, - { - method: 'GET', - path: '/api/v4/channels/beta/posts', - handle: () => page([post('beta-post', 'beta', 200)]), - }, - ]) - const readOutput = outputLog() - - await showUnread({ - url: serverUrl, - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: true, - peek: 2, - }) - - const outputs = readOutput().peek as MessageOutput[] - const alphaOutput = outputs.find(({ channel }) => channel.id === 'alpha') - const betaOutput = outputs.find(({ channel }) => channel.id === 'beta') - if (!alphaOutput || !betaOutput) throw new Error('missing unread output fixture') - expectMetadata(alphaOutput, { - source: 'unread', - count: 1, - limit: 2, - since: new Date(100).toISOString(), - truncated: false, - }) - expectMetadata(betaOutput, { - source: 'unread', - count: 1, - limit: 2, - since: new Date(50).toISOString(), - truncated: false, - }) - }) - - test('unread keeps selected-team O/P and globally unique D/G only', async () => { - const selected = { ...channel('selected'), total_msg_count: 2 } - const foreign = { ...channel('foreign'), team_id: 'other-team', total_msg_count: 2 } - const dm = { - ...channel('dm'), - team_id: '', - type: 'D' as const, - name: 'me__other', - total_msg_count: 2, - } - const { requests } = installRouteFetch([ - ...commonRoutes(), - { - method: 'GET', - path: '/api/v4/users/me/channels', - handle: () => [selected, foreign, dm, dm], - }, - { - method: 'GET', - path: '/api/v4/users/me/teams/team/channels/members', - handle: () => [ - { channel_id: 'selected', msg_count: 1, mention_count: 0, last_viewed_at: 1 }, - { channel_id: 'foreign', msg_count: 1, mention_count: 0, last_viewed_at: 1 }, - ], - }, - { - method: 'GET', - path: '/api/v4/channels/dm/members/me', - handle: () => ({ channel_id: 'dm', msg_count: 1, mention_count: 0, last_viewed_at: 1 }), - }, - { - method: 'GET', - path: '/api/v4/users/other', - handle: () => ({ id: 'other', username: 'other' }), - }, - ]) - const readOutput = outputLog() - - await showUnread({ - url: serverUrl, - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: true, - }) - - expect( - readOutput() - .unread.map(({ channel }: MessageOutput) => channel.id) - .sort(), - ).toEqual(['dm', 'selected']) - expect( - requests.filter(({ url }) => url.pathname === '/api/v4/channels/dm/members/me'), - ).toHaveLength(1) - }) -}) diff --git a/tests/cli-output.test.ts b/tests/cli-output.test.ts deleted file mode 100644 index 9279a3d..0000000 --- a/tests/cli-output.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { describe, expect, test } from 'vitest' -import { selectOutputMode } from '../src/cli' - -describe('global output mode selection', () => { - test.each([ - { json: true, isTTY: true, expected: 'json' }, - { json: true, isTTY: false, expected: 'json' }, - { json: false, isTTY: true, expected: 'pretty' }, - { json: false, isTTY: false, expected: 'markdown' }, - ] as const)('$expected when json=$json and isTTY=$isTTY', ({ json, isTTY, expected }) => { - expect(selectOutputMode(json, isTTY)).toBe(expected) - }) -}) diff --git a/tests/cli-retrieval.test.ts b/tests/cli-retrieval.test.ts deleted file mode 100644 index 5f0c98c..0000000 --- a/tests/cli-retrieval.test.ts +++ /dev/null @@ -1,1049 +0,0 @@ -import { afterEach, describe, expect, test, vi } from 'vitest' -import { initClient } from '../src/api/client' -import { clearUserCache } from '../src/api/users' -import { - BoundedPostIdSet, - createWatchPostHandler, - fetchDMs, - fetchThread, - hasLiteralMention, - hydrateVisibleThreads, - isExactMentionPost, - mentionSearchAfterDate, - mergeTruncation, -} from '../src/cli' -import { setActiveMattermostCredential } from '../src/preprocessing' -import type { Channel, Post, PostsResponse, Redaction, User } from '../src/types' -import { installRouteFetch } from './helpers/fake-fetch' - -afterEach(() => { - setActiveMattermostCredential(undefined) - clearUserCache() - vi.restoreAllMocks() - vi.unstubAllGlobals() -}) - -describe('literal mention filtering', () => { - test.each([ - [[false, false], 2, 2, false], - [[false, false], 3, 2, true], - [[true, false], 2, 2, true], - [[null, false], 2, 2, null], - ] as const)('merges global truncation states %j with %i candidates / %i limit', (states, candidates, limit, expected) => { - expect(mergeTruncation([...states], candidates, limit)).toBe(expected) - }) - - test('ignores empty configured literals', () => { - expect(hasLiteralMention('anything', [''])).toBe(false) - }) - - test('matches configured literals case-insensitively', () => { - expect(hasLiteralMention('Hello ARDA SEVINC', ['Arda Sevinc'])).toBe(true) - }) - - test('does not treat search-engine token matches as literal mentions', () => { - expect(hasLiteralMention('Arda discussed this separately', ['Arda Sevinc'])).toBe(false) - }) - - test.each([ - ['Arda', true], - ['(Arda)', true], - ['hello\nArda\nthere', true], - ['Ardahan', false], - ['xArda', false], - ['Arda.com', true], - ['Arda_more', true], - ['Arda-name', true], - ['Arda2', false], - ['Ardağ', false], - ['İArda', false], - ['Arda東京', false], - ['🙂Arda🙂', true], - ])('applies literal alias boundaries to %j', (message, expected) => { - expect(hasLiteralMention(message, ['Arda'])).toBe(expected) - }) - - test.each([ - ['@arda', true], - ['(@arda)', true], - ['hey\n@arda\nthere', true], - ['foo@arda', false], - ['foo@arda.com', false], - ['@arda.com', false], - ['@arda_more', false], - ['@arda-more', false], - ['@arda2', false], - ['x@arda!', false], - ])('applies username boundaries to %j', (message, expected) => { - expect(hasLiteralMention(message, ['@arda'])).toBe(expected) - }) - - test('applies the exact millisecond boundary after search retrieval', () => { - const candidate = { message: '@arda', delete_at: 0 } as Post - expect(isExactMentionPost({ ...candidate, create_at: 999 }, '@arda', 1000)).toBe(false) - expect(isExactMentionPost({ ...candidate, create_at: 1000 }, '@arda', 1000)).toBe(true) - }) - - test('widens the coarse after query by one UTC calendar day', () => { - const since = Date.UTC(2026, 6, 14, 15, 30) - expect(`after:${mentionSearchAfterDate(since)}`).toBe('after:2026-07-13') - }) - - test('dedupes repeated DM targets before fetching and enforces one global limit', async () => { - const now = Date.now() - const users = { - me: { id: 'me', username: 'me' } as User, - alice: { id: 'alice', username: 'alice' } as User, - bob: { id: 'bob', username: 'bob' } as User, - } - const channels = [ - { id: 'dm-alice', team_id: '', type: 'D', name: 'alice__me', display_name: '' } as Channel, - { id: 'dm-bob', team_id: '', type: 'D', name: 'bob__me', display_name: '' } as Channel, - ] - const makePost = (id: string, channelId: string, userId: string, createAt: number) => - ({ - id, - channel_id: channelId, - user_id: userId, - create_at: createAt, - update_at: createAt, - delete_at: 0, - edit_at: 0, - message: id, - type: '', - props: {}, - hashtags: '', - file_ids: [], - root_id: '', - reply_count: 0, - pending_post_id: '', - }) satisfies Post - const pages: Record = { - 'dm-alice': [makePost('alice-new', 'dm-alice', 'alice', now - 1)], - 'dm-bob': [makePost('bob-new', 'dm-bob', 'bob', now - 2)], - } - const { requests } = installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => users.me }, - { - method: 'GET', - path: '/api/v4/users/username/alice', - handle: () => users.alice, - }, - { method: 'GET', path: '/api/v4/users/username/bob', handle: () => users.bob }, - { method: 'GET', path: '/api/v4/users/me/channels', handle: () => channels }, - { - method: 'GET', - path: '/api/v4/channels/dm-alice/posts', - handle: () => responsePage(pages['dm-alice'] ?? []), - }, - { - method: 'GET', - path: '/api/v4/channels/dm-bob/posts', - handle: () => responsePage(pages['dm-bob'] ?? []), - }, - ]) - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) - - await fetchDMs({ - url: 'https://mattermost.test', - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: false, - user: ['alice', 'alice', 'bob'], - limit: 2, - since: '1m', - }) - - const output = JSON.parse(String(log.mock.calls.at(-1)?.[0])) as Array<{ - messages: Array<{ id: string }> - retrieval: { selection: { queryTruncated: boolean | null } } - }> - expect(output.flatMap(({ messages }) => messages.map(({ id }) => id))).toEqual([ - 'alice-new', - 'bob-new', - ]) - expect(output.every(({ retrieval }) => retrieval.selection.queryTruncated === false)).toBe(true) - expect(requests.filter(({ url }) => url.pathname.endsWith('/dm-alice/posts'))).toHaveLength(1) - expect(requests.filter(({ url }) => url.pathname.endsWith('/dm-bob/posts'))).toHaveLength(1) - }) - - test.each([ - ['blank name', { id: 'dm', team_id: '', type: 'D', name: '', display_name: '' }], - [ - 'malformed direct name', - { id: 'dm', team_id: '', type: 'D', name: 'only-one-user', display_name: '' }, - ], - [ - 'nonempty direct team', - { id: 'dm', team_id: 'team', type: 'D', name: 'me__other', display_name: '' }, - ], - ] as const)('fails closed for a discovered direct channel with %s', async (_label, malformed) => { - const { requests } = installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => ({ id: 'me', username: 'me' }) }, - { method: 'GET', path: '/api/v4/users/me/channels', handle: () => [malformed] }, - ]) - - await expect( - fetchDMs({ - url: 'https://mattermost.test', - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: false, - user: [], - limit: 1, - since: '', - }), - ).rejects.toThrow('Mattermost returned an invalid channel response.') - expect(requests.some(({ url }) => url.pathname.endsWith('/posts'))).toBe(false) - }) - - test('validates every discovered direct channel before filtered-user matching', async () => { - const malformed = { - id: 'dm-alice', - team_id: '', - type: 'D', - name: 'only-one-user', - display_name: '', - } - const validDuplicate = { - ...malformed, - name: 'alice__me', - } - const { requests } = installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => ({ id: 'me', username: 'me' }) }, - { - method: 'GET', - path: '/api/v4/users/me/channels', - handle: () => [malformed, validDuplicate], - }, - ]) - const error = vi.spyOn(console, 'error').mockImplementation(() => undefined) - - await expect( - fetchDMs({ - url: 'https://mattermost.test', - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: false, - user: ['alice'], - limit: 1, - since: '', - }), - ).rejects.toThrow('Mattermost returned an invalid channel response.') - expect(requests.some(({ url }) => url.pathname.includes('/users/username/'))).toBe(false) - expect(requests.some(({ url }) => url.pathname.endsWith('/posts'))).toBe(false) - expect(error).not.toHaveBeenCalled() - }) - - test('fails closed when a discovered direct channel excludes the current user', async () => { - const wrongParticipants = { - id: 'dm-unrelated', - team_id: '', - type: 'D', - name: 'alice__bob', - display_name: '', - } - const { requests } = installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => ({ id: 'me', username: 'me' }) }, - { method: 'GET', path: '/api/v4/users/me/channels', handle: () => [wrongParticipants] }, - ]) - - await expect( - fetchDMs({ - url: 'https://mattermost.test', - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: false, - user: [], - limit: 1, - since: '', - }), - ).rejects.toThrow('Mattermost returned an invalid channel response.') - expect(requests.some(({ url }) => url.pathname.endsWith('/posts'))).toBe(false) - }) - - test('hydrates only threads surviving the final global DM selection', async () => { - const now = Date.now() - const channels = [ - { id: 'dm-alice', team_id: '', type: 'D', name: 'alice__me', display_name: '' } as Channel, - { id: 'dm-bob', team_id: '', type: 'D', name: 'bob__me', display_name: '' } as Channel, - ] - const aliceSeed = { ...makeThreadPost('alice-seed', 'alice-root', now), channel_id: 'dm-alice' } - const bobSeed = { - ...makeThreadPost('bob-seed', 'bob-root', now - 1), - channel_id: 'dm-bob', - user_id: 'bob', - } - const aliceRoot = { - ...makeThreadPost('alice-root', '', now - 2, 1), - channel_id: 'dm-alice', - } - const { requests } = installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => ({ id: 'me', username: 'me' }) }, - { - method: 'GET', - path: '/api/v4/users/alice', - handle: () => ({ id: 'alice', username: 'alice' }), - }, - { method: 'GET', path: '/api/v4/users/me/channels', handle: () => channels }, - { - method: 'GET', - path: '/api/v4/channels/dm-alice/posts', - handle: () => responsePage([aliceSeed]), - }, - { - method: 'GET', - path: '/api/v4/channels/dm-bob/posts', - handle: () => responsePage([bobSeed]), - }, - { - method: 'GET', - path: '/api/v4/posts/alice-root/thread', - handle: () => ({ ...responsePage([aliceRoot, aliceSeed]), has_next: false }), - }, - ]) - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) - - await fetchDMs({ - url: 'https://mattermost.test', - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: true, - user: [], - limit: 1, - since: '1m', - }) - - const [output] = JSON.parse(String(log.mock.calls.at(-1)?.[0])) - expect(output.retrieval.selection.selectedCount).toBe(1) - expect(output.retrieval.visiblePostCount).toBe(2) - expect( - requests.filter(({ url }) => url.pathname.includes('/posts/alice-root/thread')), - ).toHaveLength(1) - expect(requests.some(({ url }) => url.pathname.includes('/posts/bob-root/thread'))).toBe(false) - }) -}) - -describe('watch post deduplication', () => { - test('suppresses repeats while bounding retained post ids', () => { - const ids = new BoundedPostIdSet(2) - expect(ids.add('one')).toBe(true) - expect(ids.add('one')).toBe(false) - expect(ids.add('two')).toBe(true) - expect(ids.add('three')).toBe(true) - expect(ids.add('one')).toBe(true) - }) - - test('writes synchronous redacted JSONL in event order and suppresses duplicate ids', () => { - const lines: string[] = [] - const handlePost = createWatchPostHandler({ json: true, color: false, redact: true }, (line) => - lines.push(line), - ) - const makePost = (id: string, message: string, createAt: number) => - ({ - id, - channel_id: 'channel-1', - user_id: 'user-1', - create_at: createAt, - update_at: createAt, - delete_at: 0, - edit_at: 0, - message, - type: '', - props: {}, - hashtags: '', - file_ids: [], - root_id: '', - reply_count: 0, - pending_post_id: '', - }) satisfies Post - - handlePost(makePost('one', 'first sk-abcdefghijklmnopqrstuvwxyz123456', 1), 'town', 'arda') - handlePost(makePost('two', 'second', 2), 'town', 'arda') - handlePost(makePost('one', 'duplicate', 3), 'town', 'arda') - - expect(lines).toHaveLength(2) - expect(lines.every((line) => !line.includes('\n'))).toBe(true) - expect(lines.map((line) => JSON.parse(line).postId)).toEqual(['one', 'two']) - expect(JSON.parse(lines[0] as string).message).not.toContain( - 'sk-abcdefghijklmnopqrstuvwxyz123456', - ) - }) - - test('sanitizes every remotely controlled watch presentation field', () => { - const token = `ghp_${'a'.repeat(36)}` - const lines: string[] = [] - const handlePost = createWatchPostHandler({ json: true, color: false, redact: true }, (line) => - lines.push(line), - ) - handlePost( - { - id: `post\u001b${token}`, - channel_id: `channel\u001b${token}`, - user_id: `user\u001b${token}`, - create_at: 1, - update_at: 1, - delete_at: 0, - edit_at: 0, - message: `message\u001b${token}`, - type: '', - props: {}, - hashtags: '', - file_ids: [`file\u001b${token}`], - root_id: `root\u001b${token}`, - reply_count: 0, - pending_post_id: '', - }, - `town\u001b${token}`, - `sender\u001b${token}`, - ) - - expect(lines).toHaveLength(1) - expect(lines[0]).not.toContain(token) - expect(lines[0]).not.toContain('\u001b') - expect(JSON.parse(lines[0] as string).redactions.map(({ field }: Redaction) => field)).toEqual( - expect.arrayContaining([ - 'watch.sender', - 'watch.message', - 'watch.postId', - 'watch.channelId', - 'watch.channelName', - 'watch.senderId', - 'watch.rootId', - 'watch.fileId', - ]), - ) - }) - - test('protects the active credential in JSON watch output with --no-redact', () => { - const credential = '9xuqwrwgstrb3mzrxb83nb357a' - initClient('https://mattermost.test', credential) - const lines: string[] = [] - const handlePost = createWatchPostHandler({ json: true, color: false, redact: false }, (line) => - lines.push(line), - ) - handlePost( - { - id: 'post', - channel_id: 'channel', - user_id: 'user', - create_at: 1, - update_at: 1, - delete_at: 0, - edit_at: 0, - message: `active=${credential} unrelated=aaaaaaaaaaaaaaaaaaaaaaaaaa`, - type: '', - props: {}, - hashtags: '', - file_ids: [], - root_id: '', - reply_count: 0, - pending_post_id: '', - }, - 'town', - 'sender', - ) - expect(lines[0]).not.toContain(credential) - expect(lines[0]).toContain('aaaaaaaaaaaaaaaaaaaaaaaaaa') - expect(JSON.parse(lines[0] as string).redactions).toEqual([ - expect.objectContaining({ type: 'mattermost_credential', field: 'watch.message' }), - ]) - }) -}) - -describe('thread retrieval metadata', () => { - test('hydrates a reply seed with its root and siblings once', async () => { - const root = makeThreadPost('root', '', 1, 2) - const reply = makeThreadPost('reply', 'root', 2) - const sibling = makeThreadPost('sibling', 'root', 3) - const { requests } = installRouteFetch([ - { - method: 'GET', - path: '/api/v4/posts/root/thread', - handle: () => ({ ...responsePage([root, reply, sibling]), has_next: false }), - }, - ]) - initClient('https://mattermost.test', 'token') - - const result = await hydrateVisibleThreads([reply, sibling], true) - - expect(result.posts.map(({ id }) => id)).toEqual(['reply', 'sibling', 'root']) - expect(result.visibleThreads).toEqual({ - status: 'complete', - hydratedRootCount: 1, - failedRootIds: [], - }) - expect(requests).toHaveLength(1) - }) - - test('hydrates a selected root with its replies', async () => { - const root = makeThreadPost('root', '', 1, 1) - const reply = makeThreadPost('reply', 'root', 2) - const { requests } = installRouteFetch([ - { - method: 'GET', - path: '/api/v4/posts/root/thread', - handle: () => ({ ...responsePage([root, reply]), has_next: false }), - }, - ]) - initClient('https://mattermost.test', 'token') - - expect((await hydrateVisibleThreads([root], true)).posts.map(({ id }) => id)).toEqual([ - 'root', - 'reply', - ]) - expect(requests).toHaveLength(1) - }) - - test('skips complete seed threads and makes no requests when threads are disabled', async () => { - const root = makeThreadPost('root', '', 1, 1) - const reply = makeThreadPost('reply', 'root', 2) - const { requests } = installRouteFetch([]) - initClient('https://mattermost.test', 'token') - - expect((await hydrateVisibleThreads([root, reply], true)).visibleThreads.status).toBe( - 'complete', - ) - expect((await hydrateVisibleThreads([reply], false)).visibleThreads.status).toBe( - 'not_requested', - ) - expect(requests).toHaveLength(0) - }) - - test('isolates failures, sanitizes warnings, and preserves other hydrated roots', async () => { - const badRootId = 'bad\u001b[31m-root' - const goodSeed = makeThreadPost('good-seed', 'good-root', 2) - const badSeed = makeThreadPost('bad-seed', badRootId, 3) - const goodRoot = makeThreadPost('good-root', '', 1, 1) - installRouteFetch([ - { - method: 'GET', - path: '/api/v4/posts/good-root/thread', - handle: () => ({ ...responsePage([goodRoot, goodSeed]), has_next: false }), - }, - { - method: 'GET', - path: `/api/v4/posts/${badRootId}/thread`, - handle: () => { - throw new Error('server included secret') - }, - }, - ]) - initClient('https://mattermost.test', 'token') - const error = vi.spyOn(console, 'error').mockImplementation(() => undefined) - - const result = await hydrateVisibleThreads([goodSeed, badSeed], true) - - expect(result.posts.map(({ id }) => id)).toEqual(['good-seed', 'bad-seed', 'good-root']) - expect(result.visibleThreads).toEqual({ - status: 'partial', - hydratedRootCount: 1, - failedRootIds: [badRootId], - }) - const warning = String(error.mock.calls.at(-1)?.[0]) - expect(warning).not.toContain('\u001b') - expect(warning).not.toContain('server included secret') - }) - - test('preserves accumulated thread context when a later page fails', async () => { - const root = makeThreadPost('root', '', 1, 3) - const seed = makeThreadPost('seed', 'root', 3) - const context = makeThreadPost('context', 'root', 2) - installRouteFetch([ - { - method: 'GET', - path: '/api/v4/posts/root/thread', - handle: ({ url }) => { - if (url.searchParams.has('fromPost')) throw new Error('page two failed') - return { ...responsePage([root, context, seed]), has_next: true } - }, - }, - ]) - initClient('https://mattermost.test', 'token') - vi.spyOn(console, 'error').mockImplementation(() => undefined) - - const result = await hydrateVisibleThreads([seed], true) - - expect(result.posts.map(({ id }) => id)).toEqual(['seed', 'root', 'context']) - expect(result.visibleThreads).toEqual({ - status: 'partial', - hydratedRootCount: 0, - failedRootIds: ['root'], - }) - }) - - test('never exceeds four concurrent thread requests', async () => { - const seeds = Array.from({ length: 9 }, (_, index) => - makeThreadPost(`seed-${index}`, `root-${index}`, index + 10), - ) - let active = 0 - let maxActive = 0 - vi.stubGlobal( - 'fetch', - vi.fn(async (input: string | URL | Request) => { - active += 1 - maxActive = Math.max(maxActive, active) - await new Promise((resolve) => setTimeout(resolve, 1)) - const url = new URL(typeof input === 'string' || input instanceof URL ? input : input.url) - const rootId = url.pathname.split('/').at(-2) as string - const index = Number(rootId.slice('root-'.length)) - const root = makeThreadPost(rootId, '', index, 1) - active -= 1 - return Response.json({ ...responsePage([root, seeds[index] as Post]), has_next: false }) - }), - ) - initClient('https://mattermost.test', 'token') - - expect((await hydrateVisibleThreads(seeds, true)).visibleThreads.status).toBe('complete') - expect(maxActive).toBe(4) - }) - - test('reports concurrent failed roots in seed selection order', async () => { - const seeds = [ - makeThreadPost('seed-first', 'root-first', 1), - makeThreadPost('seed-second', 'root-second', 2), - ] - vi.stubGlobal( - 'fetch', - vi.fn(async (input: string | URL | Request) => { - const url = new URL(typeof input === 'string' || input instanceof URL ? input : input.url) - if (url.pathname.includes('root-first')) { - await new Promise((resolve) => setTimeout(resolve, 2)) - return Response.json({ ...responsePage([seeds[0] as Post]), has_next: false }) - } - return Response.json({ ...responsePage([seeds[1] as Post]), has_next: false }) - }), - ) - initClient('https://mattermost.test', 'token') - vi.spyOn(console, 'error').mockImplementation(() => undefined) - - expect((await hydrateVisibleThreads(seeds, true)).visibleThreads.failedRootIds).toEqual([ - 'root-first', - 'root-second', - ]) - }) - - test('uses a reply channel and reports a missing root as partial', async () => { - const reply = { - id: 'reply', - channel_id: 'channel', - user_id: 'me', - create_at: 1, - update_at: 1, - delete_at: 0, - edit_at: 0, - message: 'reply', - type: '', - props: {}, - hashtags: '', - file_ids: [], - root_id: 'missing-root', - reply_count: 0, - pending_post_id: '', - } satisfies Post - installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => ({ id: 'me', username: 'me' }) }, - { - method: 'GET', - path: '/api/v4/posts/missing-root/thread', - handle: () => ({ ...responsePage([reply]), has_next: false }), - }, - { - method: 'GET', - path: '/api/v4/channels/channel', - handle: () => ({ - id: 'channel', - team_id: 'team', - type: 'O', - name: 'general', - display_name: 'General', - }), - }, - ]) - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) - - await fetchThread({ - url: 'https://mattermost.test', - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: true, - postId: 'missing-root', - }) - - const [output] = JSON.parse(String(log.mock.calls.at(-1)?.[0])) - expect(output.channel.id).toBe('channel') - expect(output.retrieval.selection).toMatchObject({ - source: 'thread', - selectedCount: 1, - requestedLimit: null, - since: null, - queryTruncated: false, - }) - expect(output.retrieval.visibleThreads).toEqual({ - status: 'partial', - hydratedRootCount: 0, - failedRootIds: ['missing-root'], - }) - }) - - test('classifies malformed channel metadata as unavailable and sanitizes its ID', async () => { - const token = `ghp_${'a'.repeat(36)}` - const unsafeRoot = `root\u001b${token}` - const unsafeChannel = `channel\u001b${token}` - const reply = { - ...makeThreadPost('reply', unsafeRoot, 1), - channel_id: unsafeChannel, - user_id: 'me', - } - installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => ({ id: 'me', username: 'me' }) }, - { - method: 'GET', - path: `/api/v4/posts/${encodeURIComponent(unsafeRoot)}/thread`, - handle: () => ({ ...responsePage([reply]), has_next: false }), - }, - { - method: 'GET', - path: `/api/v4/channels/${encodeURIComponent(unsafeChannel)}`, - handle: () => null, - }, - ]) - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) - const error = vi.spyOn(console, 'error').mockImplementation(() => undefined) - - await fetchThread({ - url: 'https://mattermost.test', - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: true, - postId: unsafeRoot, - }) - - const serialized = String(log.mock.calls.at(-1)?.[0]) - const [output] = JSON.parse(serialized) - expect(output.channel).toEqual({ - id: expect.not.stringContaining(token), - type: 'unknown', - name: 'unknown', - metadataStatus: 'unavailable', - }) - expect( - error.mock.calls - .flat() - .map(String) - .filter((message) => message.startsWith('Warning: Channel metadata is unavailable')), - ).toHaveLength(1) - expect(serialized).not.toContain(token) - expect(String(error.mock.calls.at(-1)?.[0])).not.toContain(token) - expect(serialized).not.toContain('\u001b') - }) - - test.each([ - ['null body', null], - ['non-object body', 'channel'], - ['blank id', { id: '', team_id: 'team', type: 'O', name: 'general', display_name: 'General' }], - [ - 'mismatched id', - { id: 'other', team_id: 'team', type: 'O', name: 'general', display_name: 'General' }, - ], - [ - 'unknown type', - { id: 'channel', team_id: 'team', type: 'X', name: 'general', display_name: 'General' }, - ], - [ - 'blank public name', - { id: 'channel', team_id: 'team', type: 'O', name: ' ', display_name: 'General' }, - ], - [ - 'non-string display name', - { id: 'channel', team_id: 'team', type: 'O', name: 'general', display_name: null }, - ], - ['missing team id', { id: 'channel', type: 'O', name: 'general', display_name: 'General' }], - [ - 'blank public team id', - { id: 'channel', team_id: '', type: 'O', name: 'general', display_name: 'General' }, - ], - [ - 'malformed direct name', - { id: 'channel', team_id: '', type: 'D', name: 'one-user', display_name: '' }, - ], - [ - 'direct channel excluding the current user', - { id: 'channel', team_id: '', type: 'D', name: 'alice__bob', display_name: '' }, - ], - [ - 'nonempty direct team id', - { id: 'channel', team_id: 'team', type: 'D', name: 'me__me', display_name: '' }, - ], - [ - 'blank group name', - { id: 'channel', team_id: '', type: 'G', name: '', display_name: 'Group' }, - ], - [ - 'nonempty group team id', - { id: 'channel', team_id: 'team', type: 'G', name: 'group', display_name: 'Group' }, - ], - ] as const)('classifies malformed 200 channel metadata with %s as unavailable', async (_label, payload) => { - const root = makeThreadPost('root', '', 1) - installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => ({ id: 'me', username: 'me' }) }, - { - method: 'GET', - path: '/api/v4/posts/root/thread', - handle: () => ({ ...responsePage([root]), has_next: false }), - }, - { method: 'GET', path: '/api/v4/channels/channel', handle: () => payload }, - ]) - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) - const error = vi.spyOn(console, 'error').mockImplementation(() => undefined) - - await fetchThread({ - url: 'https://mattermost.test', - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: true, - postId: 'root', - }) - - const [output] = JSON.parse(String(log.mock.calls.at(-1)?.[0])) - expect(output.channel).toEqual({ - id: 'channel', - type: 'unknown', - name: 'unknown', - metadataStatus: 'unavailable', - }) - expect(output.messages.map(({ id }: { id: string }) => id)).toEqual(['root']) - expect( - error.mock.calls - .flat() - .map(String) - .filter((message) => message.startsWith('Warning: Channel metadata is unavailable')), - ).toEqual(['Warning: Channel metadata is unavailable for channel.']) - }) - - test.each([ - ['direct', { id: 'channel', team_id: '', type: 'D', name: 'me__me', display_name: '' }], - ['group', { id: 'channel', team_id: '', type: 'G', name: 'group', display_name: 'Group' }], - ] as const)('accepts a valid %s channel metadata shape', async (_label, payload) => { - const root = makeThreadPost('root', '', 1) - installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => ({ id: 'me', username: 'me' }) }, - { - method: 'GET', - path: '/api/v4/posts/root/thread', - handle: () => ({ ...responsePage([root]), has_next: false }), - }, - { method: 'GET', path: '/api/v4/channels/channel', handle: () => payload }, - ]) - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) - - await fetchThread({ - url: 'https://mattermost.test', - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: true, - postId: 'root', - }) - - const [output] = JSON.parse(String(log.mock.calls.at(-1)?.[0])) - expect(output.channel.metadataStatus).toBe('resolved') - expect(output.channel.type).toBe(_label === 'direct' ? 'dm' : 'group') - }) - - test.each([ - ['403', 403], - ['404', 404], - ['500', 500], - ['network', null], - ] as const)( - 'preserves a fetched thread when channel metadata fails with %s', - async (_name, status) => { - const remoteSecret = `sk-${'z'.repeat(40)}` - const root = makeThreadPost('root', '', 1) - vi.stubGlobal( - 'fetch', - vi.fn(async (input: string | URL | Request) => { - const path = new URL( - typeof input === 'string' || input instanceof URL ? input : input.url, - ).pathname - if (path.endsWith('/users/me')) { - return Response.json({ id: 'me', username: 'me' }) - } - if (path.endsWith('/posts/root/thread')) { - return Response.json({ ...responsePage([root]), has_next: false }) - } - if (path.endsWith('/channels/channel')) { - if (status === null) throw new Error(`transport ${remoteSecret}`) - return Response.json({ message: remoteSecret }, { status }) - } - throw new Error(`unexpected route ${path}`) - }), - ) - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) - const error = vi.spyOn(console, 'error').mockImplementation(() => undefined) - - await fetchThread({ - url: 'https://mattermost.test', - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: true, - postId: 'root', - }) - - const serialized = String(log.mock.calls.at(-1)?.[0]) - const [output] = JSON.parse(serialized) - expect(output.messages.map(({ id }: { id: string }) => id)).toEqual(['root']) - expect(output.channel).toEqual({ - id: 'channel', - type: 'unknown', - name: 'unknown', - metadataStatus: 'unavailable', - }) - expect( - error.mock.calls - .flat() - .map(String) - .filter((message) => message.startsWith('Warning: Channel metadata is unavailable')), - ).toEqual(['Warning: Channel metadata is unavailable for channel.']) - expect(`${serialized}\n${error.mock.calls.flat().join('\n')}`).not.toContain(remoteSecret) - }, - 10_000, - ) - - test('keeps a channel authentication failure fail-closed', async () => { - const root = makeThreadPost('root', '', 1) - vi.stubGlobal( - 'fetch', - vi.fn(async (input: string | URL | Request) => { - const path = new URL(typeof input === 'string' || input instanceof URL ? input : input.url) - .pathname - if (path.endsWith('/users/me')) return Response.json({ id: 'me', username: 'me' }) - if (path.endsWith('/posts/root/thread')) { - return Response.json({ ...responsePage([root]), has_next: false }) - } - if (path.endsWith('/channels/channel')) { - return Response.json({ message: 'REMOTE_SECRET_BODY' }, { status: 401 }) - } - throw new Error(`unexpected route ${path}`) - }), - ) - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) - const error = vi.spyOn(console, 'error').mockImplementation(() => undefined) - - await expect( - fetchThread({ - url: 'https://mattermost.test', - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: true, - postId: 'root', - }), - ).rejects.toThrow('API request failed: 401') - expect(log).not.toHaveBeenCalled() - expect(error.mock.calls.flat().join('\n')).not.toContain('REMOTE_SECRET_BODY') - expect(error.mock.calls.flat().join('\n')).not.toContain('Channel metadata is unavailable') - }) - - test('does not count an unproven root as hydrated in mm thread metadata', async () => { - const root = makeThreadPost('root', '', 1, 2) - installRouteFetch([ - { method: 'GET', path: '/api/v4/users/me', handle: () => ({ id: 'me', username: 'me' }) }, - { - method: 'GET', - path: '/api/v4/posts/root/thread', - handle: () => responsePage([root]), - }, - { - method: 'GET', - path: '/api/v4/channels/channel', - handle: () => ({ - id: 'channel', - team_id: 'team', - type: 'O', - name: 'general', - display_name: 'General', - }), - }, - ]) - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) - vi.spyOn(console, 'error').mockImplementation(() => undefined) - - await fetchThread({ - url: 'https://mattermost.test', - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: true, - postId: 'root', - }) - - const [output] = JSON.parse(String(log.mock.calls.at(-1)?.[0])) - expect(output.retrieval.visibleThreads).toEqual({ - status: 'partial', - hydratedRootCount: 0, - failedRootIds: ['root'], - }) - }) -}) - -function makeThreadPost(id: string, rootId: string, createAt: number, replyCount = 0): Post { - return { - id, - channel_id: 'channel', - user_id: 'me', - create_at: createAt, - update_at: createAt, - delete_at: 0, - edit_at: 0, - message: id, - type: '', - props: {}, - hashtags: '', - file_ids: [], - root_id: rootId, - reply_count: replyCount, - pending_post_id: '', - } -} - -function responsePage(posts: Post[]): PostsResponse { - return { - order: posts.map(({ id }) => id), - posts: Object.fromEntries(posts.map((post) => [post.id, post])), - } as PostsResponse -} diff --git a/tests/config-doctor.test.ts b/tests/config-doctor.test.ts deleted file mode 100644 index 1f08619..0000000 --- a/tests/config-doctor.test.ts +++ /dev/null @@ -1,262 +0,0 @@ -import { chmod, mkdtemp, writeFile } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { describe, expect, it, vi } from 'vitest' -import { resolveConfigState } from '../src/config' -import { formatDoctorReport, runDoctor } from '../src/doctor' -import { preprocess } from '../src/preprocessing' - -async function configFile(content: string, mode = 0o600): Promise { - const directory = await mkdtemp(join(tmpdir(), 'mm-doctor-')) - const path = join(directory, 'config.toml') - await writeFile(path, content) - await chmod(path, mode) - return path -} - -describe('doctor config resolution', () => { - it('registers a config-file token at resolution before output validation', async () => { - const token = 'file-active-token' - const path = await configFile(`url = "https://file.example"\ntoken = "${token}"\n`) - await resolveConfigState({}, {}, path) - expect(preprocess(`invalid=${token}`, { redact: false }).text).toBe( - 'invalid=[REDACTED:mattermost_credential]', - ) - }) - - it('reports CLI > env > file sources without exposing values', async () => { - const path = await configFile('url = "https://file.example"\ntoken = "file-secret"\n') - const state = await resolveConfigState( - { url: 'https://cli.example' }, - { MM_URL: 'https://env.example', MM_TOKEN: 'env-secret' }, - path, - ) - - expect(state.urlSource).toBe('cli') - expect(state.tokenSource).toBe('env') - expect(state.url).toBe('https://cli.example') - expect(state.token).toBe('env-secret') - }) - - it('detects insecure config permissions', async () => { - const path = await configFile('token = "file-secret"\n', 0o644) - const state = await resolveConfigState({}, {}, path) - const report = await runDoctor(state) - - expect(report.checks[0]).toMatchObject({ status: 'fail' }) - expect(JSON.stringify(report)).not.toContain('file-secret') - }) - - it('fails insecure permissions when an environment token overrides a stored file token', async () => { - const path = await configFile('url = "https://file.example"\ntoken = "file-secret"\n', 0o644) - const state = await resolveConfigState({}, { MM_TOKEN: 'env-secret' }, path) - const fetcher = vi.fn(async () => Response.json({})) as unknown as typeof fetch - const report = await runDoctor(state, { fetcher }) - - expect(state).toMatchObject({ urlSource: 'file', tokenSource: 'env' }) - expect(report.checks[0]).toMatchObject({ status: 'fail' }) - expect(JSON.stringify(report)).not.toContain('file-secret') - expect(JSON.stringify(report)).not.toContain('env-secret') - }) - - it('fails closed on an unreadable or malformed config file', async () => { - const path = await configFile('not valid = [toml') - const state = await resolveConfigState({}, {}, path) - - expect(state.fileError).toBe('parse') - expect((await runDoctor(state)).checks[0]).toMatchObject({ status: 'fail' }) - }) - - it('distinguishes a missing file from a config read failure', async () => { - const directory = await mkdtemp(join(tmpdir(), 'mm-doctor-directory-')) - const readFailure = await resolveConfigState({}, {}, directory) - const missing = await resolveConfigState({}, {}, join(directory, 'missing.toml')) - - expect(readFailure).toMatchObject({ fileExists: true, fileError: 'read' }) - expect(missing).toMatchObject({ fileExists: false, fileError: undefined }) - }) - - it('only warns for insecure permissions when the file stores no token', async () => { - const path = await configFile('url = "https://file.example"\n', 0o644) - const state = await resolveConfigState({}, { MM_TOKEN: 'env-secret' }, path) - const fetcher = vi - .fn() - .mockResolvedValueOnce( - Response.json({ status: 'OK', database_status: 'OK', filestore_status: 'OK' }), - ) - .mockResolvedValueOnce( - Response.json({ id: 'user-id', username: 'arda' }), - ) as unknown as typeof fetch - - expect((await runDoctor(state, { fetcher })).checks[0]).toMatchObject({ status: 'warn' }) - }) - - it('fails incomplete config even when tokenless file permissions would otherwise warn', async () => { - const path = await configFile('url = "https://file.example"\n', 0o644) - const state = await resolveConfigState({}, {}, path) - const fetcher = vi.fn(async () => - Response.json({ status: 'OK', database_status: 'OK', filestore_status: 'OK' }), - ) as unknown as typeof fetch - const report = await runDoctor(state, { fetcher }) - - expect(report.ok).toBe(false) - expect(report.checks[0]).toMatchObject({ - status: 'fail', - message: 'configuration is incomplete', - }) - expect(report.checks[1]).toMatchObject({ status: 'pass' }) - expect(report.checks[2]).toMatchObject({ status: 'skipped' }) - }) -}) - -describe('doctor checks', () => { - it('runs ping without auth and skips authentication when the token is missing', async () => { - const fetcher = vi.fn(async () => - Response.json({ status: 'OK', database_status: 'OK' }), - ) as unknown as typeof fetch - const state = await resolveConfigState({ url: 'https://mm.example' }, {}, '/missing') - const report = await runDoctor(state, { fetcher }) - - expect(fetcher).toHaveBeenCalledOnce() - expect(fetcher).toHaveBeenCalledWith( - 'https://mm.example/api/v4/system/ping?get_server_status=true', - expect.objectContaining({ - method: 'GET', - headers: undefined, - signal: expect.any(AbortSignal), - }), - ) - expect(report.checks[0]).toMatchObject({ status: 'fail' }) - expect(report.checks[1]).toMatchObject({ status: 'warn' }) - expect(report.checks[1]?.details).toMatchObject({ filestoreStatus: 'unknown' }) - expect(report.checks[2]).toMatchObject({ status: 'skipped' }) - }) - - it('whitelists server and user fields', async () => { - const token = 'super-secret-token' - const fetcher = vi - .fn() - .mockResolvedValueOnce( - Response.json({ - status: 'OK', - database_status: 'OK', - filestore_status: 'OK', - secret: token, - }), - ) - .mockResolvedValueOnce( - Response.json({ id: 'user-id', username: 'arda', email: token, roles: token }), - ) as unknown as typeof fetch - const state = await resolveConfigState({ url: 'https://mm.example', token }, {}, '/missing') - const report = await runDoctor(state, { fetcher }) - const output = JSON.stringify(report) - - expect(report.ok).toBe(true) - expect(output).not.toContain(token) - expect(output).not.toContain('email') - expect(report.checks[2]?.details).toEqual({ id: 'user-id', username: 'arda' }) - }) - - it('sanitizes failures and does not abort before all checks are present', async () => { - const token = 'never-print-this' - const fetcher = vi.fn( - async () => new Response(`${token}\nremote body`, { status: 401 }), - ) as unknown as typeof fetch - const state = await resolveConfigState({ url: 'https://mm.example', token }, {}, '/missing') - const report = await runDoctor(state, { fetcher }) - const output = `${JSON.stringify(report)}\n${formatDoctorReport(report)}` - - expect(report.ok).toBe(false) - expect(report.checks).toHaveLength(3) - expect(output).not.toContain(token) - expect(output).not.toContain('remote body') - expect(report.checks[1]).toMatchObject({ status: 'fail', details: { httpStatus: 401 } }) - expect(report.checks[2]).toMatchObject({ status: 'fail', details: { httpStatus: 401 } }) - }) - - it('fails unhealthy server values and skips auth for unsafe URLs', async () => { - const unhealthyFetcher = vi.fn(async () => - Response.json({ status: 'OK', database_status: 'DOWN', filestore_status: 'OK' }), - ) as unknown as typeof fetch - const healthyConfig = await resolveConfigState({ url: 'https://mm.example' }, {}, '/missing') - const unhealthy = await runDoctor(healthyConfig, { fetcher: unhealthyFetcher }) - expect(unhealthy.checks[1]).toMatchObject({ status: 'fail' }) - - const unsafeConfig = await resolveConfigState( - { url: 'http://mm.example', token: 'secret' }, - {}, - '/missing', - ) - const neverFetch = vi.fn() as unknown as typeof fetch - const unsafe = await runDoctor(unsafeConfig, { fetcher: neverFetch }) - expect(neverFetch).not.toHaveBeenCalled() - expect(unsafe.checks[1]).toMatchObject({ status: 'fail' }) - expect(unsafe.checks[2]).toMatchObject({ status: 'skipped' }) - }) - - it('redacts configured tokens and sanitizes terminal controls in all remote strings', async () => { - const token = 'super-secret-token' - const osc = '\u001b]8;;https://evil.example\u0007click\u001b]8;;\u0007' - const boundaryProbe = '\u001bghp_abcdefghijklmnopqrstuvwxyz1234567890' - const state = await resolveConfigState({ url: 'https://mm.example', token }, {}, '/missing') - - for (const redact of [true, false]) { - const fetcher = vi - .fn() - .mockResolvedValueOnce( - Response.json({ - status: 'OK', - database_status: `${token}${osc}`, - filestore_status: 'OK', - }), - ) - .mockResolvedValueOnce( - Response.json({ id: `${token}${osc}`, username: `arda${osc}${boundaryProbe}` }), - ) as unknown as typeof fetch - const report = await runDoctor(state, { fetcher, redact }) - const output = `${JSON.stringify(report)}\n${formatDoctorReport(report)}` - expect(output).not.toContain(token) - expect(output).not.toContain('\u001b') - expect(output).not.toContain('\u0007') - expect(output).toContain('\\u001b') - expect(output).toContain('[REDACTED:token]') - if (redact) expect(output).not.toContain('ghp_abcdefghijklmnopqrstuvwxyz1234567890') - } - }) - - it('times out each request independently and still attempts authentication', async () => { - const fetcher = vi - .fn() - .mockImplementationOnce(() => new Promise(() => {})) - .mockResolvedValueOnce( - Response.json({ id: 'user-id', username: 'arda' }), - ) as unknown as typeof fetch - const state = await resolveConfigState( - { url: 'https://mm.example', token: 'token' }, - {}, - '/missing', - ) - const report = await runDoctor(state, { fetcher, timeoutMs: 5 }) - - expect(fetcher).toHaveBeenCalledTimes(2) - expect(report.checks[1]).toMatchObject({ status: 'fail' }) - expect(report.checks[2]).toMatchObject({ status: 'pass' }) - }) - - it('rejects empty authentication identity fields', async () => { - const fetcher = vi - .fn() - .mockResolvedValueOnce( - Response.json({ status: 'OK', database_status: 'OK', filestore_status: 'OK' }), - ) - .mockResolvedValueOnce(Response.json({ id: ' ', username: '' })) as unknown as typeof fetch - const state = await resolveConfigState( - { url: 'https://mm.example', token: 'token' }, - {}, - '/missing', - ) - const report = await runDoctor(state, { fetcher }) - - expect(report.checks[2]).toMatchObject({ status: 'fail' }) - }) -}) diff --git a/tests/cursor-cli.test.ts b/tests/cursor-cli.test.ts deleted file mode 100644 index 5b7402c..0000000 --- a/tests/cursor-cli.test.ts +++ /dev/null @@ -1,389 +0,0 @@ -import { afterEach, describe, expect, test, vi } from 'vitest' -import { clearUserCache } from '../src/api/users' -import { fetchChannel, fetchDMs, fetchGroupDMs } from '../src/cli' -import { decodeChannelHistoryCursor, encodeChannelHistoryCursor } from '../src/cursor' -import type { Channel, ChannelOptions, Post, PostsResponse } from '../src/types' -import { installRouteFetch } from './helpers/fake-fetch' - -const serverUrl = 'https://mattermost.test' -const me = { id: 'me', username: 'me' } -const team = { id: 'team', name: 'team', display_name: 'Team', type: 'O' } -const general = { - id: 'general', - team_id: 'team', - type: 'O', - name: 'general', - display_name: 'General', - header: '', - purpose: '', - last_post_at: 0, - total_msg_count: 0, - creator_id: 'me', -} satisfies Channel -const direct = { - ...general, - id: 'dm', - team_id: '', - type: 'D', - name: 'me__other', - display_name: '', -} satisfies Channel -const group = { - ...general, - id: 'group', - team_id: '', - type: 'G', - name: 'group', - display_name: 'Crew', -} satisfies Channel - -function post(id: string, createAt: number, overrides: Partial = {}): Post { - return { - id, - channel_id: 'general', - user_id: 'me', - create_at: createAt, - update_at: createAt, - delete_at: 0, - edit_at: 0, - message: id, - type: '', - props: {}, - hashtags: '', - file_ids: [], - root_id: '', - reply_count: 0, - pending_post_id: '', - ...overrides, - } -} - -function page(items: Post[], extra: Partial = {}): PostsResponse { - return { - order: items.map(({ id }) => id), - posts: Object.fromEntries(items.map((item) => [item.id, item])), - ...extra, - } as PostsResponse -} - -const options = (overrides: Partial = {}): ChannelOptions => ({ - url: serverUrl, - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: false, - channel: 'general', - limit: 2, - since: '', - ...overrides, -}) - -function routes(handlePosts: (url: URL) => PostsResponse, resolved = general) { - return [ - { method: 'GET', path: '/api/v4/users/me', handle: () => me }, - { method: 'GET', path: '/api/v4/users/me/teams', handle: () => [team] }, - { - method: 'GET', - path: '/api/v4/teams/team/channels/name/general', - handle: () => resolved, - }, - { - method: 'GET', - path: '/api/v4/channels/general/posts', - handle: ({ url }: { url: URL }) => handlePosts(url), - }, - ] -} - -function captureJSON() { - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) - return () => JSON.parse(String(log.mock.calls.at(-1)?.[0])) -} - -afterEach(() => { - clearUserCache() - vi.useRealTimers() - vi.restoreAllMocks() - vi.unstubAllGlobals() -}) - -describe('channel history cursor integration', () => { - test('pages across equal timestamps without gaps, duplicates, or newly arrived posts', async () => { - vi.useFakeTimers() - vi.setSystemTime(new Date('2026-07-14T12:00:00Z')) - installRouteFetch( - routes((url) => - url.searchParams.get('page') === '0' - ? page([ - post('newer', Date.now()), - post('peer-a', Date.now() - 1), - post('peer-b', Date.now() - 1), - ]) - : page([post('older', Date.now() - 2)]), - ), - ) - const firstOutput = captureJSON() - await fetchChannel(options({ since: '24h' })) - const first = firstOutput()[0] - const cursor = first.retrieval.selection.nextCursor as string - const decoded = decodeChannelHistoryCursor(cursor) - expect(first.retrieval.selection.selectedCount).toBe(2) - expect(decoded.since).toBe(Date.now() - 86_400_000) - expect(decoded.safeBeforePostId).toBe('newer') - - vi.restoreAllMocks() - const { requests } = installRouteFetch( - routes((url) => - url.searchParams.get('page') === '0' - ? page([ - post('arrived-after-page-one', Date.now() + 10), - post('peer-b', Date.now() - 1), - post('older', Date.now() - 2), - ]) - : page([]), - ), - ) - const secondOutput = captureJSON() - await fetchChannel(options({ cursor })) - const second = secondOutput()[0] - expect(second.messages.map(({ id }: { id: string }) => id).sort()).toEqual(['older', 'peer-b']) - expect(second.retrieval.selection.inputCursor).toBe(cursor) - expect(second.retrieval.selection.nextCursor).toBeNull() - expect(second.retrieval.selection.since).toBe('2026-07-13T12:00:00.000Z') - expect( - requests.find(({ url }) => url.pathname.endsWith('/posts'))?.url.searchParams.get('before'), - ).toBe('newer') - }) - - test('preserves an unknown empty resume as an unchanged retry cursor', async () => { - const cursor = encodeChannelHistoryCursor({ - v: 1, - scope: 'channel', - channelId: 'general', - boundary: { createAt: 100, id: 'anchor' }, - since: null, - }) - installRouteFetch(routes(() => page([], { first_inaccessible_post_time: 1 }))) - const output = captureJSON() - - await fetchChannel(options({ cursor })) - - expect(output()).toEqual([ - expect.objectContaining({ - messages: [], - retrieval: expect.objectContaining({ - visiblePostCount: 0, - selection: expect.objectContaining({ - selectedCount: 0, - queryTruncated: null, - inputCursor: cursor, - nextCursor: cursor, - }), - }), - }), - ]) - }) - - test('keeps a next cursor for a nonempty result with unknown completeness', async () => { - installRouteFetch( - routes(() => page([post('visible', 100)], { first_inaccessible_post_time: 1 })), - ) - const output = captureJSON() - - await fetchChannel(options()) - - const selection = output()[0].retrieval.selection - expect(selection).toMatchObject({ selectedCount: 1, queryTruncated: null, inputCursor: null }) - expect(selection.nextCursor).toEqual(expect.any(String)) - }) - - test('drops a dead safe anchor from the regenerated cursor and the following resume', async () => { - const inputCursor = encodeChannelHistoryCursor({ - v: 1, - scope: 'channel', - channelId: 'general', - boundary: { createAt: 200, id: 'anchor' }, - since: null, - safeBeforePostId: 'deleted-safe-anchor', - }) - const firstRequests = installRouteFetch( - routes((url) => - url.searchParams.has('before') - ? page([]) - : page([post('peer-b', 200), post('peer-c', 200), post('older', 100)]), - ), - ).requests - const firstOutput = captureJSON() - - await fetchChannel(options({ cursor: inputCursor, limit: 1 })) - - const regenerated = firstOutput()[0].retrieval.selection.nextCursor as string - expect(decodeChannelHistoryCursor(regenerated).safeBeforePostId).toBeUndefined() - expect( - firstRequests - .filter(({ url }) => url.pathname.endsWith('/posts')) - .map(({ url }) => url.searchParams.get('before')), - ).toEqual(['deleted-safe-anchor', null]) - - vi.restoreAllMocks() - const secondRequests = installRouteFetch( - routes(() => page([post('peer-c', 200), post('older', 100)])), - ).requests - const secondOutput = captureJSON() - - await fetchChannel(options({ cursor: regenerated, limit: 1 })) - - expect( - secondRequests - .filter(({ url }) => url.pathname.endsWith('/posts')) - .every(({ url }) => !url.searchParams.has('before')), - ).toBe(true) - expect(secondOutput()[0].messages[0].id).toBe('peer-c') - }) - - test('selects the cursor page before hydrating threads', async () => { - const root = post('root', 300, { reply_count: 1 }) - const sibling = post('sibling', 200) - const reply = post('reply', 400, { root_id: 'root' }) - const { requests } = installRouteFetch([ - ...routes(() => page([root, sibling])), - { - method: 'GET', - path: '/api/v4/posts/root/thread', - handle: () => page([root, reply], { has_next: false }), - }, - ]) - const output = captureJSON() - - await fetchChannel(options({ limit: 1, threads: true })) - - const result = output()[0] - expect(result.retrieval.selection).toMatchObject({ selectedCount: 1, queryTruncated: true }) - expect(decodeChannelHistoryCursor(result.retrieval.selection.nextCursor).boundary.id).toBe( - 'root', - ) - expect(result.retrieval.visiblePostCount).toBe(2) - expect(JSON.stringify(result.messages)).toContain('reply') - expect(JSON.stringify(result.messages)).not.toContain('sibling') - expect(requests.filter(({ url }) => url.pathname.includes('/thread'))).toHaveLength(1) - }) - - test.each([ - ['direct message', direct, fetchDMs], - ['group DM', group, fetchGroupDMs], - ] as const)('resumes an explicit %s channel', async (_label, conversation, run) => { - const cursor = encodeChannelHistoryCursor({ - v: 1, - scope: 'channel', - channelId: conversation.id, - boundary: { createAt: 200, id: 'anchor' }, - since: null, - }) - const conversationPosts = [ - post('peer', 200, { channel_id: conversation.id }), - post('older', 100, { channel_id: conversation.id }), - ] - const extraRoutes = - conversation.type === 'D' - ? [ - { - method: 'GET', - path: '/api/v4/users/other', - handle: () => ({ id: 'other', username: 'other' }), - }, - ] - : [] - const { requests } = installRouteFetch([ - { method: 'GET', path: `/api/v4/channels/${conversation.id}`, handle: () => conversation }, - { method: 'GET', path: '/api/v4/users/me', handle: () => me }, - ...extraRoutes, - { - method: 'GET', - path: `/api/v4/channels/${conversation.id}/posts`, - handle: () => page(conversationPosts), - }, - ]) - const output = captureJSON() - - await run({ ...options(), user: [], channel: conversation.id, cursor }) - - expect(output()[0].retrieval.selection).toMatchObject({ - selectedCount: 2, - queryTruncated: false, - inputCursor: cursor, - nextCursor: null, - }) - expect(requests.some(({ url }) => url.pathname.endsWith(`/${conversation.id}/posts`))).toBe( - true, - ) - }) - - test.each([ - ['direct message', direct, fetchDMs], - ['group DM', group, fetchGroupDMs], - ] as const)('rejects an explicit %s cursor mismatch before post history', async (_label, conversation, run) => { - const cursor = encodeChannelHistoryCursor({ - v: 1, - scope: 'channel', - channelId: 'other-channel', - boundary: { createAt: 200, id: 'anchor' }, - since: null, - }) - const { requests } = installRouteFetch([ - { method: 'GET', path: `/api/v4/channels/${conversation.id}`, handle: () => conversation }, - { method: 'GET', path: '/api/v4/users/me', handle: () => me }, - ]) - - const invocation = run({ ...options(), user: [], channel: conversation.id, cursor }) - await expect(invocation).rejects.toThrow('Cursor does not match the selected channel.') - expect(requests.some(({ url }) => url.pathname.endsWith('/posts'))).toBe(false) - }) - - test('rejects mismatch before requesting posts', async () => { - const cursor = encodeChannelHistoryCursor({ - v: 1, - scope: 'channel', - channelId: 'other', - boundary: { createAt: 100, id: 'anchor' }, - since: null, - }) - const { requests } = installRouteFetch(routes(() => page([]))) - await expect(fetchChannel(options({ cursor }))).rejects.toThrow( - 'Cursor does not match the selected channel.', - ) - expect(requests.some(({ url }) => url.pathname.endsWith('/posts'))).toBe(false) - }) - - test('rejects an empty cursor before requesting posts', async () => { - const { requests } = installRouteFetch(routes(() => page([]))) - await expect(fetchChannel(options({ cursor: '' }))).rejects.toThrow('Invalid cursor.') - expect(requests).toHaveLength(0) - }) - - test('rejects malformed, unsupported, and explicit-since combinations before fetching', async () => { - const fake = vi.fn() - vi.stubGlobal('fetch', fake) - const validCursor = encodeChannelHistoryCursor({ - v: 1, - scope: 'channel', - channelId: 'general', - boundary: { createAt: 100, id: 'anchor' }, - since: null, - }) - await expect( - fetchChannel(options({ cursor: validCursor, sinceExplicit: true })), - ).rejects.toThrow('A cursor cannot be combined with --since.') - await expect( - fetchDMs({ ...options(), user: [], channel: undefined, cursor: validCursor }), - ).rejects.toThrow('A cursor requires --channel') - await expect( - fetchDMs({ ...options(), user: ['alice'], channel: 'dm', cursor: validCursor }), - ).rejects.toThrow('A cursor cannot be combined with --user.') - await expect( - fetchGroupDMs({ ...options(), channel: undefined, cursor: validCursor }), - ).rejects.toThrow('A cursor requires --channel') - expect(fake).not.toHaveBeenCalled() - }) -}) diff --git a/tests/cursor-commander.test.ts b/tests/cursor-commander.test.ts deleted file mode 100644 index c99201c..0000000 --- a/tests/cursor-commander.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { spawnSync } from 'node:child_process' -import { describe, expect, test } from 'vitest' -import { encodeChannelHistoryCursor } from '../src/cursor' - -const validCursor = encodeChannelHistoryCursor({ - v: 1, - scope: 'channel', - channelId: 'general', - boundary: { createAt: 100, id: 'anchor' }, - since: null, -}) - -function run(args: string[]) { - return spawnSync( - 'bun', - ['src/index.ts', '--url', 'http://127.0.0.1:1', '--token', 'test-token', ...args], - { encoding: 'utf8', timeout: 5_000 }, - ) -} - -describe('Commander cursor validation', () => { - test('rejects a malformed cursor before any network failure can occur', () => { - const result = run(['channel', 'general', '--cursor', '']) - - expect(result.status).toBe(1) - expect(result.stderr).toContain('Error: Invalid cursor.') - expect(result.stderr).not.toContain('fetch failed') - }) - - test('distinguishes explicit --since from its Commander default', () => { - const explicit = run(['channel', 'general', '--cursor', validCursor, '--since', '7d']) - expect(explicit.status).toBe(1) - expect(explicit.stderr).toContain('A cursor cannot be combined with --since.') - expect(explicit.stderr).not.toContain('fetch failed') - - const defaulted = run(['channel', 'general', '--cursor', validCursor]) - expect(defaulted.status).toBe(1) - expect(defaulted.stderr).not.toContain('A cursor cannot be combined with --since.') - expect(defaulted.stderr).not.toContain('Invalid cursor.') - expect(defaulted.stderr).toContain('Unable to connect') - }) -}) diff --git a/tests/cursor.test.ts b/tests/cursor.test.ts deleted file mode 100644 index f37ccba..0000000 --- a/tests/cursor.test.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { describe, expect, test } from 'vitest' -import { - comparePostIds, - decodeChannelHistoryCursor, - encodeChannelHistoryCursor, -} from '../src/cursor' - -const cursor = { - v: 1 as const, - scope: 'channel' as const, - channelId: 'channel', - boundary: { createAt: 123, id: 'post' }, - since: 10, -} - -describe('channel history cursors', () => { - test('round trips as bounded base64url', () => { - const encoded = encodeChannelHistoryCursor(cursor) - expect(encoded).toMatch(/^[A-Za-z0-9_-]+$/) - expect(decodeChannelHistoryCursor(encoded)).toEqual(cursor) - }) - - test.each(['', 'not json', 'e30=', 'a'.repeat(2049)])('rejects malformed cursor %j', (value) => { - expect(() => decodeChannelHistoryCursor(value)).toThrow('Invalid cursor.') - }) - - test('rejects unsupported versions and invalid fields', () => { - const encode = (value: unknown) => Buffer.from(JSON.stringify(value)).toString('base64url') - expect(() => decodeChannelHistoryCursor(encode({ ...cursor, v: 2 }))).toThrow('Invalid cursor.') - expect(() => - decodeChannelHistoryCursor(encode({ ...cursor, boundary: { createAt: -1, id: '' } })), - ).toThrow('Invalid cursor.') - expect(() => decodeChannelHistoryCursor(encode({ ...cursor, extra: true }))).toThrow( - 'Invalid cursor.', - ) - expect(() => - decodeChannelHistoryCursor( - encode({ ...cursor, boundary: { ...cursor.boundary, extra: true } }), - ), - ).toThrow('Invalid cursor.') - expect(() => decodeChannelHistoryCursor(encode({ ...cursor, since: 124 }))).toThrow( - 'Invalid cursor.', - ) - expect(() => encodeChannelHistoryCursor({ ...cursor, channelId: 'not safe!' })).toThrow( - 'Invalid cursor.', - ) - }) - - test('uses one deterministic ASCII ID ordering', () => { - expect(['z', 'A', '_', '-'].sort(comparePostIds)).toEqual(['-', 'A', '_', 'z']) - }) -}) diff --git a/tests/e2e/send-live.e2e.ts b/tests/e2e/send-live.e2e.ts deleted file mode 100644 index 0148054..0000000 --- a/tests/e2e/send-live.e2e.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { spawnSync } from 'node:child_process' -import { describe, expect, test } from 'vitest' -import { MAX_MESSAGE_CHARACTERS } from '../../src/input' -import { LONG_MARKDOWN, LONG_MARKDOWN_CHARACTERS, SHORT_MARKDOWN } from '../fixtures/markdown' - -const url = process.env.MM_E2E_URL -const token = process.env.MM_E2E_TOKEN -if (!url || !token) throw new Error('Disposable Mattermost E2E credentials are required.') -const parsedUrl = new URL(url) -if (parsedUrl.hostname !== '127.0.0.1' && parsedUrl.hostname !== 'localhost') { - throw new Error('Refusing to run message-write E2E against a non-loopback Mattermost server.') -} - -interface Receipt { - status: 'dry_run' | 'sent' - destination: { - type: 'dm' | 'group' - label: string - channelId: string | null - willCreate: boolean - } - post?: { id: string; createAt: string; pendingPostId: string; permalink: string } -} - -async function api(path: string, init: RequestInit = {}): Promise { - const response = await fetch(`${url}/api/v4${path}`, { - ...init, - headers: { - Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json', - ...init.headers, - }, - }) - if (!response.ok) throw new Error(`Mattermost E2E API request failed: ${response.status}.`) - return (await response.json()) as T -} - -function cli(args: string[], input?: string): Receipt { - const result = spawnSync('node', ['dist/index.js', '--json', ...args], { - cwd: process.cwd(), - encoding: 'utf8', - env: { ...process.env, MM_URL: url, MM_TOKEN: token }, - input, - }) - if (result.status !== 0) { - throw new Error(`CLI failed (${result.status}): ${result.stderr}`) - } - return JSON.parse(result.stdout) as Receipt -} - -describe.sequential('disposable Mattermost message sending', () => { - test('dry-runs and sends short Markdown exactly once to a direct message', async () => { - const dryRun = cli(['send', 'dm', 'alice', '--dry-run']) - expect(dryRun).toEqual({ - status: 'dry_run', - destination: { - type: 'dm', - label: '@alice', - channelId: null, - willCreate: true, - }, - }) - - const message = `${SHORT_MARKDOWN}\n\n` - const sent = cli(['send', 'dm', 'alice'], message) - expect(sent.status).toBe('sent') - expect(sent.destination).toMatchObject({ - type: 'dm', - label: '@alice', - willCreate: false, - }) - expect(sent.post?.permalink).toBe(`${url}/_redirect/pl/${sent.post?.id}`) - expect(sent.post?.pendingPostId).toMatch(/^[0-9a-f-]{36}$/) - - const post = await api<{ - id: string - channel_id: string - user_id: string - message: string - }>(`/posts/${sent.post?.id}`) - const me = await api<{ id: string }>('/users/me') - expect(post).toMatchObject({ - id: sent.post?.id, - channel_id: sent.destination.channelId, - user_id: me.id, - message, - }) - - const page = await api<{ order: string[]; posts: Record }>( - `/channels/${sent.destination.channelId}/posts?per_page=200`, - ) - expect(page.order.filter((id) => page.posts[id]?.message === message)).toHaveLength(1) - }) - - test('dry-runs and sends long Markdown exactly once to an existing group conversation', async () => { - const users = await Promise.all( - ['sender', 'alice', 'bob'].map((username) => - api<{ id: string }>(`/users/username/${username}`), - ), - ) - const group = await api<{ id: string }>('/channels/group', { - method: 'POST', - body: JSON.stringify(users.map((user) => user.id)), - }) - - const dryRun = cli(['send', 'group', group.id, '--dry-run']) - expect(dryRun).toMatchObject({ - status: 'dry_run', - destination: { type: 'group', channelId: group.id, willCreate: false }, - }) - const before = await api<{ order: string[] }>(`/channels/${group.id}/posts?per_page=200`) - - const message = `${LONG_MARKDOWN}\n\n` - expect([...message].length).toBeGreaterThan(LONG_MARKDOWN_CHARACTERS) - expect([...message].length).toBeLessThan(MAX_MESSAGE_CHARACTERS) - const sent = cli(['send', 'group', group.id], message) - expect(sent).toMatchObject({ - status: 'sent', - destination: { type: 'group', channelId: group.id, willCreate: false }, - }) - - const post = await api<{ message: string; channel_id: string }>(`/posts/${sent.post?.id}`) - expect(post).toMatchObject({ channel_id: group.id, message }) - const after = await api<{ order: string[]; posts: Record }>( - `/channels/${group.id}/posts?per_page=200`, - ) - expect(after.order).toHaveLength(before.order.length + 1) - expect(after.order.filter((id) => after.posts[id]?.message === message)).toHaveLength(1) - }) -}) diff --git a/tests/fixtures/markdown.ts b/tests/fixtures/markdown.ts deleted file mode 100644 index f5f863a..0000000 --- a/tests/fixtures/markdown.ts +++ /dev/null @@ -1,55 +0,0 @@ -export const SHORT_MARKDOWN = `# Release ready ✅ - -**Status:** shipped with [runbook](https://example.test/runbook). - -- [x] API healthy -- [x] migrations applied - -> Verify the canary before broad rollout. - -\`\`\`ts -const ready = true -\`\`\` -` - -function buildLongMarkdown(): string { - let message = `# Extended deployment report 🌍 - -This intentionally large fixture exercises structured Markdown without relying on generated prose. -` - let section = 1 - while ([...message].length < 15_500) { - message += ` -## Service ${section} - -| Check | Result | Detail | -| --- | --- | --- | -| health | ✅ | [probe](https://example.test/health/${section}) | -| queue | ✅ | **drained** | - -- [x] deploy completed -- [x] metrics reviewed -- [ ] observe for 30 minutes - -> Service ${section} remained inside its latency budget. - -\`\`\`json -{"service":${section},"status":"healthy","regions":["eu","us"],"rollback":false} -\`\`\` -` - section += 1 - } - return `${message} ---- - -End of report. _Keep this final newline._ -` -} - -export const LONG_MARKDOWN = buildLongMarkdown() -export const LONG_MARKDOWN_BYTES = new TextEncoder().encode(LONG_MARKDOWN).byteLength -export const LONG_MARKDOWN_CHARACTERS = [...LONG_MARKDOWN].length - -if (LONG_MARKDOWN_CHARACTERS < 15_500 || LONG_MARKDOWN_CHARACTERS >= 16_200) { - throw new Error('Long Markdown fixture escaped its intended character range.') -} diff --git a/tests/formatters/headers.test.ts b/tests/formatters/headers.test.ts deleted file mode 100644 index d3cecf9..0000000 --- a/tests/formatters/headers.test.ts +++ /dev/null @@ -1,281 +0,0 @@ -import { describe, expect, test } from 'vitest' -import { formatMarkdown } from '../../src/formatters/markdown' -import { formatPretty } from '../../src/formatters/pretty' -import type { MessageOutput, ProcessedChannel } from '../../src/types' - -function makeOutput(channel: ProcessedChannel): MessageOutput { - return { - channel, - messages: [ - { - id: 'msg1', - permalink: 'https://mattermost.example.com/_redirect/pl/msg1', - user: 'alice', - userId: 'u1', - text: 'hello', - timestamp: new Date('2026-02-21T10:00:00Z'), - updatedAt: new Date('2026-02-21T10:00:00Z'), - isDeleted: false, - postType: '', - isSystem: false, - isPinned: false, - files: [], - fileDetails: [], - attachments: [], - reactions: [], - }, - ], - redactions: [], - retrieval: { - selection: { - source: 'recent', - selectedCount: 1, - requestedLimit: 1, - since: null, - queryTruncated: true, - inputCursor: null, - nextCursor: null, - }, - visibleThreads: { status: 'not_requested', hydratedRootCount: 0, failedRootIds: [] }, - visiblePostCount: 1, - deletedPostsIncluded: false, - }, - } -} - -function withTruncation(output: MessageOutput, queryTruncated: boolean | null): MessageOutput { - return { - ...output, - retrieval: { - ...output.retrieval, - selection: { ...output.retrieval.selection, queryTruncated }, - }, - } -} - -function withNextCursor(output: MessageOutput, nextCursor: string | null): MessageOutput { - return { - ...output, - retrieval: { - ...output.retrieval, - selection: { ...output.retrieval.selection, nextCursor }, - }, - } -} - -describe('formatter channel headers', () => { - const dmChannel: ProcessedChannel = { - id: 'ch1', - type: 'dm', - name: '@bob', - metadataStatus: 'resolved', - } - const publicChannel: ProcessedChannel = { - id: 'ch2', - type: 'public', - name: 'general', - displayName: 'General', - metadataStatus: 'resolved', - } - const privateChannel: ProcessedChannel = { - id: 'ch3', - type: 'private', - name: 'secret-stuff', - metadataStatus: 'resolved', - } - const groupChannel: ProcessedChannel = { - id: 'ch4', - type: 'group', - name: 'Design Crew', - metadataStatus: 'resolved', - } - const unknownChannel: ProcessedChannel = { - id: 'channelid', - type: 'unknown', - name: 'unknown', - metadataStatus: 'unavailable', - } - - describe('pretty formatter (no color)', () => { - test('DM header shows "DMs with @user"', () => { - const output = formatPretty([makeOutput(dmChannel)], { color: false }) - expect(output).toContain('DMs with @bob') - }) - - test('public channel header shows "#channel (DisplayName)"', () => { - const output = formatPretty([makeOutput(publicChannel)], { color: false }) - expect(output).toContain('#general (General)') - }) - - test('private channel header shows "#channel" without display name', () => { - const output = formatPretty([makeOutput(privateChannel)], { color: false }) - expect(output).toContain('#secret-stuff') - expect(output).not.toContain('undefined') - }) - - test('group DM header shows its display label without a hash', () => { - const output = formatPretty([makeOutput(groupChannel)], { color: false }) - expect(output).toContain('Group DM: Design Crew') - expect(output).not.toContain('#Design Crew') - }) - - test('unresolved channel header exposes the stable ID without inventing a type', () => { - const output = formatPretty([makeOutput(unknownChannel)], { color: false }) - expect(output).toContain('Unknown channel (channelid)') - expect(output).not.toContain('#unknown') - }) - - test('shows a compact post id, permalink, and coverage warning', () => { - const output = formatPretty([makeOutput(publicChannel)], { color: false }) - expect(output).toContain('msg1 https://mattermost.example.com/_redirect/pl/msg1') - expect(output).toContain('Coverage: 1 selected, 1 visible; query truncated') - }) - - test('prints only a present next cursor on the final line', () => { - expect(formatPretty([makeOutput(publicChannel)], { color: false })).not.toContain( - 'Next cursor:', - ) - const output = formatPretty([withNextCursor(makeOutput(publicChannel), 'opaque_123')], { - color: false, - }) - expect(output.trimEnd().endsWith('Next cursor: opaque_123')).toBe(true) - }) - - test.each([ - [false, 'query complete'], - [null, 'query completeness unknown'], - ] as const)('reports %j query coverage honestly', (state, wording) => { - const output = formatPretty([withTruncation(makeOutput(publicChannel), state)], { - color: false, - }) - expect(output).toContain(wording) - }) - - test('shows current-state markers, file metadata, attachments, and reactions', () => { - const fixture = makeOutput(publicChannel) - const message = fixture.messages[0] - if (!message) throw new Error('missing formatter fixture message') - message.editedAt = new Date('2026-02-21T10:01:00Z') - message.isPinned = true - message.fileDetails = [{ id: 'file1', name: 'report.txt' }] - message.files = ['file1'] - message.attachments = [ - { title: 'Deploy', titleLink: 'https://example.test/deploy', text: 'Passed' }, - ] - message.reactions = [ - { emoji: 'white_check_mark', count: 1, actors: [{ id: 'u2', username: 'bob' }] }, - ] - - const output = formatPretty([fixture], { color: false }) - expect(output).toContain('[edited] [pinned]') - expect(output).toContain('report.txt (file1)') - expect(output).toContain('Attachment: Deploy') - expect(output).toContain(':white_check_mark: 1 (bob)') - }) - }) - - describe('markdown formatter', () => { - test('prints only a present next cursor on the final line', () => { - expect(formatMarkdown([makeOutput(publicChannel)])).not.toContain('Next cursor:') - const output = formatMarkdown([withNextCursor(makeOutput(publicChannel), 'opaque_123')]) - expect(output.trimEnd().endsWith('Next cursor: `opaque_123`')).toBe(true) - }) - test('DM header uses ## DMs with @user', () => { - const output = formatMarkdown([makeOutput(dmChannel)]) - expect(output).toContain('## DMs with @bob') - }) - - test('public channel header uses ## #channel (DisplayName)', () => { - const output = formatMarkdown([makeOutput(publicChannel)]) - expect(output).toContain('## #general (General)') - }) - - test('private channel header omits display name when absent', () => { - const output = formatMarkdown([makeOutput(privateChannel)]) - expect(output).toContain('## #secret\\-stuff') - expect(output).not.toContain('undefined') - }) - - test('unresolved channel header exposes the stable ID without inventing a type', () => { - const output = formatMarkdown([makeOutput(unknownChannel)]) - expect(output).toContain('## Unknown channel (channelid)') - expect(output).not.toContain('## #unknown') - }) - - test('links the stable post id and reports coverage', () => { - const output = formatMarkdown([makeOutput(publicChannel)]) - expect(output).toContain('[msg1]()') - expect(output).toContain('Coverage: 1 selected, 1 visible; query truncated') - }) - - test('keeps a permalink with parentheses parseable as one markdown destination', () => { - const fixture = makeOutput(publicChannel) - const message = fixture.messages[0] - if (!message) throw new Error('missing formatter fixture message') - message.permalink = 'https://mattermost.example.com/chat)/_redirect/pl/post%28special%29' - expect(formatMarkdown([fixture])).toContain( - '[msg1]()', - ) - }) - - test.each([ - [false, 'query complete'], - [null, 'query completeness unknown'], - ] as const)('reports %j query coverage honestly', (state, wording) => { - const output = formatMarkdown([withTruncation(makeOutput(publicChannel), state)]) - expect(output).toContain(wording) - }) - - test('uses safe markdown link destinations for attachment links', () => { - const fixture = makeOutput(publicChannel) - const message = fixture.messages[0] - if (!message) throw new Error('missing formatter fixture message') - message.isSystem = true - message.postType = 'system_webhook' - message.attachments = [{ title: 'Deploy', titleLink: 'https://example.test/a>b\\c' }] - message.reactions = [{ emoji: 'eyes', count: 1, actors: [{ id: 'u2' }] }] - - const output = formatMarkdown([fixture]) - expect(output).toContain('[system:system\\_webhook]') - expect(output).toContain('[Deploy]()') - expect(output).toContain(':eyes: 1 (u2)') - }) - - test('escapes remote markdown, quotes multiline fields, and rejects unsafe links', () => { - const fixture = makeOutput(publicChannel) - const message = fixture.messages[0] - if (!message) throw new Error('missing formatter fixture message') - message.user = '[admin](https://evil.test)' - message.text = 'one\ntwo *bold*' - message.attachments = [ - { - title: '[click](https://evil.test)', - titleLink: 'javascript:alert(1)', - text: 'first\nsecond', - }, - ] - - const output = formatMarkdown([fixture]) - expect(output).toContain('**\\[admin\\]\\(https://evil\\.test\\)**') - expect(output).toContain('> one\n> two \\*bold\\*') - expect(output).toContain('> first\n> second') - expect(output).not.toContain('javascript:') - expect(output).not.toContain('[click](https://evil.test)') - }) - - test('neutralizes block and table markdown from remote text', () => { - const fixture = makeOutput(publicChannel) - const message = fixture.messages[0] - if (!message) throw new Error('missing formatter fixture message') - message.text = '# heading\n---\n- item\n+ item\n~~strike~~\na | b' - - const output = formatMarkdown([fixture]) - expect(output).toContain('> \\# heading') - expect(output).toContain('> \\-\\-\\-') - expect(output).toContain('> \\- item') - expect(output).toContain('> \\+ item') - expect(output).toContain('> \\~\\~strike\\~\\~') - expect(output).toContain('> a \\| b') - }) - }) -}) diff --git a/tests/formatters/watch.test.ts b/tests/formatters/watch.test.ts deleted file mode 100644 index 5ea2376..0000000 --- a/tests/formatters/watch.test.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { describe, expect, test } from 'vitest' -import { formatWatchEvent, formatWatchJSON } from '../../src/formatters' -import type { WatchEvent } from '../../src/types' - -const event: WatchEvent = { - type: 'posted', - postId: 'post-1', - channelId: 'channel-1', - channelName: 'town-square', - sender: 'arda\\nadmin', - senderId: 'user-1', - message: 'hello\nworld sk-****-7890', - timestamp: '2026-07-14T12:34:00.000Z', - fileIds: [], - redactions: [{ type: 'OpenAI API Key', masked: 'sk-****-7890', position: 12 }], -} - -describe('watch formatters', () => { - test('emits one stable JSON object per line without diagnostics', () => { - const line = formatWatchJSON(event) - expect(line).not.toContain('\n') - expect(JSON.parse(line)).toEqual(event) - }) - - test('renders sanitized human output on one line', () => { - expect(formatWatchEvent(event, false)).toMatch( - /^\[\d{2}:\d{2}\] arda\\nadmin: hello world sk-\*\*\*\*-7890$/, - ) - }) -}) diff --git a/tests/group-dms.test.ts b/tests/group-dms.test.ts deleted file mode 100644 index add5c2e..0000000 --- a/tests/group-dms.test.ts +++ /dev/null @@ -1,285 +0,0 @@ -import { afterEach, describe, expect, test, vi } from 'vitest' -import { clearUserCache } from '../src/api/users' -import { fetchDMs, fetchGroupDMs } from '../src/cli' -import type { Channel, DMsOptions, GroupDMsOptions, Post, User } from '../src/types' -import { installRouteFetch } from './helpers/fake-fetch' - -const me = { id: 'me', username: 'me' } as User -const alice = { id: 'alice', username: 'alice' } as User - -function group(id: string, displayName = ''): Channel { - return { - id, - team_id: '', - type: 'G', - name: `${id}-internal`, - display_name: displayName, - } as Channel -} - -function post(id: string, channelId: string, createAt: number): Post { - return { - id, - channel_id: channelId, - user_id: 'alice', - create_at: createAt, - update_at: createAt, - delete_at: 0, - edit_at: 0, - message: id, - type: '', - props: {}, - hashtags: '', - file_ids: [], - root_id: '', - reply_count: 0, - pending_post_id: '', - } -} - -function page(posts: Post[]) { - return { - order: posts.map(({ id }) => id), - posts: Object.fromEntries(posts.map((item) => [item.id, item])), - has_next: false, - } -} - -function options(overrides: Partial = {}): GroupDMsOptions { - return { - url: 'https://mattermost.test', - token: 'token', - json: true, - color: false, - relative: false, - redact: true, - threads: false, - limit: 50, - since: '7d', - ...overrides, - } -} - -function userBatchRoute() { - return { method: 'POST', path: '/api/v4/users/ids', handle: () => [alice] } -} - -afterEach(() => { - clearUserCache() - vi.restoreAllMocks() - vi.unstubAllGlobals() -}) - -describe('group DM retrieval', () => { - test('discovers only group channels and labels display names without a channel prefix', async () => { - const channel = group('group-one', 'Launch