From c2263ff3c91b5d9e5e8fcd638b754fa1fb09a33e Mon Sep 17 00:00:00 2001 From: Armando Anaya Date: Tue, 28 Jul 2026 18:49:41 -0700 Subject: [PATCH] =?UTF-8?q?feat(cli):=20flow=20commands=20=E2=80=94=20the?= =?UTF-8?q?=20whole=20cycle=20without=20touching=20Python=20(#34)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Twenty new commands take the CLI from four to the full cycle: project create / list, schema apply / list, ingest, batch approve / start / complete / promote, job list / next / progress / start / mark / complete, release publish / list / verify, export, format list and backfill-thumbnails. Every one calls the SDK in-process — no HTTP hop and no token dance — and every one takes --json. Also lands `visionset init`, closing #104: the first acceptance criterion here is a scripted shell test driving the cycle via CLI only, and without a command-line way to create a workspace that is unsatisfiable. examples/cli_end_to_end.sh walks init → export using nothing but `visionset`, asserts the --json shapes and one deliberate refusal, and runs twice in CI. It needs no ffmpeg, no jq and no server. Two kernel reads, no migration: ProjectService.get_by_name (case-insensitive, like the index) and ReleaseService.get_by_tag (case-sensitive, like the index). They live there because the two rules are opposites and a surface re-deriving either from prose would eventually get one wrong. --json shapes agree key-for-key with the REST wire models, gated by tests/cli/test_json_contract.py over fifteen resources — a test may import both packages where the packages themselves may not. --- .github/workflows/ci.yml | 7 + README.md | 27 +- docs/README.md | 2 +- docs/batches.md | 30 ++ docs/cli.md | 303 ++++++++++++++++- docs/datasets.md | 5 + docs/examples.md | 60 +++- docs/ingest.md | 39 +++ docs/jobs.md | 29 ++ docs/projects.md | 25 ++ docs/releases.md | 36 ++ docs/schemas.md | 43 +++ docs/workspaces.md | 24 ++ examples/README.md | 25 +- examples/cli_end_to_end.sh | 151 +++++++++ src/visionset/cli/_errors.py | 16 + src/visionset/cli/_json.py | 307 ++++++++++++++++++ src/visionset/cli/_output.py | 105 ++++++ src/visionset/cli/_resolve.py | 78 +++++ src/visionset/cli/batches.py | 191 +++++++++++ src/visionset/cli/export.py | 86 +++++ src/visionset/cli/formats.py | 44 +++ src/visionset/cli/ingest.py | 179 ++++++++++ src/visionset/cli/init.py | 62 ++++ src/visionset/cli/jobs.py | 190 +++++++++++ src/visionset/cli/main.py | 38 ++- src/visionset/cli/projects.py | 72 ++++ src/visionset/cli/releases.py | 175 ++++++++++ src/visionset/cli/schemas.py | 145 +++++++++ src/visionset/cli/tokens.py | 44 +-- .../kernel/services/project_service.py | 38 +++ .../kernel/services/release_service.py | 22 ++ tests/cli/_flow.py | 157 +++++++++ tests/cli/test_batch_commands.py | 190 +++++++++++ tests/cli/test_export_commands.py | 197 +++++++++++ tests/cli/test_full_cycle.py | 130 ++++++++ tests/cli/test_ingest_commands.py | 178 ++++++++++ tests/cli/test_init.py | 120 +++++++ tests/cli/test_job_commands.py | 179 ++++++++++ tests/cli/test_json_contract.py | 145 +++++++++ tests/cli/test_output.py | 52 +++ tests/cli/test_project_commands.py | 130 ++++++++ tests/cli/test_release_commands.py | 199 ++++++++++++ tests/cli/test_schema_commands.py | 159 +++++++++ tests/examples/test_cli_end_to_end.py | 113 +++++++ tests/fixtures/samples.py | 177 ++++++++++ tests/kernel/test_project_service.py | 53 +++ tests/kernel/test_release_service.py | 55 ++++ 48 files changed, 4771 insertions(+), 61 deletions(-) create mode 100755 examples/cli_end_to_end.sh create mode 100644 src/visionset/cli/_json.py create mode 100644 src/visionset/cli/_output.py create mode 100644 src/visionset/cli/_resolve.py create mode 100644 src/visionset/cli/batches.py create mode 100644 src/visionset/cli/export.py create mode 100644 src/visionset/cli/formats.py create mode 100644 src/visionset/cli/ingest.py create mode 100644 src/visionset/cli/init.py create mode 100644 src/visionset/cli/jobs.py create mode 100644 src/visionset/cli/projects.py create mode 100644 src/visionset/cli/releases.py create mode 100644 src/visionset/cli/schemas.py create mode 100644 tests/cli/_flow.py create mode 100644 tests/cli/test_batch_commands.py create mode 100644 tests/cli/test_export_commands.py create mode 100644 tests/cli/test_full_cycle.py create mode 100644 tests/cli/test_ingest_commands.py create mode 100644 tests/cli/test_init.py create mode 100644 tests/cli/test_job_commands.py create mode 100644 tests/cli/test_json_contract.py create mode 100644 tests/cli/test_output.py create mode 100644 tests/cli/test_project_commands.py create mode 100644 tests/cli/test_release_commands.py create mode 100644 tests/cli/test_schema_commands.py create mode 100644 tests/examples/test_cli_end_to_end.py create mode 100644 tests/fixtures/samples.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fca0d403..49be003b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,13 @@ jobs: - name: Ingest end-to-end example run: uv run python examples/ingest_end_to_end.py + # The CLI half, and the only one that proves the *installed console script* + # works from a real shell — which CliRunner, running in-process, cannot. + # `uv run bash` puts the virtualenv's bin/ on PATH so `visionset` and + # `python3` are the same installation. Stills only, so no ffmpeg needed. + - name: CLI end-to-end example + run: uv run bash examples/cli_end_to_end.sh + frontend: runs-on: ubuntu-latest steps: diff --git a/README.md b/README.md index 3e6362e6..4b4fa068 100644 --- a/README.md +++ b/README.md @@ -16,14 +16,31 @@ plain `pip` package. ```bash pip install visionset # coming soon -python -c "from visionset.kernel.services import WorkspaceService; WorkspaceService.init('.').close()" +visionset init # a workspace here visionset ui # API at http://127.0.0.1:8000, app at /ui ``` -Creating the workspace is a Python call for now — there is no `visionset init` yet. `visionset ui` -run outside a workspace refuses with one sentence and exit 1; it never creates one, because a -command that silently made a workspace out of whatever directory you were standing in is how data -ends up somewhere nobody chose. See [docs/cli.md](docs/cli.md). +`init` is the only command that creates a workspace, and it refuses a directory that already holds +something. `visionset ui` run outside one refuses with one sentence and exit 1; it never creates +one, because a command that silently made a workspace out of whatever directory you were standing +in is how data ends up somewhere nobody chose. + +Or drive the whole cycle from the terminal, without a server: + +```bash +visionset project create road-signs +visionset schema apply schema.json --project road-signs +BATCH=$(visionset ingest ./incoming --project road-signs) +visionset batch approve "$BATCH" --jobs-of 100 && visionset batch start "$BATCH" +# …annotate, then… +visionset batch complete "$BATCH" && visionset batch promote "$BATCH" +visionset release publish --tag v1.0 --project road-signs --split 0.7,0.15,0.15 +visionset export --project road-signs --release v1.0 --format dummy --out ./out +``` + +Every command takes `--json` for scripting, and the shapes are the REST API's. See +[docs/cli.md](docs/cli.md), or [`examples/cli_end_to_end.sh`](examples/cli_end_to_end.sh) for that +walk with its assertions still in it. Prefer to see the SDK first? [`examples/sdk_end_to_end.py`](examples/sdk_end_to_end.py) drives an empty directory to a hash-verified release in one pass, generating its own images — no server, diff --git a/docs/README.md b/docs/README.md index c5468587..975bcc38 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,4 +22,4 @@ contracts (kernel purity, headless annotator) are described there and enforced i | [examples.md](examples.md) | The two runnable examples: the whole cycle in one pass, ingest on its own, and what each is built to demonstrate | | [api.md](api.md) | The REST surface: the conventions every endpoint follows (paths, UUID ids, the list envelope, gates as query parameters), the one error body, why clients branch on `code` and not on the status, what decides 404 / 409 / 422, what a 5xx does and does not tell you, and which codes are worth retrying | | [auth.md](auth.md) | Who may call it: per-workspace API tokens, why only a digest is stored, why every refusal is one identical 401, immediate revocation, the `visionset token` commands, and how a protected route is built | -| [cli.md](cli.md) | The command line: the three exit codes, why stdout is data and stderr is prose, why `--workspace` follows the subcommand, and what `visionset ui` starts — the resolved workspace it states, the bundle it serves, and what happens when nobody built one | +| [cli.md](cli.md) | The command line: the whole cycle as a script, the three exit codes (and why one of them also means "no"), why stdout is data and stderr is prose, what `--json` promises and how it stays the API's shape, why `--workspace` follows the subcommand, and what `visionset init` and `visionset ui` each do | diff --git a/docs/batches.md b/docs/batches.md index 1085eb4a..6b603e07 100644 --- a/docs/batches.md +++ b/docs/batches.md @@ -154,6 +154,36 @@ progress and the membership rows. of work never deletes the work. Neither the assets nor any blob are touched either — see [projects.md](projects.md) for why blobs are never deleted. +## At a terminal + +```bash +visionset batch list --project road-signs +visionset batch approve "$BATCH" --jobs-of 100 +visionset batch start "$BATCH" +visionset batch complete "$BATCH" +visionset batch promote "$BATCH" +``` + +Each is one service call, and the listing carries the progress counts because a batch's name and +state do not say whether anybody has started on it. + +**`--jobs-of N` is `BySize`; with no flag the batch becomes one job.** There is no flag for +`BySegments`, and that is a decision rather than an omission: its own contract is that the caller +has already decided the split, and the only caller that ever holds an exact partition is a program — +which has the SDK and the API. It is also the one partition that can be *wrong*, with four distinct +refusals, and putting it behind a shell's quoting of tuples of UUIDs is a way to meet all of them. +If it is ever wanted it arrives as `--segments FILE.json`. + +`--jobs-of` carries `min=1` at the Click layer, because `BySize.size` is `gt=0` and a pydantic error +is not a `VisionSetError` — it would print a traceback rather than a sentence. + +**There is no `batch create`, and no membership editing**, for the reason there is none over HTTP: a +batch is born from an ingest. `BatchService` still has all four methods; this is a decision about +the surfaces. + +`promote` is here rather than under a dataset group because `DatasetService.promote` takes a *batch* +id and derives the dataset from it — the same argument its route makes. + ## Over HTTP The [API](api.md) is this service with the curation half left off. diff --git a/docs/cli.md b/docs/cli.md index a3be9381..5a4c5b34 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -4,21 +4,72 @@ the kernel in-process — there is no HTTP hop, and no command reaches a capability the SDK does not already have. +Every command below takes `[--json]` and `[--workspace]` unless the table says otherwise, and both +are omitted from this listing to keep it readable. + ``` visionset --version -visionset ui [--host] [--port] [--reload] [--workspace] -visionset token create --name NAME [--workspace] -visionset token list [--workspace] -visionset token revoke NAME [--yes] [--workspace] -visionset mcp # not implemented yet +visionset init [PATH] [--name NAME] # neither --json nor --workspace + +visionset project create NAME [--description TEXT] +visionset project list +visionset schema apply FILE --project P [--allow-destructive] +visionset schema list --project P + +visionset ingest PATH --project P [--fps N] [--batch-name NAME] +visionset batch list --project P +visionset batch approve BATCH_ID [--jobs-of N] +visionset batch start|complete|promote BATCH_ID + +visionset job list --batch BATCH_ID +visionset job next JOB_ID [-n COUNT] +visionset job progress|start|complete JOB_ID +visionset job mark JOB_ID ASSET_ID --progress STATE + +visionset release publish --tag T --project P [--split TRAIN,VAL,TEST] [--seed N] +visionset release list --project P +visionset release verify TAG --project P +visionset export --project P --release TAG --format F --out DIR [--allow-lossy] +visionset format list # no --workspace: it opens nothing + +visionset backfill-thumbnails --project P +visionset token create --name NAME +visionset token list +visionset token revoke NAME [--yes] +visionset ui [--host] [--port] [--reload] # no --json +visionset mcp # not implemented yet +``` + +## The cycle, as a script + +```bash +WS=$(visionset init ./datasets/robots) +export VISIONSET_WORKSPACE="$WS" # say it once; no command needs -w after this + +visionset project create road-signs +visionset schema apply schema.json --project road-signs +BATCH=$(visionset ingest ./incoming --project road-signs) + +visionset batch approve "$BATCH" --jobs-of 100 +visionset batch start "$BATCH" +# …annotate, in the app or through `visionset job mark`… +visionset batch complete "$BATCH" +visionset batch promote "$BATCH" + +visionset release publish --tag v1.0 --project road-signs --split 0.7,0.15,0.15 +visionset release verify v1.0 --project road-signs && \ + visionset export --project road-signs --release v1.0 --format dummy --out ./out ``` +[`examples/cli_end_to_end.sh`](../examples/cli_end_to_end.sh) is that walk with its assertions +still in it, and it runs in CI on every change. + ## Three exit codes, and no others | | Meaning | | --- | --- | | **0** | the command did what it said | -| **1** | a domain refusal — one sentence on stderr, no traceback | +| **1** | a domain refusal — one sentence on stderr, no traceback — **or the answer is no** | | **2** | a usage error, raised and formatted by Click itself | One non-zero code for the whole `VisionSetError` family, rather than a table mapping each error to @@ -40,23 +91,91 @@ $ echo $? ``` Only `VisionSetError` is caught. An `OSError`, a `KeyboardInterrupt` or a bug keeps its traceback, -because folding one into `Error: [Errno 2] ...` would hide the thing that identifies it. +because folding one into `Error: [Errno 2] ...` would hide the thing that identifies it. Where a +kernel call refuses with something outside that family — a non-positive `--fps`, a `--split` whose +fractions do not add up, a schema file that is not JSON — the CLI catches it **before** the call and +raises Click's own usage error, so it lands at 2 rather than as a traceback. + +**Code 1 also means "the answer is no".** `visionset release verify` runs, finds damage, and exits 1 +with one sentence naming it; nothing refused, and nothing about the command failed. That is what +`grep` and `diff` already mean by a non-zero exit, and it is the only way a script branches on the +result without parsing output: + +```bash +visionset release verify v1.0 --project road-signs && ./train.sh +``` + +A malformed UUID is a **2**, not a 1 — Click's own type refuses it, and the request could not have +named anything. That is the same call [the REST surface](api.md#the-two-shapes-of-422) makes when it +answers 422 rather than 404. A *name* that matches nothing is a 1, because it could have. ## Stdout is data; stderr is everything a person reads ```bash TOKEN=$(visionset token create --name ci) # exactly the secret, nothing else +WS=$(visionset init ./datasets/robots) # exactly the resolved workspace root +BATCH=$(visionset ingest ./incoming -p road) # exactly the batch id visionset token list | tail -n +2 # the rows without the header ``` -The secret goes to stdout alone; the "shown once" warning goes to stderr, so it survives that -redirection and is still seen. `token list` prints a header even with zero rows, so `tail -n +2` is -stable, and it names its three columns one at a time rather than dumping the model — a field added -to `Token` cannot leak into output nobody re-read. +A command whose result is one thing puts that one thing on stdout and everything else on stderr: +`init` the workspace root, `project create` the new id, `ingest` the batch id, `schema apply` the +new version number. The "shown once" warning, the per-file ingest report and every "created …" line +go to stderr, so they survive the redirection that most needs them. + +Every listing prints a header even with zero rows, so `tail -n +2` is stable, and **the first column +of every listing is the id**, so `awk '{print $1}'` is too: + +```bash +visionset job list --batch "$BATCH" | tail -n +2 | awk '{print $1}' +``` + +That rule matters because a name may hold internal whitespace — `normalize_name` strips the outside +and deliberately preserves the middle — so a name-first column would break the moment somebody typed +`road signs east`. `token list` is the one listing that leads with a name, because a token has no id +anybody types; use `--json` there instead. + +Every listing names its columns one at a time rather than dumping a model, for the reason +`token list` established: a field added to a domain model cannot leak into output nobody re-read. Plain columns, never `rich.table`: box drawing pads to `$COLUMNS` and wraps, which makes output width-dependent and neither testable nor `cut`-able. +## `--json`, and what it promises + +Columns are for a person. `--json` is for a program, and it is available on every command that +prints anything: + +```bash +visionset release list --project road-signs --json | jq '.items[] | .tag' +visionset batch list --project road-signs --json | jq '.items[] | select(.progress.unannotated > 0)' +``` + +The contract: + +- **One JSON document per invocation**, on stdout, indented, with a trailing newline. Not + JSON-lines — a listing is a single value, so a partial read is never mistaken for a whole one. +- **A listing is `{"items": [...], "total": n}`**, never a bare array, and an empty one is + `{"items": [], "total": 0}` rather than an error. The same envelope, and the same argument for it, + as [the REST API's](api.md#conventions). +- **A single resource or report is the bare object.** +- `--json` changes what stdout *is*. It does not silence stderr: a warning is still a warning. + +**The shapes deliberately agree, key for key, with the REST API's**, so a script moves between +`curl | jq` and `visionset --json | jq` without relearning field names. That agreement is not a +convention anybody remembers — `tests/cli/test_json_contract.py` asserts, for fifteen resources, +that the CLI's projection has exactly the wire model's fields *and* that the wire model validates +it, which catches a timestamp in the wrong format that a key comparison would miss. + +Two shapes have no REST counterpart, because no route publishes them, and the CLI defines them +first: `export`'s report (`release_id`, `format`, `directory`, `file_count`, `total_bytes`) and +`backfill-thumbnails`' (`project_id`, `examined`, `filled`, `missing`, `unreadable`). + +Three fields are deliberately **never** published, in either surface: an asset's `uri` and a +source's `path`, which are absolute paths on this machine, and a batch's `asset_ids`, which for +fifty thousand frames must not travel on every read of its name. A source publishes `name` — the +last component of its path — instead. + ## `--workspace` comes after the subcommand `--workspace` / `-w` is declared on **each command**, not on the root callback, and that is a Click @@ -65,6 +184,13 @@ the callback would have to *precede* the subcommand — `visionset --workspace X ci` would work and `visionset token create --name ci --workspace X` would fail with "No such option". Nobody types the first one. +`--json` is per command for the identical reason, and so is every other option here. The two +commands without `--workspace` are the ones that need none: `visionset format list` reads installed +distributions, which is a fact about the process; `visionset init` takes a positional `PATH`, +because it names where to *make* a workspace rather than which one to use — and for that reason it +never walks, never reads `$VISIONSET_WORKSPACE`, and never trades the directory you named for its +parent. + Which workspace a command lands in, when the flag is absent, is [one rule shared by every surface](workspaces.md#which-workspace-when-nobody-said): the flag, then `VISIONSET_WORKSPACE`, then the nearest workspace at or above the working directory, then the @@ -72,6 +198,33 @@ working directory. **Only that third case walks upward.** A flag and an environm somebody *stating* which workspace, and trading a stated directory for its parent is how a credential gets minted into the wrong one. +## `visionset init` + +Creates a workspace, which every other command needs and none of them makes. + +``` +$ visionset init ./datasets/robots +Created workspace 'robots' at /home/you/datasets/robots. +/home/you/datasets/robots +Next: visionset token create --name , then visionset ui. +``` + +The root is the only thing on stdout, so `WS=$(visionset init ./robots)` is exactly the path — and +it is the *resolved* path, which is the useful answer when you typed `.`. + +| | | +| --- | --- | +| `PATH` | Where to create it. Defaults to the working directory. Missing or empty are both fine; anything else is refused, so a typo cannot turn a home directory into a workspace. | +| `--name` | The workspace's name. Defaults to the directory's own. | + +Creating one where a workspace already sits is refused too — the remedy is to use it, not to make a +second. Both refusals are one sentence at exit 1. + +Deliberately **not** folded into `visionset ui`, which would have to create a workspace when the +directory looked empty: that breaks "`init` creates, `open` never does" and means a mistyped path +silently becomes a new empty workspace instead of an error. See +[workspaces.md](workspaces.md#at-a-terminal). + ## `visionset ui` Starts the server against the resolved workspace, serving the REST API at the root and the compiled @@ -121,6 +274,83 @@ Why `/ui` rather than `/`: the API already owns the root, so an app served from claim `/projects/abc` as one of its own client routes. See [api.md](api.md#where-the-ui-lives). +## The flow commands + +One command, one SDK call, with the rationale in the topic doc rather than here. + +### `visionset project` + +`create NAME [--description TEXT]` → `ProjectService.create`, which writes the project and its one +dataset in a single transaction. `list` → `ProjectService.list`. + +Everything downstream takes `--project` / `-p`, which accepts **a name or an id**. A name matches +case-insensitively, the way the unique index compares. There is no `rename` and no `delete`: both +are administration rather than flow, and both want the cascade explained. See +[projects.md](projects.md#at-a-terminal). + +### `visionset schema` + +`apply FILE --project P [--allow-destructive]` → `SchemaService.create_version`. The file is JSON +and is **the same document** `POST /projects/{id}/schema/versions` takes. `list --project P` → +`SchemaService.list_versions`; the last one is active. + +Versions are 1..N and none of them changes, so `apply` always *adds* one. A change that removes or +narrows something is refused until `--allow-destructive`; one that would orphan existing annotations +has no override at all. See [schemas.md](schemas.md#at-a-terminal). + +### `visionset ingest` + +`PATH --project P [--fps N] [--batch-name NAME]` — **the one command that is two SDK calls**: +`SourceService.register_images` or `register_video`, dispatched on whether the path is a directory, +then `IngestService.ingest`. Registration is idempotent, so re-running the same line registers once; +content addressing means it also creates no asset it created before, which is the remedy for an +interrupted run. The batch id goes to stdout. + +`--fps` is video-only and a usage error on a folder. The run is **synchronous**, and there is no +`--resume`: polling needs a second process, which is what `visionset ui` and +`GET /ingest-jobs/{id}` are for. See [ingest.md](ingest.md#at-a-terminal). + +### `visionset batch` + +`list --project P`, then the one-way walk `approve [--jobs-of N]` → `start` → `complete`, then +`promote`. Each maps to the `BatchService` method of the same name, except `promote`, which is +`DatasetService.promote` — it takes a *batch* id and derives the dataset, which is why it lives here. + +`--jobs-of N` is the `BySize` partition; with no flag the batch becomes one job. There is no +`batch create` and no membership editing: a batch is born from an ingest. See +[batches.md](batches.md#at-a-terminal). + +### `visionset job` + +`list --batch B`, `next JOB [-n N]`, `progress JOB`, `start JOB`, `mark JOB ASSET --progress STATE`, +`complete JOB`. Each is one `JobService` call. + +**`--progress annotated` records that somebody labeled an asset, and the CLI writes no labels** — +geometry comes from a canvas or a model, not from typing. A release published off a batch driven +this way reports `annotation_count: 0`, and its manifest says so. These commands exist because the +lifecycle must be drivable from a script, not because this is how labelling happens. See +[jobs.md](jobs.md#at-a-terminal). + +### `visionset release` and `visionset export` + +`release publish --tag T --project P [--split TRAIN,VAL,TEST] [--seed N]` → `ReleaseService.publish`. +`release list --project P`, and `release verify TAG --project P`, whose **exit code is the answer**. + +`export --project P --release TAG --format F --out DIR [--allow-lossy]` resolves the format through +the plugin registry and hands the instance to `ReleaseService.export` — the kernel is forbidden from +finding a plugin itself. `format list` says which are installed. + +A release tag is **case-sensitive** where a project name is not: a tag is an identifier, not a label +somebody reads. `--allow-lossy` is a third gate word beside `--yes` and `--allow-destructive`, never +merged with either. See [releases.md](releases.md#at-a-terminal). + +### `visionset backfill-thumbnails` + +`--project P` → `IngestService.backfill_thumbnails`. Renders the previews of assets that have none — +a preview is a cache, not an identity, so an asset whose bytes will not render keeps a null one and +is reported here rather than having failed its ingest. Idempotent. See +[ingest.md](ingest.md#the-backfill). + ## `visionset token` Issuing, listing and revoking per-workspace API tokens. Covered in full — including why only a @@ -134,13 +364,52 @@ its target by import string or subprocess for the same reason `ui` does — impo ## For contributors -`cli/_errors.py` owns the exit codes and `domain_errors()`; `cli/_workspace.py` owns -`WorkspaceOption` and `opened_workspace()`. A new command is a module beside them, a function taking -`workspace: WorkspaceOption = None` **last**, and one registration line in `cli/main.py`. Wrap every -kernel call in `opened_workspace()` — it composes the open, the close and the refusal, and it closes -in a `finally` so no `visionset.db-wal` is left behind. +Five private modules carry everything a command needs: + +| | | +| --- | --- | +| `cli/_errors.py` | the exit codes and `domain_errors()` | +| `cli/_workspace.py` | `WorkspaceOption` and `opened_workspace()` | +| `cli/_output.py` | `JsonOption`, the column formatter, `document()`, `note()` | +| `cli/_json.py` | one hand-written projection per resource | +| `cli/_resolve.py` | `ProjectOption`, and turning a name or a tag into the thing it names | + +A new command is a module beside them and one registration line in `cli/main.py` — groups by +`add_typer`, bare commands by `app.command("name")(fn)`, which is where they are registered rather +than at their definition site because a decorator there would import `main` and `main` imports them. +The signature ends `json_out: JsonOption = False, workspace: WorkspaceOption = None`, with +`workspace` **last**. Wrap every kernel call in `opened_workspace()` — it composes the open, the +close and the refusal, and it closes in a `finally` so no `visionset.db-wal` is left behind. + +**A command maps to exactly one service call, and says so in its docstring when it does not.** +`ingest` is the only one that does not, and its module explains why. + +**Never `model_dump()` a domain model into `--json`.** Write the projection in `_json.py` and add +the pair to `tests/cli/test_json_contract.py`, which asserts key-for-key parity with the REST wire +model. That test may import both `visionset.cli` and `visionset.server` because `tests/` is outside +the package the independence contract governs — the packages themselves must not. + +**A bound the domain enforces with a pydantic `Field` has to be mirrored in the Typer option**, or +the refusal arrives as a traceback: a pydantic `ValidationError` and a bare `ValueError` are not +`VisionSetError`s and `domain_errors()` deliberately does not catch either. `--jobs-of` carries +`min=1`, `--fps` is checked in the body (Typer has no `min_open`), and `--split` is parsed into a +`SplitRecipe` inside a `try`. Commands are tested through `typer.testing.CliRunner` against the real `visionset.cli.main:app`, with `result.stdout` and `result.stderr` asserted separately. There is no `conftest.py` anywhere in this repository; each module declares its own fixtures, including an autouse one that clears -`VISIONSET_WORKSPACE` so a developer with it exported gets CI's results. +`VISIONSET_WORKSPACE` so a developer with it exported gets CI's results. **Use +`monkeypatch.setenv(VAR, "")` if any command the module exercises can write `os.environ`, and +`delenv(VAR, raising=False)` otherwise** — `delenv` records no undo when the variable was already +absent, so a written one leaks into every later module. Only `ui` writes it today. + +**A test module's basename must be unique across the whole suite.** With no `__init__.py` anywhere, +pytest imports a test module under its bare basename, so `tests/cli/test_batches.py` beside +`tests/server/test_batches.py` is a collection error rather than two modules — which is why the CLI +ones are `test__commands.py`. Private helpers are exempt: `tests/cli/_flow.py` and +`tests/server/_flow.py` coexist because they are imported by their full dotted path, which PEP 420 +namespace packages resolve. + +`tests/cli/_flow.py` walks the CLI up to a given rung *by invoking the CLI*, so the ladder is itself +under test on the way up; `tests/cli/test_full_cycle.py` uses none of it, because the point there is +that the whole walk is readable in one function. diff --git a/docs/datasets.md b/docs/datasets.md index 2a9740ed..af22f65c 100644 --- a/docs/datasets.md +++ b/docs/datasets.md @@ -86,6 +86,11 @@ make the audit record load-bearing for behaviour, so that reading it wrong and d thing become the same bug. Promotion answers only *what does this batch have that the trunk does not*. +At a terminal that gate is `visionset batch promote BATCH_ID`, which prints the ids of the assets it +admitted — none, when there were none to admit, because promotion is a union and a second call is a +no-op. The rest of curation is deliberately not on the CLI: `remove_asset` is a gallery's operation, +and the log above is what makes it reviewable. + ## Curating: `remove_asset` ```python diff --git a/docs/examples.md b/docs/examples.md index be8a5961..50e299b3 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -7,21 +7,26 @@ files that do several of them at once. There are two, one per milestone: | --- | --- | --- | | [`sdk_end_to_end.py`](../examples/sdk_end_to_end.py) | an empty directory to a release whose every byte can be re-hashed and checked | M1 | | [`ingest_end_to_end.py`](../examples/ingest_end_to_end.py) | a ten-second clip and a folder of stills to an approved, partitioned batch | M2 | +| [`cli_end_to_end.sh`](../examples/cli_end_to_end.sh) | the same cycle again, from a shell, using nothing but the `visionset` command | M3 | ```bash uv run python examples/sdk_end_to_end.py uv run python examples/ingest_end_to_end.py # needs ffmpeg +uv run bash examples/cli_end_to_end.sh ``` Each is its milestone's exit criterion made executable, and each runs in CI **twice**: once as a pytest smoke test ([M1](../tests/examples/test_sdk_end_to_end.py), -[M2](../tests/examples/test_ingest_end_to_end.py)) that asserts on outcomes, and once as a plain +[M2](../tests/examples/test_ingest_end_to_end.py), +[M3](../tests/examples/test_cli_end_to_end.py)) that asserts on outcomes, and once as a plain script, which is the only way to prove it still works from a clean checkout. -The two overlap by design and neither subsumes the other. The SDK example walks the whole cycle +They overlap by design and none subsumes another. The SDK example walks the whole cycle and treats ingest as one stage of thirteen; the ingest example stops at an approved batch and spends its length on where assets come from — two sources over one file, dedup, progress and the -per-file report. +per-file report. The CLI example walks the whole cycle a second time, and what it proves is not the +cycle but the *surface*: that the installed console script reaches every stage of it, that ids +travel on stdout, and that `--json` is stable enough to assert on. --- @@ -174,3 +179,52 @@ The generation command is `tests/fixtures/media.write_video`'s, duplicated rathe that module is a test fixture, it imports pytest, and its answer to a missing binary is `pytest.skip`, which means nothing in a script. The stills, by contrast, are Pillow's work — a real dependency since #16, so a second hand-rolled PNG encoder beside it would be archaeology. + + +--- + +# The CLI example + +## What it does + +`examples/cli_end_to_end.sh` is M3's exit criterion — *the full cycle without touching Python* — +written as the thing that criterion describes. It runs `visionset init`, `project create`, +`schema apply`, `ingest`, `batch approve/start/complete/promote`, a `job` loop, `release +publish/verify`, `format list` and `export`, and then asserts. + +## Three things it is built to demonstrate + +**Ids travel on stdout.** `WS=$(visionset init "$DEST/ws")`, `BATCH=$(visionset ingest …)`, and +every listing read with `tail -n +2 | awk '{print $1}'` — which works because a header always +prints and the first column is always an id. Nothing here parses prose. + +**The workspace is stated once.** `export VISIONSET_WORKSPACE="$WS"` after `init`, and no command +after that carries `-w`. That is the environment-variable branch of the resolution rule, and it is +the branch a script should use: the flag is for a one-off, and the upward walk is for a person +standing in a project directory. + +**`--json` is stable enough to assert on.** Step 9 pipes `release list --json` through `python3` +and checks the envelope, the tag, the asset count and the split recipe — which *is* the "`--json` +outputs stable, documented shapes" acceptance criterion, tested rather than promised. + +There is a fourth, at the end: a deliberate refusal. Publishing `v1.0` twice exits 1 with one +sentence on stderr, and the script asserts that it did. A command inside an `if` condition does not +trip `set -e`, which is what makes demonstrating a failure safe. + +## What it deliberately does not need + +**No ffmpeg**, so it runs anywhere the package installs — stills only, six of them plus one +`notes.txt` that is deliberately not an image, so the per-file report has something in it. **No +`jq`**, because the column format is designed to be read with `awk`. **No `curl` and no server**: +the CLI calls the SDK in-process, which is the whole point of it being a sibling of the REST API +rather than a client of it. + +`python3` appears twice — once to write PNGs, because a shell cannot, and once to assert on a JSON +document. Neither touches the SDK. + +## The honest note it carries + +Every asset in this run is marked `annotated` and carries **no labels**. Drawing a box is the app's +job; `visionset job mark` records that somebody did it. So the release reports +`annotation_count: 0`, the manifest says so, and the smoke test asserts it — rather than the script +quietly leaving the impression that a terminal can label images. diff --git a/docs/ingest.md b/docs/ingest.md index cb7512f7..ce59a300 100644 --- a/docs/ingest.md +++ b/docs/ingest.md @@ -270,6 +270,10 @@ this file" and a blob that is not there is not a file. There is no progress to poll: a backfill has no `IngestJob` row. If that is ever wanted it is a task of its own, not a flag on this one. +At a terminal this is `visionset backfill-thumbnails --project P`; its report is the +`ThumbnailBackfill` above, printed as counts on stderr with the unreadable files in a table. It is +the only command for a kernel read that no route exposes. + ## The target batch With no `batch_id`, the run creates a draft named `batch_name` or, failing that, after the @@ -280,6 +284,41 @@ Membership is everything the run ingested, deduplicated assets included: a dupli data, but it is part of what the run was asked to gather. Order is ingest order, which is filename order for a directory and frame order for a clip. +## At a terminal + +```bash +BATCH=$(visionset ingest ./incoming --project road-signs --batch-name day-one) +visionset ingest ./clip.mp4 --project road-signs --fps 5 +``` + +**One command where registration has two methods**, dispatched on whether the path is a directory. +That is the ergonomic mirror of the split above: the *caller* does not have to say which of the two +they have, because the filesystem already knows, and the source they end up with carries the kind, +the path and the rate from then on. + +It is the only command in the CLI that is two SDK calls, and its module says so. Both are safe to +repeat: registration is idempotent on `(kind, path, extraction_fps)`, and content addressing means a +second run creates nothing the first already did. + +The batch id goes to stdout alone, so `BATCH=$(visionset ingest …)` is the whole idiom. The per-file +report goes to stderr, one line per refused file, so a redirected stdout stays a single id. + +**`--fps` is video-only, and a usage error on a folder.** Silently ignoring it would let somebody +believe they had chosen a rate. It is also checked for being positive before the call, because +`register_video` refuses a non-positive rate with a bare `ValueError` — not a `VisionSetError`, so +it would print a traceback rather than a sentence. A missing path is exit 2 for the same reason +(`FileNotFoundError`), which is why the argument carries Click's own `exists=True`. + +**The run is synchronous, and the CLI never calls `enqueue`.** A queued job needs a worker to pick +it up, and a CLI process has none — a detached job would simply never run. Polling is what the +server is for: `visionset ui`, then `GET /ingest-jobs/{id}`. + +**Interrupting a run leaves the job row at `running`, and there is no `--resume`.** The remedy needs +no new vocabulary: run the same line again. Registration finds the same source, `enqueue` does not +consult other jobs, and content addressing means the new run creates nothing the old one already +created. The stuck row stays as the only evidence that something was interrupted, which is the same +posture the kernel takes about a crashed process. + ## Over HTTP The [API](api.md) is `enqueue` and `resume` with a worker between them. diff --git a/docs/jobs.md b/docs/jobs.md index 7361cc2f..64308014 100644 --- a/docs/jobs.md +++ b/docs/jobs.md @@ -147,6 +147,35 @@ cross-table query: `Repository.list` takes a single `parent_id`. That is N + 1 r deliberately — see [persistence.md](persistence.md). When it starts to cost, the fix is a method on the port, never a SQLAlchemy import in a service. +## At a terminal + +```bash +visionset job list --batch "$BATCH" +visionset job start "$JOB" +visionset job next "$JOB" -n 50 +visionset job mark "$JOB" "$ASSET" --progress annotated +visionset job progress "$JOB" +visionset job complete "$JOB" +``` + +Each is one `JobService` call. `next` and `mark` are what make the lifecycle drivable from a script +at all — a batch cannot be completed until every asset has settled, and nothing else settles one. +`JobService.mark`'s own docstring invites the second by name. + +**Say the wart out loud: `--progress annotated` records that somebody labeled an asset, and the CLI +writes no labels.** Geometry comes from a canvas or a model, not from typing. A release published +off a batch driven entirely this way carries `annotation_count: 0`, and its manifest honestly says +so. These commands exist because the *lifecycle* must be reachable from a terminal, not because this +is how labelling is meant to happen. + +`--progress` is rendered from `AssetProgress` itself, so a wrong value exits 2 listing every legal +one, and `job progress`'s columns are read off the same enum — a sixth state cannot be silently +missing from the table. `-n` carries `min=1`, because `next_pending` refuses a non-positive count +with a bare `ValueError`. + +Jobs and assets are addressed by **id only**: neither has a name, and both ids come off the previous +command's stdout. + ## Over HTTP The [API](api.md) is this service, one route per method. diff --git a/docs/projects.md b/docs/projects.md index 42f5fccc..721cd306 100644 --- a/docs/projects.md +++ b/docs/projects.md @@ -52,6 +52,31 @@ whitespace, NFC-normalized on the way in. See in two places. `rename` passes `exclude=project_id`, so correcting only the capitalization of a name is not a collision with itself. +## At a terminal + +```bash +visionset project create road-signs --description "Motorway signage" +visionset project list +``` + +`create` writes the project and its dataset in one transaction, and prints the new id on stdout +alone. `list` leads with the id, so `awk '{print $1}'` is stable even for a name holding internal +whitespace. + +**Every downstream command takes `--project` / `-p`, and it accepts a name or an id.** A +well-formed UUID is treated as an id; anything else is a name, matched case-insensitively through +`ProjectService.get_by_name`. That method is a kernel read rather than a scan written in the CLI +because the comparison is not obvious and it is not the only one — a release tag is unique per +dataset and **case-sensitive**, the opposite rule — so a surface re-deriving either from prose would +eventually get one of them wrong. + +A project whose *name* is a well-formed UUID string is unreachable by name. Harmless: the same +string reaches it as an id. + +There is deliberately no `visionset project rename` and no `visionset project delete`. Both are +administration rather than flow, and a delete wants the prompt and the cascade above spelled out at +the point of use; landing them together is how that gets written once. + ## Deleting a project Deletion is guarded by a parameter, not by a prompt: diff --git a/docs/releases.md b/docs/releases.md index d43d724e..efd8e0b0 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -193,6 +193,42 @@ Counting is `ReleaseService`'s rather than the plugin's, deliberately: a number thing it describes is not checkable, and an exporter that writes nothing must report zero rather than say what it meant to do. +## At a terminal + +```bash +visionset release publish --tag v1.0 --project road-signs --split 0.7,0.15,0.15 --seed 42 +visionset release list --project road-signs +visionset release verify v1.0 --project road-signs +visionset export --project road-signs --release v1.0 --format dummy --out ./out +``` + +`--split` is **one** option rather than three, because a split is one concept, `0.7,0.15,0.15` is +how it is written everywhere, and one flag means one refusal to word. `--seed` stays separate; it is +not a fraction. Fractions that do not add up are exit 2 — `SplitRecipe` refuses them with a pydantic +error, which is not a `VisionSetError` — so the CLI parses the recipe before the call. + +**A tag is case-sensitive where a project name is not.** Both comparisons live in the kernel beside +the index that enforces them (`ReleaseService.get_by_tag`, `ProjectService.get_by_name`), because +they are opposites and a surface re-deriving either would eventually pick the wrong one. + +**`release verify` exits 1 when the answer is no.** Nothing refused — the check ran and found +damage — but a non-zero exit is what `grep` and `diff` already mean, and the only way a script +branches on the result without parsing output: + +```bash +visionset release verify v1.0 --project road-signs && ./train.sh +``` + +For `export`, the CLI resolves the format name through the plugin registry and hands the *instance* +to `ReleaseService.export`, because the kernel is forbidden from importing the registry. It resolves +it with `pick`, never a dict lookup: a `KeyError` is outside the `VisionSetError` tree and a typo +would answer with a traceback instead of the list of installed formats. `visionset format list` +prints that list without opening a workspace at all. + +`--allow-lossy` is the third gate word, never folded into `--yes` or `--allow-destructive`. And +`dummy` — the only exporter this repository ships — writes nothing, so a `file_count` of 0 in its +report is an export that ran, not one that failed. + ## Over HTTP The [API](api.md) is this service, one route per method, plus the format listing. diff --git a/docs/schemas.md b/docs/schemas.md index 887ebc9d..6ef11a8f 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -201,6 +201,49 @@ active version. The judgement itself is shared rather than reimplemented: `Attri is the one method that answers "does this attribute take this value", and both a default and a label go through it. See [annotations.md](annotations.md). +## At a terminal + +```bash +visionset schema apply schema.json --project road-signs +visionset schema list --project road-signs +``` + +The file is **JSON**, and it is byte-for-byte the same document +`POST /projects/{id}/schema/versions` takes: + +```json +{ + "classes": [ + { + "name": "sign", + "geometry": "bbox", + "color": "#ff0000", + "attributes": [ + {"name": "occluded", "kind": "boolean", "required": false, "default": false} + ] + } + ] +} +``` + +That is a tested claim rather than a promise: `tests/cli/test_json_contract.py` asserts the CLI's +`label_class` and `attribute` projections have exactly `LabelClassBody`'s and `AttributeBody`'s +fields, and `tests/cli/test_schemas.py` validates the example document as a request body. + +**JSON and not YAML.** A second format means a runtime dependency in every wheel, a second parser +to keep honest, and two shapes that can disagree — while the surface a schema file has to +interoperate with speaks JSON already. `yq . schema.yaml | visionset schema apply /dev/stdin` is one +pipe away for whoever wants one. + +**The document parses through the domain models themselves**, so `LabelClass`'s and `Attribute`'s +own validators do the refusing and no rule here is restated in the CLI. Those refusals are **exit +2**, not 1: a pydantic `ValidationError` is not a `VisionSetError`, and a malformed file is a usage +error in the same sense a malformed request body is a 422. The message carries the domain's own +words and the path to the offending field (`classes.0.name`). + +`--allow-destructive` is the flag for the first of the two gates above. The second — a change that +would orphan annotations — has no flag, here as everywhere. + ## Concurrency The next version number is computed from the versions already stored, so two writers can diff --git a/docs/workspaces.md b/docs/workspaces.md index f6832278..42eec722 100644 --- a/docs/workspaces.md +++ b/docs/workspaces.md @@ -201,6 +201,30 @@ workspaces. An empty `VISIONSET_WORKSPACE` falls through to case 3 rather than resolving to `Path("")`, because a shell cannot tell `VISIONSET_WORKSPACE=` from an unset variable. +## At a terminal + +```bash +visionset init ./datasets/robots # or `visionset init` for the working directory +``` + +Three decisions worth stating, because they are the opposite of every other command's: + +- **A positional `PATH`, not `--workspace`.** Every other command takes a workspace that exists; + this one names where to make one. The flag would read as "operate on this", which is not what is + being said. +- **It does not use `resolve_workspace_root`.** Walking upward to find a place to *create* + something is precisely the failure mode the resolver's precedence argues against — and here it + would be irreversible, since the answer is a new workspace in somebody else's directory rather + than a command that touched the wrong one. It does not read `$VISIONSET_WORKSPACE` either. +- **It closes the workspace it just made.** `WorkspaceService.init` hands one back *open*; a command + that returned without closing would strand `visionset.db-wal` beside it for the next reader to + recover. + +The root goes to stdout alone, so `WS=$(visionset init ./robots)` is exactly the path — resolved, +which is the useful answer when you typed `.`. `WorkspaceNotEmpty` and `WorkspaceAlreadyExists` both +arrive as one sentence at exit 1, and neither earns a hint: their own messages already name the +remedy. + ## `format_version` lives in one place The database stamp in `_visionset_meta` is the sole authority. There is no sidecar marker diff --git a/examples/README.md b/examples/README.md index e94e83e4..c463b063 100644 --- a/examples/README.md +++ b/examples/README.md @@ -7,6 +7,7 @@ and `**/workspace-data/` is git-ignored by design. | --- | --- | | [`sdk_end_to_end.py`](sdk_end_to_end.py) | The whole cycle in one pass: workspace → project → schema → synthetic frames → batch → jobs → annotations → curated trunk → verified release | | [`ingest_end_to_end.py`](ingest_end_to_end.py) | Where assets come from: a generated 10 s clip → source at 5 fps → 50 deduplicated assets → approved batch of 2 jobs, plus a re-run that creates nothing, the same clip at a second rate, and a folder of stills with one unreadable file. **Needs ffmpeg.** | +| [`cli_end_to_end.sh`](cli_end_to_end.sh) | The same cycle from a shell, using nothing but the `visionset` command: init → project → schema → ingest → batch → jobs → release → export, with the `--json` shapes asserted and one deliberate refusal. No ffmpeg, no `jq`, no server. | ## Running the SDK end-to-end example @@ -67,5 +68,25 @@ with WorkspaceService.open("examples/workspace-data/ingest-e2e") as workspace: print(source.kind.value, job.state.value, job.processed, job.total, job.failures) ``` -[`docs/examples.md`](../docs/examples.md) walks through what each stage of both examples does and -why. +## Running the CLI end-to-end example + +```bash +uv run bash examples/cli_end_to_end.sh # into examples/workspace-data/cli-e2e +uv run bash examples/cli_end_to_end.sh ./scratch # or wherever you like +``` + +Same destination rules as above. `uv run bash` rather than plain `bash` is the one requirement: it +puts the virtualenv's `bin/` on `PATH`, so `visionset` and `python3` are the same installation. + +It leaves a workspace at `/ws` — beside its inputs rather than over them, because +`init` refuses a directory that already holds something — with one project, one schema version, six +assets in a completed batch, and a verified `v1.0` release carrying no annotations. + +```bash +export VISIONSET_WORKSPACE=examples/workspace-data/cli-e2e/ws +visionset release list --project road-signs --json | python3 -m json.tool +visionset release verify v1.0 --project road-signs && echo "still intact" +``` + +[`docs/examples.md`](../docs/examples.md) walks through what each stage of all three examples does +and why. diff --git a/examples/cli_end_to_end.sh b/examples/cli_end_to_end.sh new file mode 100755 index 00000000..fd1834aa --- /dev/null +++ b/examples/cli_end_to_end.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# The whole cycle without touching Python — M3's exit criterion, in a shell. +# +# uv run bash examples/cli_end_to_end.sh [DESTINATION] +# +# Its two siblings walk the same ground through the SDK. This one uses only +# `visionset`, which is the claim worth proving: every capability of the kernel +# is reachable from a terminal, ids travel on stdout, refusals travel on stderr, +# and the exit code is what a script branches on. +# +# `uv run bash` rather than plain `bash`: it puts the virtualenv's `bin/` on +# PATH, so `visionset` and `python3` are the same installation. That is this +# script's one requirement. +# +# **No ffmpeg.** Stills only, so it runs anywhere the package installs. The +# ingest example is where video lives. +# +# **No jq.** Listings are read with `tail -n +2 | awk '{print $1}'`, which is +# what the always-printed header and the id-first column order exist for; the +# one JSON assertion goes through `python3`, which is already here. +# +# python3 is used for exactly two things — generating images, because a PNG +# cannot be written in shell, and asserting on one JSON document. Neither +# touches the SDK. + +set -euo pipefail + +DEST=${1:-"$(cd "$(dirname "$0")" && pwd)/workspace-data/cli-e2e"} +NAMED=${1:+yes} + +say() { printf '\n=== %s\n' "$*"; } + +# Only ever removes a directory this script made, and only when it was not +# named on the command line — the same rule the Python examples follow. +if [ -z "${NAMED:-}" ] && [ -e "$DEST" ]; then + if [ -f "$DEST/ws/visionset.db" ]; then + rm -rf "$DEST" + else + echo "$DEST exists and is not a workspace this example made; refusing to clear it" >&2 + exit 1 + fi +fi +mkdir -p "$DEST/incoming" + +say "0. six distinct stills, and one file that is deliberately not an image" +python3 - "$DEST/incoming" <<'PY' +import sys +from pathlib import Path + +from PIL import Image + +incoming = Path(sys.argv[1]) +for index in range(6): + Image.new("RGB", (32, 24), (index * 40 % 256, 80, 160)).save(incoming / f"frame_{index}.png") +(incoming / "notes.txt").write_text("not an image\n", encoding="utf-8") +PY + +cat > "$DEST/schema.json" <<'JSON' +{ + "classes": [ + { + "name": "sign", + "geometry": "bbox", + "color": "#ff0000", + "attributes": [{"name": "occluded", "kind": "boolean", "default": false}] + } + ] +} +JSON + +say "1. a workspace — its root is the only thing on stdout" +# Beside the inputs rather than over them: `init` refuses a directory that +# already holds something, which is the guard that stops a typo turning a home +# directory into a workspace. +WS=$(visionset init "$DEST/ws") +# Stated once, so no later command needs -w. This is the environment-variable +# branch of the resolution rule, which is the one a script should use. +export VISIONSET_WORKSPACE="$WS" + +say "2. a project, and the schema its annotations will be judged against" +visionset project create road-signs --description "CLI end-to-end example" +visionset schema apply "$DEST/schema.json" --project road-signs +visionset schema list --project road-signs + +say "3. ingest — one path in, one batch id out" +BATCH=$(visionset ingest "$DEST/incoming" --project road-signs --batch-name stills) +visionset batch list --project road-signs + +say "4. freeze the membership and cut it into jobs of three" +visionset batch approve "$BATCH" --jobs-of 3 +visionset batch start "$BATCH" + +say "5. work through each job" +# Every asset here is marked `annotated` and carries **no labels** — drawing a +# box is the app's job, not a terminal's. So the release below reports +# annotation_count 0, and that is what its manifest honestly says. +JOBS=$(visionset job list --batch "$BATCH" | tail -n +2 | awk '{print $1}') +[ -n "$JOBS" ] || { echo "expected the approved batch to have jobs" >&2; exit 1; } +for JOB in $JOBS; do + visionset job start "$JOB" + ASSETS=$(visionset job next "$JOB" -n 100 | tail -n +2 | awk '{print $1}') + [ -n "$ASSETS" ] || { echo "expected job $JOB to have assets" >&2; exit 1; } + for ASSET in $ASSETS; do + visionset job mark "$JOB" "$ASSET" --progress annotated + done + visionset job progress "$JOB" + visionset job complete "$JOB" +done + +say "6. close the batch, and let its finished assets into the trunk" +visionset batch complete "$BATCH" +visionset batch promote "$BATCH" + +say "7. publish, and check the freeze — exit 0 is the assertion" +visionset release publish --tag v1.0 --project road-signs --split 0.5,0.25,0.25 +visionset release verify v1.0 --project road-signs + +say "8. export in an installed format" +# `dummy` is the only exporter this repository ships and it writes nothing, so a +# file_count of 0 below is the honest report of an export that ran. +visionset format list +visionset export --project road-signs --release v1.0 --format dummy --out "$DEST/export" --json + +say "9. the release as a program reads it" +visionset release list --project road-signs --json > "$DEST/releases.json" +python3 - "$DEST/releases.json" <<'PY' +import json +import sys + +document = json.load(open(sys.argv[1], encoding="utf-8")) +assert set(document) == {"items", "total"}, document +assert document["total"] == 1, document +release = document["items"][0] +assert release["tag"] == "v1.0", release +assert release["asset_count"] == 6, release +assert release["annotation_count"] == 0, release +assert release["split"] == {"train": 0.5, "val": 0.25, "test": 0.25, "seed": 0}, release +print("--json shapes are what the docs say they are") +PY + +say "10. and a refusal, because a script has to be able to branch on one" +# A command inside an `if` condition does not trip `set -e`, which is what makes +# demonstrating a failure safe. A release is never edited, so the second publish +# under the same tag is refused with one sentence on stderr and exit 1. +if visionset release publish --tag v1.0 --project road-signs 2>/dev/null; then + echo "expected the duplicate tag to be refused" >&2 + exit 1 +fi +echo "the duplicate tag was refused, as it should be" + +say "done — the workspace is at $WS" diff --git a/src/visionset/cli/_errors.py b/src/visionset/cli/_errors.py index ec12f82e..715ae0b0 100644 --- a/src/visionset/cli/_errors.py +++ b/src/visionset/cli/_errors.py @@ -37,6 +37,22 @@ EXIT_DOMAIN_ERROR: Final = 1 """Every ``VisionSetError``. See the module docstring for why it is not a table.""" +EXIT_ANSWER_IS_NO: Final = 1 +"""A command that asked a question and got "no" — ``release verify`` on damage. + +The **same number** as ``EXIT_DOMAIN_ERROR``, and named separately so that the +second meaning is written down rather than inferred from a literal. This is +``grep``'s and ``diff``'s convention, and it is the only way a script can branch +on the answer without grepping the output — which is exactly the coupling +``--json`` exists to avoid. It does not stretch the contract: such a command +still prints one sentence on stderr and no traceback, which is what code 1 has +always described. + +Merging the two constants would lose the distinction the day a caller wants +"could not check" told apart from "checked, and it is broken"; splitting the +*values* would mean a shell had to learn a table to answer "did that work?". +""" + _HINTS: Final[dict[type[BaseException], str]] = { # The kernel's sentence ends in "use WorkspaceService.init to create one", # which is a Python API a person at a terminal has no way to call. Rewriting diff --git a/src/visionset/cli/_json.py b/src/visionset/cli/_json.py new file mode 100644 index 00000000..226ea904 --- /dev/null +++ b/src/visionset/cli/_json.py @@ -0,0 +1,307 @@ +# usage: from visionset.cli import _json +"""What ``--json`` publishes: one hand-written projection per resource. + +**A field reaches a script because somebody wrote it here.** That is +``tokens.py``'s rule — its listing names three columns one at a time rather than +dumping the model — promoted to the shape a program parses. The alternative, +``model_dump()`` on a domain model, would publish whatever the domain happens to +hold today and silently republish whatever it holds tomorrow. Three fields make +the point, and each is already absent from the wire model the server publishes: + +- ``Asset.uri`` and ``Source.path`` are absolute paths on this machine. A script + reading them learns the layout of somebody's disk and nothing it can use. +- ``Batch.asset_ids`` is a batch's whole roll call, which for fifty thousand + frames must not travel on every read of its name. + +**These shapes deliberately agree, key for key, with the REST API's wire models.** +Not by importing them — import-linter forbids ``visionset.cli`` importing +``visionset.server``, and rightly: the surfaces are siblings — but by +``tests/cli/test_json_contract.py``, which imports both and asserts each pair has +the same keys *and* that the projection round-trips through the wire model. A +test may do what neither package may. What that buys is one shape for one +concept, so a script moves between ``curl | jq`` and ``visionset --json | jq`` +without relearning the field names. + +Two things here have no wire partner and are the CLI's own: an export report +(the API returns the archive itself, not a description of it) and a thumbnail +backfill (no route reaches it). Both are named in the docs as the CLI defining +the shape first, for #35 to follow. + +Leaf encoding is explicit everywhere: UUIDs as strings, enums as ``.value``, +paths as strings, and timestamps in **pydantic's** format — microseconds, ``Z`` +— which is why :func:`_moment` is not ``_output.moment``. That one is for a +column and stops at seconds; sharing it would break the parity gate in the one +way key-set comparison cannot see. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from datetime import UTC, datetime +from pathlib import PurePath +from typing import Any +from uuid import UUID + +from visionset.kernel.domain import ( + AnnotationJob, + AnnotationSchema, + Asset, + AssetProgress, + Attribute, + Batch, + Dataset, + ExportResult, + IngestFailure, + IngestJob, + LabelClass, + Project, + Release, + ReleaseVerification, + Source, + SplitRecipe, + ThumbnailBackfill, + VideoProvenance, +) +from visionset.kernel.ports import Exporter + + +def _moment(when: datetime) -> str: + """A timestamp the way pydantic writes one, because the parity gate compares.""" + return when.astimezone(UTC).isoformat().replace("+00:00", "Z") + + +def page(items: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """The collection envelope, identical to the REST API's. + + ``{"items": [...], "total": n}`` and never a bare array — an array cannot + grow a field without breaking every client, which is the argument + ``docs/api.md`` already makes. ``total`` is how many matched, which for a CLI + that does not page is always ``len(items)``; it is here so that the day a + listing grows ``--limit``, the shape does not move. + """ + return {"items": list(items), "total": len(items)} + + +# --- projects and schemas ---------------------------------------------------- + + +def project(value: Project) -> dict[str, Any]: + """A project. ``workspace_id`` is absent: a command speaks for one workspace.""" + return {"id": str(value.id), "name": value.name, "description": value.description} + + +def dataset(value: Dataset) -> dict[str, Any]: + """A project's one dataset.""" + return { + "id": str(value.id), + "project_id": str(value.project_id), + "name": value.name, + "description": value.description, + } + + +def attribute(value: Attribute) -> dict[str, Any]: + """One attribute of a label class. Also the *input* shape ``schema apply`` reads.""" + return { + "name": value.name, + "kind": value.kind, + "required": value.required, + "options": None if value.options is None else list(value.options), + "default": value.default, + } + + +def label_class(value: LabelClass) -> dict[str, Any]: + """One class of a schema version. Also the *input* shape ``schema apply`` reads.""" + return { + "name": value.name, + "geometry": value.geometry.value, + "color": value.color, + "attributes": [attribute(a) for a in value.attributes], + } + + +def schema_version(value: AnnotationSchema) -> dict[str, Any]: + """One version of a project's schema. Its own UUID is absent: nothing addresses it.""" + return { + "project_id": str(value.project_id), + "version": value.version, + "classes": [label_class(c) for c in value.classes], + } + + +# --- sources, ingest and assets ---------------------------------------------- + + +def video_provenance(value: VideoProvenance) -> dict[str, Any]: + """What a clip turned out to be, flattened the way the wire flattens it.""" + return { + "width": value.metadata.width, + "height": value.metadata.height, + "fps": value.metadata.fps, + "duration_seconds": value.metadata.duration_seconds, + "codec": value.metadata.codec, + "extraction_fps": value.extraction_fps, + } + + +def source(value: Source) -> dict[str, Any]: + """A registered origin. ``path`` is absent; ``name`` is its last component.""" + return { + "id": str(value.id), + "project_id": str(value.project_id), + "kind": value.kind.value, + "name": PurePath(value.path).name, + "registered_at": _moment(value.registered_at), + "video": None if value.video is None else video_provenance(value.video), + } + + +def ingest_failure(value: IngestFailure) -> dict[str, Any]: + """One file a run could not use, and why.""" + return {"name": value.name, "kind": value.kind.value, "reason": value.reason} + + +def ingest_job(value: IngestJob) -> dict[str, Any]: + """One run of one source, counters and per-item report included.""" + return { + "id": str(value.id), + "source_id": str(value.source_id), + "state": value.state.value, + "error": value.error, + "batch_id": None if value.batch_id is None else str(value.batch_id), + "batch_name": value.batch_name, + "processed": value.processed, + "total": value.total, + "failures": [ingest_failure(f) for f in value.failures], + } + + +def asset(value: Asset) -> dict[str, Any]: + """One image. ``uri`` is absent: it is a path on this machine.""" + return { + "id": str(value.id), + "project_id": str(value.project_id), + "modality": value.modality, + "content_hash": value.content_hash, + "width": value.width, + "height": value.height, + "format": None if value.format is None else value.format.value, + "source_id": None if value.source_id is None else str(value.source_id), + "frame_index": value.frame_index, + "frame_timestamp": value.frame_timestamp, + "thumbnail_hash": value.thumbnail_hash, + } + + +def thumbnail_backfill(value: ThumbnailBackfill) -> dict[str, Any]: + """A preview pass over a project. **CLI-defined**: no route reaches this.""" + return { + "project_id": str(value.project_id), + "examined": value.examined, + "filled": [str(i) for i in value.filled], + "missing": [str(i) for i in value.missing], + "unreadable": [ingest_failure(f) for f in value.unreadable], + } + + +# --- batches and jobs -------------------------------------------------------- + + +def progress_counts(counts: Mapping[AssetProgress, int]) -> dict[str, Any]: + """Five named fields and a total, not an open map — the wire model's own reason.""" + return { + "unannotated": counts[AssetProgress.UNANNOTATED], + "annotated": counts[AssetProgress.ANNOTATED], + "skipped": counts[AssetProgress.SKIPPED], + "review_pending": counts[AssetProgress.REVIEW_PENDING], + "accepted": counts[AssetProgress.ACCEPTED], + "total": sum(counts.values()), + } + + +def batch(value: Batch, counts: Mapping[AssetProgress, int]) -> dict[str, Any]: + """A batch and where its assets have got to. ``asset_ids`` is absent.""" + return { + "id": str(value.id), + "project_id": str(value.project_id), + "name": value.name, + "state": value.state.value, + "schema_version": value.schema_version, + "asset_count": len(value.asset_ids), + "progress": progress_counts(counts), + } + + +def job(value: AnnotationJob, *, batch_id: UUID) -> dict[str, Any]: + """One segment of a batch. ``task_group_id`` and the per-asset map are absent.""" + return { + "id": str(value.id), + "batch_id": str(batch_id), + "state": value.state.value, + "asset_count": len(value.progress), + } + + +def asset_progress(asset_id: UUID, progress: AssetProgress) -> dict[str, Any]: + """Where one asset of a job has got to.""" + return {"asset_id": str(asset_id), "progress": progress.value} + + +# --- releases, exports and formats ------------------------------------------- + + +def split_recipe(value: SplitRecipe) -> dict[str, Any]: + """How a release is cut for training.""" + return {"train": value.train, "val": value.val, "test": value.test, "seed": value.seed} + + +def release(value: Release) -> dict[str, Any]: + """A published snapshot of a dataset.""" + return { + "id": str(value.id), + "dataset_id": str(value.dataset_id), + "tag": value.tag, + "manifest_hash": value.manifest_hash, + "schema_version": value.schema_version, + "asset_count": value.asset_count, + "annotation_count": value.annotation_count, + "split": None if value.split is None else split_recipe(value.split), + "created_at": _moment(value.created_at), + "visionset_version": value.visionset_version, + } + + +def release_verification(value: ReleaseVerification) -> dict[str, Any]: + """The result of re-hashing everything a release names. ``ok`` is derived.""" + return { + "release_id": str(value.release_id), + "manifest_hash": value.manifest_hash, + "manifest_intact": value.manifest_intact, + "ok": value.ok, + "checked": value.checked, + "missing": list(value.missing), + "corrupt": list(value.corrupt), + "cache_mismatches": list(value.cache_mismatches), + } + + +def export_format(value: Exporter) -> dict[str, Any]: + """One installed exporter.""" + return {"name": value.format_name, "lossy": value.lossy} + + +def export_result(value: ExportResult) -> dict[str, Any]: + """What an export left on disk. **CLI-defined**: the API returns the archive. + + ``directory`` is here where ``Asset.uri`` is not, and the difference is who + chose it: this is the path the caller typed on ``--out``, so echoing it tells + them nothing they did not already say. + """ + return { + "release_id": str(value.release_id), + "format": value.format_name, + "directory": str(value.directory), + "file_count": value.file_count, + "total_bytes": value.total_bytes, + } diff --git a/src/visionset/cli/_output.py b/src/visionset/cli/_output.py new file mode 100644 index 00000000..f024e64f --- /dev/null +++ b/src/visionset/cli/_output.py @@ -0,0 +1,105 @@ +# usage: from visionset.cli._output import JsonOption, document, note, table +"""How a command prints: columns for a person, JSON for a program, prose to stderr. + +**Stdout is data; stderr is everything a person reads.** That rule predates this +module — ``token create`` put the secret alone on stdout so +``TOKEN=$(visionset token create --name ci)`` is exactly the secret — and this is +where it becomes the same three functions for every command. :func:`table` and +:func:`document` write to stdout and nothing else does; :func:`note` writes to +stderr and never competes with them. + +``--json`` is declared **per command**, not on the root callback, for the identical +Click reason ``--workspace`` is (see ``_workspace.py``): a group's parser stops at +the first non-option token, so an option on ``@app.callback()`` would have to +*precede* the subcommand — ``visionset --json project list`` would work and +``visionset project list --json`` would fail with "No such option". Nobody types +the first one. + +**Plain columns, never ``rich.table``.** Box drawing pads to ``$COLUMNS`` and +wraps, which makes output width-dependent and neither testable nor ``cut``-able. +The header prints even when there are no rows, so ``| tail -n +2`` is stable, and +**the first column of every listing is the id**, so ``awk '{print $1}'`` is stable +too — a name may hold internal whitespace, because ``normalize_name`` strips the +outside and deliberately preserves the middle. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from datetime import UTC, datetime +from typing import Annotated, Any, Final + +import typer + +JsonOption = Annotated[ + bool, + typer.Option("--json", help="Print one JSON document on stdout instead of columns."), +] +"""``--json``, for a command whose output a program might read. + +Module-level so that ``get_type_hints`` resolves it in the importing module's +globals under ``from __future__ import annotations`` — the same constraint that +puts ``WorkspaceOption`` at module level. Bound to a parameter named ``json_out`` +rather than ``json``, which would shadow the module every caller imports. +""" + +TIMESTAMP_FORMAT: Final = "%Y-%m-%dT%H:%M:%SZ" +"""Seconds, UTC, no offset. A listing is read by a person; microseconds are not. + +Deliberately **not** the format ``_json.py`` uses. That one has to agree with +pydantic's, because the JSON shapes are gated against the server's wire models; +this one has to be readable in a column. Sharing them would break the parity gate +in a way key-set comparison cannot see. +""" + +NEVER: Final = "-" +"""What a column shows for a timestamp that has not happened.""" + + +def moment(when: datetime | None) -> str: + """A timestamp as a column shows it.""" + return NEVER if when is None else when.astimezone(UTC).strftime(TIMESTAMP_FORMAT) + + +def widths(columns: Sequence[str], rows: Sequence[Sequence[str]]) -> list[int]: + """How wide each column has to be to hold its header and every cell.""" + return [max([len(header), *(len(row[i]) for row in rows)]) for i, header in enumerate(columns)] + + +def row(cells: Sequence[str], column_widths: Sequence[int]) -> str: + """One line of a table: cells left-justified, two spaces apart, no trailing pad. + + ``strict=True`` on the zip is the guard: a row with the wrong number of cells + raises here rather than silently losing its last column. + """ + return " ".join( + cell.ljust(width) for cell, width in zip(cells, column_widths, strict=True) + ).rstrip() + + +def table(columns: Sequence[str], rows: Sequence[Sequence[str]]) -> None: + """Print a listing on stdout, header first, even when there are no rows.""" + column_widths = widths(columns, rows) + typer.echo(row(columns, column_widths)) + for cells in rows: + typer.echo(row(cells, column_widths)) + + +def document(payload: Mapping[str, Any]) -> None: + """Print one JSON document on stdout — the whole of ``--json``'s output. + + One document per invocation, not JSON-lines: a listing is a single value, so + ``| jq '.items[]'`` works and a partial read is never mistaken for a whole one. + + ``json.dumps`` is called with **no** ``default=``, on purpose. A ``UUID``, a + ``datetime`` or a ``Path`` reaching this function is a projection in ``_json`` + that forgot to encode a leaf, and that must be a ``TypeError`` a test catches + rather than a silent ``str()`` nobody re-reads. + """ + typer.echo(json.dumps(payload, indent=2)) + + +def note(message: str) -> None: + """Say something to the person, on stderr, where it survives a redirection.""" + typer.echo(message, err=True) diff --git a/src/visionset/cli/_resolve.py b/src/visionset/cli/_resolve.py new file mode 100644 index 00000000..049cb6b3 --- /dev/null +++ b/src/visionset/cli/_resolve.py @@ -0,0 +1,78 @@ +# usage: from visionset.cli._resolve import ProjectOption, resolve_project, resolve_release +"""Turning what somebody typed into the thing they meant. + +A person at a terminal types ``--project road-signs``, not a UUID they have to +find first. Two resources can be named that way and no more: + +- a **project**, whose name is unique per workspace **case-insensitively**; +- a **release**, whose tag is unique per dataset and **case-sensitive**. + +Those two rules are opposites, and neither is spelled here. ``get_by_name`` and +``get_by_tag`` are kernel reads for exactly that reason — the comparison belongs +beside the index that enforces it, and a surface re-deriving one from prose is a +second spelling free to drift. What this module owns is only the dispatch: a +well-formed UUID is an id, anything else is a name. + +**Batches, jobs and assets are addressed by id and nothing else.** A batch has a +name but it is not unique — an ingest names one after its source, and re-ingesting +the same folder makes a second batch with a name just as good — so resolving one +by name would have to pick, and picking is worse than refusing. Their ids come +off the previous command's stdout, which is what the one-datum rule is for. + +A malformed id is Click's refusal at **exit 2**, not a kernel ``*NotFound`` at +exit 1: the same call the API makes, where a malformed UUID is 422 rather than +404 because the request could not have named anything. That is why the id-only +parameters are typed ``UUID`` and this module is not involved. +""" + +from __future__ import annotations + +from typing import Annotated +from uuid import UUID + +import typer + +from visionset.kernel.domain import Project, Release +from visionset.kernel.services import ProjectService, ReleaseService, WorkspaceService + +ProjectOption = Annotated[ + str, + typer.Option("--project", "-p", help="The project, by name or by id."), +] +"""``--project`` / ``-p``, for a command scoped to one project. + +Typed ``str`` rather than ``UUID`` precisely so a name gets through. The cost is +that a value which is neither reaches the kernel and comes back as +``ProjectNotFound`` at exit 1 — which is right, because unlike a malformed id it +*could* have named something. + +Module-level for the ``get_type_hints`` reason ``WorkspaceOption`` is. +""" + + +def resolve_project(workspace: WorkspaceService, reference: str) -> Project: + """The project that reference names, by id if it parses as one, else by name. + + A project whose *name* is a well-formed UUID string is unreachable by name. + That is harmless: the same string reaches it as an id, and it is the same + string either way. + """ + projects = ProjectService(workspace) + try: + project_id = UUID(reference) + except ValueError: + return projects.get_by_name(reference) + return projects.get(project_id) + + +def resolve_release(workspace: WorkspaceService, reference: str, tag: str) -> Release: + """The release under that tag, in the dataset of the project that reference names. + + Two lookups rather than one, because a release tag is unique per *dataset* + and a dataset is reached through its project. The intermediate read is not + waste: it is what makes an unknown project say so, instead of reporting a + perfectly good tag as missing. + """ + project = resolve_project(workspace, reference) + dataset = ProjectService(workspace).get_dataset(project.id) + return ReleaseService(workspace).get_by_tag(dataset.id, tag) diff --git a/src/visionset/cli/batches.py b/src/visionset/cli/batches.py new file mode 100644 index 00000000..9e8d9d9a --- /dev/null +++ b/src/visionset/cli/batches.py @@ -0,0 +1,191 @@ +# usage: from visionset.cli.batches import batch_app +"""``visionset batch`` — the lifecycle, and the gate into the trunk. + +Five commands, each exactly one service call: ``list``, then the one-way walk +``approve`` → ``start`` → ``complete``, then ``promote``. + +**There is no ``batch create``, and none of the membership commands.** A batch is +born from an ingest; curating one out of an arbitrary subset of assets has no +caller until a gallery exists to pick that subset in. ``BatchService`` still has +all four methods — this is a decision about the surface, not about the SDK, and +it is the same one the REST API made. + +``--jobs-of N`` is the ``BySize`` partition, and no flag spells ``BySegments``. +That variant's own docstring says the caller has already decided the split, and +the only caller that ever holds an exact partition is a program — which has the +SDK and the API. It is also the one partition that can be *wrong*, with four +distinct refusals; asking somebody to type tuples of UUIDs past a shell's quoting +is a way to meet all four. If it is ever wanted it arrives as ``--segments +FILE.json`` and nothing here moves. + +``promote`` lives under ``batch`` rather than under a dataset group because +``DatasetService.promote`` takes a *batch* id and derives the dataset from it — +the same reason its route hangs off ``/batches/{id}``. +""" + +from __future__ import annotations + +from typing import Annotated, Final +from uuid import UUID + +import typer + +from visionset.cli import _json +from visionset.cli._output import JsonOption, document, note, table +from visionset.cli._resolve import ProjectOption, resolve_project +from visionset.cli._workspace import WorkspaceOption, opened_workspace +from visionset.kernel.domain import AssetProgress, BySize, Partition +from visionset.kernel.services import BatchService, DatasetService, JobService + +batch_app = typer.Typer(help="Move batches through the annotation lifecycle.", no_args_is_help=True) + +BatchArgument = Annotated[UUID, typer.Argument(help="The batch, by id.")] +"""The batch a command acts on. Ids only — batch names are not unique. + +Module-level for the ``get_type_hints`` reason ``WorkspaceOption`` is. +""" + +_COLUMNS: Final = ("ID", "NAME", "STATE", "SCHEMA", "ASSETS", "ANNOTATED", "SETTLED") + +_NO_SCHEMA: Final = "-" +"""What a draft shows: approval is what pins a version, and it never moves after.""" + +_ACTOR: Final = "cli" +"""Who the dataset change log records for a promotion made at a terminal.""" + + +def _echo(batch_id: UUID, state: str, json_out: bool, payload: dict[str, object]) -> None: + """One shape for the four lifecycle commands: the batch, or a line and its id.""" + if json_out: + document(payload) + return + note(f"Batch {batch_id} is now {state}.") + typer.echo(str(batch_id)) + + +@batch_app.command("list") +def batch_list( + project: ProjectOption, + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """List a project's batches with where their assets have got to.""" + with opened_workspace(workspace) as service: + resolved = resolve_project(service, project) + batches = BatchService(service).list(resolved.id) + jobs = JobService(service) + # One progress read per batch, which is exactly what the REST listing + # does. The counts are the point of the listing: a batch's name and state + # do not say whether anybody has started on it. + counts = [jobs.batch_progress(batch.id) for batch in batches] + if json_out: + document(_json.page([_json.batch(b, c) for b, c in zip(batches, counts, strict=True)])) + return + table( + _COLUMNS, + [ + ( + str(batch.id), + batch.name, + batch.state.value, + _NO_SCHEMA if batch.schema_version is None else str(batch.schema_version), + str(len(batch.asset_ids)), + str(count[AssetProgress.ANNOTATED]), + str(sum(count.values()) - count[AssetProgress.UNANNOTATED]), + ) + for batch, count in zip(batches, counts, strict=True) + ], + ) + if not batches: + note(f"Project {resolved.name!r} has no batches yet.") + + +@batch_app.command("approve") +def batch_approve( + batch: BatchArgument, + jobs_of: Annotated[ + int | None, + typer.Option( + "--jobs-of", + min=1, + help="Cut into jobs of this many assets. Default: one job for the whole batch.", + ), + ] = None, + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """Freeze a batch's membership, pin the schema, and cut it into jobs. + + Approval is one-way. There is no route back to `draft`, because the jobs are + already partitioned against the pinned schema version — and a later + `schema apply` does not move that pin. + """ + # ``min=1`` rather than a check in the body: ``BySize.size`` is ``gt=0``, and + # a pydantic ``ValidationError`` from constructing one is not a + # ``VisionSetError``, so Click has to refuse zero before the domain sees it. + partition: Partition | None = None if jobs_of is None else BySize(size=jobs_of) + with opened_workspace(workspace) as service: + batches = BatchService(service) + approved = batches.approve(batch, partition) + counts = JobService(service).batch_progress(approved.id) + job_count = len(batches.jobs(approved.id)) + if json_out: + document(_json.batch(approved, counts)) + return + note( + f"Approved batch {approved.name!r} against schema version " + f"{approved.schema_version}, in {job_count} job(s)." + ) + typer.echo(str(approved.id)) + + +@batch_app.command("start") +def batch_start( + batch: BatchArgument, + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """Open an approved batch for annotation.""" + with opened_workspace(workspace) as service: + started = BatchService(service).start(batch) + counts = JobService(service).batch_progress(started.id) + _echo(started.id, started.state.value, json_out, _json.batch(started, counts)) + + +@batch_app.command("complete") +def batch_complete( + batch: BatchArgument, + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """Close a batch, once every one of its jobs is complete. + + Derived means recomputed, not automatic: this reads the jobs and refuses + while any is outstanding. + """ + with opened_workspace(workspace) as service: + completed = BatchService(service).complete(batch) + counts = JobService(service).batch_progress(completed.id) + _echo(completed.id, completed.state.value, json_out, _json.batch(completed, counts)) + + +@batch_app.command("promote") +def batch_promote( + batch: BatchArgument, + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """Move a completed batch's finished assets into the project's dataset. + + A union against what is already there, so promoting twice adds nothing and + logs nothing. Only `annotated` and `accepted` assets travel — a `skipped` one + was a decision, and it is honoured. + """ + with opened_workspace(workspace) as service: + promoted = DatasetService(service).promote(batch, actor=_ACTOR) + if json_out: + document(_json.page([_json.asset(a) for a in promoted])) + return + note(f"Promoted {len(promoted)} asset(s) into the dataset.") + for asset in promoted: + typer.echo(str(asset.id)) diff --git a/src/visionset/cli/export.py b/src/visionset/cli/export.py new file mode 100644 index 00000000..4ad5e027 --- /dev/null +++ b/src/visionset/cli/export.py @@ -0,0 +1,86 @@ +# usage: from visionset.cli.export import export +"""``visionset export`` — a release, an installed format, a directory. + +The kernel takes a plugin instance; it does not find one. ``ReleaseService.export`` +is handed an ``Exporter``, because import-linter forbids ``visionset.kernel`` +importing ``visionset.formats`` — a plugin registry is discovery at runtime, and +the kernel is the part that must not do any. So resolving a *name* to a plugin is +the surface's job, and this module does it the one supported way: +``formats.registry.exporter(name)``, never a dict lookup, because a ``KeyError`` +is outside the ``VisionSetError`` tree and would answer a typo with a traceback. + +**``--allow-lossy`` is a third gate word, never folded into ``--yes``.** ``--yes`` +guards destroying data and ``--allow-destructive`` guards narrowing a contract; +this guards emitting an incomplete *copy* of something that stays intact. Whether +a format is lossy is declared by the format, once, by whoever knows what it can +express — not asked per release, which would give a different answer as the data +drifts. + +The destination is the caller's. It is created if missing and **never emptied**, +so a second export into the same directory leaves the first run's files there and +the counts describe the directory afterwards rather than this run alone. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Annotated + +import typer + +from visionset.cli import _json +from visionset.cli._output import JsonOption, document, note +from visionset.cli._resolve import ProjectOption, resolve_release +from visionset.cli._workspace import WorkspaceOption, opened_workspace +from visionset.formats.registry import exporter +from visionset.kernel.services import ReleaseService + + +def export( + project: ProjectOption, + release: Annotated[str, typer.Option("--release", help="The release tag.")], + # ``format_name``, not ``format``: the builtin would be shadowed for the rest + # of the module. Typer takes the flag's spelling from the option, not the + # parameter, so ``--format`` is unaffected. + format_name: Annotated[ + str, typer.Option("--format", "-f", help="An installed exporter's name.") + ], + out: Annotated[ + Path, + typer.Option( + "--out", + "-o", + file_okay=False, + help="Where to write. Created if missing; never emptied.", + ), + ], + allow_lossy: Annotated[ + bool, + typer.Option( + "--allow-lossy", + help="Accept a format that cannot carry everything the release holds.", + ), + ] = False, + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """Write a release out in an installed format. + + `visionset format list` says which formats are installed. A name that is not + among them is refused with the list, at exit 1. + """ + with opened_workspace(workspace) as service: + # Inside the block on purpose: ``ExportFormatNotFound`` is a + # ``VisionSetError`` naming every installed format, and ``opened_workspace`` + # is what turns it into one sentence and exit 1. + plugin = exporter(format_name) + found = resolve_release(service, project, release) + result = ReleaseService(service).export(found.id, plugin, out, allow_lossy=allow_lossy) + if json_out: + document(_json.export_result(result)) + return + note( + f"Exported {found.tag!r} as {result.format_name}: " + f"{result.file_count} file(s), {result.total_bytes} byte(s)." + ) + typer.echo(str(result.directory)) diff --git a/src/visionset/cli/formats.py b/src/visionset/cli/formats.py new file mode 100644 index 00000000..bb0fa932 --- /dev/null +++ b/src/visionset/cli/formats.py @@ -0,0 +1,44 @@ +# usage: from visionset.cli.formats import format_app +"""``visionset format`` — which exporters this installation actually has. + +One command, and the only one besides ``init`` and ``--version`` that opens no +workspace: plugins are discovered from installed distributions through +``importlib.metadata``, which is a fact about the process rather than about any +dataset. Running it outside a workspace works, and that is the point — you ask +what is available *before* choosing an ``--format``. + +It earns a command because the valid values of ``export --format`` depend on what +somebody installed, and without this the only way to find out is to guess and +read the refusal. + +``lossy`` is declared by the format itself. A lossy exporter is not broken; it is +one whose file layout cannot express everything a release can hold — a bbox-only +format asked for a polygon — and ``export`` refuses it until ``--allow-lossy``. +""" + +from __future__ import annotations + +from typing import Final + +import typer + +from visionset.cli import _json +from visionset.cli._output import JsonOption, document, note, table +from visionset.formats.registry import exporters + +format_app = typer.Typer(help="Inspect installed export formats.", no_args_is_help=True) + +_COLUMNS: Final = ("NAME", "LOSSY") + + +@format_app.command("list") +def format_list(json_out: JsonOption = False) -> None: + """List the installed exporters, by name.""" + found = exporters() + installed = [found[name] for name in sorted(found)] + if json_out: + document(_json.page([_json.export_format(p) for p in installed])) + return + table(_COLUMNS, [(p.format_name, "yes" if p.lossy else "no") for p in installed]) + if not installed: + note("No exporters are installed.") diff --git a/src/visionset/cli/ingest.py b/src/visionset/cli/ingest.py new file mode 100644 index 00000000..492dcf4f --- /dev/null +++ b/src/visionset/cli/ingest.py @@ -0,0 +1,179 @@ +# usage: from visionset.cli.ingest import backfill_thumbnails, ingest +"""``visionset ingest`` — a path in, a batch out. And the preview backfill. + +**The one command in the CLI that is two service calls**, and it earns it. +``SourceService`` has two registration methods because a clip needs a rate and a +probe while a folder needs neither; ``IngestService`` has one ``ingest`` because +the source already carries the kind, the path and the rate. A person typing a +path does not want to say which of the two it is, and does not have to — the +dispatch is ``path.is_dir()``. + +Registering twice is free: registration is idempotent on +``(kind, path, extraction_fps)``, so running this again on the same folder finds +the same source. Ingesting again is nearly free too — content addressing means a +re-run creates no assets it created before — which is also the remedy for the one +gap this command has: interrupting it leaves the job row at ``running``, and +there is no ``--resume``, because re-running does the right thing and needs no +new vocabulary. + +**The run is synchronous, and nothing polls it.** The kernel writes progress to +the job row for a *second process* to read (that is what ``visionset ui`` and +``GET /ingest-jobs/{id}`` are for); a CLI that queued the work would have no +worker to run it. So this blocks, says so on stderr first, and prints the batch +id when it is done. + +**The batch id goes to stdout, alone** — it is what the next command in a script +needs, which is the whole one-datum rule:: + + BATCH=$(visionset ingest ./incoming --project road-signs) + +``backfill-thumbnails`` lives here rather than under a group because it has no +object group to join and it is the other half of what ingest writes: a preview is +a cache, so a missing one is a thing to fill in later rather than a failure to +report at the time. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Annotated, Final + +import typer + +from visionset.cli import _json +from visionset.cli._output import JsonOption, document, note, table +from visionset.cli._resolve import ProjectOption, resolve_project +from visionset.cli._workspace import WorkspaceOption, opened_workspace +from visionset.kernel.domain import IngestResult +from visionset.kernel.ports import DEFAULT_EXTRACTION_FPS +from visionset.kernel.services import IngestService, SourceService + +_FAILURE_COLUMNS: Final = ("FILE", "KIND", "REASON") + + +def _report(result: IngestResult) -> None: + """Say what the run did, on stderr, with the refused files named one per line.""" + note( + f"Ingested {result.created} new and {result.deduplicated} already-known " + f"assets into batch {result.batch_id}." + ) + if result.failures: + note(f"{result.failed} file(s) could not be used:") + for failure in result.failures: + note(f" {failure.name} {failure.kind.value} {failure.reason}") + + +def ingest( + source: Annotated[ + Path, + typer.Argument( + exists=True, + readable=True, + help="A directory of stills, or a video file.", + ), + ], + project: ProjectOption, + fps: Annotated[ + float | None, + typer.Option( + "--fps", + help=( + "Frames per second to extract. Video sources only; defaults to " + f"{DEFAULT_EXTRACTION_FPS}." + ), + ), + ] = None, + batch_name: Annotated[ + str | None, + typer.Option( + "--batch-name", + help="Name the batch this run fills. Defaults to the source's own name.", + ), + ] = None, + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """Register a source and ingest it, into one batch. + + A directory is read top level only, sorted, with no filter on the suffix — + anything that is not an image is reported per file and the run carries on. + A video file is decomposed into frames at `--fps`. + + Files are addressed by content, so ingesting the same bytes twice gives one + asset. That is what makes re-running this safe after an interruption. + """ + # ``typer.Option`` can express ``min=`` but not Click's ``min_open``, so a + # ``gt=0`` bound has to be checked here. It has to be checked *somewhere*: + # ``SourceService.register_video`` refuses a non-positive rate with a bare + # ``ValueError``, which is not a ``VisionSetError`` and would print a + # traceback rather than a sentence. + if fps is not None and fps <= 0: + raise typer.BadParameter("--fps must be greater than zero") + if fps is not None and source.is_dir(): + raise typer.BadParameter( + f"--fps applies to a video source; {source} is a directory of stills" + ) + + with opened_workspace(workspace) as service: + resolved = resolve_project(service, project) + sources = SourceService(service) + if source.is_dir(): + registered = sources.register_images(resolved.id, source) + else: + registered = sources.register_video( + resolved.id, + source, + extraction_fps=DEFAULT_EXTRACTION_FPS if fps is None else fps, + ) + note(f"Reading {registered.kind.value.replace('_', ' ')} {source}…") + result = IngestService(service).ingest(registered.id, batch_name=batch_name) + + if json_out: + document( + { + "source": _json.source(registered), + "job_id": str(result.job_id), + "batch_id": str(result.batch_id), + "created": result.created, + "deduplicated": result.deduplicated, + "failed": result.failed, + "failures": [_json.ingest_failure(f) for f in result.failures], + } + ) + return + _report(result) + typer.echo(str(result.batch_id)) + + +def backfill_thumbnails( + project: ProjectOption, + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """Render the missing previews of a project's assets. + + A preview is a cache, not an identity — its hash is in no release manifest + and no verification recomputes it — so an asset whose bytes will not render + keeps a null one and is reported here rather than having failed its ingest. + Idempotent: assets that already have one are not re-rendered. + + `missing` and `unreadable` are different damage. The first is a content blob + that is gone, which no preview pass can repair; the second is bytes that are + there and will not decode. + """ + with opened_workspace(workspace) as service: + resolved = resolve_project(service, project) + report = IngestService(service).backfill_thumbnails(resolved.id) + if json_out: + document(_json.thumbnail_backfill(report)) + return + note( + f"Examined {report.examined} asset(s) without a preview in {resolved.name!r}: " + f"{len(report.filled)} filled, {len(report.missing)} with no content blob, " + f"{len(report.unreadable)} unreadable." + ) + if report.unreadable: + table( + _FAILURE_COLUMNS, + [(f.name, f.kind.value, f.reason) for f in report.unreadable], + ) diff --git a/src/visionset/cli/init.py b/src/visionset/cli/init.py new file mode 100644 index 00000000..3733ddb0 --- /dev/null +++ b/src/visionset/cli/init.py @@ -0,0 +1,62 @@ +# usage: from visionset.cli.init import init +"""``visionset init`` — make a workspace, which every other command needs. + +Every other command takes a workspace that already exists, and finds it through +``resolve_workspace_root``. This one names where to *make* one, and that is why +it is the single command with no ``--workspace``: a positional ``PATH`` reads +correctly, and **nothing here walks upward**. Trading a stated directory for its +parent is how a workspace gets created in the wrong place — the argument that +kept the upward walk off the flag and the environment variable in the first +place, applied to the one operation where it would be irreversible. + +It does not reuse ``opened_workspace()`` either, which opens an *existing* +workspace. It needs ``domain_errors()`` on its own, and a ``close()`` in a +``finally``, because ``init`` hands back a workspace that is **open** and a +process that leaves one open strands ``visionset.db-wal`` beside it. + +**The root goes to stdout, alone**, on ``token create``'s rule: it is this +command's one piece of data, so ``WS=$(visionset init ./datasets)`` is exactly the +path and the two "what next" lines still reach the person on stderr. The path +printed is the *resolved* one, which is the useful answer — ``init .`` prints +where "." actually was. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Annotated + +import typer + +from visionset.cli._errors import domain_errors +from visionset.cli._output import note +from visionset.kernel.services import WorkspaceService + + +def init( + path: Annotated[ + Path, + typer.Argument(help="Where to create the workspace. Created if missing."), + ] = Path(), + name: Annotated[ + str | None, + typer.Option("--name", help="The workspace's name. Defaults to the directory's own."), + ] = None, +) -> None: + """Create a workspace here, or at PATH. + + The directory may be missing or empty; anything else is refused, so a typo + can never turn somebody's home directory into a workspace. Creating one where + a workspace already sits is refused too — the remedy is to use it, not to + make a second. + """ + with domain_errors(): + workspace = WorkspaceService.init(path, name=name) + try: + root = workspace.root + created = workspace.workspace.name + finally: + workspace.close() + note(f"Created workspace {created!r} at {root}.") + typer.echo(str(root)) + note("Next: visionset token create --name , then visionset ui.") diff --git a/src/visionset/cli/jobs.py b/src/visionset/cli/jobs.py new file mode 100644 index 00000000..c2afdf05 --- /dev/null +++ b/src/visionset/cli/jobs.py @@ -0,0 +1,190 @@ +# usage: from visionset.cli.jobs import job_app +"""``visionset job`` — the annotator's unit of work, driven from a shell. + +Six commands, each one service call: ``list``, ``next``, ``progress``, ``start``, +``mark``, ``complete``. + +``next`` and ``mark`` are not in #34's own list of deliverables, and without them +"the full cycle without touching Python" is not true — a batch cannot be +completed until every asset has settled, and nothing else here settles one. +``JobService.mark``'s docstring invites the second by name; the first is how a +shell learns which asset ids are still outstanding without this module rebuilding +the job-to-asset join the API does server-side. + +**Say the wart out loud.** ``--progress annotated`` records that somebody labeled +an asset, and the CLI writes no labels — geometry comes from a canvas or a model, +not from typing. A release published off a batch driven this way carries +``annotation_count: 0``, and that is what its manifest honestly says. The command +exists because the *lifecycle* has to be reachable from a script, not because it +is how labelling is meant to happen. + +Ids only, for jobs and assets both: neither has a name, and both ids come off the +previous command's stdout. +""" + +from __future__ import annotations + +from typing import Annotated, Final +from uuid import UUID + +import typer + +from visionset.cli import _json +from visionset.cli._output import JsonOption, document, note, table +from visionset.cli._workspace import WorkspaceOption, opened_workspace +from visionset.kernel.domain import AssetProgress +from visionset.kernel.services import BatchService, JobService + +job_app = typer.Typer(help="Drive annotation jobs.", no_args_is_help=True) + +JobArgument = Annotated[UUID, typer.Argument(help="The annotation job, by id.")] +"""Module-level for the ``get_type_hints`` reason ``WorkspaceOption`` is.""" + +_COLUMNS: Final = ("ID", "STATE", "ASSETS") + +_ASSET_COLUMNS: Final = ("ID", "CONTENT_HASH", "WIDTH", "HEIGHT") + +_PROGRESS_COLUMNS: Final = tuple(state.value.upper() for state in AssetProgress) + ("TOTAL",) +"""Read off the enum, so a sixth state cannot be silently missing from the table.""" + +_UNKNOWN: Final = "-" +"""What an asset with no recorded dimensions shows.""" + + +@job_app.command("list") +def job_list( + batch: Annotated[UUID, typer.Option("--batch", help="The batch whose jobs to list.")], + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """List a batch's jobs, in segment order. A draft batch has none.""" + with opened_workspace(workspace) as service: + jobs = BatchService(service).jobs(batch) + if json_out: + document(_json.page([_json.job(j, batch_id=batch) for j in jobs])) + return + table(_COLUMNS, [(str(j.id), j.state.value, str(len(j.progress))) for j in jobs]) + if not jobs: + note(f"Batch {batch} has no jobs; approve it first.") + + +@job_app.command("next") +def job_next( + job: JobArgument, + count: Annotated[ + int, + typer.Option("--count", "-n", min=1, help="How many to hand back."), + ] = 10, + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """The next assets of a job still awaiting annotation, in batch order. + + Order is the stored position, not insertion luck, so two callers asking for + the next ten get the same ten. + """ + # ``min=1`` because ``next_pending`` refuses a non-positive count with a bare + # ``ValueError``, which would print a traceback rather than a sentence. + with opened_workspace(workspace) as service: + assets = JobService(service).next_pending(job, count) + if json_out: + document(_json.page([_json.asset(a) for a in assets])) + return + table( + _ASSET_COLUMNS, + [ + ( + str(a.id), + a.content_hash, + _UNKNOWN if a.width is None else str(a.width), + _UNKNOWN if a.height is None else str(a.height), + ) + for a in assets + ], + ) + if not assets: + note(f"Job {job} has nothing left awaiting annotation.") + + +@job_app.command("progress") +def job_progress( + job: JobArgument, + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """How many of a job's assets sit in each state.""" + with opened_workspace(workspace) as service: + counts = JobService(service).job_progress(job) + if json_out: + document(_json.progress_counts(counts)) + return + table( + _PROGRESS_COLUMNS, + [tuple(str(counts[state]) for state in AssetProgress) + (str(sum(counts.values())),)], + ) + + +@job_app.command("start") +def job_start( + job: JobArgument, + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """Take a pending job.""" + with opened_workspace(workspace) as service: + service_jobs = JobService(service) + started = service_jobs.start(job) + batch = service_jobs.batch(started.id) + if json_out: + document(_json.job(started, batch_id=batch.id)) + return + note(f"Job {started.id} is now {started.state.value}.") + typer.echo(str(started.id)) + + +@job_app.command("mark") +def job_mark( + job: JobArgument, + asset: Annotated[UUID, typer.Argument(help="The asset in that job.")], + progress: Annotated[ + AssetProgress, + typer.Option("--progress", help="Where the asset has got to."), + ], + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """Record where one asset of a job has got to. + + Marking a state the asset already holds is a no-op — but the batch gate + fires first, so writing into a batch nobody opened is refused even when the + value would not change. + """ + with opened_workspace(workspace) as service: + JobService(service).mark(job, asset, progress) + if json_out: + document(_json.asset_progress(asset, progress)) + return + note(f"Asset {asset} is now {progress.value}.") + + +@job_app.command("complete") +def job_complete( + job: JobArgument, + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """Close a job, once every one of its assets has settled. + + Settled means annotated, skipped or accepted — review is optional, so a + reviewed-and-accepted asset and a plainly annotated one both count. + Completing a job never completes its batch; `batch complete` derives that. + """ + with opened_workspace(workspace) as service: + service_jobs = JobService(service) + completed = service_jobs.complete(job) + batch = service_jobs.batch(completed.id) + if json_out: + document(_json.job(completed, batch_id=batch.id)) + return + note(f"Job {completed.id} is now {completed.state.value}.") + typer.echo(str(completed.id)) diff --git a/src/visionset/cli/main.py b/src/visionset/cli/main.py index 9795266f..05615273 100644 --- a/src/visionset/cli/main.py +++ b/src/visionset/cli/main.py @@ -7,6 +7,15 @@ import typer from visionset import __version__ +from visionset.cli.batches import batch_app +from visionset.cli.export import export +from visionset.cli.formats import format_app +from visionset.cli.ingest import backfill_thumbnails, ingest +from visionset.cli.init import init +from visionset.cli.jobs import job_app +from visionset.cli.projects import project_app +from visionset.cli.releases import release_app +from visionset.cli.schemas import schema_app from visionset.cli.tokens import token_app from visionset.cli.ui import ui @@ -15,11 +24,32 @@ help="Robomous VisionSet — local-first dataset creation for computer vision.", no_args_is_help=True, ) + +# Registration is in cycle order — make a workspace, make a project, give it a +# schema, put images in it, work through them, publish, export — and ``--help`` +# keeps it, because Typer preserves declaration order rather than sorting. It +# keeps it *within each kind*: bare commands are listed before groups, so the +# listing reads as two passes over the cycle rather than one. That is Typer's +# own layout and not worth fighting; ``docs/cli.md``'s synopsis is where the +# cycle is shown in one sequence. +# +# Bare commands are registered here rather than decorated at their definition +# site: a ``@app.command()`` in ``ui.py`` would have to import this module, which +# imports ``ui.py``. Typer reads a command's annotations out of its *defining* +# module's globals either way, which is what lets the shared ``WorkspaceOption`` +# and ``JsonOption`` aliases resolve there. The name is spelled out rather than +# derived from the function, so ``backfill-thumbnails`` is not a guess. +app.command("init")(init) +app.add_typer(project_app, name="project") +app.add_typer(schema_app, name="schema") +app.command("ingest")(ingest) +app.add_typer(batch_app, name="batch") +app.add_typer(job_app, name="job") +app.add_typer(release_app, name="release") +app.command("export")(export) +app.add_typer(format_app, name="format") +app.command("backfill-thumbnails")(backfill_thumbnails) app.add_typer(token_app, name="token") -# Registered here rather than decorated at its definition site: a ``@app.command()`` -# in ``ui.py`` would have to import this module, which imports ``ui.py``. Typer -# reads a command's annotations out of its *defining* module's globals either -# way, which is what lets the shared ``WorkspaceOption`` alias resolve there. app.command()(ui) diff --git a/src/visionset/cli/projects.py b/src/visionset/cli/projects.py new file mode 100644 index 00000000..1d8ca0ab --- /dev/null +++ b/src/visionset/cli/projects.py @@ -0,0 +1,72 @@ +# usage: from visionset.cli.projects import project_app +"""``visionset project`` — the container everything else hangs off. + +Two commands, and both are one ``ProjectService`` call. The rules — a name unique +per workspace case-insensitively, a dataset created in the same transaction, a +blank name refused — are the kernel's and not one of them is restated here. + +``create`` takes its name **positionally** where ``token create`` takes ``--name``, +and the difference is what the name is. A token's name is metadata attached to a +credential whose actual output is the secret; a project's name *is* the project, +the way ``token revoke NAME`` already treats one. Its id goes to stdout alone, so +``P=$(visionset project create road-signs)`` works — though every other command +also takes the name, which is usually what a person types. + +There is deliberately no ``rename`` and no ``delete``. Both are administration +rather than flow, and both want a confirmation prompt and the cascade explained; +landing them together is how that gets documented once instead of twice. +""" + +from __future__ import annotations + +from typing import Annotated, Final + +import typer + +from visionset.cli import _json +from visionset.cli._output import JsonOption, document, note, table +from visionset.cli._workspace import WorkspaceOption, opened_workspace +from visionset.kernel.services import ProjectService + +project_app = typer.Typer(help="Create and list projects.", no_args_is_help=True) + +_COLUMNS: Final = ("ID", "NAME", "DESCRIPTION") + +_NONE: Final = "-" +"""What an absent description shows, so the column never collapses.""" + + +@project_app.command("create") +def project_create( + name: Annotated[str, typer.Argument(help="Unique in this workspace, ignoring case.")], + description: Annotated[ + str | None, typer.Option("--description", help="Free text, for people.") + ] = None, + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """Create a project and its empty dataset.""" + with opened_workspace(workspace) as service: + created = ProjectService(service).create(name, description) + if json_out: + document(_json.project(created)) + return + note(f"Created project {created.name!r}.") + typer.echo(str(created.id)) + + +@project_app.command("list") +def project_list( + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """List this workspace's projects, oldest first.""" + with opened_workspace(workspace) as service: + projects = ProjectService(service).list() + root = service.root + if json_out: + document(_json.page([_json.project(p) for p in projects])) + return + table(_COLUMNS, [(str(p.id), p.name, p.description or _NONE) for p in projects]) + if not projects: + note(f"No projects in {root}.") diff --git a/src/visionset/cli/releases.py b/src/visionset/cli/releases.py new file mode 100644 index 00000000..9e0984f4 --- /dev/null +++ b/src/visionset/cli/releases.py @@ -0,0 +1,175 @@ +# usage: from visionset.cli.releases import release_app +"""``visionset release`` — freezing a dataset, and proving it is still frozen. + +Three commands: ``publish``, ``list``, ``verify``. A release is the one truly +immutable artifact here, so there is no ``edit`` and no ``delete`` — the fix for +a wrong release is another release under another tag. + +``--split "0.7,0.15,0.15"`` is **one** option rather than three, because a split +is one concept, that is how it is written everywhere, and one flag means one +refusal to word. ``--seed`` stays separate; it is not a fraction. The recipe is +*stored*, not applied — folds are computed on demand from the frozen asset set, +keyed on content hash, which is why they come out the same on every machine. + +**``verify`` exits 1 when the answer is no.** Not because anything refused — +nothing did, the check ran and reported damage — but because that is the only way +a script branches on the result without grepping output, and it is what ``grep`` +and ``diff`` already mean by a non-zero exit. See ``EXIT_ANSWER_IS_NO`` in +``_errors.py``, where the two meanings of code 1 are written down. + +A tag is **case-sensitive** where a project name is not. Both rules live in the +kernel beside the index that enforces them, and neither is restated here. +""" + +from __future__ import annotations + +from typing import Annotated, Final + +import typer + +from visionset.cli import _json +from visionset.cli._errors import EXIT_ANSWER_IS_NO +from visionset.cli._output import JsonOption, document, moment, note, table +from visionset.cli._resolve import ProjectOption, resolve_project, resolve_release +from visionset.cli._workspace import WorkspaceOption, opened_workspace +from visionset.kernel.domain import SplitRecipe +from visionset.kernel.services import ProjectService, ReleaseService + +release_app = typer.Typer(help="Publish and verify releases.", no_args_is_help=True) + +_COLUMNS: Final = ("ID", "TAG", "ASSETS", "ANNOTATIONS", "SCHEMA", "CREATED") + +_SPLIT_PARTS: Final = 3 +"""``--split`` is train, val and test — exactly three numbers, in that order.""" + + +def _split_of(value: str | None, seed: int) -> SplitRecipe | None: + """``"0.7,0.15,0.15"`` as a recipe, or a usage error saying what is wrong. + + ``SplitRecipe`` refuses fractions that do not add up, with a pydantic + ``ValidationError`` — not a ``VisionSetError``, so it would print a traceback + rather than a sentence. Caught here and re-raised as Click's own refusal, + which is what exit 2 is for. + """ + if value is None: + return None + parts = value.split(",") + if len(parts) != _SPLIT_PARTS: + raise typer.BadParameter("--split takes three fractions: TRAIN,VAL,TEST") + try: + train, val, test = (float(part) for part in parts) + except ValueError as exc: + raise typer.BadParameter(f"--split takes three numbers, not {value!r}") from exc + try: + return SplitRecipe(train=train, val=val, test=test, seed=seed) + except ValueError as exc: + raise typer.BadParameter(f"--split {value!r}: {exc}") from exc + + +@release_app.command("publish") +def release_publish( + tag: Annotated[str, typer.Option("--tag", help="The release's name in this dataset.")], + project: ProjectOption, + split: Annotated[ + str | None, + typer.Option( + "--split", + metavar="TRAIN,VAL,TEST", + help="Fractions adding up to 1.0, e.g. 0.7,0.15,0.15.", + ), + ] = None, + seed: Annotated[int, typer.Option("--seed", help="Fixes the fold assignment.")] = 0, + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """Freeze the project's dataset as it stands, under a tag. + + What is frozen: every asset in the trunk by content hash, every annotation on + those assets copied rather than referenced, and the active schema version. + What is not: the time, the tag and the release id, which live on the row — + so publishing twice from an unchanged dataset produces byte-identical + manifests that share one blob. + """ + recipe = _split_of(split, seed) + with opened_workspace(workspace) as service: + resolved = resolve_project(service, project) + dataset = ProjectService(service).get_dataset(resolved.id) + published = ReleaseService(service).publish(dataset.id, tag, split=recipe) + if json_out: + document(_json.release(published)) + return + note( + f"Published {published.tag!r}: {published.asset_count} asset(s), " + f"{published.annotation_count} annotation(s), schema version " + f"{published.schema_version}." + ) + typer.echo(str(published.id)) + + +@release_app.command("list") +def release_list( + project: ProjectOption, + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """List a project's releases, oldest first.""" + with opened_workspace(workspace) as service: + resolved = resolve_project(service, project) + dataset = ProjectService(service).get_dataset(resolved.id) + releases = ReleaseService(service).list(dataset.id) + if json_out: + document(_json.page([_json.release(r) for r in releases])) + return + table( + _COLUMNS, + [ + ( + str(r.id), + r.tag, + str(r.asset_count), + str(r.annotation_count), + str(r.schema_version), + moment(r.created_at), + ) + for r in releases + ], + ) + if not releases: + note(f"Project {resolved.name!r} has published no releases.") + + +@release_app.command("verify") +def release_verify( + tag: Annotated[str, typer.Argument(help="The release tag, case-sensitively.")], + project: ProjectOption, + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """Re-read and re-hash everything a release names. + + Exits 0 when the release is intact and **1 when it is not** — the answer is + the exit code, so `visionset release verify v1.0 -p road && train.sh` is a + sensible thing to write. + + A manifest that fails its own hash stops the walk: reporting assets missing + on the strength of a tampered inventory would be worse than saying nothing. + """ + with opened_workspace(workspace) as service: + release = resolve_release(service, project, tag) + report = ReleaseService(service).verify(release.id) + if json_out: + document(_json.release_verification(report)) + elif report.ok: + note(f"Release {release.tag!r} verifies: {report.checked} blob(s) intact.") + else: + if not report.manifest_intact: + note(f"Release {release.tag!r}: the manifest itself does not match its hash.") + for label, hashes in ( + ("missing", report.missing), + ("corrupt", report.corrupt), + ("stale in the row's cache", report.cache_mismatches), + ): + for value in hashes: + note(f" {label}: {value}") + if not report.ok: + raise typer.Exit(code=EXIT_ANSWER_IS_NO) diff --git a/src/visionset/cli/schemas.py b/src/visionset/cli/schemas.py new file mode 100644 index 00000000..d2912cc7 --- /dev/null +++ b/src/visionset/cli/schemas.py @@ -0,0 +1,145 @@ +# usage: from visionset.cli.schemas import schema_app +"""``visionset schema`` — applying a schema version from a file. + +The document is **JSON**, read with the standard library, and it is +byte-for-byte the same document ``POST /projects/{id}/schema/versions`` takes:: + + {"classes": [{"name": "sign", "geometry": "bbox", "color": "#ff0000", + "attributes": [{"name": "occluded", "kind": "boolean", + "required": true, "options": null, + "default": false}]}]} + +No YAML, and the reason is not taste: a second file format means a runtime +dependency in every wheel, a second parser to keep honest, and two shapes that +can disagree — while the surface a schema file has to interoperate with, the REST +API, speaks JSON already. ``yq . schema.yaml`` is one pipe away for whoever wants +one. + +**The document parses through the domain, not through a hand-written reader.** +``TypeAdapter(tuple[LabelClass, ...])`` runs ``LabelClass``'s and ``Attribute``'s +own validators — the same ones ``AttributeBody._the_domain_accepts_it`` calls on +the server side — so a ``select`` with no options, a duplicate attribute name or a +blank class name is refused in the kernel's own wording and nothing is restated +here. + +Two failures on the way in are **not** ``VisionSetError``: a file that is not JSON +(``json.JSONDecodeError``) and a document that is JSON but not this shape (a +pydantic ``ValidationError``). ``domain_errors()`` deliberately does not catch +either, so both become ``typer.BadParameter`` at **exit 2** — the CLI's 422, +matching the call the server makes when it moves the same failure into request +parsing. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Annotated, Final + +import typer +from pydantic import TypeAdapter, ValidationError + +from visionset.cli import _json +from visionset.cli._output import JsonOption, document, note, table +from visionset.cli._resolve import ProjectOption, resolve_project +from visionset.cli._workspace import WorkspaceOption, opened_workspace +from visionset.kernel.domain import LabelClass +from visionset.kernel.services import SchemaService + +schema_app = typer.Typer(help="Apply and inspect annotation schemas.", no_args_is_help=True) + +_COLUMNS: Final = ("VERSION", "CLASSES", "GEOMETRIES") + +_CLASSES: Final = TypeAdapter(tuple[LabelClass, ...]) +"""The document's one field, parsed by the domain models themselves.""" + + +def _read_classes(file: Path) -> tuple[LabelClass, ...]: + """The classes in that file, or a usage error naming what is wrong with it.""" + try: + loaded = json.loads(file.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise typer.BadParameter(f"{file} is not valid JSON: {exc}") from exc + if not isinstance(loaded, dict) or "classes" not in loaded: + raise typer.BadParameter(f'{file} must be an object with a "classes" list') + try: + return _CLASSES.validate_python(loaded["classes"]) + except ValidationError as exc: + details = "; ".join( + f"classes.{'.'.join(str(part) for part in error['loc'])}: {error['msg']}" + for error in exc.errors() + ) + raise typer.BadParameter(f"{file}: {details}") from exc + + +@schema_app.command("apply") +def schema_apply( + file: Annotated[ + Path, + typer.Argument( + exists=True, + dir_okay=False, + readable=True, + help='A JSON document: {"classes": [...]}.', + ), + ], + project: ProjectOption, + allow_destructive: Annotated[ + bool, + typer.Option( + "--allow-destructive", + help="Accept a change that narrows the contract by removing something.", + ), + ] = False, + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """Add the next schema version from a JSON file. + + Versions are 1..N and none of them ever changes, so this always *adds* one — + there is no edit and no rollback, and applying an unchanged document still + makes a new version. + + A change that removes a class or an attribute, or narrows one, is refused + until `--allow-destructive`. A change that would orphan annotations already + written under an affected class has **no** override, deliberately. + """ + classes = _read_classes(file) + with opened_workspace(workspace) as service: + resolved = resolve_project(service, project) + version = SchemaService(service).create_version( + resolved.id, classes, allow_destructive=allow_destructive + ) + if json_out: + document(_json.schema_version(version)) + return + note(f"Applied schema version {version.version} to {resolved.name!r}.") + typer.echo(str(version.version)) + + +@schema_app.command("list") +def schema_list( + project: ProjectOption, + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """List a project's schema versions, oldest first. The last one is active.""" + with opened_workspace(workspace) as service: + resolved = resolve_project(service, project) + versions = SchemaService(service).list_versions(resolved.id) + if json_out: + document(_json.page([_json.schema_version(v) for v in versions])) + return + table( + _COLUMNS, + [ + ( + str(v.version), + str(len(v.classes)), + ",".join(sorted({c.geometry.value for c in v.classes})), + ) + for v in versions + ], + ) + if not versions: + note(f"Project {resolved.name!r} has no schema yet.") diff --git a/src/visionset/cli/tokens.py b/src/visionset/cli/tokens.py index 47023b9c..10782447 100644 --- a/src/visionset/cli/tokens.py +++ b/src/visionset/cli/tokens.py @@ -21,31 +21,25 @@ from __future__ import annotations -from datetime import UTC, datetime from typing import Annotated, Final import typer +from visionset.cli._output import moment, note, table from visionset.cli._workspace import WorkspaceOption, opened_workspace from visionset.kernel.services import TokenService token_app = typer.Typer(help="Manage API tokens.", no_args_is_help=True) _COLUMNS: Final = ("NAME", "CREATED", "REVOKED") +"""NAME first, unlike every other listing, which leads with an id. -_TIMESTAMP_FORMAT: Final = "%Y-%m-%dT%H:%M:%SZ" -"""Seconds, UTC, no offset. A listing is read by a person; microseconds are not.""" - -_NEVER: Final = "-" -"""What a live token shows in the REVOKED column.""" - - -def _moment(when: datetime | None) -> str: - return _NEVER if when is None else when.astimezone(UTC).strftime(_TIMESTAMP_FORMAT) - - -def _row(cells: tuple[str, ...], widths: list[int]) -> str: - return " ".join(cell.ljust(w) for cell, w in zip(cells, widths, strict=True)).rstrip() +Kept as it is because a token has no id a person would ever type — ``revoke`` +takes the name — and moving the column would break scripts for no gain. The +consequence is that ``awk '{print $1}'`` is not safe *here*, since a token name +may hold internal whitespace; ``docs/cli.md`` says so where it states the +id-first rule for the flow listings. +""" @token_app.command("create") @@ -57,7 +51,7 @@ def token_create( with opened_workspace(workspace) as service: issued = TokenService(service).create(name) root = service.root - typer.echo(f"Created token {issued.token.name!r} in {root}.", err=True) + note(f"Created token {issued.token.name!r} in {root}.") typer.echo(issued.secret) typer.secho( "This secret is shown once and cannot be recovered. Store it now.", @@ -69,8 +63,8 @@ def token_create( @token_app.command("revoke") def token_revoke( name: Annotated[str, typer.Argument(help="The token to burn.")], - workspace: WorkspaceOption = None, yes: Annotated[bool, typer.Option("--yes", "-y", help="Do not ask.")] = False, + workspace: WorkspaceOption = None, ) -> None: """Burn a token. Every client holding its secret stops working. @@ -88,10 +82,7 @@ def token_revoke( # The kernel's no-op, surfaced. Exit 0, and do not ask: a retried # ``token revoke ci`` must be safe, and prompting to redo something # already done invites a "yes" that means nothing. - typer.echo( - f"Token {token.name!r} was already revoked at {_moment(token.revoked_at)}.", - err=True, - ) + note(f"Token {token.name!r} was already revoked at {moment(token.revoked_at)}.") return if not yes: # ``ConfirmationRequired`` exists because the kernel has no terminal; @@ -105,7 +96,7 @@ def token_revoke( ) tokens.revoke(token.id, confirm=True) burned = token.name - typer.echo(f"Revoked token {burned!r}.", err=True) + note(f"Revoked token {burned!r}.") @token_app.command("list") @@ -113,17 +104,12 @@ def token_list(workspace: WorkspaceOption = None) -> None: """List this workspace's tokens, revoked ones included. Never their secrets.""" with opened_workspace(workspace) as service: rows = [ - (token.name, _moment(token.created_at), _moment(token.revoked_at)) + (token.name, moment(token.created_at), moment(token.revoked_at)) for token in TokenService(service).list() ] root = service.root - widths = [ - max([len(header), *(len(row[i]) for row in rows)]) for i, header in enumerate(_COLUMNS) - ] # The header prints whether or not there are rows, so ``| tail -n +2`` is # stable; the "none" note goes to stderr, where notes go. - typer.echo(_row(_COLUMNS, widths)) - for row in rows: - typer.echo(_row(row, widths)) + table(_COLUMNS, rows) if not rows: - typer.echo(f"No tokens in {root}.", err=True) + note(f"No tokens in {root}.") diff --git a/src/visionset/kernel/services/project_service.py b/src/visionset/kernel/services/project_service.py index 9b703b86..68e80e74 100644 --- a/src/visionset/kernel/services/project_service.py +++ b/src/visionset/kernel/services/project_service.py @@ -60,6 +60,23 @@ def get(self, project_id: UUID) -> Project: with self._workspace.unit_of_work() as uow: return self._require(uow, project_id) + def get_by_name(self, name: str) -> Project: + """The project an operator would name, resolved case-insensitively. + + Here rather than in a surface, on ``TokenService.get_by_name``'s + precedent, because the comparison is not obvious and it is not the only + one: a project name is unique **case-insensitively** while a release tag + is case-sensitive. A CLI or an MCP tool that re-derived either rule from + prose would be a second spelling of it, free to drift from the index that + actually enforces it. + + Raises: + InvalidName: the name is blank once stripped. + ProjectNotFound: no project in this workspace holds that name. + """ + with self._workspace.unit_of_work() as uow: + return self.require_project_named(uow, name) + def list(self) -> list[Project]: """Every project in this workspace, in the order they were created.""" with self._workspace.unit_of_work() as uow: @@ -165,6 +182,27 @@ def _require(self, uow: UnitOfWork, project_id: UUID) -> Project: ) return project + def require_project_named(self, uow: UnitOfWork, name: str) -> Project: + """The project holding that name, compared the way the index compares. + + Unicode case folding here, ASCII ``COLLATE NOCASE`` in the index — the + service is where the normalized string is in hand, so it is the stricter + of the two. ``require_project_name`` is its opposite number and answers a + different question: that one refuses a name because it is *taken*, this + one resolves a name because it is. + + Public, and taking a ``uow``, for the reason ``JobService.require_job`` + is: a caller resolving a project inside its own transaction must not have + to spell the comparison a second time. + """ + wanted = self._workspace.normalize_project_name(name).casefold() + for project in uow.projects.list(self._workspace.workspace_id): + if project.name.casefold() == wanted: + return project + raise ProjectNotFound( + f"no project named {name!r} in workspace {self._workspace.workspace.name!r}" + ) + def require_dataset(self, uow: UnitOfWork, project_id: UUID) -> Dataset: """The project's one dataset. diff --git a/src/visionset/kernel/services/release_service.py b/src/visionset/kernel/services/release_service.py index 9c9a29d6..c0124157 100644 --- a/src/visionset/kernel/services/release_service.py +++ b/src/visionset/kernel/services/release_service.py @@ -214,6 +214,28 @@ def get(self, release_id: UUID) -> Release: with self._workspace.unit_of_work() as uow: return self._require_release(uow, release_id) + def get_by_tag(self, dataset_id: UUID, tag: str) -> Release: + """The release published under that tag in that dataset. + + **Case-sensitive**, matching ``publish``: a tag is an identifier, not a + label somebody reads, and ``uq_release_dataset_tag`` compares it exactly. + That is the whole reason this lives here rather than in a surface — it is + the *opposite* rule to ``ProjectService.get_by_name``'s, and a caller + re-deriving either from prose would eventually get one of them wrong. + + Raises: + DatasetNotFound: no such dataset in this workspace. + InvalidName: the tag is blank once stripped. + ReleaseNotFound: that dataset has no release under that tag. + """ + cleaned = normalize_name(tag, what="release tag") + with self._workspace.unit_of_work() as uow: + dataset = self._datasets.require_dataset(uow, dataset_id) + for release in uow.releases.list(dataset.id): + if release.tag == cleaned: + return release + raise ReleaseNotFound(f"dataset {dataset.name!r} has no release tagged {cleaned!r}") + def manifest(self, release_id: UUID) -> Manifest: """The frozen document this release names, read back out of the blob store. diff --git a/tests/cli/_flow.py b/tests/cli/_flow.py new file mode 100644 index 00000000..c5401674 --- /dev/null +++ b/tests/cli/_flow.py @@ -0,0 +1,157 @@ +# usage: from tests.cli._flow import ok, run, workspace +"""Walking the CLI up to a given rung, by invoking the CLI. + +Plain functions, the way ``tests/server/_flow.py`` and ``tests/fixtures/media.py`` +are plain functions — there is no ``conftest.py`` anywhere in this repository and +this is not the module that starts one. + +**Every rung is reached by running commands, never by calling the SDK.** A helper +that reached for ``BatchService`` to build "an approved batch" would test the +later command against a state no user can produce; building it with +``visionset batch approve`` means the ladder is itself under test on the way up. +The one exception is reading state *back* for an assertion, which a test does +through the kernel — output is what is being checked, so it cannot also be the +evidence. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from click.testing import Result +from tests.fixtures.media import write_images, write_unsupported_file +from typer.testing import CliRunner + +from visionset.cli.main import app + +runner = CliRunner() + +SCHEMA_DOCUMENT = { + "classes": [ + { + "name": "sign", + "geometry": "bbox", + "color": "#ff0000", + "attributes": [{"name": "occluded", "kind": "boolean", "default": False}], + } + ] +} +"""The smallest schema that is not trivial: one class, one optional attribute.""" + +IMAGE_COUNT = 6 +"""Six stills, so ``--jobs-of 3`` cuts exactly two jobs with no remainder.""" + + +def run(root: Path, *argv: str) -> Result: + """Invoke the real app against a workspace, without asserting anything.""" + return runner.invoke(app, [*argv, "--workspace", str(root)]) + + +def ok(root: Path, *argv: str) -> str: + """Invoke, insist it worked, and hand back stdout — usually an id.""" + result = run(root, *argv) + assert result.exit_code == 0, result.output + return result.stdout.strip() + + +def payload(root: Path, *argv: str) -> dict: + """The ``--json`` document a command printed, parsed.""" + return json.loads(ok(root, *argv, "--json")) + + +def workspace(tmp_path: Path) -> Path: + """A workspace created the way a person creates one.""" + root = tmp_path / "ws" + result = runner.invoke(app, ["init", str(root)]) + assert result.exit_code == 0, result.output + return root + + +def project(root: Path, name: str = "road-signs") -> str: + """A project, named.""" + ok(root, "project", "create", name) + return name + + +def schema_file(tmp_path: Path) -> Path: + """``SCHEMA_DOCUMENT`` on disk, ready for ``schema apply``.""" + path = tmp_path / "schema.json" + path.write_text(json.dumps(SCHEMA_DOCUMENT), encoding="utf-8") + return path + + +def stills(tmp_path: Path, *, count: int = IMAGE_COUNT, stray: bool = False) -> Path: + """A folder of distinct images, optionally with one file that is not an image.""" + directory = tmp_path / "incoming" + write_images(directory, count=count) + if stray: + write_unsupported_file(directory / "notes.txt") + return directory + + +def schemad_project(root: Path, tmp_path: Path, name: str = "road-signs") -> str: + """A project with schema version 1 applied.""" + project(root, name) + ok(root, "schema", "apply", str(schema_file(tmp_path)), "--project", name) + return name + + +def ingested_batch(root: Path, tmp_path: Path, *, stray: bool = False) -> tuple[str, str]: + """A project with a schema and one draft batch full of stills.""" + name = schemad_project(root, tmp_path) + batch = ok( + root, + "ingest", + str(stills(tmp_path, stray=stray)), + "--project", + name, + "--batch-name", + "stills", + ) + return name, batch + + +def started_batch(root: Path, tmp_path: Path, *, jobs_of: int | None = None) -> tuple[str, str]: + """The same, approved and opened for annotation.""" + name, batch = ingested_batch(root, tmp_path) + approve = ["batch", "approve", batch] + if jobs_of is not None: + approve += ["--jobs-of", str(jobs_of)] + ok(root, *approve) + ok(root, "batch", "start", batch) + return name, batch + + +def jobs_of(root: Path, batch: str) -> list[str]: + """The ids in a batch's job listing, the way a shell reads them.""" + return [line.split()[0] for line in ok(root, "job", "list", "--batch", batch).splitlines()[1:]] + + +def completed_batch( + root: Path, tmp_path: Path, *, jobs_of_size: int | None = None +) -> tuple[str, str]: + """Every asset marked ``annotated``, every job closed, the batch closed.""" + name, batch = started_batch(root, tmp_path, jobs_of=jobs_of_size) + for job in jobs_of(root, batch): + ok(root, "job", "start", job) + listing = ok(root, "job", "next", job, "-n", "100").splitlines()[1:] + for line in listing: + ok(root, "job", "mark", job, line.split()[0], "--progress", "annotated") + ok(root, "job", "complete", job) + ok(root, "batch", "complete", batch) + return name, batch + + +def promoted_project(root: Path, tmp_path: Path) -> str: + """A dataset with something in it, which is what publishing needs.""" + name, batch = completed_batch(root, tmp_path) + ok(root, "batch", "promote", batch) + return name + + +def published_release(root: Path, tmp_path: Path, tag: str = "v1.0") -> str: + """A published release, and the project it belongs to.""" + name = promoted_project(root, tmp_path) + ok(root, "release", "publish", "--tag", tag, "--project", name) + return name diff --git a/tests/cli/test_batch_commands.py b/tests/cli/test_batch_commands.py new file mode 100644 index 00000000..3e62f00a --- /dev/null +++ b/tests/cli/test_batch_commands.py @@ -0,0 +1,190 @@ +"""``visionset batch`` — the one-way lifecycle, and the gate into the trunk.""" + +from __future__ import annotations + +from pathlib import Path +from uuid import uuid4 + +import pytest +from tests.cli._flow import ( + completed_batch, + ingested_batch, + jobs_of, + ok, + payload, + run, + started_batch, + workspace, +) + +from visionset.kernel.services import ( + WORKSPACE_ENV_VAR, + DatasetService, + ProjectService, + WorkspaceService, +) + + +@pytest.fixture(autouse=True) +def _no_ambient_workspace(monkeypatch: pytest.MonkeyPatch) -> None: + """A developer with ``VISIONSET_WORKSPACE`` exported gets CI's results.""" + monkeypatch.delenv(WORKSPACE_ENV_VAR, raising=False) + + +@pytest.fixture() +def root(tmp_path: Path) -> Path: + return workspace(tmp_path) + + +def _trunk_size(root: Path, name: str) -> int: + with WorkspaceService.open(root) as service: + project = ProjectService(service).get_by_name(name) + dataset = ProjectService(service).get_dataset(project.id) + return len(DatasetService(service).assets(dataset.id)) + + +# --- list -------------------------------------------------------------------- + + +def test_list_leads_with_the_id_and_names_the_state(root: Path, tmp_path: Path) -> None: + name, batch = ingested_batch(root, tmp_path) + rows = ok(root, "batch", "list", "-p", name).splitlines() + assert rows[0].split() == ["ID", "NAME", "STATE", "SCHEMA", "ASSETS", "ANNOTATED", "SETTLED"] + assert rows[1].split()[:5] == [batch, "stills", "draft", "-", "6"] + + +def test_a_draft_shows_no_pinned_schema(root: Path, tmp_path: Path) -> None: + # Approval is what pins a version, and it never moves after — so a draft + # showing one would be a claim nothing supports. + name, _ = ingested_batch(root, tmp_path) + assert payload(root, "batch", "list", "-p", name)["items"][0]["schema_version"] is None + + +def test_list_json_carries_the_progress_counts(root: Path, tmp_path: Path) -> None: + name, _ = started_batch(root, tmp_path) + progress = payload(root, "batch", "list", "-p", name)["items"][0]["progress"] + assert progress == { + "unannotated": 6, + "annotated": 0, + "skipped": 0, + "review_pending": 0, + "accepted": 0, + "total": 6, + } + + +def test_an_empty_listing_still_prints_its_header(root: Path, tmp_path: Path) -> None: + ok(root, "project", "create", "empty") + result = run(root, "batch", "list", "-p", "empty") + assert len(result.stdout.splitlines()) == 1 + assert "no batches yet" in result.stderr + + +# --- approve ----------------------------------------------------------------- + + +def test_approve_with_no_flag_cuts_one_job(root: Path, tmp_path: Path) -> None: + _, batch = ingested_batch(root, tmp_path) + result = run(root, "batch", "approve", batch) + assert result.exit_code == 0, result.output + assert "in 1 job(s)" in result.stderr + assert len(jobs_of(root, batch)) == 1 + + +def test_jobs_of_cuts_by_size(root: Path, tmp_path: Path) -> None: + _, batch = ingested_batch(root, tmp_path) + ok(root, "batch", "approve", batch, "--jobs-of", "3") + assert len(jobs_of(root, batch)) == 2 + + +def test_the_last_job_takes_the_remainder(root: Path, tmp_path: Path) -> None: + _, batch = ingested_batch(root, tmp_path) + ok(root, "batch", "approve", batch, "--jobs-of", "4") + sizes = [ + int(line.split()[2]) for line in ok(root, "job", "list", "--batch", batch).splitlines()[1:] + ] + assert sizes == [4, 2] + + +def test_approve_pins_the_active_schema_version(root: Path, tmp_path: Path) -> None: + _, batch = ingested_batch(root, tmp_path) + assert payload(root, "batch", "approve", batch)["schema_version"] == 1 + + +def test_jobs_of_zero_exits_two(root: Path, tmp_path: Path) -> None: + # ``BySize.size`` is ``gt=0`` and a pydantic error would print a traceback, + # so Click's ``min=1`` has to catch it first. + _, batch = ingested_batch(root, tmp_path) + result = run(root, "batch", "approve", batch, "--jobs-of", "0") + assert result.exit_code == 2, result.output + + +def test_approving_twice_exits_one(root: Path, tmp_path: Path) -> None: + # One-way: there is no route back to draft, because the jobs are already cut + # against the pin. + _, batch = ingested_batch(root, tmp_path) + ok(root, "batch", "approve", batch) + assert run(root, "batch", "approve", batch).exit_code == 1 + + +def test_a_malformed_batch_id_exits_two(root: Path) -> None: + # Click's ``UUID`` type, and the same call the API makes: a malformed id is + # 422 rather than 404, because the request could not have named anything. + assert run(root, "batch", "approve", "not-a-uuid").exit_code == 2 + + +def test_an_unknown_batch_id_exits_one(root: Path) -> None: + assert run(root, "batch", "approve", str(uuid4())).exit_code == 1 + + +# --- start, complete --------------------------------------------------------- + + +def test_start_opens_an_approved_batch(root: Path, tmp_path: Path) -> None: + _, batch = ingested_batch(root, tmp_path) + ok(root, "batch", "approve", batch) + assert payload(root, "batch", "start", batch)["state"] == "in_annotation" + + +def test_starting_a_draft_exits_one(root: Path, tmp_path: Path) -> None: + _, batch = ingested_batch(root, tmp_path) + assert run(root, "batch", "start", batch).exit_code == 1 + + +def test_complete_with_an_outstanding_job_exits_one(root: Path, tmp_path: Path) -> None: + # Derived means recomputed, not automatic. + _, batch = started_batch(root, tmp_path) + result = run(root, "batch", "complete", batch) + assert result.exit_code == 1, result.output + assert "Error:" in result.stderr + + +def test_complete_closes_a_finished_batch(root: Path, tmp_path: Path) -> None: + name, _ = completed_batch(root, tmp_path) + assert payload(root, "batch", "list", "-p", name)["items"][0]["state"] == "completed" + + +# --- promote ----------------------------------------------------------------- + + +def test_promote_moves_the_finished_assets_into_the_trunk(root: Path, tmp_path: Path) -> None: + name, batch = completed_batch(root, tmp_path) + ids = ok(root, "batch", "promote", batch).splitlines() + assert len(ids) == 6 + assert _trunk_size(root, name) == 6 + + +def test_promoting_twice_adds_nothing(root: Path, tmp_path: Path) -> None: + # A union against current membership, so a retried command is safe and the + # change log stays quiet. + name, batch = completed_batch(root, tmp_path) + ok(root, "batch", "promote", batch) + assert payload(root, "batch", "promote", batch) == {"items": [], "total": 0} + assert _trunk_size(root, name) == 6 + + +def test_promoting_an_unfinished_batch_exits_one(root: Path, tmp_path: Path) -> None: + _, batch = started_batch(root, tmp_path) + result = run(root, "batch", "promote", batch) + assert result.exit_code == 1, result.output + assert result.stdout == "" diff --git a/tests/cli/test_export_commands.py b/tests/cli/test_export_commands.py new file mode 100644 index 00000000..5b47dc3e --- /dev/null +++ b/tests/cli/test_export_commands.py @@ -0,0 +1,197 @@ +"""``visionset export`` and ``visionset format list``. + +The only installed exporter is ``dummy``, and it **writes nothing** — so +``file_count: 0`` here is the honest report of an export that ran, not evidence +of one that failed. The counts are taken by walking the destination afterwards, +which is what makes them checkable at all. + +The lossy gate is exercised against an exporter registered for the test through +``importlib.metadata``, because no installed one declares itself lossy — and a +gate nothing ever trips is a gate nobody has tested. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterator +from pathlib import Path + +import pytest +from tests.cli._flow import payload, published_release, run, workspace +from typer.testing import CliRunner + +from visionset.cli.main import app +from visionset.formats import registry +from visionset.kernel.domain import Manifest, Release +from visionset.kernel.services import WORKSPACE_ENV_VAR + + +class LossyExporter: + """A format that cannot carry everything a release holds. Writes one file.""" + + format_name = "lossy-sample" + lossy = True + + def export(self, release: Release, manifest: Manifest, dest: Path) -> None: + (dest / "labels.txt").write_text(f"{len(manifest.assets)}\n", encoding="utf-8") + + +@pytest.fixture(autouse=True) +def _no_ambient_workspace(monkeypatch: pytest.MonkeyPatch) -> None: + """A developer with ``VISIONSET_WORKSPACE`` exported gets CI's results.""" + monkeypatch.delenv(WORKSPACE_ENV_VAR, raising=False) + + +@pytest.fixture() +def root(tmp_path: Path) -> Path: + return workspace(tmp_path) + + +@pytest.fixture() +def lossy(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + """Add a lossy exporter to what the registry finds, for one test. + + The registry itself is not stubbed — ``exporters()`` still scans the entry + point group — so what is being tested is the command's use of it. + """ + real = registry.exporters + + def with_lossy() -> dict[str, object]: + return {**real(), LossyExporter.format_name: LossyExporter()} + + monkeypatch.setattr(registry, "exporters", with_lossy) + yield + + +# --- format list ------------------------------------------------------------- + + +def test_format_list_names_the_installed_exporters() -> None: + # No ``--workspace``: this command opens nothing, so ``_flow.ok`` (which + # always appends the flag) cannot be used and the runner is called directly. + result = CliRunner().invoke(app, ["format", "list"]) + assert result.exit_code == 0, result.output + rows = result.stdout.splitlines() + assert rows[0].split() == ["NAME", "LOSSY"] + assert rows[1].split() == ["dummy", "no"] + + +def test_format_list_needs_no_workspace_at_all( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # Plugins are a fact about the process, not about any dataset — and you ask + # what is available *before* choosing a ``--format``. + monkeypatch.chdir(tmp_path) + result = CliRunner().invoke(app, ["format", "list"]) + assert result.exit_code == 0, result.output + assert "dummy" in result.stdout + + +def test_format_list_json_is_the_envelope() -> None: + result = CliRunner().invoke(app, ["format", "list", "--json"]) + assert result.exit_code == 0, result.output + assert json.loads(result.stdout)["items"] == [{"name": "dummy", "lossy": False}] + + +# --- export ------------------------------------------------------------------ + + +def test_export_writes_into_the_directory_it_was_given(root: Path, tmp_path: Path) -> None: + name = published_release(root, tmp_path) + out = tmp_path / "out" + document = payload( + root, "export", "-p", name, "--release", "v1.0", "--format", "dummy", "--out", str(out) + ) + assert document["directory"] == str(out) + assert out.is_dir() + + +def test_the_dummy_exporter_reports_zero_files_and_that_is_correct( + root: Path, tmp_path: Path +) -> None: + name = published_release(root, tmp_path) + document = payload( + root, + "export", + "-p", + name, + "--release", + "v1.0", + "--format", + "dummy", + "--out", + str(tmp_path / "out"), + ) + assert document == { + "release_id": document["release_id"], + "format": "dummy", + "directory": str(tmp_path / "out"), + "file_count": 0, + "total_bytes": 0, + } + + +def test_an_unknown_format_exits_one_naming_what_is_installed(root: Path, tmp_path: Path) -> None: + # ``registry.pick`` refuses with a ``VisionSetError`` listing the installed + # set; a dict lookup would raise ``KeyError`` and print a traceback. + name = published_release(root, tmp_path) + result = run( + root, "export", "-p", name, "--release", "v1.0", "--format", "yolo", "--out", str(tmp_path) + ) + assert result.exit_code == 1, result.output + assert "dummy" in result.stderr + + +def test_an_unknown_release_tag_exits_one(root: Path, tmp_path: Path) -> None: + name = published_release(root, tmp_path) + result = run( + root, "export", "-p", name, "--release", "v9.9", "--format", "dummy", "--out", str(tmp_path) + ) + assert result.exit_code == 1, result.output + + +def test_out_pointing_at_a_file_exits_two(root: Path, tmp_path: Path) -> None: + name = published_release(root, tmp_path) + occupied = tmp_path / "already-a-file" + occupied.write_text("mine", encoding="utf-8") + result = run( + root, + "export", + "-p", + name, + "--release", + "v1.0", + "--format", + "dummy", + "--out", + str(occupied), + ) + assert result.exit_code == 2, result.output + + +# --- the lossy gate ---------------------------------------------------------- + + +def test_a_lossy_format_exits_one_until_the_flag(root: Path, tmp_path: Path, lossy: None) -> None: + # A third gate word, never folded into ``--yes``: this guards emitting an + # incomplete *copy* of something that stays intact. + name = published_release(root, tmp_path) + out = tmp_path / "out" + argv = [ + "export", + "-p", + name, + "--release", + "v1.0", + "--format", + "lossy-sample", + "--out", + str(out), + ] + refused = run(root, *argv) + assert refused.exit_code == 1, refused.output + assert not out.exists() + + document = payload(root, *argv, "--allow-lossy") + assert document["file_count"] == 1 + assert (out / "labels.txt").is_file() diff --git a/tests/cli/test_full_cycle.py b/tests/cli/test_full_cycle.py new file mode 100644 index 00000000..dc0e2a6e --- /dev/null +++ b/tests/cli/test_full_cycle.py @@ -0,0 +1,130 @@ +"""The whole cycle in one function, driven exactly as a script would drive it. + +The ``tests/server/test_external_client.py`` precedent, and it uses **none** of +``tests/cli/_flow.py`` for the same reason that module uses none of the server +helpers: the point is that the walk is visible in one place, ids travelling from +one command's stdout into the next command's argv, with every exit code asserted +on the way rather than only the final state. + +``examples/cli_end_to_end.sh`` is its sibling and proves something this cannot — +that the *installed console script* works from a real shell. This one proves what +the script cannot: that stdout and stderr are separated correctly at every step. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from tests.fixtures.media import write_images, write_unsupported_file +from typer.testing import CliRunner + +from visionset.cli.main import app +from visionset.kernel.services import WORKSPACE_ENV_VAR + +runner = CliRunner() + +SCHEMA = { + "classes": [ + { + "name": "sign", + "geometry": "bbox", + "attributes": [{"name": "occluded", "kind": "boolean"}], + } + ] +} + + +@pytest.fixture(autouse=True) +def _no_ambient_workspace(monkeypatch: pytest.MonkeyPatch) -> None: + """A developer with ``VISIONSET_WORKSPACE`` exported gets CI's results.""" + monkeypatch.delenv(WORKSPACE_ENV_VAR, raising=False) + + +def data(*argv: str) -> str: + """Run, insist it worked, and insist stdout held nothing but the data.""" + result = runner.invoke(app, list(argv)) + assert result.exit_code == 0, result.output + return result.stdout.strip() + + +def test_the_whole_cycle_runs_from_the_command_line(tmp_path: Path) -> None: + incoming = tmp_path / "incoming" + write_images(incoming, count=6) + write_unsupported_file(incoming / "notes.txt") + schema = tmp_path / "schema.json" + schema.write_text(json.dumps(SCHEMA), encoding="utf-8") + + # 1. A workspace, named by its own stdout from here on. + root = data("init", str(tmp_path / "ws")) + ws = ["--workspace", root] + + # 2. A project and a schema for it. + data("project", "create", "road-signs", *ws) + assert data("schema", "apply", str(schema), "-p", "road-signs", *ws) == "1" + + # 3. Images in, one batch out. The stray file is reported, not fatal. + batch = data("ingest", str(incoming), "-p", "road-signs", *ws) + + # 4. Freeze the membership and cut it into two jobs of three. + data("batch", "approve", batch, "--jobs-of", "3", *ws) + data("batch", "start", batch, *ws) + + # 5. Work through each job the way a shell reads a listing. + jobs = [line.split()[0] for line in data("job", "list", "--batch", batch, *ws).splitlines()[1:]] + assert len(jobs) == 2 + for job in jobs: + data("job", "start", job, *ws) + listing = data("job", "next", job, "-n", "100", *ws).splitlines()[1:] + assert len(listing) == 3 + for line in listing: + data("job", "mark", job, line.split()[0], "--progress", "annotated", *ws) + data("job", "complete", job, *ws) + + # 6. Close the batch and let its finished assets into the trunk. + data("batch", "complete", batch, *ws) + assert len(data("batch", "promote", batch, *ws).splitlines()) == 6 + + # 7. Freeze it, and check the freeze. + data("release", "publish", "--tag", "v1.0", "-p", "road-signs", "--split", "0.5,0.25,0.25", *ws) + verified = runner.invoke(app, ["release", "verify", "v1.0", "-p", "road-signs", *ws]) + assert verified.exit_code == 0, verified.output + + # 8. Write it out in the one installed format, which writes nothing — so a + # zero file count here is the honest report rather than a failure. + exported = json.loads( + data( + "export", + "-p", + "road-signs", + "--release", + "v1.0", + "--format", + "dummy", + "--out", + str(tmp_path / "out"), + "--json", + *ws, + ) + ) + assert exported["format"] == "dummy" + assert exported["file_count"] == 0 + + # 9. The release as a program reads it. + releases = json.loads(data("release", "list", "-p", "road-signs", "--json", *ws)) + assert releases["total"] == 1 + assert releases["items"][0]["asset_count"] == 6 + # No annotations, because the CLI marks progress and writes no labels. + assert releases["items"][0]["annotation_count"] == 0 + + +def test_a_refusal_on_the_way_through_exits_one_and_says_so(tmp_path: Path) -> None: + # The other half of what a script needs: a non-zero exit it can branch on, + # one sentence on stderr, and nothing at all on stdout. + root = data("init", str(tmp_path / "ws")) + data("project", "create", "road-signs", "--workspace", root) + result = runner.invoke(app, ["project", "create", "ROAD-SIGNS", "--workspace", root]) + assert result.exit_code == 1 + assert result.stdout == "" + assert result.stderr.startswith("Error:") diff --git a/tests/cli/test_ingest_commands.py b/tests/cli/test_ingest_commands.py new file mode 100644 index 00000000..64f0590b --- /dev/null +++ b/tests/cli/test_ingest_commands.py @@ -0,0 +1,178 @@ +"""``visionset ingest`` — one path in, one batch out, and the per-file report. + +The two things worth pinning: the **dispatch** on ``is_dir()`` (one command +standing in for two registration methods), and that the failure modes which are +*not* ``VisionSetError`` — a missing path, a non-positive rate, ``--fps`` on a +folder — are refused by Click at exit 2 rather than reaching the kernel and +printing a traceback. +""" + +from __future__ import annotations + +from pathlib import Path +from uuid import UUID + +import pytest +from tests.cli._flow import ok, payload, run, schemad_project, stills, workspace +from tests.fixtures.media import require_ffmpeg, write_video + +from visionset.kernel.domain import SourceKind +from visionset.kernel.services import ( + WORKSPACE_ENV_VAR, + BatchService, + ProjectService, + SourceService, + WorkspaceService, +) + + +@pytest.fixture(autouse=True) +def _no_ambient_workspace(monkeypatch: pytest.MonkeyPatch) -> None: + """A developer with ``VISIONSET_WORKSPACE`` exported gets CI's results.""" + monkeypatch.delenv(WORKSPACE_ENV_VAR, raising=False) + + +@pytest.fixture() +def root(tmp_path: Path) -> Path: + root = workspace(tmp_path) + schemad_project(root, tmp_path) + return root + + +def _sources(root: Path) -> list[SourceKind]: + with WorkspaceService.open(root) as service: + project = ProjectService(service).get_by_name("road-signs") + return [s.kind for s in SourceService(service).list(project.id)] + + +def _batch_size(root: Path, batch: str) -> int: + with WorkspaceService.open(root) as service: + return len(BatchService(service).assets(UUID(batch))) + + +# --- a directory of stills --------------------------------------------------- + + +def test_the_batch_id_is_the_only_thing_on_stdout(root: Path, tmp_path: Path) -> None: + result = run(root, "ingest", str(stills(tmp_path)), "-p", "road-signs") + assert result.exit_code == 0, result.output + assert "\n" not in result.stdout.strip() + assert _batch_size(root, result.stdout.strip()) == 6 + + +def test_a_directory_registers_as_an_image_source(root: Path, tmp_path: Path) -> None: + ok(root, "ingest", str(stills(tmp_path)), "-p", "road-signs") + assert _sources(root) == [SourceKind.IMAGE_DIRECTORY] + + +def test_the_batch_takes_the_sources_name_by_default(root: Path, tmp_path: Path) -> None: + ok(root, "ingest", str(stills(tmp_path)), "-p", "road-signs") + rows = ok(root, "batch", "list", "-p", "road-signs").splitlines() + assert rows[1].split()[1] == "incoming" + + +def test_batch_name_overrides_it(root: Path, tmp_path: Path) -> None: + ok(root, "ingest", str(stills(tmp_path)), "-p", "road-signs", "--batch-name", "day-one") + rows = ok(root, "batch", "list", "-p", "road-signs").splitlines() + assert rows[1].split()[1] == "day-one" + + +def test_a_file_that_is_not_an_image_is_reported_and_the_run_carries_on( + root: Path, tmp_path: Path +) -> None: + result = run(root, "ingest", str(stills(tmp_path, stray=True)), "-p", "road-signs") + assert result.exit_code == 0, result.output + assert "notes.txt" in result.stderr + assert _batch_size(root, result.stdout.strip()) == 6 + + +def test_json_carries_the_counts_and_the_failures(root: Path, tmp_path: Path) -> None: + document = payload(root, "ingest", str(stills(tmp_path, stray=True)), "-p", "road-signs") + assert document["created"] == 6 + assert document["deduplicated"] == 0 + assert document["failed"] == 1 + assert document["failures"][0]["kind"] == "unsupported" + assert document["source"]["kind"] == "image_directory" + + +def test_ingesting_the_same_folder_twice_creates_no_new_assets(root: Path, tmp_path: Path) -> None: + # Registration is idempotent and content addressing does the rest, which is + # why an interrupted run needs no ``--resume``: you run the same line again. + folder = stills(tmp_path) + ok(root, "ingest", str(folder), "-p", "road-signs") + document = payload(root, "ingest", str(folder), "-p", "road-signs") + assert document["created"] == 0 + assert document["deduplicated"] == 6 + assert _sources(root) == [SourceKind.IMAGE_DIRECTORY] + + +# --- refusals Click has to make ---------------------------------------------- + + +def test_a_path_that_is_not_there_exits_two(root: Path, tmp_path: Path) -> None: + # ``canonical_path`` raises ``FileNotFoundError``, which is not a + # ``VisionSetError`` and would print a traceback. + result = run(root, "ingest", str(tmp_path / "absent"), "-p", "road-signs") + assert result.exit_code == 2, result.output + + +def test_a_non_positive_rate_exits_two(root: Path, tmp_path: Path) -> None: + # ``register_video`` raises a bare ``ValueError`` for this, and Typer cannot + # express ``gt=0`` — hence the explicit check. + clip = tmp_path / "clip.mp4" + clip.write_bytes(b"") + result = run(root, "ingest", str(clip), "-p", "road-signs", "--fps", "0") + assert result.exit_code == 2, result.output + assert "greater than zero" in result.output + + +def test_fps_on_a_directory_exits_two(root: Path, tmp_path: Path) -> None: + # A rate has no meaning for stills, and silently ignoring it would let + # somebody believe they had chosen one. + result = run(root, "ingest", str(stills(tmp_path)), "-p", "road-signs", "--fps", "5") + assert result.exit_code == 2, result.output + assert "directory of stills" in result.output + + +def test_an_unknown_project_exits_one(root: Path, tmp_path: Path) -> None: + result = run(root, "ingest", str(stills(tmp_path)), "-p", "nope") + assert result.exit_code == 1, result.output + assert result.stdout == "" + + +# --- a clip ------------------------------------------------------------------ + + +def test_a_video_registers_at_the_rate_it_was_given(root: Path, tmp_path: Path) -> None: + require_ffmpeg() + # 96x72 rather than the fixture default: below roughly that, ``testsrc``'s + # per-frame movement falls under what the encoder resolves, consecutive + # frames come out byte-identical, and content addressing deduplicates them — + # which would read as this command losing frames. + clip = write_video(tmp_path / "clip.mp4", size=(96, 72), fps=10, duration_seconds=2.0) + document = payload(root, "ingest", str(clip.path), "-p", "road-signs", "--fps", "5") + assert document["source"]["kind"] == "video" + assert document["source"]["video"]["extraction_fps"] == 5.0 + assert document["created"] == 10 + + +# --- the preview backfill ---------------------------------------------------- + + +def test_backfill_reports_a_project_whose_previews_are_already_there( + root: Path, tmp_path: Path +) -> None: + # Ingest caches a preview per asset, so the backfill is a no-op — and its + # report says examined 0 rather than pretending to have done work. + ok(root, "ingest", str(stills(tmp_path)), "-p", "road-signs") + document = payload(root, "backfill-thumbnails", "-p", "road-signs") + assert document["examined"] == 0 + assert document["filled"] == [] + assert document["unreadable"] == [] + + +def test_backfill_says_what_it_examined_on_stderr(root: Path, tmp_path: Path) -> None: + ok(root, "ingest", str(stills(tmp_path)), "-p", "road-signs") + result = run(root, "backfill-thumbnails", "-p", "road-signs") + assert result.exit_code == 0, result.output + assert "Examined 0 asset(s)" in result.stderr diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py new file mode 100644 index 00000000..4f724221 --- /dev/null +++ b/tests/cli/test_init.py @@ -0,0 +1,120 @@ +"""``visionset init`` — the one command that creates rather than opens. + +Three things it must get right and one it must never do: create where nothing is, +refuse where something already is, put the resolved root alone on stdout, and +**never walk upward** to find a place to create a workspace. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from visionset.cli.main import app +from visionset.kernel.services import WORKSPACE_ENV_VAR, WorkspaceService + +runner = CliRunner() + + +@pytest.fixture(autouse=True) +def _no_ambient_workspace(monkeypatch: pytest.MonkeyPatch) -> None: + """A developer with ``VISIONSET_WORKSPACE`` exported gets CI's results.""" + monkeypatch.delenv(WORKSPACE_ENV_VAR, raising=False) + + +# --- creating ---------------------------------------------------------------- + + +def test_it_makes_a_workspace_a_later_command_can_open(tmp_path: Path) -> None: + result = runner.invoke(app, ["init", str(tmp_path / "ws")]) + assert result.exit_code == 0, result.output + WorkspaceService.open(tmp_path / "ws").close() + + +def test_the_resolved_root_is_the_only_thing_on_stdout(tmp_path: Path) -> None: + # ``WS=$(visionset init ./ws)`` has to be exactly the path, so that the two + # "what next" lines can still reach the person on stderr. + result = runner.invoke(app, ["init", str(tmp_path / "ws")]) + assert result.stdout.strip() == str((tmp_path / "ws").resolve()) + assert "Created workspace" in result.stderr + assert "visionset ui" in result.stderr + + +def test_it_names_the_workspace_after_its_directory(tmp_path: Path) -> None: + runner.invoke(app, ["init", str(tmp_path / "robots")]) + with WorkspaceService.open(tmp_path / "robots") as service: + assert service.workspace.name == "robots" + + +def test_name_overrides_the_directory(tmp_path: Path) -> None: + runner.invoke(app, ["init", str(tmp_path / "ws"), "--name", "Field trial"]) + with WorkspaceService.open(tmp_path / "ws") as service: + assert service.workspace.name == "Field trial" + + +def test_with_no_path_it_uses_the_working_directory( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + here = tmp_path / "here" + here.mkdir() + monkeypatch.chdir(here) + result = runner.invoke(app, ["init"]) + assert result.exit_code == 0, result.output + assert (here / "visionset.db").is_file() + + +# --- refusing ---------------------------------------------------------------- + + +def test_a_second_init_over_a_workspace_exits_one(tmp_path: Path) -> None: + runner.invoke(app, ["init", str(tmp_path / "ws")]) + result = runner.invoke(app, ["init", str(tmp_path / "ws")]) + assert result.exit_code == 1, result.output + assert result.stdout == "" + assert "Error:" in result.stderr + + +def test_a_directory_holding_something_else_exits_one(tmp_path: Path) -> None: + # The guard that stops a typo turning a home directory into a workspace. + (tmp_path / "occupied").mkdir() + (tmp_path / "occupied" / "notes.txt").write_text("mine", encoding="utf-8") + result = runner.invoke(app, ["init", str(tmp_path / "occupied")]) + assert result.exit_code == 1, result.output + assert not (tmp_path / "occupied" / "visionset.db").exists() + + +# --- what it must never do --------------------------------------------------- + + +def test_it_does_not_walk_up_to_an_existing_workspace(tmp_path: Path) -> None: + # The sibling of the kernel's ``test_an_explicit_path_does_not_walk_upward``, + # and the reason this command has a positional PATH rather than + # ``--workspace``: naming where to *make* one must never be traded for its + # parent, which is the one case where the trade is irreversible. + runner.invoke(app, ["init", str(tmp_path / "outer")]) + below = tmp_path / "outer" / "nested" + result = runner.invoke(app, ["init", str(below)]) + assert result.exit_code == 0, result.output + assert (below / "visionset.db").is_file() + + +def test_it_ignores_the_environment_variable( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # ``--workspace`` and ``$VISIONSET_WORKSPACE`` both say "operate on this one". + # Neither answers "where should a new one go", so this command consults + # neither, and the argument is the whole of its input. + monkeypatch.setenv(WORKSPACE_ENV_VAR, str(tmp_path / "elsewhere")) + result = runner.invoke(app, ["init", str(tmp_path / "ws")]) + assert result.exit_code == 0, result.output + assert (tmp_path / "ws" / "visionset.db").is_file() + assert not (tmp_path / "elsewhere").exists() + + +def test_it_leaves_no_write_ahead_log_behind(tmp_path: Path) -> None: + # ``init`` hands back an *open* workspace; a command that forgot to close it + # would strand ``visionset.db-wal`` for the next reader to recover. + runner.invoke(app, ["init", str(tmp_path / "ws")]) + assert sorted(p.name for p in (tmp_path / "ws").iterdir()) == ["blobs", "visionset.db"] diff --git a/tests/cli/test_job_commands.py b/tests/cli/test_job_commands.py new file mode 100644 index 00000000..2eccd110 --- /dev/null +++ b/tests/cli/test_job_commands.py @@ -0,0 +1,179 @@ +"""``visionset job`` — the six commands that make the lifecycle drivable.""" + +from __future__ import annotations + +from pathlib import Path +from uuid import uuid4 + +import pytest +from tests.cli._flow import ingested_batch, jobs_of, ok, payload, run, started_batch, workspace + +from visionset.kernel.domain import AssetProgress +from visionset.kernel.services import WORKSPACE_ENV_VAR + + +@pytest.fixture(autouse=True) +def _no_ambient_workspace(monkeypatch: pytest.MonkeyPatch) -> None: + """A developer with ``VISIONSET_WORKSPACE`` exported gets CI's results.""" + monkeypatch.delenv(WORKSPACE_ENV_VAR, raising=False) + + +@pytest.fixture() +def root(tmp_path: Path) -> Path: + return workspace(tmp_path) + + +def _assets(root: Path, job: str) -> list[str]: + return [line.split()[0] for line in ok(root, "job", "next", job, "-n", "100").splitlines()[1:]] + + +# --- list -------------------------------------------------------------------- + + +def test_a_draft_batch_has_no_jobs(root: Path, tmp_path: Path) -> None: + _, batch = ingested_batch(root, tmp_path) + result = run(root, "job", "list", "--batch", batch) + assert result.stdout.splitlines() == ["ID STATE ASSETS"] + assert "approve it first" in result.stderr + + +def test_list_leads_with_the_id(root: Path, tmp_path: Path) -> None: + _, batch = started_batch(root, tmp_path, jobs_of=3) + rows = ok(root, "job", "list", "--batch", batch).splitlines() + assert rows[0].split() == ["ID", "STATE", "ASSETS"] + assert [line.split()[1:] for line in rows[1:]] == [["pending", "3"], ["pending", "3"]] + + +def test_list_json_names_the_batch_each_job_belongs_to(root: Path, tmp_path: Path) -> None: + # ``task_group_id`` is absent and ``batch_id`` is here instead, because the + # batch is what leads to the pinned schema the work is judged against. + _, batch = started_batch(root, tmp_path) + item = payload(root, "job", "list", "--batch", batch)["items"][0] + assert item["batch_id"] == batch + assert "task_group_id" not in item + + +# --- next -------------------------------------------------------------------- + + +def test_next_hands_back_the_assets_awaiting_annotation(root: Path, tmp_path: Path) -> None: + _, batch = started_batch(root, tmp_path, jobs_of=3) + job = jobs_of(root, batch)[0] + assert len(_assets(root, job)) == 3 + + +def test_next_bounds_what_it_returns(root: Path, tmp_path: Path) -> None: + _, batch = started_batch(root, tmp_path) + job = jobs_of(root, batch)[0] + assert payload(root, "job", "next", job, "-n", "2")["total"] == 2 + + +def test_next_is_stable_across_two_calls(root: Path, tmp_path: Path) -> None: + # Order is the stored position, not insertion luck. + _, batch = started_batch(root, tmp_path) + job = jobs_of(root, batch)[0] + assert _assets(root, job) == _assets(root, job) + + +def test_a_count_of_zero_exits_two(root: Path, tmp_path: Path) -> None: + # ``next_pending`` refuses a non-positive count with a bare ``ValueError``. + _, batch = started_batch(root, tmp_path) + assert run(root, "job", "next", jobs_of(root, batch)[0], "-n", "0").exit_code == 2 + + +def test_next_says_so_when_nothing_is_left(root: Path, tmp_path: Path) -> None: + _, batch = started_batch(root, tmp_path) + job = jobs_of(root, batch)[0] + ok(root, "job", "start", job) + for asset in _assets(root, job): + ok(root, "job", "mark", job, asset, "--progress", "annotated") + result = run(root, "job", "next", job) + assert result.exit_code == 0, result.output + assert "nothing left" in result.stderr + + +# --- progress ---------------------------------------------------------------- + + +def test_progress_names_every_state_the_enum_has(root: Path, tmp_path: Path) -> None: + # The columns are read off ``AssetProgress``, so a sixth state cannot be + # silently missing from the table. + _, batch = started_batch(root, tmp_path) + header = ok(root, "job", "progress", jobs_of(root, batch)[0]).splitlines()[0].split() + assert header == [state.value.upper() for state in AssetProgress] + ["TOTAL"] + + +def test_progress_json_agrees_with_the_wire_shape(root: Path, tmp_path: Path) -> None: + _, batch = started_batch(root, tmp_path, jobs_of=3) + counts = payload(root, "job", "progress", jobs_of(root, batch)[0]) + assert counts["unannotated"] == 3 + assert counts["total"] == 3 + + +# --- start, mark, complete --------------------------------------------------- + + +def test_start_takes_a_pending_job(root: Path, tmp_path: Path) -> None: + _, batch = started_batch(root, tmp_path) + assert payload(root, "job", "start", jobs_of(root, batch)[0])["state"] == "in_progress" + + +def test_mark_records_the_state(root: Path, tmp_path: Path) -> None: + _, batch = started_batch(root, tmp_path) + job = jobs_of(root, batch)[0] + ok(root, "job", "start", job) + asset = _assets(root, job)[0] + assert payload(root, "job", "mark", job, asset, "--progress", "skipped") == { + "asset_id": asset, + "progress": "skipped", + } + + +def test_an_unknown_progress_state_exits_two_listing_the_real_ones( + root: Path, tmp_path: Path +) -> None: + # Typer renders the ``StrEnum`` as a Click choice, so the refusal names every + # legal value without this module restating them. + _, batch = started_batch(root, tmp_path) + result = run(root, "job", "mark", jobs_of(root, batch)[0], str(uuid4()), "--progress", "bogus") + assert result.exit_code == 2, result.output + assert "annotated" in result.output + + +def test_marking_an_asset_that_is_not_in_the_job_exits_one(root: Path, tmp_path: Path) -> None: + _, batch = started_batch(root, tmp_path) + job = jobs_of(root, batch)[0] + ok(root, "job", "start", job) + result = run(root, "job", "mark", job, str(uuid4()), "--progress", "annotated") + assert result.exit_code == 1, result.output + assert "Error:" in result.stderr + + +def test_marking_into_a_batch_nobody_opened_exits_one(root: Path, tmp_path: Path) -> None: + # The batch gate fires before the value is even looked at. + _, batch = ingested_batch(root, tmp_path) + ok(root, "batch", "approve", batch) + job = jobs_of(root, batch)[0] + result = run(root, "job", "mark", job, str(uuid4()), "--progress", "annotated") + assert result.exit_code == 1, result.output + + +def test_completing_with_an_unsettled_asset_exits_one(root: Path, tmp_path: Path) -> None: + _, batch = started_batch(root, tmp_path) + job = jobs_of(root, batch)[0] + ok(root, "job", "start", job) + result = run(root, "job", "complete", job) + assert result.exit_code == 1, result.output + assert "Error:" in result.stderr + + +def test_completing_a_job_does_not_complete_its_batch(root: Path, tmp_path: Path) -> None: + # One machine in two places is one too many: ``batch complete`` derives that + # itself, and refuses while any job is outstanding. + name, batch = started_batch(root, tmp_path, jobs_of=3) + job = jobs_of(root, batch)[0] + ok(root, "job", "start", job) + for asset in _assets(root, job): + ok(root, "job", "mark", job, asset, "--progress", "annotated") + ok(root, "job", "complete", job) + assert payload(root, "batch", "list", "-p", name)["items"][0]["state"] == "in_annotation" diff --git a/tests/cli/test_json_contract.py b/tests/cli/test_json_contract.py new file mode 100644 index 00000000..3bca737e --- /dev/null +++ b/tests/cli/test_json_contract.py @@ -0,0 +1,145 @@ +"""``--json`` and the REST API publish the same shape for the same concept. + +The two packages may not import each other — import-linter's independence +contract — so nothing enforces the agreement from inside ``src/``. A test can: +``tests/`` is outside the ``visionset`` package, so this module imports both +``visionset.cli._json`` and ``visionset.server.models`` and asserts, per pair: + +1. the projection's keys are exactly the wire model's fields; +2. the wire model *validates* the projection — which catches encoding drift a + key-set comparison cannot see, chiefly a timestamp in the wrong format or a + UUID handed over as an object; +3. the projection is JSON-serializable with no ``default=``, so a leaf somebody + forgot to encode is a ``TypeError`` here rather than a silent ``str()``. + +Two projections are deliberately ungated, and named at the bottom: the CLI +defines those shapes first because no route publishes them. +""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from typing import Any + +import pytest +from pydantic import BaseModel +from tests.fixtures.samples import ( + ASSET, + BATCH, + COUNTS, + EXPORT_RESULT, + INGEST_FAILURE, + INGEST_JOB, + JOB, + PROJECT, + RELEASE, + SCHEMA_VERSION, + SOURCE, + SPLIT, + THUMBNAIL_BACKFILL, + VERIFICATION, +) + +from visionset.cli import _json +from visionset.formats._dummy import DummyExporter +from visionset.server import models + +# One row per pair: a label, the projected payload, and the wire model it must +# agree with. Built eagerly — every projection runs at import, so a leaf that +# does not encode fails collection rather than one parametrized case. +PAIRS: list[tuple[str, dict[str, Any], type[BaseModel]]] = [ + ("project", _json.project(PROJECT), models.ProjectOut), + ("schema_version", _json.schema_version(SCHEMA_VERSION), models.SchemaVersionOut), + ("label_class", _json.label_class(SCHEMA_VERSION.classes[0]), models.LabelClassBody), + ( + "attribute", + _json.attribute(SCHEMA_VERSION.classes[0].attributes[0]), + models.AttributeBody, + ), + ("source", _json.source(SOURCE), models.SourceOut), + ("video_provenance", _json.video_provenance(SOURCE.require_video()), models.VideoProvenanceOut), + ("ingest_job", _json.ingest_job(INGEST_JOB), models.IngestJobOut), + ("ingest_failure", _json.ingest_failure(INGEST_FAILURE), models.IngestFailureOut), + ("asset", _json.asset(ASSET), models.AssetOut), + ("progress_counts", _json.progress_counts(COUNTS), models.ProgressCounts), + ("batch", _json.batch(BATCH, COUNTS), models.BatchOut), + ("job", _json.job(JOB, batch_id=BATCH.id), models.JobOut), + ("release", _json.release(RELEASE), models.ReleaseOut), + ("split_recipe", _json.split_recipe(SPLIT), models.SplitRecipeBody), + ( + "release_verification", + _json.release_verification(VERIFICATION), + models.ReleaseVerificationOut, + ), + ("export_format", _json.export_format(DummyExporter()), models.FormatOut), +] + +IDS = [label for label, _, _ in PAIRS] + + +# --- parity with the wire models --------------------------------------------- + + +@pytest.mark.parametrize(("payload", "wire"), [(p, w) for _, p, w in PAIRS], ids=IDS) +def test_the_projection_publishes_exactly_the_fields_the_wire_model_does( + payload: dict[str, Any], wire: type[BaseModel] +) -> None: + assert set(payload) == set(wire.model_fields) + + +@pytest.mark.parametrize(("payload", "wire"), [(p, w) for _, p, w in PAIRS], ids=IDS) +def test_the_wire_model_accepts_the_projection_verbatim( + payload: dict[str, Any], wire: type[BaseModel] +) -> None: + # Stronger than key-set parity and one line: it is what proves a UUID left as + # a string, an enum left as its value, and a timestamp in pydantic's own + # format rather than the human one the columns use. + wire.model_validate(payload) + + +@pytest.mark.parametrize("payload", [p for _, p, _ in PAIRS], ids=IDS) +def test_the_projection_serializes_with_no_default_encoder(payload: dict[str, Any]) -> None: + json.dumps(payload) + + +# --- the envelope ------------------------------------------------------------ + + +def test_a_listing_is_an_object_with_items_and_a_total() -> None: + assert _json.page([{"id": "a"}, {"id": "b"}]) == { + "items": [{"id": "a"}, {"id": "b"}], + "total": 2, + } + + +def test_an_empty_listing_is_still_an_object() -> None: + # Never a bare array, and never a 404's moral equivalent: an empty collection + # is a collection. + assert _json.page([]) == {"items": [], "total": 0} + + +# --- the timestamp format the parity gate depends on ------------------------- + + +def test_a_timestamp_keeps_its_microseconds_and_ends_in_z() -> None: + # Deliberately *not* ``_output.moment``'s format, which stops at seconds. A + # single shared helper would pass every key-set assertion above and fail the + # round-trip one. + when = datetime(2026, 7, 28, 12, 34, 56, 789012, tzinfo=UTC) + assert _json._moment(when) == "2026-07-28T12:34:56.789012Z" + + +# --- the two shapes with no wire partner ------------------------------------- + + +@pytest.mark.parametrize( + "payload", + [_json.export_result(EXPORT_RESULT), _json.thumbnail_backfill(THUMBNAIL_BACKFILL)], + ids=["export_result", "thumbnail_backfill"], +) +def test_a_cli_defined_shape_still_serializes(payload: dict[str, Any]) -> None: + # No route publishes either, so there is nothing to be parity-gated against. + # What still has to hold is that every leaf is encoded — which is the failure + # these two would otherwise be free to have. + json.dumps(payload) diff --git a/tests/cli/test_output.py b/tests/cli/test_output.py new file mode 100644 index 00000000..192edf8a --- /dev/null +++ b/tests/cli/test_output.py @@ -0,0 +1,52 @@ +"""The column formatter, on its own. + +Pure string functions, so they can be swept without running a command: a ragged +row, a cell wider than its header, a header wider than every cell, and the +trailing-space rule that makes the output diffable. +""" + +from __future__ import annotations + +import pytest + +from visionset.cli._output import row, table, widths + + +def test_a_column_is_as_wide_as_its_widest_cell() -> None: + assert widths(("ID", "NAME"), [("1", "alpha"), ("22", "b")]) == [2, 5] + + +def test_a_column_with_no_rows_is_as_wide_as_its_header() -> None: + assert widths(("ID", "NAME"), []) == [2, 4] + + +def test_a_header_wider_than_every_cell_still_sets_the_width() -> None: + assert widths(("DESCRIPTION",), [("x",)]) == [11] + + +def test_cells_are_left_justified_and_two_spaces_apart() -> None: + assert row(("1", "alpha"), [2, 5]) == "1 alpha" + + +def test_the_last_cell_is_not_padded() -> None: + # Trailing whitespace would make the output differ from what a person sees + # and would break a naive equality assertion for no benefit. + assert row(("1", "a"), [2, 5]) == "1 a" + + +def test_a_row_with_the_wrong_number_of_cells_raises() -> None: + # ``strict=True`` on the zip. Silently dropping the extra cell would lose a + # column from a listing and nothing would say so. + with pytest.raises(ValueError): + row(("1", "a", "extra"), [2, 5]) + + +def test_the_header_prints_even_with_no_rows(capsys: pytest.CaptureFixture[str]) -> None: + # What makes ``| tail -n +2`` stable whether or not anything matched. + table(("ID", "NAME"), []) + assert capsys.readouterr().out == "ID NAME\n" + + +def test_a_table_prints_its_header_then_its_rows(capsys: pytest.CaptureFixture[str]) -> None: + table(("ID", "NAME"), [("1", "alpha"), ("22", "b")]) + assert capsys.readouterr().out.splitlines() == ["ID NAME", "1 alpha", "22 b"] diff --git a/tests/cli/test_project_commands.py b/tests/cli/test_project_commands.py new file mode 100644 index 00000000..0b71a974 --- /dev/null +++ b/tests/cli/test_project_commands.py @@ -0,0 +1,130 @@ +"""``visionset project`` — creating, listing, and being addressable by name. + +Name-or-id addressing is exercised here rather than in ``_resolve``'s own module, +because what matters is that a *command* accepts both. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from tests.cli._flow import ok, payload, run, workspace + +from visionset.kernel.services import WORKSPACE_ENV_VAR, ProjectService, WorkspaceService + + +@pytest.fixture(autouse=True) +def _no_ambient_workspace(monkeypatch: pytest.MonkeyPatch) -> None: + """A developer with ``VISIONSET_WORKSPACE`` exported gets CI's results.""" + monkeypatch.delenv(WORKSPACE_ENV_VAR, raising=False) + + +@pytest.fixture() +def root(tmp_path: Path) -> Path: + return workspace(tmp_path) + + +def _stored(root: Path) -> list[str]: + with WorkspaceService.open(root) as service: + return [p.name for p in ProjectService(service).list()] + + +# --- create ------------------------------------------------------------------ + + +def test_create_writes_the_project(root: Path) -> None: + ok(root, "project", "create", "road-signs") + assert _stored(root) == ["road-signs"] + + +def test_the_new_id_is_the_only_thing_on_stdout(root: Path) -> None: + result = run(root, "project", "create", "road-signs") + assert result.stdout.strip().count("\n") == 0 + assert "Created project" in result.stderr + + +def test_create_carries_a_description(root: Path) -> None: + assert ( + payload(root, "project", "create", "x", "--description", "field trial")["description"] + == "field trial" + ) + + +def test_a_blank_name_exits_one(root: Path) -> None: + result = run(root, "project", "create", " ") + assert result.exit_code == 1, result.output + assert result.stdout == "" + assert "Error:" in result.stderr + + +def test_a_repeated_name_exits_one(root: Path) -> None: + ok(root, "project", "create", "road-signs") + result = run(root, "project", "create", "ROAD-SIGNS") + assert result.exit_code == 1, result.output + assert _stored(root) == ["road-signs"] + + +# --- list -------------------------------------------------------------------- + + +def test_list_leads_with_the_id(root: Path) -> None: + # The rule the whole shell idiom rests on: ``awk '{print $1}'`` must be + # stable even when a name holds internal whitespace, which normalization + # deliberately preserves. + created = ok(root, "project", "create", "road signs east") + rows = ok(root, "project", "list").splitlines() + assert rows[0].split() == ["ID", "NAME", "DESCRIPTION"] + assert rows[1].split()[0] == created + + +def test_an_empty_listing_still_prints_its_header(root: Path) -> None: + result = run(root, "project", "list") + assert result.stdout.splitlines() == ["ID NAME DESCRIPTION"] + assert "No projects" in result.stderr + + +def test_list_json_is_the_envelope(root: Path) -> None: + ok(root, "project", "create", "a") + ok(root, "project", "create", "b") + document = payload(root, "project", "list") + assert set(document) == {"items", "total"} + assert document["total"] == 2 + assert [item["name"] for item in document["items"]] == ["a", "b"] + + +def test_an_empty_listing_json_is_an_object(root: Path) -> None: + assert payload(root, "project", "list") == {"items": [], "total": 0} + + +def test_json_puts_nothing_on_stdout_but_the_document(root: Path) -> None: + result = run(root, "project", "create", "a", "--json") + json.loads(result.stdout) + assert result.stderr == "" + + +# --- addressing -------------------------------------------------------------- + + +def test_a_project_is_reachable_by_name(root: Path) -> None: + ok(root, "project", "create", "road-signs") + assert ok(root, "schema", "list", "--project", "road-signs") == "VERSION CLASSES GEOMETRIES" + + +def test_a_project_is_reachable_by_id(root: Path) -> None: + created = ok(root, "project", "create", "road-signs") + assert ok(root, "schema", "list", "--project", created) == "VERSION CLASSES GEOMETRIES" + + +def test_a_name_matches_ignoring_case(root: Path) -> None: + # The comparison the unique index makes, and the reason ``get_by_name`` is a + # kernel read rather than a scan written here. + ok(root, "project", "create", "Road-Signs") + assert run(root, "schema", "list", "--project", "road-SIGNS").exit_code == 0 + + +def test_an_unknown_project_exits_one(root: Path) -> None: + result = run(root, "schema", "list", "--project", "nope") + assert result.exit_code == 1, result.output + assert "Error:" in result.stderr diff --git a/tests/cli/test_release_commands.py b/tests/cli/test_release_commands.py new file mode 100644 index 00000000..9f2c30d8 --- /dev/null +++ b/tests/cli/test_release_commands.py @@ -0,0 +1,199 @@ +"""``visionset release`` — publishing, and the command whose exit code is the answer.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from tests.cli._flow import ( + ok, + payload, + promoted_project, + published_release, + run, + schemad_project, + workspace, +) + +from visionset.kernel.services import ( + WORKSPACE_ENV_VAR, + ProjectService, + ReleaseService, + WorkspaceService, +) + + +@pytest.fixture(autouse=True) +def _no_ambient_workspace(monkeypatch: pytest.MonkeyPatch) -> None: + """A developer with ``VISIONSET_WORKSPACE`` exported gets CI's results.""" + monkeypatch.delenv(WORKSPACE_ENV_VAR, raising=False) + + +@pytest.fixture() +def root(tmp_path: Path) -> Path: + return workspace(tmp_path) + + +def _manifest_blob(root: Path, name: str, tag: str) -> Path: + with WorkspaceService.open(root) as service: + project = ProjectService(service).get_by_name(name) + dataset = ProjectService(service).get_dataset(project.id) + release = ReleaseService(service).get_by_tag(dataset.id, tag) + digest = release.manifest_hash + # The blob store's own sharding, ``///``. + return root / "blobs" / digest[:2] / digest[2:4] / digest + + +# --- publish ----------------------------------------------------------------- + + +def test_publish_freezes_the_trunk(root: Path, tmp_path: Path) -> None: + name = promoted_project(root, tmp_path) + document = payload(root, "release", "publish", "--tag", "v1.0", "-p", name) + assert document["tag"] == "v1.0" + assert document["asset_count"] == 6 + assert document["schema_version"] == 1 + + +def test_a_release_of_a_cli_driven_batch_carries_no_annotations(root: Path, tmp_path: Path) -> None: + # Said out loud rather than hidden: ``job mark --progress annotated`` records + # that somebody labeled an asset, and the CLI writes no labels. The manifest + # is honest about it. + name = promoted_project(root, tmp_path) + assert payload(root, "release", "publish", "--tag", "v1.0", "-p", name)["annotation_count"] == 0 + + +def test_publishing_an_empty_trunk_exits_one(root: Path, tmp_path: Path) -> None: + name = schemad_project(root, tmp_path) + result = run(root, "release", "publish", "--tag", "v1.0", "-p", name) + assert result.exit_code == 1, result.output + assert "Error:" in result.stderr + + +def test_a_repeated_tag_exits_one(root: Path, tmp_path: Path) -> None: + # A release is never edited, so the remedy named in the refusal is a new tag. + name = published_release(root, tmp_path) + assert run(root, "release", "publish", "--tag", "v1.0", "-p", name).exit_code == 1 + + +def test_a_tag_is_case_sensitive(root: Path, tmp_path: Path) -> None: + # The opposite rule to a project name, which is why both live in the kernel. + name = published_release(root, tmp_path) + assert run(root, "release", "publish", "--tag", "V1.0", "-p", name).exit_code == 0 + + +# --- the split --------------------------------------------------------------- + + +def test_split_is_stored_on_the_release(root: Path, tmp_path: Path) -> None: + name = promoted_project(root, tmp_path) + document = payload( + root, "release", "publish", "--tag", "v1.0", "-p", name, "--split", "0.5,0.25,0.25" + ) + assert document["split"] == {"train": 0.5, "val": 0.25, "test": 0.25, "seed": 0} + + +def test_seed_reaches_the_recipe(root: Path, tmp_path: Path) -> None: + name = promoted_project(root, tmp_path) + document = payload( + root, + "release", + "publish", + "--tag", + "v1.0", + "-p", + name, + "--split", + "0.7,0.15,0.15", + "--seed", + "42", + ) + assert document["split"]["seed"] == 42 + + +def test_no_split_leaves_it_null(root: Path, tmp_path: Path) -> None: + name = promoted_project(root, tmp_path) + assert payload(root, "release", "publish", "--tag", "v1.0", "-p", name)["split"] is None + + +@pytest.mark.parametrize( + ("value", "why"), + [ + ("0.5,0.5", "too few"), + ("0.5,0.25,0.15,0.1", "too many"), + ("a,b,c", "not numbers"), + ("0.5,0.5,0.5", "does not add up"), + ], +) +def test_a_malformed_split_exits_two(root: Path, tmp_path: Path, value: str, why: str) -> None: + # ``SplitRecipe`` refuses the last one with a pydantic error, which is not a + # ``VisionSetError`` and would print a traceback. + name = promoted_project(root, tmp_path) + result = run(root, "release", "publish", "--tag", "v1.0", "-p", name, "--split", value) + assert result.exit_code == 2, f"{why}: {result.output}" + + +# --- list -------------------------------------------------------------------- + + +def test_list_leads_with_the_id(root: Path, tmp_path: Path) -> None: + name = published_release(root, tmp_path) + rows = ok(root, "release", "list", "-p", name).splitlines() + assert rows[0].split() == ["ID", "TAG", "ASSETS", "ANNOTATIONS", "SCHEMA", "CREATED"] + assert rows[1].split()[1] == "v1.0" + + +def test_an_empty_listing_still_prints_its_header(root: Path, tmp_path: Path) -> None: + name = promoted_project(root, tmp_path) + result = run(root, "release", "list", "-p", name) + assert len(result.stdout.splitlines()) == 1 + assert "published no releases" in result.stderr + + +def test_list_json_is_the_envelope(root: Path, tmp_path: Path) -> None: + name = published_release(root, tmp_path) + document = payload(root, "release", "list", "-p", name) + assert document["total"] == 1 + assert document["items"][0]["tag"] == "v1.0" + + +# --- verify: the exit code is the answer ------------------------------------- + + +def test_verify_exits_zero_when_the_release_is_intact(root: Path, tmp_path: Path) -> None: + name = published_release(root, tmp_path) + result = run(root, "release", "verify", "v1.0", "-p", name) + assert result.exit_code == 0, result.output + assert "verifies" in result.stderr + + +def test_verify_json_carries_the_derived_ok(root: Path, tmp_path: Path) -> None: + name = published_release(root, tmp_path) + document = payload(root, "release", "verify", "v1.0", "-p", name) + assert document["ok"] is True + assert document["checked"] == 6 + + +def test_verify_exits_one_when_the_manifest_has_been_altered(root: Path, tmp_path: Path) -> None: + # Not a refusal — the check ran and the answer is no. Exit 1 is what lets + # ``verify && train.sh`` mean something. + name = published_release(root, tmp_path) + _manifest_blob(root, name, "v1.0").write_bytes(b"{}") + result = run(root, "release", "verify", "v1.0", "-p", name) + assert result.exit_code == 1, result.output + assert "does not match its hash" in result.stderr + + +def test_verify_json_still_prints_when_the_answer_is_no(root: Path, tmp_path: Path) -> None: + name = published_release(root, tmp_path) + _manifest_blob(root, name, "v1.0").write_bytes(b"{}") + result = run(root, "release", "verify", "v1.0", "-p", name, "--json") + assert result.exit_code == 1, result.output + assert '"ok": false' in result.stdout + + +def test_an_unknown_tag_exits_one(root: Path, tmp_path: Path) -> None: + name = published_release(root, tmp_path) + result = run(root, "release", "verify", "v9.9", "-p", name) + assert result.exit_code == 1, result.output + assert "Error:" in result.stderr diff --git a/tests/cli/test_schema_commands.py b/tests/cli/test_schema_commands.py new file mode 100644 index 00000000..0bd9af1c --- /dev/null +++ b/tests/cli/test_schema_commands.py @@ -0,0 +1,159 @@ +"""``visionset schema apply`` — a JSON document, refused in the right place. + +Three failure modes and three exit codes, which is the point of the module: a +file that is not JSON and a document the domain will not accept are **usage** +errors at exit 2, while a change the *project's history* refuses is a domain +error at exit 1. Only the last is a ``VisionSetError``, and only the last can be +retried with a flag. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from tests.cli._flow import SCHEMA_DOCUMENT, ok, payload, run, schema_file, workspace + +from visionset.kernel.services import WORKSPACE_ENV_VAR +from visionset.server.models import SchemaVersionCreate + + +@pytest.fixture(autouse=True) +def _no_ambient_workspace(monkeypatch: pytest.MonkeyPatch) -> None: + """A developer with ``VISIONSET_WORKSPACE`` exported gets CI's results.""" + monkeypatch.delenv(WORKSPACE_ENV_VAR, raising=False) + + +@pytest.fixture() +def root(tmp_path: Path) -> Path: + root = workspace(tmp_path) + ok(root, "project", "create", "road-signs") + return root + + +def _document(tmp_path: Path, classes: list[dict]) -> Path: + path = tmp_path / "custom.json" + path.write_text(json.dumps({"classes": classes}), encoding="utf-8") + return path + + +# --- applying ---------------------------------------------------------------- + + +def test_apply_creates_version_one(root: Path, tmp_path: Path) -> None: + assert ok(root, "schema", "apply", str(schema_file(tmp_path)), "-p", "road-signs") == "1" + + +def test_applying_again_creates_the_next_version(root: Path, tmp_path: Path) -> None: + # Versions are 1..N and none of them changes, so an unchanged document still + # adds one — there is no edit and no rollback. + file = schema_file(tmp_path) + ok(root, "schema", "apply", str(file), "-p", "road-signs") + assert ok(root, "schema", "apply", str(file), "-p", "road-signs") == "2" + + +def test_the_classes_survive_the_round_trip(root: Path, tmp_path: Path) -> None: + document = payload(root, "schema", "apply", str(schema_file(tmp_path)), "-p", "road-signs") + assert [c["name"] for c in document["classes"]] == ["sign"] + assert document["classes"][0]["attributes"][0]["name"] == "occluded" + + +def test_the_same_document_is_a_valid_request_body(tmp_path: Path) -> None: + # The cross-surface claim, tested rather than promised: one schema file works + # against ``visionset schema apply`` and against + # ``POST /projects/{id}/schema/versions``. + SchemaVersionCreate.model_validate(SCHEMA_DOCUMENT) + + +# --- refusing a bad file: exit 2 --------------------------------------------- + + +def test_a_file_that_is_not_json_exits_two(root: Path, tmp_path: Path) -> None: + path = tmp_path / "broken.json" + path.write_text("{not json", encoding="utf-8") + result = run(root, "schema", "apply", str(path), "-p", "road-signs") + assert result.exit_code == 2, result.output + assert "not valid JSON" in result.output + + +def test_a_document_with_no_classes_key_exits_two(root: Path, tmp_path: Path) -> None: + path = tmp_path / "empty.json" + path.write_text(json.dumps({"labels": []}), encoding="utf-8") + result = run(root, "schema", "apply", str(path), "-p", "road-signs") + assert result.exit_code == 2, result.output + assert "classes" in result.output + + +def test_a_file_that_is_not_there_exits_two(root: Path, tmp_path: Path) -> None: + result = run(root, "schema", "apply", str(tmp_path / "absent.json"), "-p", "road-signs") + assert result.exit_code == 2, result.output + + +def test_a_select_with_no_options_exits_two_in_the_domains_words( + root: Path, tmp_path: Path +) -> None: + # The document parses *through* ``LabelClass``, so the kernel's own validator + # is what refuses — nothing about attributes is restated in the CLI. + path = _document( + tmp_path, + [ + { + "name": "sign", + "geometry": "bbox", + "attributes": [{"name": "condition", "kind": "select"}], + } + ], + ) + result = run(root, "schema", "apply", str(path), "-p", "road-signs") + assert result.exit_code == 2, result.output + assert "options" in result.output + + +def test_a_blank_class_name_exits_two_and_says_where(root: Path, tmp_path: Path) -> None: + path = _document(tmp_path, [{"name": " ", "geometry": "bbox"}]) + result = run(root, "schema", "apply", str(path), "-p", "road-signs") + assert result.exit_code == 2, result.output + assert "classes.0.name" in result.output + + +def test_an_unimplemented_geometry_exits_one(root: Path, tmp_path: Path) -> None: + # ``mask`` is a legal ``GeometryType`` member, so the document parses; it is + # the *service* that refuses it. A domain refusal, therefore exit 1. + path = _document(tmp_path, [{"name": "road", "geometry": "mask"}]) + result = run(root, "schema", "apply", str(path), "-p", "road-signs") + assert result.exit_code == 1, result.output + assert "Error:" in result.stderr + + +# --- refusing a narrowing change: exit 1, retryable with a flag --------------- + + +def test_removing_a_class_exits_one_until_the_flag(root: Path, tmp_path: Path) -> None: + ok(root, "schema", "apply", str(schema_file(tmp_path)), "-p", "road-signs") + narrowed = _document(tmp_path, [{"name": "lane", "geometry": "bbox"}]) + refused = run(root, "schema", "apply", str(narrowed), "-p", "road-signs") + assert refused.exit_code == 1, refused.output + assert ok(root, "schema", "apply", str(narrowed), "-p", "road-signs", "--allow-destructive") + + +# --- list -------------------------------------------------------------------- + + +def test_list_is_empty_for_a_schemaless_project(root: Path) -> None: + result = run(root, "schema", "list", "-p", "road-signs") + assert result.stdout.splitlines() == ["VERSION CLASSES GEOMETRIES"] + assert "no schema yet" in result.stderr + + +def test_list_names_each_versions_geometries(root: Path, tmp_path: Path) -> None: + ok(root, "schema", "apply", str(schema_file(tmp_path)), "-p", "road-signs") + rows = ok(root, "schema", "list", "-p", "road-signs").splitlines() + assert rows[1].split() == ["1", "1", "bbox"] + + +def test_list_json_is_the_envelope(root: Path, tmp_path: Path) -> None: + ok(root, "schema", "apply", str(schema_file(tmp_path)), "-p", "road-signs") + document = payload(root, "schema", "list", "-p", "road-signs") + assert document["total"] == 1 + assert document["items"][0]["version"] == 1 diff --git a/tests/examples/test_cli_end_to_end.py b/tests/examples/test_cli_end_to_end.py new file mode 100644 index 00000000..7c3b8a6a --- /dev/null +++ b/tests/examples/test_cli_end_to_end.py @@ -0,0 +1,113 @@ +"""``examples/cli_end_to_end.sh`` still runs, and still leaves what it claims. + +By **subprocess**, not by importing anything — which is the whole point, and the +one thing its two sibling smoke tests cannot do. ``CliRunner`` calls the Typer +app in-process; only running the real ``visionset`` binary proves +``[project.scripts]`` still points somewhere and that the console script works +from a shell that knows nothing about Python. + +Assertions are about **outcomes**, per the sibling modules: the exit code, then +the workspace reopened through the SDK. The narration is never grepped — it is +for a person reading the run, and pinning it would make every wording change a +test failure. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from visionset.kernel.services import ( + WORKSPACE_ENV_VAR, + ProjectService, + ReleaseService, + WorkspaceService, +) + +SCRIPT = Path(__file__).resolve().parents[2] / "examples" / "cli_end_to_end.sh" + +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="the example is a bash script") + + +@pytest.fixture(scope="module") +def destination(tmp_path_factory: pytest.TempPathFactory) -> Path: + return tmp_path_factory.mktemp("workspace") / "cli-e2e" + + +@pytest.fixture(scope="module") +def run(destination: Path) -> subprocess.CompletedProcess[str]: + """One run of the script, shared by every assertion below.""" + # An *assert*, not a skip: a silently skipped CLI test looks exactly like a + # passing one, which is the posture ``require_ffmpeg`` already takes. + assert shutil.which("visionset") is not None, "the console script is not on PATH" + # ``setenv``-style rather than deleting: the script exports the variable, and + # a developer with it already exported must get CI's result. Empty is the + # same as unset both to ``resolve_workspace_root`` and to a shell. + environment = {**os.environ, WORKSPACE_ENV_VAR: ""} + return subprocess.run( + ["bash", str(SCRIPT), str(destination)], + capture_output=True, + text=True, + env=environment, + check=False, + ) + + +@pytest.fixture(scope="module") +def workspace(run: subprocess.CompletedProcess[str], destination: Path) -> Path: + assert run.returncode == 0, run.stderr + return destination / "ws" + + +def test_the_script_runs_to_the_end(run: subprocess.CompletedProcess[str]) -> None: + assert run.returncode == 0, run.stderr + + +def test_it_leaves_one_project_with_a_schema(workspace: Path) -> None: + with WorkspaceService.open(workspace) as service: + projects = ProjectService(service).list() + assert [p.name for p in projects] == ["road-signs"] + + +def test_it_leaves_a_release_of_every_still(workspace: Path) -> None: + with WorkspaceService.open(workspace) as service: + project = ProjectService(service).get_by_name("road-signs") + dataset = ProjectService(service).get_dataset(project.id) + releases = ReleaseService(service).list(dataset.id) + assert [r.tag for r in releases] == ["v1.0"] + assert releases[0].asset_count == 6 + # The stray ``notes.txt`` is reported per file and never becomes an asset. + assert releases[0].schema_version == 1 + + +def test_the_release_it_published_verifies(workspace: Path) -> None: + with WorkspaceService.open(workspace) as service: + project = ProjectService(service).get_by_name("road-signs") + dataset = ProjectService(service).get_dataset(project.id) + releases = ReleaseService(service) + report = releases.verify(releases.list(dataset.id)[0].id) + assert report.ok + assert report.checked == 6 + + +def test_it_carries_no_annotations_and_says_so(workspace: Path) -> None: + # The honest consequence of driving the lifecycle from a terminal: + # ``job mark --progress annotated`` records that somebody labeled an asset, + # and the CLI writes no labels. + with WorkspaceService.open(workspace) as service: + project = ProjectService(service).get_by_name("road-signs") + dataset = ProjectService(service).get_dataset(project.id) + release = ReleaseService(service).list(dataset.id)[0] + assert release.annotation_count == 0 + + +def test_it_leaves_the_export_directory_it_was_given(destination: Path, workspace: Path) -> None: + # ``dummy`` writes nothing, so the directory is empty — but it exists, which + # is what proves the export ran rather than being skipped. + assert (destination / "export").is_dir() + assert workspace.is_dir() diff --git a/tests/fixtures/samples.py b/tests/fixtures/samples.py new file mode 100644 index 00000000..314ecd3f --- /dev/null +++ b/tests/fixtures/samples.py @@ -0,0 +1,177 @@ +# usage: from tests.fixtures.samples import PROJECT, RELEASE +"""One fully-populated instance of every domain model a surface publishes. + +The `tests/fixtures/media.py` precedent: a plain module of module-level values, +no pytest import and no fixtures, so anything may reach for it. It exists for +`tests/cli/test_json_contract.py`, which compares the CLI's JSON projections +against the server's wire models field by field. + +**Every optional field is populated.** A sample carrying `None` where a nested +model belongs would let the projection of that nested model go unchecked, which +is exactly the drift the parity gate is for. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path +from uuid import uuid4 + +from visionset.kernel.domain import ( + AnnotationJob, + AnnotationJobState, + AnnotationSchema, + Asset, + AssetProgress, + Attribute, + Batch, + BatchState, + ExportResult, + GeometryType, + ImageFormat, + IngestFailure, + IngestFailureKind, + IngestJob, + IngestState, + LabelClass, + Project, + Release, + ReleaseVerification, + Source, + SourceKind, + SplitRecipe, + ThumbnailBackfill, + VideoMetadata, + VideoProvenance, +) + +_HASH = "0" * 64 +_WHEN = datetime(2026, 7, 28, 12, 34, 56, 789012, tzinfo=UTC) + +PROJECT = Project( + id=uuid4(), workspace_id=uuid4(), name="road-signs", description="a sample project" +) + +SCHEMA_VERSION = AnnotationSchema( + project_id=PROJECT.id, + version=3, + classes=( + LabelClass( + name="sign", + geometry=GeometryType.BBOX, + color="#ff0000", + attributes=( + Attribute( + name="condition", + kind="select", + required=True, + options=("clean", "faded"), + default="clean", + ), + ), + ), + ), +) + +SOURCE = Source( + project_id=PROJECT.id, + kind=SourceKind.VIDEO, + path=str(Path("/workspace/incoming/clip.mp4")), + registered_at=_WHEN, + capture_params={"lens": "wide"}, + video=VideoProvenance( + metadata=VideoMetadata( + width=160, height=120, fps=10.0, duration_seconds=10.0, codec="h264" + ), + extraction_fps=5.0, + ), +) + +INGEST_FAILURE = IngestFailure( + name="notes.txt", kind=IngestFailureKind.UNSUPPORTED, reason="not a recognizable image" +) + +BATCH = Batch( + project_id=PROJECT.id, + name="clip-5fps", + state=BatchState.IN_ANNOTATION, + schema_version=3, + asset_ids=[uuid4(), uuid4()], +) + +INGEST_JOB = IngestJob( + source_id=SOURCE.id, + state=IngestState.COMPLETED, + error=None, + batch_id=BATCH.id, + batch_name=BATCH.name, + processed=2, + total=3, + failures=(INGEST_FAILURE,), +) + +ASSET = Asset( + project_id=PROJECT.id, + content_hash=_HASH, + uri=str(Path("/workspace/blobs") / _HASH), + width=160, + height=120, + format=ImageFormat.PNG, + source_id=SOURCE.id, + frame_index=4, + frame_timestamp=0.8, + thumbnail_hash="1" * 64, +) + +COUNTS = { + AssetProgress.UNANNOTATED: 1, + AssetProgress.ANNOTATED: 2, + AssetProgress.SKIPPED: 3, + AssetProgress.REVIEW_PENDING: 4, + AssetProgress.ACCEPTED: 5, +} + +JOB = AnnotationJob( + task_group_id=uuid4(), + state=AnnotationJobState.IN_PROGRESS, + progress=dict.fromkeys(BATCH.asset_ids, AssetProgress.UNANNOTATED), +) + +SPLIT = SplitRecipe(train=0.7, val=0.15, test=0.15, seed=42) + +RELEASE = Release( + dataset_id=uuid4(), + tag="v1.0", + manifest_hash=_HASH, + schema_version=3, + asset_count=2, + annotation_count=5, + split=SPLIT, + created_at=_WHEN, + visionset_version="0.0.1.dev0", +) + +VERIFICATION = ReleaseVerification( + release_id=RELEASE.id, + manifest_hash=_HASH, + manifest_intact=True, + checked=2, + missing=("2" * 64,), + corrupt=("3" * 64,), + cache_mismatches=("asset_count",), +) + +EXPORT_RESULT = ExportResult( + release_id=RELEASE.id, + format_name="dummy", + directory=Path("/workspace/exports/dummy"), + file_count=7, + total_bytes=4096, +) + +THUMBNAIL_BACKFILL = ThumbnailBackfill( + project_id=PROJECT.id, + filled=(uuid4(),), + missing=(uuid4(),), + unreadable=(INGEST_FAILURE,), +) diff --git a/tests/kernel/test_project_service.py b/tests/kernel/test_project_service.py index d70abff5..d2973f85 100644 --- a/tests/kernel/test_project_service.py +++ b/tests/kernel/test_project_service.py @@ -228,6 +228,59 @@ def test_a_project_from_another_workspace_reads_as_missing(tmp_path: Path) -> No second_workspace.close() +def test_a_project_reads_back_by_name(tmp_path: Path) -> None: + workspace, projects = _service(tmp_path) + project = projects.create("signs") + assert projects.get_by_name("signs") == project + workspace.close() + + +@pytest.mark.parametrize("spelling", ["SIGNS", "Signs", "sIgNs"]) +def test_a_name_resolves_ignoring_case(tmp_path: Path, spelling: str) -> None: + # The comparison the unique index makes. It lives here rather than in a + # surface because it is not obvious and it is not the only one: a release tag + # is unique per dataset and case-*sensitive*, so a caller re-deriving either + # rule from prose would eventually get one of them wrong. + workspace, projects = _service(tmp_path) + project = projects.create("signs") + assert projects.get_by_name(spelling) == project + workspace.close() + + +def test_a_name_resolves_after_normalization(tmp_path: Path) -> None: + workspace, projects = _service(tmp_path) + project = projects.create("signs") + assert projects.get_by_name(" signs ") == project + workspace.close() + + +def test_getting_an_unknown_name_is_refused(tmp_path: Path) -> None: + workspace, projects = _service(tmp_path) + with pytest.raises(ProjectNotFound, match="no project named"): + projects.get_by_name("signs") + workspace.close() + + +@pytest.mark.parametrize("blank", ["", " ", "\t\n"]) +def test_getting_a_blank_name_is_refused_as_a_name(tmp_path: Path, blank: str) -> None: + # ``InvalidName`` rather than ``ProjectNotFound``: a blank string never named + # anything, which is a different answer from naming something absent. + workspace, projects = _service(tmp_path) + with pytest.raises(InvalidName): + projects.get_by_name(blank) + workspace.close() + + +def test_a_project_from_another_workspace_does_not_resolve_by_name(tmp_path: Path) -> None: + first_workspace, first = _service(tmp_path, "one") + second_workspace, second = _service(tmp_path, "two") + second.create("signs") + with pytest.raises(ProjectNotFound): + first.get_by_name("signs") + first_workspace.close() + second_workspace.close() + + def test_a_fresh_workspace_has_no_projects(tmp_path: Path) -> None: workspace, projects = _service(tmp_path) assert projects.list() == [] diff --git a/tests/kernel/test_release_service.py b/tests/kernel/test_release_service.py index 11eede86..6ff0a378 100644 --- a/tests/kernel/test_release_service.py +++ b/tests/kernel/test_release_service.py @@ -281,6 +281,61 @@ def test_promoting_a_second_batch_does_not_change_an_earlier_release(tmp_path: P fixture.close() +# --- reading a release back by its tag ---------------------------------------- + + +def test_a_release_reads_back_by_its_tag(tmp_path: Path) -> None: + fixture = Fixture(tmp_path) + release = fixture.releases.publish(fixture.ready(), "v1") + assert fixture.releases.get_by_tag(fixture.dataset_id, "v1") == release + fixture.close() + + +def test_a_tag_resolves_after_normalization(tmp_path: Path) -> None: + fixture = Fixture(tmp_path) + release = fixture.releases.publish(fixture.ready(), "v1") + assert fixture.releases.get_by_tag(fixture.dataset_id, " v1 ") == release + fixture.close() + + +def test_a_tag_is_matched_case_sensitively(tmp_path: Path) -> None: + # The opposite of ``ProjectService.get_by_name``, and deliberately so: a tag + # is an identifier rather than a label somebody reads, and the unique index + # compares it exactly. Both rules live beside the index that enforces them so + # that no surface has to re-derive either. + fixture = Fixture(tmp_path) + upper = fixture.releases.publish(fixture.ready(), "V1") + with pytest.raises(ReleaseNotFound): + fixture.releases.get_by_tag(fixture.dataset_id, "v1") + assert fixture.releases.get_by_tag(fixture.dataset_id, "V1") == upper + fixture.close() + + +def test_getting_an_unknown_tag_is_refused(tmp_path: Path) -> None: + fixture = Fixture(tmp_path) + fixture.releases.publish(fixture.ready(), "v1") + with pytest.raises(ReleaseNotFound, match="no release tagged"): + fixture.releases.get_by_tag(fixture.dataset_id, "v2") + fixture.close() + + +@pytest.mark.parametrize("blank", ["", " "]) +def test_getting_a_blank_tag_is_refused_as_a_name(tmp_path: Path, blank: str) -> None: + fixture = Fixture(tmp_path) + fixture.releases.publish(fixture.ready(), "v1") + with pytest.raises(InvalidName): + fixture.releases.get_by_tag(fixture.dataset_id, blank) + fixture.close() + + +def test_getting_a_tag_from_an_unknown_dataset_is_refused(tmp_path: Path) -> None: + fixture = Fixture(tmp_path) + fixture.releases.publish(fixture.ready(), "v1") + with pytest.raises(DatasetNotFound): + fixture.releases.get_by_tag(uuid4(), "v1") + fixture.close() + + # --- what publishing refuses --------------------------------------------------