From 8a0d486ba2d66df7c5c28fac163044495b5317e6 Mon Sep 17 00:00:00 2001 From: Armando Anaya Date: Tue, 25 Aug 2026 23:53:52 -0700 Subject: [PATCH 1/5] feat(kernel): a pre-label run is over one job, and an asset page can be one job's The annotation.pre_label queue row names its annotation job beside its batch; PreLabelRun carries annotation_job_id and a row without it is skipped by readers and refused by the handler. pre_label() and the handler take a job; JobAction declares pre_label on an open job of an in_annotation batch, on BatchAction.PRE_LABEL's precedent. JobService gains require_pre_labelable, live_job, latest_pre_label_run and pre_label_runs; open_jobs_of is the fan-out primitive. BatchService.asset_page keeps only one job's assets when asked. --- src/visionset/inference/__init__.py | 2 + src/visionset/inference/prelabel.py | 62 ++++--- src/visionset/jobs/prelabel.py | 20 ++- src/visionset/kernel/domain/__init__.py | 4 + src/visionset/kernel/domain/capabilities.py | 36 +++- src/visionset/kernel/domain/inference.py | 46 +++-- .../kernel/services/batch_service.py | 53 +++--- src/visionset/kernel/services/job_service.py | 62 +++++++ tests/inference/test_prelabel.py | 162 +++++++++++++----- tests/jobs/test_prelabel_handler.py | 149 +++++++++++++--- tests/kernel/test_batch_asset_page.py | 53 +++++- tests/kernel/test_capabilities.py | 30 +++- tests/kernel/test_inference_connections.py | 8 +- tests/kernel/test_pre_label_runs.py | 94 ++++++++-- 14 files changed, 609 insertions(+), 172 deletions(-) diff --git a/src/visionset/inference/__init__.py b/src/visionset/inference/__init__.py index 1859d928..6c443d4b 100644 --- a/src/visionset/inference/__init__.py +++ b/src/visionset/inference/__init__.py @@ -91,6 +91,7 @@ detectable_classes, effective_produces, no_detectable_class_message, + open_jobs_of, planned, pre_label, prompt_plan, @@ -203,6 +204,7 @@ "measure", "no_detectable_class_message", "not_set_up_message", + "open_jobs_of", "published_digests", "purge", "MINIMUM_FRAGMENT_SHARE", diff --git a/src/visionset/inference/prelabel.py b/src/visionset/inference/prelabel.py index 3b2d4593..0aaaa17d 100644 --- a/src/visionset/inference/prelabel.py +++ b/src/visionset/inference/prelabel.py @@ -1,5 +1,5 @@ # usage: from visionset.inference import pre_label -"""Labeling a batch nobody has opened — the orchestration behind the background job. +"""Labeling a job nobody has opened — the orchestration behind the background job. **Here rather than in a handler, because every surface would need the same thing.** A route, a command and a tool would each have to resolve a connection, @@ -36,6 +36,7 @@ from visionset.inference.providers import ProviderPool, resident from visionset.kernel.domain import ( + OPEN_JOB_STATES, PRE_LABELABLE_STATES, Annotation, AnnotationJob, @@ -69,6 +70,7 @@ BatchService, InferenceConnectionService, IngestService, + JobService, ProjectService, SchemaService, WorkspaceService, @@ -378,6 +380,16 @@ class a shape this model produces could be written as. return selected +def open_jobs_of(workspace: WorkspaceService, batch_id: UUID) -> list[AnnotationJob]: + """The batch's jobs a run may still be asked over, in segment order. + + What every fan-out iterates: a batch launch is one launch per open job, + and a finished job is passed over rather than refused, because a batch + finished half in the annotator is the ordinary case. + """ + return [job for job in BatchService(workspace).jobs(batch_id) if job.state in OPEN_JOB_STATES] + + def served_for( workspace: WorkspaceService, connection_id: UUID, *, pool: ProviderPool | None = None ) -> ServedFamily: @@ -441,7 +453,7 @@ def planned( def pre_label( workspace: WorkspaceService, *, - batch_id: UUID, + job_id: UUID, connection_id: UUID, minimum_confidence: float = DEFAULT_MINIMUM_CONFIDENCE, replace_model_labels: bool = False, @@ -451,13 +463,13 @@ def pre_label( should_stop: Callable[[], bool] | None = None, pool: ProviderPool | None = None, ) -> PreLabelOutcome: - """Ask a model about every untouched asset in a batch, and enter what it finds. + """Ask a model about every untouched asset in a job, and enter what it finds. The order of the lookups is the order of the refusals a caller most needs: the connection first, because "this build cannot run that kind" is an answer - about a setup somebody is part-way through, then the batch's own state, - then the schema, because a batch with nowhere to write what the model - produces is refused before a single image is read. + about a setup somebody is part-way through, then the job's own state and its + batch's, then the schema, because a batch with nowhere to write what the + model produces is refused before a single image is read. An asset untouched when the run started but worked by somebody before the run reaches it is passed over, not fatal — the batch is open for @@ -502,9 +514,10 @@ def pre_label( UnsupportedPrompt: that connection's model answers places, not words. GeometryNotProduced: ``geometries`` names a shape the model does not produce, or no shape at all. - BatchNotFound: no such batch in this workspace. - BatchNotInAnnotation: the batch is not open for annotation, so a model - cannot pre-label it. + JobNotFound: no such job in this workspace. + BatchNotInAnnotation: the job's batch is not open for annotation, so a + model cannot pre-label it. + JobFinished: the job is completed. WorkspaceCorrupt: the batch is open but pinned no schema version — a broken invariant, since approval is what pins one. SchemaHasNoDetectableClass: the pinned schema holds no class a shape @@ -521,8 +534,7 @@ def pre_label( declared = resolved.served(connection, workspace_root=workspace.root) produces = effective_produces(declared.produces, geometries) - batches = BatchService(workspace) - batch = batches.require_pre_labelable(batch_id) + job, batch = JobService(workspace).require_pre_labelable(job_id) schema = require_detectable_schema(workspace, batch, produces) # Announced from inside the run rather than derived by the caller, so the # plan a surface reports is the one this run is about to prompt with and the @@ -537,15 +549,14 @@ def pre_label( for label_class in schema.classes } - jobs = batches.jobs(batch_id) - targets = _targets(workspace, jobs, replace_model_labels=replace_model_labels) + targets = _targets(workspace, job, replace_model_labels=replace_model_labels) total = len(targets) annotations_service = AnnotationService(workspace) ingest = IngestService(workspace) considered = labeled = written = skipped = discarded = out_of_bounds = replaced = 0 model_ref: str | None = None - for job_id, asset_id, replacing in targets: + for asset_id, replacing in targets: # Between assets, which is the only place stopping is honest: the last # asset is committed and the next has not been touched. if should_stop is not None and should_stop(): @@ -600,7 +611,7 @@ def pre_label( superseded = ( sum( 1 - for annotation in annotations_service.for_asset(job_id, asset_id) + for annotation in annotations_service.for_asset(job.id, asset_id) if annotation.provenance == "model" ) if replacing @@ -608,7 +619,7 @@ def pre_label( ) try: annotations_service.enter_unreviewed( - job_id, in_bounds, replacing={asset_id} if replacing else () + job.id, in_bounds, replacing={asset_id} if replacing else () ) except AssetNotWritable: # The batch is `in_annotation`, so somebody working in it @@ -641,15 +652,15 @@ def pre_label( def _targets( workspace: WorkspaceService, - jobs: Sequence[AnnotationJob], + job: AnnotationJob, *, replace_model_labels: bool, -) -> tuple[tuple[UUID, UUID, bool], ...]: - """Every ``(job, asset, replacing)`` this run will reach, in a stable order. +) -> tuple[tuple[UUID, bool], ...]: + """Every ``(asset, replacing)`` this run will reach, in a stable order. - Read off the jobs rather than off the batch's membership, because the write - needs the job that carries the asset and a batch of any size is partitioned - into several. + Read off the job's own progress rather than off its batch's membership: a + batch of any size is partitioned into several jobs, and only the one asked + about is this run's to write into. Progress alone does not prove untouched: ``annotated -> skipped -> unannotated`` is legal and deletes no labels, so an asset can read @@ -663,16 +674,15 @@ def _targets( only path that may remove what an earlier run wrote. """ candidates = [ - (job.id, asset_id, progress is AssetProgress.PRE_LABELED) - for job in jobs + (asset_id, progress is AssetProgress.PRE_LABELED) for asset_id, progress in job.progress.items() if progress is AssetProgress.UNANNOTATED or (replace_model_labels and progress is AssetProgress.PRE_LABELED) ] with workspace.unit_of_work() as uow: return tuple( - (job_id, asset_id, replacing) - for job_id, asset_id, replacing in candidates + (asset_id, replacing) + for asset_id, replacing in candidates if replacing or not uow.annotations.list(asset_id) ) diff --git a/src/visionset/jobs/prelabel.py b/src/visionset/jobs/prelabel.py index ec491db2..d4a1d0e3 100644 --- a/src/visionset/jobs/prelabel.py +++ b/src/visionset/jobs/prelabel.py @@ -1,5 +1,5 @@ # usage: registered as job type "annotation.pre_label" -"""The pre-labeling handler: ask a model about a batch, and enter what it finds. +"""The pre-labeling handler: ask a model about a job, and enter what it finds. **A background job because of how long it is, not how complex.** A batch is hundreds of images and each is a forward pass; a request that waited for them @@ -30,6 +30,7 @@ from visionset.jobs.context import workspace_for from visionset.jobs.registry import HandlerRef, register from visionset.kernel.domain import ( + ANNOTATION_JOB_KEY, BATCH_JOB_KEY, CONNECTION_JOB_KEY, PRE_LABEL_CONFIDENCE_KEY, @@ -53,6 +54,7 @@ def payload_for( + job_id: UUID, batch_id: UUID, connection_id: UUID, minimum_confidence: float, @@ -61,7 +63,7 @@ def payload_for( ) -> dict[str, JsonValue]: """The payload this handler expects, built where the type is known.""" return pre_label_job_payload( - batch_id, connection_id, minimum_confidence, replace_model_labels, geometries + job_id, batch_id, connection_id, minimum_confidence, replace_model_labels, geometries ) @@ -70,13 +72,13 @@ def run( payload: dict[str, JsonValue], reporter: ProgressReporter, ) -> dict[str, JsonValue]: - """Pre-label the named batch's untouched assets and say what that came to. + """Pre-label the named job's untouched assets and say what that came to. ``is_cancelled`` is consulted **both up front and passed through as should_stop**, which is where this differs from the transfer handlers: what follows is a loop over assets rather than one library call, and every iteration boundary is a point at which the last asset is committed and the - next is untouched. Stopping there leaves a batch partly pre-labeled, which is + next is untouched. Stopping there leaves the job partly pre-labeled, which is a coherent state precisely because one frame is one transaction. **What it reports is assets.** A job row's ``processed`` and ``total`` are an @@ -111,6 +113,13 @@ def run( """ if reporter.is_cancelled(): return {} + named_job = payload.get(ANNOTATION_JOB_KEY) + if named_job is None: + raise ValueError( + f"pre-label row carries no {ANNOTATION_JOB_KEY!r}; a row enqueued before the key " + "existed names a batch but no job, and a run is entered into one job's assets" + ) + job_id = UUID(str(named_job)) batch_id = UUID(str(payload[BATCH_JOB_KEY])) connection_id = UUID(str(payload[CONNECTION_JOB_KEY])) minimum_confidence = float(str(payload[PRE_LABEL_CONFIDENCE_KEY])) @@ -126,7 +135,7 @@ def run( workspace = workspace_for(workspace_root) outcome = pre_label( workspace, - batch_id=batch_id, + job_id=job_id, connection_id=connection_id, minimum_confidence=minimum_confidence, replace_model_labels=replace_model_labels, @@ -135,6 +144,7 @@ def run( should_stop=reporter.is_cancelled, ) return { + "annotation_job_id": str(job_id), "batch_id": str(batch_id), "assets_considered": outcome.assets_considered, "assets_labeled": outcome.assets_labeled, diff --git a/src/visionset/kernel/domain/__init__.py b/src/visionset/kernel/domain/__init__.py index a8777d31..bf43fccf 100644 --- a/src/visionset/kernel/domain/__init__.py +++ b/src/visionset/kernel/domain/__init__.py @@ -34,6 +34,7 @@ CONNECTION_GATES, CONNECTION_KINDS, ENDPOINT_TYPES, + JOB_GATES, JOB_MOVES, UNNAMED_EDGES, AssetAction, @@ -74,6 +75,7 @@ PolylineGeometry, ) from visionset.kernel.domain.inference import ( + ANNOTATION_JOB_KEY, BATCH_JOB_KEY, CHECKABLE_STATES, COMMIT_PATTERN, @@ -270,6 +272,7 @@ "ConnectionAction", "UNNAMED_EDGES", "JOB_MOVES", + "JOB_GATES", "BATCH_MOVES", "BATCH_GATES", "CONNECTION_GATES", @@ -283,6 +286,7 @@ "INTEGRITY_CHECK_JOB_TYPE", "PRE_LABEL_JOB_TYPE", "BATCH_JOB_KEY", + "ANNOTATION_JOB_KEY", "CONNECTION_JOB_KEY", "PRE_LABEL_CONFIDENCE_KEY", "PRE_LABEL_GEOMETRIES_KEY", diff --git a/src/visionset/kernel/domain/capabilities.py b/src/visionset/kernel/domain/capabilities.py index 59c1b3a4..d69d1a28 100644 --- a/src/visionset/kernel/domain/capabilities.py +++ b/src/visionset/kernel/domain/capabilities.py @@ -99,9 +99,10 @@ class BatchAction(OpenVocabulary): class JobAction(OpenVocabulary): - """What can be asked of an annotation job.""" + """What can be asked of an annotation job. Declaration order is display order.""" START = "start" + PRE_LABEL = "pre_label" COMPLETE = "complete" @@ -237,7 +238,20 @@ def offered_from(self, current: S, transitions: Mapping[S, frozenset[S]]) -> boo AnnotationJobState.COMPLETED, frozenset({AnnotationJobState.IN_PROGRESS}) ), } -"""Both job actions are moves in ``JOB_TRANSITIONS``.""" +"""The two job actions that are moves in ``JOB_TRANSITIONS``.""" + + +JOB_GATES: Final[Mapping[JobAction, frozenset[AnnotationJobState]]] = { + JobAction.PRE_LABEL: OPEN_JOB_STATES, +} +"""The one job action that changes no job state, and so appears in no table row. + +``pre_label`` is declared from the job's own openness and the batch's, on +``BatchAction.PRE_LABEL``'s precedent: whether the pinned schema declares an +askable class, and whether this machine can run the model, are not facts about +the job, and hiding the control on either ground would leave their refusals +with nowhere to be shown. +""" ASSET_MOVES: Final[Mapping[AssetAction, Move[AssetProgress]]] = { @@ -465,10 +479,10 @@ def job_actions( ) -> list[JobAction]: """Everything this job can be asked to do, given its batch and its assets. - Both actions need the batch open — ``JobService`` runs ``require_open_batch`` - before it consults ``JOB_TRANSITIONS``, so a job that is otherwise startable - inside an ``approved`` batch declares nothing, which is precisely the - dimension the browser's mirror dropped. + Both moves need the batch open, and so does the one gate — ``JobService`` + runs ``require_open_batch`` before it consults ``JOB_TRANSITIONS``, so a job + that is otherwise startable inside an ``approved`` batch declares nothing, + which is precisely the dimension the browser's mirror dropped. ``complete`` is refined here rather than caveated, unlike a batch's: the kernel's extra condition is that every asset is in ``SETTLED_PROGRESS``, and @@ -480,8 +494,14 @@ def job_actions( return [ action for action in JobAction - if JOB_MOVES[action].offered_from(state, JOB_TRANSITIONS) - and (settled or action is not JobAction.COMPLETE) + if ( + ( + JOB_MOVES[action].offered_from(state, JOB_TRANSITIONS) + and (settled or action is not JobAction.COMPLETE) + ) + if action in JOB_MOVES + else state in JOB_GATES[action] + ) ] diff --git a/src/visionset/kernel/domain/inference.py b/src/visionset/kernel/domain/inference.py index ba767d1a..c1db7591 100644 --- a/src/visionset/kernel/domain/inference.py +++ b/src/visionset/kernel/domain/inference.py @@ -470,6 +470,9 @@ def connection_job_payload(connection_id: UUID) -> dict[str, JsonValue]: BATCH_JOB_KEY: Final = "batch_id" """Which batch a background job is about, inside its payload.""" +ANNOTATION_JOB_KEY: Final = "annotation_job_id" +"""Which annotation job a pre-labeling run is over, inside its payload.""" + PRE_LABEL_CONFIDENCE_KEY: Final = "minimum_confidence" """The floor a run applies to what the model returns, inside its payload.""" @@ -481,6 +484,7 @@ def connection_job_payload(connection_id: UUID) -> dict[str, JsonValue]: def pre_label_job_payload( + job_id: UUID, batch_id: UUID, connection_id: UUID, minimum_confidence: float, @@ -489,19 +493,17 @@ def pre_label_job_payload( ) -> dict[str, JsonValue]: """The payload a pre-labeling job carries. Built here, read here. - Five facts and no more: which batch, which connection answers, the floor - the run applies, whether it may supersede its own earlier labels, and which - of the model's shapes it writes — a selection somebody made at launch, kept - so a queued run executes what was asked rather than whatever the model - declares by the time it is claimed. Everything else the handler needs — the - phrases, the asset set — is derived on the other side from the batch - itself, because a payload that carried them would be a copy of state that - can move underneath it. + Six facts and no more: which annotation job, which batch it is a segment + of, which connection answers, the floor the run applies, whether it may + supersede its own earlier labels, and which of the model's shapes it + writes. The batch is carried beside the job so a batch listing can find + its runs without a join; the job is the unit the run is over. """ selected: JsonValue = ( None if geometries is None else [shape.value for shape in sorted(geometries)] ) return { + ANNOTATION_JOB_KEY: str(job_id), BATCH_JOB_KEY: str(batch_id), CONNECTION_JOB_KEY: str(connection_id), PRE_LABEL_CONFIDENCE_KEY: minimum_confidence, @@ -692,16 +694,17 @@ class ConnectionJobs(BaseModel): class PreLabelRun(BaseModel): - """A batch's most recent pre-labeling run, read off its job row. - - ``ConnectionJob``'s model, applied to a batch instead of a connection: a run - outlives the request that launched it and the page that asked, so the only - way a screen can show one it did not itself launch — a reopened dialog, a - second tab, a run somebody started from the terminal — is for the batch it - lists to say so. Derived, never stored: persisting a copy would be a second - encoding of numbers the job row already holds. Not a subclass of - ``ConnectionJob``, because its identity is a batch rather than a connection — - the shape is shared by imitation, not by inheritance. + """A job's most recent pre-labeling run, read off its queue row. + + ``ConnectionJob``'s model, applied to an annotation job instead of a + connection: a run outlives the request that launched it and the page that + asked, so the only way a screen can show one it did not itself launch — a + reopened dialog, a second tab, a run somebody started from the terminal — is + for the job it lists to say so. Derived, never stored: persisting a copy + would be a second encoding of numbers the job row already holds. Not a + subclass of ``ConnectionJob``, because its identity is an annotation job + rather than a connection — the shape is shared by imitation, not by + inheritance. **Named for what its handler counts.** ``prelabel.py``'s unit is assets, so ``assets_processed``/``assets_total`` are named here rather than borrowing a @@ -725,6 +728,7 @@ class PreLabelRun(BaseModel): JOB_TYPE: ClassVar[str] = PRE_LABEL_JOB_TYPE batch_id: UUID + annotation_job_id: UUID job_id: UUID state: BackgroundJobState #: Assets looked at so far, clamped to :attr:`assets_total` for @@ -780,13 +784,16 @@ def of(cls, job: BackgroundJob) -> Self: Raises: ValueError: the job is not a pre-labeling run, or its payload names - no batch. + no batch or no annotation job. """ if job.type != cls.JOB_TYPE: raise ValueError(f"job {job.id} is a {job.type!r}, not a {cls.JOB_TYPE!r}") named = job.payload.get(BATCH_JOB_KEY) if not isinstance(named, str): raise ValueError(f"job {job.id} names no batch") + named_job = job.payload.get(ANNOTATION_JOB_KEY) + if not isinstance(named_job, str): + raise ValueError(f"job {job.id} names no annotation job") result = job.result stopped_early = result.get("stopped_early") assets_labeled = result.get("assets_labeled") @@ -795,6 +802,7 @@ def of(cls, job: BackgroundJob) -> Self: annotations_replaced = result.get("annotations_replaced") return cls( batch_id=UUID(named), + annotation_job_id=UUID(named_job), job_id=job.id, state=job.state, assets_processed=_at_most(job.processed, job.total), diff --git a/src/visionset/kernel/services/batch_service.py b/src/visionset/kernel/services/batch_service.py index 418c7206..68f56534 100644 --- a/src/visionset/kernel/services/batch_service.py +++ b/src/visionset/kernel/services/batch_service.py @@ -40,12 +40,12 @@ from pydantic import BaseModel, ConfigDict from visionset.kernel.domain import ( + ANNOTATION_JOB_KEY, BATCH_JOB_KEY, BATCH_TRANSITIONS, CORRECTABLE_STATES, DELETABLE_STATES, EDITABLE_STATES, - LIVE_JOB_STATES, PRE_LABEL_JOB_TYPE, PRE_LABELABLE_STATES, REPINNABLE_STATES, @@ -56,7 +56,6 @@ Asset, AssetProgress, AssetSort, - BackgroundJob, Batch, BatchApproved, BatchCompleted, @@ -89,6 +88,7 @@ ConfirmationRequired, DestructiveSchemaChange, EmptyBatch, + JobNotFound, ProjectNotFound, SchemaChangeWouldOrphan, WorkspaceCorrupt, @@ -131,27 +131,12 @@ def jobs(self, batch_id: UUID) -> list[AnnotationJob]: with self._workspace.unit_of_work() as uow: return jobs_of(uow, self.require_batch(uow, batch_id)) - def live_job(self, batch_id: UUID, *, job_type: str) -> BackgroundJob | None: - """That kind of work already under way against this batch, if any. - - ``InferenceConnectionService.live_job``'s counterpart, over - :data:`~visionset.kernel.domain.inference.BATCH_JOB_KEY` instead of a - connection's. What a route asks so that a second request joins the run - already in flight instead of paying for the same inference twice — see - that method for the coalescing it does and does not promise. - """ - for job in self._workspace.job_queue.list(states=LIVE_JOB_STATES, types={job_type}): - if job.payload.get(BATCH_JOB_KEY) == str(batch_id): - return job - return None - def latest_pre_label_job(self, batch_id: UUID) -> PreLabelRun | None: """That batch's most recent pre-labeling run, live or settled, if it has one. - ``live_job``'s sibling: the same question about the same job type, asked - over every state rather than only the live ones — a dialog reopened - after a cancelled run or a failure needs the *last* thing that happened - here, not only one still in flight. Delegates to :meth:`pre_label_runs` + Asked over every state rather than only the live ones — a dialog + reopened after a cancelled run or a failure needs the *last* thing that + happened here, not only one still in flight. Delegates to :meth:`pre_label_runs` rather than repeating its payload match: the queue read costs the same either way, so a second body here would only be a second place for that match to drift from the first. @@ -171,14 +156,16 @@ def pre_label_runs(self) -> Mapping[UUID, PreLabelRun]: running now* and *what happened last time*. The queue answers newest-first, so the first job seen for a batch is that batch's latest. - A job whose payload names no batch is skipped rather than raised over: - it cannot be a run this method is about, and a batch listing is the - wrong place to discover a malformed row. + A job whose payload names no batch or no annotation job is skipped + rather than raised over: it cannot be a run this method is about, and + a batch listing is the wrong place to discover a malformed row. """ latest: dict[UUID, PreLabelRun] = {} for job in self._workspace.job_queue.list(types={PRE_LABEL_JOB_TYPE}): named = job.payload.get(BATCH_JOB_KEY) - if not isinstance(named, str): + if not isinstance(named, str) or not isinstance( + job.payload.get(ANNOTATION_JOB_KEY), str + ): continue batch_id = UUID(named) if batch_id not in latest: @@ -205,6 +192,7 @@ def asset_page( self, batch_id: UUID, *, + job: UUID | None = None, progress: frozenset[AssetProgress] | None = None, sort: AssetSort = AssetSort.MEMBERSHIP, limit: int | None = None, @@ -215,24 +203,31 @@ def asset_page( Placement is read once off the jobs, the summary once off the store, and only the window's assets are hydrated — the listing used to read every asset of the batch for every page. ``confidence`` orders by the lowest model score, - unscored last, ties in membership order. + unscored last, ties in membership order. ``job`` keeps only the assets that + job carries; a job the batch does not have is refused, and a draft — which + has no jobs — matches nothing. Raises: BatchNotFound: no such batch in this workspace. + JobNotFound: ``job`` names a job that is not one of this batch's. """ with self._workspace.unit_of_work() as uow: batch = self.require_batch(uow, batch_id) + jobs = jobs_of(uow, batch) + if job is not None and jobs and all(one.id != job for one in jobs): + raise JobNotFound(f"no job {job} in batch {batch.name!r}") placement = { - asset_id: (job.id, job.state, state) - for job in jobs_of(uow, batch) - for asset_id, state in job.progress.items() + asset_id: (job_.id, job_.state, state) + for job_ in jobs + for asset_id, state in job_.progress.items() } summary = uow.annotation_summary(batch.id) nothing = AnnotationSummary(count=0) ids = [ asset_id for asset_id in batch.asset_ids - if progress is None or placement.get(asset_id, (None, None, None))[2] in progress + if (progress is None or placement.get(asset_id, (None, None, None))[2] in progress) + and (job is None or placement.get(asset_id, (None, None, None))[0] == job) ] if sort is AssetSort.CONFIDENCE: position = {asset_id: at for at, asset_id in enumerate(batch.asset_ids)} diff --git a/src/visionset/kernel/services/job_service.py b/src/visionset/kernel/services/job_service.py index 32e7b98a..f9845403 100644 --- a/src/visionset/kernel/services/job_service.py +++ b/src/visionset/kernel/services/job_service.py @@ -34,20 +34,28 @@ from __future__ import annotations +from collections.abc import Mapping from datetime import UTC, datetime from uuid import UUID from visionset.kernel.domain import ( + ANNOTATION_JOB_KEY, ASSET_PROGRESS_TRANSITIONS, + BATCH_JOB_KEY, JOB_TRANSITIONS, + LIVE_JOB_STATES, OPEN_JOB_STATES, + PRE_LABEL_JOB_TYPE, + PRE_LABELABLE_STATES, SETTLED_PROGRESS, AnnotationJob, AnnotationJobState, Asset, AssetProgress, + BackgroundJob, Batch, BatchState, + PreLabelRun, normalize_name, require_move, ) @@ -99,6 +107,60 @@ def batch(self, job_id: UUID) -> Batch: with self._workspace.unit_of_work() as uow: return self.batch_of(uow, self.require_job(uow, job_id)) + def require_pre_labelable(self, job_id: UUID) -> tuple[AnnotationJob, Batch]: + """The job and its batch, if a model may pre-label the job right now. + + ``BatchService.require_pre_labelable``'s construction, one level down: + the batch has to be open for annotation and the job must not be + finished. Everything else pre-labeling can be refused for is a fact + this service cannot see. + + Raises: + JobNotFound: no such job in this workspace. + WorkspaceCorrupt: the job's task group is gone. + BatchNotInAnnotation: the batch is not ``in_annotation``. + JobFinished: the job is ``completed``. + """ + with self._workspace.unit_of_work() as uow: + job = self.require_job(uow, job_id) + batch = self.batch_of(uow, job) + if batch.state not in PRE_LABELABLE_STATES: + raise BatchNotInAnnotation( + f"batch {batch.name!r} is {batch.state.value!r}, not " + f"{BatchState.IN_ANNOTATION.value!r}; a model cannot pre-label a job " + f"of a batch nobody opened" + ) + self.require_open_job(job) + return job, batch + + def live_job(self, job_id: UUID, *, job_type: str) -> BackgroundJob | None: + """That kind of work already under way against this job, if any.""" + for row in self._workspace.job_queue.list(states=LIVE_JOB_STATES, types={job_type}): + if row.payload.get(ANNOTATION_JOB_KEY) == str(job_id): + return row + return None + + def latest_pre_label_run(self, job_id: UUID) -> PreLabelRun | None: + """This job's most recent pre-labeling run, live or settled, if it has one.""" + return self.pre_label_runs().get(job_id) + + def pre_label_runs(self) -> Mapping[UUID, PreLabelRun]: + """Every job's most recent pre-labeling run, read from the queue at once. + + ``BatchService.pre_label_runs`` keyed by the annotation job instead of + the batch; the same one read for a whole listing. The queue answers + newest-first, so the first row seen for a job is that job's latest. + """ + latest: dict[UUID, PreLabelRun] = {} + for row in self._workspace.job_queue.list(types={PRE_LABEL_JOB_TYPE}): + named = row.payload.get(ANNOTATION_JOB_KEY) + if not isinstance(named, str) or not isinstance(row.payload.get(BATCH_JOB_KEY), str): + continue + job_id = UUID(named) + if job_id not in latest: + latest[job_id] = PreLabelRun.of(row) + return latest + def next_pending(self, job_id: UUID, count: int) -> list[Asset]: """The next assets waiting to be annotated, in the batch's own order. diff --git a/tests/inference/test_prelabel.py b/tests/inference/test_prelabel.py index 8aeba292..8c05908f 100644 --- a/tests/inference/test_prelabel.py +++ b/tests/inference/test_prelabel.py @@ -18,6 +18,7 @@ detectable_classes, effective_produces, no_detectable_class_message, + open_jobs_of, planned, pre_label, prompt_plan, @@ -28,6 +29,7 @@ BatchNotFound, BatchNotInAnnotation, GeometryNotProduced, + JobFinished, ProjectNotFound, SchemaHasNoDetectableClass, UnsupportedPrompt, @@ -41,6 +43,7 @@ AssetProgress, Attribute, BboxGeometry, + BySize, ConnectionType, GeometryType, InferenceConnection, @@ -265,6 +268,10 @@ def _asset(self, seed: str, *, width: int | None = None, height: int | None = No ) ).id + @property + def job_id(self) -> UUID: + return self._job_id + def job(self) -> AnnotationJob: return self.jobs.get(self._job_id) @@ -427,7 +434,7 @@ def test_every_untouched_asset_is_labeled_and_enters_pre_labeled(prelabel_fixtur queue: a model's guess never claims to be work a person has already judged.""" outcome = pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, pool=prelabel_fixture.pool, ) @@ -445,7 +452,7 @@ def test_an_asset_somebody_worked_is_passed_over_silently(prelabel_fixture: Fixt outcome = pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, pool=prelabel_fixture.pool, ) @@ -467,7 +474,7 @@ def test_a_skipped_and_restored_asset_is_passed_over_silently(prelabel_fixture: outcome = pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, pool=prelabel_fixture.pool, ) @@ -493,7 +500,7 @@ def move_it(asset_id: UUID) -> None: outcome = pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, pool=prelabel_fixture.pool, ) @@ -511,14 +518,14 @@ def move_it(asset_id: UUID) -> None: def test_a_second_run_picks_up_only_what_is_still_untouched(prelabel_fixture: Fixture) -> None: pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, pool=prelabel_fixture.pool, ) again = pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, pool=prelabel_fixture.pool, ) @@ -530,7 +537,7 @@ def test_a_second_run_picks_up_only_what_is_still_untouched(prelabel_fixture: Fi def test_every_written_label_carries_its_provenance(prelabel_fixture: Fixture) -> None: pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, pool=prelabel_fixture.pool, ) @@ -547,7 +554,7 @@ def test_the_phrases_are_the_schema_s_askable_classes(prelabel_fixture: Fixture) """The schema is the prompt, so nothing comes back that cannot be written.""" pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, pool=prelabel_fixture.pool, ) @@ -561,7 +568,7 @@ def test_a_schema_with_no_producible_class_is_refused_before_anything_loads( with pytest.raises(SchemaHasNoDetectableClass): pre_label( polygon_only_fixture.workspace, - batch_id=polygon_only_fixture.batch.id, + job_id=polygon_only_fixture.job_id, connection_id=polygon_only_fixture.connection.id, pool=polygon_only_fixture.pool, ) @@ -651,7 +658,7 @@ def test_a_run_announces_the_plan_it_is_about_to_prompt_with( pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, on_plan=seen.append, pool=prelabel_fixture.pool, @@ -674,7 +681,7 @@ def test_a_refused_run_announces_no_plan(polygon_only_fixture: Fixture) -> None: with pytest.raises(SchemaHasNoDetectableClass): pre_label( polygon_only_fixture.workspace, - batch_id=polygon_only_fixture.batch.id, + job_id=polygon_only_fixture.job_id, connection_id=polygon_only_fixture.connection.id, on_plan=seen.append, pool=polygon_only_fixture.pool, @@ -694,7 +701,7 @@ class demands an attribute a model cannot supply is refused up front, with pytest.raises(SchemaHasNoDetectableClass): pre_label( required_attribute_fixture.workspace, - batch_id=required_attribute_fixture.batch.id, + job_id=required_attribute_fixture.job_id, connection_id=required_attribute_fixture.connection.id, pool=required_attribute_fixture.pool, ) @@ -707,7 +714,7 @@ def test_a_point_prompt_connection_is_refused(segmenter_fixture: Fixture) -> Non with pytest.raises(UnsupportedPrompt): pre_label( segmenter_fixture.workspace, - batch_id=segmenter_fixture.batch.id, + job_id=segmenter_fixture.job_id, connection_id=segmenter_fixture.connection.id, pool=segmenter_fixture.pool, ) @@ -730,9 +737,12 @@ def test_a_batch_that_is_not_in_annotation_is_refused_before_anything_loads( asset_id = uow.assets.add( Asset(project_id=project.id, content_hash=content_hash, uri="/draft.png") ).id - # A draft, never approved: `require_pre_labelable` refuses every state - # but `in_annotation`, and a draft is the one furthest from it. + # Approved but never started: approval is what partitions the batch into + # jobs, so this is the earliest state a job exists to ask about at all, + # and `require_pre_labelable` admits only `in_annotation`. batch = batches.create(project.id, "first", [asset_id]) + batches.approve(batch.id) + job = batches.jobs(batch.id)[0] connection = connections.create( "draft-connection", connection_type=ConnectionType.HTTP, @@ -743,20 +753,80 @@ def test_a_batch_that_is_not_in_annotation_is_refused_before_anything_loads( pool = FakeProviderPool() with pytest.raises(BatchNotInAnnotation): - pre_label(workspace, batch_id=batch.id, connection_id=connection.id, pool=pool) + pre_label(workspace, job_id=job.id, connection_id=connection.id, pool=pool) assert pool.calls == 0 finally: workspace.close() +def _two_job_batch(fixture: Fixture, name: str) -> tuple[UUID, AnnotationJob, AnnotationJob]: + """A batch of ``fixture``'s project cut into two jobs, open and the first started.""" + assets = [fixture._asset(f"{name}-{index}") for index in range(3)] + batch = fixture.batches.create(fixture.project.id, name, assets) + fixture.batches.approve(batch.id, BySize(size=2)) + fixture.batches.start(batch.id) + first, second = fixture.batches.jobs(batch.id) + fixture.jobs.start(first.id) + return batch.id, first, second + + +def test_a_run_reaches_only_the_job_it_was_asked_about(prelabel_fixture: Fixture) -> None: + """Two jobs of one batch: the run over the first leaves the second untouched.""" + _, first, second = _two_job_batch(prelabel_fixture, "two-jobs") + + outcome = pre_label( + prelabel_fixture.workspace, + job_id=first.id, + connection_id=prelabel_fixture.connection.id, + pool=prelabel_fixture.pool, + ) + + assert outcome.assets_considered == 2 + untouched = prelabel_fixture.jobs.get(second.id) + assert all(p is AssetProgress.UNANNOTATED for p in untouched.progress.values()) + for asset_id in untouched.progress: + assert prelabel_fixture.annotations.for_asset(second.id, asset_id) == [] + + +def test_a_completed_job_is_refused_before_anything_loads(prelabel_fixture: Fixture) -> None: + for asset_id in prelabel_fixture.assets: + prelabel_fixture.mark(asset_id, AssetProgress.SKIPPED) + prelabel_fixture.jobs.complete(prelabel_fixture.job_id) + + with pytest.raises(JobFinished): + pre_label( + prelabel_fixture.workspace, + job_id=prelabel_fixture.job_id, + connection_id=prelabel_fixture.connection.id, + pool=prelabel_fixture.pool, + ) + assert prelabel_fixture.pool.calls == 0 + + +def test_open_jobs_of_lists_the_batchs_open_jobs_in_segment_order( + prelabel_fixture: Fixture, +) -> None: + batch_id, first, second = _two_job_batch(prelabel_fixture, "two-jobs") + + listed = open_jobs_of(prelabel_fixture.workspace, batch_id) + assert [job.id for job in listed] == [first.id, second.id] + + for asset_id in first.progress: + prelabel_fixture.jobs.mark(first.id, asset_id, AssetProgress.SKIPPED) + prelabel_fixture.jobs.complete(first.id) + + still_open = open_jobs_of(prelabel_fixture.workspace, batch_id) + assert [job.id for job in still_open] == [second.id] + + def test_an_asset_the_model_found_nothing_on_stays_untouched( empty_answer_fixture: Fixture, ) -> None: """ "Found nothing" and "reviewed and found empty" are different facts.""" outcome = pre_label( empty_answer_fixture.workspace, - batch_id=empty_answer_fixture.batch.id, + job_id=empty_answer_fixture.job_id, connection_id=empty_answer_fixture.connection.id, pool=empty_answer_fixture.pool, ) @@ -772,7 +842,7 @@ def test_stopping_leaves_what_was_entered_entered(prelabel_fixture: Fixture) -> outcome = pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, should_stop=lambda: next(seen), pool=prelabel_fixture.pool, @@ -794,7 +864,7 @@ def test_progress_is_reported_in_assets(prelabel_fixture: Fixture) -> None: pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, on_progress=lambda done, total: reported.append((done, total)), pool=prelabel_fixture.pool, @@ -812,7 +882,7 @@ def test_a_merged_span_is_discarded_and_the_run_completes( while the mappable region beside it is entered normally.""" outcome = pre_label( merged_span_fixture.workspace, - batch_id=merged_span_fixture.batch.id, + job_id=merged_span_fixture.job_id, connection_id=merged_span_fixture.connection.id, pool=merged_span_fixture.pool, ) @@ -837,7 +907,7 @@ def test_a_capitalized_class_receives_the_casefolded_answer( spelling rather than the model's.""" pre_label( capitalized_class_fixture.workspace, - batch_id=capitalized_class_fixture.batch.id, + job_id=capitalized_class_fixture.job_id, connection_id=capitalized_class_fixture.connection.id, pool=capitalized_class_fixture.pool, ) @@ -854,7 +924,7 @@ def test_an_asset_whose_only_regions_are_unmappable_stays_unannotated( is left untouched, not entered with zero annotations.""" outcome = pre_label( only_unmappable_fixture.workspace, - batch_id=only_unmappable_fixture.batch.id, + job_id=only_unmappable_fixture.job_id, connection_id=only_unmappable_fixture.connection.id, pool=only_unmappable_fixture.pool, ) @@ -872,7 +942,7 @@ def test_a_region_wholly_outside_a_measured_asset_is_discarded( ) -> None: outcome = pre_label( partly_off_frame_fixture.workspace, - batch_id=partly_off_frame_fixture.batch.id, + job_id=partly_off_frame_fixture.job_id, connection_id=partly_off_frame_fixture.connection.id, pool=partly_off_frame_fixture.pool, ) @@ -893,7 +963,7 @@ def test_an_asset_whose_only_regions_are_off_frame_stays_unannotated( ) -> None: outcome = pre_label( only_off_frame_fixture.workspace, - batch_id=only_off_frame_fixture.batch.id, + job_id=only_off_frame_fixture.job_id, connection_id=only_off_frame_fixture.connection.id, pool=only_off_frame_fixture.pool, ) @@ -913,7 +983,7 @@ def test_stopping_after_an_off_frame_region_keeps_its_count( outcome = pre_label( early_stop_off_frame_fixture.workspace, - batch_id=early_stop_off_frame_fixture.batch.id, + job_id=early_stop_off_frame_fixture.job_id, connection_id=early_stop_off_frame_fixture.connection.id, should_stop=lambda: next(seen), pool=early_stop_off_frame_fixture.pool, @@ -973,7 +1043,7 @@ def test_a_polygon_only_schema_runs_against_a_polygon_producing_model(tmp_path: try: outcome = pre_label( fixture.workspace, - batch_id=fixture.batch.id, + job_id=fixture.job_id, connection_id=fixture.connection.id, pool=fixture.pool, ) @@ -1005,7 +1075,7 @@ def test_a_region_in_a_shape_its_class_does_not_admit_is_discarded(tmp_path: Pat try: outcome = pre_label( fixture.workspace, - batch_id=fixture.batch.id, + job_id=fixture.job_id, connection_id=fixture.connection.id, pool=fixture.pool, ) @@ -1037,7 +1107,7 @@ def test_a_region_in_a_shape_the_model_never_declared_is_discarded(tmp_path: Pat try: outcome = pre_label( fixture.workspace, - batch_id=fixture.batch.id, + job_id=fixture.job_id, connection_id=fixture.connection.id, pool=fixture.pool, ) @@ -1077,7 +1147,7 @@ def test_a_selection_narrows_what_a_two_shape_model_writes(tmp_path: Path) -> No try: outcome = pre_label( fixture.workspace, - batch_id=fixture.batch.id, + job_id=fixture.job_id, connection_id=fixture.connection.id, geometries=BOXES, pool=fixture.pool, @@ -1099,7 +1169,7 @@ def test_no_selection_writes_every_shape_the_model_produces(tmp_path: Path) -> N try: outcome = pre_label( fixture.workspace, - batch_id=fixture.batch.id, + job_id=fixture.job_id, connection_id=fixture.connection.id, geometries=None, pool=fixture.pool, @@ -1117,7 +1187,7 @@ def test_a_selection_outside_what_the_model_produces_is_refused_before_anything_ with pytest.raises(GeometryNotProduced) as refused: pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, geometries=POLYGONS, pool=prelabel_fixture.pool, @@ -1134,7 +1204,7 @@ def test_an_empty_selection_is_refused_rather_than_run_as_a_no_op( with pytest.raises(GeometryNotProduced): pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, geometries=frozenset(), pool=prelabel_fixture.pool, @@ -1150,7 +1220,7 @@ def test_the_announced_plan_carries_the_effective_shapes(tmp_path: Path) -> None try: pre_label( fixture.workspace, - batch_id=fixture.batch.id, + job_id=fixture.job_id, connection_id=fixture.connection.id, geometries=POLYGONS, on_plan=seen.append, @@ -1171,7 +1241,7 @@ def test_a_selection_that_leaves_the_schema_nothing_to_hold_is_refused(tmp_path: with pytest.raises(SchemaHasNoDetectableClass) as refused: pre_label( fixture.workspace, - batch_id=fixture.batch.id, + job_id=fixture.job_id, connection_id=fixture.connection.id, geometries=BOXES, pool=fixture.pool, @@ -1388,7 +1458,7 @@ def test_a_replacing_run_rewrites_every_pre_labeled_frame_and_counts_what_it_rep ) -> None: first = pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, pool=prelabel_fixture.pool, ) @@ -1398,7 +1468,7 @@ def test_a_replacing_run_rewrites_every_pre_labeled_frame_and_counts_what_it_rep again = pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, replace_model_labels=True, pool=prelabel_fixture.pool, @@ -1419,7 +1489,7 @@ def test_a_replacing_run_leaves_a_persons_frame_alone(prelabel_fixture: Fixture) """Confirmed means judged: `pre_labeled -> annotated` by `mark`, labels untouched.""" pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, pool=prelabel_fixture.pool, ) @@ -1429,7 +1499,7 @@ def test_a_replacing_run_leaves_a_persons_frame_alone(prelabel_fixture: Fixture) again = pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, replace_model_labels=True, pool=prelabel_fixture.pool, @@ -1444,7 +1514,7 @@ def test_a_replacing_run_leaves_a_persons_frame_alone(prelabel_fixture: Fixture) def test_a_replacing_run_also_enters_frames_still_untouched(prelabel_fixture: Fixture) -> None: pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, pool=prelabel_fixture.pool, ) @@ -1456,7 +1526,7 @@ def test_a_replacing_run_also_enters_frames_still_untouched(prelabel_fixture: Fi again = pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, replace_model_labels=True, pool=prelabel_fixture.pool, @@ -1473,7 +1543,7 @@ def test_a_replacing_run_that_finds_nothing_now_returns_the_frame_to_unannotated ) -> None: pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, pool=prelabel_fixture.pool, ) @@ -1481,7 +1551,7 @@ def test_a_replacing_run_that_finds_nothing_now_returns_the_frame_to_unannotated again = pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, replace_model_labels=True, pool=prelabel_fixture.pool, @@ -1501,7 +1571,7 @@ def test_a_pre_labeled_frame_taken_over_mid_run_is_skipped_by_a_replacing_run( ) -> None: pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, pool=prelabel_fixture.pool, ) @@ -1515,7 +1585,7 @@ def confirm_it(asset_id: UUID) -> None: again = pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, replace_model_labels=True, pool=prelabel_fixture.pool, @@ -1531,13 +1601,13 @@ def test_an_unflagged_second_run_still_never_touches_a_pre_labeled_frame( ) -> None: pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, pool=prelabel_fixture.pool, ) again = pre_label( prelabel_fixture.workspace, - batch_id=prelabel_fixture.batch.id, + job_id=prelabel_fixture.job_id, connection_id=prelabel_fixture.connection.id, pool=prelabel_fixture.pool, ) diff --git a/tests/jobs/test_prelabel_handler.py b/tests/jobs/test_prelabel_handler.py index 139e172d..91dd4574 100644 --- a/tests/jobs/test_prelabel_handler.py +++ b/tests/jobs/test_prelabel_handler.py @@ -19,9 +19,11 @@ PRE_LABEL_JOB_TYPE, Asset, AssetPrediction, + AssetProgress, BackgroundJobSpec, BackgroundJobState, BboxGeometry, + BySize, ConnectionType, GeometryType, LabelClass, @@ -47,16 +49,17 @@ def test_the_type_is_registered_and_idempotent() -> None: def test_the_payload_is_built_where_the_type_is_known() -> None: - batch_id, connection_id = uuid4(), uuid4() + job_id, batch_id, connection_id = uuid4(), uuid4(), uuid4() - payload = prelabel.payload_for(batch_id, connection_id, 0.35) + payload = prelabel.payload_for(job_id, batch_id, connection_id, 0.35) + assert payload["annotation_job_id"] == str(job_id) assert payload["batch_id"] == str(batch_id) assert payload["connection_id"] == str(connection_id) assert payload["minimum_confidence"] == 0.35 assert payload["replace_model_labels"] is False assert ( - prelabel.payload_for(batch_id, connection_id, 0.35, replace_model_labels=True)[ + prelabel.payload_for(job_id, batch_id, connection_id, 0.35, replace_model_labels=True)[ "replace_model_labels" ] is True @@ -76,7 +79,8 @@ def is_cancelled(self) -> bool: def report(self, **_: object) -> None: raise AssertionError("a cancelled run reports nothing") - assert prelabel.run(tmp_path, prelabel.payload_for(uuid4(), uuid4(), 0.35), Cancelled()) == {} + payload = prelabel.payload_for(uuid4(), uuid4(), uuid4(), 0.35) + assert prelabel.run(tmp_path, payload, Cancelled()) == {} assert calls == [] @@ -106,7 +110,7 @@ def test_a_finished_run_reports_progress_and_returns_the_outcome( def fake_pre_label( workspace: object, *, - batch_id: UUID, + job_id: UUID, connection_id: UUID, minimum_confidence: float, replace_model_labels: bool, @@ -114,7 +118,7 @@ def fake_pre_label( on_progress: Any, should_stop: Any, ) -> PreLabelOutcome: - captured["batch_id"] = batch_id + captured["job_id"] = job_id captured["connection_id"] = connection_id captured["minimum_confidence"] = minimum_confidence captured["replace_model_labels"] = replace_model_labels @@ -133,13 +137,16 @@ def fake_pre_label( monkeypatch.setattr(prelabel, "pre_label", fake_pre_label) monkeypatch.setattr(prelabel, "workspace_for", lambda root: object()) reporter = Reporter() - batch_id, connection_id = uuid4(), uuid4() + job_id, batch_id, connection_id = uuid4(), uuid4(), uuid4() result = prelabel.run( - Path("/does/not/matter"), prelabel.payload_for(batch_id, connection_id, 0.4), reporter + Path("/does/not/matter"), + prelabel.payload_for(job_id, batch_id, connection_id, 0.4), + reporter, ) assert result == { + "annotation_job_id": str(job_id), "batch_id": str(batch_id), "assets_considered": 2, "assets_labeled": 2, @@ -151,7 +158,7 @@ def fake_pre_label( "regions_out_of_bounds": 0, "annotations_replaced": 1, } - assert captured["batch_id"] == batch_id + assert captured["job_id"] == job_id assert captured["connection_id"] == connection_id assert captured["minimum_confidence"] == 0.4 assert captured["replace_model_labels"] is False @@ -174,7 +181,7 @@ def fake_pre_label(workspace: object, **kwargs: Any) -> PreLabelOutcome: monkeypatch.setattr(prelabel, "pre_label", fake_pre_label) monkeypatch.setattr(prelabel, "workspace_for", lambda root: object()) - payload = prelabel.payload_for(uuid4(), uuid4(), 0.4) + payload = prelabel.payload_for(uuid4(), uuid4(), uuid4(), 0.4) del payload["replace_model_labels"] prelabel.run(Path("/does/not/matter"), payload, Reporter()) @@ -183,10 +190,11 @@ def fake_pre_label(workspace: object, **kwargs: Any) -> PreLabelOutcome: def test_the_payload_carries_the_shape_selection_and_omits_it_by_default() -> None: - batch_id, connection_id = uuid4(), uuid4() + job_id, batch_id, connection_id = uuid4(), uuid4(), uuid4() - assert prelabel.payload_for(batch_id, connection_id, 0.35)["geometries"] is None + assert prelabel.payload_for(job_id, batch_id, connection_id, 0.35)["geometries"] is None assert prelabel.payload_for( + job_id, batch_id, connection_id, 0.35, @@ -207,7 +215,9 @@ def fake_pre_label(workspace: object, **kwargs: Any) -> PreLabelOutcome: prelabel.run( Path("/does/not/matter"), - prelabel.payload_for(uuid4(), uuid4(), 0.4, geometries=frozenset({GeometryType.BBOX})), + prelabel.payload_for( + uuid4(), uuid4(), uuid4(), 0.4, geometries=frozenset({GeometryType.BBOX}) + ), Reporter(), ) @@ -225,7 +235,7 @@ def fake_pre_label(workspace: object, **kwargs: Any) -> PreLabelOutcome: monkeypatch.setattr(prelabel, "pre_label", fake_pre_label) monkeypatch.setattr(prelabel, "workspace_for", lambda root: object()) - payload = prelabel.payload_for(uuid4(), uuid4(), 0.4) + payload = prelabel.payload_for(uuid4(), uuid4(), uuid4(), 0.4) del payload["geometries"] prelabel.run(Path("/does/not/matter"), payload, Reporter()) @@ -245,7 +255,7 @@ def fake_pre_label(workspace: object, **kwargs: Any) -> PreLabelOutcome: prelabel.run( Path("/does/not/matter"), - prelabel.payload_for(uuid4(), uuid4(), 0.4, replace_model_labels=True), + prelabel.payload_for(uuid4(), uuid4(), uuid4(), 0.4, replace_model_labels=True), Reporter(), ) @@ -343,25 +353,25 @@ def queued_job( endpoint_url="https://example.invalid/predict", ) - job = workspace.job_queue.enqueue( + background_job = workspace.job_queue.enqueue( BackgroundJobSpec( type=PRE_LABEL_JOB_TYPE, - payload=prelabel.payload_for(batch.id, connection.id, 0.35), + payload=prelabel.payload_for(job_id, batch.id, connection.id, 0.35), idempotent=True, ) ) - yield workspace, job.id + yield workspace, job_id, background_job.id workspace.close() def test_the_row_settles_succeeded_with_nobody_polling( - queued_job: tuple[WorkspaceService, UUID], + queued_job: tuple[WorkspaceService, UUID, UUID], ) -> None: """Nothing here reads the row in a loop: `drain()` is the dispatcher's own claim loop run to completion, and the line after it reads the settled row once — the property `weights.py` states and `test_dispatcher.py` holds for every handler through the same seam.""" - workspace, job_id = queued_job + workspace, job_id, background_job_id = queued_job runner = JobRunner( workspace.job_queue, workspace.root, @@ -373,10 +383,107 @@ def test_the_row_settles_succeeded_with_nobody_polling( assert runner.drain() == 1 - stored = workspace.job_queue.get(job_id) + stored = workspace.job_queue.get(background_job_id) assert stored is not None assert stored.state is BackgroundJobState.SUCCEEDED assert stored.result is not None + assert stored.result["annotation_job_id"] == str(job_id) assert stored.result["assets_considered"] == 1 assert stored.result["assets_labeled"] == 1 assert stored.result["model_ref"] == "acme/detector@abc123" + + +def test_a_row_enqueued_before_the_job_key_existed_fails_naming_the_situation( + queued_job: tuple[WorkspaceService, UUID, UUID], +) -> None: + """A payload naming a batch and no job cannot be attributed to a job, and the + row says so instead of dying on a bare `KeyError`.""" + workspace, job_id, _ = queued_job + legacy_payload = prelabel.payload_for(job_id, uuid4(), uuid4(), 0.35) + del legacy_payload["annotation_job_id"] + legacy = workspace.job_queue.enqueue( + BackgroundJobSpec(type=PRE_LABEL_JOB_TYPE, payload=legacy_payload, idempotent=True) + ) + runner = JobRunner( + workspace.job_queue, + workspace.root, + event_bus=workspace.event_bus, + workers=1, + progress_min_interval_s=0, + executor_factory=lambda _: InlineExecutor(), + ) + + runner.drain() + + stored = workspace.job_queue.get(legacy.id) + assert stored is not None + assert stored.state is BackgroundJobState.FAILED + assert "a row enqueued before the key existed" in (stored.error or "") + + +def test_the_handler_pre_labels_only_the_named_job( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Two jobs of one batch: enqueueing and draining the first job's run leaves + the second job's assets untouched, on the same terms as `pre_label` itself.""" + monkeypatch.setattr(prelabel_engine, "resident", lambda: _FakePool()) + workspace = WorkspaceService.init(tmp_path / "ws") + try: + project = ProjectService(workspace).create("prelabel-two-jobs") + SchemaService(workspace).create_version( + project.id, [LabelClass(name="post", geometries=(GeometryType.BBOX,))] + ) + asset_ids = [] + with workspace.unit_of_work() as uow: + for name in ("one", "two", "three"): + content_hash = workspace.blob_store.put(BytesIO(f"{name}-asset".encode())) + asset_ids.append( + uow.assets.add( + Asset( + project_id=project.id, + content_hash=content_hash, + uri=f"/tmp/{name}.png", + ) + ).id + ) + batches = BatchService(workspace) + batch = batches.create(project.id, "two-jobs", asset_ids) + batches.approve(batch.id, BySize(size=2)) + batches.start(batch.id) + first, second = batches.jobs(batch.id) + JobService(workspace).start(first.id) + connection = InferenceConnectionService(workspace).create( + "e2e-connection", + connection_type=ConnectionType.HTTP, + model_id="acme/detector", + model_revision="abc123", + endpoint_url="https://example.invalid/predict", + ) + background_job = workspace.job_queue.enqueue( + BackgroundJobSpec( + type=PRE_LABEL_JOB_TYPE, + payload=prelabel.payload_for(first.id, batch.id, connection.id, 0.35), + idempotent=True, + ) + ) + runner = JobRunner( + workspace.job_queue, + workspace.root, + event_bus=workspace.event_bus, + workers=1, + progress_min_interval_s=0, + executor_factory=lambda _: InlineExecutor(), + ) + + assert runner.drain() == 1 + + stored = workspace.job_queue.get(background_job.id) + assert stored is not None + assert stored.state is BackgroundJobState.SUCCEEDED + assert stored.result is not None + assert stored.result["annotation_job_id"] == str(first.id) + + untouched = JobService(workspace).get(second.id) + assert all(p is AssetProgress.UNANNOTATED for p in untouched.progress.values()) + finally: + workspace.close() diff --git a/tests/kernel/test_batch_asset_page.py b/tests/kernel/test_batch_asset_page.py index b5eb6271..ee3e3859 100644 --- a/tests/kernel/test_batch_asset_page.py +++ b/tests/kernel/test_batch_asset_page.py @@ -3,7 +3,7 @@ from datetime import UTC, datetime from io import BytesIO from pathlib import Path -from uuid import UUID +from uuid import UUID, uuid4 import pytest @@ -13,10 +13,11 @@ AssetProgress, AssetSort, BboxGeometry, + BySize, GeometryType, LabelClass, ) -from visionset.kernel.errors import BatchNotFound +from visionset.kernel.errors import BatchNotFound, JobNotFound from visionset.kernel.services import ( AnnotationService, BatchService, @@ -169,7 +170,51 @@ def test_a_draft_has_no_placement_and_a_progress_filter_over_it_is_empty(fx: Fix def test_unknown_batch_is_refused(fx: Fixture) -> None: - from uuid import uuid4 - with pytest.raises(BatchNotFound): fx.batches.asset_page(uuid4()) + + +def test_a_job_filter_keeps_only_that_jobs_assets(fx: Fixture) -> None: + assets = fx.assets(4) + batch = fx.batches.create(fx.project, "b", assets) + fx.batches.approve(batch.id, BySize(size=2)) + first, second = fx.batches.jobs(batch.id) + + placed, total = fx.batches.asset_page(batch.id, job=first.id) + + assert total == 2 + assert [one.asset.id for one in placed] == assets[:2] + assert all(one.job_id == first.id for one in placed) + + +def test_a_job_filter_composes_with_progress(fx: Fixture) -> None: + assets = fx.assets(4) + batch = fx.batches.create(fx.project, "b", assets) + fx.batches.approve(batch.id, BySize(size=2)) + first, _ = fx.batches.jobs(batch.id) + fx.batches.start(batch.id) + fx.jobs.mark(first.id, assets[0], AssetProgress.SKIPPED) + + placed, total = fx.batches.asset_page( + batch.id, job=first.id, progress=frozenset({AssetProgress.UNANNOTATED}) + ) + + assert total == 1 and placed[0].asset.id == assets[1] + + +def test_a_job_of_another_batch_is_refused(fx: Fixture) -> None: + assets = fx.assets(2) + batch = fx.batches.create(fx.project, "b", assets) + other = fx.batches.create(fx.project, "o", fx.assets(1)) + fx.batches.approve(batch.id) + fx.batches.approve(other.id) + theirs = fx.batches.jobs(other.id)[0] + + with pytest.raises(JobNotFound): + fx.batches.asset_page(batch.id, job=theirs.id) + + +def test_a_draft_with_a_job_filter_is_empty(fx: Fixture) -> None: + batch = fx.batches.create(fx.project, "b", fx.assets(2)) + placed, total = fx.batches.asset_page(batch.id, job=uuid4()) + assert (placed, total) == ([], 0) diff --git a/tests/kernel/test_capabilities.py b/tests/kernel/test_capabilities.py index 64c7b364..87c5162b 100644 --- a/tests/kernel/test_capabilities.py +++ b/tests/kernel/test_capabilities.py @@ -60,6 +60,7 @@ ENDPOINT_TYPES, EVERY_CONNECTION_TYPE, EVERY_SETUP_STATE, + JOB_GATES, JOB_MOVES, JOB_TRANSITIONS, OPEN_JOB_STATES, @@ -306,7 +307,8 @@ def test_every_action_is_decided_by_exactly_one_source() -> None: """ assert set(BATCH_MOVES) | set(BATCH_GATES) == set(BatchAction) assert not set(BATCH_MOVES) & set(BATCH_GATES) - assert set(JOB_MOVES) == set(JobAction) + assert set(JOB_MOVES) | set(JOB_GATES) == set(JobAction) + assert not set(JOB_MOVES) & set(JOB_GATES) assert set(ASSET_MOVES) | {AssetAction.ANNOTATE} == set(AssetAction) # A connection has no moves at all: nothing in this slice changes # `setup_state`, so every one of its actions is decided by a gate. @@ -450,7 +452,7 @@ def test_pre_label_is_offered_only_while_a_batch_is_being_annotated() -> None: ] -@pytest.mark.parametrize("action", list(JobAction), ids=lambda a: a.value) +@pytest.mark.parametrize("action", list(JOB_MOVES), ids=lambda a: a.value) @pytest.mark.parametrize( "scenario", JOB_SCENARIOS, @@ -493,6 +495,30 @@ def _run_job(fixture: Fixture, job_id: UUID, action: JobAction) -> Any: return (fixture.jobs.start if action is JobAction.START else fixture.jobs.complete)(job_id) +@pytest.mark.parametrize( + ("batch_state", "job_state", "declared"), + [ + (BatchState.APPROVED, AnnotationJobState.PENDING, False), + (BatchState.IN_ANNOTATION, AnnotationJobState.PENDING, True), + (BatchState.IN_ANNOTATION, AnnotationJobState.IN_PROGRESS, True), + (BatchState.IN_ANNOTATION, AnnotationJobState.COMPLETED, False), + (BatchState.COMPLETED, AnnotationJobState.COMPLETED, False), + ], +) +def test_pre_label_is_declared_on_an_open_job_of_an_open_batch( + batch_state: BatchState, job_state: AnnotationJobState, declared: bool +) -> None: + actions = job_actions(job_state, batch_state=batch_state, progress=[]) + assert (JobAction.PRE_LABEL in actions) is declared + + +def test_job_actions_are_declared_in_display_order() -> None: + actions = job_actions( + AnnotationJobState.PENDING, batch_state=BatchState.IN_ANNOTATION, progress=[] + ) + assert actions == [JobAction.START, JobAction.PRE_LABEL] + + # --- enforcement: batch assets ------------------------------------------------ #: Settled progress, spelled as the tuple the scenarios below iterate. The set diff --git a/tests/kernel/test_inference_connections.py b/tests/kernel/test_inference_connections.py index b51412b7..cea5dca5 100644 --- a/tests/kernel/test_inference_connections.py +++ b/tests/kernel/test_inference_connections.py @@ -624,7 +624,7 @@ def test_the_model_a_row_already_names_is_not_a_move(connections) -> None: # no def test_a_pre_label_payload_round_trips_its_three_facts() -> None: batch_id, connection_id = uuid4(), uuid4() - payload = pre_label_job_payload(batch_id, connection_id, 0.35) + payload = pre_label_job_payload(uuid4(), batch_id, connection_id, 0.35) assert payload[BATCH_JOB_KEY] == str(batch_id) assert payload[CONNECTION_JOB_KEY] == str(connection_id) @@ -636,7 +636,11 @@ def test_a_pre_label_payload_keeps_the_shape_selection_sorted_and_json_plain() - """Sorted values, never enum members: a queue row is JSON, and two launches naming the same shapes in a different order are the same selection.""" payload = pre_label_job_payload( - uuid4(), uuid4(), 0.35, geometries=frozenset({GeometryType.POLYGON, GeometryType.BBOX}) + uuid4(), + uuid4(), + uuid4(), + 0.35, + geometries=frozenset({GeometryType.POLYGON, GeometryType.BBOX}), ) assert payload[PRE_LABEL_GEOMETRIES_KEY] == ["bbox", "polygon"] diff --git a/tests/kernel/test_pre_label_runs.py b/tests/kernel/test_pre_label_runs.py index 51c494f9..e5bd737a 100644 --- a/tests/kernel/test_pre_label_runs.py +++ b/tests/kernel/test_pre_label_runs.py @@ -15,6 +15,9 @@ import pytest from visionset.kernel.domain import ( + ANNOTATION_JOB_KEY, + BATCH_JOB_KEY, + CONNECTION_JOB_KEY, PRE_LABEL_JOB_TYPE, WEIGHT_DOWNLOAD_JOB_TYPE, BackgroundJob, @@ -25,7 +28,7 @@ connection_job_payload, pre_label_job_payload, ) -from visionset.kernel.services import BatchService, WorkspaceService +from visionset.kernel.services import BatchService, JobService, WorkspaceService @pytest.fixture() @@ -42,11 +45,13 @@ def batches(workspace: WorkspaceService) -> BatchService: return BatchService(workspace) -def enqueue_run(workspace: WorkspaceService, batch_id: UUID) -> BackgroundJob: +def enqueue_run( + workspace: WorkspaceService, batch_id: UUID, job_id: UUID | None = None +) -> BackgroundJob: return workspace.job_queue.enqueue( BackgroundJobSpec( type=PRE_LABEL_JOB_TYPE, - payload=pre_label_job_payload(batch_id, uuid4(), 0.35), + payload=pre_label_job_payload(job_id or uuid4(), batch_id, uuid4(), 0.35), idempotent=True, ) ) @@ -60,7 +65,7 @@ def test_a_job_row_reads_as_assets() -> None: batch_id = uuid4() job = BackgroundJob( type=PRE_LABEL_JOB_TYPE, - payload=pre_label_job_payload(batch_id, uuid4(), 0.35), + payload=pre_label_job_payload(uuid4(), batch_id, uuid4(), 0.35), state=BackgroundJobState.RUNNING, processed=12, total=48, @@ -78,7 +83,7 @@ def test_progress_is_clamped_to_its_total() -> None: """The two counts can disagree by a cosmetic amount; a listing must not 500.""" job = BackgroundJob( type=PRE_LABEL_JOB_TYPE, - payload=pre_label_job_payload(uuid4(), uuid4(), 0.35), + payload=pre_label_job_payload(uuid4(), uuid4(), uuid4(), 0.35), processed=50, total=48, ) @@ -90,6 +95,7 @@ def test_the_type_refuses_progress_above_its_total() -> None: with pytest.raises(ValueError, match="cannot have processed"): PreLabelRun( batch_id=uuid4(), + annotation_job_id=uuid4(), job_id=uuid4(), state=BackgroundJobState.RUNNING, assets_processed=50, @@ -113,6 +119,38 @@ def test_a_job_naming_no_batch_is_refused() -> None: PreLabelRun.of(job) +def test_a_run_names_the_annotation_job_it_is_over() -> None: + job_id, batch_id = uuid4(), uuid4() + payload = pre_label_job_payload(job_id, batch_id, uuid4(), 0.35) + assert payload[ANNOTATION_JOB_KEY] == str(job_id) + assert payload[BATCH_JOB_KEY] == str(batch_id) + row = BackgroundJob( + id=uuid4(), type=PRE_LABEL_JOB_TYPE, state=BackgroundJobState.QUEUED, payload=payload + ) + assert PreLabelRun.of(row).annotation_job_id == job_id + + +def test_a_row_naming_no_annotation_job_is_refused_and_skipped( + workspace: WorkspaceService, batches: BatchService +) -> None: + """A row enqueued before the key existed is not a run this build can attribute.""" + batch_id = uuid4() + legacy = workspace.job_queue.enqueue( + BackgroundJobSpec( + type=PRE_LABEL_JOB_TYPE, + payload={ + BATCH_JOB_KEY: str(batch_id), + CONNECTION_JOB_KEY: str(uuid4()), + "minimum_confidence": 0.35, + }, + idempotent=True, + ) + ) + with pytest.raises(ValueError, match="annotation job"): + PreLabelRun.of(legacy) + assert batches.pre_label_runs() == {} + + # --- the handler's own outcome --------------------------------------------------- @@ -120,7 +158,7 @@ def test_the_outcome_is_null_before_the_job_settles() -> None: """A queued or running job has no result yet — nothing to read it out of.""" job = BackgroundJob( type=PRE_LABEL_JOB_TYPE, - payload=pre_label_job_payload(uuid4(), uuid4(), 0.35), + payload=pre_label_job_payload(uuid4(), uuid4(), uuid4(), 0.35), state=BackgroundJobState.RUNNING, processed=3, total=8, @@ -138,7 +176,7 @@ def test_the_outcome_is_null_before_the_job_settles() -> None: def test_a_succeeded_run_carries_the_handlers_outcome() -> None: job = BackgroundJob( type=PRE_LABEL_JOB_TYPE, - payload=pre_label_job_payload(uuid4(), uuid4(), 0.35), + payload=pre_label_job_payload(uuid4(), uuid4(), uuid4(), 0.35), state=BackgroundJobState.SUCCEEDED, processed=8, total=8, @@ -167,7 +205,7 @@ def test_a_succeeded_run_carries_the_handlers_outcome() -> None: def test_a_pre_label_result_ignores_boolean_and_malformed_counts() -> None: job = BackgroundJob( type=PRE_LABEL_JOB_TYPE, - payload=pre_label_job_payload(uuid4(), uuid4(), 0.35), + payload=pre_label_job_payload(uuid4(), uuid4(), uuid4(), 0.35), state=BackgroundJobState.SUCCEEDED, result={ "assets_labeled": True, @@ -189,7 +227,7 @@ def test_a_failed_run_keeps_the_sentence_and_has_no_outcome() -> None: """A failure never reaches the point of building a result dict.""" job = BackgroundJob( type=PRE_LABEL_JOB_TYPE, - payload=pre_label_job_payload(uuid4(), uuid4(), 0.35), + payload=pre_label_job_payload(uuid4(), uuid4(), uuid4(), 0.35), state=BackgroundJobState.FAILED, error="the model server is unreachable", processed=3, @@ -207,7 +245,7 @@ def test_a_cancelled_run_still_carries_its_outcome() -> None: """Stopping partway is a coherent outcome, not the absence of one.""" job = BackgroundJob( type=PRE_LABEL_JOB_TYPE, - payload=pre_label_job_payload(uuid4(), uuid4(), 0.35), + payload=pre_label_job_payload(uuid4(), uuid4(), uuid4(), 0.35), state=BackgroundJobState.CANCELLED, processed=12, total=48, @@ -388,3 +426,39 @@ def test_other_job_types_are_not_read_as_pre_label_runs( ) assert batches.pre_label_runs() == {} + + +# --- what a job reports, through the service -------------------------------------- + + +def test_a_job_remembers_its_own_run_and_not_its_siblings(tmp_path: Path) -> None: + from tests.inference.test_prelabel import Fixture + + fixture = Fixture(tmp_path) + try: + jobs = JobService(fixture.workspace) + sibling = uuid4() + mine = enqueue_run(fixture.workspace, fixture.batch.id, fixture.job_id) + enqueue_run(fixture.workspace, fixture.batch.id, sibling) + + latest = jobs.latest_pre_label_run(fixture.job_id) + assert latest is not None + assert latest.job_id == mine.id + live = jobs.live_job(fixture.job_id, job_type=PRE_LABEL_JOB_TYPE) + assert live is not None + assert live.id == mine.id + assert set(jobs.pre_label_runs()) == {fixture.job_id, sibling} + finally: + fixture.close() + + +def test_a_jobs_newest_run_is_the_one_reported(workspace: WorkspaceService) -> None: + """A job pre-labeled twice reports the second attempt, not the first.""" + job_id = uuid4() + enqueue_run(workspace, uuid4(), job_id) + second = enqueue_run(workspace, uuid4(), job_id) + + latest = JobService(workspace).latest_pre_label_run(job_id) + + assert latest is not None + assert latest.job_id == second.id From 9f19e81d7d884a2f49de494ccba50a0aa09738a1 Mon Sep 17 00:00:00 2001 From: Armando Anaya Date: Tue, 25 Aug 2026 23:53:52 -0700 Subject: [PATCH 2/5] feat(api): pre-label a job; batch and project launches fan out one row per open job POST /jobs/{job_id}/pre-label queues one run (202, Location) and refuses a finished job. The batch and project launches answer PreLabelFanOutOut, one row per open job with annotation_job_id, the queued or joined background job and joined. JobOut carries pre_label_run; PreLabelRunOut names its annotation job. GET /batches/{batch_id}/assets takes job= to keep one job's assets, 404 JOB_NOT_FOUND for a job the batch lacks and the empty page over a draft. The launch logic the three routes share lives once in routes/_prelabel.py. --- frontend/ui-core/src/generated/api.ts | 334 ++++++++++++++++++----- frontend/ui-core/src/generated/checks.ts | 23 +- openapi.json | 272 ++++++++++++++---- src/visionset/server/models.py | 46 ++-- src/visionset/server/routes/_prelabel.py | 134 +++++++++ src/visionset/server/routes/batches.py | 268 ++++++++---------- src/visionset/server/routes/jobs.py | 141 +++++++++- src/visionset/wire/__init__.py | 16 +- tests/fixtures/wire_capabilities.json | 4 +- tests/server/test_batches.py | 19 ++ tests/server/test_jobs.py | 4 +- tests/server/test_prelabel_route.py | 100 ++++++- 12 files changed, 1033 insertions(+), 328 deletions(-) create mode 100644 src/visionset/server/routes/_prelabel.py diff --git a/frontend/ui-core/src/generated/api.ts b/frontend/ui-core/src/generated/api.ts index 6d8703ac..0604c9b9 100644 --- a/frontend/ui-core/src/generated/api.ts +++ b/frontend/ui-core/src/generated/api.ts @@ -239,9 +239,11 @@ export interface paths { * An offset past the end is an empty list and a 200, never a 404. The 404 belongs * to the batch itself, which is resolved first: an unknown one is `BATCH_NOT_FOUND`. * - * `job_id` and `progress` are null while the batch is a draft, because a draft - * has no jobs — so a `progress` filter over a draft matches nothing. Bytes are - * not here: an asset is named by its hashes, and + * `job` narrows to the assets one job carries, composing with `progress`; a job + * this batch does not have is 404 `JOB_NOT_FOUND`. `job_id` and `progress` on + * each item are null while the batch is a draft, which has no jobs — so there, + * a `progress` filter or a `job` filter matches nothing rather than refusing. + * Bytes are not here: an asset is named by its hashes, and * `GET /projects/{project_id}/assets/{asset_id}/content` is what serves them. */ get: operations["list_batch_assets"]; @@ -427,6 +429,16 @@ export interface paths { * Pre Label Batch * @description Ask a model to label every untouched asset in this batch, and answer at once. * + * **One row per open job, and the job is the unit.** This launch fans out over + * the batch's jobs that are still open and queues for each the same + * `annotation.pre_label` row `POST /jobs/{job_id}/pre-label` queues, or joins + * the one already queued or running for that job (`joined`). A finished job is + * passed over, so a batch whose every job is complete answers an empty page. + * Each row is polled, cancelled and remembered per job: + * `GET /background-jobs/{id}` for progress counted in that job's assets, + * `JobOut.pre_label_run` afterwards. Nothing here reports one total across + * jobs, because nothing here is one run. + * * The `pre_label` action. Labels land at `pre_labeled`, never at `annotated`: * nobody judged them, so they arrive editable and correctable rather than * claiming to be somebody's work — and, being unjudged, they never reach the @@ -438,11 +450,11 @@ export interface paths { * that still carries annotations from an earlier round that was skipped and * then restored: that sequence deletes no labels, so progress alone does not * prove an asset untouched. A run never writes over what a person did in this - * batch, and never writes twice over what a model did — a plain second run + * job, and never writes twice over what a model did — a plain second run * extends an earlier one onto whatever is still untouched. * `replace_model_labels` widens it to every frame still `pre_labeled` and * supersedes those labels with this run's answer, one frame per transaction; - * a frame anyone edited, confirmed or skipped in this batch is never touched, + * a frame anyone edited, confirmed or skipped in this job is never touched, * and a frame the model now finds nothing on returns to `unannotated`. A * replacing request arriving while a run is in flight joins that run, * whichever flag it carries. @@ -467,14 +479,14 @@ export interface paths { * and it is kept on the queued row, so a run claimed later executes what was * asked. * - * **202, not 200.** A batch is hundreds of forward passes, so this follows the + * **202, not 200.** A job is hundreds of forward passes, so this follows the * launch-and-poll contract the export and weight-download routes use: poll `GET - * /background-jobs/{id}` — the `Location` header names it — until `state` is - * `succeeded`, then re-read the batch's assets. Progress on the row is counted - * in assets. + * /background-jobs/{id}` for each row until `state` is `succeeded`, then + * re-read the batch's assets. Progress on a row is counted in assets. There is + * no `Location` header, because there is no single row for it to name. * * **Everything a caller can be told now is told now**, and no refusal creates a - * job — so a caller holding a job id holds one that will run. These refusals + * row — so a caller holding a row's id holds one that will run. These refusals * are about the request, and the caller can act on each. They are checked in * this order, and it is the order `pre_label` itself checks in, so a request * wrong about the connection and the batch both always names the connection: @@ -494,10 +506,11 @@ export interface paths { * `WORKSPACE_CORRUPT`. Neither is worth resending unchanged: there is no * state here a caller can change, so the remedy is the one the message names. * - * **Asking twice joins the run already in flight rather than starting a second - * one.** A request arriving while this batch has a pre-labeling run queued or - * running is answered with that run's id, so a double-click and a second tab - * watch one run instead of paying for the same inference twice. + * **Asking twice joins the runs already in flight rather than starting second + * ones.** A request arriving while a job here has a pre-labeling run queued or + * running is answered with that run's row and `joined` true, so a double-click + * and a second tab watch one run per job instead of paying for the same + * inference twice. */ post: operations["pre_label_batch"]; delete?: never; @@ -1645,6 +1658,103 @@ export interface paths { patch?: never; trace?: never; }; + "/jobs/{job_id}/pre-label": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Pre Label Job + * @description Ask a model to label every untouched asset in this job, and answer at once. + * + * The `pre_label` action. Labels land at `pre_labeled`, never at `annotated`: + * nobody judged them, so they arrive editable and correctable rather than + * claiming to be somebody's work — and, being unjudged, they never reach the + * Dataset until a person has taken them over. + * + * **Only assets nothing has touched — which is stronger than reading + * `unannotated`.** An asset already `pre_labeled`, annotated, skipped, + * awaiting review or accepted is passed over, and so is an `unannotated` one + * that still carries annotations from an earlier round that was skipped and + * then restored: that sequence deletes no labels, so progress alone does not + * prove an asset untouched. A run never writes over what a person did in this + * job, and never writes twice over what a model did — a plain second run + * extends an earlier one onto whatever is still untouched. + * `replace_model_labels` widens it to every frame still `pre_labeled` and + * supersedes those labels with this run's answer, one frame per transaction; + * a frame anyone edited, confirmed or skipped in this job is never touched, + * and a frame the model now finds nothing on returns to `unannotated`. A + * replacing request arriving while a run is in flight joins that run, + * whichever flag it carries. + * + * **The batch's pinned schema is the prompt, narrowed to what this run + * writes.** The model is asked for each class the schema declares that admits + * one of the shapes the run writes and demands no attribute a prediction + * cannot supply; an answer naming one of those classes, matched + * case-insensitively, is written under the schema's own spelling, and an + * answer naming none of them is discarded. A schema with no such class has + * nowhere for a prediction to land and is refused — so the same schema is + * askable of a model that answers polygons and refused for one that answers + * boxes. `GET /batches/{batch_id}/pre-label` with the same `connection_id` + * (and the same `geometries`) reads the narrowing before launching. + * + * **What the run writes is every shape the model produces, unless + * `geometries` says which.** A model declaring both a box and a polygon + * writes both for every region it answers with — the kernel writes one + * annotation per emitted region and pairs nothing — and `geometries` filters + * that to the shapes named: a region in any other shape is discarded and + * counted in `regions_discarded`. The selection is per run, not per class, + * and it is kept on the queued row, so a run claimed later executes what was + * asked. + * + * **202, not 200.** A job is hundreds of forward passes, so this follows the + * launch-and-poll contract the export and weight-download routes use: poll `GET + * /background-jobs/{id}` — the `Location` header names it — until `state` is + * `succeeded`, then re-read the job's assets. Progress on the row is counted + * in assets, and `JobOut.pre_label_run` remembers the same row afterwards. + * + * **Everything a caller can be told now is told now**, and no refusal creates a + * job — so a caller holding a job id holds one that will run. These refusals + * are about the request, and the caller can act on each. They are checked in + * this order, and it is the order `pre_label` itself checks in, so a request + * wrong about the connection and the job both always names the connection: + * an unknown connection is 404 `INFERENCE_CONNECTION_NOT_FOUND`; a + * connection not set up yet is 409 `INFERENCE_CONNECTION_NOT_SET_UP` — its + * weights not here, or its endpoint not yet asked what it answers; a + * connection whose model answers places rather than words is 422 + * `UNSUPPORTED_PROMPT`; a `geometries` naming a shape the model does not + * produce is 422 `GEOMETRY_NOT_PRODUCED`. An unknown job is 404 + * `JOB_NOT_FOUND`; a job whose batch is not `in_annotation` is 409 + * `BATCH_NOT_IN_ANNOTATION`; a job already `completed` is 409 `JOB_FINISHED`, + * and there is no remedy on this route — settled work is corrected through a + * new batch rather than reopened. A pinned schema with no class the selected + * shapes can be written as is 409 `SCHEMA_HAS_NO_DETECTABLE_CLASS`. + * + * Two failures are about this installation rather than about the request, and + * answer 500 carrying the message that says which: a machine without the + * optional local runtime is `LOCAL_INFERENCE_UNAVAILABLE` and carries the + * exact command that installs it, and a workspace whose records no longer + * hold together — a batch pinned to a schema version that is not stored — is + * `WORKSPACE_CORRUPT`. Neither is worth resending unchanged: there is no + * state here a caller can change, so the remedy is the one the message names. + * + * **Asking twice joins the run already in flight rather than starting a second + * one.** A request arriving while this job has a pre-labeling run queued or + * running is answered with that run's id, so a double-click and a second tab + * watch one run instead of paying for the same inference twice — and so does + * `POST /batches/{batch_id}/pre-label`, whose fan-out reaches this same job. + */ + post: operations["pre_label_job"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/jobs/{job_id}/progress": { parameters: { query?: never; @@ -1965,15 +2075,18 @@ export interface paths { * Pre Label Project Batches * @description Ask a model to label every untouched asset across this project's open batches. * - * **One row per batch, and the batch stays the unit.** This launch fans out - * over the project's batches that are open for annotation — every one of - * them, or exactly the `batch_ids` named — and for each one queues the same - * `annotation.pre_label` job `POST /batches/{batch_id}/pre-label` queues, or - * joins the one already queued or running for that batch (`joined`). Each - * row is polled, cancelled and remembered per batch, exactly as a - * single-batch launch is: `GET /background-jobs/{id}` for progress counted - * in that batch's assets, `BatchOut.pre_label_run` afterwards. Nothing here - * reports one total across batches, because nothing here is one run. + * **One row per open job of each selected batch, and the job is the unit.** + * This launch fans out over the project's batches that are open for + * annotation — every one of them, or exactly the `batch_ids` named — and + * within each over the jobs still open, queueing for each the same + * `annotation.pre_label` row `POST /jobs/{job_id}/pre-label` queues, or + * joining the one already queued or running for that job (`joined`). A + * finished job is passed over, so a selected batch whose every job is + * complete contributes no row. Each row is polled, cancelled and remembered + * per job, exactly as a single-job launch is: `GET /background-jobs/{id}` + * for progress counted in that job's assets, `JobOut.pre_label_run` + * afterwards. Nothing here reports one total across jobs, because nothing + * here is one run. * * **Refused whole, up front, and no refusal creates a row.** The connection * is checked first, as the single-batch launch checks it: an unknown @@ -4130,10 +4243,10 @@ export interface components { }; /** * JobAction - * @description What can be asked of an annotation job. + * @description What can be asked of an annotation job. Declaration order is display order. * @enum {string} */ - JobAction: "start" | "complete" | (string & {}); + JobAction: "start" | "pre_label" | "complete" | (string & {}); /** * JobAssign * @description The name to record for this job, or null to clear it. @@ -4144,7 +4257,8 @@ export interface components { }; /** * JobOut - * @description One annotator's unit of work over a segment of a batch. + * @description One annotator's unit of work over a segment of a batch, and its most recent + * pre-labeling run, `null` where none ever ran. */ JobOut: { /** Allowed Actions */ @@ -4163,6 +4277,7 @@ export interface components { * Format: uuid */ id: string; + pre_label_run: components["schemas"]["PreLabelRunOut"] | null; state: components["schemas"]["AnnotationJobState"]; }; /** @@ -4334,6 +4449,37 @@ export interface components { * @enum {string} */ PreLabelExclusionReason: "no_producible_geometry" | "required_attribute" | (string & {}); + /** + * PreLabelFanOutItemOut + * @description One job's row in a launch that fanned out over several. + */ + PreLabelFanOutItemOut: { + /** + * Annotation Job Id + * Format: uuid + */ + annotation_job_id: string; + /** + * Batch Id + * Format: uuid + */ + batch_id: string; + /** Batch Name */ + batch_name: string; + job: components["schemas"]["BackgroundJobOut"]; + /** Joined */ + joined: boolean; + }; + /** + * PreLabelFanOutOut + * @description Every job the launch fanned out over, one row each, in batch then segment order. + */ + PreLabelFanOutOut: { + /** Items */ + items: components["schemas"]["PreLabelFanOutItemOut"][]; + /** Total */ + total: number; + }; /** * PreLabelPlanOut * @description The words a run would ask a model for over this batch, and the shapes it would write. @@ -4386,16 +4532,19 @@ export interface components { }; /** * PreLabelRunOut - * @description A batch's most recent pre-labeling run: which job, how far, and what it found. + * @description The most recent pre-labeling run: which job, how far, and what it found. + * + * `annotation_job_id` is the job the run is over; `job_id` is the queue row to + * poll. * - * Present whenever pre-labeling has ever been asked for on this batch, and - * describing the most recent run — including one this session did not launch. - * A dialog reopened after a reload, in a second tab, or after a run started - * from the terminal reads the same state from here rather than from a job id - * a component happened to keep. + * Present whenever pre-labeling has ever been asked for, and describing the + * most recent run — including one this session did not launch. A dialog + * reopened after a reload, in a second tab, or after a run started from the + * terminal reads the same state from here rather than from a job id a + * component happened to keep. * * **Assets, where a download counts bytes and a check counts files.** The - * handler owns a loop over the batch's untouched assets and knows the whole + * handler owns a loop over the job's untouched assets and knows the whole * set before the first forward pass, so both its progress and its total are * counted in the unit its own work is over. * @@ -4408,6 +4557,11 @@ export interface components { * contract is to write only where nothing has been written. */ PreLabelRunOut: { + /** + * Annotation Job Id + * Format: uuid + */ + annotation_job_id: string; /** Annotations Replaced */ annotations_replaced: number | null; /** Assets Labeled */ @@ -4520,32 +4674,6 @@ export interface components { /** Total */ total: number; }; - /** - * ProjectPreLabelItemOut - * @description One batch's row in a project-wide launch. - */ - ProjectPreLabelItemOut: { - /** - * Batch Id - * Format: uuid - */ - batch_id: string; - /** Batch Name */ - batch_name: string; - job: components["schemas"]["BackgroundJobOut"]; - /** Joined */ - joined: boolean; - }; - /** - * ProjectPreLabelOut - * @description Every batch the launch fanned out over, one row each, in selection order. - */ - ProjectPreLabelOut: { - /** Items */ - items: components["schemas"]["ProjectPreLabelItemOut"][]; - /** Total */ - total: number; - }; /** * ProjectPreLabelRequest * @description Which model should pre-label this project's open batches, and which batches. @@ -5750,6 +5878,8 @@ export interface operations { progress?: components["schemas"]["AssetProgress"][] | null; /** @description `membership` is stored order; `confidence` is lowest model confidence first, unscored last, ties in membership order. */ sort?: components["schemas"]["AssetSort"]; + /** @description Keep only the assets this job carries. Omit for the whole batch. */ + job?: string | null; }; header?: never; path: { @@ -6298,7 +6428,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["BackgroundJobOut"]; + "application/json": components["schemas"]["PreLabelFanOutOut"]; }; }; /** @description Missing or invalid bearer token */ @@ -8891,6 +9021,86 @@ export interface operations { }; }; }; + pre_label_job: { + parameters: { + query?: never; + header?: never; + path: { + job_id: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["PreLabelRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 202: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BackgroundJobOut"]; + }; + }; + /** @description Missing or invalid bearer token */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorBody"]; + }; + }; + /** @description No such resource */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorBody"]; + }; + }; + /** @description The resource's state refuses this request */ + 409: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorBody"]; + }; + }; + /** @description The request payload is not processable */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorBody"]; + }; + }; + /** @description Unhandled server error, with an incident id */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorBody"]; + }; + }; + /** @description The workspace is busy; retry after the header says */ + 503: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorBody"]; + }; + }; + }; + }; get_job_progress: { parameters: { query?: never; @@ -9888,7 +10098,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["ProjectPreLabelOut"]; + "application/json": components["schemas"]["PreLabelFanOutOut"]; }; }; /** @description Missing or invalid bearer token */ @@ -11741,7 +11951,7 @@ export interface KnownMembers { AssetAction: "annotate" | "skip" | "restore" | "confirm" | "submit_for_review" | "accept" | "return_to_annotator"; BatchAction: "approve" | "start" | "complete" | "repin" | "promote" | "create_correction" | "pre_label" | "edit_membership" | "delete"; ConnectionAction: "download_weights" | "check_integrity" | "test_endpoint" | "update" | "update_model" | "delete"; - JobAction: "start" | "complete"; + JobAction: "start" | "pre_label" | "complete"; ModelCapability: "point_suggest" | "text_detect"; PreLabelExclusionReason: "no_producible_geometry" | "required_attribute"; SuggestParameter: "tolerance"; diff --git a/frontend/ui-core/src/generated/checks.ts b/frontend/ui-core/src/generated/checks.ts index f2b43697..4be6a319 100644 --- a/frontend/ui-core/src/generated/checks.ts +++ b/frontend/ui-core/src/generated/checks.ts @@ -100,7 +100,7 @@ export const checkBatchState: Check = /*#__PURE__*/ oneOf(["draft", "approved", "in_annotation", "completed"] as const); export const checkPreLabelRunOut: Check = - /*#__PURE__*/ object({ "annotations_replaced": [true, either([isInteger, isNull] as const)], "assets_labeled": [true, either([isInteger, isNull] as const)], "assets_processed": [true, isInteger], "assets_total": [true, either([isInteger, isNull] as const)], "error": [true, either([isString, isNull] as const)], "error_code": [true, either([isString, isNull] as const)], "job_id": [true, isString], "regions_discarded": [true, either([isInteger, isNull] as const)], "regions_out_of_bounds": [true, either([isInteger, isNull] as const)], "state": [true, checkBackgroundJobState], "stopped_early": [true, either([isBoolean, isNull] as const)] } as const); + /*#__PURE__*/ object({ "annotation_job_id": [true, isString], "annotations_replaced": [true, either([isInteger, isNull] as const)], "assets_labeled": [true, either([isInteger, isNull] as const)], "assets_processed": [true, isInteger], "assets_total": [true, either([isInteger, isNull] as const)], "error": [true, either([isString, isNull] as const)], "error_code": [true, either([isString, isNull] as const)], "job_id": [true, isString], "regions_discarded": [true, either([isInteger, isNull] as const)], "regions_out_of_bounds": [true, either([isInteger, isNull] as const)], "state": [true, checkBackgroundJobState], "stopped_early": [true, either([isBoolean, isNull] as const)] } as const); export const checkProgressCounts: Check = /*#__PURE__*/ object({ "accepted": [true, isInteger], "annotated": [true, isInteger], "pre_labeled": [true, isInteger], "review_pending": [true, isInteger], "skipped": [true, isInteger], "total": [true, isInteger], "unannotated": [true, isInteger] } as const); @@ -238,14 +238,20 @@ export const checkAnnotationJobState: Check = /*#__PURE__*/ oneOf(["pending", "in_progress", "completed"] as const); export const checkJobAction: Check = - /*#__PURE__*/ openOneOf(["start", "complete"] as const); + /*#__PURE__*/ openOneOf(["start", "pre_label", "complete"] as const); export const checkJobOut: Check = - /*#__PURE__*/ object({ "allowed_actions": [true, arrayOf(checkJobAction)], "asset_count": [true, isInteger], "assignee": [true, either([isString, isNull] as const)], "batch_id": [true, isString], "id": [true, isString], "state": [true, checkAnnotationJobState] } as const); + /*#__PURE__*/ object({ "allowed_actions": [true, arrayOf(checkJobAction)], "asset_count": [true, isInteger], "assignee": [true, either([isString, isNull] as const)], "batch_id": [true, isString], "id": [true, isString], "pre_label_run": [true, either([checkPreLabelRunOut, isNull] as const)], "state": [true, checkAnnotationJobState] } as const); export const checkJobPage: Check = /*#__PURE__*/ object({ "items": [true, arrayOf(checkJobOut)], "total": [true, isInteger] } as const); +export const checkPreLabelFanOutItemOut: Check = + /*#__PURE__*/ object({ "annotation_job_id": [true, isString], "batch_id": [true, isString], "batch_name": [true, isString], "job": [true, checkBackgroundJobOut], "joined": [true, isBoolean] } as const); + +export const checkPreLabelFanOutOut: Check = + /*#__PURE__*/ object({ "items": [true, arrayOf(checkPreLabelFanOutItemOut)], "total": [true, isInteger] } as const); + export const checkPreLabelExclusionReason: Check = /*#__PURE__*/ openOneOf(["no_producible_geometry", "required_attribute"] as const); @@ -261,12 +267,6 @@ export const checkProjectOut: Check = export const checkProjectPage: Check = /*#__PURE__*/ object({ "items": [true, arrayOf(checkProjectOut)], "total": [true, isInteger] } as const); -export const checkProjectPreLabelItemOut: Check = - /*#__PURE__*/ object({ "batch_id": [true, isString], "batch_name": [true, isString], "job": [true, checkBackgroundJobOut], "joined": [true, isBoolean] } as const); - -export const checkProjectPreLabelOut: Check = - /*#__PURE__*/ object({ "items": [true, arrayOf(checkProjectPreLabelItemOut)], "total": [true, isInteger] } as const); - export const checkProjectStatsOut: Check = /*#__PURE__*/ object({ "annotated_asset_count": [true, isInteger], "annotated_pct": [true, isNumber], "annotation_count": [true, isInteger], "asset_count": [true, isInteger], "class_count": [true, isInteger], "classes": [true, arrayOf(checkClassCountOut)], "last_ingest_at": [false, either([isString, isNull] as const)], "project_id": [true, isString] } as const); @@ -443,9 +443,10 @@ export const checkListReleases = checkReleasePage; export const checkListSchemaVersions = checkSchemaVersionPage; export const checkListSources = checkSourcePage; export const checkNextPendingAssets = checkAssetPage; -export const checkPreLabelBatch = checkBackgroundJobOut; +export const checkPreLabelBatch = checkPreLabelFanOutOut; +export const checkPreLabelJob = checkBackgroundJobOut; export const checkPreLabelPlan = checkPreLabelPlanOut; -export const checkPreLabelProjectBatches = checkProjectPreLabelOut; +export const checkPreLabelProjectBatches = checkPreLabelFanOutOut; export const checkPreviewSchemaChange = checkSchemaChangePreviewOut; export const checkPromoteBatch = checkAssetPage; export const checkPublishRelease = checkReleaseOut; diff --git a/openapi.json b/openapi.json index 97aec23b..9ecc0bc5 100644 --- a/openapi.json +++ b/openapi.json @@ -3259,9 +3259,10 @@ "type": "object" }, "JobAction": { - "description": "What can be asked of an annotation job.", + "description": "What can be asked of an annotation job. Declaration order is display order.", "enum": [ "start", + "pre_label", "complete" ], "title": "JobAction", @@ -3291,7 +3292,7 @@ "type": "object" }, "JobOut": { - "description": "One annotator's unit of work over a segment of a batch.", + "description": "One annotator's unit of work over a segment of a batch, and its most recent\npre-labeling run, `null` where none ever ran.", "properties": { "allowed_actions": { "items": { @@ -3325,6 +3326,16 @@ "title": "Id", "type": "string" }, + "pre_label_run": { + "anyOf": [ + { + "$ref": "#/components/schemas/PreLabelRunOut" + }, + { + "type": "null" + } + ] + }, "state": { "$ref": "#/components/schemas/AnnotationJobState" } @@ -3335,7 +3346,8 @@ "state", "assignee", "asset_count", - "allowed_actions" + "allowed_actions", + "pre_label_run" ], "title": "JobOut", "type": "object" @@ -3596,6 +3608,63 @@ "type": "string", "x-visionset-open": true }, + "PreLabelFanOutItemOut": { + "description": "One job's row in a launch that fanned out over several.", + "properties": { + "annotation_job_id": { + "format": "uuid", + "title": "Annotation Job Id", + "type": "string" + }, + "batch_id": { + "format": "uuid", + "title": "Batch Id", + "type": "string" + }, + "batch_name": { + "title": "Batch Name", + "type": "string" + }, + "job": { + "$ref": "#/components/schemas/BackgroundJobOut" + }, + "joined": { + "title": "Joined", + "type": "boolean" + } + }, + "required": [ + "batch_id", + "batch_name", + "annotation_job_id", + "job", + "joined" + ], + "title": "PreLabelFanOutItemOut", + "type": "object" + }, + "PreLabelFanOutOut": { + "description": "Every job the launch fanned out over, one row each, in batch then segment order.", + "properties": { + "items": { + "items": { + "$ref": "#/components/schemas/PreLabelFanOutItemOut" + }, + "title": "Items", + "type": "array" + }, + "total": { + "title": "Total", + "type": "integer" + } + }, + "required": [ + "items", + "total" + ], + "title": "PreLabelFanOutOut", + "type": "object" + }, "PreLabelPlanOut": { "description": "The words a run would ask a model for over this batch, and the shapes it would write.\n\nA run's prompt is the batch's pinned schema, narrowed to the classes the\nmodel's declared shapes can be written as. That narrowing is invisible in\nthe run's result \u2014 a schema whose `vehicle` class requires a `color`\nattribute yields no vehicles and no explanation \u2014 so it is published here,\nbefore a run starts, with the left-out classes named beside the asked-for\nones.\n\nEvery class the pinned schema declares appears in exactly one of the two\nlists, both in the schema's own declaration order. A batch whose schema has\nno askable class at all is refused rather than answered with an empty\n`asked_classes`: pre-labeling it is impossible, not merely unproductive.", "properties": { @@ -3678,8 +3747,13 @@ "type": "object" }, "PreLabelRunOut": { - "description": "A batch's most recent pre-labeling run: which job, how far, and what it found.\n\nPresent whenever pre-labeling has ever been asked for on this batch, and\ndescribing the most recent run \u2014 including one this session did not launch.\nA dialog reopened after a reload, in a second tab, or after a run started\nfrom the terminal reads the same state from here rather than from a job id\na component happened to keep.\n\n**Assets, where a download counts bytes and a check counts files.** The\nhandler owns a loop over the batch's untouched assets and knows the whole\nset before the first forward pass, so both its progress and its total are\ncounted in the unit its own work is over.\n\n**The outcome, once the job has one.** `stopped_early`, `assets_labeled`,\n`regions_discarded`, `regions_out_of_bounds` and `annotations_replaced` are\nthe handler's own account of what a settled run did.\nThey are `null` while the job is still `queued` or `running`, and `null`\nwhere it ended `failed` before producing one \u2014 but a `cancelled` run still\ncarries them: stopping partway is a coherent outcome for a handler whose\ncontract is to write only where nothing has been written.", + "description": "The most recent pre-labeling run: which job, how far, and what it found.\n\n`annotation_job_id` is the job the run is over; `job_id` is the queue row to\npoll.\n\nPresent whenever pre-labeling has ever been asked for, and describing the\nmost recent run \u2014 including one this session did not launch. A dialog\nreopened after a reload, in a second tab, or after a run started from the\nterminal reads the same state from here rather than from a job id a\ncomponent happened to keep.\n\n**Assets, where a download counts bytes and a check counts files.** The\nhandler owns a loop over the job's untouched assets and knows the whole\nset before the first forward pass, so both its progress and its total are\ncounted in the unit its own work is over.\n\n**The outcome, once the job has one.** `stopped_early`, `assets_labeled`,\n`regions_discarded`, `regions_out_of_bounds` and `annotations_replaced` are\nthe handler's own account of what a settled run did.\nThey are `null` while the job is still `queued` or `running`, and `null`\nwhere it ended `failed` before producing one \u2014 but a `cancelled` run still\ncarries them: stopping partway is a coherent outcome for a handler whose\ncontract is to write only where nothing has been written.", "properties": { + "annotation_job_id": { + "format": "uuid", + "title": "Annotation Job Id", + "type": "string" + }, "annotations_replaced": { "anyOf": [ { @@ -3782,6 +3856,7 @@ } }, "required": [ + "annotation_job_id", "job_id", "state", "assets_processed", @@ -3968,57 +4043,6 @@ "title": "ProjectPage", "type": "object" }, - "ProjectPreLabelItemOut": { - "description": "One batch's row in a project-wide launch.", - "properties": { - "batch_id": { - "format": "uuid", - "title": "Batch Id", - "type": "string" - }, - "batch_name": { - "title": "Batch Name", - "type": "string" - }, - "job": { - "$ref": "#/components/schemas/BackgroundJobOut" - }, - "joined": { - "title": "Joined", - "type": "boolean" - } - }, - "required": [ - "batch_id", - "batch_name", - "job", - "joined" - ], - "title": "ProjectPreLabelItemOut", - "type": "object" - }, - "ProjectPreLabelOut": { - "description": "Every batch the launch fanned out over, one row each, in selection order.", - "properties": { - "items": { - "items": { - "$ref": "#/components/schemas/ProjectPreLabelItemOut" - }, - "title": "Items", - "type": "array" - }, - "total": { - "title": "Total", - "type": "integer" - } - }, - "required": [ - "items", - "total" - ], - "title": "ProjectPreLabelOut", - "type": "object" - }, "ProjectPreLabelRequest": { "additionalProperties": false, "description": "Which model should pre-label this project's open batches, and which batches.\n\n`batch_ids` absent means every batch of the project that is open for\nannotation; present means exactly those \u2014 a batch outside the project is\n404, one not open is 409, an empty list names nothing and is 409 too, and\nthe request is refused whole, never partly launched.", @@ -6149,7 +6173,7 @@ ] }, "get": { - "description": "The batch's assets, with where each has got to and its labels in two numbers.\n\nMembership order by default, so reading twice gives the same sequence and an\ningest into an existing batch appends rather than reshuffles; `sort=confidence`\nputs the frame whose weakest model label scores lowest first, unscored frames\nlast, ties in membership order. `progress` narrows to the states named, and\n`total` is the size of what matched \u2014 the whole batch when nothing narrows it.\nAn offset past the end is an empty list and a 200, never a 404. The 404 belongs\nto the batch itself, which is resolved first: an unknown one is `BATCH_NOT_FOUND`.\n\n`job_id` and `progress` are null while the batch is a draft, because a draft\nhas no jobs \u2014 so a `progress` filter over a draft matches nothing. Bytes are\nnot here: an asset is named by its hashes, and\n`GET /projects/{project_id}/assets/{asset_id}/content` is what serves them.", + "description": "The batch's assets, with where each has got to and its labels in two numbers.\n\nMembership order by default, so reading twice gives the same sequence and an\ningest into an existing batch appends rather than reshuffles; `sort=confidence`\nputs the frame whose weakest model label scores lowest first, unscored frames\nlast, ties in membership order. `progress` narrows to the states named, and\n`total` is the size of what matched \u2014 the whole batch when nothing narrows it.\nAn offset past the end is an empty list and a 200, never a 404. The 404 belongs\nto the batch itself, which is resolved first: an unknown one is `BATCH_NOT_FOUND`.\n\n`job` narrows to the assets one job carries, composing with `progress`; a job\nthis batch does not have is 404 `JOB_NOT_FOUND`. `job_id` and `progress` on\neach item are null while the batch is a draft, which has no jobs \u2014 so there,\na `progress` filter or a `job` filter matches nothing rather than refusing.\nBytes are not here: an asset is named by its hashes, and\n`GET /projects/{project_id}/assets/{asset_id}/content` is what serves them.", "operationId": "list_batch_assets", "parameters": [ { @@ -6225,6 +6249,25 @@ "default": "membership", "description": "`membership` is stored order; `confidence` is lowest model confidence first, unscored last, ties in membership order." } + }, + { + "description": "Keep only the assets this job carries. Omit for the whole batch.", + "in": "query", + "name": "job", + "required": false, + "schema": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Keep only the assets this job carries. Omit for the whole batch.", + "title": "Job" + } } ], "responses": { @@ -6834,7 +6877,7 @@ ] }, "post": { - "description": "Ask a model to label every untouched asset in this batch, and answer at once.\n\nThe `pre_label` action. Labels land at `pre_labeled`, never at `annotated`:\nnobody judged them, so they arrive editable and correctable rather than\nclaiming to be somebody's work \u2014 and, being unjudged, they never reach the\nDataset until a person has taken them over.\n\n**Only assets nothing has touched \u2014 which is stronger than reading\n`unannotated`.** An asset already `pre_labeled`, annotated, skipped,\nawaiting review or accepted is passed over, and so is an `unannotated` one\nthat still carries annotations from an earlier round that was skipped and\nthen restored: that sequence deletes no labels, so progress alone does not\nprove an asset untouched. A run never writes over what a person did in this\nbatch, and never writes twice over what a model did \u2014 a plain second run\nextends an earlier one onto whatever is still untouched.\n`replace_model_labels` widens it to every frame still `pre_labeled` and\nsupersedes those labels with this run's answer, one frame per transaction;\na frame anyone edited, confirmed or skipped in this batch is never touched,\nand a frame the model now finds nothing on returns to `unannotated`. A\nreplacing request arriving while a run is in flight joins that run,\nwhichever flag it carries.\n\n**The batch's pinned schema is the prompt, narrowed to what this run\nwrites.** The model is asked for each class the schema declares that admits\none of the shapes the run writes and demands no attribute a prediction\ncannot supply; an answer naming one of those classes, matched\ncase-insensitively, is written under the schema's own spelling, and an\nanswer naming none of them is discarded. A schema with no such class has\nnowhere for a prediction to land and is refused \u2014 so the same schema is\naskable of a model that answers polygons and refused for one that answers\nboxes. `GET` this path with the same `connection_id` (and the same\n`geometries`) to read the narrowing before launching.\n\n**What the run writes is every shape the model produces, unless\n`geometries` says which.** A model declaring both a box and a polygon\nwrites both for every region it answers with \u2014 the kernel writes one\nannotation per emitted region and pairs nothing \u2014 and `geometries` filters\nthat to the shapes named: a region in any other shape is discarded and\ncounted in `regions_discarded`. The selection is per run, not per class,\nand it is kept on the queued row, so a run claimed later executes what was\nasked.\n\n**202, not 200.** A batch is hundreds of forward passes, so this follows the\nlaunch-and-poll contract the export and weight-download routes use: poll `GET\n/background-jobs/{id}` \u2014 the `Location` header names it \u2014 until `state` is\n`succeeded`, then re-read the batch's assets. Progress on the row is counted\nin assets.\n\n**Everything a caller can be told now is told now**, and no refusal creates a\njob \u2014 so a caller holding a job id holds one that will run. These refusals\nare about the request, and the caller can act on each. They are checked in\nthis order, and it is the order `pre_label` itself checks in, so a request\nwrong about the connection and the batch both always names the connection:\na connection not set up yet is 409 `INFERENCE_CONNECTION_NOT_SET_UP` \u2014 its\nweights not here, or its endpoint not yet asked what it answers; a\nconnection whose model answers places rather than words is 422\n`UNSUPPORTED_PROMPT`; a `geometries` naming a shape the model does not\nproduce is 422 `GEOMETRY_NOT_PRODUCED`; a batch that is not `in_annotation`\nis 409 `BATCH_NOT_IN_ANNOTATION`; a pinned schema with no class the\nselected shapes can be written as is 409 `SCHEMA_HAS_NO_DETECTABLE_CLASS`.\n\nTwo failures are about this installation rather than about the request, and\nanswer 500 carrying the message that says which: a machine without the\noptional local runtime is `LOCAL_INFERENCE_UNAVAILABLE` and carries the\nexact command that installs it, and a workspace whose records no longer\nhold together \u2014 a batch pinned to a schema version that is not stored \u2014 is\n`WORKSPACE_CORRUPT`. Neither is worth resending unchanged: there is no\nstate here a caller can change, so the remedy is the one the message names.\n\n**Asking twice joins the run already in flight rather than starting a second\none.** A request arriving while this batch has a pre-labeling run queued or\nrunning is answered with that run's id, so a double-click and a second tab\nwatch one run instead of paying for the same inference twice.", + "description": "Ask a model to label every untouched asset in this batch, and answer at once.\n\n**One row per open job, and the job is the unit.** This launch fans out over\nthe batch's jobs that are still open and queues for each the same\n`annotation.pre_label` row `POST /jobs/{job_id}/pre-label` queues, or joins\nthe one already queued or running for that job (`joined`). A finished job is\npassed over, so a batch whose every job is complete answers an empty page.\nEach row is polled, cancelled and remembered per job:\n`GET /background-jobs/{id}` for progress counted in that job's assets,\n`JobOut.pre_label_run` afterwards. Nothing here reports one total across\njobs, because nothing here is one run.\n\nThe `pre_label` action. Labels land at `pre_labeled`, never at `annotated`:\nnobody judged them, so they arrive editable and correctable rather than\nclaiming to be somebody's work \u2014 and, being unjudged, they never reach the\nDataset until a person has taken them over.\n\n**Only assets nothing has touched \u2014 which is stronger than reading\n`unannotated`.** An asset already `pre_labeled`, annotated, skipped,\nawaiting review or accepted is passed over, and so is an `unannotated` one\nthat still carries annotations from an earlier round that was skipped and\nthen restored: that sequence deletes no labels, so progress alone does not\nprove an asset untouched. A run never writes over what a person did in this\njob, and never writes twice over what a model did \u2014 a plain second run\nextends an earlier one onto whatever is still untouched.\n`replace_model_labels` widens it to every frame still `pre_labeled` and\nsupersedes those labels with this run's answer, one frame per transaction;\na frame anyone edited, confirmed or skipped in this job is never touched,\nand a frame the model now finds nothing on returns to `unannotated`. A\nreplacing request arriving while a run is in flight joins that run,\nwhichever flag it carries.\n\n**The batch's pinned schema is the prompt, narrowed to what this run\nwrites.** The model is asked for each class the schema declares that admits\none of the shapes the run writes and demands no attribute a prediction\ncannot supply; an answer naming one of those classes, matched\ncase-insensitively, is written under the schema's own spelling, and an\nanswer naming none of them is discarded. A schema with no such class has\nnowhere for a prediction to land and is refused \u2014 so the same schema is\naskable of a model that answers polygons and refused for one that answers\nboxes. `GET` this path with the same `connection_id` (and the same\n`geometries`) to read the narrowing before launching.\n\n**What the run writes is every shape the model produces, unless\n`geometries` says which.** A model declaring both a box and a polygon\nwrites both for every region it answers with \u2014 the kernel writes one\nannotation per emitted region and pairs nothing \u2014 and `geometries` filters\nthat to the shapes named: a region in any other shape is discarded and\ncounted in `regions_discarded`. The selection is per run, not per class,\nand it is kept on the queued row, so a run claimed later executes what was\nasked.\n\n**202, not 200.** A job is hundreds of forward passes, so this follows the\nlaunch-and-poll contract the export and weight-download routes use: poll `GET\n/background-jobs/{id}` for each row until `state` is `succeeded`, then\nre-read the batch's assets. Progress on a row is counted in assets. There is\nno `Location` header, because there is no single row for it to name.\n\n**Everything a caller can be told now is told now**, and no refusal creates a\nrow \u2014 so a caller holding a row's id holds one that will run. These refusals\nare about the request, and the caller can act on each. They are checked in\nthis order, and it is the order `pre_label` itself checks in, so a request\nwrong about the connection and the batch both always names the connection:\na connection not set up yet is 409 `INFERENCE_CONNECTION_NOT_SET_UP` \u2014 its\nweights not here, or its endpoint not yet asked what it answers; a\nconnection whose model answers places rather than words is 422\n`UNSUPPORTED_PROMPT`; a `geometries` naming a shape the model does not\nproduce is 422 `GEOMETRY_NOT_PRODUCED`; a batch that is not `in_annotation`\nis 409 `BATCH_NOT_IN_ANNOTATION`; a pinned schema with no class the\nselected shapes can be written as is 409 `SCHEMA_HAS_NO_DETECTABLE_CLASS`.\n\nTwo failures are about this installation rather than about the request, and\nanswer 500 carrying the message that says which: a machine without the\noptional local runtime is `LOCAL_INFERENCE_UNAVAILABLE` and carries the\nexact command that installs it, and a workspace whose records no longer\nhold together \u2014 a batch pinned to a schema version that is not stored \u2014 is\n`WORKSPACE_CORRUPT`. Neither is worth resending unchanged: there is no\nstate here a caller can change, so the remedy is the one the message names.\n\n**Asking twice joins the runs already in flight rather than starting second\nones.** A request arriving while a job here has a pre-labeling run queued or\nrunning is answered with that run's row and `joined` true, so a double-click\nand a second tab watch one run per job instead of paying for the same\ninference twice.", "operationId": "pre_label_batch", "parameters": [ { @@ -6863,7 +6906,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/BackgroundJobOut" + "$ref": "#/components/schemas/PreLabelFanOutOut" } } }, @@ -10330,6 +10373,115 @@ ] } }, + "/jobs/{job_id}/pre-label": { + "post": { + "description": "Ask a model to label every untouched asset in this job, and answer at once.\n\nThe `pre_label` action. Labels land at `pre_labeled`, never at `annotated`:\nnobody judged them, so they arrive editable and correctable rather than\nclaiming to be somebody's work \u2014 and, being unjudged, they never reach the\nDataset until a person has taken them over.\n\n**Only assets nothing has touched \u2014 which is stronger than reading\n`unannotated`.** An asset already `pre_labeled`, annotated, skipped,\nawaiting review or accepted is passed over, and so is an `unannotated` one\nthat still carries annotations from an earlier round that was skipped and\nthen restored: that sequence deletes no labels, so progress alone does not\nprove an asset untouched. A run never writes over what a person did in this\njob, and never writes twice over what a model did \u2014 a plain second run\nextends an earlier one onto whatever is still untouched.\n`replace_model_labels` widens it to every frame still `pre_labeled` and\nsupersedes those labels with this run's answer, one frame per transaction;\na frame anyone edited, confirmed or skipped in this job is never touched,\nand a frame the model now finds nothing on returns to `unannotated`. A\nreplacing request arriving while a run is in flight joins that run,\nwhichever flag it carries.\n\n**The batch's pinned schema is the prompt, narrowed to what this run\nwrites.** The model is asked for each class the schema declares that admits\none of the shapes the run writes and demands no attribute a prediction\ncannot supply; an answer naming one of those classes, matched\ncase-insensitively, is written under the schema's own spelling, and an\nanswer naming none of them is discarded. A schema with no such class has\nnowhere for a prediction to land and is refused \u2014 so the same schema is\naskable of a model that answers polygons and refused for one that answers\nboxes. `GET /batches/{batch_id}/pre-label` with the same `connection_id`\n(and the same `geometries`) reads the narrowing before launching.\n\n**What the run writes is every shape the model produces, unless\n`geometries` says which.** A model declaring both a box and a polygon\nwrites both for every region it answers with \u2014 the kernel writes one\nannotation per emitted region and pairs nothing \u2014 and `geometries` filters\nthat to the shapes named: a region in any other shape is discarded and\ncounted in `regions_discarded`. The selection is per run, not per class,\nand it is kept on the queued row, so a run claimed later executes what was\nasked.\n\n**202, not 200.** A job is hundreds of forward passes, so this follows the\nlaunch-and-poll contract the export and weight-download routes use: poll `GET\n/background-jobs/{id}` \u2014 the `Location` header names it \u2014 until `state` is\n`succeeded`, then re-read the job's assets. Progress on the row is counted\nin assets, and `JobOut.pre_label_run` remembers the same row afterwards.\n\n**Everything a caller can be told now is told now**, and no refusal creates a\njob \u2014 so a caller holding a job id holds one that will run. These refusals\nare about the request, and the caller can act on each. They are checked in\nthis order, and it is the order `pre_label` itself checks in, so a request\nwrong about the connection and the job both always names the connection:\nan unknown connection is 404 `INFERENCE_CONNECTION_NOT_FOUND`; a\nconnection not set up yet is 409 `INFERENCE_CONNECTION_NOT_SET_UP` \u2014 its\nweights not here, or its endpoint not yet asked what it answers; a\nconnection whose model answers places rather than words is 422\n`UNSUPPORTED_PROMPT`; a `geometries` naming a shape the model does not\nproduce is 422 `GEOMETRY_NOT_PRODUCED`. An unknown job is 404\n`JOB_NOT_FOUND`; a job whose batch is not `in_annotation` is 409\n`BATCH_NOT_IN_ANNOTATION`; a job already `completed` is 409 `JOB_FINISHED`,\nand there is no remedy on this route \u2014 settled work is corrected through a\nnew batch rather than reopened. A pinned schema with no class the selected\nshapes can be written as is 409 `SCHEMA_HAS_NO_DETECTABLE_CLASS`.\n\nTwo failures are about this installation rather than about the request, and\nanswer 500 carrying the message that says which: a machine without the\noptional local runtime is `LOCAL_INFERENCE_UNAVAILABLE` and carries the\nexact command that installs it, and a workspace whose records no longer\nhold together \u2014 a batch pinned to a schema version that is not stored \u2014 is\n`WORKSPACE_CORRUPT`. Neither is worth resending unchanged: there is no\nstate here a caller can change, so the remedy is the one the message names.\n\n**Asking twice joins the run already in flight rather than starting a second\none.** A request arriving while this job has a pre-labeling run queued or\nrunning is answered with that run's id, so a double-click and a second tab\nwatch one run instead of paying for the same inference twice \u2014 and so does\n`POST /batches/{batch_id}/pre-label`, whose fan-out reaches this same job.", + "operationId": "pre_label_job", + "parameters": [ + { + "in": "path", + "name": "job_id", + "required": true, + "schema": { + "format": "uuid", + "title": "Job Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PreLabelRequest" + } + } + }, + "required": true + }, + "responses": { + "202": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BackgroundJobOut" + } + } + }, + "description": "Successful Response" + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + }, + "description": "Missing or invalid bearer token" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + }, + "description": "No such resource" + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + }, + "description": "The resource's state refuses this request" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + }, + "description": "The request payload is not processable" + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + }, + "description": "Unhandled server error, with an incident id" + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + }, + "description": "The workspace is busy; retry after the header says" + } + }, + "security": [ + { + "HTTPBearer": [] + } + ], + "summary": "Pre Label Job", + "tags": [ + "jobs" + ] + } + }, "/jobs/{job_id}/progress": { "get": { "description": "How many of this job's assets sit in each state.\n\nEvery state is a field, including the ones nobody is in, so a client charting\nprogress never has to guard a lookup.", @@ -11675,7 +11827,7 @@ }, "/projects/{project_id}/batches/pre-label": { "post": { - "description": "Ask a model to label every untouched asset across this project's open batches.\n\n**One row per batch, and the batch stays the unit.** This launch fans out\nover the project's batches that are open for annotation \u2014 every one of\nthem, or exactly the `batch_ids` named \u2014 and for each one queues the same\n`annotation.pre_label` job `POST /batches/{batch_id}/pre-label` queues, or\njoins the one already queued or running for that batch (`joined`). Each\nrow is polled, cancelled and remembered per batch, exactly as a\nsingle-batch launch is: `GET /background-jobs/{id}` for progress counted\nin that batch's assets, `BatchOut.pre_label_run` afterwards. Nothing here\nreports one total across batches, because nothing here is one run.\n\n**Refused whole, up front, and no refusal creates a row.** The connection\nis checked first, as the single-batch launch checks it: an unknown\nconnection is 404 `INFERENCE_CONNECTION_NOT_FOUND`, one not set up yet is\n409 `INFERENCE_CONNECTION_NOT_SET_UP`, a model that answers places rather\nthan words is 422 `UNSUPPORTED_PROMPT`, and a `geometries` naming a shape\nthe model does not produce is 422 `GEOMETRY_NOT_PRODUCED`. Then the\nselection: an unknown project is 404 `PROJECT_NOT_FOUND`; a named batch\noutside this project is 404 `BATCH_NOT_FOUND`; a named batch not\n`in_annotation`, a project with no open batch at all, or an empty\n`batch_ids`, is 409 `BATCH_NOT_IN_ANNOTATION`; any selected batch whose\npinned schema has no class the selected shapes can be written as is 409\n`SCHEMA_HAS_NO_DETECTABLE_CLASS`, and the message names the batch so the\ncaller can leave it out by name and ask again. A partly launched project\nwould leave rows the caller was never told about, which is why the whole\nrequest is refused instead.\n\nWhat each run writes, passes over and counts is the single-batch launch's\ncontract, `geometries` included; read `POST /batches/{batch_id}/pre-label`.", + "description": "Ask a model to label every untouched asset across this project's open batches.\n\n**One row per open job of each selected batch, and the job is the unit.**\nThis launch fans out over the project's batches that are open for\nannotation \u2014 every one of them, or exactly the `batch_ids` named \u2014 and\nwithin each over the jobs still open, queueing for each the same\n`annotation.pre_label` row `POST /jobs/{job_id}/pre-label` queues, or\njoining the one already queued or running for that job (`joined`). A\nfinished job is passed over, so a selected batch whose every job is\ncomplete contributes no row. Each row is polled, cancelled and remembered\nper job, exactly as a single-job launch is: `GET /background-jobs/{id}`\nfor progress counted in that job's assets, `JobOut.pre_label_run`\nafterwards. Nothing here reports one total across jobs, because nothing\nhere is one run.\n\n**Refused whole, up front, and no refusal creates a row.** The connection\nis checked first, as the single-batch launch checks it: an unknown\nconnection is 404 `INFERENCE_CONNECTION_NOT_FOUND`, one not set up yet is\n409 `INFERENCE_CONNECTION_NOT_SET_UP`, a model that answers places rather\nthan words is 422 `UNSUPPORTED_PROMPT`, and a `geometries` naming a shape\nthe model does not produce is 422 `GEOMETRY_NOT_PRODUCED`. Then the\nselection: an unknown project is 404 `PROJECT_NOT_FOUND`; a named batch\noutside this project is 404 `BATCH_NOT_FOUND`; a named batch not\n`in_annotation`, a project with no open batch at all, or an empty\n`batch_ids`, is 409 `BATCH_NOT_IN_ANNOTATION`; any selected batch whose\npinned schema has no class the selected shapes can be written as is 409\n`SCHEMA_HAS_NO_DETECTABLE_CLASS`, and the message names the batch so the\ncaller can leave it out by name and ask again. A partly launched project\nwould leave rows the caller was never told about, which is why the whole\nrequest is refused instead.\n\nWhat each run writes, passes over and counts is the single-batch launch's\ncontract, `geometries` included; read `POST /batches/{batch_id}/pre-label`.", "operationId": "pre_label_project_batches", "parameters": [ { @@ -11704,7 +11856,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ProjectPreLabelOut" + "$ref": "#/components/schemas/PreLabelFanOutOut" } } }, diff --git a/src/visionset/server/models.py b/src/visionset/server/models.py index 33ac6b4e..52679c4e 100644 --- a/src/visionset/server/models.py +++ b/src/visionset/server/models.py @@ -204,6 +204,10 @@ class Page[T](BaseModel): list[AssetProgress] | None, Query(description="Keep only assets in these states. Repeat the parameter per state."), ] +JobQuery = Annotated[ + UUID | None, + Query(description="Keep only the assets this job carries. Omit for the whole batch."), +] GeometriesQuery = Annotated[ list[GeometryType] | None, Query( @@ -1030,16 +1034,19 @@ def of(cls, counts: dict[AssetProgress, int]) -> Self: class PreLabelRunOut(BaseModel): - """A batch's most recent pre-labeling run: which job, how far, and what it found. + """The most recent pre-labeling run: which job, how far, and what it found. + + `annotation_job_id` is the job the run is over; `job_id` is the queue row to + poll. - Present whenever pre-labeling has ever been asked for on this batch, and - describing the most recent run — including one this session did not launch. - A dialog reopened after a reload, in a second tab, or after a run started - from the terminal reads the same state from here rather than from a job id - a component happened to keep. + Present whenever pre-labeling has ever been asked for, and describing the + most recent run — including one this session did not launch. A dialog + reopened after a reload, in a second tab, or after a run started from the + terminal reads the same state from here rather than from a job id a + component happened to keep. **Assets, where a download counts bytes and a check counts files.** The - handler owns a loop over the batch's untouched assets and knows the whole + handler owns a loop over the job's untouched assets and knows the whole set before the first forward pass, so both its progress and its total are counted in the unit its own work is over. @@ -1052,6 +1059,7 @@ class PreLabelRunOut(BaseModel): contract is to write only where nothing has been written. """ + annotation_job_id: UUID job_id: UUID state: BackgroundJobState #: Assets looked at so far. @@ -1088,6 +1096,7 @@ class PreLabelRunOut(BaseModel): @classmethod def of(cls, run: PreLabelRun) -> Self: return cls( + annotation_job_id=run.annotation_job_id, job_id=run.job_id, state=run.state, assets_processed=run.assets_processed, @@ -1428,21 +1437,23 @@ class ProjectPreLabelRequest(BaseModel): geometries: list[GeometryType] | None = Field(default=None, min_length=1) -class ProjectPreLabelItemOut(BaseModel): - """One batch's row in a project-wide launch.""" +class PreLabelFanOutItemOut(BaseModel): + """One job's row in a launch that fanned out over several.""" batch_id: UUID batch_name: str + #: The annotation job this row is over — the unit a run is over. + annotation_job_id: UUID #: The `annotation.pre_label` row to poll — `GET /background-jobs/{id}` — - #: the same row `BatchOut.pre_label_run` remembers. + #: the same row `JobOut.pre_label_run` remembers. job: BackgroundJobOut #: True when the row existed before this request: a run already queued or - #: running for that batch was joined rather than started again. + #: running for that job was joined rather than started again. joined: bool -class ProjectPreLabelOut(Page[ProjectPreLabelItemOut]): - """Every batch the launch fanned out over, one row each, in selection order.""" +class PreLabelFanOutOut(Page[PreLabelFanOutItemOut]): + """Every job the launch fanned out over, one row each, in batch then segment order.""" # --- jobs -------------------------------------------------------------------- @@ -1456,7 +1467,8 @@ class ProjectPreLabelOut(Page[ProjectPreLabelItemOut]): # too; a job of fifty thousand assets must not ship one on every read, and the # paged batch listing is where per-asset detail lives. class JobOut(BaseModel): - """One annotator's unit of work over a segment of a batch.""" + """One annotator's unit of work over a segment of a batch, and its most recent + pre-labeling run, `null` where none ever ran.""" id: UUID batch_id: UUID @@ -1464,13 +1476,14 @@ class JobOut(BaseModel): assignee: str | None asset_count: int allowed_actions: list[JobAction] + pre_label_run: PreLabelRunOut | None - # ``batch`` whole rather than an id: both actions need the batch open, which + # ``batch`` whole rather than an id: every action needs the batch open, which # is the dimension a client re-deriving these rules dropped. The per-asset map # stays unpublished and is still read here, because ``complete`` is refined by # whether every asset has settled — a refinement that costs no extra read. @classmethod - def of(cls, job: AnnotationJob, *, batch: Batch) -> Self: + def of(cls, job: AnnotationJob, *, batch: Batch, pre_label_run: PreLabelRun | None) -> Self: return cls( id=job.id, batch_id=batch.id, @@ -1480,6 +1493,7 @@ def of(cls, job: AnnotationJob, *, batch: Batch) -> Self: allowed_actions=job_actions( job.state, batch_state=batch.state, progress=job.progress.values() ), + pre_label_run=None if pre_label_run is None else PreLabelRunOut.of(pre_label_run), ) diff --git a/src/visionset/server/routes/_prelabel.py b/src/visionset/server/routes/_prelabel.py new file mode 100644 index 00000000..14a937dc --- /dev/null +++ b/src/visionset/server/routes/_prelabel.py @@ -0,0 +1,134 @@ +# usage: from visionset.server.routes import _prelabel +"""What the three pre-label launches share: the connection gate, the shape +selection and the enqueue-or-join. One spelling, so a job launched alone and a +job launched inside a batch or project fan-out get the same row. + +Underscored because it carries no router: ``routes/__init__`` lists the modules +``create_app`` includes, and this is not one of them. +""" + +from __future__ import annotations + +from uuid import UUID + +from visionset.inference import ( + STUB_MODEL_ID, + capabilities_of, + effective_produces, + not_set_up_message, + produces_of, + unsupported_prompt_message, + with_families, +) +from visionset.inference import require as require_local_inference +from visionset.jobs.prelabel import JOB_TYPE as pre_label_job_type +from visionset.jobs.prelabel import payload_for as pre_label_payload_for +from visionset.kernel.domain import ( + AnnotationJob, + BackgroundJob, + BackgroundJobSpec, + Batch, + ConnectionSetupState, + ConnectionType, + GeometryType, + InferenceConnection, + ModelCapability, +) +from visionset.kernel.errors import InferenceConnectionNotSetUp, UnsupportedPrompt +from visionset.kernel.services import InferenceConnectionService, JobService +from visionset.server.dependencies import WorkspaceDep + + +def text_detect_connection(workspace: WorkspaceDep, connection_id: UUID) -> InferenceConnection: + """The connection, once it is known to be set up and to answer words. + + The gate the plan and every launch share, so they cannot refuse + differently. It runs before anything about the batch or the job, matching + ``pre_label``'s own order: what a build can run is an answer about a setup + somebody is part-way through, independent of any batch's state, and a + caller most needs it first. + + The runtime is demanded here rather than inside a worker, on the download + route's terms: a refusal a request can make is a refusal the request makes, + and discovering a missing install mid-run would put an install command on a + failed row somebody has to go and find. Not for the stub, which needs + neither the runtime nor the network, and not for an ``http`` connection + either — the gate is about a model that would load here, and an endpoint + loads nothing here. + + ``setup_state`` is checked before the capability read: ``model_family`` is + written only by a completed weight download or a tested ``http`` endpoint, + so a connection that has finished neither reads no capabilities at all, and + ``UNSUPPORTED_PROMPT`` for that would claim the model answers places rather + than words when nothing has yet said what it answers. Capabilities are + derived from the family rather than stored on the row, the same way the + connection wire model asks for them. + + Raises: + InferenceConnectionNotFound: no such connection. + InferenceConnectionNotSetUp: its weights are not here, or its endpoint + has not been asked what it answers. + UnsupportedPrompt: its model answers places rather than words. + LocalInferenceUnavailable: a local connection on a machine without the + optional runtime. + """ + connection = InferenceConnectionService(workspace).get(connection_id) + (connection,) = with_families(workspace, [connection]) + if connection.connection_type is ConnectionType.LOCAL and connection.model_id != STUB_MODEL_ID: + require_local_inference() + if connection.setup_state is not ConnectionSetupState.READY or not connection.model_family: + raise InferenceConnectionNotSetUp(not_set_up_message(connection)) + if ModelCapability.TEXT_DETECT not in capabilities_of(connection.model_family): + raise UnsupportedPrompt(unsupported_prompt_message(connection.name)) + return connection + + +def selected_produces( + connection: InferenceConnection, geometries: list[GeometryType] | None +) -> frozenset[GeometryType]: + """The shapes a run writes — the model's, narrowed to a request's selection. + + Checked right after the connection and before the batch, on every pre-label + surface, so the plan, the job launch and both fan-outs refuse a bad + selection in one place in the order and with no row queued. + + Raises: + GeometryNotProduced: the selection names a shape the model does not produce. + """ + return effective_produces( + produces_of(connection.model_family), + None if geometries is None else frozenset(geometries), + ) + + +def launch( + workspace: WorkspaceDep, + job: AnnotationJob, + batch: Batch, + *, + connection_id: UUID, + minimum_confidence: float, + replace_model_labels: bool, + geometries: frozenset[GeometryType] | None, +) -> tuple[BackgroundJob, bool]: + """The run for this job: joined if one is live, queued otherwise. + + The second element says which — true where the row existed before this + call, so a fan-out can report per row whether it started anything. + """ + running = JobService(workspace).live_job(job.id, job_type=pre_label_job_type) + row = running or workspace.job_queue.enqueue( + BackgroundJobSpec( + type=pre_label_job_type, + payload=pre_label_payload_for( + job.id, + batch.id, + connection_id, + minimum_confidence, + replace_model_labels, + geometries, + ), + idempotent=True, + ) + ) + return row, running is not None diff --git a/src/visionset/server/routes/batches.py b/src/visionset/server/routes/batches.py index d02ae053..23a4fce9 100644 --- a/src/visionset/server/routes/batches.py +++ b/src/visionset/server/routes/batches.py @@ -32,38 +32,18 @@ from typing import Annotated from uuid import UUID -from fastapi import Query, Response, status +from fastapi import Query, status from visionset.inference import ( - STUB_MODEL_ID, - capabilities_of, - effective_produces, - not_set_up_message, - produces_of, + open_jobs_of, prompt_plan, require_detectable_schema, select_pre_labelable, - unsupported_prompt_message, - with_families, ) -from visionset.inference import require as require_local_inference -from visionset.jobs.prelabel import JOB_TYPE as pre_label_job_type -from visionset.jobs.prelabel import payload_for as pre_label_payload_for -from visionset.kernel.domain import ( - AssetSort, - BackgroundJobSpec, - ConnectionSetupState, - ConnectionType, - GeometryType, - InferenceConnection, - MembershipChange, - ModelCapability, -) -from visionset.kernel.errors import InferenceConnectionNotSetUp, UnsupportedPrompt +from visionset.kernel.domain import AssetSort, MembershipChange from visionset.kernel.services import ( BatchService, DatasetService, - InferenceConnectionService, JobService, ProjectService, ) @@ -86,16 +66,18 @@ GeometriesQuery, JobOut, JobPage, + JobQuery, LimitQuery, OffsetQuery, + PreLabelFanOutItemOut, + PreLabelFanOutOut, PreLabelPlanOut, PreLabelRequest, ProgressQuery, - ProjectPreLabelItemOut, - ProjectPreLabelOut, ProjectPreLabelRequest, SortQuery, ) +from visionset.server.routes._prelabel import launch, selected_produces, text_detect_connection project_router = protected_router(prefix="/projects/{project_id}/batches", tags=["batches"]) router = protected_router(prefix="/batches", tags=["batches"]) @@ -116,68 +98,6 @@ def _promoted(workspace: WorkspaceDep, project_id: UUID) -> frozenset[UUID]: return DatasetService(workspace).member_asset_ids(dataset.id) -def _text_detect_connection(workspace: WorkspaceDep, connection_id: UUID) -> InferenceConnection: - """The connection, once it is known to be set up and to answer words. - - The gate the plan and the launch share, so the two cannot refuse - differently. It runs before anything about the batch, matching - ``pre_label``'s own order: what a build can run is an answer about a setup - somebody is part-way through, independent of this batch's state, and a - caller most needs it first. - - The runtime is demanded here rather than inside a worker, on the download - route's terms: a refusal a request can make is a refusal the request makes, - and discovering a missing install mid-run would put an install command on a - failed row somebody has to go and find. Not for the stub, which needs - neither the runtime nor the network, and not for an ``http`` connection - either — the gate is about a model that would load here, and an endpoint - loads nothing here. - - ``setup_state`` is checked before the capability read: ``model_family`` is - written only by a completed weight download or a tested ``http`` endpoint, - so a connection that has finished neither reads no capabilities at all, and - ``UNSUPPORTED_PROMPT`` for that would claim the model answers places rather - than words when nothing has yet said what it answers. Capabilities are - derived from the family rather than stored on the row, the same way the - connection wire model asks for them. - - Raises: - InferenceConnectionNotFound: no such connection. - InferenceConnectionNotSetUp: its weights are not here, or its endpoint - has not been asked what it answers. - UnsupportedPrompt: its model answers places rather than words. - LocalInferenceUnavailable: a local connection on a machine without the - optional runtime. - """ - connection = InferenceConnectionService(workspace).get(connection_id) - (connection,) = with_families(workspace, [connection]) - if connection.connection_type is ConnectionType.LOCAL and connection.model_id != STUB_MODEL_ID: - require_local_inference() - if connection.setup_state is not ConnectionSetupState.READY or not connection.model_family: - raise InferenceConnectionNotSetUp(not_set_up_message(connection)) - if ModelCapability.TEXT_DETECT not in capabilities_of(connection.model_family): - raise UnsupportedPrompt(unsupported_prompt_message(connection.name)) - return connection - - -def _selected_produces( - connection: InferenceConnection, geometries: list[GeometryType] | None -) -> frozenset[GeometryType]: - """The shapes a run writes — the model's, narrowed to a request's selection. - - Checked right after the connection and before the batch, on every pre-label - surface, so the plan, the launch and the project launch refuse a bad - selection in one place in the order and with no row queued. - - Raises: - GeometryNotProduced: the selection names a shape the model does not produce. - """ - return effective_produces( - produces_of(connection.model_family), - None if geometries is None else frozenset(geometries), - ) - - @project_router.post("", status_code=201, responses=documented(404)) def create_batch(workspace: WorkspaceDep, project_id: UUID, body: BatchCreate) -> BatchOut: """Start a draft batch over a chosen set of the project's assets. @@ -415,8 +335,8 @@ def pre_label_plan( with the install command, and a batch open for annotation but pinning no schema version is a broken invariant and answers 500 `WORKSPACE_CORRUPT`. """ - connection = _text_detect_connection(workspace, connection_id) - produces = _selected_produces(connection, geometries) + connection = text_detect_connection(workspace, connection_id) + produces = selected_produces(connection, geometries) batch = BatchService(workspace).require_pre_labelable(batch_id) schema = require_detectable_schema(workspace, batch, produces) return PreLabelPlanOut.of(prompt_plan(schema, produces)) @@ -430,12 +350,21 @@ def pre_label_plan( def pre_label_batch( workspace: WorkspaceDep, runner: RunnerDep, - response: Response, batch_id: UUID, body: PreLabelRequest, -) -> BackgroundJobOut: +) -> PreLabelFanOutOut: """Ask a model to label every untouched asset in this batch, and answer at once. + **One row per open job, and the job is the unit.** This launch fans out over + the batch's jobs that are still open and queues for each the same + `annotation.pre_label` row `POST /jobs/{job_id}/pre-label` queues, or joins + the one already queued or running for that job (`joined`). A finished job is + passed over, so a batch whose every job is complete answers an empty page. + Each row is polled, cancelled and remembered per job: + `GET /background-jobs/{id}` for progress counted in that job's assets, + `JobOut.pre_label_run` afterwards. Nothing here reports one total across + jobs, because nothing here is one run. + The `pre_label` action. Labels land at `pre_labeled`, never at `annotated`: nobody judged them, so they arrive editable and correctable rather than claiming to be somebody's work — and, being unjudged, they never reach the @@ -447,11 +376,11 @@ def pre_label_batch( that still carries annotations from an earlier round that was skipped and then restored: that sequence deletes no labels, so progress alone does not prove an asset untouched. A run never writes over what a person did in this - batch, and never writes twice over what a model did — a plain second run + job, and never writes twice over what a model did — a plain second run extends an earlier one onto whatever is still untouched. `replace_model_labels` widens it to every frame still `pre_labeled` and supersedes those labels with this run's answer, one frame per transaction; - a frame anyone edited, confirmed or skipped in this batch is never touched, + a frame anyone edited, confirmed or skipped in this job is never touched, and a frame the model now finds nothing on returns to `unannotated`. A replacing request arriving while a run is in flight joins that run, whichever flag it carries. @@ -476,14 +405,14 @@ def pre_label_batch( and it is kept on the queued row, so a run claimed later executes what was asked. - **202, not 200.** A batch is hundreds of forward passes, so this follows the + **202, not 200.** A job is hundreds of forward passes, so this follows the launch-and-poll contract the export and weight-download routes use: poll `GET - /background-jobs/{id}` — the `Location` header names it — until `state` is - `succeeded`, then re-read the batch's assets. Progress on the row is counted - in assets. + /background-jobs/{id}` for each row until `state` is `succeeded`, then + re-read the batch's assets. Progress on a row is counted in assets. There is + no `Location` header, because there is no single row for it to name. **Everything a caller can be told now is told now**, and no refusal creates a - job — so a caller holding a job id holds one that will run. These refusals + row — so a caller holding a row's id holds one that will run. These refusals are about the request, and the caller can act on each. They are checked in this order, and it is the order `pre_label` itself checks in, so a request wrong about the connection and the batch both always names the connection: @@ -503,36 +432,42 @@ def pre_label_batch( `WORKSPACE_CORRUPT`. Neither is worth resending unchanged: there is no state here a caller can change, so the remedy is the one the message names. - **Asking twice joins the run already in flight rather than starting a second - one.** A request arriving while this batch has a pre-labeling run queued or - running is answered with that run's id, so a double-click and a second tab - watch one run instead of paying for the same inference twice. + **Asking twice joins the runs already in flight rather than starting second + ones.** A request arriving while a job here has a pre-labeling run queued or + running is answered with that run's row and `joined` true, so a double-click + and a second tab watch one run per job instead of paying for the same + inference twice. """ - service = BatchService(workspace) - connection = _text_detect_connection(workspace, body.connection_id) - produces = _selected_produces(connection, body.geometries) - batch = service.require_pre_labelable(batch_id) + connection = text_detect_connection(workspace, body.connection_id) + produces = selected_produces(connection, body.geometries) + batch = BatchService(workspace).require_pre_labelable(batch_id) require_detectable_schema(workspace, batch, produces) - running = service.live_job(batch_id, job_type=pre_label_job_type) - job = running or workspace.job_queue.enqueue( - BackgroundJobSpec( - type=pre_label_job_type, - payload=pre_label_payload_for( - batch_id, - body.connection_id, - body.minimum_confidence, - body.replace_model_labels, - None if body.geometries is None else frozenset(body.geometries), - ), - idempotent=True, + geometries = None if body.geometries is None else frozenset(body.geometries) + items: list[PreLabelFanOutItemOut] = [] + for job in open_jobs_of(workspace, batch_id): + row, joined = launch( + workspace, + job, + batch, + connection_id=body.connection_id, + minimum_confidence=body.minimum_confidence, + replace_model_labels=body.replace_model_labels, + geometries=geometries, ) - ) - # Woken even when the answer is a run that already existed: what came back - # may be `queued`, and the dispatcher it waits for sleeps on its own interval. + items.append( + PreLabelFanOutItemOut( + batch_id=batch.id, + batch_name=batch.name, + annotation_job_id=job.id, + job=BackgroundJobOut.of(row), + joined=joined, + ) + ) + # Woken even where every row already existed: what came back may be `queued`, + # and the dispatcher it waits for sleeps on its own interval. runner.wake() - response.headers["Location"] = f"/background-jobs/{job.id}" - return BackgroundJobOut.of(job) + return PreLabelFanOutOut(items=items, total=len(items)) @project_router.post( @@ -545,18 +480,21 @@ def pre_label_project_batches( runner: RunnerDep, project_id: UUID, body: ProjectPreLabelRequest, -) -> ProjectPreLabelOut: +) -> PreLabelFanOutOut: """Ask a model to label every untouched asset across this project's open batches. - **One row per batch, and the batch stays the unit.** This launch fans out - over the project's batches that are open for annotation — every one of - them, or exactly the `batch_ids` named — and for each one queues the same - `annotation.pre_label` job `POST /batches/{batch_id}/pre-label` queues, or - joins the one already queued or running for that batch (`joined`). Each - row is polled, cancelled and remembered per batch, exactly as a - single-batch launch is: `GET /background-jobs/{id}` for progress counted - in that batch's assets, `BatchOut.pre_label_run` afterwards. Nothing here - reports one total across batches, because nothing here is one run. + **One row per open job of each selected batch, and the job is the unit.** + This launch fans out over the project's batches that are open for + annotation — every one of them, or exactly the `batch_ids` named — and + within each over the jobs still open, queueing for each the same + `annotation.pre_label` row `POST /jobs/{job_id}/pre-label` queues, or + joining the one already queued or running for that job (`joined`). A + finished job is passed over, so a selected batch whose every job is + complete contributes no row. Each row is polled, cancelled and remembered + per job, exactly as a single-job launch is: `GET /background-jobs/{id}` + for progress counted in that job's assets, `JobOut.pre_label_run` + afterwards. Nothing here reports one total across jobs, because nothing + here is one run. **Refused whole, up front, and no refusal creates a row.** The connection is checked first, as the single-batch launch checks it: an unknown @@ -577,33 +515,33 @@ def pre_label_project_batches( What each run writes, passes over and counts is the single-batch launch's contract, `geometries` included; read `POST /batches/{batch_id}/pre-label`. """ - connection = _text_detect_connection(workspace, body.connection_id) - produces = _selected_produces(connection, body.geometries) + connection = text_detect_connection(workspace, body.connection_id) + produces = selected_produces(connection, body.geometries) selected = select_pre_labelable(workspace, project_id, produces, body.batch_ids) - service = BatchService(workspace) geometries = None if body.geometries is None else frozenset(body.geometries) - items: list[ProjectPreLabelItemOut] = [] + items: list[PreLabelFanOutItemOut] = [] for batch in selected: - running = service.live_job(batch.id, job_type=pre_label_job_type) - job = running or workspace.job_queue.enqueue( - BackgroundJobSpec( - type=pre_label_job_type, - payload=pre_label_payload_for( - batch.id, connection.id, body.minimum_confidence, geometries=geometries - ), - idempotent=True, + for job in open_jobs_of(workspace, batch.id): + row, joined = launch( + workspace, + job, + batch, + connection_id=connection.id, + minimum_confidence=body.minimum_confidence, + replace_model_labels=False, + geometries=geometries, ) - ) - items.append( - ProjectPreLabelItemOut( - batch_id=batch.id, - batch_name=batch.name, - job=BackgroundJobOut.of(job), - joined=running is not None, + items.append( + PreLabelFanOutItemOut( + batch_id=batch.id, + batch_name=batch.name, + annotation_job_id=job.id, + job=BackgroundJobOut.of(row), + joined=joined, + ) ) - ) runner.wake() - return ProjectPreLabelOut(items=items, total=len(items)) + return PreLabelFanOutOut(items=items, total=len(items)) @router.get("/{batch_id}/jobs", responses=documented(404)) @@ -614,11 +552,17 @@ def list_batch_jobs(workspace: WorkspaceDep, batch_id: UUID) -> JobPage: way. """ batches = BatchService(workspace) - # The batch itself, not only the id its path already carries: both job actions - # need the batch open, so ``allowed_actions`` cannot be answered without it. + # The batch itself, not only the id its path already carries: every job action + # needs the batch open, so ``allowed_actions`` cannot be answered without it. batch = batches.get(batch_id) found = batches.jobs(batch_id) - return JobPage(items=[JobOut.of(job, batch=batch) for job in found], total=len(found)) + # One queue read for the whole page, ``list_batches``'s cost model: without it + # a batch cut into twenty jobs would ask the queue once per row. + runs = JobService(workspace).pre_label_runs() + return JobPage( + items=[JobOut.of(job, batch=batch, pre_label_run=runs.get(job.id)) for job in found], + total=len(found), + ) @router.get("/{batch_id}/assets", responses=documented(404)) @@ -629,6 +573,7 @@ def list_batch_assets( offset: OffsetQuery = 0, progress: ProgressQuery = None, sort: SortQuery = AssetSort.MEMBERSHIP, + job: JobQuery = None, ) -> BatchAssetPage: """The batch's assets, with where each has got to and its labels in two numbers. @@ -640,15 +585,18 @@ def list_batch_assets( An offset past the end is an empty list and a 200, never a 404. The 404 belongs to the batch itself, which is resolved first: an unknown one is `BATCH_NOT_FOUND`. - `job_id` and `progress` are null while the batch is a draft, because a draft - has no jobs — so a `progress` filter over a draft matches nothing. Bytes are - not here: an asset is named by its hashes, and + `job` narrows to the assets one job carries, composing with `progress`; a job + this batch does not have is 404 `JOB_NOT_FOUND`. `job_id` and `progress` on + each item are null while the batch is a draft, which has no jobs — so there, + a `progress` filter or a `job` filter matches nothing rather than refusing. + Bytes are not here: an asset is named by its hashes, and `GET /projects/{project_id}/assets/{asset_id}/content` is what serves them. """ batches = BatchService(workspace) batch = batches.get(batch_id) placed, total = batches.asset_page( batch_id, + job=job, progress=None if progress is None else frozenset(progress), sort=sort, limit=limit, diff --git a/src/visionset/server/routes/jobs.py b/src/visionset/server/routes/jobs.py index 89cf38db..783a08a8 100644 --- a/src/visionset/server/routes/jobs.py +++ b/src/visionset/server/routes/jobs.py @@ -19,20 +19,24 @@ from typing import Annotated from uuid import UUID -from fastapi import Query +from fastapi import Query, Response, status +from visionset.inference import require_detectable_schema from visionset.kernel.services import JobService -from visionset.server.dependencies import WorkspaceDep, protected_router +from visionset.server.dependencies import RunnerDep, WorkspaceDep, protected_router from visionset.server.errors import documented from visionset.server.models import ( AssetOut, AssetPage, AssetProgressOut, AssetProgressSet, + BackgroundJobOut, JobAssign, JobOut, + PreLabelRequest, ProgressCounts, ) +from visionset.server.routes._prelabel import launch, selected_produces, text_detect_connection router = protected_router(prefix="/jobs", tags=["jobs"]) @@ -55,7 +59,11 @@ def get_job(workspace: WorkspaceDep, job_id: UUID) -> JobOut: job's work is judged against, which a job id alone does not. """ jobs = JobService(workspace) - return JobOut.of(jobs.get(job_id), batch=jobs.batch(job_id)) + return JobOut.of( + jobs.get(job_id), + batch=jobs.batch(job_id), + pre_label_run=jobs.latest_pre_label_run(job_id), + ) @router.get("/{job_id}/progress", responses=documented(404)) @@ -78,7 +86,122 @@ def start_job(workspace: WorkspaceDep, job_id: UUID) -> JobOut: is already in progress or finished never starts again. """ jobs = JobService(workspace) - return JobOut.of(jobs.start(job_id), batch=jobs.batch(job_id)) + return JobOut.of( + jobs.start(job_id), + batch=jobs.batch(job_id), + pre_label_run=jobs.latest_pre_label_run(job_id), + ) + + +@router.post( + "/{job_id}/pre-label", + status_code=status.HTTP_202_ACCEPTED, + responses=documented(404, 409), +) +def pre_label_job( + workspace: WorkspaceDep, + runner: RunnerDep, + response: Response, + job_id: UUID, + body: PreLabelRequest, +) -> BackgroundJobOut: + """Ask a model to label every untouched asset in this job, and answer at once. + + The `pre_label` action. Labels land at `pre_labeled`, never at `annotated`: + nobody judged them, so they arrive editable and correctable rather than + claiming to be somebody's work — and, being unjudged, they never reach the + Dataset until a person has taken them over. + + **Only assets nothing has touched — which is stronger than reading + `unannotated`.** An asset already `pre_labeled`, annotated, skipped, + awaiting review or accepted is passed over, and so is an `unannotated` one + that still carries annotations from an earlier round that was skipped and + then restored: that sequence deletes no labels, so progress alone does not + prove an asset untouched. A run never writes over what a person did in this + job, and never writes twice over what a model did — a plain second run + extends an earlier one onto whatever is still untouched. + `replace_model_labels` widens it to every frame still `pre_labeled` and + supersedes those labels with this run's answer, one frame per transaction; + a frame anyone edited, confirmed or skipped in this job is never touched, + and a frame the model now finds nothing on returns to `unannotated`. A + replacing request arriving while a run is in flight joins that run, + whichever flag it carries. + + **The batch's pinned schema is the prompt, narrowed to what this run + writes.** The model is asked for each class the schema declares that admits + one of the shapes the run writes and demands no attribute a prediction + cannot supply; an answer naming one of those classes, matched + case-insensitively, is written under the schema's own spelling, and an + answer naming none of them is discarded. A schema with no such class has + nowhere for a prediction to land and is refused — so the same schema is + askable of a model that answers polygons and refused for one that answers + boxes. `GET /batches/{batch_id}/pre-label` with the same `connection_id` + (and the same `geometries`) reads the narrowing before launching. + + **What the run writes is every shape the model produces, unless + `geometries` says which.** A model declaring both a box and a polygon + writes both for every region it answers with — the kernel writes one + annotation per emitted region and pairs nothing — and `geometries` filters + that to the shapes named: a region in any other shape is discarded and + counted in `regions_discarded`. The selection is per run, not per class, + and it is kept on the queued row, so a run claimed later executes what was + asked. + + **202, not 200.** A job is hundreds of forward passes, so this follows the + launch-and-poll contract the export and weight-download routes use: poll `GET + /background-jobs/{id}` — the `Location` header names it — until `state` is + `succeeded`, then re-read the job's assets. Progress on the row is counted + in assets, and `JobOut.pre_label_run` remembers the same row afterwards. + + **Everything a caller can be told now is told now**, and no refusal creates a + job — so a caller holding a job id holds one that will run. These refusals + are about the request, and the caller can act on each. They are checked in + this order, and it is the order `pre_label` itself checks in, so a request + wrong about the connection and the job both always names the connection: + an unknown connection is 404 `INFERENCE_CONNECTION_NOT_FOUND`; a + connection not set up yet is 409 `INFERENCE_CONNECTION_NOT_SET_UP` — its + weights not here, or its endpoint not yet asked what it answers; a + connection whose model answers places rather than words is 422 + `UNSUPPORTED_PROMPT`; a `geometries` naming a shape the model does not + produce is 422 `GEOMETRY_NOT_PRODUCED`. An unknown job is 404 + `JOB_NOT_FOUND`; a job whose batch is not `in_annotation` is 409 + `BATCH_NOT_IN_ANNOTATION`; a job already `completed` is 409 `JOB_FINISHED`, + and there is no remedy on this route — settled work is corrected through a + new batch rather than reopened. A pinned schema with no class the selected + shapes can be written as is 409 `SCHEMA_HAS_NO_DETECTABLE_CLASS`. + + Two failures are about this installation rather than about the request, and + answer 500 carrying the message that says which: a machine without the + optional local runtime is `LOCAL_INFERENCE_UNAVAILABLE` and carries the + exact command that installs it, and a workspace whose records no longer + hold together — a batch pinned to a schema version that is not stored — is + `WORKSPACE_CORRUPT`. Neither is worth resending unchanged: there is no + state here a caller can change, so the remedy is the one the message names. + + **Asking twice joins the run already in flight rather than starting a second + one.** A request arriving while this job has a pre-labeling run queued or + running is answered with that run's id, so a double-click and a second tab + watch one run instead of paying for the same inference twice — and so does + `POST /batches/{batch_id}/pre-label`, whose fan-out reaches this same job. + """ + connection = text_detect_connection(workspace, body.connection_id) + produces = selected_produces(connection, body.geometries) + job, batch = JobService(workspace).require_pre_labelable(job_id) + require_detectable_schema(workspace, batch, produces) + row, _ = launch( + workspace, + job, + batch, + connection_id=body.connection_id, + minimum_confidence=body.minimum_confidence, + replace_model_labels=body.replace_model_labels, + geometries=None if body.geometries is None else frozenset(body.geometries), + ) + # Woken even when the answer is a run that already existed: what came back + # may be `queued`, and the dispatcher it waits for sleeps on its own interval. + runner.wake() + response.headers["Location"] = f"/background-jobs/{row.id}" + return BackgroundJobOut.of(row) @router.post("/{job_id}/complete", responses=documented(404, 409)) @@ -100,7 +223,11 @@ def complete_job(workspace: WorkspaceDep, job_id: UUID) -> JobOut: derives that from all of them. """ jobs = JobService(workspace) - return JobOut.of(jobs.complete(job_id), batch=jobs.batch(job_id)) + return JobOut.of( + jobs.complete(job_id), + batch=jobs.batch(job_id), + pre_label_run=jobs.latest_pre_label_run(job_id), + ) @router.put("/{job_id}/assignee", responses=documented(404)) @@ -112,7 +239,9 @@ def assign_job(workspace: WorkspaceDep, job_id: UUID, body: JobAssign) -> JobOut """ service = JobService(workspace) job = service.assign(job_id, body.assignee) - return JobOut.of(job, batch=service.batch(job_id)) + return JobOut.of( + job, batch=service.batch(job_id), pre_label_run=service.latest_pre_label_run(job_id) + ) @router.get("/{job_id}/next", responses=documented(404)) diff --git a/src/visionset/wire/__init__.py b/src/visionset/wire/__init__.py index 6e76e37e..3defda7f 100644 --- a/src/visionset/wire/__init__.py +++ b/src/visionset/wire/__init__.py @@ -453,7 +453,7 @@ def progress_counts(counts: Mapping[AssetProgress, int]) -> dict[str, Any]: def pre_label_run(value: PreLabelRun) -> dict[str, Any]: - """A batch's most recent pre-labeling run: which job, how far, and what it found. + """An annotation job's most recent pre-labeling run: how far it got, and what it found. Assets, on ``weight_download``'s and ``integrity_check``'s terms: the handler's own unit, named where its job type is known. ``stopped_early``, @@ -462,6 +462,7 @@ def pre_label_run(value: PreLabelRun) -> dict[str, Any]: cancelled run still carries them, a failed one never does. """ return { + "annotation_job_id": str(value.annotation_job_id), "job_id": str(value.job_id), "state": value.state.value, "assets_processed": value.assets_processed, @@ -531,13 +532,23 @@ def batch( } -def job(value: AnnotationJob, *, batch_id: UUID, batch_state: BatchState) -> dict[str, Any]: +def job( + value: AnnotationJob, + *, + batch_id: UUID, + batch_state: BatchState, + pre_labeled: PreLabelRun | None = None, +) -> dict[str, Any]: """One segment of a batch. ``task_group_id`` and the per-asset map are absent. ``batch_state`` is not published here — ``BatchOut`` owns it — but nothing can be said about what this job may do without it: both of its actions need the batch open. The per-asset map stays unpublished and is still *read*, because ``complete`` is refined by whether every asset has settled. + + ``pre_labeled`` is this job's own most recent pre-labeling run, on the same + terms ``batch``'s own field reads the batch's: a caller with no view of the + queue passes nothing and ``pre_label_run`` is null. """ return { "id": str(value.id), @@ -551,6 +562,7 @@ def job(value: AnnotationJob, *, batch_id: UUID, batch_state: BatchState) -> dic value.state, batch_state=batch_state, progress=value.progress.values() ) ], + "pre_label_run": None if pre_labeled is None else pre_label_run(pre_labeled), } diff --git a/tests/fixtures/wire_capabilities.json b/tests/fixtures/wire_capabilities.json index d425bf74..89dea10f 100644 --- a/tests/fixtures/wire_capabilities.json +++ b/tests/fixtures/wire_capabilities.json @@ -48,10 +48,12 @@ "JOB_ACTIONS": { "completed": [], "in_progress": [ + "pre_label", "complete" ], "pending": [ - "start" + "start", + "pre_label" ] } } diff --git a/tests/server/test_batches.py b/tests/server/test_batches.py index 5061296f..d9603822 100644 --- a/tests/server/test_batches.py +++ b/tests/server/test_batches.py @@ -1136,6 +1136,25 @@ def test_a_progress_filter_over_a_draft_is_an_empty_page(client: TestClient, ing assert body == {"items": [], "total": 0} +def test_a_job_filter_lists_only_that_jobs_assets(client: TestClient, ingested: str) -> None: + client.post(f"/batches/{ingested}/approve", json={"partition": {"kind": "by_size", "size": 2}}) + first = client.get(f"/batches/{ingested}/jobs").json()["items"][0]["id"] + + page = client.get(f"/batches/{ingested}/assets", params={"job": first}).json() + + assert page["total"] == 2 + assert {one["job_id"] for one in page["items"]} == {first} + + +def test_a_job_of_another_batch_is_404(client: TestClient, ingested: str) -> None: + client.post(f"/batches/{ingested}/approve", json={"partition": {"kind": "by_size", "size": 2}}) + + response = client.get(f"/batches/{ingested}/assets", params={"job": str(uuid4())}) + + assert response.status_code == 404 + assert response.json()["code"] == "JOB_NOT_FOUND" + + def test_sorting_by_confidence_puts_the_weakest_first_and_unscored_last( client: TestClient, runner: InlineDispatcher, tmp_path: Path ) -> None: diff --git a/tests/server/test_jobs.py b/tests/server/test_jobs.py index 5c925a66..b145dd9c 100644 --- a/tests/server/test_jobs.py +++ b/tests/server/test_jobs.py @@ -280,8 +280,10 @@ def test_a_job_completes_once_every_asset_is_settled( "state": "completed", "assignee": None, "asset_count": 3, - # `JOB_TRANSITIONS[completed]` is empty, so a finished job declares nothing. + # `JOB_TRANSITIONS[completed]` is empty and `pre_label` is gated on the job + # being open, so a finished job declares nothing. "allowed_actions": [], + "pre_label_run": None, } diff --git a/tests/server/test_prelabel_route.py b/tests/server/test_prelabel_route.py index f8e3e262..f0bb8ddb 100644 --- a/tests/server/test_prelabel_route.py +++ b/tests/server/test_prelabel_route.py @@ -26,7 +26,7 @@ from visionset.inference.registry import capabilities, registered, served from visionset.jobs import prelabel as prelabel_handler from visionset.kernel.domain import DownloadSize, GeometryType, ModelCapability -from visionset.server.routes import batches as batches_routes +from visionset.server.routes import _prelabel as prelabel_routes from visionset.server.routes import inference as inference_routes #: A plain box class with no required attribute, so it is exactly what @@ -86,7 +86,7 @@ def _local_setup_is_faked(monkeypatch: pytest.MonkeyPatch) -> None: size lookup a create form reads before anybody commits to a download. """ monkeypatch.setattr(inference_routes, "require_local_inference", lambda: None) - monkeypatch.setattr(batches_routes, "require_local_inference", lambda: None) + monkeypatch.setattr(prelabel_routes, "require_local_inference", lambda: None) monkeypatch.setattr(weights_module, "download", lambda connection, *, into, on_bytes=None: into) monkeypatch.setattr( weights_module, @@ -157,6 +157,9 @@ class OpenBatch: project_id: str id: str connection_id: str + #: The batch's first annotation job — empty for a batch nobody has approved, + #: which has no jobs to name. + job_id: str = "" def _open_batch( @@ -176,7 +179,12 @@ def _open_batch( connection_id = _connection(client, runner, monkeypatch, family=family, capability=capability) assert client.post(f"/batches/{batch_id}/approve").status_code == 200 assert client.post(f"/batches/{batch_id}/start").status_code == 200 - return OpenBatch(project_id=project_id, id=batch_id, connection_id=connection_id) + return OpenBatch( + project_id=project_id, + id=batch_id, + connection_id=connection_id, + job_id=client.get(f"/batches/{batch_id}/jobs").json()["items"][0]["id"], + ) @pytest.fixture() @@ -256,10 +264,81 @@ def test_pre_labeling_answers_202_and_points_at_the_job( json={"connection_id": in_annotation_batch.connection_id, "minimum_confidence": 0.35}, ) + assert response.status_code == 202, response.text + assert response.json()["items"][0]["job"]["type"] == "annotation.pre_label" + + +def test_pre_labeling_a_job_answers_202_and_points_at_the_job( + client: TestClient, in_annotation_batch: OpenBatch +) -> None: + response = client.post( + f"/jobs/{in_annotation_batch.job_id}/pre-label", + json={"connection_id": in_annotation_batch.connection_id, "minimum_confidence": 0.35}, + ) assert response.status_code == 202, response.text assert response.headers["location"] == f"/background-jobs/{response.json()['id']}" +def test_the_job_remembers_its_run(client: TestClient, in_annotation_batch: OpenBatch) -> None: + launched = client.post( + f"/jobs/{in_annotation_batch.job_id}/pre-label", + json={"connection_id": in_annotation_batch.connection_id}, + ).json() + job = client.get(f"/jobs/{in_annotation_batch.job_id}").json() + assert job["pre_label_run"]["job_id"] == launched["id"] + assert job["pre_label_run"]["annotation_job_id"] == in_annotation_batch.job_id + listed = client.get(f"/batches/{in_annotation_batch.id}/jobs").json()["items"][0] + assert listed["pre_label_run"]["job_id"] == launched["id"] + + +def test_an_open_job_declares_pre_label_and_a_finished_one_does_not( + client: TestClient, in_annotation_batch: OpenBatch +) -> None: + job_id = in_annotation_batch.job_id + assert "pre_label" in client.get(f"/jobs/{job_id}").json()["allowed_actions"] + + for asset in client.get(f"/batches/{in_annotation_batch.id}/assets").json()["items"]: + client.put(f"/jobs/{job_id}/assets/{asset['id']}/progress", json={"progress": "skipped"}) + assert client.post(f"/jobs/{job_id}/start").status_code == 200 + assert client.post(f"/jobs/{job_id}/complete").status_code == 200 + + assert "pre_label" not in client.get(f"/jobs/{job_id}").json()["allowed_actions"] + + +def test_a_finished_job_is_refused_and_queues_nothing( + client: TestClient, in_annotation_batch: OpenBatch +) -> None: + job_id = in_annotation_batch.job_id + for asset in client.get(f"/batches/{in_annotation_batch.id}/assets").json()["items"]: + client.put(f"/jobs/{job_id}/assets/{asset['id']}/progress", json={"progress": "skipped"}) + assert client.post(f"/jobs/{job_id}/start").status_code == 200 + assert client.post(f"/jobs/{job_id}/complete").status_code == 200 + before = _pre_label_job_count(client) + + response = client.post( + f"/jobs/{job_id}/pre-label", json={"connection_id": in_annotation_batch.connection_id} + ) + + assert response.status_code == 409 + assert response.json()["code"] == "JOB_FINISHED" + assert _pre_label_job_count(client) == before + + +def test_the_batch_launch_fans_out_one_row_per_open_job( + client: TestClient, in_annotation_batch: OpenBatch +) -> None: + response = client.post( + f"/batches/{in_annotation_batch.id}/pre-label", + json={"connection_id": in_annotation_batch.connection_id}, + ) + assert response.status_code == 202, response.text + body = response.json() + assert body["total"] == 1 + assert body["items"][0]["annotation_job_id"] == in_annotation_batch.job_id + assert body["items"][0]["batch_id"] == in_annotation_batch.id + assert body["items"][0]["joined"] is False + + def test_asking_twice_joins_the_run_already_in_flight( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -271,7 +350,7 @@ def test_asking_twice_joins_the_run_already_in_flight( with api_client(tmp_path / "ws", dispatcher=manual) as client: manual.bind(client.app.state.workspace_handle) monkeypatch.setattr(inference_routes, "require_local_inference", lambda: None) - monkeypatch.setattr(batches_routes, "require_local_inference", lambda: None) + monkeypatch.setattr(prelabel_routes, "require_local_inference", lambda: None) monkeypatch.setattr( weights_module, "download", lambda connection, *, into, on_bytes=None: into ) @@ -292,7 +371,8 @@ def test_asking_twice_joins_the_run_already_in_flight( second = client.post(f"/batches/{batch.id}/pre-label", json=body) assert first.status_code == second.status_code == 202 - assert first.json()["id"] == second.json()["id"] + assert first.json()["items"][0]["job"]["id"] == second.json()["items"][0]["job"]["id"] + assert second.json()["items"][0]["joined"] is True assert _pre_label_job_count(client) == 1 @@ -334,7 +414,7 @@ def test_a_settled_run_is_readable_with_no_job_id_of_its_own( run = reopened["pre_label_run"] assert run is not None - assert run["job_id"] == launched["id"] + assert run["job_id"] == launched["items"][0]["job"]["id"] assert run["state"] == "failed" assert run["error"] assert run["annotations_replaced"] is None @@ -375,11 +455,12 @@ def test_the_replace_flag_defaults_off_and_reaches_the_run_when_set( json={"connection_id": in_annotation_batch.connection_id, "replace_model_labels": True}, ) assert flagged.status_code == 202, flagged.text - assert flagged.json()["id"] != plain.json()["id"] + flagged_row = flagged.json()["items"][0]["job"]["id"] + assert flagged_row != plain.json()["items"][0]["job"]["id"] assert captured["replace_model_labels"] is True run = client.get(f"/batches/{in_annotation_batch.id}").json()["pre_label_run"] - assert run["job_id"] == flagged.json()["id"] + assert run["job_id"] == flagged_row assert run["state"] == "succeeded" assert run["annotations_replaced"] == 2 @@ -740,7 +821,7 @@ def test_an_http_connection_that_answers_words_is_not_gated_on_the_local_runtime def _never() -> None: raise AssertionError("the local runtime must not be demanded for an http connection") - monkeypatch.setattr(batches_routes, "require_local_inference", _never) + monkeypatch.setattr(prelabel_routes, "require_local_inference", _never) with serving_endpoint(capability="text_detect") as endpoint: made = client.post( "/inference/connections", @@ -842,6 +923,7 @@ def test_the_project_launch_fans_out_one_row_per_open_batch( body = response.json() assert body["total"] == 2 assert [item["batch_id"] for item in body["items"]] == [in_annotation_batch.id, second] + assert all(item["annotation_job_id"] for item in body["items"]) assert all(item["job"]["type"] == "annotation.pre_label" for item in body["items"]) assert all(item["joined"] is False for item in body["items"]) assert {item["batch_name"] for item in body["items"]} == { From 56a6ed117389cec6f01b0dc456d70623bbc9f0d1 Mon Sep 17 00:00:00 2001 From: Armando Anaya Date: Tue, 25 Aug 2026 23:53:52 -0700 Subject: [PATCH 3/5] feat(cli,mcp): job pre-label; batch and project pre-label run per open job visionset job pre-label and the pre_label_job tool run one job; the batch and project verbs loop each batch's open jobs and report one outcome per job, with its job_id. Job outputs on both surfaces carry pre_label_run, and list_batch_assets takes job_id. --- docs/content/mcp-tools.md | 7 +- src/visionset/cli/batches.py | 65 +++++++++------ src/visionset/cli/jobs.py | 84 +++++++++++++++++-- src/visionset/cli/projects.py | 40 +++++---- src/visionset/mcp/batches.py | 122 ++++++++++++++++----------- src/visionset/mcp/jobs.py | 138 ++++++++++++++++++++++++++++++- src/visionset/mcp/main.py | 1 + tests/cli/test_batch_commands.py | 86 +++++++++++++++---- tests/cli/test_job_commands.py | 73 +++++++++++++++- tests/cli/test_json_contract.py | 1 + tests/mcp/test_batch_tools.py | 118 +++++++++++++++++++------- tests/mcp/test_job_tools.py | 30 +++++++ tests/mcp/test_registration.py | 1 + 13 files changed, 621 insertions(+), 145 deletions(-) diff --git a/docs/content/mcp-tools.md b/docs/content/mcp-tools.md index 2202d2e6..57783601 100644 --- a/docs/content/mcp-tools.md +++ b/docs/content/mcp-tools.md @@ -11,7 +11,7 @@ error envelope, and the three gate words. ## Always offered -52 tools, in the order an agent meets them: make a project, give it a schema, put images in it, work through them, promote, publish, export. +53 tools, in the order an agent meets them: make a project, give it a schema, put images in it, work through them, promote, publish, export. | Tool | Takes | What it does | | --- | --- | --- | @@ -34,10 +34,11 @@ error envelope, and the three gate words. | `approve_batch` | `batch_id`, `jobs_of`? | Freeze a batch, pin the project's active schema, and cut it into jobs. | | `start_batch` | `batch_id` | Open an approved batch for annotation. | | `get_pre_label_plan` | `batch_id`, `connection`, `geometries`? | Which classes a pre-labeling run of that connection over this batch would ask about, which it would leave out, and what shapes it would write. | -| `pre_label_batch` | `batch_id`, `connection`, `minimum_confidence`?, `replace_model_labels`?, `geometries`? | Ask a model to label every untouched asset in a batch. This blocks until it is done. | +| `pre_label_batch` | `batch_id`, `connection`, `minimum_confidence`?, `replace_model_labels`?, `geometries`? | Ask a model to label every untouched asset in a batch, one run per open job. | | `pre_label_project` | `project`, `connection`, `minimum_confidence`?, `batch_ids`?, `geometries`? | Ask a model to label untouched assets across a project's open batches. Blocks until done. | +| `pre_label_job` | `job_id`, `connection`, `minimum_confidence`?, `replace_model_labels`?, `geometries`? | Ask a model to label every untouched asset in one job. This blocks until it is done. | | `repin_batch` | `batch_id`, `allow_destructive`? | Move a batch's schema pin onto the project's *current* active version. | -| `list_batch_assets` | `batch_id`, `limit`?, `offset`?, `progress`?, `sort`? | List a batch's assets, with the job, progress and label summary each carries. | +| `list_batch_assets` | `batch_id`, `limit`?, `offset`?, `progress`?, `sort`?, `job_id`? | List a batch's assets, with the job, progress and label summary each carries. | | `create_batch` | `project`, `name`, `asset_ids`? | Start a draft batch over a chosen set of a project's assets. | | `add_batch_assets` | `batch_id`, `asset_ids` | Put assets into a draft batch. | | `remove_batch_assets` | `batch_id`, `asset_ids` | Take assets out of a draft batch. This does not delete anything. | diff --git a/src/visionset/cli/batches.py b/src/visionset/cli/batches.py index 98e4c7a7..45f7df8c 100644 --- a/src/visionset/cli/batches.py +++ b/src/visionset/cli/batches.py @@ -40,7 +40,9 @@ from visionset.inference import ( DEFAULT_MINIMUM_CONFIDENCE, PreLabelExclusionReason, + PreLabelOutcome, PreLabelPlan, + open_jobs_of, pre_label, shapes_prose, ) @@ -274,35 +276,52 @@ def batch_pre_label( json_out: JsonOption = False, workspace: WorkspaceOption = None, ) -> None: - """Ask a model to label every untouched asset in an open batch. + """Ask a model to label every untouched asset in every open job of a batch. This blocks because a terminal has no dispatcher to claim an enqueued run. """ with opened_workspace(workspace) as service: - connections = InferenceConnectionService(service) - outcome = pre_label( - service, - batch_id=batch, - connection_id=_resolve(connections, connection), - minimum_confidence=minimum_confidence, - replace_model_labels=replace_model_labels, - geometries=selected_geometries(geometry), - on_plan=announce_plan, - on_progress=lambda done, total: note(f"Pre-labeling {done}/{total} asset(s)."), - ) + connection_id = _resolve(InferenceConnectionService(service), connection) + BatchService(service).require_pre_labelable(batch) + geometries = selected_geometries(geometry) + items: list[tuple[UUID, PreLabelOutcome]] = [] + for job in open_jobs_of(service, batch): + note(f"Job {job.id}:") + outcome = pre_label( + service, + job_id=job.id, + connection_id=connection_id, + minimum_confidence=minimum_confidence, + replace_model_labels=replace_model_labels, + geometries=geometries, + on_plan=announce_plan if not items else None, + on_progress=lambda done, total: note(f"Pre-labeling {done}/{total} asset(s)."), + ) + items.append((job.id, outcome)) + if not items: + note("No open job to pre-label.") + typer.echo("0") + return + written = sum(outcome.annotations_written for _, outcome in items) if json_out: - document(asdict(outcome)) + document( + { + "items": [{"job_id": str(job_id), **asdict(outcome)} for job_id, outcome in items], + "annotations_written": written, + } + ) return - replaced = ( - f", replaced {outcome.annotations_replaced} earlier model label(s)" - if outcome.annotations_replaced - else "" - ) - note( - f"Pre-labeled {outcome.assets_labeled} asset(s), " - f"wrote {outcome.annotations_written} annotation(s){replaced}." - ) - typer.echo(str(outcome.annotations_written)) + for _, outcome in items: + replaced = ( + f", replaced {outcome.annotations_replaced} earlier model label(s)" + if outcome.annotations_replaced + else "" + ) + note( + f"Pre-labeled {outcome.assets_labeled} asset(s), " + f"wrote {outcome.annotations_written} annotation(s){replaced}." + ) + typer.echo(str(written)) @batch_app.command("complete") diff --git a/src/visionset/cli/jobs.py b/src/visionset/cli/jobs.py index 023ff39e..7868a14d 100644 --- a/src/visionset/cli/jobs.py +++ b/src/visionset/cli/jobs.py @@ -1,8 +1,9 @@ # usage: from visionset.cli.jobs import job_app """``visionset job`` — the annotator's unit of work, driven from a shell. -Six commands, each one service call: ``list``, ``next``, ``progress``, ``start``, -``mark``, ``complete``. +Seven commands: ``list``, ``next``, ``progress``, ``start``, ``mark``, +``complete``, and ``pre-label``, which invokes shared inference inline because a +terminal has no dispatcher — ``batch pre-label``'s pattern, one job down. ``next`` and ``mark`` are what make "the full cycle without touching Python" true: a batch cannot be completed until every asset has settled, and nothing else @@ -23,6 +24,7 @@ from __future__ import annotations +from dataclasses import asdict from typing import Annotated, Final from uuid import UUID @@ -31,8 +33,11 @@ from visionset import wire from visionset.cli._output import JsonOption, document, note, table from visionset.cli._workspace import WorkspaceOption, opened_workspace +from visionset.cli.batches import GeometryOption, announce_plan, selected_geometries +from visionset.cli.inference import ConnectionArgument, _resolve +from visionset.inference import DEFAULT_MINIMUM_CONFIDENCE, pre_label from visionset.kernel.domain import AssetProgress -from visionset.kernel.services import BatchService, JobService +from visionset.kernel.services import BatchService, InferenceConnectionService, JobService job_app = typer.Typer(help="Drive annotation jobs.", no_args_is_help=True) @@ -63,8 +68,18 @@ def job_list( # need the batch open, so ``allowed_actions`` cannot be answered without it. found = batches.get(batch) jobs = batches.jobs(batch) + runs = JobService(service).pre_label_runs() if json_out: - document(wire.page([wire.job(j, batch_id=found.id, batch_state=found.state) for j in jobs])) + document( + wire.page( + [ + wire.job( + j, batch_id=found.id, batch_state=found.state, pre_labeled=runs.get(j.id) + ) + for j in jobs + ] + ) + ) return table( _COLUMNS, @@ -141,8 +156,9 @@ def job_start( service_jobs = JobService(service) started = service_jobs.start(job) batch = service_jobs.batch(started.id) + run = service_jobs.latest_pre_label_run(started.id) if json_out: - document(wire.job(started, batch_id=batch.id, batch_state=batch.state)) + document(wire.job(started, batch_id=batch.id, batch_state=batch.state, pre_labeled=run)) return note(f"Job {started.id} is now {started.state.value}.") typer.echo(str(started.id)) @@ -189,8 +205,64 @@ def job_complete( service_jobs = JobService(service) completed = service_jobs.complete(job) batch = service_jobs.batch(completed.id) + run = service_jobs.latest_pre_label_run(completed.id) if json_out: - document(wire.job(completed, batch_id=batch.id, batch_state=batch.state)) + document(wire.job(completed, batch_id=batch.id, batch_state=batch.state, pre_labeled=run)) return note(f"Job {completed.id} is now {completed.state.value}.") typer.echo(str(completed.id)) + + +@job_app.command("pre-label") +def job_pre_label( + job: JobArgument, + connection: ConnectionArgument, + minimum_confidence: Annotated[ + float, + typer.Option( + "--minimum-confidence", + min=0.0, + max=1.0, + help="The floor a prediction must clear to be written, in [0, 1].", + ), + ] = DEFAULT_MINIMUM_CONFIDENCE, + replace_model_labels: Annotated[ + bool, + typer.Option( + "--replace-model-labels", + help="Also rewrite the model labels on frames still pre-labeled (nobody has " + "touched them). Cannot be undone.", + ), + ] = False, + geometry: GeometryOption = None, + json_out: JsonOption = False, + workspace: WorkspaceOption = None, +) -> None: + """Ask a model to label every untouched asset in one job of an open batch. + + Blocks because a terminal has no dispatcher to claim an enqueued run. + """ + with opened_workspace(workspace) as service: + outcome = pre_label( + service, + job_id=job, + connection_id=_resolve(InferenceConnectionService(service), connection), + minimum_confidence=minimum_confidence, + replace_model_labels=replace_model_labels, + geometries=selected_geometries(geometry), + on_plan=announce_plan, + on_progress=lambda done, total: note(f"Pre-labeling {done}/{total} asset(s)."), + ) + if json_out: + document({"job_id": str(job), **asdict(outcome)}) + return + replaced = ( + f", replaced {outcome.annotations_replaced} earlier model label(s)" + if outcome.annotations_replaced + else "" + ) + note( + f"Pre-labeled {outcome.assets_labeled} asset(s), " + f"wrote {outcome.annotations_written} annotation(s){replaced}." + ) + typer.echo(str(outcome.annotations_written)) diff --git a/src/visionset/cli/projects.py b/src/visionset/cli/projects.py index b2c91b4b..2ce5d2a4 100644 --- a/src/visionset/cli/projects.py +++ b/src/visionset/cli/projects.py @@ -38,6 +38,7 @@ from visionset.inference import ( DEFAULT_MINIMUM_CONFIDENCE, effective_produces, + open_jobs_of, pre_label, select_pre_labelable, served_for, @@ -143,32 +144,39 @@ def project_pre_label( outcomes = [] for one in selected: note(f"Batch {one.name!r}:") - outcome = pre_label( - service, - batch_id=one.id, - connection_id=connection_id, - minimum_confidence=minimum_confidence, - geometries=geometries, - on_plan=announce_plan, - on_progress=_progress_note(one.name), - ) - outcomes.append((one, outcome)) - written = sum(outcome.annotations_written for _, outcome in outcomes) + progress = _progress_note(one.name) + for index, job in enumerate(open_jobs_of(service, one.id)): + outcome = pre_label( + service, + job_id=job.id, + connection_id=connection_id, + minimum_confidence=minimum_confidence, + geometries=geometries, + on_plan=announce_plan if index == 0 else None, + on_progress=progress, + ) + outcomes.append((one, job.id, outcome)) + written = sum(outcome.annotations_written for _, _, outcome in outcomes) if json_out: document( { "items": [ - {"batch_id": str(one.id), "batch_name": one.name, **asdict(outcome)} - for one, outcome in outcomes + { + "batch_id": str(one.id), + "batch_name": one.name, + "job_id": str(job_id), + **asdict(outcome), + } + for one, job_id, outcome in outcomes ], "annotations_written": written, } ) return - for one, outcome in outcomes: + for one, job_id, outcome in outcomes: note( - f"Batch {one.name!r}: pre-labeled {outcome.assets_labeled} asset(s), " + f"Batch {one.name!r} job {job_id}: pre-labeled {outcome.assets_labeled} asset(s), " f"wrote {outcome.annotations_written} annotation(s)." ) - note(f"Pre-labeled {len(outcomes)} batch(es), wrote {written} annotation(s).") + note(f"Pre-labeled {len(selected)} batch(es), wrote {written} annotation(s).") typer.echo(str(written)) diff --git a/src/visionset/mcp/batches.py b/src/visionset/mcp/batches.py index f87ed3cb..a6f2d845 100644 --- a/src/visionset/mcp/batches.py +++ b/src/visionset/mcp/batches.py @@ -51,6 +51,7 @@ PreLabelOutcome, PreLabelPlan, effective_produces, + open_jobs_of, planned, pre_label, select_pre_labelable, @@ -95,8 +96,10 @@ def _batch_payload(workspace: WorkspaceService, batch_id: UUID) -> dict[str, Any """The batch, its progress and its jobs — the shape most tools return.""" batches = BatchService(workspace) batch = batches.get(batch_id) - counts = JobService(workspace).batch_progress(batch.id) + job_service = JobService(workspace) + counts = job_service.batch_progress(batch.id) jobs = batches.jobs(batch.id) + runs = job_service.pre_label_runs() return { **wire.batch( batch, @@ -104,7 +107,10 @@ def _batch_payload(workspace: WorkspaceService, batch_id: UUID) -> dict[str, Any promoted=_promoted(workspace, batch.project_id), pre_labeled=batches.latest_pre_label_job(batch.id), ), - "jobs": [wire.job(j, batch_id=batch.id, batch_state=batch.state) for j in jobs], + "jobs": [ + wire.job(j, batch_id=batch.id, batch_state=batch.state, pre_labeled=runs.get(j.id)) + for j in jobs + ], } @@ -383,7 +389,14 @@ def pre_label_batch( ] = False, geometries: Geometries = None, ) -> dict[str, Any]: - """Ask a model to label every untouched asset in a batch. This blocks until it is done. + """Ask a model to label every untouched asset in a batch, one run per open job. + + This blocks until every job is done. The job is the unit this runs over — + each open job of the batch gets its own run, reported as its own item in + `items`, and a job already `completed` is passed over rather than refused, + since a batch finished half in the annotator is the ordinary case. + `pre_label_job` is the same run for a single job, when that is the unit you + already hold. `download_connection_weights`'s pattern, not a shortcut: a stdio server has no background worker, so a tool that queued this work would answer with a @@ -446,35 +459,41 @@ class the schema declares that the shapes this run writes can be written `regions_discarded`. Naming a shape the model does not produce is refused before anything runs. - `plan` in the result names both halves: `asked_classes` is what this run + `items` holds one outcome per open job, in the same shape `pre_label_job` + returns, with its own `job_id` and `plan`; a finished job is passed over. + Each `plan` names both halves: `asked_classes` is what that job's run actually asked about, and `excluded_classes` names every class of the pinned schema it could not, each with every reason; `produces` says what shape the - run wrote. `schema_version` is the pin both were derived from. Read it - whenever `assets_labeled` is lower than expected — a run that asked about - two of a schema's five classes labels nothing under the other three, and - the counters alone cannot say so. `get_pre_label_plan` answers the same - thing without running anything. + run wrote. `schema_version` is the pin both were derived from — the same + pin for every job, since it is the batch's. Read a job's plan whenever its + `assets_labeled` is lower than expected — a run that asked about two of a + schema's five classes labels nothing under the other three, and the + counters alone cannot say so. `get_pre_label_plan` answers the same thing + without running anything. `annotations_written` at the top level is the + total across every item. Also refused before anything runs: a batch that is not `in_annotation`, a connection whose model answers places rather than words, and a deployment without the local runtime — with the install command in the message. """ - # Captured from the run rather than derived beside it: a plan read from the - # schema separately could differ from the one the run prompted with, and - # that it is the same list is the whole reason for reporting it. - seen: list[PreLabelPlan] = [] with opened_workspace() as workspace: resolved_connection = resolve_connection(workspace, connection) - outcome = pre_label( - workspace, - batch_id=identifier(batch_id, what="batch_id"), - connection_id=resolved_connection.id, - minimum_confidence=minimum_confidence, - replace_model_labels=replace_model_labels, - geometries=_selection(geometries), - on_plan=seen.append, - ) - return _pre_label_outcome(outcome, seen[0]) + batch_uuid = identifier(batch_id, what="batch_id") + BatchService(workspace).require_pre_labelable(batch_uuid) + items: list[dict[str, Any]] = [] + for job in open_jobs_of(workspace, batch_uuid): + seen: list[PreLabelPlan] = [] + outcome = pre_label( + workspace, + job_id=job.id, + connection_id=resolved_connection.id, + minimum_confidence=minimum_confidence, + replace_model_labels=replace_model_labels, + geometries=_selection(geometries), + on_plan=seen.append, + ) + items.append({"job_id": str(job.id), **_pre_label_outcome(outcome, seen[0])}) + return {"items": items, "annotations_written": sum(i["annotations_written"] for i in items)} def pre_label_project( @@ -500,12 +519,11 @@ def pre_label_project( ) -> dict[str, Any]: """Ask a model to label untouched assets across a project's open batches. Blocks until done. - `pre_label_batch`, one batch after another: the batch stays the unit, each - run writes what that tool writes and reports what it reports — `geometries` - included, the same selection for every batch — and `items` holds one such - outcome per batch with its `plan`; `annotations_written` is the total. - Every batch of the project that is `in_annotation` is run, or exactly - `batch_ids`. + The job is still the unit — `pre_label_batch` fanned out over every open + batch of the project, or exactly `batch_ids`, each in turn: `geometries` is + the same selection for every one, and `items` holds one outcome per open + job across the whole selection, each with its `batch_id`, `job_id` and + `plan`; `annotations_written` is the total. The connection is checked first: an unknown connection, one not set up yet, one whose model answers places rather than words, or a `geometries` @@ -538,22 +556,24 @@ def pre_label_project( ) items: list[dict[str, Any]] = [] for batch in selected: - seen: list[PreLabelPlan] = [] - outcome = pre_label( - workspace, - batch_id=batch.id, - connection_id=resolved_connection.id, - minimum_confidence=minimum_confidence, - geometries=_selection(geometries), - on_plan=seen.append, - ) - items.append( - { - "batch_id": str(batch.id), - "batch_name": batch.name, - **_pre_label_outcome(outcome, seen[0]), - } - ) + for job in open_jobs_of(workspace, batch.id): + seen: list[PreLabelPlan] = [] + outcome = pre_label( + workspace, + job_id=job.id, + connection_id=resolved_connection.id, + minimum_confidence=minimum_confidence, + geometries=_selection(geometries), + on_plan=seen.append, + ) + items.append( + { + "batch_id": str(batch.id), + "batch_name": batch.name, + "job_id": str(job.id), + **_pre_label_outcome(outcome, seen[0]), + } + ) return { "items": items, "annotations_written": sum(item["annotations_written"] for item in items), @@ -666,6 +686,9 @@ def list_batch_assets( ) ), ] = AssetSort.MEMBERSHIP, + job_id: Annotated[ + str | None, Field(description="Keep only the assets this job carries.") + ] = None, ) -> dict[str, Any]: """List a batch's assets, with the job, progress and label summary each carries. @@ -675,9 +698,11 @@ def list_batch_assets( `annotation_count` is every label on the asset; `min_confidence` is the lowest score among the labels a model wrote, on that model's own scale, or null. - `job_id` and `progress` are both null exactly while the batch is a draft, - because a draft has no jobs, so a `progress` filter over a draft matches - nothing. Use `get_asset_image` on any `id` here to see the pixels. + `job_id` and `progress` on each item are both null exactly while the batch is + a draft. `job_id` narrows to that job's assets; a job this batch does not + have is refused, and over a draft — which has no jobs — it matches nothing, + same as a `progress` filter. Use `get_asset_image` on any `id` here to see + the pixels. """ with opened_workspace() as workspace: resolved = identifier(batch_id, what="batch_id") @@ -685,6 +710,7 @@ def list_batch_assets( batch = service.get(resolved) placed, total = service.asset_page( resolved, + job=None if job_id is None else identifier(job_id, what="job_id"), progress=frozenset(progress) if progress else None, sort=sort, limit=limit, diff --git a/src/visionset/mcp/jobs.py b/src/visionset/mcp/jobs.py index fa634b70..7f223f86 100644 --- a/src/visionset/mcp/jobs.py +++ b/src/visionset/mcp/jobs.py @@ -17,6 +17,10 @@ terminates. It returns only ``unannotated`` assets in stored order, so calling it after each write walks the job exactly once with no bookkeeping on the agent's side. + +``pre_label_job`` asks a model to do the first pass instead: ``pre_label_batch`` +narrowed to one job, for the caller that already holds a job id rather than a +batch's whole list of them. """ from __future__ import annotations @@ -26,11 +30,13 @@ from pydantic import Field from visionset import wire +from visionset.inference import DEFAULT_MINIMUM_CONFIDENCE, PreLabelPlan, pre_label from visionset.kernel.domain import AssetProgress from visionset.kernel.services import JobService from visionset.mcp._autostart import autostarted -from visionset.mcp._resolve import identifier +from visionset.mcp._resolve import ConnectionRef, identifier, resolve_connection from visionset.mcp._workspace import opened_workspace +from visionset.mcp.batches import Geometries, _pre_label_outcome, _selection JobRef = Annotated[str, Field(description="The annotation job, by id.")] """The job a tool acts on. Module-level for the ``inspect.signature`` reason.""" @@ -47,7 +53,12 @@ def _job_payload( job = service.get(job_id) batch = service.batch(job.id) payload = { - **wire.job(job, batch_id=batch.id, batch_state=batch.state), + **wire.job( + job, + batch_id=batch.id, + batch_state=batch.state, + pre_labeled=service.latest_pre_label_run(job.id), + ), "batch_state": batch.state.value, "schema_version": batch.schema_version, "progress": wire.progress_counts(service.job_progress(job.id)), @@ -65,6 +76,8 @@ def get_job(job_id: JobRef) -> dict[str, Any]: `progress.unannotated` is how much is left; the job can be completed once it, `progress.pre_labeled` and `progress.review_pending` are all zero. + `pre_label_run` is this job's own most recent `pre_label_job` run, or null + if nothing has pre-labeled it yet. `job_id` comes from `approve_batch`, `get_batch` or `list_batch_assets`. It is not the `ingest_job_id` an `ingest` run returns — that names the run that @@ -74,6 +87,127 @@ def get_job(job_id: JobRef) -> dict[str, Any]: return _job_payload(JobService(workspace), identifier(job_id, what="job_id")) +def pre_label_job( + job_id: JobRef, + connection: ConnectionRef, + minimum_confidence: Annotated[ + float, + Field( + ge=0.0, + le=1.0, + description=( + "The floor a prediction must clear to be written, in [0, 1]. Tuned for a " + "text-prompt model's prompt-affinity score — a point-prompt model's mask " + "quality is a different scale and does not share a threshold with this." + ), + ), + ] = DEFAULT_MINIMUM_CONFIDENCE, + replace_model_labels: Annotated[ + bool, + Field( + description=( + "Also rewrite the model labels on frames still `pre_labeled` — labels a " + "model wrote and nobody has edited, confirmed or skipped — superseding them " + "with this run's answer. Frames anyone touched in this job are never " + "affected. This cannot be undone; read `get_job`'s `progress.pre_labeled` " + "first." + ) + ), + ] = False, + geometries: Geometries = None, +) -> dict[str, Any]: + """Ask a model to label every untouched asset in one job. This blocks until it is done. + + `pre_label_batch`'s run, narrowed to the one job you already hold: a + `batch_id` starts this from `open_jobs_of`'s whole list, this starts it + from one entry in that list. **The batch's pinned schema is still the + prompt** — the model is asked for each class the schema declares that the + shapes this run writes can be written as, exactly as a batch-wide run + would ask, because the schema is pinned to the batch, not chosen per job. + **A job already `completed` is refused, not passed over** — unlike + `pre_label_batch`, which skips a finished job on its way to the next one, + naming one job here is a decision to run that job, and a finished one has + nothing left to write. + + This call runs one forward pass per untouched asset, so the wait is + roughly that many times one image's inference time. + + **Interrupting is safe.** A plain run only ever writes to an asset nothing + has touched, and commits one asset's labels in the same transaction as its + move to `pre_labeled` — so a cut-off call has entered some prefix of the + assets it was reaching and touched nothing else, and calling this again + resumes with whatever is still untouched — plus, where + `replace_model_labels` is set, the frames still `pre_labeled` — rather than + starting over or double-writing what already landed. + + **Only assets nothing has touched — not merely assets reading + `unannotated`.** An asset already `pre_labeled`, annotated, skipped, + awaiting review or accepted is passed over, and so is an `unannotated` one + that still carries annotations from a round that was skipped and later + restored, since that sequence deletes no labels. A frame this tool already + pre-labeled is therefore never re-asked about by a plain call, at any confidence. + **`replace_model_labels` is the deliberate exception**: it also reaches every + frame still `pre_labeled` and supersedes the model's labels there with this + call's answer, one frame per transaction — a frame the model now finds + nothing on returns to `unannotated`, and `annotations_replaced` in the + result says how many labels went. A frame anyone edited, confirmed or + skipped in this job is never touched either way. What is written lands at + `pre_labeled`, never at `annotated` — nobody judged it, so it stays + editable and out of the Dataset until somebody does. An asset somebody + starts working while this call is still running is passed over the same + way rather than failing the whole call; `assets_skipped` in the result + says how many. + + **A region that could not be written as the class it named is discarded, + not fatal.** A label naming no phrase asked for, or a shape the class does + not admit or the model never declared, is passed over the same way. A + text-prompted detector answers with text decoded from spans over the + prompt, not a choice from the classes it was asked about, so a span + crossing the boundary between two phrases can answer with neither of + them; a model declaring two shapes may also answer in the one its class + does not take. `regions_discarded` in the result says how many. + + **A mapped region with no overlap with a measured asset is discarded + separately.** `regions_out_of_bounds` in the result says how many; an + asset without dimensions remains eligible. + + **What the run writes is every shape the model produces unless `geometries` + says which.** A model declaring both a box and a polygon writes both for + every region it answers with, unpaired; `geometries` filters that to the + shapes named, and a region in any other shape is counted in + `regions_discarded`. Naming a shape the model does not produce is refused + before anything runs. + + `plan` in the result names both halves: `asked_classes` is what this run + actually asked about, and `excluded_classes` names every class of the pinned + schema it could not, each with every reason; `produces` says what shape the + run wrote. `schema_version` is the pin both were derived from. Read it + whenever `assets_labeled` is lower than expected — a run that asked about + two of a schema's five classes labels nothing under the other three, and + the counters alone cannot say so. `get_pre_label_plan` answers the same + thing without running anything. + + Also refused before anything runs: a job that is `completed`, a batch that + is not `in_annotation`, a connection whose model answers places rather + than words, and a deployment without the local runtime — with the install + command in the message. + """ + seen: list[PreLabelPlan] = [] + with opened_workspace() as workspace: + resolved_connection = resolve_connection(workspace, connection) + resolved_job = identifier(job_id, what="job_id") + outcome = pre_label( + workspace, + job_id=resolved_job, + connection_id=resolved_connection.id, + minimum_confidence=minimum_confidence, + replace_model_labels=replace_model_labels, + geometries=_selection(geometries), + on_plan=seen.append, + ) + return {"job_id": str(resolved_job), **_pre_label_outcome(outcome, seen[0])} + + def complete_job(job_id: JobRef) -> dict[str, Any]: """Close a job, once every one of its assets has been settled. diff --git a/src/visionset/mcp/main.py b/src/visionset/mcp/main.py index 4de19a9f..fb4d7f2d 100644 --- a/src/visionset/mcp/main.py +++ b/src/visionset/mcp/main.py @@ -101,6 +101,7 @@ (batches.get_pre_label_plan, READS), (batches.pre_label_batch, WRITES), (batches.pre_label_project, WRITES), + (jobs.pre_label_job, WRITES), (batches.repin_batch, WRITES), (batches.list_batch_assets, READS), (batches.create_batch, WRITES), diff --git a/tests/cli/test_batch_commands.py b/tests/cli/test_batch_commands.py index cdc40fce..50684e12 100644 --- a/tests/cli/test_batch_commands.py +++ b/tests/cli/test_batch_commands.py @@ -403,9 +403,10 @@ def test_pre_label_json_emits_the_complete_outcome( ) -> None: _, batch = started_batch(root, tmp_path) - outcome = payload(root, "batch", "pre-label", batch, _connection(root)) + body = payload(root, "batch", "pre-label", batch, _connection(root)) - assert outcome == { + assert body["items"][0] == { + "job_id": jobs_of(root, batch)[0], "assets_considered": 6, "assets_labeled": 6, "annotations_written": 6, @@ -418,6 +419,51 @@ def test_pre_label_json_emits_the_complete_outcome( } +def test_batch_pre_label_json_reports_one_item_per_job( + root: Path, tmp_path: Path, predicting: _FakePredictor +) -> None: + _, batch = started_batch(root, tmp_path) + job = jobs_of(root, batch)[0] + + body = payload(root, "batch", "pre-label", batch, _connection(root)) + + assert [item["job_id"] for item in body["items"]] == [job] + assert body["annotations_written"] == body["items"][0]["annotations_written"] + + +def test_batch_pre_label_runs_a_job_at_a_time_across_two_jobs( + root: Path, tmp_path: Path, predicting: _FakePredictor +) -> None: + _, batch = started_batch(root, tmp_path, jobs_of=3) + jobs = jobs_of(root, batch) + + body = payload(root, "batch", "pre-label", batch, _connection(root)) + + assert [item["job_id"] for item in body["items"]] == jobs + assert [item["annotations_written"] for item in body["items"]] == [3, 3] + assert body["annotations_written"] == 6 + + +def test_batch_pre_label_with_no_open_job_notes_it_and_echoes_zero( + root: Path, tmp_path: Path +) -> None: + # Every job closed but the batch itself still ``in_annotation`` — reachable + # because ``batch complete`` is a separate, deliberate step. + _, batch = started_batch(root, tmp_path) + for job in jobs_of(root, batch): + ok(root, "job", "start", job) + for line in ok(root, "job", "next", job, "-n", "100").splitlines()[1:]: + ok(root, "job", "mark", job, line.split()[0], "--progress", "annotated") + ok(root, "job", "complete", job) + connection = _connection(root) + + result = run(root, "batch", "pre-label", batch, connection) + + assert result.exit_code == 0, result.output + assert result.stdout == "0\n" + assert "No open job to pre-label." in result.stderr + + def test_pre_label_replace_model_labels_rewrites_the_first_runs_frames( root: Path, tmp_path: Path, predicting: _FakePredictor ) -> None: @@ -430,8 +476,8 @@ def test_pre_label_replace_model_labels_rewrites_the_first_runs_frames( assert result.exit_code == 0, result.output assert result.stdout == "6\n" assert "replaced 6 earlier model label(s)" in result.stderr - outcome = payload(root, "batch", "pre-label", batch, connection, "--replace-model-labels") - assert outcome["annotations_replaced"] == 6 + body = payload(root, "batch", "pre-label", batch, connection, "--replace-model-labels") + assert body["items"][0]["annotations_replaced"] == 6 def test_batch_pre_label_help_lists_the_replace_option() -> None: @@ -496,10 +542,10 @@ def test_pre_label_without_geometry_writes_every_shape_the_model_produces( ) -> None: batch = _both_shapes_batch(root, tmp_path) - outcome = payload(root, "batch", "pre-label", batch, _connection(root)) + body = payload(root, "batch", "pre-label", batch, _connection(root)) - assert outcome["annotations_written"] == 12 - assert outcome["regions_discarded"] == 0 + assert body["items"][0]["annotations_written"] == 12 + assert body["items"][0]["regions_discarded"] == 0 def test_pre_label_geometry_json_counts_the_discarded_shape( @@ -507,10 +553,10 @@ def test_pre_label_geometry_json_counts_the_discarded_shape( ) -> None: batch = _both_shapes_batch(root, tmp_path) - outcome = payload(root, "batch", "pre-label", batch, _connection(root), "--geometry", "polygon") + body = payload(root, "batch", "pre-label", batch, _connection(root), "--geometry", "polygon") - assert outcome["annotations_written"] == 6 - assert outcome["regions_discarded"] == 6 + assert body["items"][0]["annotations_written"] == 6 + assert body["items"][0]["regions_discarded"] == 6 def test_pre_label_geometry_repeats( @@ -518,7 +564,7 @@ def test_pre_label_geometry_repeats( ) -> None: batch = _both_shapes_batch(root, tmp_path) - outcome = payload( + body = payload( root, "batch", "pre-label", @@ -530,8 +576,8 @@ def test_pre_label_geometry_repeats( "polygon", ) - assert outcome["annotations_written"] == 12 - assert outcome["regions_discarded"] == 0 + assert body["items"][0]["annotations_written"] == 12 + assert body["items"][0]["regions_discarded"] == 0 def test_pre_label_geometry_outside_what_the_model_produces_exits_1( @@ -648,8 +694,14 @@ def test_project_pre_label_runs_every_open_batch( assert result.stdout == "8\n" assert "Pre-labeling 'stills' 6/6 asset(s)." in result.stderr assert "Pre-labeling 'more' 2/2 asset(s)." in result.stderr - assert "Batch 'stills': pre-labeled 6 asset(s), wrote 6 annotation(s)." in result.stderr - assert "Batch 'more': pre-labeled 2 asset(s), wrote 2 annotation(s)." in result.stderr + assert ( + f"Batch 'stills' job {jobs_of(root, first)[0]}: " + "pre-labeled 6 asset(s), wrote 6 annotation(s)." in result.stderr + ) + assert ( + f"Batch 'more' job {jobs_of(root, second)[0]}: " + "pre-labeled 2 asset(s), wrote 2 annotation(s)." in result.stderr + ) assert "Pre-labeled 2 batch(es), wrote 8 annotation(s)." in result.stderr listed = {row["id"]: row for row in payload(root, "batch", "list", "-p", name)["items"]} assert listed[first]["progress"]["pre_labeled"] == 6 @@ -681,6 +733,10 @@ def test_project_pre_label_json_lists_one_outcome_per_batch( assert [item["batch_id"] for item in outcome["items"]] == [first, second] assert [item["batch_name"] for item in outcome["items"]] == ["stills", "more"] + assert [item["job_id"] for item in outcome["items"]] == [ + jobs_of(root, first)[0], + jobs_of(root, second)[0], + ] assert [item["annotations_written"] for item in outcome["items"]] == [6, 2] assert outcome["annotations_written"] == 8 diff --git a/tests/cli/test_job_commands.py b/tests/cli/test_job_commands.py index 791dc46a..6cd4b2f4 100644 --- a/tests/cli/test_job_commands.py +++ b/tests/cli/test_job_commands.py @@ -1,4 +1,4 @@ -"""``visionset job`` — the six commands that make the lifecycle drivable.""" +"""``visionset job`` — the seven commands that make the lifecycle drivable.""" from __future__ import annotations @@ -12,12 +12,16 @@ ok, payload, run, + runner, started_batch, usage_error, workspace, ) +from tests.cli.test_batch_commands import _connection, _FakePool, _FakePredictor -from visionset.kernel.domain import AssetProgress +from visionset.cli.main import app +from visionset.inference import prelabel as prelabel_module +from visionset.kernel.domain import AssetProgress, GeometryType from visionset.kernel.services import WORKSPACE_ENV_VAR @@ -32,6 +36,19 @@ def root(tmp_path: Path) -> Path: return workspace(tmp_path) +@pytest.fixture() +def predicting(monkeypatch: pytest.MonkeyPatch) -> _FakePredictor: + """``test_batch_commands``'s fixture, redefined here: an imported fixture + shadowed by a same-named parameter reads as an unused import to ruff.""" + predictor = _FakePredictor() + monkeypatch.setattr( + prelabel_module, + "resident", + lambda: _FakePool(predictor, produces=frozenset({GeometryType.BBOX})), + ) + return predictor + + def _assets(root: Path, job: str) -> list[str]: return [line.split()[0] for line in ok(root, "job", "next", job, "-n", "100").splitlines()[1:]] @@ -189,3 +206,55 @@ def test_completing_a_job_does_not_complete_its_batch(root: Path, tmp_path: Path ok(root, "job", "mark", job, asset, "--progress", "annotated") ok(root, "job", "complete", job) assert payload(root, "batch", "list", "-p", name)["items"][0]["state"] == "in_annotation" + + +# --- pre-label --------------------------------------------------------------- + + +def test_job_pre_label_writes_the_named_jobs_untouched_assets( + root: Path, tmp_path: Path, predicting: _FakePredictor +) -> None: + _, batch = started_batch(root, tmp_path, jobs_of=3) + job, other = jobs_of(root, batch) + connection = _connection(root) + + result = run(root, "job", "pre-label", job, connection) + + assert result.exit_code == 0, result.output + assert result.stdout == "3\n" + assert "Pre-labeling 1/3 asset(s)." in result.stderr + assert "Pre-labeled 3 asset(s), wrote 3 annotation(s)." in result.stderr + assert payload(root, "job", "progress", job)["pre_labeled"] == 3 + assert payload(root, "job", "progress", other)["pre_labeled"] == 0 + + +def test_job_pre_label_json_emits_the_job_id( + root: Path, tmp_path: Path, predicting: _FakePredictor +) -> None: + _, batch = started_batch(root, tmp_path) + job = jobs_of(root, batch)[0] + + outcome = payload(root, "job", "pre-label", job, _connection(root)) + + assert outcome["job_id"] == job + assert outcome["annotations_written"] == 6 + + +def test_job_pre_label_refuses_a_completed_job( + root: Path, tmp_path: Path, predicting: _FakePredictor +) -> None: + _, batch = started_batch(root, tmp_path) + job = jobs_of(root, batch)[0] + ok(root, "job", "start", job) + for asset in _assets(root, job): + ok(root, "job", "mark", job, asset, "--progress", "annotated") + ok(root, "job", "complete", job) + + result = run(root, "job", "pre-label", job, _connection(root)) + + assert result.exit_code == 1, result.output + assert result.stdout == "" + + +def test_job_help_lists_pre_label() -> None: + assert "pre-label" in runner.invoke(app, ["job", "--help"]).stdout diff --git a/tests/cli/test_json_contract.py b/tests/cli/test_json_contract.py index 19333afe..f51b148c 100644 --- a/tests/cli/test_json_contract.py +++ b/tests/cli/test_json_contract.py @@ -244,6 +244,7 @@ def test_an_empty_listing_is_still_an_object() -> None: def test_a_settled_pre_label_run_has_wire_parity_when_nested_in_a_batch() -> None: run = PreLabelRun( batch_id=BATCH.id, + annotation_job_id=JOB.id, job_id=uuid4(), state=BackgroundJobState.SUCCEEDED, assets_processed=2, diff --git a/tests/mcp/test_batch_tools.py b/tests/mcp/test_batch_tools.py index 35a639c9..a06b275c 100644 --- a/tests/mcp/test_batch_tools.py +++ b/tests/mcp/test_batch_tools.py @@ -562,31 +562,69 @@ def _connection() -> str: def test_pre_labeling_blocks_and_returns_what_it_wrote( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - _, batch_id, _job = open_batch(monkeypatch, tmp_path, count=2) + _, batch_id, job_id = open_batch(monkeypatch, tmp_path, count=2) connection_id = _connection() _predicting(monkeypatch, label="sign") outcome = payload(call("pre_label_batch", batch_id=batch_id, connection=connection_id)) - assert outcome == { - "assets_considered": 2, - "assets_labeled": 2, - "annotations_written": 2, - "annotations_replaced": 0, - "model_ref": "acme/detector@abc123", - "assets_skipped": 0, - "regions_discarded": 0, - "regions_out_of_bounds": 0, - "plan": { - "schema_version": 1, - "asked_classes": ["sign"], - "produces": ["bbox"], - "excluded_classes": [], - }, - } + assert outcome["annotations_written"] == 2 + assert outcome["items"] == [ + { + "job_id": job_id, + "assets_considered": 2, + "assets_labeled": 2, + "annotations_written": 2, + "annotations_replaced": 0, + "model_ref": "acme/detector@abc123", + "assets_skipped": 0, + "regions_discarded": 0, + "regions_out_of_bounds": 0, + "plan": { + "schema_version": 1, + "asked_classes": ["sign"], + "produces": ["bbox"], + "excluded_classes": [], + }, + } + ] assert payload(call("get_batch", batch_id=batch_id))["progress"]["pre_labeled"] == 2 +def test_pre_labeling_a_batch_runs_one_item_per_open_job( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _, batch_id = ingested(monkeypatch, tmp_path, count=4) + approved = payload(call("approve_batch", batch_id=batch_id, jobs_of=2)) + job_ids = [j["id"] for j in approved["jobs"]] + payload(call("start_batch", batch_id=batch_id)) + connection_id = _connection() + _predicting(monkeypatch, label="sign") + + outcome = payload(call("pre_label_batch", batch_id=batch_id, connection=connection_id)) + + assert len(job_ids) == 2 + assert {item["job_id"] for item in outcome["items"]} == set(job_ids) + assert outcome["annotations_written"] == sum( + item["annotations_written"] for item in outcome["items"] + ) + assert outcome["annotations_written"] == 4 + + +def test_pre_labeling_a_batch_with_no_open_job_writes_nothing( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _, batch_id, job_id = open_batch(monkeypatch, tmp_path, count=1) + asset_id = payload(call("list_batch_assets", batch_id=batch_id))["items"][0]["id"] + payload(call("set_asset_progress", job_id=job_id, asset_id=asset_id, progress="skipped")) + payload(call("complete_job", job_id=job_id)) + connection_id = _connection() + + outcome = payload(call("pre_label_batch", batch_id=batch_id, connection=connection_id)) + + assert outcome == {"items": [], "annotations_written": 0} + + def test_a_replacing_run_rewrites_the_frames_the_first_run_labeled( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -604,9 +642,10 @@ def test_a_replacing_run_rewrites_the_frames_the_first_run_labeled( ) ) - assert again["assets_considered"] == 2 - assert again["annotations_written"] == 2 - assert again["annotations_replaced"] == 2 + item = again["items"][0] + assert item["assets_considered"] == 2 + assert item["annotations_written"] == 2 + assert item["annotations_replaced"] == 2 assert payload(call("get_batch", batch_id=batch_id))["progress"]["pre_labeled"] == 2 @@ -625,10 +664,11 @@ def test_a_run_that_labeled_nothing_says_what_it_asked_about( _predicting(monkeypatch, label="centerline") outcome = payload(call("pre_label_batch", batch_id=batch_id, connection=connection_id)) + item = outcome["items"][0] - assert outcome["assets_labeled"] == 0 - assert outcome["plan"]["asked_classes"] == ["sign"] - assert outcome["plan"]["excluded_classes"] == [ + assert item["assets_labeled"] == 0 + assert item["plan"]["asked_classes"] == ["sign"] + assert item["plan"]["excluded_classes"] == [ {"name": "centerline", "reasons": ["no_producible_geometry"]}, {"name": "crossing", "reasons": ["no_producible_geometry", "required_attribute"]}, ] @@ -796,17 +836,19 @@ def test_a_selection_writes_only_those_shapes_and_the_run_reports_it( outcome = payload( call("pre_label_batch", batch_id=batch_id, connection=connection_id, geometries=["bbox"]) ) + item = outcome["items"][0] assert outcome["annotations_written"] == 2 - assert outcome["regions_discarded"] == 2 - assert outcome["plan"]["produces"] == ["bbox"] - assert outcome["plan"]["asked_classes"] == ["sign"] + assert item["regions_discarded"] == 2 + assert item["plan"]["produces"] == ["bbox"] + assert item["plan"]["asked_classes"] == ["sign"] other = _another_open_batch(monkeypatch, tmp_path, project) unselected = payload(call("pre_label_batch", batch_id=other, connection=connection_id)) + unselected_item = unselected["items"][0] assert unselected["annotations_written"] == 4 - assert unselected["regions_discarded"] == 0 - assert unselected["plan"]["produces"] == ["bbox", "polygon"] + assert unselected_item["regions_discarded"] == 0 + assert unselected_item["plan"]["produces"] == ["bbox", "polygon"] def test_a_selection_outside_what_the_model_produces_is_refused_and_writes_nothing( @@ -889,6 +931,20 @@ def test_an_empty_progress_list_means_no_filter( assert payload(call("list_batch_assets", batch_id=batch_id, progress=[]))["total"] == 2 +def test_a_job_filter_lists_only_that_jobs_assets( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _, batch_id = ingested(monkeypatch, tmp_path, count=4) + approved = payload(call("approve_batch", batch_id=batch_id, jobs_of=2)) + payload(call("start_batch", batch_id=batch_id)) + first = approved["jobs"][0]["id"] + + listed = payload(call("list_batch_assets", batch_id=batch_id, job_id=first)) + + assert listed["total"] == 2 + assert {a["job_id"] for a in listed["items"]} == {first} + + # --- pre-labeling a project: the batch, fanned out ---------------------------- @@ -905,7 +961,7 @@ def _another_open_batch(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, project def test_pre_labeling_a_project_runs_every_open_batch_and_reports_each( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - project, first, _job = open_batch(monkeypatch, tmp_path, count=2) + project, first, job_id = open_batch(monkeypatch, tmp_path, count=2) second = _another_open_batch(monkeypatch, tmp_path, project) connection_id = _connection() _predicting(monkeypatch, label="sign") @@ -913,6 +969,7 @@ def test_pre_labeling_a_project_runs_every_open_batch_and_reports_each( outcome = payload(call("pre_label_project", project=project, connection=connection_id)) assert [item["batch_id"] for item in outcome["items"]] == [first, second] + assert outcome["items"][0]["job_id"] == job_id assert all(item["annotations_written"] == 2 for item in outcome["items"]) assert all(item["plan"]["asked_classes"] == ["sign"] for item in outcome["items"]) assert outcome["annotations_written"] == 4 @@ -938,7 +995,7 @@ def test_pre_labeling_a_project_narrows_to_the_named_batches( def test_pre_labeling_a_project_carries_the_selection_to_every_batch( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - project, first, _job = open_batch(monkeypatch, tmp_path, count=2, classes=BOTH_SHAPES_CLASSES) + project, first, job_id = open_batch(monkeypatch, tmp_path, count=2, classes=BOTH_SHAPES_CLASSES) second = _another_open_batch(monkeypatch, tmp_path, project) connection_id = _connection() _predicting(monkeypatch, both_shapes=True) @@ -948,6 +1005,7 @@ def test_pre_labeling_a_project_carries_the_selection_to_every_batch( ) assert [item["batch_id"] for item in outcome["items"]] == [first, second] + assert outcome["items"][0]["job_id"] == job_id assert all(item["annotations_written"] == 2 for item in outcome["items"]) assert all(item["regions_discarded"] == 2 for item in outcome["items"]) assert all(item["plan"]["produces"] == ["bbox"] for item in outcome["items"]) diff --git a/tests/mcp/test_job_tools.py b/tests/mcp/test_job_tools.py index b0c20788..21d82e68 100644 --- a/tests/mcp/test_job_tools.py +++ b/tests/mcp/test_job_tools.py @@ -7,6 +7,7 @@ import pytest from tests.mcp._flow import BBOX, call, error, ingested, open_batch, payload, tool_schemas +from tests.mcp.test_batch_tools import _connection, _predicting def _add(job_id: str, asset_id: str) -> dict[str, Any]: @@ -85,6 +86,35 @@ def test_a_job_names_the_batch_and_the_schema_its_work_is_judged_against( assert job["progress"]["unannotated"] == 2 +def test_pre_labeling_a_job_blocks_and_returns_what_it_wrote( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _, _batch_id, job_id = open_batch(monkeypatch, tmp_path, count=3) + connection_id = _connection() + _predicting(monkeypatch, label="sign") + + out = payload(call("pre_label_job", job_id=job_id, connection=connection_id)) + + assert out["job_id"] == job_id + assert out["assets_labeled"] == 3 + assert "plan" in out + + +def test_pre_labeling_a_completed_job_is_refused( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _, batch_id, job_id = open_batch(monkeypatch, tmp_path, count=1) + asset_id = payload(call("list_batch_assets", batch_id=batch_id))["items"][0]["id"] + payload(call("set_asset_progress", job_id=job_id, asset_id=asset_id, progress="skipped")) + payload(call("complete_job", job_id=job_id)) + connection_id = _connection() + _predicting(monkeypatch) + + refused = error(call("pre_label_job", job_id=job_id, connection=connection_id)) + + assert "does not re-open" in refused["message"] + + def test_next_pending_returns_only_unannotated_assets_and_shrinks_as_you_work( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: diff --git a/tests/mcp/test_registration.py b/tests/mcp/test_registration.py index 317be6a9..28ea581a 100644 --- a/tests/mcp/test_registration.py +++ b/tests/mcp/test_registration.py @@ -42,6 +42,7 @@ "get_pre_label_plan", "pre_label_batch", "pre_label_project", + "pre_label_job", "repin_batch", "complete_batch", "list_batch_assets", From 3660fe505399ed0adb4627a0d54b009350baeecb Mon Sep 17 00:00:00 2001 From: Armando Anaya Date: Tue, 25 Aug 2026 23:53:52 -0700 Subject: [PATCH 4/5] feat(ui): the gallery enters annotation by job, as an accordion of jobs Once a batch is open its frames are shown per job: at most one panel open, every panel closable, the first job with work left open on arrival. A collapsed header states the job's frames, state, annotated count, assignee and a bar; the open panel carries the door (Annotate takes a pending job and opens it; Continue and View only open), Pre-label gated on the job's allowed_actions, the assignee, segment chips counted from the job's own progress, the order select, the job's timeline and only its frames. The thumbnail-size control is one shared setting on the header's caption line. The header keeps Start annotating only as the approved to in_annotation transition. The project's Annotate control opens a batch's one job directly and the gallery otherwise. The virtualised grid, selection and bulk bar move unchanged into FrameGrid; JobsStrip and JobRow are gone. --- frontend/app/cycle/cycle.spec.ts | 39 +- frontend/app/e2e/_visual.ts | 1 + frontend/app/e2e/_wire.ts | 4 +- frontend/app/e2e/annotate.spec.ts | 1 + frontend/app/e2e/gallery.spec.ts | 174 +- frontend/app/e2e/navigation.spec.ts | 1 + frontend/app/e2e/project-nav.spec.ts | 72 +- frontend/app/e2e/viewport.spec.ts | 1 + frontend/app/src/routes.tsx | 5 + .../src/annotator/addClassProvenance.test.tsx | 1 + .../src/annotator/drawingClass.test.tsx | 1 + .../src/annotator/editorNotice.test.tsx | 1 + .../src/annotator/frameGallery.test.tsx | 1 + frontend/ui-core/src/annotator/jobQueries.ts | 8 +- .../ui-core/src/annotator/pinBadge.test.tsx | 1 + .../src/annotator/suggestFlow.test.tsx | 1 + .../ui-core/src/annotator/topBar.test.tsx | 1 + .../src/annotator/viewportFloor.test.tsx | 1 + .../ui-core/src/data/capabilities.test.ts | 2 +- frontend/ui-core/src/data/capabilities.ts | 1 + frontend/ui-core/src/patterns/ProjectNav.tsx | 47 +- .../ui-core/src/patterns/projectNav.test.tsx | 68 +- .../ui-core/src/screens/BatchLifecycle.tsx | 8 +- frontend/ui-core/src/screens/FrameGrid.tsx | 1179 ++++++++++++ .../ui-core/src/screens/GalleryScreen.tsx | 1576 +++-------------- frontend/ui-core/src/screens/JobPanels.tsx | 466 +++++ .../ui-core/src/screens/PreLabelDialog.tsx | 124 +- frontend/ui-core/src/screens/ProjectFrame.tsx | 23 +- .../src/screens/ProjectPreLabelDialog.tsx | 30 +- .../ui-core/src/screens/ProjectScreen.tsx | 3 + frontend/ui-core/src/screens/gallery.test.tsx | 364 ++-- .../ui-core/src/screens/jobPanels.test.tsx | 582 ++++++ .../ui-core/src/screens/preLabel.test.tsx | 235 ++- .../src/screens/projectPreLabel.test.tsx | 69 +- frontend/ui-core/src/screens/queries.ts | 128 +- frontend/ui-core/src/testing/wire.fixtures.ts | 4 +- 36 files changed, 3527 insertions(+), 1696 deletions(-) create mode 100644 frontend/ui-core/src/screens/FrameGrid.tsx create mode 100644 frontend/ui-core/src/screens/JobPanels.tsx create mode 100644 frontend/ui-core/src/screens/jobPanels.test.tsx diff --git a/frontend/app/cycle/cycle.spec.ts b/frontend/app/cycle/cycle.spec.ts index efedeaf0..0f4f7e33 100644 --- a/frontend/app/cycle/cycle.spec.ts +++ b/frontend/app/cycle/cycle.spec.ts @@ -473,6 +473,41 @@ test("the whole cycle, from opening the app to a downloaded export", async ({ pa await expect(page.getByTestId("state-cycle-batch")).toHaveText("in progress"); }); + await test.step("the job panel is the way in: start the one job, and it opens", async () => { + // Only the panel's door is driven here: the cycle server has no `text_detect` + // model, so the pre-label half of the panel is exercised against the server suite. + await openProject(page, PROJECT, "batches"); + await page.getByTestId("open-batch-cycle-batch").click(); + await expect(page.getByTestId("gallery")).toBeVisible(); + // The header offers no door of its own any more; the job panel does. + await expect(page.getByTestId("start-annotating")).toHaveCount(0); + const panels = page.getByTestId("job-panels"); + await expect(panels).toBeVisible(); + // One job, with every frame still to do, so the accordion opens on it — and + // the collapsed row is the overview even so. + const header = panels.getByTestId(/^job-header-/); + await expect(header).toHaveAttribute("aria-expanded", "true"); + await expect(header).toContainText("Job 1"); + await expect(header).toContainText("pending"); + const door = panels.getByTestId(/^job-panel-/).getByTestId(/^start-job-/); + await expect(door).toHaveText("Annotate"); + const jobId = (await door.getAttribute("data-testid"))!.replace("start-job-", ""); + await door.click(); + await expect(page.getByTestId("annotation-page")).toBeVisible(); + // The door names no frame, so the job opens on its first — and the annotator + // then writes the frame it is showing into the address, as it does after any + // walk, which is why `?asset=` may already be there. + await expect(page.getByTestId("asset-position")).toContainText("1/3"); + await expect(page).toHaveURL(new RegExp(`/jobs/${jobId}(\\?asset=[0-9a-f-]+)?$`)); + await page.getByTestId("back").click(); + await expect(page.getByTestId("gallery")).toBeVisible(); + // Started: the header says so, and the door now continues rather than starts. + await expect(panels.getByTestId(/^job-header-/)).toContainText("in progress"); + await expect(panels.getByTestId(/^job-panel-/).getByTestId(/^start-job-/)).toHaveText( + "Continue", + ); + }); + await test.step("reach the annotator by clicking, on the asset that was clicked", async () => { // **The annotator is reached by clicking, and that is the reason this step // exists.** `page.goto('./jobs/' + id)` with the id read out of the API makes @@ -491,7 +526,9 @@ test("the whole cycle, from opening the app to a downloaded export", async ({ pa // visible rather than hover-gated, which a touch device would never reach. // What matters is that the annotator is reachable **by clicking**, with no id // read out of the API and no URL typed. - const tiles = page.getByTestId(/^tile-/); + // Inside the open panel: the frames belong to a job now, and three is the + // whole batch only because this batch was cut into one. + const tiles = page.getByTestId(/^job-panel-/).getByTestId(/^tile-/); await expect(tiles).toHaveCount(3); const third = tiles.nth(2); await expect(third).not.toHaveAttribute("data-pending", "true"); diff --git a/frontend/app/e2e/_visual.ts b/frontend/app/e2e/_visual.ts index 0d877f0a..ea7d29aa 100644 --- a/frontend/app/e2e/_visual.ts +++ b/frontend/app/e2e/_visual.ts @@ -357,6 +357,7 @@ export async function serveVisualApi(page: Page, options: VisualOptions = {}): P asset_count: 1, allowed_actions: jobActions("in_progress"), assignee: null, + pre_label_run: null, } satisfies Wire["JobOut"], }); } diff --git a/frontend/app/e2e/_wire.ts b/frontend/app/e2e/_wire.ts index 95277bf5..1532ab72 100644 --- a/frontend/app/e2e/_wire.ts +++ b/frontend/app/e2e/_wire.ts @@ -58,8 +58,8 @@ const BATCH_ACTIONS: Record }; const JOB_ACTIONS: Record = { - pending: ["start"], - in_progress: ["complete"], + pending: ["start", "pre_label"], + in_progress: ["pre_label", "complete"], completed: [], }; diff --git a/frontend/app/e2e/annotate.spec.ts b/frontend/app/e2e/annotate.spec.ts index a6012db3..3e855620 100644 --- a/frontend/app/e2e/annotate.spec.ts +++ b/frontend/app/e2e/annotate.spec.ts @@ -248,6 +248,7 @@ async function serveApi( settled: lifecycle.jobSettled ?? true, }), assignee: null, + pre_label_run: null, }); await page.route("**/api/**", async (route) => { const request = route.request(); diff --git a/frontend/app/e2e/gallery.spec.ts b/frontend/app/e2e/gallery.spec.ts index 403a5099..a7d9ceef 100644 --- a/frontend/app/e2e/gallery.spec.ts +++ b/frontend/app/e2e/gallery.spec.ts @@ -28,6 +28,7 @@ import { assetActions, batchActions, jobActions, type Wire } from "./_wire"; const PROJECT = "11111111-1111-4111-8111-111111111111"; const BATCH = "22222222-2222-4222-8222-222222222222"; const JOB = "33333333-3333-4333-8333-333333333333"; +const JOB_B = "55555555-5555-4555-8555-555555555555"; const SOURCE = "44444444-4444-4444-8444-444444444444"; /** A 1x1 PNG, so a tile has real bytes rather than a broken-image box. */ @@ -84,7 +85,7 @@ const SETTLED_STATES: readonly Wire["AssetProgress"][] = [ ]; function assets( - jobId: string | null, + jobOf: (at: number) => string | null, settled = false, batchState: Wire["BatchState"] = "in_annotation", removed: ReadonlySet = new Set(), @@ -99,30 +100,33 @@ function assets( .filter(({ at }) => !removed.has(`asset-${at}`)); return { total: kept.length, - items: kept.map(({ progress, at }) => ({ - id: `asset-${at}`, - project_id: PROJECT, - modality: "image", - content_hash: `${at}`.padStart(8, "0") + "deadbeef", - width: 1280, - height: 720, - format: "png", - source_id: SOURCE, - frame_index: INDEXES[at], - frame_timestamp: at, - thumbnail_hash: "cafebabe", - ingested_at: "2026-08-01T09:00:00Z", - job_id: jobId, - progress: jobId === null ? null : progress, - // The server's own answer, and the dimension the client's old mirror - // dropped: `asset_actions` returns `[]` for every frame of a batch that is - // not `in_annotation`, whatever the frame's own progress is. - allowed_actions: assetActions(jobId === null ? null : progress, { batchState }), - // Three boxes on the one card a test names ("3 boxes"); every other state - // carries none, which is what an unannotated or merely-reviewed frame is. - annotation_count: progress === "annotated" ? 3 : 0, - min_confidence: null, - })), + items: kept.map(({ progress, at }) => { + const jobId = jobOf(at); + return { + id: `asset-${at}`, + project_id: PROJECT, + modality: "image", + content_hash: `${at}`.padStart(8, "0") + "deadbeef", + width: 1280, + height: 720, + format: "png", + source_id: SOURCE, + frame_index: INDEXES[at], + frame_timestamp: at, + thumbnail_hash: "cafebabe", + ingested_at: "2026-08-01T09:00:00Z", + job_id: jobId, + progress: jobId === null ? null : progress, + // The server's own answer, and the dimension the client's old mirror + // dropped: `asset_actions` returns `[]` for every frame of a batch that is + // not `in_annotation`, whatever the frame's own progress is. + allowed_actions: assetActions(jobId === null ? null : progress, { batchState }), + // Three boxes on the one card a test names ("3 boxes"); every other state + // carries none, which is what an unannotated or merely-reviewed frame is. + annotation_count: progress === "annotated" ? 3 : 0, + min_confidence: null, + }; + }), }; } @@ -147,9 +151,45 @@ const SETTLED_COUNTS = { skipped: 45, } satisfies Wire["ProgressCounts"]; +/** + * The two jobs' counts when the fixture cuts two, each describing its own half. + * + * The accordion opens on the first job with `unannotated + pre_labeled` left, so + * the split is what decides which panel a two-job run starts on — here the first, + * with the second's frames all settled or in review. + */ +const SPLIT_COUNTS: Record = { + [JOB]: { + total: 4, + unannotated: 2, + pre_labeled: 0, + annotated: 2, + review_pending: 0, + accepted: 0, + skipped: 0, + }, + [JOB_B]: { + total: 4, + unannotated: 1, + pre_labeled: 0, + annotated: 0, + review_pending: 1, + accepted: 1, + skipped: 1, + }, +}; + interface Options { /** `draft` is the state with the approve CTA, and the state with no jobs. */ readonly state?: Wire["BatchState"]; + /** + * Cut the eight frames into two jobs, 0–3 and 4–7. + * + * Different frames on each side, which is what makes "only the open job's + * tiles" a claim with content: a page that dropped the `job` filter would show + * all eight and a stub that answered one fixed page could not tell. + */ + readonly twoJobs?: boolean; /** * Every frame settled and nothing outstanding. * @@ -165,7 +205,12 @@ interface Options { async function serveApi(page: Page, sent: Request[], options: Options = {}): Promise { const state = options.state ?? "in_annotation"; - const jobId = state === "draft" ? null : JOB; + const twoJobs = options.twoJobs === true; + const roster = twoJobs ? [JOB, JOB_B] : [JOB]; + // A draft has no jobs at all; otherwise the second half of the fixture belongs + // to the second job whenever there is one. + const jobOf = (at: number): string | null => + state === "draft" ? null : twoJobs && at >= 4 ? JOB_B : JOB; const counts = options.settled === true ? SETTLED_COUNTS : BATCH_COUNTS; const settledStates = options.settled === true; // The job moves as the requests land, for the same reason `current` does: a @@ -210,20 +255,28 @@ async function serveApi(page: Page, sent: Request[], options: Options = {}): Pro if (path === `/batches/${BATCH}/jobs`) { return route.fulfill({ json: { - items: [ - { - id: JOB, - batch_id: BATCH, - state: job, - asset_count: 48, - allowed_actions: jobActions(job), - assignee: null, - }, - ], - total: 1, + items: roster.map((id) => ({ + id, + batch_id: BATCH, + state: job, + asset_count: twoJobs ? 4 : 48, + allowed_actions: jobActions(job), + assignee: null, + pre_label_run: null, + })), + total: roster.length, } satisfies Wire["JobPage"], }); } + // The accordion reads every job's counts before it opens one, and the open + // panel's segment chips are that job's rather than the batch's — so this is + // per job, and with two jobs it is each one's own half. + if (request.method() === "GET" && /^\/jobs\/[^/]+\/progress$/.test(path)) { + const id = path.split("/")[2] as string; + return route.fulfill({ + json: (twoJobs ? (SPLIT_COUNTS[id] ?? counts) : counts) satisfies Wire["ProgressCounts"], + }); + } if (request.method() === "POST" && path === `/jobs/${JOB}/start`) { job = "in_progress"; return route.fulfill({ @@ -234,6 +287,7 @@ async function serveApi(page: Page, sent: Request[], options: Options = {}): Pro asset_count: 48, allowed_actions: jobActions(job), assignee: null, + pre_label_run: null, } satisfies Wire["JobOut"], }); } @@ -256,6 +310,7 @@ async function serveApi(page: Page, sent: Request[], options: Options = {}): Pro asset_count: 48, allowed_actions: jobActions(job), assignee: null, + pre_label_run: null, } satisfies Wire["JobOut"], }); } @@ -362,12 +417,16 @@ async function serveApi(page: Page, sent: Request[], options: Options = {}): Pro if (path === `/batches/${BATCH}/assets`) { // `current`, not `state`: an approve during the test moves it, and the // frames' declarations move with the batch exactly as the server's would. - const page_ = assets(jobId, settledStates, current, removed); + const page_ = assets(jobOf, settledStates, current, removed); // The segment toolbar is server-side now: `progress` repeats per state and - // narrows the page, exactly as `BatchService.asset_page` does. - const wanted = new URL(request.url()).searchParams.getAll("progress"); - const items = - wanted.length === 0 ? page_.items : page_.items.filter((one) => wanted.includes(one.progress ?? "")); + // narrows the page, exactly as `BatchService.asset_page` does — and so is + // the accordion's `job`, which keeps only the frames that job carries. + const params = new URL(request.url()).searchParams; + const wanted = params.getAll("progress"); + const forJob = params.get("job"); + const items = page_.items + .filter((one) => forJob === null || one.job_id === forJob) + .filter((one) => wanted.length === 0 || wanted.includes(one.progress ?? "")); return route.fulfill({ json: { total: items.length, items } satisfies Wire["BatchAssetPage"] }); } if (path === `/sources/${SOURCE}`) { @@ -424,6 +483,11 @@ async function openGallery(page: Page, sent: Request[], options: Options = {}): await page.getByTestId("token-input").fill("a-token"); await page.getByTestId("token-submit").click(); await expect(page.getByTestId("gallery-grid")).toBeVisible(); + // And it has frames in it. Inside a job panel the grid is two round trips + // ahead of its own rows — the jobs, then every job's counts, then the frames — + // so an empty grid is a normal intermediate state, and the tests that read the + // DOM in one `evaluate` with nothing to retry would sample it. + await expect(page.getByTestId("gallery-row-0")).toBeVisible(); } // --- the layout change, which is why this file exists ------------------------ @@ -668,6 +732,32 @@ test("the chosen density survives a reload", async ({ page }) => { await expect(page.getByTestId("density")).toHaveValue("0"); }); +// --- the accordion ----------------------------------------------------------- + +test("a two-job batch shows only the open job's tiles, and opening the other header swaps them", async ({ + page, +}) => { + const sent: Request[] = []; + await openGallery(page, sent, { twoJobs: true }); + + // Exactly one open. Two panels at once would be the batch-wide grid again, one + // indent further in — the screen with two truths for the same frames. + await expect(page.getByTestId(/^job-header-/)).toHaveCount(2); + await expect(page.locator('[data-testid^="job-header-"][aria-expanded="true"]')).toHaveCount(1); + + // The first job is the one with work left, so it is the one that opens. + await expect(page.getByTestId(`job-header-${JOB}`)).toHaveAttribute("aria-expanded", "true"); + await expect(page.getByTestId(`job-panel-${JOB}`).getByTestId("tile-asset-0")).toBeVisible(); + await expect(page.getByTestId("tile-asset-4")).toHaveCount(0); + + await page.getByTestId(`job-header-${JOB_B}`).click(); + + await expect(page.getByTestId(`job-panel-${JOB_B}`).getByTestId("tile-asset-4")).toBeVisible(); + await expect(page.getByTestId("tile-asset-0")).toHaveCount(0); + await expect(page.getByTestId(`job-panel-${JOB}`)).toHaveCount(0); + await expect(page.getByTestId(`job-header-${JOB}`)).toHaveAttribute("aria-expanded", "false"); +}); + // --- approval ---------------------------------------------------------------- test("a draft batch can be approved from the view it is a dead end in", async ({ page }) => { diff --git a/frontend/app/e2e/navigation.spec.ts b/frontend/app/e2e/navigation.spec.ts index 0fb3b4fd..576ee4ea 100644 --- a/frontend/app/e2e/navigation.spec.ts +++ b/frontend/app/e2e/navigation.spec.ts @@ -110,6 +110,7 @@ async function serveApi(page: Page): Promise { asset_count: 1, allowed_actions: jobActions("in_progress"), assignee: null, + pre_label_run: null, } satisfies Wire["JobOut"], }); } diff --git a/frontend/app/e2e/project-nav.spec.ts b/frontend/app/e2e/project-nav.spec.ts index cce75f25..56d983e9 100644 --- a/frontend/app/e2e/project-nav.spec.ts +++ b/frontend/app/e2e/project-nav.spec.ts @@ -16,6 +16,7 @@ const PROJECT = "11111111-1111-4111-8111-111111111111"; const BATCH = "22222222-2222-4222-8222-222222222222"; const DATASET = "44444444-4444-4444-8444-444444444444"; const JOB = "33333333-3333-4333-8333-333333333333"; +const OTHER_JOB = "55555555-5555-4555-8555-555555555555"; const NO_PROGRESS = { unannotated: 0, @@ -53,8 +54,24 @@ const PIXEL = Buffer.from( "base64", ); -/** One project, one open batch, one job — enough for every route this suite visits. */ -async function serveApi(page: Page): Promise { +/** A panel of the open batch's jobs accordion, waiting to be started. */ +function pendingJob(id: string): Wire["JobOut"] { + return { + id, + batch_id: BATCH, + state: "pending", + assignee: null, + asset_count: 3, + allowed_actions: jobActions("pending"), + pre_label_run: null, + }; +} + +/** + * One project, one open batch, and the jobs cut from it — enough for every route + * this suite visits. The job ids decide where the `Annotate` control lands. + */ +async function serveApi(page: Page, jobs: readonly string[] = [JOB]): Promise { await page.route("**/api/**", (route) => { const path = new URL(route.request().url()).pathname.replace(/^\/api/, ""); @@ -116,10 +133,18 @@ async function serveApi(page: Page): Promise { asset_count: 1, allowed_actions: jobActions("in_progress"), assignee: null, + pre_label_run: null, } satisfies Wire["JobOut"], }); } - if (path === `/jobs/${JOB}/progress`) { + if (path === `/batches/${BATCH}/jobs`) { + return route.fulfill({ + json: { items: jobs.map(pendingJob), total: jobs.length } satisfies Wire["JobPage"], + }); + } + // Every job in the roster, not only the one the annotator opens: the + // accordion holds its panels shut until all of their counts have answered. + if (/^\/jobs\/[^/]+\/progress$/.test(path)) { return route.fulfill({ json: { ...NO_PROGRESS, unannotated: 1, total: 1 } satisfies Wire["ProgressCounts"], }); @@ -168,6 +193,26 @@ async function openCold(page: Page, url: string): Promise { await page.getByTestId("token-submit").click(); } +/** + * The same cold open, held until the open batch's jobs have been answered. + * + * Where the `Annotate` control lands is decided by that answer, and its absence + * is not a failure but the fallback — the gallery. So a test that clicks before + * it arrives proves nothing either way. Nothing is fetched until the token is + * accepted, which is why the wait can be armed before the submit. + */ +async function openWithJobs(page: Page, url: string, jobs: readonly string[]): Promise { + await serveApi(page, jobs); + await page.goto(url); + await page.getByTestId("token-input").fill("a-token"); + const answered = page.waitForResponse((response) => + new URL(response.url()).pathname.endsWith(`/batches/${BATCH}/jobs`), + ); + await page.getByTestId("token-submit").click(); + await answered; + await expect(page.getByTestId("go-annotate")).toBeVisible(); +} + const SECTIONS = ["overview", "schema", "batches", "dataset"] as const; /** @@ -225,6 +270,27 @@ for (const section of SECTIONS) { }); } +test("the filled control opens the open batch's one job", async ({ page }) => { + await openWithJobs(page, `/projects/${PROJECT}/overview`, [JOB]); + await page.getByTestId("go-annotate").click(); + await expect(page.getByTestId("annotation-page")).toBeVisible(); + // The job is the identity; the control names no frame, and the annotator then + // writes the frame it opened on into the address, as it does after any walk. + await expect(page).toHaveURL(new RegExp(`/jobs/${JOB}(\\?asset=[^&]+)?$`)); +}); + +test("with more than one job the filled control lands on the gallery, where a job is chosen", async ({ + page, +}) => { + await openWithJobs(page, `/projects/${PROJECT}/overview`, [JOB, OTHER_JOB]); + await page.getByTestId("go-annotate").click(); + await expect(page).toHaveURL(new RegExp(`/projects/${PROJECT}/batches/${BATCH}$`)); + await expect(page.getByTestId("gallery")).toBeVisible(); + // Both jobs are offered, one panel each; only the open one carries a door, + // which is why the count is of headers rather than of `start-job-*`. + await expect(page.getByTestId("job-panels").getByTestId(/^job-header-/)).toHaveCount(2); +}); + test("the column frames the ingest flow and the batch gallery too, without a filled control of its own", async ({ page, }) => { diff --git a/frontend/app/e2e/viewport.spec.ts b/frontend/app/e2e/viewport.spec.ts index 30b5d417..38069a3d 100644 --- a/frontend/app/e2e/viewport.spec.ts +++ b/frontend/app/e2e/viewport.spec.ts @@ -92,6 +92,7 @@ async function serveApi(page: Page): Promise { asset_count: 1, allowed_actions: jobActions("in_progress"), assignee: null, + pre_label_run: null, } satisfies Wire["JobOut"], }); } diff --git a/frontend/app/src/routes.tsx b/frontend/app/src/routes.tsx index 928be368..2e9475e1 100644 --- a/frontend/app/src/routes.tsx +++ b/frontend/app/src/routes.tsx @@ -295,6 +295,7 @@ function Project(): JSX.Element { hrefFor={(next) => PARENT.section(projectId, next)} onIngest={() => void navigate(`/projects/${projectId}/ingest`)} onOpenBatch={(batchId) => void navigate(`/projects/${projectId}/batches/${batchId}`)} + onOpenJob={(jobId) => void navigate(`/jobs/${jobId}`)} // A deleted project's own URL is a 404 waiting to happen, so the parent is // where to land — and `replace`, because Back should not walk into it. onDeleted={() => void navigate(PARENT.projects, { replace: true })} @@ -318,6 +319,9 @@ function Project(): JSX.Element { * The asset travels as a query parameter, not a path segment: `/jobs/:jobId` is the * annotator's identity and the asset is *where to start*, which a person can change * with the next/previous buttons without the URL becoming a lie. + * + * `onOpenJob` is the job row's door: the job is the identity and no `?asset=` is + * named, so the annotator opens on the job's first frame. */ function Gallery(): JSX.Element { const { projectId, batchId } = useParams(); @@ -336,6 +340,7 @@ function Gallery(): JSX.Element { if (asset.job_id === null || asset.job_id === undefined) return; void navigate(`/jobs/${asset.job_id}?asset=${asset.id}`); }} + onOpenJob={(jobId) => void navigate(`/jobs/${jobId}`)} // The approve dialog's SCHEMA_NOT_FOUND remedy: the schema section of // the project, and spelling that URL is this file's job. onOpenSchema={() => void navigate(PARENT.schema(projectId))} diff --git a/frontend/ui-core/src/annotator/addClassProvenance.test.tsx b/frontend/ui-core/src/annotator/addClassProvenance.test.tsx index 61115f1d..2440db8d 100644 --- a/frontend/ui-core/src/annotator/addClassProvenance.test.tsx +++ b/frontend/ui-core/src/annotator/addClassProvenance.test.tsx @@ -113,6 +113,7 @@ function answer(path: string): unknown { asset_count: 1, allowed_actions: jobActions("in_progress", { settled: false }), assignee: null, + pre_label_run: null, }; } if (path === `/batches/${BATCH}`) return batch(); diff --git a/frontend/ui-core/src/annotator/drawingClass.test.tsx b/frontend/ui-core/src/annotator/drawingClass.test.tsx index b1c24fab..0e0dde76 100644 --- a/frontend/ui-core/src/annotator/drawingClass.test.tsx +++ b/frontend/ui-core/src/annotator/drawingClass.test.tsx @@ -78,6 +78,7 @@ function answer(path: string): unknown { asset_count: 2, allowed_actions: jobActions("in_progress", { settled: false }), assignee: null, + pre_label_run: null, }; } if (path === `/batches/${BATCH}`) { diff --git a/frontend/ui-core/src/annotator/editorNotice.test.tsx b/frontend/ui-core/src/annotator/editorNotice.test.tsx index 16ff27d5..bc22dac0 100644 --- a/frontend/ui-core/src/annotator/editorNotice.test.tsx +++ b/frontend/ui-core/src/annotator/editorNotice.test.tsx @@ -57,6 +57,7 @@ function answer(path: string): unknown { asset_count: 1, allowed_actions: jobActions("in_progress", { settled: false }), assignee: null, + pre_label_run: null, }; } if (path === `/batches/${BATCH}`) { diff --git a/frontend/ui-core/src/annotator/frameGallery.test.tsx b/frontend/ui-core/src/annotator/frameGallery.test.tsx index b8fe31d2..e6caa839 100644 --- a/frontend/ui-core/src/annotator/frameGallery.test.tsx +++ b/frontend/ui-core/src/annotator/frameGallery.test.tsx @@ -75,6 +75,7 @@ function answer(path: string): unknown { asset_count: FRAMES.length, allowed_actions: jobActions("in_progress", { settled: false }), assignee: null, + pre_label_run: null, }; } if (path === `/batches/${BATCH}`) { diff --git a/frontend/ui-core/src/annotator/jobQueries.ts b/frontend/ui-core/src/annotator/jobQueries.ts index 1f2ab8cd..218655c4 100644 --- a/frontend/ui-core/src/annotator/jobQueries.ts +++ b/frontend/ui-core/src/annotator/jobQueries.ts @@ -123,13 +123,15 @@ export function useJob(jobId: string): UseQueryResult { }); } -export function useJobProgress(jobId: string): UseQueryResult { +/** `null` is "no job to ask about yet" — a dialog that has not been opened. */ +export function useJobProgress(jobId: string | null): UseQueryResult { const client = useApiClient(); return useQuery({ - queryKey: jobKeys.progress(jobId), + queryKey: jobKeys.progress(jobId ?? "none"), + enabled: jobId !== null, queryFn: async () => unwrap( - await client.GET("/jobs/{job_id}/progress", { params: { path: { job_id: jobId } } }), + await client.GET("/jobs/{job_id}/progress", { params: { path: { job_id: jobId ?? "" } } }), checkGetJobProgress, ), }); diff --git a/frontend/ui-core/src/annotator/pinBadge.test.tsx b/frontend/ui-core/src/annotator/pinBadge.test.tsx index 977d8bb1..f263a056 100644 --- a/frontend/ui-core/src/annotator/pinBadge.test.tsx +++ b/frontend/ui-core/src/annotator/pinBadge.test.tsx @@ -56,6 +56,7 @@ function answer(path: string, search: string): unknown { asset_count: 1, allowed_actions: jobActions("in_progress", { settled: false }), assignee: null, + pre_label_run: null, }; } if (path === `/batches/${BATCH}`) { diff --git a/frontend/ui-core/src/annotator/suggestFlow.test.tsx b/frontend/ui-core/src/annotator/suggestFlow.test.tsx index d4c4706e..5ba63ec6 100644 --- a/frontend/ui-core/src/annotator/suggestFlow.test.tsx +++ b/frontend/ui-core/src/annotator/suggestFlow.test.tsx @@ -183,6 +183,7 @@ function answer(path: string): unknown { asset_count: 2, allowed_actions: jobActions("in_progress", { settled: false }), assignee: null, + pre_label_run: null, }; } if (path === `/batches/${BATCH}`) { diff --git a/frontend/ui-core/src/annotator/topBar.test.tsx b/frontend/ui-core/src/annotator/topBar.test.tsx index 601ac013..2abb23f7 100644 --- a/frontend/ui-core/src/annotator/topBar.test.tsx +++ b/frontend/ui-core/src/annotator/topBar.test.tsx @@ -120,6 +120,7 @@ function answer(path: string): unknown { settled: jobSettled, }), assignee: null, + pre_label_run: null, }; } if (path === `/batches/${BATCH}`) { diff --git a/frontend/ui-core/src/annotator/viewportFloor.test.tsx b/frontend/ui-core/src/annotator/viewportFloor.test.tsx index 2392c116..f27822aa 100644 --- a/frontend/ui-core/src/annotator/viewportFloor.test.tsx +++ b/frontend/ui-core/src/annotator/viewportFloor.test.tsx @@ -59,6 +59,7 @@ beforeEach(() => { asset_count: 1, allowed_actions: jobActions("in_progress", { settled: false }), assignee: null, + pre_label_run: null, } : path === `/batches/${BATCH}` ? { diff --git a/frontend/ui-core/src/data/capabilities.test.ts b/frontend/ui-core/src/data/capabilities.test.ts index 0b70dc92..f438b5bb 100644 --- a/frontend/ui-core/src/data/capabilities.test.ts +++ b/frontend/ui-core/src/data/capabilities.test.ts @@ -126,7 +126,7 @@ describe("the action names the client imports", () => { "start", ].sort(), ); - expect(Object.values(JOB_ACTION).sort()).toEqual(["complete", "start"].sort()); + expect(Object.values(JOB_ACTION).sort()).toEqual(["complete", "pre_label", "start"].sort()); expect(Object.values(ASSET_ACTION).sort()).toEqual( [ "accept", diff --git a/frontend/ui-core/src/data/capabilities.ts b/frontend/ui-core/src/data/capabilities.ts index e18bcf1f..04b0d953 100644 --- a/frontend/ui-core/src/data/capabilities.ts +++ b/frontend/ui-core/src/data/capabilities.ts @@ -79,6 +79,7 @@ export const BATCH_ACTION = { export const JOB_ACTION = { start: "start", + preLabel: "pre_label", complete: "complete", } as const satisfies Record; diff --git a/frontend/ui-core/src/patterns/ProjectNav.tsx b/frontend/ui-core/src/patterns/ProjectNav.tsx index a5853501..8150ceca 100644 --- a/frontend/ui-core/src/patterns/ProjectNav.tsx +++ b/frontend/ui-core/src/patterns/ProjectNav.tsx @@ -29,14 +29,15 @@ * ## The one filled control * * Annotate is the project's forward action, and it opens the batch that is - * currently `in_annotation`. With none open there is nowhere to send anybody, - * so the button is absent rather than grey and **Ingest takes the slot** — the - * honest next step. With two or more open it reads `Annotate ▾` and asks which, - * because the batch you pick decides which schema version you annotate under - * and a silent default would be a choice nobody made. A section whose own - * content holds the page's filled control says so through - * `contentOwnsTheAction`, and Ingest steps back to `secondary` for as long as - * that holds, so the page never shows two. + * currently `in_annotation` — straight into its one job when it has exactly + * one, and onto the gallery to pick a job otherwise. With none open there is + * nowhere to send anybody, so the button is absent rather than grey and + * **Ingest takes the slot** — the honest next step. With two or more open it + * reads `Annotate ▾` and asks which, because the batch you pick decides which + * schema version you annotate under and a silent default would be a choice + * nobody made. A section whose own content holds the page's filled control + * says so through `contentOwnsTheAction`, and Ingest steps back to `secondary` + * for as long as that holds, so the page never shows two. * * ## The rail it is not * @@ -113,6 +114,14 @@ export interface AnnotateTarget { readonly remaining: number; /** The schema version the batch is pinned to; null only for a row the wire left unpinned. */ readonly schemaVersion: number | null; + /** The batch's job ids, or undefined while unknown — unknown lands on the gallery. */ + readonly jobIds?: readonly string[] | undefined; +} + +/** Exactly one job → straight into it; otherwise the gallery, where a job is chosen. */ +export function destinationOf(target: AnnotateTarget): { kind: "job"; id: string } | { kind: "batch"; id: string } { + const [only] = target.jobIds ?? []; + return target.jobIds?.length === 1 && only !== undefined ? { kind: "job", id: only } : { kind: "batch", id: target.id }; } export interface ProjectNavProps { @@ -128,6 +137,7 @@ export interface ProjectNavProps { readonly annotate?: { readonly targets: readonly AnnotateTarget[]; readonly onOpen: (batchId: string) => void; + readonly onOpenJob?: (jobId: string) => void; }; readonly onIngest?: () => void; /** The open section's content holds the page's filled control, so Ingest steps back. */ @@ -230,7 +240,14 @@ function Strip(props: ProjectNavProps): JSX.Element { function Cta({ annotate, onIngest, contentOwnsTheAction = false, layout }: ProjectNavProps): JSX.Element | null { const wide = layout === "column" ? "w-full" : undefined; if (annotate !== undefined && annotate.targets.length > 0) { - return ; + return ( + + ); } if (onIngest === undefined) return null; return ( @@ -256,16 +273,24 @@ function Cta({ annotate, onIngest, contentOwnsTheAction = false, layout }: Proje function AnnotateAction({ targets, onOpen, + onOpenJob, className, }: { readonly targets: readonly AnnotateTarget[]; readonly onOpen: (batchId: string) => void; + readonly onOpenJob?: (jobId: string) => void; readonly className?: string | undefined; }): JSX.Element { + function go(target: AnnotateTarget): void { + const to = destinationOf(target); + if (to.kind === "job" && onOpenJob !== undefined) onOpenJob(to.id); + else onOpen(target.id); + } + const [only] = targets; if (targets.length === 1 && only !== undefined) { return ( - @@ -287,7 +312,7 @@ function AnnotateAction({ onOpen(batch.id)} + onSelect={() => go(batch)} >
{batch.name} diff --git a/frontend/ui-core/src/patterns/projectNav.test.tsx b/frontend/ui-core/src/patterns/projectNav.test.tsx index 0aa9b18e..75cdeeb1 100644 --- a/frontend/ui-core/src/patterns/projectNav.test.tsx +++ b/frontend/ui-core/src/patterns/projectNav.test.tsx @@ -25,7 +25,7 @@ function props(overrides: Partial = {}): ProjectNavProps { hrefFor: (section) => `/projects/p/${section}`, onNavigate: vi.fn(), annotate: { - targets: [{ id: "b1", name: "drive-01", remaining: 12, schemaVersion: 4 }], + targets: [{ id: "b1", name: "drive-01", remaining: 12, schemaVersion: 4, jobIds: ["j1"] }], onOpen: vi.fn(), }, onIngest: vi.fn(), @@ -135,6 +135,72 @@ describe("ProjectNav", () => { expect(onOpen).toHaveBeenCalledWith("b1"); }); + it("jumps into the annotator when the one open batch has one job", async () => { + const onOpen = vi.fn(); + const onOpenJob = vi.fn(); + render( + , + ); + await userEvent.click(screen.getByTestId("go-annotate")); + expect(onOpenJob).toHaveBeenCalledWith("j1"); + expect(onOpen).not.toHaveBeenCalled(); + }); + + it("lands on the gallery to pick a job when the batch has several, or while they are unknown", async () => { + for (const jobIds of [["j1", "j2"], undefined]) { + const onOpen = vi.fn(); + const onOpenJob = vi.fn(); + const { unmount } = render( + , + ); + await userEvent.click(screen.getByTestId("go-annotate")); + expect(onOpen).toHaveBeenCalledWith("b1"); + expect(onOpenJob).not.toHaveBeenCalled(); + unmount(); + } + }); + + it("applies the job rule to the batch picked from the dropdown", async () => { + const onOpen = vi.fn(); + const onOpenJob = vi.fn(); + render( + , + ); + await userEvent.click(screen.getByTestId("go-annotate")); + await userEvent.click(await screen.findByTestId("annotate-batch-drive-02")); + expect(onOpenJob).toHaveBeenCalledWith("j2"); + await userEvent.click(screen.getByTestId("go-annotate")); + await userEvent.click(await screen.findByTestId("annotate-batch-drive-01")); + expect(onOpen).toHaveBeenCalledWith("b1"); + }); + it("keeps rename and delete behind the overflow", async () => { const onRename = vi.fn(); render(); diff --git a/frontend/ui-core/src/screens/BatchLifecycle.tsx b/frontend/ui-core/src/screens/BatchLifecycle.tsx index 75acbf04..ff03f71b 100644 --- a/frontend/ui-core/src/screens/BatchLifecycle.tsx +++ b/frontend/ui-core/src/screens/BatchLifecycle.tsx @@ -23,7 +23,7 @@ */ import { useState, type JSX } from "react"; -import { Play, SquareCheck } from "lucide-react"; +import { SquareCheck } from "lucide-react"; import { asApiError } from "../data/errors"; import { refusalProse } from "../data/refusals"; @@ -333,9 +333,8 @@ export function ApproveDialog({ * gallery send the identical mutation rather than two spellings of it. * * Deliberately does not navigate. Landing back on the gallery re-reads the - * batch as `in_annotation`, and the header then offers Pre-label beside Open - * annotator — the choice a batch of any size is worth making explicitly, not - * one a jump straight into the annotator would skip past. + * batch as `in_annotation`, and the job panels then offer Start annotating and + * Pre-label per job. */ export function StartAnnotatingButton({ batch, @@ -356,7 +355,6 @@ export function StartAnnotatingButton({ disabled={start.isPending} onClick={() => start.mutate()} > -
diff --git a/frontend/ui-core/src/screens/ProjectFrame.tsx b/frontend/ui-core/src/screens/ProjectFrame.tsx index 6d1eb94f..4d093237 100644 --- a/frontend/ui-core/src/screens/ProjectFrame.tsx +++ b/frontend/ui-core/src/screens/ProjectFrame.tsx @@ -16,7 +16,7 @@ * second filled control beside it would be two answers to "what now?". */ -import { useState, type JSX, type ReactNode } from "react"; +import { useMemo, useState, type JSX, type ReactNode } from "react"; import { refusalProse } from "../data/refusals"; import { asApiError } from "../data/errors"; @@ -38,10 +38,12 @@ import { useActiveSchema, useBatches, useDeleteProject, + useJobsOfBatches, useProject, useProjectStats, useRenameProject, type Batch, + type Job, } from "./queries"; import type { FormEvent } from "react"; @@ -68,6 +70,7 @@ export interface ProjectFrameProps { */ readonly cta?: { readonly onOpenBatch?: (batchId: string) => void; + readonly onOpenJob?: (jobId: string) => void; readonly onIngest?: () => void; readonly contentOwnsTheAction?: boolean; }; @@ -92,7 +95,10 @@ export interface ProjectFrameProps { * direction. The copy is not decoration — the array belongs to the query cache, * and `reverse` mutates in place. */ -export function openForAnnotation(batches: readonly Batch[] | undefined): readonly AnnotateTarget[] { +export function openForAnnotation( + batches: readonly Batch[] | undefined, + jobs?: ReadonlyMap, +): readonly AnnotateTarget[] { return [...(batches ?? [])] .filter((batch) => batch.state === "in_annotation") .reverse() @@ -101,6 +107,7 @@ export function openForAnnotation(batches: readonly Batch[] | undefined): readon name: batch.name, remaining: batch.progress.unannotated, schemaVersion: batch.schema_version ?? null, + jobIds: jobs?.get(batch.id)?.map((job) => job.id), })); } @@ -121,14 +128,22 @@ export function ProjectFrame({ const [renaming, setRenaming] = useState(false); const [deleting, setDeleting] = useState(false); - const open = cta?.onOpenBatch === undefined ? [] : openForAnnotation(batches.data?.items); + // Hooks cannot be conditional, so this always runs — with an empty list, and + // therefore no request, whenever the host wired no `onOpenJob` to apply the + // job rule with. + const openIds = useMemo( + () => (batches.data?.items ?? []).filter((batch) => batch.state === "in_annotation").map((batch) => batch.id), + [batches.data], + ); + const jobs = useJobsOfBatches(cta?.onOpenJob === undefined ? [] : openIds); + const open = cta?.onOpenBatch === undefined ? [] : openForAnnotation(batches.data?.items, jobs); const nav: ProjectNavData = { sections, active, onNavigate, ...(hrefFor === undefined ? {} : { hrefFor }), ...(open.length > 0 && cta?.onOpenBatch !== undefined - ? { annotate: { targets: open, onOpen: cta.onOpenBatch } } + ? { annotate: { targets: open, onOpen: cta.onOpenBatch, ...(cta.onOpenJob === undefined ? {} : { onOpenJob: cta.onOpenJob }) } } : {}), ...(cta?.onIngest === undefined ? {} : { onIngest: cta.onIngest }), contentOwnsTheAction: cta?.contentOwnsTheAction ?? false, diff --git a/frontend/ui-core/src/screens/ProjectPreLabelDialog.tsx b/frontend/ui-core/src/screens/ProjectPreLabelDialog.tsx index bb390b1f..1cf530d1 100644 --- a/frontend/ui-core/src/screens/ProjectPreLabelDialog.tsx +++ b/frontend/ui-core/src/screens/ProjectPreLabelDialog.tsx @@ -43,9 +43,29 @@ import { usePreLabelProject, type Batch, type GeometryType, - type ProjectPreLabelOut, + type PreLabelFanOutOut, } from "./queries"; +/** + * One row per job. A batch that fanned out to several jobs would otherwise + * repeat its name over indistinguishable rows, so a row past the first for + * that batch is suffixed by its 1-based position among them. + */ +function resultRows( + items: PreLabelFanOutOut["items"], +): readonly { item: PreLabelFanOutOut["items"][number]; label: string }[] { + const counts = new Map(); + for (const item of items) counts.set(item.batch_id, (counts.get(item.batch_id) ?? 0) + 1); + const seen = new Map(); + return items.map((item) => { + const position = (seen.get(item.batch_id) ?? 0) + 1; + seen.set(item.batch_id, position); + const label = + (counts.get(item.batch_id) ?? 0) > 1 ? `${item.batch_name} · job ${position}` : item.batch_name; + return { item, label }; + }); +} + export interface ProjectPreLabelButtonProps { readonly projectId: string; readonly batches: readonly Batch[]; @@ -111,7 +131,7 @@ function ProjectPreLabelDialog({ const [checked, setChecked] = useState>( () => new Set(batches.filter((one) => one.progress.unannotated > 0).map((one) => one.id)), ); - const [result, setResult] = useState(null); + const [result, setResult] = useState(null); const launch = usePreLabelProject(projectId); const active = candidates.find((row) => row.id === connectionId) ?? candidates[0]; const shapes = active?.produces ?? []; @@ -249,13 +269,13 @@ function ProjectPreLabelDialog({ ) : (
    - {result.items.map((item) => ( -
  • + {resultRows(result.items).map(({ item, label }) => ( +
  • {item.joined ? "already running — joined" : "queued"} diff --git a/frontend/ui-core/src/screens/ProjectScreen.tsx b/frontend/ui-core/src/screens/ProjectScreen.tsx index 42ec78f7..124d53ed 100644 --- a/frontend/ui-core/src/screens/ProjectScreen.tsx +++ b/frontend/ui-core/src/screens/ProjectScreen.tsx @@ -149,6 +149,7 @@ export interface ProjectScreenProps { /** Route changes, supplied by the app. See `ProjectsScreen`'s note. */ readonly onIngest?: () => void; readonly onOpenBatch?: (batchId: string) => void; + readonly onOpenJob?: (jobId: string) => void; /** * Where to go once the project is gone. Absent means the overflow menu still * deletes, and the caller is left on a screen whose subject no longer exists — @@ -170,6 +171,7 @@ export function ProjectScreen({ projectId, onIngest, onOpenBatch, + onOpenJob, onDeleted, tab, onTabChange, @@ -456,6 +458,7 @@ export function ProjectScreen({ chain="frame" cta={{ ...(onOpenBatch === undefined ? {} : { onOpenBatch }), + ...(onOpenJob === undefined ? {} : { onOpenJob }), ...(onIngest === undefined ? {} : { onIngest }), contentOwnsTheAction: overviewOwnsTheAction, }} diff --git a/frontend/ui-core/src/screens/gallery.test.tsx b/frontend/ui-core/src/screens/gallery.test.tsx index d7cc10bd..8acdff7f 100644 --- a/frontend/ui-core/src/screens/gallery.test.tsx +++ b/frontend/ui-core/src/screens/gallery.test.tsx @@ -130,6 +130,54 @@ function batch(overrides: Record = {}): Record }; } +/** + * The jobs a non-draft batch has, and each one's counts. + * + * Every fixture past `draft` needs both, because the gallery shows a batch's + * frames **inside a job**: with no roster and no per-job progress there is no + * open panel, and therefore no grid, no segments and no timeline to assert + * against. The counts are the *job's* — the segment chips read them, not the + * batch's. + */ +function oneJob( + options: { + state?: JobState; + batchState?: BatchState; + assetCount?: number; + assignee?: string | null; + counts?: Record; + } = {}, +): void { + const { + state = "in_progress", + batchState = "in_annotation", + assetCount = 3, + assignee = null, + counts = {}, + } = options; + on("GET", /\/jobs$/, { + status: 200, + body: { + items: [ + { + id: JOB, + batch_id: BATCH, + state, + asset_count: assetCount, + assignee, + pre_label_run: null, + allowed_actions: jobActions(state, { batchState }), + }, + ], + total: 1, + }, + }); + on("GET", /\/jobs\/[^/]+\/progress$/, { + status: 200, + body: { ...NO_PROGRESS, total: assetCount, unannotated: assetCount, ...counts }, + }); +} + describe("the batch table", () => { it("shows one action per state, and the last one is promote", async () => { on("GET", /\/batches$/, { @@ -348,6 +396,7 @@ describe("the gallery", () => { } it("asks for the first window with the page size", async () => { + on("GET", /\/batches\/[^/]+$/, { status: 200, body: batch() }); on("GET", /\/assets$/, { status: 200, body: assets(100, 0, 250) }); render(mount()); @@ -432,6 +481,7 @@ describe("the gallery", () => { // Null means *unknown*, not "never" — an asset ingested before the column // existed is legitimately unstamped, and inventing a date would be worse than // the omission. + on("GET", /\/batches\/[^/]+$/, { status: 200, body: batch() }); on("GET", /\/assets$/, { status: 200, body: { total: 1, items: [asset(0, { ingested_at: null, source_id: null })] }, @@ -472,24 +522,24 @@ describe("the gallery", () => { expect(screen.queryByTestId("approve-batch")).toBeNull(); }); - it("counts the segments off the batch rather than off the loaded page", async () => { + it("counts the segments off the open job rather than off the loaded page", async () => { on("GET", /\/batches\/[^/]+$/, { status: 200, - body: batch({ - state: "in_annotation", - asset_count: 48, - progress: { - total: 48, - unannotated: 30, - pre_labeled: 2, - annotated: 8, - review_pending: 5, - accepted: 1, - skipped: 4, - }, - }), + body: batch({ state: "in_annotation", asset_count: 48 }), + }); + oneJob({ + assetCount: 48, + counts: { + total: 48, + unannotated: 30, + pre_labeled: 2, + annotated: 8, + review_pending: 5, + accepted: 1, + skipped: 4, + }, }); - // Five loaded out of forty-eight: the counts must describe the batch, not the + // Five loaded out of forty-eight: the counts must describe the job, not the // window. A filter whose numbers described the page would be a filter that // lies about the collection it is filtering. on("GET", /\/assets$/, { status: 200, body: mixed() }); @@ -514,6 +564,7 @@ describe("the gallery", () => { body: batch({ state: "in_annotation", schema_version: 1, progress: { ...NO_PROGRESS, total: 3, unannotated: 2, pre_labeled: 1 } }), }); + oneJob({ counts: { total: 3, unannotated: 2, pre_labeled: 1 } }); on("GET", /\/assets$/, { status: 200, body: assets(3) }); render(mount()); await screen.findByTestId("segment-pre_labeled"); @@ -542,6 +593,7 @@ describe("the gallery", () => { body: batch({ state: "in_annotation", schema_version: 1, progress: { ...NO_PROGRESS, total: 2, pre_labeled: 2 } }), }); + oneJob({ assetCount: 2, counts: { total: 2, pre_labeled: 2 } }); on("GET", /\/assets$/, { status: 200, body: assets(2) }); render(mount()); const order = (await screen.findByTestId("sort-order")) as HTMLSelectElement; @@ -557,6 +609,7 @@ describe("the gallery", () => { }); it("shows a draft no sort control, because a draft has no scores", async () => { + on("GET", /\/batches\/[^/]+$/, { status: 200, body: batch() }); on("GET", /\/assets$/, { status: 200, body: assets(2) }); render(mount()); await screen.findByTestId("tile-asset-0"); @@ -569,6 +622,7 @@ describe("the gallery", () => { body: batch({ state: "in_annotation", schema_version: 1, progress: { ...NO_PROGRESS, total: 2, annotated: 1, pre_labeled: 1 } }), }); + oneJob({ assetCount: 2, counts: { total: 2, annotated: 1, pre_labeled: 1 } }); on("GET", /\/assets$/, { status: 200, body: { @@ -594,6 +648,7 @@ describe("the gallery", () => { body: batch({ state: "in_annotation", schema_version: 1, progress: { ...NO_PROGRESS, total: 3, unannotated: 3 } }), }); + oneJob({ counts: { total: 3, unannotated: 3 } }); handlers.push((request) => { const url = new URL(request.url); if (request.method === "GET" && url.pathname.endsWith("/assets")) { @@ -710,13 +765,14 @@ describe("the gallery", () => { status: 200, body: batch({ state: "approved", progress: { ...NO_PROGRESS, total: 5, unannotated: 5 } }), }); + oneJob({ state: "pending", batchState: "approved", assetCount: 5, counts: { total: 5, unannotated: 5 } }); on("GET", /\/assets$/, { status: 200, body: mixed() }); render(mount()); await waitFor(() => expect(screen.queryByTestId("segments")).not.toBeNull()); // The other half of the claim above: hidden *before* approval, not removed. - expect(screen.queryByTestId("timeline")).not.toBeNull(); + expect(await screen.findByTestId("timeline")).not.toBeNull(); expect(screen.queryByTestId("select-asset-0")).not.toBeNull(); expect(screen.queryByTestId("state-asset-0")).not.toBeNull(); }); @@ -729,6 +785,7 @@ describe("the gallery", () => { status: 200, body: batch({ state: "in_annotation", progress: { ...NO_PROGRESS, total: 5, annotated: 5 } }), }); + oneJob({ assetCount: 5, counts: { total: 5, annotated: 5 } }); on("GET", /\/assets$/, { status: 200, body: mixed() }); render(mount()); @@ -761,6 +818,7 @@ describe("the gallery", () => { }); it("keeps the empty state for a batch with nothing in it", async () => { + on("GET", /\/batches\/[^/]+$/, { status: 200, body: batch() }); on("GET", /\/assets$/, { status: 200, body: assets(0, 0, 0) }); render(mount()); await waitFor(() => expect(screen.queryByText("This batch is empty")).not.toBeNull()); @@ -854,6 +912,7 @@ describe("the gallery", () => { // render here for the intended layout. expect(globalThis.ResizeObserver).toBeUndefined(); + on("GET", /\/batches\/[^/]+$/, { status: 200, body: batch() }); on("GET", /\/assets$/, { status: 200, body: assets(6, 0, 6) }); render(mount()); const grid = await screen.findByTestId("gallery-grid"); @@ -861,6 +920,7 @@ describe("the gallery", () => { }); it("has no scrollable box of its own any more", async () => { + on("GET", /\/batches\/[^/]+$/, { status: 200, body: batch() }); on("GET", /\/assets$/, { status: 200, body: assets(6, 0, 6) }); render(mount()); @@ -981,6 +1041,7 @@ describe("finishing a batch", () => { asset_count: 48, allowed_actions: jobActions(state as JobState), assignee: null, + pre_label_run: null, }; } @@ -1192,6 +1253,9 @@ describe("the bulk bar", () => { progress: { ...NO_PROGRESS, total: states.length, unannotated: states.length }, }), }); + // Unused by a draft, which has no jobs and keeps its flat grid — and the one + // thing without which every other state renders no grid at all. + oneJob({ batchState, assetCount: states.length, counts: { total: states.length, unannotated: states.length } }); on("GET", /\/assets$/, { status: 200, body: { total: states.length, items: states.map((one, at) => tile(at, one, batchState)) }, @@ -1368,6 +1432,7 @@ describe("the bulk bar", () => { progress: { ...NO_PROGRESS, total: 2, pre_labeled: 2 }, }), }); + oneJob({ assetCount: 2, counts: { total: 2, pre_labeled: 2 } }); let swept = false; handlers.push((request) => { const url = new URL(request.url); @@ -1416,6 +1481,7 @@ describe("the bulk bar", () => { progress: { ...NO_PROGRESS, total: 2, pre_labeled: 2 }, }), }); + oneJob({ assetCount: 2, counts: { total: 2, pre_labeled: 2 } }); let moved = false; handlers.push((request) => { const url = new URL(request.url); @@ -1587,6 +1653,7 @@ describe("the bulk bar", () => { progress: { ...NO_PROGRESS, total: 2, review_pending: 1, annotated: 1 }, }), }); + oneJob({ assetCount: 2, counts: { total: 2, review_pending: 1, annotated: 1 } }); handlers.push((request) => { const url = new URL(request.url); if (request.method === "GET" && url.pathname.endsWith("/assets")) { @@ -1926,13 +1993,14 @@ describe("the bulk bar", () => { }); /** - * The way into the annotator, which must not close behind you. + * The way into the annotator, and it is a job's panel that holds it. * - * Drawing `Start annotating` only while some frame is `unannotated` leaves a batch - * whose work is finished with no action in its header at all — while the badge - * beside the empty space goes on saying `in progress`. + * A batch's frames are partitioned into jobs, so "start annotating" is a question + * about one job: which frames, and taking them (`start`) before anyone else does. + * The header's single door could answer neither — it guessed a frame and sent no + * mutation at all — so the door moved into the panel that knows. */ -describe("the gallery header's way into the annotator", () => { +describe("the job panel's way into the annotator", () => { function frames(batchState: BatchState, ...states: string[]): Record { return { total: states.length, @@ -1958,12 +2026,9 @@ describe("the gallery header's way into the annotator", () => { }; } - async function open(...states: string[]): Promise> { - return openIn("in_annotation", ...states); - } - - async function openIn( + async function openWith( batchState: BatchState, + jobState: JobState, ...states: string[] ): Promise> { on("GET", /\/batches\/[^/]+$/, { @@ -1976,64 +2041,126 @@ describe("the gallery header's way into the annotator", () => { }); on("GET", /\/assets$/, { status: 200, body: frames(batchState, ...states) }); on("GET", /\/annotations$/, { status: 200, body: [] }); - - const opened = vi.fn(); - render(mount()); + on("GET", /\/jobs$/, { + status: 200, + body: { + items: [ + { + id: JOB, + batch_id: BATCH, + state: jobState, + asset_count: states.length, + assignee: null, + pre_label_run: null, + allowed_actions: jobActions(jobState, { batchState }), + }, + ], + total: 1, + }, + }); + on("GET", /\/jobs\/[^/]+\/progress$/, { + status: 200, + body: { ...NO_PROGRESS, total: states.length, unannotated: states.length }, + }); + + const openedJob = vi.fn(); + render( + mount( + , + ), + ); + // The accordion first: the frames are *inside* a panel now, so there are no + // tiles at all until the job roster and its counts have both landed. + await screen.findByTestId("job-panels"); await screen.findByTestId("tile-asset-0"); - return opened; + return openedJob; } - it("still offers a way in when every frame is settled", async () => { - await open("annotated", "skipped", "skipped"); - // The defect: this was absent, and nothing else in the header offered one. - expect(screen.getByTestId("start-annotating").textContent).toContain("Open annotator"); - }); - - it("opens a skipped frame, which the annotator can un-skip from", async () => { - const opened = await open("skipped", "skipped"); - await userEvent.click(screen.getByTestId("start-annotating")); - // Not filtered out. The annotator lists a job's assets with no progress - // filter and carries `Un-skip` on its toolbar, so a skipped frame is a - // legitimate thing to open — and with everything skipped it is the only thing. - expect(opened).toHaveBeenCalledWith(expect.objectContaining({ id: "asset-0" })); - }); - - it("still starts on the first waiting frame when there is one", async () => { - const opened = await open("annotated", "unannotated", "unannotated"); - expect(screen.getByTestId("start-annotating").textContent).toContain("Start annotating"); - await userEvent.click(screen.getByTestId("start-annotating")); - expect(opened).toHaveBeenCalledWith(expect.objectContaining({ id: "asset-1" })); - }); - - /** - * The third question the label had to start asking (F2). - * - * "Whether there is a frame to open" and "whether any is still waiting" were - * the only two, so a completed batch read `Open annotator` and opened a fully - * live editor whose every save the kernel refuses. The door is the same; the - * word on it is now honest about what is behind it. - */ - it("says View when the frames cannot be written to", async () => { - await openIn("completed", "annotated", "skipped"); - expect(screen.getByTestId("start-annotating").textContent).toContain("View frames"); - expect(screen.getByTestId("start-annotating").textContent).not.toContain("annotator"); + it("offers no door in the header once the batch is open", async () => { + await openWith("in_annotation", "pending", "unannotated"); + expect(screen.queryByTestId("start-annotating")).toBeNull(); + // Pre-label is a job's action now, so every trigger on the page is inside the + // accordion. Asserting a batch-keyed testid is absent would pass on a page + // that still had a header mount, because the testid is keyed by job. + const panels = screen.getByTestId("job-panels"); + const triggers = [...document.querySelectorAll('[data-testid^="pre-label-"]')]; + expect(triggers.length).toBeGreaterThan(0); + expect(triggers.every((node) => panels.contains(node))).toBe(true); }); - it("says View on the tiles too, since a tile is the other door", async () => { - await openIn("completed", "annotated", "skipped"); + it("starts a pending job, then opens it", async () => { + on("POST", /\/jobs\/[^/]+\/start$/, { + status: 200, + body: { + id: JOB, + batch_id: BATCH, + state: "in_progress", + asset_count: 1, + assignee: null, + pre_label_run: null, + allowed_actions: jobActions("in_progress"), + }, + }); + const openedJob = await openWith("in_annotation", "pending", "unannotated"); + const door = screen.getByTestId(`start-job-${JOB}`); + expect(door.textContent).toContain("Annotate"); + expect(door.querySelector("svg")).not.toBeNull(); + expect(door.dataset.variant).toBe("secondary"); + await userEvent.click(door); + await waitFor(() => expect(openedJob).toHaveBeenCalledWith(JOB)); + expect(sent.filter((r) => r.method === "POST").map((r) => new URL(r.url).pathname)).toEqual([ + `/jobs/${JOB}/start`, + ]); + }); + + it("continues an in-progress job without starting it again", async () => { + const openedJob = await openWith("in_annotation", "in_progress", "unannotated"); + const door = screen.getByTestId(`start-job-${JOB}`); + expect(door.textContent).toContain("Continue"); + await userEvent.click(door); + expect(openedJob).toHaveBeenCalledWith(JOB); + expect(sent.some((r) => r.method === "POST")).toBe(false); + }); + + it("says View on a pending job the batch has not started yet", async () => { + // An `approved` batch's jobs are `pending` and declare nothing, so there is + // no taking them from here — and "Continue" over a job nobody has opened is + // the label promising a state that does not exist. + const openedJob = await openWith("approved", "pending", "unannotated"); + const door = screen.getByTestId(`start-job-${JOB}`); + expect(door.textContent).toContain("View"); + await userEvent.click(door); + expect(openedJob).toHaveBeenCalledWith(JOB); + expect(sent.some((r) => r.method === "POST")).toBe(false); + }); + + it("says View on a finished job and on a finished batch", async () => { + await openWith("completed", "completed", "annotated", "skipped"); + expect(screen.getByTestId(`start-job-${JOB}`).textContent).toContain("View"); expect(screen.getByTestId("open-asset-0").textContent).toBe("View"); - expect(screen.getByTestId("open-asset-0").getAttribute("aria-label")).toMatch(/^View frame/); }); - it("still opens, because looking at finished work is the point of the door", async () => { - const opened = await openIn("completed", "annotated", "skipped"); - await userEvent.click(screen.getByTestId("start-annotating")); - expect(opened).toHaveBeenCalledWith(expect.objectContaining({ id: "asset-0" })); + it("shows the refusal when the start is refused, and does not navigate", async () => { + on("POST", /\/jobs\/[^/]+\/start$/, { + status: 409, + body: { code: "BATCH_NOT_IN_ANNOTATION", message: "batch 'drive-01' is 'completed'" }, + }); + const openedJob = await openWith("in_annotation", "pending", "unannotated"); + await userEvent.click(screen.getByTestId(`start-job-${JOB}`)); + + const said = await screen.findByText("This batch is not open for annotation any more."); + expect(said.textContent).not.toContain("BATCH_NOT_IN_ANNOTATION"); + expect(openedJob).not.toHaveBeenCalled(); }); - it("says Open on a batch that can be written to", async () => { - await open("annotated", "skipped"); - expect(screen.getByTestId("open-asset-0").textContent).toBe("Open"); + it("offers Pre-label in the job's panel, gated on the job's declaration", async () => { + await openWith("in_annotation", "pending", "unannotated"); + expect(screen.getByTestId(`pre-label-${JOB}`)).toBeTruthy(); }); /** @@ -2044,14 +2171,14 @@ describe("the gallery header's way into the annotator", () => { * link to the dataset either, which is where a promotion's evidence lives. */ it("offers Promote once the batch is completed", async () => { - await openIn("completed", "annotated", "skipped"); + await openWith("completed", "completed", "annotated", "skipped"); expect(screen.queryByTestId("promote-drive-01")).not.toBeNull(); }); it("offers no Promote before the batch is completed", async () => { // Capability-gated, not state-guessed: `PROMOTABLE_STATES` is the kernel's // and the wire declares it. - await open("annotated", "unannotated"); + await openWith("in_annotation", "in_progress", "annotated", "unannotated"); expect(screen.queryByTestId("promote-drive-01")).toBeNull(); }); }); @@ -2103,6 +2230,7 @@ describe("the gallery header's own next step", () => { }); on("GET", /\/assets$/, { status: 200, body: frames(batchState, ...states) }); on("GET", /\/annotations$/, { status: 200, body: [] }); + oneJob({ batchState, assetCount: states.length, counts: { total: states.length, unannotated: states.length } }); render(mount()); await screen.findByTestId("tile-asset-0"); @@ -2111,21 +2239,26 @@ describe("the gallery header's own next step", () => { it("offers Start annotating on an approved batch, not View frames", async () => { await openIn("approved", "unannotated", "unannotated"); expect(screen.getByTestId("start-batch").textContent).toContain("Start annotating"); - // The per-frame door is not offered beside it — a batch that has not - // started has nothing settled yet for that door to show. expect(screen.queryByTestId("start-annotating")).toBeNull(); }); - it("offers no Start on a completed batch, only View frames", async () => { + it("draws no icon on the approved batch's Start annotating", async () => { + // The header is a row of word-only controls; a glyph on one of them reads as + // a different kind of control rather than as emphasis. + await openIn("approved", "unannotated", "unannotated"); + expect(screen.getByTestId("start-batch").querySelector("svg")).toBeNull(); + }); + + it("offers no Start on a completed batch, and no header door either", async () => { await openIn("completed", "annotated", "skipped"); expect(screen.queryByTestId("start-batch")).toBeNull(); - expect(screen.getByTestId("start-annotating").textContent).toContain("View frames"); + expect(screen.queryByTestId("start-annotating")).toBeNull(); }); - it("offers no Start on an in_annotation batch, only the annotator entry", async () => { + it("offers no Start on an in_annotation batch, whose door is in the job's panel", async () => { await openIn("in_annotation", "annotated", "unannotated", "unannotated"); expect(screen.queryByTestId("start-batch")).toBeNull(); - expect(screen.getByTestId("start-annotating")).toBeTruthy(); + expect(screen.queryByTestId("start-annotating")).toBeNull(); }); it("performs the batch's own start rather than navigating into the annotator", async () => { @@ -2218,7 +2351,7 @@ describe("the gallery header's own next step", () => { }); }); -describe("the jobs strip", () => { +describe("the jobs accordion", () => { const OTHER_JOB = "88888888-8888-4888-8888-888888888888"; function jobRow(assignee: string | null, id: string = JOB) { @@ -2229,16 +2362,32 @@ describe("the jobs strip", () => { asset_count: 3, allowed_actions: jobActions("in_progress"), assignee, + pre_label_run: null, }; } function renderGallery(): void { on("GET", /\/batches\/[^/]+$/, { status: 200, body: batch({ state: "in_annotation" }) }); on("GET", /\/assets$/, { status: 200, body: { items: [], total: 0 } }); + // Registered last, so a test that wants its own answer for a job's counts + // still wins: handlers are consulted in registration order. + on("GET", /\/jobs\/[^/]+\/progress$/, { + status: 200, + body: { ...NO_PROGRESS, total: 3, unannotated: 3 }, + }); render(mount()); } - it("shows each job's assignee, and an Assign control when there is none", async () => { + /** + * The assignee editor is in the **open panel**, not on every row: a collapsed + * header is an overview and names who has the job, and the control that changes + * that is one of the things opening a panel is for. + */ + async function openPanel(): Promise> { + return within(await screen.findByTestId(`job-panel-${JOB}`)); + } + + it("names each job's assignee on its own header, open or not", async () => { handlers.push((request) => { const url = new URL(request.url); if (url.pathname === `/batches/${BATCH}/jobs`) @@ -2252,9 +2401,16 @@ describe("the jobs strip", () => { return undefined; }); renderGallery(); - const strip = within(await screen.findByTestId("jobs-strip")); - expect(strip.getByText("Dana Reyes")).toBeTruthy(); - expect(strip.getByRole("button", { name: "Assign" })).toBeTruthy(); + + // The overview half of the claim: who has a job is legible without opening + // it, and an unassigned one says so rather than saying nothing. + expect((await screen.findByTestId(`job-row-${JOB}`)).textContent).toContain("Dana Reyes"); + expect(screen.getByTestId(`job-row-${OTHER_JOB}`).textContent).toContain("—"); + + // ...and the editor is in whichever panel is open, once. + const panel = await openPanel(); + expect(panel.getByRole("button", { name: "Dana Reyes" })).toBeTruthy(); + expect(screen.queryAllByLabelText(/Assignee for job/)).toHaveLength(0); }); it("says a missing batch in words when the jobs cannot be read", async () => { @@ -2290,10 +2446,10 @@ describe("the jobs strip", () => { return undefined; }); renderGallery(); - const strip = within(await screen.findByTestId("jobs-strip")); - await userEvent.click(strip.getByRole("button", { name: /assign/i })); + const panel = await openPanel(); + await userEvent.click(panel.getByRole("button", { name: /assign/i })); await userEvent.keyboard("Dana Reyes{Enter}"); - const said = await strip.findByRole("alert"); + const said = await panel.findByRole("alert"); expect(said.textContent).toContain("That job is no longer on record."); expect(said.textContent).not.toContain(JOB); expect(said.textContent).not.toContain("JOB_NOT_FOUND"); @@ -2315,13 +2471,13 @@ describe("the jobs strip", () => { return undefined; }); renderGallery(); - const strip = within(await screen.findByTestId("jobs-strip")); - await userEvent.click(strip.getByRole("button", { name: /assign/i })); + const panel = await openPanel(); + await userEvent.click(panel.getByRole("button", { name: /assign/i })); await userEvent.keyboard("Dana Reyes{Enter}"); const put = sent.find((request) => request.method === "PUT"); expect(put).toBeTruthy(); expect(JSON.parse(bodies.get(put!) ?? "")).toEqual({ assignee: "Dana Reyes" }); - expect(await strip.findByText("Dana Reyes")).toBeTruthy(); + expect(await panel.findByText("Dana Reyes")).toBeTruthy(); }); it("commits the typed name on blur, not only on Enter", async () => { @@ -2334,9 +2490,9 @@ describe("the jobs strip", () => { return undefined; }); renderGallery(); - const strip = within(await screen.findByTestId("jobs-strip")); - await userEvent.click(strip.getByRole("button", { name: /assign/i })); - await userEvent.type(strip.getByLabelText(/Assignee for job/), "Dana Reyes"); + const panel = await openPanel(); + await userEvent.click(panel.getByRole("button", { name: /assign/i })); + await userEvent.type(panel.getByLabelText(/Assignee for job/), "Dana Reyes"); await userEvent.tab(); const put = sent.find((request) => request.method === "PUT"); expect(put).toBeTruthy(); @@ -2351,11 +2507,11 @@ describe("the jobs strip", () => { return undefined; }); renderGallery(); - const strip = within(await screen.findByTestId("jobs-strip")); - await userEvent.click(strip.getByRole("button", { name: /assign/i })); - await userEvent.type(strip.getByLabelText(/Assignee for job/), "Dana Reyes"); + const panel = await openPanel(); + await userEvent.click(panel.getByRole("button", { name: /assign/i })); + await userEvent.type(panel.getByLabelText(/Assignee for job/), "Dana Reyes"); await userEvent.keyboard("{Escape}"); - expect(await strip.findByRole("button", { name: "Assign" })).toBeTruthy(); + expect(await panel.findByRole("button", { name: "Assign" })).toBeTruthy(); expect(sent.some((request) => request.method === "PUT")).toBe(false); }); @@ -2367,10 +2523,10 @@ describe("the jobs strip", () => { return undefined; }); renderGallery(); - const strip = within(await screen.findByTestId("jobs-strip")); - await userEvent.click(strip.getByRole("button", { name: /assign/i })); + const panel = await openPanel(); + await userEvent.click(panel.getByRole("button", { name: /assign/i })); await userEvent.tab(); - expect(await strip.findByRole("button", { name: "Assign" })).toBeTruthy(); + expect(await panel.findByRole("button", { name: "Assign" })).toBeTruthy(); expect(sent.some((request) => request.method === "PUT")).toBe(false); }); @@ -2383,6 +2539,6 @@ describe("the jobs strip", () => { }); renderGallery(); expect(await screen.findByText("jobs are unreachable")).toBeTruthy(); - expect(screen.queryByTestId("jobs-strip")).toBeNull(); + expect(screen.queryByTestId("job-panels")).toBeNull(); }); }); diff --git a/frontend/ui-core/src/screens/jobPanels.test.tsx b/frontend/ui-core/src/screens/jobPanels.test.tsx new file mode 100644 index 00000000..a33c7cb5 --- /dev/null +++ b/frontend/ui-core/src/screens/jobPanels.test.tsx @@ -0,0 +1,582 @@ +/** + * The gallery as an accordion of jobs. + * + * The claim under all of these is one sentence: **once a batch has jobs, its + * frames are shown per job, and at most one job's are on screen.** A screen that + * showed the batch's frames beside a strip of jobs had two truths for the same + * pictures — the strip's job and the grid's batch — and a person working one job + * saw everybody else's frames. + * + * The fetch-stub harness is `gallery.test.tsx`'s, reused rather than reinvented: + * `handlers` consulted in registration order, `on()` for a path, `mount()` for the + * provider. What is new here is that `/assets` **answers by the `job` query + * parameter**, because a stub that handed every job the same page could not tell + * a request that carries `job=` from one that does not. + */ + +import { QueryClient } from "@tanstack/react-query"; +import { act, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; +import { userEvent } from "@testing-library/user-event"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { JSX, ReactNode } from "react"; + +import { ApiProvider } from "../data/ApiProvider"; +import { writeToken } from "../data/session"; +import { GalleryScreen } from "./GalleryScreen"; +import { defaultOpenJob } from "./JobPanels"; +import { assetActions, batchActions, jobActions } from "../testing/wire.fixtures.js"; +import type { Job } from "./queries"; +import type { components } from "../generated/api.js"; + +type BatchState = components["schemas"]["BatchState"]; +type JobState = components["schemas"]["AnnotationJobState"]; + +const API = "http://visionset.test"; +const PROJECT = "11111111-1111-4111-8111-111111111111"; +const BATCH = "55555555-5555-4555-8555-555555555555"; +const JOB_A = "77777777-7777-4777-8777-777777777777"; +const JOB_B = "88888888-8888-4888-8888-888888888888"; + +type Answer = { status: number; body?: unknown }; +let handlers: ((request: Request) => Answer | undefined)[] = []; +const sent: Request[] = []; + +const NO_PROGRESS = { + unannotated: 0, + pre_labeled: 0, + annotated: 0, + skipped: 0, + review_pending: 0, + accepted: 0, + total: 0, +}; + +/** Each job's counts, keyed by job id — what `/jobs/{id}/progress` answers. */ +let progress: Map>; + +/** The jobs `/batches/{id}/jobs` answers with. Mutable, so a job can vanish mid-test. */ +let roster: readonly Job[]; + +/** The screen's own client, so a test can force the re-fetch a mutation would. */ +let client: QueryClient; + +beforeEach(() => { + handlers = []; + sent.length = 0; + roster = []; + progress = new Map([ + [JOB_A, { ...NO_PROGRESS, total: 2, unannotated: 2 }], + [JOB_B, { ...NO_PROGRESS, total: 1, unannotated: 1 }], + ]); + writeToken("a-token"); + vi.stubGlobal("fetch", async (request: Request) => { + sent.push(request); + for (const handler of handlers) { + const answer = handler(request); + if (answer !== undefined) { + return new Response(answer.status === 204 ? null : JSON.stringify(answer.body ?? null), { + status: answer.status, + headers: { "content-type": "application/json" }, + }); + } + } + return new Response(JSON.stringify({ code: "NO_STUB", message: request.url }), { + status: 500, + headers: { "content-type": "application/json" }, + }); + }); +}); + +afterEach(() => { + vi.unstubAllGlobals(); + globalThis.sessionStorage.clear(); + globalThis.localStorage.clear(); +}); + +/** A thunk rather than a value where the answer has to change mid-test. */ +function on(method: string, pattern: RegExp, answer: Answer | (() => Answer)): void { + handlers.push((request) => + request.method === method && pattern.test(new URL(request.url).pathname) + ? typeof answer === "function" + ? answer() + : answer + : undefined, + ); +} + +function mount(node: ReactNode): JSX.Element { + client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return ( + + {node} + + ); +} + +/** Re-read what a mutation would have invalidated, and let the answers land. */ +async function refetch(queryKey: readonly unknown[]): Promise { + await act(async () => { + await client.invalidateQueries({ queryKey: [...queryKey] }); + }); +} + +function progressIs(jobId: string, counts: Record): void { + progress.set(jobId, { ...NO_PROGRESS, ...counts }); +} + +function batch(overrides: Record = {}): Record { + const state = (overrides.state as BatchState | undefined) ?? "in_annotation"; + return { + id: BATCH, + project_id: PROJECT, + name: "drive-01", + state, + schema_version: 2, + asset_count: 3, + progress: { ...NO_PROGRESS, total: 3, unannotated: 3 }, + allowed_actions: batchActions(state), + promoted_asset_count: 0, + parent_batch_id: null, + pre_label_run: null, + ...overrides, + }; +} + +function job(id: string, assetCount: number, over: Record = {}): Job { + const state = (over.state as JobState | undefined) ?? "in_progress"; + return { + id, + batch_id: BATCH, + state, + asset_count: assetCount, + assignee: null, + pre_label_run: null, + allowed_actions: jobActions(state), + ...over, + } as Job; +} + +function asset(index: number, jobId: string): Record { + return { + id: `asset-${index}`, + project_id: PROJECT, + modality: "image", + content_hash: `${index}`.padStart(8, "0") + "deadbeef", + width: 1280, + height: 720, + format: "jpeg", + // Null, so nothing here reaches `GET /sources/{id}`: the provenance line is + // `gallery.test.tsx`'s subject, not this file's. + source_id: null, + frame_index: index, + frame_timestamp: index, + thumbnail_hash: "cafebabe", + ingested_at: "2026-08-01T09:00:00Z", + job_id: jobId, + progress: "unannotated", + allowed_actions: assetActions("unannotated"), + annotation_count: 0, + min_confidence: null, + }; +} + +/** Which frames each job carries. Frame numbers stay in batch order. */ +const FRAMES: Record = { [JOB_A]: [0, 1], [JOB_B]: [2] }; + +/** + * Every read this screen makes, with two jobs behind it. + * + * `/assets` answers **by the `job` parameter** rather than one fixed page: a + * request that forgot the filter would otherwise be indistinguishable from one + * that carried it. + */ +function stubs(batchOverrides: Record = {}, jobs?: readonly Job[]): void { + on("GET", /\/batches\/[^/]+$/, { status: 200, body: batch(batchOverrides) }); + on("GET", /\/batches$/, { status: 200, body: { items: [batch(batchOverrides)], total: 1 } }); + roster = jobs ?? [job(JOB_A, 2), job(JOB_B, 1)]; + on("GET", /\/jobs$/, () => ({ + status: 200, + body: { items: roster, total: roster.length }, + })); + handlers.push((request) => { + const url = new URL(request.url); + if (request.method !== "GET") return undefined; + const forProgress = /\/jobs\/([^/]+)\/progress$/.exec(url.pathname); + if (forProgress !== null) { + const counts = progress.get(forProgress[1] as string); + return counts === undefined + ? { status: 404, body: { code: "JOB_NOT_FOUND", message: "no such job" } } + : { status: 200, body: counts }; + } + if (url.pathname.endsWith("/assets")) { + const jobId = url.searchParams.get("job"); + const indexes = jobId === null ? [0, 1, 2] : (FRAMES[jobId] ?? []); + return { + status: 200, + body: { total: indexes.length, items: indexes.map((at) => asset(at, jobId ?? JOB_A)) }, + }; + } + return undefined; + }); +} + +function renderGallery(): void { + render( + mount( + , + ), + ); +} + +/** The `/assets` requests, most recent last. */ +function assetRequests(): URL[] { + return sent + .filter((one) => new URL(one.url).pathname.endsWith("/assets")) + .map((one) => new URL(one.url)); +} + +describe("which panel opens", () => { + it("opens exactly one panel, the first job with work left", async () => { + progressIs(JOB_A, { total: 2, annotated: 2 }); + progressIs(JOB_B, { total: 1, unannotated: 1 }); + stubs(); + renderGallery(); + + await screen.findByTestId(`job-panel-${JOB_B}`); + expect(screen.queryByTestId(`job-panel-${JOB_A}`)).toBeNull(); + expect(screen.getByTestId(`job-header-${JOB_B}`).getAttribute("aria-expanded")).toBe("true"); + expect(screen.getByTestId(`job-header-${JOB_A}`).getAttribute("aria-expanded")).toBe("false"); + }); + + it("counts a model's first pass as work left, not as work done", async () => { + // `pre_labeled` frames are the ones somebody has to look at next, which is + // the whole reason the rule sums them with `unannotated`. + progressIs(JOB_A, { total: 2, annotated: 2 }); + progressIs(JOB_B, { total: 1, pre_labeled: 1 }); + stubs(); + renderGallery(); + + await screen.findByTestId(`job-panel-${JOB_B}`); + }); + + it("falls back to the first job when nothing is left", async () => { + progressIs(JOB_A, { total: 2, annotated: 2 }); + progressIs(JOB_B, { total: 1, skipped: 1 }); + stubs(); + renderGallery(); + + await screen.findByTestId(`job-panel-${JOB_A}`); + expect(screen.queryByTestId(`job-panel-${JOB_B}`)).toBeNull(); + }); + + it("opening another job closes the open one, and clicking the open header closes it", async () => { + stubs(); + renderGallery(); + await screen.findByTestId(`job-panel-${JOB_A}`); + + await userEvent.click(screen.getByTestId(`job-header-${JOB_B}`)); + await screen.findByTestId(`job-panel-${JOB_B}`); + expect(screen.queryByTestId(`job-panel-${JOB_A}`)).toBeNull(); + + // Every panel closes: the accordion read as an index of the batch's jobs is + // a state somebody asks for, and a control that cannot undo itself is not a + // toggle. + await userEvent.click(screen.getByTestId(`job-header-${JOB_B}`)); + expect(screen.queryByTestId(`job-panel-${JOB_B}`)).toBeNull(); + for (const id of [JOB_A, JOB_B]) { + expect(screen.getByTestId(`job-header-${id}`).getAttribute("aria-expanded")).toBe("false"); + } + + await userEvent.click(screen.getByTestId(`job-header-${JOB_B}`)); + expect(await screen.findByTestId(`job-panel-${JOB_B}`)).toBeTruthy(); + }); + + it("stops describing frames once nothing is open", async () => { + stubs(); + renderGallery(); + await screen.findByTestId(`job-panel-${JOB_A}`); + // The header's provenance line is assembled from the open panel's window. + await waitFor(() => expect(screen.getByTestId("batch-facts").textContent).toContain("1280×720")); + + await userEvent.click(screen.getByTestId(`job-header-${JOB_A}`)); + await waitFor(() => + expect(screen.getByTestId("batch-facts").textContent).not.toContain("1280×720"), + ); + }); + + it("keeps the open panel open when finishing its last frame moves the default on", async () => { + // The default is a *latch*, not a derivation: re-read every render, finishing + // the open job's last frame would make the next job the first with work left + // and shut the panel under the person still looking at it. + progressIs(JOB_A, { total: 2, unannotated: 2 }); + progressIs(JOB_B, { total: 1, unannotated: 1 }); + stubs(); + renderGallery(); + await screen.findByTestId(`job-panel-${JOB_A}`); + + progressIs(JOB_A, { total: 2, annotated: 2 }); + await refetch(["jobs"]); + + expect(screen.queryByTestId(`job-panel-${JOB_A}`)).not.toBeNull(); + expect(screen.queryByTestId(`job-panel-${JOB_B}`)).toBeNull(); + }); + + it("falls back to the default when the open job is no longer in the batch", async () => { + stubs(); + renderGallery(); + await screen.findByTestId(`job-panel-${JOB_A}`); + await userEvent.click(screen.getByTestId(`job-header-${JOB_B}`)); + await screen.findByTestId(`job-panel-${JOB_B}`); + + // A job that has stopped existing cannot stay open, and holding its id would + // leave the accordion closed over a batch that has jobs. + roster = [job(JOB_A, 2)]; + await refetch(["batches"]); + + expect(await screen.findByTestId(`job-panel-${JOB_A}`)).toBeTruthy(); + expect(screen.queryByTestId(`job-header-${JOB_B}`)).toBeNull(); + }); + + it("opens nothing until every job's progress has answered", async () => { + // A panel opened off half-read counts is a panel that flips to a different + // job once the rest land — the wrong job's frames, then a jump. The gate sits + // under `globalThis.fetch` *before* the client is built, because + // `openapi-fetch` reads that reference once at `createClient()` time. + progressIs(JOB_A, { total: 2, annotated: 2 }); + progressIs(JOB_B, { total: 1, unannotated: 1 }); + const inner = globalThis.fetch; + let release: (() => void) | undefined; + const held = new Promise((resolve) => { + release = resolve; + }); + vi.stubGlobal("fetch", async (request: Request) => { + if (new URL(request.url).pathname === `/jobs/${JOB_B}/progress`) await held; + return inner(request); + }); + + stubs(); + renderGallery(); + await screen.findByTestId(`job-header-${JOB_A}`); + expect(screen.queryByTestId(`job-panel-${JOB_A}`)).toBeNull(); + expect(screen.queryByTestId(`job-panel-${JOB_B}`)).toBeNull(); + + release?.(); + // And it is B that opens — the job the withheld counts turn out to name. + await screen.findByTestId(`job-panel-${JOB_B}`); + }); +}); + +describe("what the open panel asks for", () => { + it("asks for the open job's frames only, and counts its segments off its own progress", async () => { + progressIs(JOB_A, { total: 2, annotated: 2 }); + progressIs(JOB_B, { total: 1, unannotated: 1 }); + stubs(); + renderGallery(); + + const panel = within(await screen.findByTestId(`job-panel-${JOB_B}`)); + await waitFor(() => expect(assetRequests().at(-1)?.searchParams.get("job")).toBe(JOB_B)); + // One, not three: the batch holds three frames and this job holds one, so a + // count off the batch would be the filter lying about what it filters. + expect(panel.getByTestId("segment-all").textContent).toContain("All (1)"); + expect(panel.getByTestId(`tile-asset-2`)).toBeTruthy(); + expect(screen.queryByTestId("tile-asset-0")).toBeNull(); + }); + + it("resets the segment filter when the open job changes", async () => { + progressIs(JOB_A, { total: 2, unannotated: 1, annotated: 1 }); + progressIs(JOB_B, { total: 1, unannotated: 1 }); + stubs(); + renderGallery(); + + const first = within(await screen.findByTestId(`job-panel-${JOB_A}`)); + fireEvent.click(first.getByTestId("segment-done")); + expect(first.getByTestId("segment-done").getAttribute("aria-pressed")).toBe("true"); + + await userEvent.click(screen.getByTestId(`job-header-${JOB_B}`)); + const second = within(await screen.findByTestId(`job-panel-${JOB_B}`)); + // The filter is about the job you were looking at, and carrying it over is + // how a person opens a job and is told it has no frames. + expect(second.getByTestId("segment-all").getAttribute("aria-pressed")).toBe("true"); + await waitFor(() => + expect(assetRequests().at(-1)?.searchParams.getAll("progress")).toEqual([]), + ); + }); +}); + +describe("the collapsed header is the overview", () => { + it("renders the frames, the state, the annotated count, the assignee and a bar", async () => { + progressIs(JOB_A, { total: 2, annotated: 1, unannotated: 1 }); + stubs({}, [job(JOB_A, 2, { assignee: "Dana Reyes" }), job(JOB_B, 1)]); + renderGallery(); + + const row = await screen.findByTestId(`job-row-${JOB_A}`); + await waitFor(() => expect(row.textContent).toContain("1 of 2 annotated")); + expect(row.textContent).toContain("Job 1"); + expect(row.textContent).toContain("2 frames"); + expect(row.textContent).toContain("in progress"); + expect(row.textContent).toContain("Dana Reyes"); + // The bar, and only the bar: `BatchProgressBar` would draw its readout under + // the track and say "1 of 2 annotated" a second time. + const bar = within(row).getByRole("progressbar"); + expect(bar.getAttribute("aria-valuenow")).toBe("50"); + expect(row.textContent?.match(/1 of 2 annotated/g)).toHaveLength(1); + // The bar and the assignee are siblings of the control: a `progressbar` is + // content a `