From dfef30df39bc536e8fe4e616d14f9138589e11e1 Mon Sep 17 00:00:00 2001 From: Silo Server Developer Date: Thu, 8 Oct 2026 07:40:36 +0200 Subject: [PATCH 1/3] feat(nodes): add guarded worker commissioning and retirement --- .../admin_node_capabilities_unavailable.json | 7 + ...in_node_drain_requires_authentication.json | 7 + .../admin_node_drain_unavailable.json | 7 + contracts/api/v2/fixtures/index.json | 58 + contracts/api/v2/migration.json | 320 +++- contracts/api/v2/openapi.json | 1336 +++++++++++++++-- contracts/api/v2/route-inventory.json | 238 ++- docs/transcode-node-api.md | 99 ++ internal/api/handlers/admin_node_drain.go | 122 ++ internal/api/handlers/nodes.go | 1 + internal/api/router.go | 2 + internal/apiv2/admin_node_configuration.go | 11 +- internal/apiv2/admin_node_creation_db_test.go | 360 +++++ internal/apiv2/admin_node_drain.go | 117 ++ internal/apiv2/admin_node_drain_db_test.go | 303 ++++ .../apiv2/admin_node_drain_fixtures_test.go | 11 + internal/apiv2/document.go | 1 + internal/apiv2/fixtures_test.go | 3 +- internal/apiv2/router.go | 1 + internal/contractledger/ledger_test.go | 4 + internal/mediasample/caps.go | 17 + internal/mediasample/caps_lifetime_test.go | 187 +++ internal/nodeconfig/drain.go | 34 + internal/nodepool/admin_configuration.go | 6 +- internal/nodepool/admin_configuration_test.go | 24 +- internal/nodepool/admin_drain.go | 149 ++ internal/nodepool/admin_drain_test.go | 266 ++++ internal/nodepool/repository.go | 13 +- internal/playback/copy_seek_anchor.go | 22 +- .../copy_seek_anchor_lifetime_test.go | 173 +++ internal/playback/subtitle_cache.go | 21 +- .../playback/subtitle_fill_lifetime_test.go | 55 + internal/proxy/drain.go | 74 + internal/proxy/drain_subtitle_test.go | 163 ++ internal/proxy/drain_test.go | 132 ++ internal/proxy/protocol_registry.go | 7 + internal/proxy/reprobe.go | 2 +- internal/proxy/server.go | 23 +- internal/proxy/streaming_protocol.go | 3 +- internal/routeinventory/classify.go | 3 + internal/tonemap/preflight.go | 29 + internal/tonemap/preflight_lifetime_test.go | 143 ++ internal/transcodenode/drain.go | 86 ++ .../transcodenode/drain_copy_seek_test.go | 163 ++ internal/transcodenode/drain_detached_test.go | 122 ++ internal/transcodenode/drain_test.go | 77 + internal/transcodenode/protocol_registry.go | 7 + internal/transcodenode/reprobe.go | 2 +- .../transcodenode/segment_ack_protocol.go | 3 +- internal/transcodenode/server.go | 30 +- internal/workerdrain/gate.go | 191 +++ internal/workerdrain/gate_test.go | 177 +++ ...20261008031132_add_worker_drain_fences.sql | 47 + scripts/ci/db-contracts.txt | 11 + web/src/api/v2/operations.ts | 4 + web/src/api/v2/schema.ts | 1046 ++++++++++--- .../queries/admin/nodeConfiguration.test.tsx | 6 + web/src/hooks/queries/admin/nodes.ts | 2 +- 58 files changed, 6063 insertions(+), 465 deletions(-) create mode 100644 contracts/api/v2/fixtures/admin_node_capabilities_unavailable.json create mode 100644 contracts/api/v2/fixtures/admin_node_drain_requires_authentication.json create mode 100644 contracts/api/v2/fixtures/admin_node_drain_unavailable.json create mode 100644 internal/api/handlers/admin_node_drain.go create mode 100644 internal/apiv2/admin_node_creation_db_test.go create mode 100644 internal/apiv2/admin_node_drain.go create mode 100644 internal/apiv2/admin_node_drain_db_test.go create mode 100644 internal/apiv2/admin_node_drain_fixtures_test.go create mode 100644 internal/mediasample/caps_lifetime_test.go create mode 100644 internal/nodeconfig/drain.go create mode 100644 internal/nodepool/admin_drain.go create mode 100644 internal/nodepool/admin_drain_test.go create mode 100644 internal/playback/copy_seek_anchor_lifetime_test.go create mode 100644 internal/playback/subtitle_fill_lifetime_test.go create mode 100644 internal/proxy/drain.go create mode 100644 internal/proxy/drain_subtitle_test.go create mode 100644 internal/proxy/drain_test.go create mode 100644 internal/tonemap/preflight_lifetime_test.go create mode 100644 internal/transcodenode/drain.go create mode 100644 internal/transcodenode/drain_copy_seek_test.go create mode 100644 internal/transcodenode/drain_detached_test.go create mode 100644 internal/transcodenode/drain_test.go create mode 100644 internal/workerdrain/gate.go create mode 100644 internal/workerdrain/gate_test.go create mode 100644 migrations/sql/20261008031132_add_worker_drain_fences.sql diff --git a/contracts/api/v2/fixtures/admin_node_capabilities_unavailable.json b/contracts/api/v2/fixtures/admin_node_capabilities_unavailable.json new file mode 100644 index 0000000000..a3ff12152a --- /dev/null +++ b/contracts/api/v2/fixtures/admin_node_capabilities_unavailable.json @@ -0,0 +1,7 @@ +{ + "revision": "c94aa508b5791bb321829684aa314520de10b17ac8658f8168de92a13bf9b781", + "state": "not_configured", + "allowed": false, + "worker_drain": false, + "disabled_creation": false +} diff --git a/contracts/api/v2/fixtures/admin_node_drain_requires_authentication.json b/contracts/api/v2/fixtures/admin_node_drain_requires_authentication.json new file mode 100644 index 0000000000..206396612c --- /dev/null +++ b/contracts/api/v2/fixtures/admin_node_drain_requires_authentication.json @@ -0,0 +1,7 @@ +{ + "type": "https://siloserver.org/docs/api/v2/problems/authentication_required", + "title": "Authentication required", + "status": 401, + "detail": "Authentication is required.", + "instance": "urn:silo:request:0000000000000000000002e0" +} diff --git a/contracts/api/v2/fixtures/admin_node_drain_unavailable.json b/contracts/api/v2/fixtures/admin_node_drain_unavailable.json new file mode 100644 index 0000000000..74189f666e --- /dev/null +++ b/contracts/api/v2/fixtures/admin_node_drain_unavailable.json @@ -0,0 +1,7 @@ +{ + "type": "https://siloserver.org/docs/api/v2/problems/dependency_unavailable", + "title": "Dependency unavailable", + "status": 503, + "detail": "The worker drain service is not available on this server.", + "instance": "urn:silo:request:0000000000000000000002df" +} diff --git a/contracts/api/v2/fixtures/index.json b/contracts/api/v2/fixtures/index.json index 4d6a901186..8ec1eda5b3 100644 --- a/contracts/api/v2/fixtures/index.json +++ b/contracts/api/v2/fixtures/index.json @@ -2338,6 +2338,64 @@ "schema": "#/components/schemas/Problem", "body_file": "admin_network_access_unknown_host.json" }, + { + "name": "admin_node_capabilities_unavailable", + "operation_id": "getAdminNodeCapabilities", + "scenario": "The administrator discovers unavailable drain and disabled commissioning services without guessing a build version.", + "request": { + "method": "GET", + "path": "/api/v2/admin/nodes/capabilities", + "headers": { + "Authorization": "Bearer tok-admin" + } + }, + "expected_status": 200, + "response_headers": { + "Cache-Control": "private, no-cache", + "Content-Type": "application/json", + "ETag": "\"d35adbeff1618be39295a58c4631eeb22f37843f8fcd921108ab5bf25bb52c4f\"" + }, + "response_media_type": "application/json", + "schema": "#/components/schemas/AdminNodeDrainCapabilities", + "body_file": "admin_node_capabilities_unavailable.json" + }, + { + "name": "admin_node_drain_requires_authentication", + "operation_id": "getAdminNodeDrain", + "scenario": "A worker retirement observation requires acting administrator authority.", + "request": { + "method": "GET", + "path": "/api/v2/admin/nodes/17/drain" + }, + "expected_status": 401, + "response_headers": { + "Cache-Control": "no-store", + "Content-Type": "application/problem+json" + }, + "response_media_type": "application/problem+json", + "schema": "#/components/schemas/Problem", + "body_file": "admin_node_drain_requires_authentication.json" + }, + { + "name": "admin_node_drain_unavailable", + "operation_id": "getAdminNodeDrain", + "scenario": "A missing drain service cannot return a retirement receipt.", + "request": { + "method": "GET", + "path": "/api/v2/admin/nodes/17/drain", + "headers": { + "Authorization": "Bearer tok-admin" + } + }, + "expected_status": 503, + "response_headers": { + "Cache-Control": "no-store", + "Content-Type": "application/problem+json" + }, + "response_media_type": "application/problem+json", + "schema": "#/components/schemas/Problem", + "body_file": "admin_node_drain_unavailable.json" + }, { "name": "admin_password_reset_email", "operation_id": "createAdminUserPasswordReset", diff --git a/contracts/api/v2/migration.json b/contracts/api/v2/migration.json index fa639db4bf..5c0e881542 100644 --- a/contracts/api/v2/migration.json +++ b/contracts/api/v2/migration.json @@ -10,7 +10,7 @@ "consumer_method": "Client call sites were extracted by scripts/apiv2-ledger/extract_consumers.py from web/src (this repo) and the origin/main trees of silo-apple and silo-android at the commits pinned in source_trees (re-resolve a sibling site with git show : in that repo, never against a moving branch): path literals and templates passed to HTTP/WebSocket calls, plus paths built by helper functions that return a template, templates rooted at the API base URL, Kotlin buildString/const val builders, and Swift let/static let path bindings, with interpolations normalized to a wildcard and matched against inventory paths by method (scripts/apiv2-ledger/match_consumers.py). scripts/apiv2-ledger/sweep_uncredited.py then greps the last two static segments of every inventory path across all three trees and every hit not within four lines of a credited site was resolved by hand. Server-internal and compat consumers are Go URL builders and callers in this repo. Sites marked match=manual were resolved by hand where the path is assembled from variables the scripts cannot follow. Call-site files are repository-root-relative for their repo (web: src/... under web/; apple: iosApp/...; android: shared/, android-shared/, androidApp/, androidTvApp/; server: this repository root). Absence of a first-party consumer is not proof of disuse.", "field_reference": "internal/contractledger validates this file against migration.schema.json and reconciles it against the route inventory: listener, namespace, method, path, registration_index, handler, handler_kind, source_file, route_group, middleware_chain, auth_class, auth_traits, conditional, conditions, delegates_to (inventory empty string is null here), request_kind, response_media_kind, streams, and upgrades_websocket are copied from the inventory row and must match it, except that dynamic_plugin_proxy rows record both media kinds as dynamic_proxy; profile_required and admin_required are derived from auth_traits. Every other field is curated by hand and preserved when scripts/apiv2-ledger/build_ledger.py merges a regenerated inventory into this file.", "totals": { - "entries": 782 + "entries": 786 }, "entries": [ { @@ -45374,6 +45374,113 @@ }, "notes": "API-listener hand-off of the /api/v2/ subtree to the sealed apiv2 listener (internal/apiv2). The operations behind it are v2 operations, not legacy rows." }, + { + "listener": "proxy", + "namespace": "legacy_unversioned", + "method": "GET", + "path": "/admin/drain", + "registration_index": 0, + "handler": "(*internal/proxy.Server).handleDrain", + "handler_kind": "method", + "source_file": "internal/proxy/server.go", + "route_group": "/", + "middleware_chain": 29, + "auth_class": "node_bearer", + "auth_traits": [ + "cors", + "node_bearer" + ], + "profile_required": false, + "admin_required": false, + "conditional": false, + "conditions": [], + "delegates_to": null, + "request_kind": "none", + "response_media_kind": "json", + "streams": false, + "upgrades_websocket": false, + "capability_endpoint": null, + "consumers": [ + "internal" + ], + "consumer_call_sites": [ + { + "repo": "server", + "file": "internal/api/handlers/admin_node_drain.go", + "line": 68, + "types": [], + "match": "manual" + } + ], + "section": "node-proxy", + "release_flow": "operational", + "tier": 2, + "disposition": "ported", + "disposition_rule": "listener_delegation", + "disposition_rationale": "No exclusion, removal, or ratified redesign applies. The node listeners have no /api/v2 namespace: this route is retained at its version-neutral path on the node listener and described through the typed worker protocol extension (`x-silo-worker-protocols`), with no /api/v2 alias (plan constraint 4; api-contract.md section 8, version-neutral legacy routes are retired individually and are not aliases into v2). Its section PR confirms retention or retires it individually.", + "owner": null, + "review_state": "ratified", + "v2": { + "method": null, + "path": null, + "operation_id": null + }, + "notes": "Retained on proxy listener with node_bearer; v2 unset by design; description: x-silo-worker-protocols proxy GET /admin/drain. This read-only worker acknowledgment reports the current durable admission fence and conservative execution, request and permit counts; zero seals admission. It never creates or cancels the fence. v2 is null by design for node-listener rows: the route keeps its version-neutral path and is described by the worker protocol extension (`x-silo-worker-protocols`), not aliased under /api/v2." + }, + { + "listener": "proxy", + "namespace": "legacy_unversioned", + "method": "POST", + "path": "/admin/drain", + "registration_index": 0, + "handler": "(*internal/proxy.Server).handleDrain", + "handler_kind": "method", + "source_file": "internal/proxy/server.go", + "route_group": "/", + "middleware_chain": 29, + "auth_class": "node_bearer", + "auth_traits": [ + "cors", + "node_bearer" + ], + "profile_required": false, + "admin_required": false, + "conditional": false, + "conditions": [], + "delegates_to": null, + "request_kind": "unknown", + "response_media_kind": "json", + "streams": false, + "upgrades_websocket": false, + "capability_endpoint": null, + "consumers": [ + "internal" + ], + "consumer_call_sites": [ + { + "repo": "server", + "file": "internal/api/handlers/admin_node_drain.go", + "line": 68, + "types": [], + "match": "manual" + } + ], + "section": "node-proxy", + "release_flow": "operational", + "tier": 2, + "disposition": "ported", + "disposition_rule": "listener_delegation", + "disposition_rationale": "No exclusion, removal, or ratified redesign applies. The node listeners have no /api/v2 namespace: this route is retained at its version-neutral path on the node listener and described through the typed worker protocol extension (`x-silo-worker-protocols`), with no /api/v2 alias (plan constraint 4; api-contract.md section 8, version-neutral legacy routes are retired individually and are not aliases into v2). Its section PR confirms retention or retires it individually.", + "owner": null, + "review_state": "ratified", + "v2": { + "method": null, + "path": null, + "operation_id": null + }, + "notes": "Retained on proxy listener with node_bearer; v2 unset by design; description: x-silo-worker-protocols proxy POST /admin/drain. This read-only worker acknowledgment reports the current durable admission fence and conservative execution, request and permit counts; zero seals admission. It never creates or cancels the fence. v2 is null by design for node-listener rows: the route keeps its version-neutral path and is described by the worker protocol extension (`x-silo-worker-protocols`), not aliased under /api/v2.", + "retry_safety": "natural_idempotent" + }, { "listener": "proxy", "namespace": "legacy_unversioned", @@ -45384,7 +45491,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 28, + "middleware_chain": 29, "auth_class": "node_bearer", "auth_traits": [ "cors", @@ -45444,7 +45551,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 28, + "middleware_chain": 29, "auth_class": "node_bearer", "auth_traits": [ "cors", @@ -45497,7 +45604,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 28, + "middleware_chain": 29, "auth_class": "node_bearer", "auth_traits": [ "cors", @@ -45550,7 +45657,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 26, + "middleware_chain": 27, "auth_class": "public", "auth_traits": [ "cors" @@ -45609,7 +45716,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -45662,7 +45769,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -45715,7 +45822,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 28, + "middleware_chain": 29, "auth_class": "node_bearer", "auth_traits": [ "cors", @@ -45790,7 +45897,7 @@ "handler_kind": "expression", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 26, + "middleware_chain": 27, "auth_class": "public", "auth_traits": [ "cors" @@ -45834,7 +45941,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 28, + "middleware_chain": 29, "auth_class": "node_bearer", "auth_traits": [ "cors", @@ -45887,7 +45994,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 28, + "middleware_chain": 29, "auth_class": "node_bearer", "auth_traits": [ "cors", @@ -45941,7 +46048,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 28, + "middleware_chain": 29, "auth_class": "node_bearer", "auth_traits": [ "cors", @@ -45995,7 +46102,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 28, + "middleware_chain": 29, "auth_class": "node_bearer", "auth_traits": [ "cors", @@ -46040,7 +46147,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 28, + "middleware_chain": 29, "auth_class": "node_bearer", "auth_traits": [ "cors", @@ -46085,7 +46192,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -46146,7 +46253,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -46207,7 +46314,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -46268,7 +46375,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -46329,7 +46436,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -46390,7 +46497,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -46451,7 +46558,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -46496,7 +46603,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -46541,7 +46648,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -46594,7 +46701,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -46647,7 +46754,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -46708,7 +46815,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -46769,7 +46876,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -46829,7 +46936,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -46882,7 +46989,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -46935,7 +47042,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -46988,7 +47095,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -47041,7 +47148,7 @@ "handler_kind": "method", "source_file": "internal/proxy/server.go", "route_group": "/", - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -47091,6 +47198,111 @@ }, "notes": "Retained on proxy listener with public; v2 unset by design; description: x-silo-worker-protocols proxy GET /stream/v3/{session_id}/segment/{name}. v2 is null by design for node-listener rows: the route keeps its version-neutral path and is described by the worker protocol extension (`x-silo-worker-protocols`), not aliased under /api/v2." }, + { + "listener": "transcode_node", + "namespace": "legacy_unversioned", + "method": "GET", + "path": "/admin/drain", + "registration_index": 0, + "handler": "(*internal/transcodenode.Server).handleDrain", + "handler_kind": "method", + "source_file": "internal/transcodenode/server.go", + "route_group": "/", + "middleware_chain": 30, + "auth_class": "node_bearer", + "auth_traits": [ + "node_bearer" + ], + "profile_required": false, + "admin_required": false, + "conditional": false, + "conditions": [], + "delegates_to": null, + "request_kind": "none", + "response_media_kind": "json", + "streams": false, + "upgrades_websocket": false, + "capability_endpoint": null, + "consumers": [ + "internal" + ], + "consumer_call_sites": [ + { + "repo": "server", + "file": "internal/api/handlers/admin_node_drain.go", + "line": 68, + "types": [], + "match": "manual" + } + ], + "section": "node-transcode", + "release_flow": "operational", + "tier": 2, + "disposition": "ported", + "disposition_rule": "listener_delegation", + "disposition_rationale": "No exclusion, removal, or ratified redesign applies. The node listeners have no /api/v2 namespace: this route is retained at its version-neutral path on the node listener and described through the typed worker protocol extension (`x-silo-worker-protocols`), with no /api/v2 alias (plan constraint 4; api-contract.md section 8, version-neutral legacy routes are retired individually and are not aliases into v2). Its section PR confirms retention or retires it individually.", + "owner": null, + "review_state": "ratified", + "v2": { + "method": null, + "path": null, + "operation_id": null + }, + "notes": "Retained on transcode_node listener with node_bearer; v2 unset by design; description: x-silo-worker-protocols transcode_node GET /admin/drain. This read-only worker acknowledgment reports the current durable admission fence and conservative execution, request and permit counts; zero seals admission. It never creates or cancels the fence. v2 is null by design for node-listener rows: the route keeps its version-neutral path and is described by the worker protocol extension (`x-silo-worker-protocols`), not aliased under /api/v2." + }, + { + "listener": "transcode_node", + "namespace": "legacy_unversioned", + "method": "POST", + "path": "/admin/drain", + "registration_index": 0, + "handler": "(*internal/transcodenode.Server).handleDrain", + "handler_kind": "method", + "source_file": "internal/transcodenode/server.go", + "route_group": "/", + "middleware_chain": 30, + "auth_class": "node_bearer", + "auth_traits": [ + "node_bearer" + ], + "profile_required": false, + "admin_required": false, + "conditional": false, + "conditions": [], + "delegates_to": null, + "request_kind": "unknown", + "response_media_kind": "json", + "streams": false, + "upgrades_websocket": false, + "capability_endpoint": null, + "consumers": [ + "internal" + ], + "consumer_call_sites": [ + { + "repo": "server", + "file": "internal/api/handlers/admin_node_drain.go", + "line": 68, + "types": [], + "match": "manual" + } + ], + "section": "node-transcode", + "release_flow": "operational", + "tier": 2, + "disposition": "ported", + "disposition_rule": "listener_delegation", + "disposition_rationale": "No exclusion, removal, or ratified redesign applies. The node listeners have no /api/v2 namespace: this route is retained at its version-neutral path on the node listener and described through the typed worker protocol extension (`x-silo-worker-protocols`), with no /api/v2 alias (plan constraint 4; api-contract.md section 8, version-neutral legacy routes are retired individually and are not aliases into v2). Its section PR confirms retention or retires it individually.", + "owner": null, + "review_state": "ratified", + "v2": { + "method": null, + "path": null, + "operation_id": null + }, + "notes": "Retained on transcode_node listener with node_bearer; v2 unset by design; description: x-silo-worker-protocols transcode_node POST /admin/drain. This read-only worker acknowledgment reports the current durable admission fence and conservative execution, request and permit counts; zero seals admission. It never creates or cancels the fence. v2 is null by design for node-listener rows: the route keeps its version-neutral path and is described by the worker protocol extension (`x-silo-worker-protocols`), not aliased under /api/v2.", + "retry_safety": "natural_idempotent" + }, { "listener": "transcode_node", "namespace": "legacy_unversioned", @@ -47101,7 +47313,7 @@ "handler_kind": "method", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -47160,7 +47372,7 @@ "handler_kind": "method", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -47212,7 +47424,7 @@ "handler_kind": "method", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -47264,7 +47476,7 @@ "handler_kind": "method", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 0, + "middleware_chain": 26, "auth_class": "public", "auth_traits": [], "profile_required": false, @@ -47321,7 +47533,7 @@ "handler_kind": "method", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -47373,7 +47585,7 @@ "handler_kind": "method", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -47426,7 +47638,7 @@ "handler_kind": "method", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -47478,7 +47690,7 @@ "handler_kind": "method", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -47530,7 +47742,7 @@ "handler_kind": "method", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -47584,7 +47796,7 @@ "handler_kind": "method", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -47636,7 +47848,7 @@ "handler_kind": "method", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -47710,7 +47922,7 @@ "handler_kind": "method", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -47762,7 +47974,7 @@ "handler_kind": "expression", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 0, + "middleware_chain": 26, "auth_class": "public", "auth_traits": [], "profile_required": false, @@ -47804,7 +48016,7 @@ "handler_kind": "method", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -47863,7 +48075,7 @@ "handler_kind": "method", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -47922,7 +48134,7 @@ "handler_kind": "method", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -47966,7 +48178,7 @@ "handler_kind": "method", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -48028,7 +48240,7 @@ "handler_kind": "method", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -48081,7 +48293,7 @@ "handler_kind": "method", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -48155,7 +48367,7 @@ "handler_kind": "method", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -48229,7 +48441,7 @@ "handler_kind": "method", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -48304,7 +48516,7 @@ "handler_kind": "method", "source_file": "internal/transcodenode/server.go", "route_group": "/", - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" diff --git a/contracts/api/v2/openapi.json b/contracts/api/v2/openapi.json index 307956fb3a..87231ec54c 100644 --- a/contracts/api/v2/openapi.json +++ b/contracts/api/v2/openapi.json @@ -8931,6 +8931,11 @@ "AdminNodeCreateBody": { "additionalProperties": false, "properties": { + "enabled": { + "default": true, + "description": "Create disabled for worker commissioning; activate separately with a guarded configuration update.", + "type": "boolean" + }, "group": { "maxLength": 1024, "type": "string" @@ -8982,6 +8987,108 @@ ], "type": "object" }, + "AdminNodeDrain": { + "additionalProperties": false, + "properties": { + "active_jobs": { + "format": "int64", + "type": "integer" + }, + "active_requests": { + "format": "int64", + "type": "integer" + }, + "active_reservations": { + "description": "Conservative existing media permits retained to expiry or a positive session deny; absence from Redis is never retirement proof.", + "format": "int64", + "type": "integer" + }, + "config_etag": { + "type": "string" + }, + "drained": { + "type": "boolean" + }, + "fence_id": { + "type": "string" + }, + "fenced": { + "type": "boolean" + }, + "native_server_id": { + "type": "string" + }, + "node_id": { + "description": "Opaque identifier", + "examples": [ + "1" + ], + "minLength": 1, + "type": "string" + }, + "observed_at": { + "description": "RFC 3339 instant in UTC with millisecond precision", + "format": "date-time", + "pattern": "^(?:[1-9][0-9]{3}|0[1-9][0-9]{2}|00[1-9][0-9]|000[2-9])-", + "patternDescription": "a non-zero RFC 3339 instant", + "type": "string" + }, + "worker_instance_id": { + "type": "string" + } + }, + "required": [ + "node_id", + "config_etag", + "fence_id", + "worker_instance_id", + "native_server_id", + "fenced", + "drained", + "active_jobs", + "active_requests", + "active_reservations", + "observed_at" + ], + "type": "object" + }, + "AdminNodeDrainCapabilities": { + "additionalProperties": false, + "properties": { + "allowed": { + "description": "Whether the current principal may use the capability", + "type": "boolean" + }, + "disabled_creation": { + "type": "boolean" + }, + "revision": { + "description": "Opaque revision of this document", + "type": "string" + }, + "state": { + "description": "Support and configuration state, not health", + "enum": [ + "available", + "disabled", + "not_configured", + "unsupported" + ], + "type": "string" + }, + "worker_drain": { + "type": "boolean" + } + }, + "required": [ + "worker_drain", + "disabled_creation", + "revision", + "state", + "allowed" + ], + "type": "object" + }, "AdminNodeHealth": { "additionalProperties": false, "properties": { @@ -84229,7 +84336,7 @@ "bearerAuth": [] } ], - "summary": "Persist node configuration and durable pool invalidation. An identical natural URL configuration resolves a repeated create; conflicting configuration returns 409. No worker probe or replica completion acknowledgement.", + "summary": "Persist node configuration and durable pool invalidation. Omitted enabled defaults to true; enabled=false commissions without placement until a separate guarded activation. An identical natural URL configuration, including enabled, resolves a repeated create; conflicting configuration returns 409. No worker probe or replica completion acknowledgement.", "tags": [ "admin-nodes" ], @@ -84239,10 +84346,18 @@ "x-silo-service-backed": true } }, - "/api/v2/admin/nodes/force-reload": { - "post": { - "operationId": "forceReloadAdminNodes", + "/api/v2/admin/nodes/capabilities": { + "get": { + "operationId": "getAdminNodeCapabilities", "parameters": [ + { + "description": "Optional first precondition, evaluated before If-None-Match: a tag that does not match the current representation is 412 precondition_failed.", + "in": "header", + "name": "If-Match", + "schema": { + "type": "string" + } + }, { "description": "Optional. When present, it must name the authenticated account's primary profile; an absent header is accepted.", "in": "header", @@ -84264,6 +84379,13 @@ ], "type": "string" } + }, + { + "in": "header", + "name": "If-None-Match", + "schema": { + "type": "string" + } } ], "responses": { @@ -84271,11 +84393,35 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AdminNodeReloadOutputBody" + "$ref": "#/components/schemas/AdminNodeDrainCapabilities" } } }, - "description": "OK" + "description": "OK", + "headers": { + "Cache-Control": { + "schema": { + "type": "string" + } + }, + "ETag": { + "description": "The strong, opaque validator of the representation; send it back in If-Match on a guarded mutation or If-None-Match on a conditional read.", + "schema": { + "type": "string" + } + } + } + }, + "304": { + "description": "The representation named by If-None-Match is current; no body.", + "headers": { + "ETag": { + "description": "The strong, opaque validator of the representation; send it back in If-Match on a guarded mutation or If-None-Match on a conditional read.", + "schema": { + "type": "string" + } + } + } }, "400": { "content": { @@ -84327,6 +84473,24 @@ }, "description": "Not Acceptable" }, + "412": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Precondition Failed", + "headers": { + "ETag": { + "description": "The strong, opaque validator of the representation; send it back in If-Match on a guarded mutation or If-None-Match on a conditional read.", + "schema": { + "type": "string" + } + } + } + }, "422": { "content": { "application/problem+json": { @@ -84373,37 +84537,19 @@ "bearerAuth": [] } ], - "summary": "Synchronously request force reload from every currently enumerated enabled node in parallel, with ten-second per-node timeout and no redirects or replay. May tear down sessions. Results are individual acknowledgements, not an atomic or durable completion receipt.", + "summary": "Discover native worker retirement fencing and disabled commissioning support; each configured worker must also confirm its private drain protocol.", "tags": [ "admin-nodes" ], "x-silo-class": "acting_admin", - "x-silo-demo-restricted": true, - "x-silo-retry-safety": "non_retryable", + "x-silo-conditional": true, "x-silo-service-backed": true } }, - "/api/v2/admin/nodes/{id}": { - "delete": { - "operationId": "deleteAdminNode", + "/api/v2/admin/nodes/force-reload": { + "post": { + "operationId": "forceReloadAdminNodes", "parameters": [ - { - "description": "The resource's current ETag, or \"*\" to overwrite deliberately. A missing field is 428 precondition_required; a stale tag is 412 precondition_failed with the current ETag.", - "in": "header", - "name": "If-Match", - "required": true, - "schema": { - "type": "string" - } - }, - { - "description": "Optional second precondition, evaluated after If-Match succeeds: \"*\" or any tag matching the current representation is 412 precondition_failed with the current ETag.", - "in": "header", - "name": "If-None-Match", - "schema": { - "type": "string" - } - }, { "description": "Optional. When present, it must name the authenticated account's primary profile; an absent header is accepted.", "in": "header", @@ -84425,21 +84571,18 @@ ], "type": "string" } - }, - { - "in": "path", - "name": "id", - "required": true, - "schema": { - "maxLength": 10, - "pattern": "^[1-9][0-9]*$", - "type": "string" - } } ], "responses": { - "204": { - "description": "No Content" + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AdminNodeReloadOutputBody" + } + } + }, + "description": "OK" }, "400": { "content": { @@ -84491,24 +84634,6 @@ }, "description": "Not Acceptable" }, - "412": { - "content": { - "application/problem+json": { - "schema": { - "$ref": "#/components/schemas/Problem" - } - } - }, - "description": "Precondition Failed", - "headers": { - "ETag": { - "description": "The strong, opaque validator of the representation; send it back in If-Match on a guarded mutation or If-None-Match on a conditional read.", - "schema": { - "type": "string" - } - } - } - }, "422": { "content": { "application/problem+json": { @@ -84519,16 +84644,6 @@ }, "description": "Unprocessable Entity" }, - "428": { - "content": { - "application/problem+json": { - "schema": { - "$ref": "#/components/schemas/Problem" - } - } - }, - "description": "Precondition Required" - }, "429": { "content": { "application/problem+json": { @@ -84565,18 +84680,19 @@ "bearerAuth": [] } ], - "summary": "Delete the original If-Match configuration and atomically persist pool invalidation for replica reconciliation. No worker teardown, session completion or all-replica acknowledgement. A subsequent 404 is not proof of this caller's outcome.", + "summary": "Synchronously request force reload from every currently enumerated enabled node in parallel, with ten-second per-node timeout and no redirects or replay. May tear down sessions. Results are individual acknowledgements, not an atomic or durable completion receipt.", "tags": [ "admin-nodes" ], "x-silo-class": "acting_admin", "x-silo-demo-restricted": true, - "x-silo-guarded": true, - "x-silo-retry-safety": "durable_dispatch", + "x-silo-retry-safety": "non_retryable", "x-silo-service-backed": true - }, - "put": { - "operationId": "updateAdminNode", + } + }, + "/api/v2/admin/nodes/{id}": { + "delete": { + "operationId": "deleteAdminNode", "parameters": [ { "description": "The resource's current ETag, or \"*\" to overwrite deliberately. A missing field is 428 precondition_required; a stale tag is 412 precondition_failed with the current ETag.", @@ -84628,34 +84744,225 @@ } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AdminNodeUpdateBody" - } - } - }, - "required": true - }, "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AdminNode" - } - } - }, - "description": "OK", - "headers": { - "ETag": { - "description": "The strong, opaque validator of the representation; send it back in If-Match on a guarded mutation or If-None-Match on a conditional read.", - "schema": { - "type": "string" - } - } - } + "204": { + "description": "No Content" + }, + "400": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Bad Request" + }, + "401": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Unauthorized" + }, + "403": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Forbidden" + }, + "404": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Not Found" + }, + "406": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Not Acceptable" + }, + "412": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Precondition Failed", + "headers": { + "ETag": { + "description": "The strong, opaque validator of the representation; send it back in If-Match on a guarded mutation or If-None-Match on a conditional read.", + "schema": { + "type": "string" + } + } + } + }, + "422": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Unprocessable Entity" + }, + "428": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Precondition Required" + }, + "429": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Too Many Requests" + }, + "500": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Internal Server Error" + }, + "503": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Service Unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + } + ], + "summary": "Delete the original If-Match configuration and atomically persist pool invalidation for replica reconciliation. No worker teardown, session completion or all-replica acknowledgement. A subsequent 404 is not proof of this caller's outcome.", + "tags": [ + "admin-nodes" + ], + "x-silo-class": "acting_admin", + "x-silo-demo-restricted": true, + "x-silo-guarded": true, + "x-silo-retry-safety": "durable_dispatch", + "x-silo-service-backed": true + }, + "put": { + "operationId": "updateAdminNode", + "parameters": [ + { + "description": "The resource's current ETag, or \"*\" to overwrite deliberately. A missing field is 428 precondition_required; a stale tag is 412 precondition_failed with the current ETag.", + "in": "header", + "name": "If-Match", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "Optional second precondition, evaluated after If-Match succeeds: \"*\" or any tag matching the current representation is 412 precondition_failed with the current ETag.", + "in": "header", + "name": "If-None-Match", + "schema": { + "type": "string" + } + }, + { + "description": "Optional. When present, it must name the authenticated account's primary profile; an absent header is accepted.", + "in": "header", + "name": "X-Profile-Id", + "schema": { + "examples": [ + "1" + ], + "type": "string" + } + }, + { + "description": "Verification proof for a PIN-locked profile, issued by POST /api/v2/profiles/{id}/verify-pin; required only when the declared profile is locked", + "in": "header", + "name": "X-Profile-Token", + "schema": { + "examples": [ + "pvt_5f3a9c1e7b2d4e8fa0c6" + ], + "type": "string" + } + }, + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "maxLength": 10, + "pattern": "^[1-9][0-9]*$", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AdminNodeUpdateBody" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AdminNode" + } + } + }, + "description": "OK", + "headers": { + "ETag": { + "description": "The strong, opaque validator of the representation; send it back in If-Match on a guarded mutation or If-None-Match on a conditional read.", + "schema": { + "type": "string" + } + } + } }, "400": { "content": { @@ -84874,11 +85181,558 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AdminNodeHealth" + "$ref": "#/components/schemas/AdminNodeHealth" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Bad Request" + }, + "401": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Unauthorized" + }, + "403": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Forbidden" + }, + "404": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Not Found" + }, + "406": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Not Acceptable" + }, + "422": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Unprocessable Entity" + }, + "429": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Too Many Requests" + }, + "500": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Internal Server Error" + }, + "503": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Service Unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + } + ], + "summary": "Synchronously observe node health and apply URL-fenced persistence and process pool updates. Unhealthy is a successful observation; no durable job or hardware support guarantee.", + "tags": [ + "admin-nodes" + ], + "x-silo-class": "acting_admin", + "x-silo-demo-restricted": true, + "x-silo-retry-safety": "natural_idempotent", + "x-silo-service-backed": true + } + }, + "/api/v2/admin/nodes/{id}/drain": { + "delete": { + "operationId": "cancelAdminNodeDrain", + "parameters": [ + { + "description": "The resource's current ETag, or \"*\" to overwrite deliberately. A missing field is 428 precondition_required; a stale tag is 412 precondition_failed with the current ETag.", + "in": "header", + "name": "If-Match", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "Optional second precondition, evaluated after If-Match succeeds: \"*\" or any tag matching the current representation is 412 precondition_failed with the current ETag.", + "in": "header", + "name": "If-None-Match", + "schema": { + "type": "string" + } + }, + { + "description": "Optional. When present, it must name the authenticated account's primary profile; an absent header is accepted.", + "in": "header", + "name": "X-Profile-Id", + "schema": { + "examples": [ + "1" + ], + "type": "string" + } + }, + { + "description": "Verification proof for a PIN-locked profile, issued by POST /api/v2/profiles/{id}/verify-pin; required only when the declared profile is locked", + "in": "header", + "name": "X-Profile-Token", + "schema": { + "examples": [ + "pvt_5f3a9c1e7b2d4e8fa0c6" + ], + "type": "string" + } + }, + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "maxLength": 10, + "pattern": "^[1-9][0-9]*$", + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "No Content" + }, + "400": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Bad Request" + }, + "401": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Unauthorized" + }, + "403": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Forbidden" + }, + "404": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Not Found" + }, + "406": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Not Acceptable" + }, + "412": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Precondition Failed", + "headers": { + "ETag": { + "description": "The strong, opaque validator of the representation; send it back in If-Match on a guarded mutation or If-None-Match on a conditional read.", + "schema": { + "type": "string" + } + } + } + }, + "422": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Unprocessable Entity" + }, + "428": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Precondition Required" + }, + "429": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Too Many Requests" + }, + "500": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Internal Server Error" + }, + "503": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Service Unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + } + ], + "summary": "Explicitly cancel the durable worker admission fence under the original If-Match validator, invalidating earlier drained receipts. Placement stays disabled until a separate guarded configuration update. Workers synchronize cancellation on the next admission.", + "tags": [ + "admin-nodes" + ], + "x-silo-class": "acting_admin", + "x-silo-demo-restricted": true, + "x-silo-guarded": true, + "x-silo-retry-safety": "natural_idempotent", + "x-silo-service-backed": true + }, + "get": { + "operationId": "getAdminNodeDrain", + "parameters": [ + { + "description": "Optional. When present, it must name the authenticated account's primary profile; an absent header is accepted.", + "in": "header", + "name": "X-Profile-Id", + "schema": { + "examples": [ + "1" + ], + "type": "string" + } + }, + { + "description": "Verification proof for a PIN-locked profile, issued by POST /api/v2/profiles/{id}/verify-pin; required only when the declared profile is locked", + "in": "header", + "name": "X-Profile-Token", + "schema": { + "examples": [ + "pvt_5f3a9c1e7b2d4e8fa0c6" + ], + "type": "string" + } + }, + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "maxLength": 10, + "pattern": "^[1-9][0-9]*$", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AdminNodeDrain" + } + } + }, + "description": "OK", + "headers": { + "Cache-Control": { + "schema": { + "type": "string" + } + }, + "ETag": { + "schema": { + "type": "string" + } + } + } + }, + "400": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Bad Request" + }, + "401": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Unauthorized" + }, + "403": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Forbidden" + }, + "404": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Not Found" + }, + "406": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Not Acceptable" + }, + "409": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Conflict" + }, + "422": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Unprocessable Entity" + }, + "429": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Too Many Requests" + }, + "500": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Internal Server Error" + }, + "503": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Service Unavailable" + } + }, + "security": [ + { + "bearerAuth": [] + } + ], + "summary": "Freshly confirm worker admission fencing and execution, egress and media-permit reservations. Zero seals admission, including late dispatch and reconstruction. A changed configuration or unavailable worker cannot yield a drained receipt.", + "tags": [ + "admin-nodes" + ], + "x-silo-class": "acting_admin", + "x-silo-service-backed": true + }, + "put": { + "operationId": "beginAdminNodeDrain", + "parameters": [ + { + "description": "The resource's current ETag, or \"*\" to overwrite deliberately. A missing field is 428 precondition_required; a stale tag is 412 precondition_failed with the current ETag.", + "in": "header", + "name": "If-Match", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "Optional second precondition, evaluated after If-Match succeeds: \"*\" or any tag matching the current representation is 412 precondition_failed with the current ETag.", + "in": "header", + "name": "If-None-Match", + "schema": { + "type": "string" + } + }, + { + "description": "Optional. When present, it must name the authenticated account's primary profile; an absent header is accepted.", + "in": "header", + "name": "X-Profile-Id", + "schema": { + "examples": [ + "1" + ], + "type": "string" + } + }, + { + "description": "Verification proof for a PIN-locked profile, issued by POST /api/v2/profiles/{id}/verify-pin; required only when the declared profile is locked", + "in": "header", + "name": "X-Profile-Token", + "schema": { + "examples": [ + "pvt_5f3a9c1e7b2d4e8fa0c6" + ], + "type": "string" + } + }, + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "maxLength": 10, + "pattern": "^[1-9][0-9]*$", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AdminNodeDrain" } } }, - "description": "OK" + "description": "OK", + "headers": { + "Cache-Control": { + "schema": { + "type": "string" + } + }, + "ETag": { + "description": "The strong, opaque validator of the representation; send it back in If-Match on a guarded mutation or If-None-Match on a conditional read.", + "schema": { + "type": "string" + } + } + } }, "400": { "content": { @@ -84930,6 +85784,34 @@ }, "description": "Not Acceptable" }, + "409": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Conflict" + }, + "412": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Precondition Failed", + "headers": { + "ETag": { + "description": "The strong, opaque validator of the representation; send it back in If-Match on a guarded mutation or If-None-Match on a conditional read.", + "schema": { + "type": "string" + } + } + } + }, "422": { "content": { "application/problem+json": { @@ -84940,6 +85822,16 @@ }, "description": "Unprocessable Entity" }, + "428": { + "content": { + "application/problem+json": { + "schema": { + "$ref": "#/components/schemas/Problem" + } + } + }, + "description": "Precondition Required" + }, "429": { "content": { "application/problem+json": { @@ -84976,12 +85868,13 @@ "bearerAuth": [] } ], - "summary": "Synchronously observe node health and apply URL-fenced persistence and process pool updates. Unhealthy is a successful observation; no durable job or hardware support guarantee.", + "summary": "Disable placement and durably fence new worker admission under the original If-Match configuration; existing admitted work continues. Then confirm the private worker. An unavailable confirmation can follow a committed fence; inspect configuration and GET drain before any explicit retry. Stored disable alone is insufficient.", "tags": [ "admin-nodes" ], "x-silo-class": "acting_admin", "x-silo-demo-restricted": true, + "x-silo-guarded": true, "x-silo-retry-safety": "natural_idempotent", "x-silo-service-backed": true } @@ -204519,6 +205412,45 @@ } } }, + { + "listener": "proxy", + "method": "GET", + "path": "/admin/drain", + "handler": "(*internal/proxy.Server).handleDrain", + "auth_class": "node_bearer", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/x-silo-worker-protocols/schemas/workerdrain_Status" + } + } + }, + "description": "OK" + }, + "401": { + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "description": "Unauthorized" + }, + "503": { + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "description": "Service Unavailable" + } + } + }, { "listener": "transcode_node", "method": "GET", @@ -204597,6 +205529,45 @@ } } }, + { + "listener": "transcode_node", + "method": "GET", + "path": "/admin/drain", + "handler": "(*internal/transcodenode.Server).handleDrain", + "auth_class": "node_bearer", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/x-silo-worker-protocols/schemas/workerdrain_Status" + } + } + }, + "description": "OK" + }, + "401": { + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "description": "Unauthorized" + }, + "503": { + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "description": "Service Unavailable" + } + } + }, { "description": "Reload configuration; active remux work is not torn down.", "retry_safety": "non_retryable", @@ -204716,6 +205687,57 @@ } } }, + { + "description": "Confirm the durable database admission fence; existing permits continue, and zero execution, request and permit reservations seals admission. A missing durable fence returns 409.", + "retry_safety": "natural_idempotent", + "listener": "proxy", + "method": "POST", + "path": "/admin/drain", + "handler": "(*internal/proxy.Server).handleDrain", + "auth_class": "node_bearer", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/x-silo-worker-protocols/schemas/workerdrain_Status" + } + } + }, + "description": "OK" + }, + "401": { + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "description": "Unauthorized" + }, + "409": { + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "description": "Conflict" + }, + "503": { + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "description": "Service Unavailable" + } + } + }, { "description": "Reload configuration and tear down active sessions and delivery authority. Failure can follow partial effects.", "retry_safety": "non_retryable", @@ -204855,6 +205877,57 @@ } } }, + { + "description": "Confirm the durable database admission fence; existing permits continue, and zero execution, request and permit reservations seals admission. A missing durable fence returns 409.", + "retry_safety": "natural_idempotent", + "listener": "transcode_node", + "method": "POST", + "path": "/admin/drain", + "handler": "(*internal/transcodenode.Server).handleDrain", + "auth_class": "node_bearer", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/x-silo-worker-protocols/schemas/workerdrain_Status" + } + } + }, + "description": "OK" + }, + "401": { + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "description": "Unauthorized" + }, + "409": { + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "description": "Conflict" + }, + "503": { + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "description": "Service Unavailable" + } + } + }, { "requestBody": { "content": { @@ -212386,6 +213459,59 @@ "reason" ], "type": "object" + }, + "workerdrain_Status": { + "additionalProperties": false, + "properties": { + "active_jobs": { + "format": "int64", + "type": "integer" + }, + "active_requests": { + "format": "int64", + "type": "integer" + }, + "active_reservations": { + "format": "int64", + "type": "integer" + }, + "drained": { + "type": "boolean" + }, + "fence_id": { + "type": "string" + }, + "fenced": { + "type": "boolean" + }, + "native_server_id": { + "type": "string" + }, + "node_id": { + "format": "int64", + "type": "integer" + }, + "observed_at": { + "format": "date-time", + "type": "string" + }, + "worker_instance_id": { + "type": "string" + } + }, + "required": [ + "node_id", + "fence_id", + "worker_instance_id", + "native_server_id", + "fenced", + "drained", + "active_jobs", + "active_requests", + "active_reservations", + "observed_at" + ], + "type": "object" } } } diff --git a/contracts/api/v2/route-inventory.json b/contracts/api/v2/route-inventory.json index 5c3f2d39f1..4b29de21e3 100644 --- a/contracts/api/v2/route-inventory.json +++ b/contracts/api/v2/route-inventory.json @@ -50,17 +50,17 @@ "id": "proxy", "entrypoint": "internal/proxy.(*Server).Handler", "description": "Proxy node listener: media relay, node control, and its own health/metrics probes.", - "route_count": 31 + "route_count": 33 }, { "id": "transcode_node", "entrypoint": "internal/transcodenode.(*Server).Handler", "description": "Transcode node listener: transcode/remux session control, artifacts, and probes.", - "route_count": 22 + "route_count": 24 } ], "totals": { - "routes": 872, + "routes": 876, "conditional_routes": 691, "streaming_routes": 44, "websocket_routes": 4 @@ -456,14 +456,22 @@ { "id": 26, "middleware": [ + "s.drainMiddleware" + ] + }, + { + "id": 27, + "middleware": [ + "s.drainMiddleware", "clientip.Middleware(s.clientIP) [when s.clientIP != nil]", "netaccess.Middleware(s.ingressTokens) [when s.ingressTokens != nil]", "cors.Handler(cors.Options{ AllowedOrigins: []string{\"*\"}, AllowedMethods: []string{\"GET\", \"HEAD\", \"OPTIONS\"}, AllowedHeaders: []string{ \"Accept\", \"Authorization\", \"Content-Type\", proxyRangeHeader, \"If-Match\", \"If-Modified-Since\", \"If-None-Match\", \"If-Range\", \"If-Unmodified-Since\", }, ExposedHeaders: []string{ \"Accept-Ranges\", \"Content-Encoding\", \"Content-Length\", \"Content-Range\", \"ETag\", \"Last-Modified\", }, MaxAge: 86400, })" ] }, { - "id": 27, + "id": 28, "middleware": [ + "s.drainMiddleware", "clientip.Middleware(s.clientIP) [when s.clientIP != nil]", "netaccess.Middleware(s.ingressTokens) [when s.ingressTokens != nil]", "cors.Handler(cors.Options{ AllowedOrigins: []string{\"*\"}, AllowedMethods: []string{\"GET\", \"HEAD\", \"OPTIONS\"}, AllowedHeaders: []string{ \"Accept\", \"Authorization\", \"Content-Type\", proxyRangeHeader, \"If-Match\", \"If-Modified-Since\", \"If-None-Match\", \"If-Range\", \"If-Unmodified-Since\", }, ExposedHeaders: []string{ \"Accept-Ranges\", \"Content-Encoding\", \"Content-Length\", \"Content-Range\", \"ETag\", \"Last-Modified\", }, MaxAge: 86400, })", @@ -471,8 +479,9 @@ ] }, { - "id": 28, + "id": 29, "middleware": [ + "s.drainMiddleware", "clientip.Middleware(s.clientIP) [when s.clientIP != nil]", "netaccess.Middleware(s.ingressTokens) [when s.ingressTokens != nil]", "cors.Handler(cors.Options{ AllowedOrigins: []string{\"*\"}, AllowedMethods: []string{\"GET\", \"HEAD\", \"OPTIONS\"}, AllowedHeaders: []string{ \"Accept\", \"Authorization\", \"Content-Type\", proxyRangeHeader, \"If-Match\", \"If-Modified-Since\", \"If-None-Match\", \"If-Range\", \"If-Unmodified-Since\", }, ExposedHeaders: []string{ \"Accept-Ranges\", \"Content-Encoding\", \"Content-Length\", \"Content-Range\", \"ETag\", \"Last-Modified\", }, MaxAge: 86400, })", @@ -480,8 +489,9 @@ ] }, { - "id": 29, + "id": 30, "middleware": [ + "s.drainMiddleware", "s.requireBearer" ] } @@ -27044,6 +27054,62 @@ "error_codes": null, "source_file": "internal/api/router.go" }, + { + "listener": "proxy", + "namespace": "legacy_unversioned", + "method": "GET", + "path": "/admin/drain", + "route_group": "/", + "handler": "(*internal/proxy.Server).handleDrain", + "handler_expr": "s.handleDrain", + "handler_kind": "method", + "handler_resolved": true, + "middleware_chain": 29, + "auth_class": "node_bearer", + "auth_traits": [ + "cors", + "node_bearer" + ], + "conditions": [], + "conditional": false, + "request_kind": "none", + "response_media_kind": "json", + "delegates_to": "", + "streams": false, + "upgrades_websocket": false, + "method_origin": "explicit", + "success_statuses": null, + "error_codes": null, + "source_file": "internal/proxy/server.go" + }, + { + "listener": "proxy", + "namespace": "legacy_unversioned", + "method": "POST", + "path": "/admin/drain", + "route_group": "/", + "handler": "(*internal/proxy.Server).handleDrain", + "handler_expr": "s.handleDrain", + "handler_kind": "method", + "handler_resolved": true, + "middleware_chain": 29, + "auth_class": "node_bearer", + "auth_traits": [ + "cors", + "node_bearer" + ], + "conditions": [], + "conditional": false, + "request_kind": "unknown", + "response_media_kind": "json", + "delegates_to": "", + "streams": false, + "upgrades_websocket": false, + "method_origin": "explicit", + "success_statuses": null, + "error_codes": null, + "source_file": "internal/proxy/server.go" + }, { "listener": "proxy", "namespace": "legacy_unversioned", @@ -27054,7 +27120,7 @@ "handler_expr": "s.handleForceReload", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 28, + "middleware_chain": 29, "auth_class": "node_bearer", "auth_traits": [ "cors", @@ -27082,7 +27148,7 @@ "handler_expr": "s.handleReloadConfig", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 28, + "middleware_chain": 29, "auth_class": "node_bearer", "auth_traits": [ "cors", @@ -27110,7 +27176,7 @@ "handler_expr": "s.handleReprobeCapabilities", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 28, + "middleware_chain": 29, "auth_class": "node_bearer", "auth_traits": [ "cors", @@ -27138,7 +27204,7 @@ "handler_expr": "s.handleHealth", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 26, + "middleware_chain": 27, "auth_class": "public", "auth_traits": [ "cors" @@ -27165,7 +27231,7 @@ "handler_expr": "observeProxy(s.telemetry, http.MethodGet, \"/downloads/file/{token}\", s.handleDownloadFile)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -27193,7 +27259,7 @@ "handler_expr": "observeProxy(s.telemetry, http.MethodHead, \"/downloads/file/{token}\", s.handleDownloadFile)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -27221,7 +27287,7 @@ "handler_expr": "s.handleHWCapabilities", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 28, + "middleware_chain": 29, "auth_class": "node_bearer", "auth_traits": [ "cors", @@ -27249,7 +27315,7 @@ "handler_expr": "promhttp.Handler()", "handler_kind": "expression", "handler_resolved": false, - "middleware_chain": 26, + "middleware_chain": 27, "auth_class": "public", "auth_traits": [ "cors" @@ -27276,7 +27342,7 @@ "handler_expr": "s.handleNetworkAccessStatus", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 28, + "middleware_chain": 29, "auth_class": "node_bearer", "auth_traits": [ "cors", @@ -27304,7 +27370,7 @@ "handler_expr": "s.handleNetworkAccessConnect", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 28, + "middleware_chain": 29, "auth_class": "node_bearer", "auth_traits": [ "cors", @@ -27332,7 +27398,7 @@ "handler_expr": "s.handleNetworkAccessDisconnect", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 28, + "middleware_chain": 29, "auth_class": "node_bearer", "auth_traits": [ "cors", @@ -27360,7 +27426,7 @@ "handler_expr": "s.handleNetworkAccessProviderStatus", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 28, + "middleware_chain": 29, "auth_class": "node_bearer", "auth_traits": [ "cors", @@ -27388,7 +27454,7 @@ "handler_expr": "s.handleStatus", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 28, + "middleware_chain": 29, "auth_class": "node_bearer", "auth_traits": [ "cors", @@ -27416,7 +27482,7 @@ "handler_expr": "observeProxy(s.telemetry, http.MethodGet, \"/stream/direct/{token}\", s.handleDirectPlay)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -27444,7 +27510,7 @@ "handler_expr": "observeProxy(s.telemetry, http.MethodHead, \"/stream/direct/{token}\", s.handleDirectPlay)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -27472,7 +27538,7 @@ "handler_expr": "observeProxy(s.telemetry, http.MethodGet, \"/stream/remux/audio-v2/{token}\", s.handleAudioV2Remux)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -27500,7 +27566,7 @@ "handler_expr": "observeProxy(s.telemetry, http.MethodHead, \"/stream/remux/audio-v2/{token}\", s.handleAudioV2Remux)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -27528,7 +27594,7 @@ "handler_expr": "observeProxy(s.telemetry, http.MethodGet, \"/stream/remux/{token}\", s.handleRemux)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -27556,7 +27622,7 @@ "handler_expr": "observeProxy(s.telemetry, http.MethodHead, \"/stream/remux/{token}\", s.handleRemux)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -27584,7 +27650,7 @@ "handler_expr": "observeProxy(s.telemetry, http.MethodGet, \"/stream/subtitles/{token}/{track}\", s.handleSubtitle)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -27612,7 +27678,7 @@ "handler_expr": "observeProxy(s.telemetry, http.MethodGet, \"/stream/subtitles/{token}/{track}/fonts\", s.handleSubtitleFonts)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -27640,7 +27706,7 @@ "handler_expr": "observeProxy(s.telemetry, http.MethodGet, \"/stream/theme/{token}\", s.handleThemeAudio)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -27668,7 +27734,7 @@ "handler_expr": "observeProxy(s.telemetry, http.MethodHead, \"/stream/theme/{token}\", s.handleThemeAudio)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -27696,7 +27762,7 @@ "handler_expr": "observeProxy(s.telemetry, http.MethodGet, \"/stream/transcode/{token}/master.m3u8\", s.handleTranscodeManifest)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -27724,7 +27790,7 @@ "handler_expr": "observeProxy(s.telemetry, http.MethodHead, \"/stream/transcode/{token}/master.m3u8\", s.handleTranscodeManifest)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -27752,7 +27818,7 @@ "handler_expr": "observeProxy(s.telemetry, http.MethodGet, \"/stream/transcode/{token}/segment/{name}\", s.handleTranscodeSegment)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -27780,7 +27846,7 @@ "handler_expr": "observeProxy(s.telemetry, http.MethodGet, \"/stream/v3/{session_id}\", s.handleGrantIdentity)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -27808,7 +27874,7 @@ "handler_expr": "observeProxy(s.telemetry, http.MethodHead, \"/stream/v3/{session_id}\", s.handleGrantIdentity)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -27836,7 +27902,7 @@ "handler_expr": "observeProxy(s.telemetry, http.MethodGet, \"/stream/v3/{session_id}/master.m3u8\", s.handleGrantTranscodeManifest)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -27864,7 +27930,7 @@ "handler_expr": "observeProxy(s.telemetry, http.MethodHead, \"/stream/v3/{session_id}/master.m3u8\", s.handleGrantTranscodeManifest)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -27892,7 +27958,7 @@ "handler_expr": "observeProxy(s.telemetry, http.MethodGet, \"/stream/v3/{session_id}/segment/{name}\", s.handleGrantTranscodeSegment)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 27, + "middleware_chain": 28, "auth_class": "public", "auth_traits": [ "cors", @@ -27910,6 +27976,60 @@ "error_codes": null, "source_file": "internal/proxy/server.go" }, + { + "listener": "transcode_node", + "namespace": "legacy_unversioned", + "method": "GET", + "path": "/admin/drain", + "route_group": "/", + "handler": "(*internal/transcodenode.Server).handleDrain", + "handler_expr": "s.handleDrain", + "handler_kind": "method", + "handler_resolved": true, + "middleware_chain": 30, + "auth_class": "node_bearer", + "auth_traits": [ + "node_bearer" + ], + "conditions": [], + "conditional": false, + "request_kind": "none", + "response_media_kind": "json", + "delegates_to": "", + "streams": false, + "upgrades_websocket": false, + "method_origin": "explicit", + "success_statuses": null, + "error_codes": null, + "source_file": "internal/transcodenode/server.go" + }, + { + "listener": "transcode_node", + "namespace": "legacy_unversioned", + "method": "POST", + "path": "/admin/drain", + "route_group": "/", + "handler": "(*internal/transcodenode.Server).handleDrain", + "handler_expr": "s.handleDrain", + "handler_kind": "method", + "handler_resolved": true, + "middleware_chain": 30, + "auth_class": "node_bearer", + "auth_traits": [ + "node_bearer" + ], + "conditions": [], + "conditional": false, + "request_kind": "unknown", + "response_media_kind": "json", + "delegates_to": "", + "streams": false, + "upgrades_websocket": false, + "method_origin": "explicit", + "success_statuses": null, + "error_codes": null, + "source_file": "internal/transcodenode/server.go" + }, { "listener": "transcode_node", "namespace": "legacy_unversioned", @@ -27920,7 +28040,7 @@ "handler_expr": "s.handleForceReload", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -27947,7 +28067,7 @@ "handler_expr": "s.handleReloadConfig", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -27974,7 +28094,7 @@ "handler_expr": "s.handleReprobeCapabilities", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -28001,7 +28121,7 @@ "handler_expr": "s.handleHealth", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 0, + "middleware_chain": 26, "auth_class": "public", "auth_traits": [], "conditions": [], @@ -28026,7 +28146,7 @@ "handler_expr": "s.handleChapterThumbnailExtract", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -28053,7 +28173,7 @@ "handler_expr": "s.handleDeleteDownloadArtifact", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -28080,7 +28200,7 @@ "handler_expr": "observeNode(s.telemetry, http.MethodGet, \"/downloads/artifacts/{artifact_id}\", s.handleDownloadArtifact)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -28107,7 +28227,7 @@ "handler_expr": "observeNode(s.telemetry, http.MethodHead, \"/downloads/artifacts/{artifact_id}\", s.handleDownloadArtifact)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -28134,7 +28254,7 @@ "handler_expr": "s.handleDownloadPrepare", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -28161,7 +28281,7 @@ "handler_expr": "s.handleDownloadPrepareProgress", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -28188,7 +28308,7 @@ "handler_expr": "s.handleHWCapabilities", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -28215,7 +28335,7 @@ "handler_expr": "s.handleMediaSample", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -28242,7 +28362,7 @@ "handler_expr": "promhttp.Handler()", "handler_kind": "expression", "handler_resolved": false, - "middleware_chain": 0, + "middleware_chain": 26, "auth_class": "public", "auth_traits": [], "conditions": [], @@ -28267,7 +28387,7 @@ "handler_expr": "observeNode(s.telemetry, http.MethodGet, \"/remux/{session_id}\", s.handleRemux)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -28294,7 +28414,7 @@ "handler_expr": "observeNode(s.telemetry, http.MethodHead, \"/remux/{session_id}\", s.handleRemux)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -28321,7 +28441,7 @@ "handler_expr": "s.handleStatus", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -28348,7 +28468,7 @@ "handler_expr": "s.handleStart", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -28375,7 +28495,7 @@ "handler_expr": "s.handleStop", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -28402,7 +28522,7 @@ "handler_expr": "observeNode(s.telemetry, http.MethodGet, \"/transcode/{session_id}/master.m3u8\", s.handleManifest)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -28429,7 +28549,7 @@ "handler_expr": "observeNode(s.telemetry, http.MethodGet, \"/transcode/{session_id}/segment/{name}\", s.handleSegment)", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -28456,7 +28576,7 @@ "handler_expr": "s.handleSegmentDownloaded", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" @@ -28483,7 +28603,7 @@ "handler_expr": "s.handleTrickplayExtract", "handler_kind": "method", "handler_resolved": true, - "middleware_chain": 29, + "middleware_chain": 30, "auth_class": "node_bearer", "auth_traits": [ "node_bearer" diff --git a/docs/transcode-node-api.md b/docs/transcode-node-api.md index 4b52bc87cc..b789184be2 100644 --- a/docs/transcode-node-api.md +++ b/docs/transcode-node-api.md @@ -39,3 +39,102 @@ VBR nor CBR rate control; libx264 or libx265 then encodes. Activity reporting uses `encoder_hw_accel`, falling back to `hw_accel` for older nodes that omit it. Stored recipe cards preserve both values; execution validates the encoder again when reconstructing a session. + +## Native worker retirement fencing + +`GET /api/v2/admin/nodes/capabilities` advertises `worker_drain`. This capability +describes the API server; a configured worker must separately answer the private +drain protocol. Unsupported or unreachable workers cannot produce a retirement +receipt. This feature adds no v1 business route or client playback requirement. + +The same endpoint advertises `disabled_creation`. Native v2 node creation +accepts optional `enabled`, defaulting to true. A controller commissions with +`enabled=false`, confirms the exact worker's identity and readiness, then uses +a separate guarded configuration update to enable placement. Natural-URL +create retries must match the stored enabled state; a delayed disabled create +cannot supersede activation. A late request after deletion may create another +disabled row, so controllers retain reconciliation authority after ambiguous +creation outcomes. Frozen v1 creation keeps its enabled=true behavior. + +`PUT /api/v2/admin/nodes/{id}/drain` requires the node's original configuration +`If-Match`. One PostgreSQL transaction disables placement and records a unique +admission fence. The API then asks the worker to confirm it. A failed reply can +follow a committed fence: reconcile the node configuration and the fresh +`GET /api/v2/admin/nodes/{id}/drain` result before another explicit write. + +Both private listeners expose bearer-protected `GET` and `POST /admin/drain`. +The worker resolves its stable registration and reads the durable fence when +admitting execution or media requests. Ordinary `enabled=false` continues to +remove placement and routine health sampling without fencing worker admission. +Retirement fences reject unknown session and transport identities, background +extracts, capability probes, and reconstruction. Work admitted before the fence +continues; retirement does not stop an encoder or transfer. + +The fresh status contains `node_id`, `fence_id`, `worker_instance_id`, `native_server_id`, `fenced`, +`drained`, `active_jobs`, `active_requests`, `active_reservations`, and +`observed_at`. The API adds `config_etag`, and verifies that the configured URL, +revision, and fence have not changed across the worker request. All responses +carry `Cache-Control: no-store`. Execution includes GPU admission and teardown +holds, detached probes, source preflights, capability listings, copy-seek keyframe +work, and subtitle-cache fills through child exit and publication/discard cleanup. +These cover the interval before job registration and after the ordinary job count +drops. Request reservations cover in-flight +execution and egress; bounded session permits cover gaps between media requests. +Permits remain until signed expiry, a positive session-deny marker, or confirmed +transport teardown. Header-authenticated media and internal transport permits +use the maximum token lifetime, currently 24 hours. Absence from Redis and quiet +bandwidth are never proof that a permit ended. + +`native_server_id` is the read-only `server.identity_id` in the worker's shared +database, matching native server discovery. The worker never initializes a +missing realm. Missing registration or realm prevents private confirmation; +a changed realm prevents another receipt from the same process. The API checks +the worker realm against its own database snapshot, requires every proof field, +and rejects observations older than the request with a 30-second clock-skew +allowance. Worker-only public `/api/v1/health` also carries `native_server_id` +and `worker_instance_id` after initial authority resolution. Public health uses +the last verified binding and is not retirement authority. Controllers compare +it with the authenticated, fresh private receipt from the exact owned process; +public identity alone authorizes nothing. The private control bearer is the +configured runtime JWT signing secret, not the at-rest settings encryption key. + +A temporary PostgreSQL failure refuses unknown identities, permit renewal, and +retirement observations. An admitted, unsealed media identity may continue only +until its original permit expiry; an outage cannot extend that permit or reopen +a sealed worker. Ordinary token playback therefore keeps its bounded continuity +without treating unavailable database authority as a drain acknowledgement. + +Only zero execution, request, and permit reservations sets `drained=true`. +That observation seals admission under the same worker mutex, so a late dispatch +or a resumed client token cannot invalidate the zero. A worker restart retains +the database fence and refuses reconstruction from the preceding process. + +The receipt applies to the addressed worker instance, not to unregistered or +orphan processes. A controller must correlate that instance with the exact +container/process it will stop. An unexpected instance change or URL replacement +requires process-lifecycle evidence before retirement; a zero from a replacement +worker cannot establish that an older process has exited. + +Controllers transfer retirement custody by deleting the exact registration with +`DELETE /api/v2/admin/nodes/{id}` and the fresh receipt's configuration validator +before stopping its owned process. A confirmed 204, exact registration absence, +and a durable record of that acknowledged deletion allow process cleanup. An +uncertain deletion outcome requires retaining the process; absence alone cannot +establish that the controller deleted a still-fenced registration. Cancellation +before guarded deletion changes the validator and refuses deletion. After +deletion, the old process refuses admissions and private proof even if a new +registration reuses its URL and configuration is reloaded. Its gate cannot adopt +another registration or server realm. + +`DELETE /api/v2/admin/nodes/{id}/drain` is an explicit guarded cancellation and +returns 204. It invalidates the previous configuration validator and leaves +placement disabled. Read a fresh validator before a separate guarded reenable. +Enabling a node or changing its URL, type, or name while its fence exists returns +409; this restriction also covers compatibility and direct configuration writers. +The worker synchronizes an +explicit cancellation on its next admission. Deleting the node cascades fence +cleanup; rolling back the migration refuses while any retirement fence remains. +Successful rollback preserves the last effective node validator in the older +configuration column, so cancellation followed by rollback cannot resurrect a +pre-fence `If-Match`. The configuration trigger resumes before the transaction +commits; a rollback fault restores the original revisions and trigger state. diff --git a/internal/api/handlers/admin_node_drain.go b/internal/api/handlers/admin_node_drain.go new file mode 100644 index 0000000000..1c01af7084 --- /dev/null +++ b/internal/api/handlers/admin_node_drain.go @@ -0,0 +1,122 @@ +package handlers + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "time" + + "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/telemetry" + "github.com/Silo-Server/silo-server/internal/workerdrain" +) + +type AdminNodeDrainStore interface { + ReadDrain(context.Context, int) (nodepool.DrainConfiguration, error) + BeginDrain(context.Context, int, func(int64) error) (nodepool.DrainConfiguration, error) + CancelDrain(context.Context, int, func(int64) error) (*nodepool.Node, error) +} + +type AdminNodeDrainView struct { + Node *nodepool.Node + Status workerdrain.Status +} + +func (h *NodeHandler) SetDrainStore(store AdminNodeDrainStore) { h.drainStore = store } + +func (h *NodeHandler) BeginAdminNodeDrain(ctx context.Context, id int, guard func(int64) error) (AdminNodeDrainView, error) { + if h == nil || h.drainStore == nil { + return AdminNodeDrainView{}, ErrAdminNodesUnavailable + } + config, err := h.drainStore.BeginDrain(ctx, id, guard) + if err != nil { + return AdminNodeDrainView{}, err + } + h.configurationChanged() + return h.observeAdminNodeDrain(ctx, config, http.MethodPost) +} + +func (h *NodeHandler) ReadAdminNodeDrain(ctx context.Context, id int) (AdminNodeDrainView, error) { + if h == nil || h.drainStore == nil { + return AdminNodeDrainView{}, ErrAdminNodesUnavailable + } + config, err := h.drainStore.ReadDrain(ctx, id) + if err != nil { + return AdminNodeDrainView{}, err + } + return h.observeAdminNodeDrain(ctx, config, http.MethodGet) +} + +func (h *NodeHandler) CancelAdminNodeDrain(ctx context.Context, id int, guard func(int64) error) (*nodepool.Node, error) { + if h == nil || h.drainStore == nil { + return nil, ErrAdminNodesUnavailable + } + node, err := h.drainStore.CancelDrain(ctx, id, guard) + if err == nil { + h.configurationChanged() + } + return node, err +} + +func (h *NodeHandler) observeAdminNodeDrain(ctx context.Context, config nodepool.DrainConfiguration, method string) (AdminNodeDrainView, error) { + if config.Node == nil || config.FenceID != "" && config.Node.Enabled { + return AdminNodeDrainView{}, nodepool.ErrNodeDrainConflict + } + request, err := http.NewRequestWithContext(ctx, method, nodepool.NodeEndpoint(config.Node.URL, "/admin/drain"), nil) + if err != nil { + return AdminNodeDrainView{}, workerdrain.ErrUnavailable + } + request.Header.Set("Authorization", "Bearer "+h.jwtSecret) + client := &http.Client{Timeout: 10 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + requestedAt := time.Now() + response, err := telemetry.DoTrustedNode(client, request, "drain") + if err != nil { + return AdminNodeDrainView{}, workerdrain.ErrUnavailable + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK { + return AdminNodeDrainView{}, workerdrain.ErrUnavailable + } + body, err := io.ReadAll(io.LimitReader(response.Body, (64<<10)+1)) + if err != nil || len(body) > 64<<10 { + return AdminNodeDrainView{}, workerdrain.ErrUnavailable + } + var fields map[string]json.RawMessage + decoder := json.NewDecoder(bytes.NewReader(body)) + if err = decoder.Decode(&fields); err != nil { + return AdminNodeDrainView{}, workerdrain.ErrUnavailable + } + for _, name := range []string{"node_id", "fence_id", "worker_instance_id", "native_server_id", "fenced", "drained", "active_jobs", "active_requests", "active_reservations", "observed_at"} { + value, exists := fields[name] + if !exists || bytes.Equal(bytes.TrimSpace(value), []byte("null")) { + return AdminNodeDrainView{}, workerdrain.ErrUnavailable + } + } + var trailing any + if err = decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + return AdminNodeDrainView{}, workerdrain.ErrUnavailable + } + var status workerdrain.Status + if err = json.Unmarshal(body, &status); err != nil { + return AdminNodeDrainView{}, workerdrain.ErrUnavailable + } + // Receipt time must belong to this request, allowing bounded clock skew + // between hosts but never accepting a cached observation as fresh proof. + if status.ObservedAt.Before(requestedAt.Add(-30*time.Second)) || status.ObservedAt.After(time.Now().Add(30*time.Second)) { + return AdminNodeDrainView{}, workerdrain.ErrUnavailable + } + if status.NodeID != config.Node.ID || status.FenceID != config.FenceID || status.NativeServerID == "" || status.NativeServerID != config.NativeServerID || status.WorkerInstanceID == "" || status.ObservedAt.IsZero() || status.ActiveJobs < 0 || status.ActiveRequests < 0 || status.ActiveReservations < 0 || status.Fenced != (config.FenceID != "") || status.Drained && (!status.Fenced || status.ActiveJobs != 0 || status.ActiveRequests != 0 || status.ActiveReservations != 0) { + return AdminNodeDrainView{}, workerdrain.ErrUnavailable + } + current, err := h.drainStore.ReadDrain(ctx, config.Node.ID) + if err != nil { + return AdminNodeDrainView{}, err + } + if current.Node == nil || current.Node.AdminRevision != config.Node.AdminRevision || current.Node.URL != config.Node.URL || current.FenceID != config.FenceID || current.NativeServerID != config.NativeServerID { + return AdminNodeDrainView{}, nodepool.ErrNodeDrainConflict + } + return AdminNodeDrainView{Node: config.Node, Status: status}, nil +} diff --git a/internal/api/handlers/nodes.go b/internal/api/handlers/nodes.go index ebe55ae387..cb8467808a 100644 --- a/internal/api/handlers/nodes.go +++ b/internal/api/handlers/nodes.go @@ -55,6 +55,7 @@ type NodeCapabilityRefresher interface { // NodeHandler handles CRUD operations and health checks for stream nodes. type NodeHandler struct { + drainStore AdminNodeDrainStore configuration AdminNodeConfigurationStore configurationWake chan struct{} repo NodeRepository diff --git a/internal/api/router.go b/internal/api/router.go index 59b3611abe..a59c73a61e 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -2707,6 +2707,8 @@ func newChiRouter(deps Dependencies) chi.Router { if deps.DB != nil { nodeHandler.SetConfigurationStore(nodepool.NewAdminConfigurationStore(deps.DB)) v2deps.AdminNodeConfiguration = nodeHandler + nodeHandler.SetDrainStore(nodepool.NewAdminConfigurationStore(deps.DB)) + v2deps.AdminNodeDrain = nodeHandler } } var rateLimitHandler *handlers.RateLimitHandler diff --git a/internal/apiv2/admin_node_configuration.go b/internal/apiv2/admin_node_configuration.go index e1197cacad..44bc91f7cd 100644 --- a/internal/apiv2/admin_node_configuration.go +++ b/internal/apiv2/admin_node_configuration.go @@ -19,6 +19,7 @@ type AdminNodeCreateBody struct { Name string `json:"name" minLength:"1" maxLength:"1024"` Type string `json:"type" enum:"proxy,transcode"` URL string `json:"url" minLength:"1" maxLength:"8192"` + Enabled *bool `json:"enabled,omitempty" default:"true" doc:"Create disabled for worker commissioning; activate separately with a guarded configuration update."` PublicURL string `json:"public_url,omitempty" maxLength:"8192"` Group string `json:"group,omitempty" maxLength:"1024"` MaxJobs *int `json:"max_jobs,omitempty" nullable:"true" maximum:"2147483647" minimum:"-2147483648"` @@ -84,6 +85,12 @@ func adminNodeConfigurationProblem(err error) error { if errors.Is(err, nodepool.ErrNodeConfigurationConflict) { return NewProblem(TypeConflict, "Node URL already has a different configuration.") } + if p, ok := errors.AsType[*pgconn.PgError](err); ok && p.Code == "23514" && p.ConstraintName == "stream_node_drain_requires_disabled" { + return NewProblem(TypeConflict, "Cancel the worker drain before enabling this node.") + } + if p, ok := errors.AsType[*pgconn.PgError](err); ok && p.Code == "23514" && p.ConstraintName == "stream_node_drain_target_fenced" { + return NewProblem(TypeConflict, "Cancel the worker drain before changing this node's identity.") + } if p, ok := errors.AsType[*pgconn.PgError](err); ok && p.Code == "23505" && p.ConstraintName == "stream_nodes_url_key" { return NewProblem(TypeConflict, "Node URL is already configured.") } @@ -102,7 +109,7 @@ func adminNodeConfigurationOutput(ctx context.Context, node *nodepool.Node, err return &AdminNodeConfigurationOutput{ETag: tag, Body: body}, nil } func registerAdminNodeConfiguration(reg *Registry) { - create := Operation{Operation: humaOp("POST", Prefix+"/admin/nodes", "createAdminNode", "admin-nodes", "Persist node configuration and durable pool invalidation. An identical natural URL configuration resolves a repeated create; conflicting configuration returns 409. No worker probe or replica completion acknowledgement."), Class: ClassActingAdmin, DemoRestricted: true, ServiceBacked: true, RetrySafety: RetrySafetyUniqueConstraint} + create := Operation{Operation: humaOp("POST", Prefix+"/admin/nodes", "createAdminNode", "admin-nodes", "Persist node configuration and durable pool invalidation. Omitted enabled defaults to true; enabled=false commissions without placement until a separate guarded activation. An identical natural URL configuration, including enabled, resolves a repeated create; conflicting configuration returns 409. No worker probe or replica completion acknowledgement."), Class: ClassActingAdmin, DemoRestricted: true, ServiceBacked: true, RetrySafety: RetrySafetyUniqueConstraint} create.DefaultStatus = http.StatusCreated create.Errors = append(create.Errors, http.StatusConflict) Register(reg, create, func(ctx context.Context, in *AdminNodeCreateInput) (*AdminNodeConfigurationOutput, error) { @@ -110,7 +117,7 @@ func registerAdminNodeConfiguration(reg *Registry) { return nil, unavailable("administrator nodes") } b := in.Body - node, err := reg.deps.AdminNodeConfiguration.CreateAdminNode(ctx, nodepool.CreateNodeInput{Name: b.Name, Type: b.Type, URL: b.URL, PublicURL: b.PublicURL, Group: b.Group, MaxJobs: b.MaxJobs, MaxBandwidthKbps: b.MaxBandwidthKbps}) + node, err := reg.deps.AdminNodeConfiguration.CreateAdminNode(ctx, nodepool.CreateNodeInput{Name: b.Name, Type: b.Type, URL: b.URL, Enabled: b.Enabled, PublicURL: b.PublicURL, Group: b.Group, MaxJobs: b.MaxJobs, MaxBandwidthKbps: b.MaxBandwidthKbps}) return adminNodeConfigurationOutput(ctx, node, err) }) update := Operation{Operation: humaOp("PUT", Prefix+"/admin/nodes/{id}", "updateAdminNode", "admin-nodes", "Update stored node configuration under the original If-Match revision. Configuration changes advance ETag and persist pool invalidation; a no-change PUT retains ETag. Disabling alone removes new placement and routine health sampling after reconciliation, preserves the last health sample, leaves existing streams serving and does not contact the worker. The response acknowledges stored configuration, not worker reload, replica completion or session teardown."), Class: ClassActingAdmin, DemoRestricted: true, ServiceBacked: true, Guarded: true, RetrySafety: RetrySafetyNaturalIdempotent} diff --git a/internal/apiv2/admin_node_creation_db_test.go b/internal/apiv2/admin_node_creation_db_test.go new file mode 100644 index 0000000000..748a81b876 --- /dev/null +++ b/internal/apiv2/admin_node_creation_db_test.go @@ -0,0 +1,360 @@ +package apiv2 + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/api/handlers" + "github.com/Silo-Server/silo-server/internal/nodepool" +) + +// Disabled registration and separate guarded activation use the registered +// native routes and production stores, including durable pool invalidation. +func nativeNodeCreationFixture(t *testing.T) (*pgxpool.Pool, *nodepool.Repository, http.Handler, string) { + t.Helper() + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("requires a migrated private PostgreSQL fixture") + } + pool, err := pgxpool.New(t.Context(), dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pool.Close) + store := nodepool.NewAdminConfigurationStore(pool) + repo := nodepool.NewRepository(pool) + owner := handlers.NewNodeHandler(repo, nil, nil, nil, nil, nil, "synthetic-node-secret") + owner.SetConfigurationStore(store) + owner.SetDrainStore(store) + deps := pilotDeps(nil, nil) + deps.AdminNodeConfiguration = owner + deps.AdminNodeDrain = owner + deps.AdminNodesRead = owner + native := NewHandler(deps) + url := "http://disabled-" + uuid.NewString() + ".invalid" + t.Cleanup(func() { + if _, err := pool.Exec(context.WithoutCancel(t.Context()), `DELETE FROM stream_nodes WHERE url=$1`, url); err != nil { + t.Errorf("cleanup disabled node: %v", err) + } + }) + return pool, repo, native, url +} + +func TestAdminNodeDisabledCreationPostgresProductionPath(t *testing.T) { + pool, repo, native, url := nativeNodeCreationFixture(t) + body := fmt.Sprintf(`{"name":"commissioned worker","type":"transcode","url":%q,"enabled":false}`, url) + created := do(t, native, http.MethodPost, Prefix+"/admin/nodes", body, bearer(adminToken)) + var node AdminNode + if created.Code != http.StatusCreated || json.Unmarshal(created.Body.Bytes(), &node) != nil || node.Enabled { + t.Fatalf("disabled registration: %d %s", created.Code, created.Body.String()) + } + if node.ConfigETag == "" || created.Header().Get("ETag") != node.ConfigETag { + t.Fatal("creation lacks committed configuration validator") + } + capabilities := do(t, native, http.MethodGet, Prefix+"/admin/nodes/capabilities", "", bearer(adminToken)) + var caps AdminNodeDrainCapabilities + if capabilities.Code != http.StatusOK || json.Unmarshal(capabilities.Body.Bytes(), &caps) != nil || !caps.DisabledCreation || caps.State != StateAvailable || caps.Allowed == nil || !*caps.Allowed { + t.Fatalf("disabled creation capability: %d %s", capabilities.Code, capabilities.Body.String()) + } + assertExcluded := func() { + t.Helper() + enabled, err := repo.ListEnabled(t.Context(), nodepool.NodeTypeTranscode) + if err != nil { + t.Fatal(err) + } + for _, current := range enabled { + if current.URL == url { + t.Fatal("commissioning node was admitted to enabled placement") + } + } + } + assertExcluded() + repeated := do(t, native, http.MethodPost, Prefix+"/admin/nodes", body, bearer(adminToken)) + var retry AdminNode + if repeated.Code != http.StatusCreated || json.Unmarshal(repeated.Body.Bytes(), &retry) != nil || retry.ID != node.ID || retry.ConfigETag != node.ConfigETag || retry.Enabled { + t.Fatalf("exact disabled retry: %d %s", repeated.Code, repeated.Body.String()) + } + for _, opposite := range []string{ + fmt.Sprintf(`{"name":"commissioned worker","type":"transcode","url":%q}`, url), + fmt.Sprintf(`{"name":"commissioned worker","type":"transcode","url":%q,"enabled":true}`, url), + } { + response := do(t, native, http.MethodPost, Prefix+"/admin/nodes", opposite, bearer(adminToken)) + if response.Code != http.StatusConflict { + t.Fatalf("opposite registration changed disabled configuration: %d %s", response.Code, response.Body.String()) + } + } + path := Prefix + "/admin/nodes/" + string(node.ID) + for _, tc := range []struct { + headers map[string]string + code int + }{{bearer(adminToken), 428}, {with(bearer(adminToken), "If-Match", `"stale"`), 412}} { + response := do(t, native, http.MethodPut, path, `{"enabled":true}`, tc.headers) + if response.Code != tc.code { + t.Fatalf("unguarded activation: %d %s", response.Code, response.Body.String()) + } + assertExcluded() + } + headers := bearer(adminToken) + headers["If-Match"] = node.ConfigETag + activated := do(t, native, http.MethodPut, path, `{"enabled":true}`, headers) + var active AdminNode + if activated.Code != http.StatusOK || json.Unmarshal(activated.Body.Bytes(), &active) != nil || !active.Enabled || active.ConfigETag == node.ConfigETag { + t.Fatalf("guarded activation: %d %s", activated.Code, activated.Body.String()) + } + late := do(t, native, http.MethodPost, Prefix+"/admin/nodes", body, bearer(adminToken)) + if late.Code != http.StatusConflict { + t.Fatalf("late registration superseded activation: %d %s", late.Code, late.Body.String()) + } + headers["If-Match"] = active.ConfigETag + removed := do(t, native, http.MethodDelete, path, "", headers) + if removed.Code != http.StatusNoContent { + t.Fatalf("guarded commissioning cleanup: %d %s", removed.Code, removed.Body.String()) + } + var remaining int + if err := pool.QueryRow(t.Context(), `SELECT count(*) FROM stream_nodes WHERE url=$1`, url).Scan(&remaining); err != nil || remaining != 0 { + t.Fatalf("cleanup left registration: %d %v", remaining, err) + } +} + +func TestAdminNodeDisabledPublicationFaultsPostgresProductionPath(t *testing.T) { + pool, _, native, url := nativeNodeCreationFixture(t) + body := fmt.Sprintf(`{"name":"publication worker","type":"proxy","url":%q,"enabled":false}`, url) + name := "creation_fault_" + strings.ReplaceAll(uuid.NewString(), "-", "") + install := func(event, condition string) { + t.Helper() + _, err := pool.Exec(t.Context(), fmt.Sprintf(`CREATE FUNCTION %[1]s() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN + IF NEW.url='%[2]s' AND (%[3]s) THEN RAISE EXCEPTION 'synthetic post-publication fault'; END IF; + RETURN NEW; END $$; + CREATE TRIGGER %[1]s AFTER %[4]s ON stream_nodes FOR EACH ROW EXECUTE FUNCTION %[1]s()`, name, url, condition, event)) + if err != nil { + t.Fatal(err) + } + } + remove := func() { + t.Helper() + if _, err := pool.Exec(context.WithoutCancel(t.Context()), "DROP TRIGGER IF EXISTS "+name+" ON stream_nodes; DROP FUNCTION IF EXISTS "+name+"()"); err != nil { + t.Errorf("cleanup publication fault: %v", err) + } + } + t.Cleanup(remove) + generation := func() int64 { + t.Helper() + var value int64 + if err := pool.QueryRow(t.Context(), `SELECT generation FROM stream_node_pool_generation WHERE singleton`).Scan(&value); err != nil { + t.Fatal(err) + } + return value + } + install("INSERT", "true") + before := generation() + failed := do(t, native, http.MethodPost, Prefix+"/admin/nodes", body, bearer(adminToken)) + if failed.Code != http.StatusInternalServerError || generation() != before { + t.Fatalf("failed creation leaked durable publication: %d %s", failed.Code, failed.Body.String()) + } + var count int + if err := pool.QueryRow(t.Context(), `SELECT count(*) FROM stream_nodes WHERE url=$1`, url).Scan(&count); err != nil || count != 0 { + t.Fatalf("failed creation leaked row: %d %v", count, err) + } + remove() + created := do(t, native, http.MethodPost, Prefix+"/admin/nodes", body, bearer(adminToken)) + var node AdminNode + if created.Code != http.StatusCreated || json.Unmarshal(created.Body.Bytes(), &node) != nil || node.Enabled { + t.Fatalf("creation after fault removal: %d %s", created.Code, created.Body.String()) + } + install("UPDATE", "NEW.enabled IS DISTINCT FROM OLD.enabled") + before = generation() + headers := bearer(adminToken) + headers["If-Match"] = node.ConfigETag + path := Prefix + "/admin/nodes/" + string(node.ID) + failed = do(t, native, http.MethodPut, path, `{"enabled":true}`, headers) + if failed.Code != http.StatusInternalServerError || generation() != before { + t.Fatalf("failed activation leaked durable publication: %d %s", failed.Code, failed.Body.String()) + } + var enabled bool + if err := pool.QueryRow(t.Context(), `SELECT enabled FROM stream_nodes WHERE url=$1`, url).Scan(&enabled); err != nil || enabled { + t.Fatalf("failed activation leaked enabled row: %t %v", enabled, err) + } + remove() + activated := do(t, native, http.MethodPut, path, `{"enabled":true}`, headers) + var active AdminNode + if activated.Code != http.StatusOK || json.Unmarshal(activated.Body.Bytes(), &active) != nil || !active.Enabled || active.ConfigETag == node.ConfigETag { + t.Fatalf("same validator cannot recover rolled-back activation: %d %s", activated.Code, activated.Body.String()) + } + headers["If-Match"] = active.ConfigETag + removed := do(t, native, http.MethodDelete, path, "", headers) + if removed.Code != http.StatusNoContent { + t.Fatalf("publication cleanup: %d %s", removed.Code, removed.Body.String()) + } + if err := pool.QueryRow(t.Context(), `SELECT count(*) FROM stream_nodes WHERE url=$1`, url).Scan(&count); err != nil || count != 0 { + t.Fatalf("publication cleanup left row: %d %v", count, err) + } +} + +func TestAdminNodeLateDisabledCreationPostgresProductionPath(t *testing.T) { + pool, repo, native, url := nativeNodeCreationFixture(t) + body := fmt.Sprintf(`{"name":"late worker","type":"transcode","url":%q,"enabled":false}`, url) + name := "creation_wait_" + strings.ReplaceAll(uuid.NewString(), "-", "") + // The DB trigger stops the real native write after its insert but before + // commit, modeling a caller that cannot yet know its registration outcome. + var key int64 + if err := pool.QueryRow(t.Context(), `SELECT nextval('stream_node_admin_revision_seq')`).Scan(&key); err != nil { + t.Fatal(err) + } + _, err := pool.Exec(t.Context(), fmt.Sprintf(`CREATE FUNCTION %[1]s() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN + IF NEW.url='%[2]s' THEN PERFORM pg_advisory_xact_lock(%[3]d); END IF; + RETURN NEW; END $$; + CREATE TRIGGER %[1]s AFTER INSERT ON stream_nodes FOR EACH ROW EXECUTE FUNCTION %[1]s()`, name, url, key)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if _, err := pool.Exec(context.WithoutCancel(t.Context()), "DROP TRIGGER IF EXISTS "+name+" ON stream_nodes; DROP FUNCTION IF EXISTS "+name+"()"); err != nil { + t.Errorf("cleanup delayed creation: %v", err) + } + }) + lock, err := pool.Begin(t.Context()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = lock.Rollback(context.WithoutCancel(t.Context())) }() + if _, err = lock.Exec(t.Context(), `SELECT pg_advisory_xact_lock($1)`, key); err != nil { + t.Fatal(err) + } + type creationResponse struct { + status int + body []byte + } + responses := make(chan creationResponse, 1) + errors := make(chan error, 1) + listener := httptest.NewServer(native) + t.Cleanup(listener.Close) + go func() { + request, err := http.NewRequestWithContext(t.Context(), http.MethodPost, listener.URL+Prefix+"/admin/nodes", strings.NewReader(body)) + if err != nil { + errors <- err + return + } + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Authorization", bearer(adminToken)["Authorization"]) + response, err := listener.Client().Do(request) + if err != nil { + errors <- err + return + } + defer func() { _ = response.Body.Close() }() + contents, err := io.ReadAll(response.Body) + if err != nil { + errors <- err + return + } + responses <- creationResponse{status: response.StatusCode, body: contents} + }() + deadline := time.Now().Add(5 * time.Second) + for { + var waiting bool + if err := pool.QueryRow(t.Context(), `SELECT EXISTS(SELECT 1 FROM pg_locks WHERE locktype='advisory' AND NOT granted AND objid::bigint=$1)`, key).Scan(&waiting); err != nil { + t.Fatal(err) + } + if waiting { + break + } + if time.Now().After(deadline) { + t.Fatal("native registration did not reach blocked DB publication") + } + time.Sleep(time.Millisecond) + } + var remaining int + if err := pool.QueryRow(t.Context(), `SELECT count(*) FROM stream_nodes WHERE url=$1`, url).Scan(&remaining); err != nil || remaining != 0 { + t.Fatalf("uncommitted registration appeared to cleanup reader: %d %v", remaining, err) + } + if err := lock.Commit(t.Context()); err != nil { + t.Fatal(err) + } + var response creationResponse + select { + case response = <-responses: + case err := <-errors: + t.Fatal(err) + case <-time.After(5 * time.Second): + t.Fatal("native delayed registration failed to complete") + } + var node AdminNode + if response.status != http.StatusCreated || json.Unmarshal(response.body, &node) != nil || node.Enabled { + t.Fatalf("late committed registration was enabled: %d %+v", response.status, node) + } + assertExcluded := func() { + t.Helper() + enabled, err := repo.ListEnabled(t.Context(), nodepool.NodeTypeTranscode) + if err != nil { + t.Fatal(err) + } + for _, current := range enabled { + if current.URL == url { + t.Fatal("late registration entered placement without activation") + } + } + } + assertExcluded() + headers := bearer(adminToken) + headers["If-Match"] = node.ConfigETag + path := Prefix + "/admin/nodes/" + string(node.ID) + if deleted := do(t, native, http.MethodDelete, path, "", headers); deleted.Code != http.StatusNoContent { + t.Fatalf("exact late registration cleanup: %d %s", deleted.Code, deleted.Body.String()) + } + // A second very late request may recreate a deleted natural URL. It remains + // disabled; controllers must reconcile it and never infer finality from 404. + recreated := do(t, native, http.MethodPost, Prefix+"/admin/nodes", body, bearer(adminToken)) + var late AdminNode + if recreated.Code != http.StatusCreated || json.Unmarshal(recreated.Body.Bytes(), &late) != nil || late.ID == node.ID || late.Enabled { + t.Fatalf("post-cleanup late request admitted work: %d %s", recreated.Code, recreated.Body.String()) + } + assertExcluded() + headers["If-Match"] = late.ConfigETag + if deleted := do(t, native, http.MethodDelete, Prefix+"/admin/nodes/"+string(late.ID), "", headers); deleted.Code != http.StatusNoContent { + t.Fatalf("recreated disabled registration cleanup: %d %s", deleted.Code, deleted.Body.String()) + } + if err := pool.QueryRow(t.Context(), `SELECT count(*) FROM stream_nodes WHERE url=$1`, url).Scan(&remaining); err != nil || remaining != 0 { + t.Fatalf("late-write cleanup left row: %d %v", remaining, err) + } +} + +func TestAdminNodeCreationPreservesEnabledDefaultsPostgresProductionPath(t *testing.T) { + pool, _, native, url := nativeNodeCreationFixture(t) + for _, tc := range []struct { + name, extra string + bridge bool + }{{"native omitted", "", false}, {"native true", `,"enabled":true`, false}, {"frozen bridge ignores false", `,"enabled":false`, true}} { + t.Run(tc.name, func(t *testing.T) { + target := url + "/" + uuid.NewString() + t.Cleanup(func() { + if _, err := pool.Exec(context.WithoutCancel(t.Context()), `DELETE FROM stream_nodes WHERE url=$1`, target); err != nil { + t.Errorf("cleanup default node: %v", err) + } + }) + body := fmt.Sprintf(`{"name":"default worker","type":"proxy","url":%q%s}`, target, tc.extra) + route := native + if tc.bridge { + owner := handlers.NewNodeHandler(nodepool.NewRepository(pool), nil, nil, nil, nil, nil, "synthetic-node-secret") + route = http.HandlerFunc(owner.HandleCreateNode) + } + created := do(t, route, http.MethodPost, Prefix+"/admin/nodes", body, bearer(adminToken)) + var node struct { + Enabled bool `json:"enabled"` + } + if created.Code != http.StatusCreated || json.Unmarshal(created.Body.Bytes(), &node) != nil || !node.Enabled { + t.Fatalf("creation default changed: %d %s", created.Code, created.Body.String()) + } + }) + } +} diff --git a/internal/apiv2/admin_node_drain.go b/internal/apiv2/admin_node_drain.go new file mode 100644 index 0000000000..30eaf3c868 --- /dev/null +++ b/internal/apiv2/admin_node_drain.go @@ -0,0 +1,117 @@ +package apiv2 + +import ( + "context" + "errors" + "net/http" + + "github.com/Silo-Server/silo-server/internal/api/handlers" + "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/workerdrain" +) + +type AdminNodeDrainService interface { + BeginAdminNodeDrain(context.Context, int, func(int64) error) (handlers.AdminNodeDrainView, error) + ReadAdminNodeDrain(context.Context, int) (handlers.AdminNodeDrainView, error) + CancelAdminNodeDrain(context.Context, int, func(int64) error) (*nodepool.Node, error) +} + +type AdminNodeDrainCapabilities struct { + Capability + WorkerDrain bool `json:"worker_drain"` + DisabledCreation bool `json:"disabled_creation"` +} +type AdminNodeDrainCapabilitiesOutput struct { + Status int + ETag string `header:"ETag"` + CacheControl string `header:"Cache-Control"` + Body AdminNodeDrainCapabilities +} +type AdminNodeDrainInput struct { + ID string `path:"id" pattern:"^[1-9][0-9]*$" maxLength:"10"` + IfMatch string `header:"If-Match"` + IfNoneMatch string `header:"If-None-Match"` +} +type AdminNodeDrain struct { + NodeID ID `json:"node_id"` + ConfigETag string `json:"config_etag"` + FenceID string `json:"fence_id"` + WorkerInstanceID string `json:"worker_instance_id"` + NativeServerID string `json:"native_server_id"` + Fenced bool `json:"fenced"` + Drained bool `json:"drained"` + ActiveJobs int `json:"active_jobs"` + ActiveRequests int `json:"active_requests"` + ActiveReservations int `json:"active_reservations" doc:"Conservative existing media permits retained to expiry or a positive session deny; absence from Redis is never retirement proof."` + ObservedAt Instant `json:"observed_at"` +} +type AdminNodeDrainOutput struct { + ETag string `header:"ETag"` + CacheControl string `header:"Cache-Control"` + Body AdminNodeDrain +} + +func adminNodeDrainOutput(ctx context.Context, view handlers.AdminNodeDrainView, err error) (*AdminNodeDrainOutput, error) { + if err != nil { + if errors.Is(err, workerdrain.ErrUnavailable) || errors.Is(err, nodepool.ErrNodeDrainAuthorityUnavailable) { + return nil, unavailable("worker drain confirmation; inspect current configuration before another explicit command") + } + if errors.Is(err, nodepool.ErrNodeDrainConflict) { + return nil, NewProblem(TypeConflict, "Node configuration changed during worker confirmation. Read current configuration.") + } + return nil, adminNodeConfigurationProblem(err) + } + if view.Node == nil || view.Node.AdminRevision <= 0 { + return nil, NewProblem(TypeInternalError, "Worker drain acknowledgement unavailable.") + } + tag := adminNodeConfigurationTag(ctx, view.Node.ID, view.Node.AdminRevision).String() + s := view.Status + return &AdminNodeDrainOutput{ETag: tag, CacheControl: playbackCacheControl, Body: AdminNodeDrain{NodeID: IDFromInt(int64(view.Node.ID)), ConfigETag: tag, FenceID: s.FenceID, WorkerInstanceID: s.WorkerInstanceID, NativeServerID: s.NativeServerID, Fenced: s.Fenced, Drained: s.Drained, ActiveJobs: s.ActiveJobs, ActiveRequests: s.ActiveRequests, ActiveReservations: s.ActiveReservations, ObservedAt: NewInstant(s.ObservedAt)}}, nil +} + +func registerAdminNodeDrain(reg *Registry) { + Register(reg, Operation{Operation: humaOp("GET", Prefix+"/admin/nodes/capabilities", "getAdminNodeCapabilities", "admin-nodes", "Discover native worker retirement fencing and disabled commissioning support; each configured worker must also confirm its private drain protocol."), Class: ClassActingAdmin, ServiceBacked: true}, func(_ context.Context, _ *CapabilityInput) (*AdminNodeDrainCapabilitiesOutput, error) { + return &AdminNodeDrainCapabilitiesOutput{Body: AdminNodeDrainCapabilities{Capability: Capability{State: configuredCapabilityState(reg.deps.AdminNodeDrain != nil || reg.deps.AdminNodeConfiguration != nil)}, WorkerDrain: reg.deps.AdminNodeDrain != nil, DisabledCreation: reg.deps.AdminNodeConfiguration != nil}}, nil + }) + read := Operation{Operation: humaOp("GET", Prefix+"/admin/nodes/{id}/drain", "getAdminNodeDrain", "admin-nodes", "Freshly confirm worker admission fencing and execution, egress and media-permit reservations. Zero seals admission, including late dispatch and reconstruction. A changed configuration or unavailable worker cannot yield a drained receipt."), Class: ClassActingAdmin, ServiceBacked: true} + read.Errors = append(read.Errors, http.StatusConflict) + Register(reg, read, func(ctx context.Context, in *AdminNodeCommandInput) (*AdminNodeDrainOutput, error) { + id, err := adminNodeCommandID(in) + if err != nil { + return nil, err + } + if reg.deps.AdminNodeDrain == nil { + return nil, unavailable("worker drain") + } + view, err := reg.deps.AdminNodeDrain.ReadAdminNodeDrain(ctx, id) + return adminNodeDrainOutput(ctx, view, err) + }) + begin := Operation{Operation: humaOp("PUT", Prefix+"/admin/nodes/{id}/drain", "beginAdminNodeDrain", "admin-nodes", "Disable placement and durably fence new worker admission under the original If-Match configuration; existing admitted work continues. Then confirm the private worker. An unavailable confirmation can follow a committed fence; inspect configuration and GET drain before any explicit retry. Stored disable alone is insufficient."), Class: ClassActingAdmin, ServiceBacked: true, DemoRestricted: true, Guarded: true, RetrySafety: RetrySafetyNaturalIdempotent} + begin.Errors = append(begin.Errors, http.StatusConflict) + Register(reg, begin, func(ctx context.Context, in *AdminNodeDrainInput) (*AdminNodeDrainOutput, error) { + id, err := adminNodeCommandID(&AdminNodeCommandInput{ID: in.ID}) + if err != nil { + return nil, err + } + if reg.deps.AdminNodeDrain == nil { + return nil, unavailable("worker drain") + } + view, err := reg.deps.AdminNodeDrain.BeginAdminNodeDrain(ctx, id, adminNodeConfigurationGuard(ctx, id, in.IfMatch, in.IfNoneMatch)) + return adminNodeDrainOutput(ctx, view, err) + }) + cancel := Operation{Operation: humaOp("DELETE", Prefix+"/admin/nodes/{id}/drain", "cancelAdminNodeDrain", "admin-nodes", "Explicitly cancel the durable worker admission fence under the original If-Match validator, invalidating earlier drained receipts. Placement stays disabled until a separate guarded configuration update. Workers synchronize cancellation on the next admission."), Class: ClassActingAdmin, ServiceBacked: true, DemoRestricted: true, Guarded: true, RetrySafety: RetrySafetyNaturalIdempotent} + Register(reg, cancel, func(ctx context.Context, in *AdminNodeDrainInput) (*struct{}, error) { + id, err := adminNodeCommandID(&AdminNodeCommandInput{ID: in.ID}) + if err != nil { + return nil, err + } + if reg.deps.AdminNodeDrain == nil { + return nil, unavailable("worker drain") + } + _, err = reg.deps.AdminNodeDrain.CancelAdminNodeDrain(ctx, id, adminNodeConfigurationGuard(ctx, id, in.IfMatch, in.IfNoneMatch)) + if err != nil { + return nil, adminNodeConfigurationProblem(err) + } + return &struct{}{}, nil + }) +} diff --git a/internal/apiv2/admin_node_drain_db_test.go b/internal/apiv2/admin_node_drain_db_test.go new file mode 100644 index 0000000000..785f5eb296 --- /dev/null +++ b/internal/apiv2/admin_node_drain_db_test.go @@ -0,0 +1,303 @@ +package apiv2 + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/api/handlers" + "github.com/Silo-Server/silo-server/internal/cache" + "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/nodeconfig" + "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/proxy" + "github.com/Silo-Server/silo-server/internal/streamtoken" +) + +// The registered native routes, durable production stores, config watcher and +// assembled private proxy listener are all exercised together. Only the reply +// loss/concurrent mutation wrapper is synthetic. +func TestAdminNodeDrainPostgresProductionPath(t *testing.T) { + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("requires a migrated private PostgreSQL fixture") + } + pool, err := pgxpool.New(t.Context(), dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pool.Close) + if _, err = pool.Exec(t.Context(), `INSERT INTO server_settings(key,value) VALUES ('server.identity_id',$1) ON CONFLICT(key) DO NOTHING`, uuid.NewString()); err != nil { + t.Fatal(err) + } + store := nodepool.NewAdminConfigurationStore(pool) + secret := "synthetic-worker-drain-secret" + listener := httptest.NewUnstartedServer(nil) + url := "http://" + listener.Listener.Addr().String() + name := "drain-" + uuid.NewString() + watchCtx, cancelWatch := context.WithCancel(t.Context()) + defer cancelWatch() + watcher := nodeconfig.NewWatcher(pool, nil, &cache.NoopEventBus{}, nodeconfig.BootstrapOverrides{Mode: "proxy", NodeURL: url, NodeName: name}) + scratchDir := t.TempDir() + watcher.OnLoad(func(cfg *config.Config) { cfg.Auth.JWTSecret = secret; cfg.Playback.TranscodeDir = scratchDir }) + if err = watcher.Start(watchCtx); err != nil { + listener.Close() + t.Fatal(err) + } + worker := proxy.NewServer(watcher, nil).Handler() + unregistered := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/admin/drain", nil) + request.Header.Set("Authorization", "Bearer "+secret) + worker.ServeHTTP(unregistered, request) + if unregistered.Code != http.StatusServiceUnavailable { + listener.Close() + t.Fatalf("worker before registration had authority: %d %s", unregistered.Code, unregistered.Body.String()) + } + wrongControl := httptest.NewRecorder() + request = httptest.NewRequest(http.MethodGet, "/admin/drain", nil) + request.Header.Set("Authorization", "Bearer synthetic-at-rest-master-key") + worker.ServeHTTP(wrongControl, request) + if wrongControl.Code != http.StatusUnauthorized { + listener.Close() + t.Fatalf("master key substituted for runtime control bearer: %d", wrongControl.Code) + } + node, err := store.Create(t.Context(), nodepool.CreateNodeInput{Name: name, Type: nodepool.NodeTypeProxy, URL: url}) + if err != nil { + listener.Close() + t.Fatal(err) + } + t.Cleanup(func() { + ctx := context.WithoutCancel(t.Context()) + if _, err := pool.Exec(ctx, `DELETE FROM stream_nodes WHERE id=$1`, node.ID); err != nil { + t.Errorf("cleanup node: %v", err) + } + var remaining int + if err := pool.QueryRow(ctx, `SELECT COUNT(*) FROM stream_node_drain_fences WHERE node_id=$1`, node.ID).Scan(&remaining); err != nil || remaining != 0 { + t.Errorf("cleanup fence: %d %v", remaining, err) + } + }) + request = httptest.NewRequest(http.MethodPost, "/admin/reload-config", nil) + request.Header.Set("Authorization", "Bearer "+secret) + reloaded := httptest.NewRecorder() + worker.ServeHTTP(reloaded, request) + if reloaded.Code != http.StatusOK && reloaded.Code != http.StatusNoContent { + listener.Close() + t.Fatalf("private registration reload failed: %d %s", reloaded.Code, reloaded.Body.String()) + } + if got, ok := watcher.NodeRowID(); !ok || got != node.ID { + listener.Close() + t.Fatalf("worker identity: %d %t", got, ok) + } + var fault atomic.Int64 + listener.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/admin/drain" { + worker.ServeHTTP(w, r) + return + } + recorder := httptest.NewRecorder() + worker.ServeHTTP(recorder, r) + switch fault.Load() { + case 1: + connection, _, err := w.(http.Hijacker).Hijack() + if err == nil { + _ = connection.Close() + } + return + case 2: + if _, _, err := store.Update(r.Context(), node.ID, nodepool.UpdateNodeInput{Group: new("concurrent-" + uuid.NewString())}, func(int64) error { return nil }); err != nil { + t.Errorf("inject concurrent update: %v", err) + } + case 3, 4, 5, 7, 8, 9: + var encoded map[string]any + if err := json.Unmarshal(recorder.Body.Bytes(), &encoded); err != nil { + t.Errorf("decode fault response: %v", err) + } + switch fault.Load() { + case 3: + delete(encoded, "active_jobs") + case 4: + encoded["active_requests"] = nil + case 5: + delete(encoded, "active_reservations") + case 7: + encoded["native_server_id"] = "foreign-deployment" + case 8: + delete(encoded, "native_server_id") + case 9: + encoded["observed_at"] = "2000-01-01T00:00:00Z" + } + recorder.Body.Reset() + _ = json.NewEncoder(recorder.Body).Encode(encoded) + case 6: + _, _ = recorder.Body.WriteString(`{}`) + } + for name, values := range recorder.Header() { + w.Header()[name] = values + } + w.WriteHeader(recorder.Code) + _, _ = w.Write(recorder.Body.Bytes()) + }) + listener.Start() + defer listener.Close() + owner := handlers.NewNodeHandler(nodepool.NewRepository(pool), nil, nil, nil, nil, nil, secret) + owner.SetConfigurationStore(store) + owner.SetDrainStore(store) + deps := pilotDeps(nil, nil) + deps.AdminNodeConfiguration = owner + deps.AdminNodeDrain = owner + deps.AdminNodesRead = owner + native := NewHandler(deps) + path := fmt.Sprintf("%s/admin/nodes/%d/drain", Prefix, node.ID) + readTag := func() string { + t.Helper() + response := do(t, native, http.MethodGet, Prefix+"/admin/nodes", "", bearer(adminToken)) + if response.Code != http.StatusOK { + t.Fatal(response.Code, response.Body.String()) + } + var page Collection[AdminNode] + if err := json.Unmarshal(response.Body.Bytes(), &page); err != nil { + t.Fatal(err) + } + for _, item := range page.Items { + if item.ID == IDFromInt(int64(node.ID)) { + return item.ConfigETag + } + } + t.Fatal("node absent from native list") + return "" + } + tag := readTag() + for _, tc := range []struct { + headers map[string]string + code int + }{{nil, 401}, {bearer(memberToken), 403}, {bearer(adminToken), 428}} { + response := do(t, native, http.MethodPut, path, "", tc.headers) + if response.Code != tc.code { + t.Fatalf("authority: %d %s", response.Code, response.Body.String()) + } + } + stale := bearer(adminToken) + stale["If-Match"] = `"stale"` + if response := do(t, native, http.MethodPut, path, "", stale); response.Code != 412 { + t.Fatal(response.Code, response.Body.String()) + } + before, err := store.ReadDrain(t.Context(), node.ID) + if err != nil || before.FenceID != "" || !before.Node.Enabled { + t.Fatalf("rejected writes mutated: %+v %v", before, err) + } + valid := bearer(adminToken) + valid["If-Match"] = tag + fault.Store(1) + if response := do(t, native, http.MethodPut, path, "", valid); response.Code != 503 { + t.Fatal(response.Code, response.Body.String()) + } + durable, err := store.ReadDrain(t.Context(), node.ID) + if err != nil || durable.FenceID == "" || durable.Node.Enabled { + t.Fatalf("lost reply lost fence: %+v %v", durable, err) + } + fault.Store(0) + response := do(t, native, http.MethodGet, path, "", bearer(adminToken)) + var proof AdminNodeDrain + if response.Code != 200 || json.Unmarshal(response.Body.Bytes(), &proof) != nil || !proof.Drained || !proof.Fenced || proof.FenceID != durable.FenceID || proof.ConfigETag == tag || proof.WorkerInstanceID == "" || proof.NativeServerID != durable.NativeServerID || response.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("fresh sealed proof: %d %s", response.Code, response.Body.String()) + } + valid["If-Match"] = proof.ConfigETag + for _, mode := range []int64{3, 4, 5, 6, 7, 8, 9} { + fault.Store(mode) + if response := do(t, native, http.MethodGet, path, "", bearer(adminToken)); response.Code != 503 { + t.Fatalf("incomplete worker proof %d accepted: %d %s", mode, response.Code, response.Body.String()) + } + } + fault.Store(0) + unchanged, err := store.ReadDrain(t.Context(), node.ID) + if err != nil || unchanged.FenceID != durable.FenceID || unchanged.Node.AdminRevision != durable.Node.AdminRevision { + t.Fatalf("malformed proof changed authority: %+v %v", unchanged, err) + } + configPath := fmt.Sprintf("%s/admin/nodes/%d", Prefix, node.ID) + if response := do(t, native, http.MethodPut, configPath, `{"enabled":true}`, valid); response.Code != 409 { + t.Fatalf("reenabled fenced worker: %d %s", response.Code, response.Body.String()) + } + for _, body := range []string{`{"url":"http://replacement.invalid"}`, `{"name":"replacement"}`} { + if response := do(t, native, http.MethodPut, configPath, body, valid); response.Code != 409 { + t.Fatalf("repointed fenced worker: %d %s", response.Code, response.Body.String()) + } + } + fault.Store(2) + if response := do(t, native, http.MethodGet, path, "", bearer(adminToken)); response.Code != 409 { + t.Fatalf("changed revision yielded proof: %d %s", response.Code, response.Body.String()) + } + fault.Store(0) + if response := do(t, native, http.MethodDelete, path, "", valid); response.Code != 412 { + t.Fatalf("stale cancellation: %d %s", response.Code, response.Body.String()) + } + valid["If-Match"] = readTag() + response = do(t, native, http.MethodDelete, path, "", valid) + if response.Code != 204 { + t.Fatalf("cancellation did not invalidate proof: %d %s", response.Code, response.Body.String()) + } + canceled, err := store.ReadDrain(t.Context(), node.ID) + if err != nil || canceled.FenceID != "" || canceled.Node.Enabled { + t.Fatalf("cancellation state: %+v %v", canceled, err) + } + currentTag := readTag() + if currentTag == valid["If-Match"] { + t.Fatal("cancellation did not invalidate previous configuration validator") + } + valid["If-Match"] = currentTag + if response := do(t, native, http.MethodPut, configPath, `{"enabled":true}`, valid); response.Code != 200 { + t.Fatalf("explicit reenable: %d %s", response.Code, response.Body.String()) + } + // An acknowledged guarded registration delete closes the gap between the + // final zero receipt and exact process retirement. Cancellation before the + // delete would invalidate this original validator; afterward the old node + // identity cannot authorize another admission, even when its URL is reused. + valid["If-Match"] = readTag() + response = do(t, native, http.MethodPut, path, "", valid) + if response.Code != http.StatusOK || json.Unmarshal(response.Body.Bytes(), &proof) != nil || !proof.Drained { + t.Fatalf("retirement custody proof: %d %s", response.Code, response.Body.String()) + } + valid["If-Match"] = proof.ConfigETag + response = do(t, native, http.MethodDelete, configPath, "", valid) + if response.Code != http.StatusNoContent { + t.Fatalf("guarded retirement registration removal: %d %s", response.Code, response.Body.String()) + } + privateRead := func(method, target string) *httptest.ResponseRecorder { + t.Helper() + request := httptest.NewRequest(method, target, nil) + request.Header.Set("Authorization", "Bearer "+secret) + response := httptest.NewRecorder() + worker.ServeHTTP(response, request) + return response + } + if response := privateRead(http.MethodGet, "/admin/drain"); response.Code != http.StatusServiceUnavailable { + t.Fatalf("deleted registration remained authoritative: %d %s", response.Code, response.Body.String()) + } + oldID := node.ID + replacement, err := store.Create(t.Context(), nodepool.CreateNodeInput{Name: name, Type: nodepool.NodeTypeProxy, URL: url, Enabled: new(false)}) + if err != nil || replacement.ID == oldID || replacement.Enabled { + t.Fatalf("disabled replacement registration: %+v %v", replacement, err) + } + node = replacement // Cleanup owns the exact replacement as well. + if response := privateRead(http.MethodPost, "/admin/reload-config"); response.Code != http.StatusNoContent { + t.Fatalf("replacement watcher reload: %d %s", response.Code, response.Body.String()) + } + if response := privateRead(http.MethodGet, "/admin/drain"); response.Code != http.StatusServiceUnavailable { + t.Fatalf("old process adopted replacement authority: %d %s", response.Code, response.Body.String()) + } + token, err := streamtoken.Sign(streamtoken.Claims{SessionID: "after-registration-delete", MediaPath: "/unused", PlayMethod: "direct"}, secret, time.Hour) + if err != nil { + t.Fatal(err) + } + if response := privateRead(http.MethodGet, "/stream/direct/"+token); response.Code != http.StatusServiceUnavailable { + t.Fatalf("old process reopened admission after URL reuse: %d %s", response.Code, response.Body.String()) + } +} diff --git a/internal/apiv2/admin_node_drain_fixtures_test.go b/internal/apiv2/admin_node_drain_fixtures_test.go new file mode 100644 index 0000000000..57ba875e61 --- /dev/null +++ b/internal/apiv2/admin_node_drain_fixtures_test.go @@ -0,0 +1,11 @@ +package apiv2 + +import "net/http" + +func adminNodeDrainFixtureCases() []fixtureCase { + return []fixtureCase{ + {name: "admin_node_capabilities_unavailable", operationID: "getAdminNodeCapabilities", method: http.MethodGet, path: Prefix + "/admin/nodes/capabilities", headers: bearer(adminToken), status: http.StatusOK, schema: "#/components/schemas/AdminNodeDrainCapabilities", assertHeaders: []string{"Content-Type", "Cache-Control", "ETag"}, scenario: "The administrator discovers unavailable drain and disabled commissioning services without guessing a build version."}, + {name: "admin_node_drain_unavailable", operationID: "getAdminNodeDrain", method: http.MethodGet, path: Prefix + "/admin/nodes/17/drain", headers: bearer(adminToken), status: http.StatusServiceUnavailable, schema: "#/components/schemas/Problem", assertHeaders: []string{"Content-Type", "Cache-Control"}, scenario: "A missing drain service cannot return a retirement receipt."}, + {name: "admin_node_drain_requires_authentication", operationID: "getAdminNodeDrain", method: http.MethodGet, path: Prefix + "/admin/nodes/17/drain", status: http.StatusUnauthorized, schema: "#/components/schemas/Problem", assertHeaders: []string{"Content-Type", "Cache-Control"}, scenario: "A worker retirement observation requires acting administrator authority."}, + } +} diff --git a/internal/apiv2/document.go b/internal/apiv2/document.go index 4859a1c9f3..815091237b 100644 --- a/internal/apiv2/document.go +++ b/internal/apiv2/document.go @@ -629,6 +629,7 @@ func registerAll(reg *Registry) { registerAdminNodeCommands(reg) registerAdminNodeReload(reg) registerAdminNodeConfiguration(reg) + registerAdminNodeDrain(reg) registerAdminServerStatus(reg) registerAdminBrandingAssets(reg) registerAdminJellyfinCompatWeb(reg) diff --git a/internal/apiv2/fixtures_test.go b/internal/apiv2/fixtures_test.go index b60ab79842..910bf48cab 100644 --- a/internal/apiv2/fixtures_test.go +++ b/internal/apiv2/fixtures_test.go @@ -1744,7 +1744,8 @@ func fixtureCases() []fixtureCase { cases = append(cases, deviceSignInFixtureCases()...) cases = append(cases, externalSignInFixtureCases()...) cases = append(cases, adminDownloadPreparationControlFixtureCases()...) - return append(cases, loginSessionFixtureCases()...) + cases = append(cases, loginSessionFixtureCases()...) + return append(cases, adminNodeDrainFixtureCases()...) } func loginSessionFixtureCases() []fixtureCase { diff --git a/internal/apiv2/router.go b/internal/apiv2/router.go index 77a02ba74f..c53b9a87d4 100644 --- a/internal/apiv2/router.go +++ b/internal/apiv2/router.go @@ -192,6 +192,7 @@ type Dependencies struct { AdminNodeCommands AdminNodeCommandsService AdminNodeReload AdminNodeReloadService AdminNodeConfiguration AdminNodeConfigurationService + AdminNodeDrain AdminNodeDrainService AdminServerStatus AdminServerStatusService AdminServerRestart AdminServerRestartService AdminBrandingAssets AdminBrandingAssetService diff --git a/internal/contractledger/ledger_test.go b/internal/contractledger/ledger_test.go index d91f2c27b0..c577ee1a05 100644 --- a/internal/contractledger/ledger_test.go +++ b/internal/contractledger/ledger_test.go @@ -792,6 +792,8 @@ func TestConcurrencyMarkingIsRestricted(t *testing.T) { // is empty today; the reconcile test refuses an unmapped guarded operation // that is not listed here. var guardedWithoutLegacyRow = map[string]string{ + "beginAdminNodeDrain": "V2-only durable worker admission fence: v1 disable removes placement but cannot confirm retirement. The node's monotonic configuration revision guards publication.", + "cancelAdminNodeDrain": "V2-only cancellation of the durable worker admission fence: the node's monotonic configuration revision guards cancellation and invalidates earlier drain receipts.", "updateRequestRoute": "V2-only request routing rule: the rule's revision from request_editor_revision_seq is its ETag.", "deleteRequestRoute": "V2-only request routing rule: deletion is guarded by the rule's revision.", "updateAdminRequestGroupLimit": "V2-only access-group request limit: the limit's revision from request_editor_revision_seq is its ETag; a group with none saved is revision zero.", @@ -1127,6 +1129,8 @@ func TestRetrySafetyMismatchesFire(t *testing.T) { // mutation that is not listed here, the same rule guardedWithoutLegacyRow // applies to concurrency. var mutationWithoutLegacyRow = map[string]string{ + "beginAdminNodeDrain": "V2-only durable worker admission fence (v1 is frozen). An unchanged fence is reused; replay with a pre-publication validator returns 412 without another fence.", + "cancelAdminNodeDrain": "V2-only cancellation of the durable worker admission fence. Cancellation leaves placement disabled and advances the configuration revision; a replay with the old validator returns 412 without effects.", "createShuffle": "V2-only shuffle playback (v1 is frozen). A replay after a lost response starts a second shuffle; the client never reads the first, which is deleted with other shuffles untouched for a week.", "advanceShuffle": "V2-only shuffle playback, state-gated on from_content_id: it advances only while that item is current, so a replay after success changes nothing and returns the same shuffle.", "skipShuffleItem": "V2-only shuffle playback, state-gated on next_content_id: it replaces the next item only while that item is next, so a replay after success changes nothing and returns the same shuffle.", diff --git a/internal/mediasample/caps.go b/internal/mediasample/caps.go index 7d0a744898..e73be9cd55 100644 --- a/internal/mediasample/caps.go +++ b/internal/mediasample/caps.go @@ -9,6 +9,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "golang.org/x/sync/singleflight" @@ -125,6 +126,16 @@ var capsCache = struct { list listFunc }{entries: make(map[string]Capabilities), list: runListing} +var capabilitiesInFlight atomic.Int64 + +// CapabilitiesInFlight reports claims on unfinished FFmpeg capability +// inventories, including pending singleflight admission and work detached from +// canceled callers. Coalesced callers each retain a claim until the inventory +// completes; zero proves that no listing is pending or running. +func CapabilitiesInFlight() int { + return int(capabilitiesInFlight.Load()) +} + // LoadCapabilities returns the inventory for the ffmpeg binary at ffmpegPath. // Inventories are cached per binary identity (resolved path, size, and // modification time), so replacing the binary in place loads a new one. @@ -141,6 +152,7 @@ func LoadCapabilities(ctx context.Context, ffmpegPath string) (Capabilities, err list := capsCache.list capsCache.Unlock() + capabilitiesInFlight.Add(1) resultCh := capsCache.group.DoChan(key, func() (any, error) { capsCache.Lock() cached, ok := capsCache.entries[key] @@ -159,8 +171,13 @@ func LoadCapabilities(ctx context.Context, ffmpegPath string) (Capabilities, err }) select { case <-ctx.Done(): + go func() { + <-resultCh + capabilitiesInFlight.Add(-1) + }() return Capabilities{}, ctx.Err() case result := <-resultCh: + capabilitiesInFlight.Add(-1) if result.Err != nil { return Capabilities{}, result.Err } diff --git a/internal/mediasample/caps_lifetime_test.go b/internal/mediasample/caps_lifetime_test.go new file mode 100644 index 0000000000..6329a87094 --- /dev/null +++ b/internal/mediasample/caps_lifetime_test.go @@ -0,0 +1,187 @@ +package mediasample + +import ( + "context" + "errors" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +func awaitNoCapabilitiesInFlight(t *testing.T) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + if CapabilitiesInFlight() == 0 { + return + } + if time.Now().After(deadline) { + t.Fatalf("CapabilitiesInFlight() = %d after completing the shared inventory", CapabilitiesInFlight()) + } + runtime.Gosched() + } +} + +func TestCapabilitiesLifetimeSurvivesCanceledCaller(t *testing.T) { + for _, canceledBeforeAdmission := range []bool{false, true} { + for _, failed := range []bool{false, true} { + name := "cancel after command start/success" + if canceledBeforeAdmission { + name = "cancel before command admission/success" + } + if failed { + name = strings.TrimSuffix(name, "success") + "command failure" + } + t.Run(name, func(t *testing.T) { + awaitNoCapabilitiesInFlight(t) + f := newFakeListings() + useListings(t, f) + binary := fakeBinary(t) + started, release := make(chan struct{}), make(chan struct{}) + unblock := sync.OnceFunc(func() { close(release) }) + t.Cleanup(func() { + unblock() + cleanupCtx, cleanupCancel := context.WithTimeout(context.WithoutCancel(t.Context()), 10*time.Second) + defer cleanupCancel() + _, _ = LoadCapabilities(cleanupCtx, binary) + awaitNoCapabilitiesInFlight(t) + }) + var announce sync.Once + var listings atomic.Int32 + var injectFailure atomic.Bool + injectFailure.Store(failed) + capsCache.Lock() + capsCache.list = func(ctx context.Context, name string, stdin []byte, args ...string) ([]byte, error) { + listings.Add(1) + if strings.Join(args, " ") == "-hide_banner -filters" { + announce.Do(func() { close(started) }) + select { + case <-release: + case <-ctx.Done(): + return nil, ctx.Err() + } + if injectFailure.Load() { + return nil, errors.New("injected FFmpeg capability failure") + } + } + return f.list(ctx, name, stdin, args...) + } + capsCache.Unlock() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + if canceledBeforeAdmission { + cancel() + } + done := make(chan error, 1) + go func() { + _, err := LoadCapabilities(ctx, binary) + done <- err + }() + if !canceledBeforeAdmission { + select { + case <-started: + case <-time.After(10 * time.Second): + t.Fatal("capability inventory did not start its FFmpeg listing") + } + cancel() + } + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled capability caller returned %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("canceled capability caller kept its request handler blocked") + } + if got := CapabilitiesInFlight(); got < 1 { + t.Fatalf("CapabilitiesInFlight() = %d while detached or pending FFmpeg listing has not returned", got) + } + unblock() + awaitNoCapabilitiesInFlight(t) + before := listings.Load() + injectFailure.Store(false) + caps, err := LoadCapabilities(t.Context(), binary) + if err != nil || !caps.HasMuxer("chromaprint") { + t.Fatalf("inventory after detached completion = %+v, %v", caps, err) + } + awaitNoCapabilitiesInFlight(t) + if failed && listings.Load() != before+4 { + t.Fatalf("failed inventory was not retried: listings=%d, before=%d", listings.Load(), before) + } + if !failed && listings.Load() != before { + t.Fatal("successful detached inventory was not cached") + } + }) + } + } +} + +func TestCapabilitiesLifetimeSurvivesCacheInvalidation(t *testing.T) { + awaitNoCapabilitiesInFlight(t) + f := newFakeListings() + useListings(t, f) + binary := fakeBinary(t) + started, release := make(chan struct{}), make(chan struct{}) + unblock := sync.OnceFunc(func() { close(release) }) + var announce sync.Once + var listings atomic.Int32 + capsCache.Lock() + capsCache.list = func(ctx context.Context, name string, stdin []byte, args ...string) ([]byte, error) { + listings.Add(1) + if strings.Join(args, " ") == "-hide_banner -filters" { + announce.Do(func() { close(started) }) + select { + case <-release: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return f.list(ctx, name, stdin, args...) + } + capsCache.Unlock() + t.Cleanup(func() { + unblock() + cleanupCtx, cleanupCancel := context.WithTimeout(context.WithoutCancel(t.Context()), 10*time.Second) + defer cleanupCancel() + _, _ = LoadCapabilities(cleanupCtx, binary) + awaitNoCapabilitiesInFlight(t) + }) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := LoadCapabilities(ctx, binary) + done <- err + }() + select { + case <-started: + case <-time.After(10 * time.Second): + t.Fatal("capability inventory did not start its listing") + } + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled caller returned %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("canceled caller remained blocked") + } + InvalidateCapabilities() + if got := CapabilitiesInFlight(); got < 1 { + t.Fatalf("CapabilitiesInFlight() = %d after invalidating a still-running inventory", got) + } + unblock() + awaitNoCapabilitiesInFlight(t) + before := listings.Load() + if _, err := LoadCapabilities(t.Context(), binary); err != nil { + t.Fatalf("inventory after invalidation: %v", err) + } + awaitNoCapabilitiesInFlight(t) + if got := listings.Load(); got != before+4 { + t.Fatalf("invalidated inventory reused an older generation: listings=%d, before=%d", got, before) + } +} diff --git a/internal/nodeconfig/drain.go b/internal/nodeconfig/drain.go new file mode 100644 index 0000000000..2c074dcce2 --- /dev/null +++ b/internal/nodeconfig/drain.go @@ -0,0 +1,34 @@ +package nodeconfig + +import ( + "context" + "errors" + "strings" + + "github.com/Silo-Server/silo-server/internal/serveridentity" + "github.com/Silo-Server/silo-server/internal/workerdrain" + "github.com/jackc/pgx/v5" +) + +func (w *Watcher) HasDrainAuthority() bool { return w != nil && w.pool != nil } + +// ReadDrainFence uses the worker's remembered stable registration, never a +// hostname match or a stale configuration cache. Missing registration and DB +// errors deny new admission and prevent a retirement receipt. +func (w *Watcher) ReadDrainFence(ctx context.Context) (int, string, string, error) { + if !w.HasDrainAuthority() { + return 0, "", "", workerdrain.ErrUnavailable + } + id, ok := w.NodeRowID() + if !ok { + return 0, "", "", workerdrain.ErrUnavailable + } + var fence, realm string + err := w.pool.QueryRow(ctx, `SELECT COALESCE(f.fence_id,''), s.value FROM stream_nodes n + LEFT JOIN stream_node_drain_fences f ON f.node_id=n.id + JOIN server_settings s ON s.key=$2 WHERE n.id=$1`, id, serveridentity.Key).Scan(&fence, &realm) + if errors.Is(err, pgx.ErrNoRows) || strings.TrimSpace(realm) == "" { + return 0, "", "", workerdrain.ErrUnavailable + } + return id, fence, strings.TrimSpace(realm), err +} diff --git a/internal/nodepool/admin_configuration.go b/internal/nodepool/admin_configuration.go index 122ff35d20..85a9e56ef3 100644 --- a/internal/nodepool/admin_configuration.go +++ b/internal/nodepool/admin_configuration.go @@ -38,7 +38,7 @@ func (s *AdminConfigurationStore) Snapshot(ctx context.Context) ([]*Node, int64, if err != nil { return nil, 0, err } - rows, err := tx.Query(ctx, `SELECT id, admin_revision FROM stream_nodes`) + rows, err := tx.Query(ctx, `SELECT id, GREATEST(admin_revision, drain_revision) FROM stream_nodes`) if err != nil { return nil, 0, err } @@ -64,7 +64,7 @@ func (s *AdminConfigurationStore) Snapshot(ctx context.Context) ([]*Node, int64, func readConfigurationRevision(ctx context.Context, tx pgx.Tx, id int) (int64, error) { var revision int64 - err := tx.QueryRow(ctx, `SELECT admin_revision FROM stream_nodes WHERE id=$1 FOR UPDATE`, id).Scan(&revision) + err := tx.QueryRow(ctx, `SELECT GREATEST(admin_revision, drain_revision) FROM stream_nodes WHERE id=$1 FOR UPDATE`, id).Scan(&revision) if errors.Is(err, pgx.ErrNoRows) { return 0, ErrNodeNotFound } @@ -123,7 +123,7 @@ func (s *AdminConfigurationStore) Create(ctx context.Context, input CreateNodeIn } func matchesCreation(node *Node, input CreateNodeInput) bool { - return node.Name == input.Name && node.Type == input.Type && node.URL == input.URL && node.Enabled && + return node.Name == input.Name && node.Type == input.Type && node.URL == input.URL && node.Enabled == input.enabledOnCreate() && reflect.DeepEqual(node.PublicURL, normalizeOverride(input.PublicURL)) && reflect.DeepEqual(node.Group, normalizeGroup(input.Group)) && reflect.DeepEqual(node.MaxJobs, normalizeCap(input.MaxJobs)) && diff --git a/internal/nodepool/admin_configuration_test.go b/internal/nodepool/admin_configuration_test.go index 1bd61e3871..664bd12771 100644 --- a/internal/nodepool/admin_configuration_test.go +++ b/internal/nodepool/admin_configuration_test.go @@ -16,14 +16,18 @@ import ( func configurationTestPool(t *testing.T) *pgxpool.Pool { t.Helper() dsn := os.Getenv("SILO_NODE_CONFIGURATION_TEST_DATABASE_URL") + dedicated := dsn != "" if dsn == "" { - t.Skip("SILO_NODE_CONFIGURATION_TEST_DATABASE_URL is not set") + dsn = os.Getenv("SILO_TEST_DATABASE_URL") + } + if dsn == "" { + t.Skip("requires a disposable PostgreSQL fixture") } cfg, err := pgxpool.ParseConfig(dsn) if err != nil { t.Fatal(err) } - if cfg.ConnConfig.Database != "silo_operations_node_configuration_test" { + if dedicated && cfg.ConnConfig.Database != "silo_operations_node_configuration_test" { t.Fatal("refusing non-owned node configuration database") } admin, err := pgxpool.NewWithConfig(t.Context(), cfg) @@ -53,16 +57,22 @@ func configurationTestPool(t *testing.T) *pgxpool.Pool { active_jobs integer NOT NULL DEFAULT 0, node_group text, max_jobs integer, max_bandwidth_kbps integer, egress_kbps integer NOT NULL DEFAULT 0, last_health_check timestamptz, created_at timestamptz NOT NULL DEFAULT now(), capabilities jsonb, capabilities_hash text, capabilities_refreshed_at timestamptz, last_stats jsonb, - hw_accel_override text, hw_device_override text, capability_drift text, capability_drift_baseline jsonb)`) + hw_accel_override text, hw_device_override text, capability_drift text, capability_drift_baseline jsonb, network_access jsonb)`) if err != nil { t.Fatal(err) } - migration, err := os.ReadFile("../../migrations/sql/20260906205459_node_admin_configuration_revisions.sql") - if err != nil { + if _, err = pool.Exec(t.Context(), `CREATE TABLE server_settings(key text PRIMARY KEY, value text NOT NULL); + INSERT INTO server_settings VALUES ('server.identity_id','synthetic-node-realm')`); err != nil { t.Fatal(err) } - if _, err = pool.Exec(t.Context(), strings.Split(string(migration), "-- +goose Down")[0]); err != nil { - t.Fatal(err) + for _, filename := range []string{"20260906205459_node_admin_configuration_revisions.sql", "20261008031132_add_worker_drain_fences.sql"} { + migration, err := os.ReadFile("../../migrations/sql/" + filename) + if err != nil { + t.Fatal(err) + } + if _, err = pool.Exec(t.Context(), strings.Split(string(migration), "-- +goose Down")[0]); err != nil { + t.Fatal(err) + } } return pool } diff --git a/internal/nodepool/admin_drain.go b/internal/nodepool/admin_drain.go new file mode 100644 index 0000000000..086c0ee28e --- /dev/null +++ b/internal/nodepool/admin_drain.go @@ -0,0 +1,149 @@ +package nodepool + +import ( + "context" + "errors" + "strings" + + "github.com/Silo-Server/silo-server/internal/serveridentity" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +var ErrNodeDrainConflict = errors.New("node drain configuration changed") +var ErrNodeDrainAuthorityUnavailable = errors.New("node drain deployment identity unavailable") + +type DrainConfiguration struct { + Node *Node + FenceID string + NativeServerID string +} + +func readDrainServerIdentity(ctx context.Context, tx pgx.Tx) (string, error) { + var id string + if err := tx.QueryRow(ctx, `SELECT value FROM server_settings WHERE key=$1`, serveridentity.Key).Scan(&id); err != nil { + return "", ErrNodeDrainAuthorityUnavailable + } + id = strings.TrimSpace(id) + if id == "" { + return "", ErrNodeDrainAuthorityUnavailable + } + return id, nil +} + +// ReadDrain pairs the configuration validator and fence in one snapshot. A +// health sample cannot change either part of this retirement authority. +func (s *AdminConfigurationStore) ReadDrain(ctx context.Context, id int) (DrainConfiguration, error) { + tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly}) + if err != nil { + return DrainConfiguration{}, err + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + node, err := NewTransactionalRepository(tx).GetByID(ctx, id) + if err != nil { + return DrainConfiguration{}, err + } + if err = tx.QueryRow(ctx, `SELECT GREATEST(admin_revision,drain_revision) FROM stream_nodes WHERE id=$1`, id).Scan(&node.AdminRevision); err != nil { + return DrainConfiguration{}, err + } + var fence string + err = tx.QueryRow(ctx, `SELECT fence_id FROM stream_node_drain_fences WHERE node_id=$1`, id).Scan(&fence) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return DrainConfiguration{}, err + } + realm, err := readDrainServerIdentity(ctx, tx) + if err != nil { + return DrainConfiguration{}, err + } + if err = tx.Commit(ctx); err != nil { + return DrainConfiguration{}, err + } + return DrainConfiguration{Node: node, FenceID: fence, NativeServerID: realm}, nil +} + +// BeginDrain disables placement and publishes the worker admission fence in +// the same guarded transaction. The fence survives API and worker restarts. +func (s *AdminConfigurationStore) BeginDrain(ctx context.Context, id int, guard func(int64) error) (DrainConfiguration, error) { + tx, err := s.pool.Begin(ctx) + if err != nil { + return DrainConfiguration{}, err + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + revision, err := readConfigurationRevision(ctx, tx, id) + if err != nil { + return DrainConfiguration{}, err + } + if err = guard(revision); err != nil { + return DrainConfiguration{}, err + } + realm, err := readDrainServerIdentity(ctx, tx) + if err != nil { + return DrainConfiguration{}, err + } + if _, err = tx.Exec(ctx, `UPDATE stream_nodes SET enabled=false WHERE id=$1`, id); err != nil { + return DrainConfiguration{}, err + } + var fence string + err = tx.QueryRow(ctx, `SELECT fence_id FROM stream_node_drain_fences WHERE node_id=$1`, id).Scan(&fence) + if errors.Is(err, pgx.ErrNoRows) { + fence = uuid.NewString() + if _, err = tx.Exec(ctx, `INSERT INTO stream_node_drain_fences(node_id,fence_id) VALUES($1,$2)`, id, fence); err != nil { + return DrainConfiguration{}, err + } + if _, err = tx.Exec(ctx, `UPDATE stream_nodes SET drain_revision=nextval('stream_node_admin_revision_seq') WHERE id=$1`, id); err != nil { + return DrainConfiguration{}, err + } + } else if err != nil { + return DrainConfiguration{}, err + } + node, err := NewTransactionalRepository(tx).GetByID(ctx, id) + if err != nil { + return DrainConfiguration{}, err + } + node.AdminRevision, err = readConfigurationRevision(ctx, tx, id) + if err != nil { + return DrainConfiguration{}, err + } + if err = tx.Commit(ctx); err != nil { + return DrainConfiguration{}, err + } + return DrainConfiguration{Node: node, FenceID: fence, NativeServerID: realm}, nil +} + +// CancelDrain removes the fence under the caller's original configuration +// validator. It deliberately leaves placement disabled; reenable is separate. +func (s *AdminConfigurationStore) CancelDrain(ctx context.Context, id int, guard func(int64) error) (*Node, error) { + tx, err := s.pool.Begin(ctx) + if err != nil { + return nil, err + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + revision, err := readConfigurationRevision(ctx, tx, id) + if err != nil { + return nil, err + } + if err = guard(revision); err != nil { + return nil, err + } + deleted, err := tx.Exec(ctx, `DELETE FROM stream_node_drain_fences WHERE node_id=$1`, id) + if err != nil { + return nil, err + } + if deleted.RowsAffected() > 0 { + if _, err = tx.Exec(ctx, `UPDATE stream_nodes SET drain_revision=nextval('stream_node_admin_revision_seq') WHERE id=$1`, id); err != nil { + return nil, err + } + } + node, err := NewTransactionalRepository(tx).GetByID(ctx, id) + if err != nil { + return nil, err + } + node.AdminRevision, err = readConfigurationRevision(ctx, tx, id) + if err != nil { + return nil, err + } + if err = tx.Commit(ctx); err != nil { + return nil, err + } + return node, nil +} diff --git a/internal/nodepool/admin_drain_test.go b/internal/nodepool/admin_drain_test.go new file mode 100644 index 0000000000..7d1db9dedb --- /dev/null +++ b/internal/nodepool/admin_drain_test.go @@ -0,0 +1,266 @@ +package nodepool + +import ( + "context" + "errors" + "os" + "strings" + "sync" + "testing" + "time" +) + +func TestAdminNodeDrainTransactions(t *testing.T) { + pool := configurationTestPool(t) + store := NewAdminConfigurationStore(pool) + ctx := t.Context() + node, err := store.Create(ctx, CreateNodeInput{Name: "drain", Type: NodeTypeProxy, URL: "http://drain.invalid"}) + if err != nil { + t.Fatal(err) + } + refused := errors.New("stale validator") + guard := func(want int64) func(int64) error { + return func(got int64) error { + if got != want { + return refused + } + return nil + } + } + if _, err = store.BeginDrain(ctx, node.ID, guard(node.AdminRevision+1)); !errors.Is(err, refused) { + t.Fatalf("stale fence write: %v", err) + } + before, err := store.ReadDrain(ctx, node.ID) + if err != nil || before.FenceID != "" || !before.Node.Enabled || before.Node.AdminRevision != node.AdminRevision { + t.Fatalf("guard rollback: %+v %v", before, err) + } + drained, err := store.BeginDrain(ctx, node.ID, guard(node.AdminRevision)) + if err != nil || drained.FenceID == "" || drained.Node.Enabled || drained.Node.AdminRevision <= node.AdminRevision { + t.Fatalf("durable fence: %+v %v", drained, err) + } + // A lost HTTP reply is reconciled by a durable read, never by host stats. + observed, err := store.ReadDrain(ctx, node.ID) + if err != nil || observed.FenceID != drained.FenceID || observed.Node.AdminRevision != drained.Node.AdminRevision { + t.Fatalf("response-loss reconciliation: %+v %v", observed, err) + } + if _, _, err = store.Update(ctx, node.ID, UpdateNodeInput{Enabled: new(true)}, guard(drained.Node.AdminRevision)); err == nil { + t.Fatal("reenabled retiring worker") + } + for _, input := range []UpdateNodeInput{{URL: new("http://replacement.invalid")}, {Name: new("replacement")}} { + if _, _, err = store.Update(ctx, node.ID, input, guard(drained.Node.AdminRevision)); err == nil { + t.Fatal("repointed retiring worker") + } + } + if _, err = pool.Exec(ctx, `UPDATE stream_nodes SET type='transcode' WHERE id=$1`, node.ID); err == nil { + t.Fatal("direct writer changed retiring worker identity") + } + unchanged, err := store.ReadDrain(ctx, node.ID) + if err != nil || unchanged.Node.URL != node.URL || unchanged.Node.Name != node.Name || unchanged.Node.Type != node.Type || unchanged.Node.AdminRevision != drained.Node.AdminRevision { + t.Fatalf("rejected repoint changed authority: %+v %v", unchanged, err) + } + if _, err = store.CancelDrain(ctx, node.ID, guard(node.AdminRevision)); !errors.Is(err, refused) { + t.Fatalf("stale cancellation: %v", err) + } + canceled, err := store.CancelDrain(ctx, node.ID, guard(drained.Node.AdminRevision)) + if err != nil || canceled.Enabled || canceled.AdminRevision <= drained.Node.AdminRevision { + t.Fatalf("explicit cancellation: %+v %v", canceled, err) + } + if _, err = store.BeginDrain(ctx, node.ID, guard(drained.Node.AdminRevision)); !errors.Is(err, refused) { + t.Fatalf("old drained receipt survived cancellation: %v", err) + } + node, _, err = store.Update(ctx, node.ID, UpdateNodeInput{Enabled: new(true)}, guard(canceled.AdminRevision)) + if err != nil || !node.Enabled { + t.Fatalf("reenable after cancellation: %+v %v", node, err) + } + newDrain, err := store.BeginDrain(ctx, node.ID, guard(node.AdminRevision)) + if err != nil || newDrain.FenceID == drained.FenceID { + t.Fatalf("new retirement identity: %+v %v", newDrain, err) + } + if err = store.Delete(ctx, node.ID, guard(newDrain.Node.AdminRevision)); err != nil { + t.Fatal(err) + } + var remaining int + if err = pool.QueryRow(ctx, `SELECT COUNT(*) FROM stream_node_drain_fences WHERE node_id=$1`, node.ID).Scan(&remaining); err != nil || remaining != 0 { + t.Fatalf("fence cleanup: %d %v", remaining, err) + } +} + +func TestAdminNodeDrainPublicationFaultRollsBack(t *testing.T) { + pool := configurationTestPool(t) + store := NewAdminConfigurationStore(pool) + ctx := t.Context() + node, err := store.Create(ctx, CreateNodeInput{Name: "publication-fault", Type: NodeTypeTranscode, URL: "http://fault.invalid"}) + if err != nil { + t.Fatal(err) + } + _, err = pool.Exec(ctx, `CREATE FUNCTION fail_drain_revision() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN + IF NEW.drain_revision<>OLD.drain_revision THEN RAISE EXCEPTION 'synthetic post-fence publication fault'; END IF; + RETURN NEW; END $$; + CREATE TRIGGER fail_drain_revision BEFORE UPDATE ON stream_nodes FOR EACH ROW EXECUTE FUNCTION fail_drain_revision()`) + if err != nil { + t.Fatal(err) + } + if _, err = store.BeginDrain(ctx, node.ID, func(int64) error { return nil }); err == nil { + t.Fatal("publication fault disappeared") + } + current, err := store.ReadDrain(ctx, node.ID) + if err != nil || current.FenceID != "" || !current.Node.Enabled || current.Node.AdminRevision != node.AdminRevision { + t.Fatalf("partial mutation survived rollback: %+v %v", current, err) + } + if _, err = pool.Exec(ctx, `DROP TRIGGER fail_drain_revision ON stream_nodes; DROP FUNCTION fail_drain_revision()`); err != nil { + t.Fatal(err) + } + green, err := store.BeginDrain(ctx, node.ID, func(int64) error { return nil }) + if err != nil || green.FenceID == "" || green.Node.Enabled { + t.Fatalf("publication recovery: %+v %v", green, err) + } +} + +func TestAdminNodeDrainConcurrentMigrationRollbackPreservesFence(t *testing.T) { + pool := configurationTestPool(t) + store := NewAdminConfigurationStore(pool) + ctx := t.Context() + node, err := store.Create(ctx, CreateNodeInput{Name: "rollback-race", Type: NodeTypeProxy, URL: "http://rollback.invalid"}) + if err != nil { + t.Fatal(err) + } + locked := make(chan struct{}) + release := make(chan struct{}) + drainResult := make(chan error, 1) + go func() { + _, err := store.BeginDrain(ctx, node.ID, func(int64) error { + close(locked) + select { + case <-release: + return nil + case <-ctx.Done(): + return ctx.Err() + } + }) + drainResult <- err + }() + <-locked + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + defer unblock() + migration, err := os.ReadFile("../../migrations/sql/20261008031132_add_worker_drain_fences.sql") + if err != nil { + t.Fatal(err) + } + down := strings.Split(string(migration), "-- +goose Down")[1] + rollbackResult := make(chan error, 1) + go func() { + tx, err := pool.Begin(ctx) + if err == nil { + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + _, err = tx.Exec(ctx, down) + } + rollbackResult <- err + }() + deadline := time.NewTimer(5 * time.Second) + defer deadline.Stop() + poll := time.NewTicker(10 * time.Millisecond) + defer poll.Stop() + waiting := false + for !waiting { + if err = pool.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM pg_locks WHERE relation='stream_nodes'::regclass AND mode='AccessExclusiveLock' AND NOT granted)`).Scan(&waiting); err != nil { + t.Fatal(err) + } + if waiting { + break + } + select { + case <-poll.C: + case <-deadline.C: + t.Fatal("rollback did not wait on the production node mutation") + } + } + unblock() + if err = <-drainResult; err != nil { + t.Fatal(err) + } + if err = <-rollbackResult; err == nil || !strings.Contains(err.Error(), "cancel worker drains") { + t.Fatalf("rollback erased concurrent fence: %v", err) + } + current, err := store.ReadDrain(ctx, node.ID) + if err != nil || current.FenceID == "" || current.Node.Enabled { + t.Fatalf("rollback lost durable retirement: %+v %v", current, err) + } +} + +func TestAdminNodeDrainMigrationRollbackPreservesConfigurationRevision(t *testing.T) { + pool := configurationTestPool(t) + ctx := t.Context() + store := NewAdminConfigurationStore(pool) + node, err := store.Create(ctx, CreateNodeInput{Name: "rollback-validator", Type: NodeTypeProxy, URL: "http://rollback-validator.invalid", Enabled: new(false)}) + if err != nil { + t.Fatal(err) + } + fenced, err := store.BeginDrain(ctx, node.ID, func(int64) error { return nil }) + if err != nil { + t.Fatal(err) + } + canceled, err := store.CancelDrain(ctx, node.ID, func(int64) error { return nil }) + if err != nil || canceled.AdminRevision <= fenced.Node.AdminRevision { + t.Fatalf("cancellation did not advance effective revision: %+v %v", canceled, err) + } + migration, err := os.ReadFile("../../migrations/sql/20261008031132_add_worker_drain_fences.sql") + if err != nil { + t.Fatal(err) + } + down := strings.Split(string(migration), "-- +goose Down")[1] + var originalStoredRevision int64 + if err := pool.QueryRow(ctx, `SELECT admin_revision FROM stream_nodes WHERE id=$1`, node.ID).Scan(&originalStoredRevision); err != nil { + t.Fatal(err) + } + // Inject after revision promotion and before re-enabling the original + // trigger. Goose's transaction must restore both the schema and pre-image. + faultTx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + fault := strings.Replace(down, "ALTER TABLE stream_nodes ENABLE TRIGGER stream_node_configuration_revision;", "SELECT 1/0;\nALTER TABLE stream_nodes ENABLE TRIGGER stream_node_configuration_revision;", 1) + if _, err := faultTx.Exec(ctx, fault); err == nil { + _ = faultTx.Rollback(context.WithoutCancel(ctx)) + t.Fatal("post-promotion rollback fault did not fire") + } + if err := faultTx.Rollback(context.WithoutCancel(ctx)); err != nil { + t.Fatal(err) + } + var restoredRevision int64 + var enabledTrigger bool + if err := pool.QueryRow(ctx, `SELECT n.admin_revision, t.tgenabled='O' FROM stream_nodes n JOIN pg_trigger t ON t.tgrelid='stream_nodes'::regclass AND t.tgname='stream_node_configuration_revision' WHERE n.id=$1`, node.ID).Scan(&restoredRevision, &enabledTrigger); err != nil || restoredRevision != originalStoredRevision || !enabledTrigger { + t.Fatalf("fault rollback lost revision/trigger state: %d/%d enabled=%t %v", restoredRevision, originalStoredRevision, enabledTrigger, err) + } + current, err := store.ReadDrain(ctx, node.ID) + if err != nil || current.Node.AdminRevision != canceled.AdminRevision || current.FenceID != "" { + t.Fatalf("fault rollback lost effective configuration: %+v %v", current, err) + } + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + if _, err := tx.Exec(ctx, down); err != nil { + t.Fatal(err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatal(err) + } + var revision int64 + if err := pool.QueryRow(ctx, `SELECT admin_revision FROM stream_nodes WHERE id=$1`, node.ID).Scan(&revision); err != nil || revision != canceled.AdminRevision || revision <= node.AdminRevision { + t.Fatalf("rollback resurrected original validator: revision=%d want=%d original=%d %v", revision, canceled.AdminRevision, node.AdminRevision, err) + } + // The bridge repository and original configuration trigger still operate + // on the rolled-back schema, and the next ordinary write remains newer. + if _, err := NewRepository(pool).Update(ctx, node.ID, UpdateNodeInput{Enabled: new(true)}); err != nil { + t.Fatal(err) + } + if err := pool.QueryRow(ctx, `SELECT admin_revision FROM stream_nodes WHERE id=$1`, node.ID).Scan(&revision); err != nil || revision <= canceled.AdminRevision { + t.Fatalf("configuration trigger not restored after rollback: %d %v", revision, err) + } + var remaining int + if err := pool.QueryRow(ctx, `SELECT count(*) FROM pg_class WHERE relnamespace=current_schema()::regnamespace AND relname='stream_node_drain_fences'`).Scan(&remaining); err != nil || remaining != 0 { + t.Fatalf("rollback fence table cleanup: %d %v", remaining, err) + } +} diff --git a/internal/nodepool/repository.go b/internal/nodepool/repository.go index d95809587d..0169962e95 100644 --- a/internal/nodepool/repository.go +++ b/internal/nodepool/repository.go @@ -227,6 +227,9 @@ type CreateNodeInput struct { Name string `json:"name"` Type string `json:"type"` URL string `json:"url"` + // Enabled is supplied by native v2 commissioning. The frozen v1 decoder + // ignores it, preserving the enabled=true default of that contract. + Enabled *bool `json:"-"` // PublicURL is meaningful for proxy nodes only; see Node.PublicURL. // Accepted on any node for symmetry with the acceleration overrides, // which are likewise scoped by what reads them rather than rejected. @@ -236,6 +239,10 @@ type CreateNodeInput struct { MaxBandwidthKbps *int `json:"max_bandwidth_kbps"` // nil or <= 0 = unlimited } +func (i CreateNodeInput) enabledOnCreate() bool { + return i.Enabled == nil || *i.Enabled +} + // Validate checks required fields and allowed values. func (i CreateNodeInput) Validate() error { if i.Name == "" { @@ -504,11 +511,11 @@ func (r *Repository) Create(ctx context.Context, input CreateNodeInput) (*Node, return nil, err } row := r.pool.QueryRow(ctx, - `INSERT INTO stream_nodes (name, type, url, public_url, node_group, max_jobs, max_bandwidth_kbps) - VALUES ($1, $2, $3, $4, $5, $6, $7) + `INSERT INTO stream_nodes (name, type, url, public_url, node_group, max_jobs, max_bandwidth_kbps, enabled) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING `+nodeColumns, input.Name, input.Type, input.URL, normalizeOverride(input.PublicURL), normalizeGroup(input.Group), - normalizeCap(input.MaxJobs), normalizeCap(input.MaxBandwidthKbps)) + normalizeCap(input.MaxJobs), normalizeCap(input.MaxBandwidthKbps), input.enabledOnCreate()) return scanNode(row) } diff --git a/internal/playback/copy_seek_anchor.go b/internal/playback/copy_seek_anchor.go index 8165c11760..ad11601d62 100644 --- a/internal/playback/copy_seek_anchor.go +++ b/internal/playback/copy_seek_anchor.go @@ -8,6 +8,7 @@ import ( "os/exec" "strconv" "strings" + "sync/atomic" "time" "golang.org/x/sync/singleflight" @@ -19,8 +20,9 @@ const ( ) var ( - copySeekProbeGroup singleflight.Group - copySeekProbeSlots = make(chan struct{}, maxConcurrentCopySeekProbes) + copySeekProbeGroup singleflight.Group + copySeekProbeSlots = make(chan struct{}, maxConcurrentCopySeekProbes) + copySeekAnchorsInFlight atomic.Int64 ) type copySeekAnchor struct { @@ -28,6 +30,14 @@ type copySeekAnchor struct { segment int } +// CopySeekAnchorsInFlight reports claims on unfinished shared seek-anchor work. +// Claims include pending admission and survive caller cancellation. Coalesced +// callers can each own a claim, so this is not an exact child-process count; +// zero means no seek-anchor work remains queued or running. +func CopySeekAnchorsInFlight() int { + return int(copySeekAnchorsInFlight.Load()) +} + // ResolveCopySeekAnchor returns the keyframe timestamp FFmpeg's input seek will // actually use for a copy-video restart. FFmpeg cannot discard the pre-roll // between that keyframe and requestedSeekSeconds while preserving -c:v copy, @@ -64,6 +74,7 @@ func ResolveCopySeekAnchor( strconv.FormatFloat(requestedSeekSeconds, 'f', 6, 64), strconv.Itoa(segmentDuration), }, "\x00") + copySeekAnchorsInFlight.Add(1) resultCh := copySeekProbeGroup.DoChan(key, func() (any, error) { probeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), copySeekProbeTimeout) defer cancel() @@ -79,8 +90,15 @@ func ResolveCopySeekAnchor( select { case <-ctx.Done(): + // The shared probe uses WithoutCancel and can outlive this caller. + // Transfer its claim until the command and slot cleanup complete. + go func() { + <-resultCh + copySeekAnchorsInFlight.Add(-1) + }() return 0, 0, ctx.Err() case result := <-resultCh: + copySeekAnchorsInFlight.Add(-1) if result.Err != nil { return 0, 0, result.Err } diff --git a/internal/playback/copy_seek_anchor_lifetime_test.go b/internal/playback/copy_seek_anchor_lifetime_test.go new file mode 100644 index 0000000000..e79cac5256 --- /dev/null +++ b/internal/playback/copy_seek_anchor_lifetime_test.go @@ -0,0 +1,173 @@ +//go:build unix + +package playback + +import ( + "bufio" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + "golang.org/x/sys/unix" +) + +func awaitCopySeekLifetime(t *testing.T, done func() bool, detail string) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for !done() { + if time.Now().After(deadline) { + t.Fatalf("copy seek-anchor lifetime: %s (claims=%d, slots=%d)", detail, CopySeekAnchorsInFlight(), len(copySeekProbeSlots)) + } + runtime.Gosched() + } +} + +func TestCopySeekAnchorLifetimeSurvivesCanceledCaller(t *testing.T) { + for _, pendingAdmission := range []bool{false, true} { + for _, failed := range []bool{false, true} { + name := "executing/success" + if pendingAdmission { + name = "queued/success" + } + if failed { + name = strings.TrimSuffix(name, "success") + "command failure" + } + t.Run(name, func(t *testing.T) { + awaitCopySeekLifetime(t, func() bool { return CopySeekAnchorsInFlight() == 0 && len(copySeekProbeSlots) == 0 }, "previous work did not finish") + dir := t.TempDir() + startedPath, releasePath := filepath.Join(dir, "started.fifo"), filepath.Join(dir, "release.fifo") + for _, path := range []string{startedPath, releasePath} { + if err := unix.Mkfifo(path, 0o600); err != nil { + t.Fatal(err) + } + } + startedPipe, err := os.OpenFile(startedPath, os.O_RDWR, 0o600) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = startedPipe.Close() }) + releasePipe, err := os.OpenFile(releasePath, os.O_RDWR, 0o600) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = releasePipe.Close() }) + faultPath := filepath.Join(dir, "fail") + if failed { + if err := os.WriteFile(faultPath, nil, 0o600); err != nil { + t.Fatal(err) + } + } + quote := func(value string) string { return "'" + strings.ReplaceAll(value, "'", "'\\''") + "'" } + ffmpeg := filepath.Join(dir, "ffmpeg") + script := fmt.Sprintf("#!/bin/sh\nif mkdir %s 2>/dev/null; then\n printf 'started\\n' > %s\n read ignored < %s\nfi\nif test -f %s; then\n printf 'injected seek probe fault\\n' >&2\n exit 7\nfi\nprintf '%%s\\n' '#tb 0: 1/1000' '0, 14500, 14750, 41, 178989, 0xd16e41c4'\n", quote(filepath.Join(dir, "first")), quote(startedPath), quote(releasePath), quote(faultPath)) + if err := os.WriteFile(ffmpeg, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + unblock := sync.OnceFunc(func() { _, _ = releasePipe.WriteString("release\n") }) + var releaseSlots func() + if pendingAdmission { + for range maxConcurrentCopySeekProbes { + copySeekProbeSlots <- struct{}{} + } + releaseSlots = sync.OnceFunc(func() { + for range maxConcurrentCopySeekProbes { + <-copySeekProbeSlots + } + }) + } else { + releaseSlots = func() {} + } + t.Cleanup(func() { + releaseSlots() + unblock() + // Join the production singleflight independently of the observer, + // including when the RED mutation makes it always return zero. + cleanupCtx, cleanupCancel := context.WithTimeout(context.WithoutCancel(t.Context()), 10*time.Second) + defer cleanupCancel() + _, _, _ = ResolveCopySeekAnchor(cleanupCtx, ffmpeg, "synthetic.mkv", 18, 2) + awaitCopySeekLifetime(t, func() bool { return CopySeekAnchorsInFlight() == 0 && len(copySeekProbeSlots) == 0 }, "released process retained work") + }) + started := make(chan error, 1) + go func() { _, err := bufio.NewReader(startedPipe).ReadString('\n'); started <- err }() + waitStarted := func() { + select { + case err := <-started: + if err != nil { + t.Fatal(err) + } + case <-time.After(10 * time.Second): + t.Fatal("seek-anchor child command did not start") + } + } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + if pendingAdmission { + cancel() + } + returned := make(chan error, 1) + go func() { + _, _, err := ResolveCopySeekAnchor(ctx, ffmpeg, "synthetic.mkv", 18, 2) + returned <- err + }() + if !pendingAdmission { + waitStarted() + cancel() + } + select { + case err := <-returned: + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled seek-anchor caller returned %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("canceled seek-anchor caller stayed blocked") + } + if got := CopySeekAnchorsInFlight(); got < 1 { + t.Fatalf("CopySeekAnchorsInFlight() = %d while shared command is queued or running", got) + } + if pendingAdmission { + releaseSlots() + waitStarted() + } + survivor := make(chan error, 1) + go func() { + anchor, segment, err := ResolveCopySeekAnchor(t.Context(), ffmpeg, "synthetic.mkv", 18, 2) + if err == nil && (anchor != 14.75 || segment != 7) { + err = fmt.Errorf("anchor=%v segment=%d", anchor, segment) + } + survivor <- err + }() + awaitCopySeekLifetime(t, func() bool { return CopySeekAnchorsInFlight() >= 2 }, "surviving caller was not admitted") + unblock() + select { + case err := <-survivor: + if failed && (err == nil || !strings.Contains(err.Error(), "injected seek probe fault")) { + t.Fatalf("faulted shared command returned %v", err) + } + if !failed && err != nil { + t.Fatalf("surviving caller: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("released shared command stayed blocked") + } + awaitCopySeekLifetime(t, func() bool { return CopySeekAnchorsInFlight() == 0 && len(copySeekProbeSlots) == 0 }, "completed command retained work") + if failed { + if err := os.Remove(faultPath); err != nil { + t.Fatal(err) + } + } + anchor, segment, err := ResolveCopySeekAnchor(t.Context(), ffmpeg, "synthetic.mkv", 18, 2) + if err != nil || anchor != 14.75 || segment != 7 { + t.Fatalf("follow-up probe anchor=%v segment=%d err=%v", anchor, segment, err) + } + awaitCopySeekLifetime(t, func() bool { return CopySeekAnchorsInFlight() == 0 && len(copySeekProbeSlots) == 0 }, "follow-up command retained work") + }) + } + } +} diff --git a/internal/playback/subtitle_cache.go b/internal/playback/subtitle_cache.go index aec1614aae..d031144ee5 100644 --- a/internal/playback/subtitle_cache.go +++ b/internal/playback/subtitle_cache.go @@ -13,6 +13,7 @@ import ( "slices" "strings" "sync" + "sync/atomic" "time" "github.com/Silo-Server/silo-server/internal/httpstream" @@ -87,6 +88,13 @@ const ( // serves, cleared window for background warms). type SUPExtractFunc func(ctx context.Context, opts StreamExtractOpts) error +var subtitleFillsInFlight atomic.Int64 + +// SubtitleFillsInFlight includes fills reserved before a background warm is +// scheduled. Claims survive the FFmpeg child and publication/discard cleanup; +// zero proves that no cache fill is pending or running in this process. +func SubtitleFillsInFlight() int { return int(subtitleFillsInFlight.Load()) } + // NewSubtitleCache builds a cache rooted under the transcode directory // returned by transcodeDir at call time (so runtime config changes are // honored). Pass nil to disable caching entirely. @@ -462,6 +470,9 @@ type SubtitleCacheFill struct { // failed flips when a temp-file write errors (e.g. disk full); the tee // keeps serving the client and Commit refuses to publish the entry. failed bool + // A failed Commit may be followed by Discard. The original fill must never + // release a newer fill for the same cache key or decrement its claim twice. + releaseOnce sync.Once } // BeginFill reserves the in-flight slot for the given track and creates the @@ -494,6 +505,7 @@ func (c *SubtitleCache) beginFill(inputPath string, trackIndex int, format strin return nil } c.inflight[key] = make(chan struct{}) + subtitleFillsInFlight.Add(1) c.mu.Unlock() tmp, err := os.CreateTemp(dir, key+".part-*") @@ -518,6 +530,7 @@ func (c *SubtitleCache) release(key string) { if done, ok := c.inflight[key]; ok { close(done) delete(c.inflight, key) + subtitleFillsInFlight.Add(-1) } c.mu.Unlock() } @@ -569,7 +582,7 @@ func (f *SubtitleCacheFill) Commit() error { f.Discard() return errors.New("source file changed during extract; cache fill discarded") } - defer f.c.release(f.key) + defer f.release() tmpPath := f.tmp.Name() if err := f.tmp.Sync(); err != nil { @@ -597,7 +610,11 @@ func (f *SubtitleCacheFill) Commit() error { // file is already gone and re-removal is a no-op). func (f *SubtitleCacheFill) Discard() { f.closeAndRemoveTmp() - f.c.release(f.key) + f.release() +} + +func (f *SubtitleCacheFill) release() { + f.releaseOnce.Do(func() { f.c.release(f.key) }) } func (f *SubtitleCacheFill) closeAndRemoveTmp() { diff --git a/internal/playback/subtitle_fill_lifetime_test.go b/internal/playback/subtitle_fill_lifetime_test.go new file mode 100644 index 0000000000..2c1eec56e2 --- /dev/null +++ b/internal/playback/subtitle_fill_lifetime_test.go @@ -0,0 +1,55 @@ +package playback + +import ( + "io" + "os" + "path/filepath" + "testing" +) + +func TestSubtitleFillClaimsCoverPublicationFaultAndRepeatedDiscard(t *testing.T) { + for _, tc := range []struct { + name string + fault bool + }{{"commit", false}, {"rename failure", true}} { + t.Run(tc.name, func(t *testing.T) { + cache, source := newTestCache(t) + before := SubtitleFillsInFlight() + fill := cache.BeginFill(source, 0) + if fill == nil || SubtitleFillsInFlight() != before+1 { + t.Fatal("fill was not reserved before scheduling") + } + if _, err := fill.Tee(io.Discard).Write([]byte("complete subtitles")); err != nil { + t.Fatal(err) + } + if tc.fault { + if err := os.Mkdir(filepath.Join(cache.dir(), fill.key), 0700); err != nil { + t.Fatal(err) + } + } + err := fill.Commit() + if (err != nil) != tc.fault { + t.Fatalf("publication result: %v", err) + } + if SubtitleFillsInFlight() != before { + t.Fatal("finished publication retained a claim") + } + if _, err := os.Stat(fill.tmp.Name()); !os.IsNotExist(err) { + t.Fatalf("publication retained temporary file: %v", err) + } + next := cache.BeginFill(source, 0) + if next == nil || SubtitleFillsInFlight() != before+1 { + t.Fatal("next fill lost admission") + } + fill.Discard() + fill.Discard() + if SubtitleFillsInFlight() != before+1 { + t.Fatal("old repeated discard released a newer fill") + } + next.Discard() + if SubtitleFillsInFlight() != before { + t.Fatal("next fill cleanup retained a claim") + } + }) + } +} diff --git a/internal/proxy/drain.go b/internal/proxy/drain.go new file mode 100644 index 0000000000..a234c176f4 --- /dev/null +++ b/internal/proxy/drain.go @@ -0,0 +1,74 @@ +package proxy + +import ( + "encoding/json" + "net/http" + "strings" + "time" + + "github.com/Silo-Server/silo-server/internal/auth" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/streamtoken" +) + +func (s *Server) drainMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := r.URL.Path + if s.drain == nil || (!strings.HasPrefix(path, "/stream/") && !strings.HasPrefix(path, "/downloads/") && path != "/hw-capabilities" && path != "/admin/reprobe-capabilities") { + next.ServeHTTP(w, r) + return + } + key := "" + expiry := time.Now().Add(playback.MaxTokenTTL) + parts := strings.Split(strings.Trim(path, "/"), "/") + if len(parts) >= 3 && parts[0] == "stream" && parts[1] == "v3" { + // Unknown/unauthenticated requests must not create retained permits. + if s.watcher != nil && s.watcher.Config() != nil && s.grants != nil && s.loginSessions != nil { + claims, err := auth.NewJWTService(s.watcher.Config().Auth.JWTSecret, 0, 0).ValidateToken(grantBearerToken(r)) + if err == nil && claims.TokenType == auth.TokenTypeAccess { + valid, err := s.loginSessions.IsValid(r.Context(), claims.SessionID) + card, found := s.grants.Get(r.Context(), parts[2]) + if err == nil && valid && found && card != nil && card.UserID == claims.UserID { + key = parts[2] + } + } + } + } else { + index := 2 + if len(parts) > 3 && parts[1] == kindRemux && parts[2] == "audio-v2" { + index = 3 + } + if len(parts) > index && s.watcher != nil && s.watcher.Config() != nil { + claims, err := streamtoken.Verify(parts[index], s.watcher.Config().Auth.JWTSecret) + if err == nil { + key = claims.SessionID + if claims.ExpiresAt != nil { + expiry = claims.ExpiresAt.Time + } + } + } + } + end, err := s.drain.Begin(r.Context(), key, expiry) + if err != nil { + http.Error(w, "worker retiring or drain authority unavailable", http.StatusServiceUnavailable) + return + } + defer end() + next.ServeHTTP(w, r) + }) +} + +func (s *Server) handleDrain(w http.ResponseWriter, r *http.Request) { + status, err := s.drain.Observe(r.Context()) + if err != nil { + http.Error(w, "worker drain authority unavailable", http.StatusServiceUnavailable) + return + } + if r.Method == http.MethodPost && !status.Fenced { + http.Error(w, "durable drain fence not requested", http.StatusConflict) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + _ = json.NewEncoder(w).Encode(status) +} diff --git a/internal/proxy/drain_subtitle_test.go b/internal/proxy/drain_subtitle_test.go new file mode 100644 index 0000000000..a3957d45fd --- /dev/null +++ b/internal/proxy/drain_subtitle_test.go @@ -0,0 +1,163 @@ +//go:build unix + +package proxy + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "golang.org/x/sys/unix" + + "github.com/Silo-Server/silo-server/internal/workerdrain" +) + +func TestProxyDrainCountsDetachedSubtitleFill(t *testing.T) { + for _, tc := range []struct { + name string + exitCode int + changeFile bool + committed int + }{{"publish", 0, false, 1}, {"command failure", 1, false, 0}, {"source mutation", 0, true, 0}} { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + startedPath, releasePath := filepath.Join(dir, "started.fifo"), filepath.Join(dir, "release.fifo") + for _, path := range []string{startedPath, releasePath} { + if err := unix.Mkfifo(path, 0600); err != nil { + t.Fatal(err) + } + } + startedPipe, err := os.OpenFile(startedPath, os.O_RDWR, 0600) + if err != nil { + t.Fatal(err) + } + defer func() { _ = startedPipe.Close() }() + releasePipe, err := os.OpenFile(releasePath, os.O_RDWR, 0600) + if err != nil { + t.Fatal(err) + } + defer func() { _ = releasePipe.Close() }() + quote := func(value string) string { return "'" + strings.ReplaceAll(value, "'", "'\\''") + "'" } + ffmpeg := filepath.Join(dir, "ffmpeg") + script := fmt.Sprintf("#!/bin/sh\ncase \" $* \" in *' -ss '*) printf window; exit 0;; esac\nprintf 'started\\n' > %s\nprintf partial\nread ignored < %s\nexit %d\n", quote(startedPath), quote(releasePath), tc.exitCode) + if err := os.WriteFile(ffmpeg, []byte(script), 0700); err != nil { + t.Fatal(err) + } + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { _, _ = releasePipe.WriteString("release\n") }) } + defer unblock() + t.Cleanup(func() { + deadline := time.Now().Add(5 * time.Second) + for { + parts, err := filepath.Glob(filepath.Join(dir, "subtitle-cache", "*.part-*")) + if err != nil || len(parts) == 0 { + return + } + if time.Now().After(deadline) { + t.Error("subtitle child cleanup retained temporary work") + return + } + runtime.Gosched() + } + }) + started := make(chan error, 1) + go func() { _, err := bufio.NewReader(startedPipe).ReadString('\n'); started <- err }() + const secret = "subtitle-control-secret" + srv := newSocketProxyServer(t, secret, nil) + srv.watcher.Config().Playback.FFmpegPath = ffmpeg + srv.watcher.Config().Playback.TranscodeDir = dir + var fence, deny atomic.Bool + srv.drain = workerdrain.New(func(context.Context) (int, string, string, error) { + if fence.Load() { + return 7, "durable-fence", "synthetic-realm", nil + } + return 7, "", "synthetic-realm", nil + }, srv.probesInFlight, func(context.Context, string) bool { return deny.Load() }) + source := writeSocketProxyMedia(t) + token := socketProxyMediaToken(t, secret, source) + listener := httptest.NewServer(srv.Handler()) + defer listener.Close() + response, err := listener.Client().Get(listener.URL + "/stream/subtitles/" + token + "/0.sup?windowed=1&position=1&duration=1") + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(response.Body) + _ = response.Body.Close() + if err != nil || response.StatusCode != http.StatusOK || string(body) != "window" { + t.Fatalf("routed foreground subtitle: %d %q %v", response.StatusCode, body, err) + } + select { + case err := <-started: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("actual background subtitle child did not start") + } + status := func() workerdrain.Status { + t.Helper() + request, err := http.NewRequest(http.MethodGet, listener.URL+"/admin/drain", nil) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Authorization", "Bearer "+secret) + response, err := listener.Client().Do(request) + if err != nil { + t.Fatal(err) + } + defer func() { _ = response.Body.Close() }() + var proof workerdrain.Status + if response.StatusCode != http.StatusOK || json.NewDecoder(response.Body).Decode(&proof) != nil { + t.Fatalf("subtitle drain status: %d", response.StatusCode) + } + return proof + } + fence.Store(true) + deny.Store(true) // Positive revocation closes the request's media permit. + busy := status() + if busy.Drained || busy.ActiveJobs < 1 || busy.ActiveRequests != 0 || busy.ActiveReservations != 0 { + t.Fatalf("detached subtitle falsely retired: %+v", busy) + } + if tc.changeFile { + if err := os.WriteFile(source, []byte("replaced source with different size"), 0600); err != nil { + t.Fatal(err) + } + } + unblock() + deadline := time.Now().Add(5 * time.Second) + for { + proof := status() + if proof.Drained { + if proof.ActiveJobs != 0 { + t.Fatal("drained receipt still has subtitle work") + } + break + } + if time.Now().After(deadline) { + t.Fatal("subtitle cache cleanup did not reach authoritative zero") + } + runtime.Gosched() + } + parts, err := filepath.Glob(filepath.Join(dir, "subtitle-cache", "*.part-*")) + if err != nil || len(parts) != 0 { + t.Fatalf("subtitle temporary cleanup: %v %v", parts, err) + } + entries, err := filepath.Glob(filepath.Join(dir, "subtitle-cache", "*.sup")) + if err != nil || len(entries) != tc.committed { + t.Fatalf("subtitle publication after lifecycle: %v %v", entries, err) + } + }) + } +} diff --git a/internal/proxy/drain_test.go b/internal/proxy/drain_test.go new file mode 100644 index 0000000000..d4ce685e95 --- /dev/null +++ b/internal/proxy/drain_test.go @@ -0,0 +1,132 @@ +package proxy + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/streamtoken" + "github.com/Silo-Server/silo-server/internal/workerdrain" +) + +func TestProxyDrainListenerPreservesExistingEgressAndRejectsLateDispatch(t *testing.T) { + secret := "drain-proxy-secret" + srv := newSocketProxyServer(t, secret, nil) + var fence atomic.Value + fence.Store("") + var deny atomic.Bool + srv.drain = workerdrain.New(func(context.Context) (int, string, string, error) { + return 7, fence.Load().(string), "synthetic-realm", nil + }, nil, func(context.Context, string) bool { return deny.Load() }) + path := writeSocketProxyMedia(t) + token := socketProxyMediaToken(t, secret, path) + listener := httptest.NewServer(srv.Handler()) + defer listener.Close() + get := func(url, auth string) (int, workerdrain.Status) { + t.Helper() + request, err := http.NewRequest(http.MethodGet, listener.URL+url, nil) + if err != nil { + t.Fatal(err) + } + if auth != "" { + request.Header.Set("Authorization", "Bearer "+auth) + } + response, err := listener.Client().Do(request) + if err != nil { + t.Fatal(err) + } + defer func() { _ = response.Body.Close() }() + var status workerdrain.Status + if url == "/admin/drain" && response.StatusCode == http.StatusOK { + if err = json.NewDecoder(response.Body).Decode(&status); err != nil { + t.Fatal(err) + } + } + return response.StatusCode, status + } + if code, _ := get("/stream/direct/"+token, ""); code != http.StatusOK { + t.Fatalf("initial media: %d", code) + } + fence.Store("durable-fence") + if code, status := get("/admin/drain", secret); code != http.StatusOK || !status.Fenced || status.Drained || status.ActiveReservations != 1 || status.NativeServerID != "synthetic-realm" { + t.Fatalf("quiet live token lost: %d %+v", code, status) + } + if code, _ := get("/stream/direct/"+token, ""); code != http.StatusOK { + t.Fatalf("existing egress interrupted: %d", code) + } + late, err := streamtoken.Sign(streamtoken.Claims{SessionID: "late", MediaPath: path, PlayMethod: "direct"}, secret, time.Hour) + if err != nil { + t.Fatal(err) + } + if code, _ := get("/stream/direct/"+late, ""); code != http.StatusServiceUnavailable { + t.Fatalf("late token admitted: %d", code) + } + if code, _ := get("/admin/drain", ""); code != http.StatusUnauthorized { + t.Fatalf("unauthenticated drain read: %d", code) + } + if code, _ := get("/admin/drain", "synthetic-at-rest-master-key"); code != http.StatusUnauthorized { + t.Fatalf("at-rest master key admitted as control bearer: %d", code) + } + deny.Store(true) + if code, status := get("/admin/drain", secret); code != http.StatusOK || !status.Drained || status.ActiveJobs != 0 || status.ActiveRequests != 0 || status.ActiveReservations != 0 { + t.Fatalf("sealed proof: %d %+v", code, status) + } + if code, _ := get("/stream/direct/"+token, ""); code != http.StatusServiceUnavailable { + t.Fatalf("late resumed egress after proof: %d", code) + } + fence.Store("") + deny.Store(false) + if code, _ := get("/stream/direct/"+late, ""); code != http.StatusOK { + t.Fatalf("explicit cancellation did not reopen: %d", code) + } +} + +func TestProxyHealthBindsRealmAndExactWorkerProcess(t *testing.T) { + var unavailable atomic.Bool + servers := []*Server{newSocketProxyServer(t, "node-control-secret", nil), newSocketProxyServer(t, "node-control-secret", nil)} + instances := make(map[string]bool) + for _, srv := range servers { + srv.drain = workerdrain.New(func(context.Context) (int, string, string, error) { + if unavailable.Load() { + return 0, "", "", workerdrain.ErrUnavailable + } + return 7, "", "synthetic-realm", nil + }, nil, nil) + unavailable.Store(true) + health := httptest.NewRecorder() + srv.Handler().ServeHTTP(health, httptest.NewRequest(http.MethodGet, "/api/v1/health", nil)) + var before healthResponse + if health.Code != http.StatusOK || json.Unmarshal(health.Body.Bytes(), &before) != nil || before.NativeServerID != "" || before.WorkerInstanceID != "" { + t.Fatalf("unregistered health synthesized a realm: %d %s", health.Code, health.Body.String()) + } + unavailable.Store(false) + health = httptest.NewRecorder() + srv.Handler().ServeHTTP(health, httptest.NewRequest(http.MethodGet, "/api/v1/health", nil)) + var public healthResponse + if health.Code != http.StatusOK || json.Unmarshal(health.Body.Bytes(), &public) != nil || public.NativeServerID != "synthetic-realm" || public.WorkerInstanceID == "" || health.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("public worker identity: %d %s", health.Code, health.Body.String()) + } + proof, err := srv.drain.Observe(t.Context()) + if err != nil || proof.NativeServerID != public.NativeServerID || proof.WorkerInstanceID != public.WorkerInstanceID || instances[proof.WorkerInstanceID] { + t.Fatalf("sibling/private identity confused: %+v %v", proof, err) + } + instances[proof.WorkerInstanceID] = true + } +} + +func TestProxyDrainListenerDoesNotRetainAnonymousGrantRequests(t *testing.T) { + srv := newGrantProxyServer(t, nil) + srv.drain = workerdrain.New(func(context.Context) (int, string, string, error) { return 7, "", "synthetic-realm", nil }, nil, nil) + response := grantRequest(t, srv, http.MethodGet, "/stream/v3/guessed-session", "") + if response.Code != http.StatusUnauthorized { + t.Fatalf("anonymous grant: %d", response.Code) + } + status, err := srv.drain.Observe(t.Context()) + if err != nil || status.ActiveReservations != 0 || status.ActiveRequests != 0 { + t.Fatalf("anonymous reservation retained: %+v %v", status, err) + } +} diff --git a/internal/proxy/protocol_registry.go b/internal/proxy/protocol_registry.go index a3682fbae2..9f57498faf 100644 --- a/internal/proxy/protocol_registry.go +++ b/internal/proxy/protocol_registry.go @@ -5,6 +5,7 @@ import ( "github.com/Silo-Server/silo-server/internal/netaccess" "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/workerdrain" "github.com/Silo-Server/silo-server/internal/workerprotocol" "github.com/danielgtaylor/huma/v2" ) @@ -15,11 +16,16 @@ func ProtocolReads(schemas huma.Registry) []workerprotocol.Operation { return []workerprotocol.Operation{ workerprotocol.JSONRead[playback.HWAccelInfo](schemas, "proxy", "/hw-capabilities", "(*internal/proxy.Server).handleHWCapabilities", 401, 503), workerprotocol.JSONRead[statusResponse](schemas, "proxy", "/status", "(*internal/proxy.Server).handleStatus", 401), + workerprotocol.JSONRead[workerdrain.Status](schemas, "proxy", "/admin/drain", "(*internal/proxy.Server).handleDrain", 401, 503), } } // ProtocolControls describes the existing worker admin commands, not native API aliases. func ProtocolControls(schemas huma.Registry) []workerprotocol.Operation { + drain := workerprotocol.JSONRead[workerdrain.Status](schemas, "proxy", "/admin/drain", "(*internal/proxy.Server).handleDrain", 401, 409, 503) + drain.Method = http.MethodPost + drain.RetrySafety = "natural_idempotent" + drain.Description = "Confirm the durable database admission fence; existing permits continue, and zero execution, request and permit reservations seals admission. A missing durable fence returns 409." reprobe := workerprotocol.JSONRead[reprobeCapabilitiesResponse](schemas, "proxy", "/admin/reprobe-capabilities", "(*internal/proxy.Server).handleReprobeCapabilities", 401, 409, 503) reprobe.Method = "POST" reprobe.RetrySafety = "non_retryable" @@ -28,6 +34,7 @@ func ProtocolControls(schemas huma.Registry) []workerprotocol.Operation { workerprotocol.EmptyCommand("proxy", "/admin/force-reload", "(*internal/proxy.Server).handleForceReload", "Reload configuration; active remux work is not torn down."), workerprotocol.EmptyCommand("proxy", "/admin/reload-config", "(*internal/proxy.Server).handleReloadConfig", "Reload configuration without tearing down active sessions. No durable replay receipt."), reprobe, + drain, } } diff --git a/internal/proxy/reprobe.go b/internal/proxy/reprobe.go index edc2e5b827..2b66baab80 100644 --- a/internal/proxy/reprobe.go +++ b/internal/proxy/reprobe.go @@ -97,5 +97,5 @@ func (s *Server) probesInFlight() int { if s.countProbesInFlight != nil { return s.countProbesInFlight() } - return playback.HWProbesInFlight() + tonemap.ProbesInFlight() + return playback.HWProbesInFlight() + playback.SubtitleFillsInFlight() + playback.CopySeekAnchorsInFlight() + tonemap.ProbesInFlight() + tonemap.PreflightsInFlight() + mediasample.CapabilitiesInFlight() } diff --git a/internal/proxy/server.go b/internal/proxy/server.go index ab2eb51961..b7e7952a04 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -37,6 +37,7 @@ import ( "github.com/Silo-Server/silo-server/internal/streamtelemetry" "github.com/Silo-Server/silo-server/internal/streamtoken" "github.com/Silo-Server/silo-server/internal/transcodeproxy" + "github.com/Silo-Server/silo-server/internal/workerdrain" ) // proxyRangeHeader is the HTTP Range header the media routes accept. @@ -44,6 +45,7 @@ const proxyRangeHeader = "Range" // Server is the HTTP handler for proxy mode. type Server struct { + drain *workerdrain.Gate watcher *nodeconfig.Watcher tracker *nodesessions.Tracker // nodeRowID resolves this proxy's stable stream_nodes identity. Production @@ -144,6 +146,11 @@ func NewServer(watcher *nodeconfig.Watcher, tracker *nodesessions.Tracker) *Serv } if watcher != nil { server.nodeRowID = watcher.NodeRowID + if watcher.HasDrainAuthority() { + server.drain = workerdrain.New(watcher.ReadDrainFence, server.probesInFlight, func(ctx context.Context, key string) bool { + return server.streamDeny != nil && server.streamDeny.Denied(ctx, key) + }) + } } return server } @@ -236,6 +243,7 @@ func (s *Server) Handler() http.Handler { func (s *Server) router() chi.Router { declareProxyMediaRoutes() r := chi.NewRouter() + r.Use(s.drainMiddleware) if s.clientIP != nil { r.Use(clientip.Middleware(s.clientIP)) } @@ -305,6 +313,8 @@ func (s *Server) router() chi.Router { r.Post("/admin/reload-config", s.handleReloadConfig) r.Post("/admin/reprobe-capabilities", s.handleReprobeCapabilities) r.Get("/status", s.handleStatus) + r.Get("/admin/drain", s.handleDrain) + r.Post("/admin/drain", s.handleDrain) // Network access providers running beside this proxy; the API fans its // admin status/connect/disconnect out to these with the node bearer. r.Get("/network-access/status", s.handleNetworkAccessStatus) @@ -358,6 +368,11 @@ func (s *Server) handleHWCapabilities(w http.ResponseWriter, r *http.Request) { // caller must keep the previous hash rather than publish the partial report, // exactly as a transcode node does. func (s *Server) buildCapabilitySnapshot(ctx context.Context) (playback.HWAccelInfo, error) { + end, err := s.drain.Begin(ctx, "", time.Time{}) + if err != nil { + return playback.HWAccelInfo{}, err + } + defer end() s.capabilityBuildMu.Lock() defer s.capabilityBuildMu.Unlock() return s.buildCapabilitySnapshotLocked(ctx) @@ -474,6 +489,7 @@ func (s *Server) storeCapabilityHash(hash string) { } type healthResponse struct { + workerdrain.Identity Status string `json:"status"` ActiveJobs int `json:"active_jobs"` EgressKbps int `json:"egress_kbps"` @@ -504,7 +520,11 @@ type healthResponse struct { NetworkAccess netaccess.NodeNetworkAccess `json:"network_access,omitempty"` } -func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { +func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { + identityCtx, cancel := context.WithTimeout(r.Context(), 2*time.Second) + defer cancel() + identity := s.drain.PublicIdentity(identityCtx) + w.Header().Set("Cache-Control", "no-store") activeJobs := 0 if s.tracker != nil { activeJobs = s.tracker.ActiveCount() @@ -516,6 +536,7 @@ func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(healthResponse{ + Identity: identity, Status: "ok", ActiveJobs: activeJobs, EgressKbps: s.egress.RateKbps(), diff --git a/internal/proxy/streaming_protocol.go b/internal/proxy/streaming_protocol.go index 08d6c395e7..d9f26c9245 100644 --- a/internal/proxy/streaming_protocol.go +++ b/internal/proxy/streaming_protocol.go @@ -9,11 +9,12 @@ import ( "github.com/danielgtaylor/huma/v2" ) +const kindRemux = "remux" + // ProtocolStreaming describes the remaining proxy remux/HLS token and grant // routes. Only these existing relays retain open-ended origin responses. func ProtocolStreaming() []workerprotocol.Operation { const ( - kindRemux = "remux" kindAudio = "audio" kindManifest = "manifest" kindSegment = "segment" diff --git a/internal/routeinventory/classify.go b/internal/routeinventory/classify.go index 268552a24e..6b7b960385 100644 --- a/internal/routeinventory/classify.go +++ b/internal/routeinventory/classify.go @@ -837,6 +837,9 @@ var infrastructureMiddleware = []string{ // token on every listener; it records the access path and never grants // or changes authorization. "netaccess.Middleware", + // drainMiddleware fences new worker admissions. Token validation identifies + // existing permits; it never grants the route's bearer or media authority. + "drainMiddleware", } func classifyAuth(middleware []string) (string, []string) { diff --git a/internal/tonemap/preflight.go b/internal/tonemap/preflight.go index 4db27b8bd2..3cd54393ea 100644 --- a/internal/tonemap/preflight.go +++ b/internal/tonemap/preflight.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "golang.org/x/sync/singleflight" @@ -63,6 +64,24 @@ var ffmpegVersionCache = struct { group singleflight.Group }{entries: make(map[string][]byte)} +var preflightsInFlight atomic.Int64 + +// PreflightsInFlight reports outstanding claims on source-preflight and +// FFmpeg identity work. Claims begin before singleflight schedules a command +// and survive a canceled caller until its shared result arrives. Coalesced +// callers each retain a claim, so the count may exceed the number of processes; +// zero proves that none of this work is pending or running. +func PreflightsInFlight() int { + return int(preflightsInFlight.Load()) +} + +func releaseDetachedPreflight(resultCh <-chan singleflight.Result) { + go func() { + <-resultCh + preflightsInFlight.Add(-1) + }() +} + var ( // ErrSourcePreflightUnavailable identifies an operational validation failure // that may succeed when the executor or probe is retried. @@ -105,6 +124,7 @@ func ValidateSourceWithRunner(ctx context.Context, request SourcePreflightReques if ok { return cachedPreflightError(entry) } + preflightsInFlight.Add(1) resultCh := sourcePreflightCache.group.DoChan(key, func() (any, error) { entry, ok := sourcePreflightCacheLookup(key, time.Now()) if ok { @@ -129,8 +149,10 @@ func ValidateSourceWithRunner(ctx context.Context, request SourcePreflightReques }) select { case <-preflightCtx.Done(): + releaseDetachedPreflight(resultCh) return fmt.Errorf("%w: %w", ErrSourcePreflightUnavailable, preflightCtx.Err()) case result := <-resultCh: + preflightsInFlight.Add(-1) if result.Err != nil { return result.Err } @@ -141,6 +163,8 @@ func ValidateSourceWithRunner(ctx context.Context, request SourcePreflightReques return cachedPreflightError(entry) } } + preflightsInFlight.Add(1) + defer preflightsInFlight.Add(-1) return runSourcePreflight(preflightCtx, request, run) } @@ -241,6 +265,8 @@ func sourcePreflightKey(ctx context.Context, request SourcePreflightRequest, run func ffmpegVersionForPreflight(ctx context.Context, ffmpegPath string, run CommandRunner) ([]byte, error) { resolved, cacheKey, cacheable := FFmpegBinaryIdentity(ffmpegPath) if !cacheable { + preflightsInFlight.Add(1) + defer preflightsInFlight.Add(-1) return runBounded(ctx, run, ffmpegPath, "-version") } ffmpegVersionCache.Lock() @@ -250,6 +276,7 @@ func ffmpegVersionForPreflight(ctx context.Context, ffmpegPath string, run Comma return result, nil } ffmpegVersionCache.Unlock() + preflightsInFlight.Add(1) resultCh := ffmpegVersionCache.group.DoChan(cacheKey, func() (any, error) { ffmpegVersionCache.Lock() cached, ok := ffmpegVersionCache.entries[cacheKey] @@ -273,8 +300,10 @@ func ffmpegVersionForPreflight(ctx context.Context, ffmpegPath string, run Comma }) select { case <-ctx.Done(): + releaseDetachedPreflight(resultCh) return nil, ctx.Err() case result := <-resultCh: + preflightsInFlight.Add(-1) if result.Err != nil { return nil, result.Err } diff --git a/internal/tonemap/preflight_lifetime_test.go b/internal/tonemap/preflight_lifetime_test.go new file mode 100644 index 0000000000..399e17550e --- /dev/null +++ b/internal/tonemap/preflight_lifetime_test.go @@ -0,0 +1,143 @@ +package tonemap + +import ( + "context" + "errors" + "os" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +func awaitNoPreflightsInFlight(t *testing.T) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + if PreflightsInFlight() == 0 { + return + } + if time.Now().After(deadline) { + t.Fatalf("PreflightsInFlight() = %d after completing the shared work", PreflightsInFlight()) + } + runtime.Gosched() + } +} + +func TestPreflightLifetimeSurvivesCanceledCaller(t *testing.T) { + for _, command := range []string{"source", "version"} { + for _, failed := range []bool{false, true} { + name := command + "/success" + if failed { + name = command + "/command failure" + } + t.Run(name, func(t *testing.T) { + awaitNoPreflightsInFlight(t) + resetSourcePreflightCache(t) + t.Setenv("TMPDIR", t.TempDir()) + request := sourcePreflightTestRequest(t) + started, release := make(chan struct{}), make(chan struct{}) + unblock := sync.OnceFunc(func() { close(release) }) + var announce sync.Once + var blockedCalls, conversions atomic.Int32 + var injectFailure atomic.Bool + injectFailure.Store(failed) + outputPath := make(chan string, 1) + runner := func(ctx context.Context, name string, args ...string) ([]byte, error) { + isVersion := len(args) == 1 && args[0] == "-version" + isConversion := !isVersion && !strings.Contains(name, "ffprobe") + if command == "version" && isVersion || command == "source" && isConversion { + blockedCalls.Add(1) + announce.Do(func() { + if isConversion { + path := args[len(args)-1] + if err := os.WriteFile(path, []byte("temporary conversion"), 0o600); err != nil { + outputPath <- "" + } else { + outputPath <- path + } + } + close(started) + }) + select { + case <-release: + case <-ctx.Done(): + return nil, ctx.Err() + } + if injectFailure.Load() { + return nil, errors.New("injected FFmpeg command failure") + } + } + if isVersion { + return []byte("ffmpeg version lifetime test"), nil + } + if strings.Contains(name, "ffprobe") { + if !strings.Contains(strings.Join(args, " "), "stream=codec_name") { + return []byte(`{"frames":[{"color_range":"tv","color_space":"bt2020nc","color_transfer":"smpte2084","color_primaries":"bt2020"}]}`), nil + } + return []byte(`{"streams":[{"codec_name":"h264","pix_fmt":"yuv420p","color_range":"tv","color_space":"bt709","color_transfer":"bt709","color_primaries":"bt709","side_data_list":[]}],"frames":[{"color_range":"tv","color_space":"bt709","color_transfer":"bt709","color_primaries":"bt709","side_data_list":[]}]}`), nil + } + conversions.Add(1) + return nil, nil + } + t.Cleanup(func() { + unblock() + // Join the real cache task even when the counter itself is the + // failing RED oracle; no background fixture may outlive cleanup. + cleanupCtx, cleanupCancel := context.WithTimeout(context.WithoutCancel(t.Context()), 10*time.Second) + defer cleanupCancel() + _ = ValidateSourceWithRunner(cleanupCtx, request, runner) + awaitNoPreflightsInFlight(t) + }) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 1) + go func() { done <- ValidateSourceWithRunner(ctx, request, runner) }() + select { + case <-started: + case <-time.After(10 * time.Second): + t.Fatal("source validation did not start its FFmpeg command") + } + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled caller returned %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("canceled source validation kept its request handler blocked") + } + if got := PreflightsInFlight(); got < 1 { + t.Fatalf("PreflightsInFlight() = %d while a detached %s command is still running", got, command) + } + unblock() + awaitNoPreflightsInFlight(t) + if command == "source" { + path := <-outputPath + if path == "" { + t.Fatal("command fixture did not write its temporary output") + } + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("source preflight did not remove temporary output: %v", err) + } + } + // An operational fault must release the claim and permit retry. + // A successful shared task must be reused rather than run again. + before := blockedCalls.Load() + injectFailure.Store(false) + if err := ValidateSourceWithRunner(t.Context(), request, runner); err != nil { + t.Fatalf("source validation after completing detached command: %v", err) + } + awaitNoPreflightsInFlight(t) + if failed && blockedCalls.Load() <= before { + t.Fatal("operational failure was incorrectly cached") + } + if !failed && blockedCalls.Load() != before { + t.Fatal("successful shared result was not reused") + } + }) + } + } +} diff --git a/internal/transcodenode/drain.go b/internal/transcodenode/drain.go new file mode 100644 index 0000000000..19dfb2086d --- /dev/null +++ b/internal/transcodenode/drain.go @@ -0,0 +1,86 @@ +package transcodenode + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "strings" + "time" + + "github.com/Silo-Server/silo-server/internal/mediasample" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/tonemap" +) + +const ( + drainStartSegment = "start" + drainTranscodeSegment = "transcode" +) + +func (s *Server) drainActiveJobs() int { + s.gpu.mu.Lock() + busy := s.gpu.workers + if s.gpu.reprobing { + busy++ + } + s.gpu.mu.Unlock() + return int(s.activeJobs.Load()) + busy + playback.HWProbesInFlight() + playback.SubtitleFillsInFlight() + playback.CopySeekAnchorsInFlight() + tonemap.ProbesInFlight() + tonemap.PreflightsInFlight() + mediasample.CapabilitiesInFlight() +} + +func (s *Server) drainMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := r.URL.Path + if s.drain == nil || path == "/admin/drain" || path == "/api/v1/health" || path == "/metrics" || path == "/status" || path == "/admin/reload-config" || path == "/admin/force-reload" || r.Method == http.MethodDelete || strings.HasSuffix(path, "/progress") || strings.HasSuffix(path, "/downloaded") { + next.ServeHTTP(w, r) + return + } + // All media and execution paths on this listener require the node bearer. + // Do not retain retirement reservations from unauthenticated requests. + if s.watcher == nil || s.watcher.Config() == nil || r.Header.Get("Authorization") != "Bearer "+s.watcher.Config().Auth.JWTSecret { + next.ServeHTTP(w, r) + return + } + key := "" + parts := strings.Split(strings.Trim(path, "/"), "/") + if len(parts) >= 2 && (parts[0] == drainTranscodeSegment || parts[0] == "remux") && parts[1] != drainStartSegment { + key = parts[1] + } + if path == "/transcode/start" { + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 64<<10)) + if err != nil { + http.Error(w, "invalid request body", http.StatusBadRequest) + return + } + r.Body = io.NopCloser(bytes.NewReader(body)) + var input struct { + SessionID string `json:"session_id"` + } + if json.Unmarshal(body, &input) == nil { + key = input.SessionID + } + } + end, err := s.drain.Begin(r.Context(), key, time.Now().Add(playback.MaxTokenTTL)) + if err != nil { + http.Error(w, "worker retiring or drain authority unavailable", http.StatusServiceUnavailable) + return + } + defer end() + next.ServeHTTP(w, r) + }) +} + +func (s *Server) handleDrain(w http.ResponseWriter, r *http.Request) { + status, err := s.drain.Observe(r.Context()) + if err != nil { + http.Error(w, "worker drain authority unavailable", http.StatusServiceUnavailable) + return + } + if r.Method == http.MethodPost && !status.Fenced { + http.Error(w, "durable drain fence not requested", http.StatusConflict) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + _ = json.NewEncoder(w).Encode(status) +} diff --git a/internal/transcodenode/drain_copy_seek_test.go b/internal/transcodenode/drain_copy_seek_test.go new file mode 100644 index 0000000000..10ad95fbc8 --- /dev/null +++ b/internal/transcodenode/drain_copy_seek_test.go @@ -0,0 +1,163 @@ +//go:build unix + +package transcodenode + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "golang.org/x/sys/unix" + + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/workerdrain" +) + +func TestTranscodeDrainCountsCanceledCopyRecoveryAfterStop(t *testing.T) { + dir := t.TempDir() + startedPath, releasePath, producerPath := filepath.Join(dir, "started.fifo"), filepath.Join(dir, "release.fifo"), filepath.Join(dir, "producer.fifo") + for _, path := range []string{startedPath, releasePath, producerPath} { + if err := unix.Mkfifo(path, 0600); err != nil { + t.Fatal(err) + } + } + startedPipe, err := os.OpenFile(startedPath, os.O_RDWR, 0600) + if err != nil { + t.Fatal(err) + } + defer func() { _ = startedPipe.Close() }() + releasePipe, err := os.OpenFile(releasePath, os.O_RDWR, 0600) + if err != nil { + t.Fatal(err) + } + defer func() { _ = releasePipe.Close() }() + quote := func(value string) string { return "'" + strings.ReplaceAll(value, "'", "'\\''") + "'" } + ffmpeg := filepath.Join(dir, "ffmpeg") + manifest := "#EXTM3U\\n#EXT-X-VERSION:7\\n#EXT-X-TARGETDURATION:3\\n#EXT-X-MEDIA-SEQUENCE:9\\n#EXT-X-MAP:URI=\"init.mp4\"\\n#EXTINF:2.669000,\\nseg_00009.m4s\\n#EXTINF:1.669000,\\nseg_00010.m4s\\n#EXTINF:1.668000,\\nseg_00011.m4s\\n" + script := fmt.Sprintf("#!/bin/sh\nfor arg in \"$@\"; do\n if [ \"$arg\" = framecrc ]; then\n printf 'started\\n' > %s\n read ignored < %s\n printf '%%s\\n' '#tb 0: 1/1000' '0, 18000, 18000, 41, 1024, 0x12345678'\n exit 0\n fi\ndone\nprintf '%%b' %s > %s\nprintf fragment > %s\nexec cat %s > /dev/null\n", quote(startedPath), quote(releasePath), quote(manifest), quote(filepath.Join(dir, "stream.m3u8")), quote(filepath.Join(dir, "seg_00011.m4s")), quote(producerPath)) + if err := os.WriteFile(ffmpeg, []byte(script), 0700); err != nil { + t.Fatal(err) + } + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { _, _ = releasePipe.WriteString("release\n") }) } + defer unblock() + started := make(chan error, 1) + go func() { _, err := bufio.NewReader(startedPipe).ReadString('\n'); started <- err }() + const sid = "copy-node-drain" + session, err := playback.StartTranscode(t.Context(), playback.TranscodeOpts{SessionID: sid, InputPath: "/media/synthetic.mkv", OutputDir: dir, FFmpegPath: ffmpeg, SeekSeconds: 18.261, StreamOriginSeconds: 18, CopySeekAnchorResolved: true, TargetCodecVideo: "copy", TargetCodecAudio: "copy", SegmentDuration: 2, StartSegmentNumber: 9}) + if err != nil { + t.Fatal(err) + } + defer func() { _ = session.Close() }() + deadline := time.Now().Add(5 * time.Second) + for { + data, err := os.ReadFile(filepath.Join(dir, "stream.m3u8")) + fragment, fragmentErr := os.Stat(filepath.Join(dir, "seg_00011.m4s")) + if err == nil && strings.Contains(string(data), "seg_00011.m4s\n") && fragmentErr == nil && fragment.Size() > 0 { + break + } + if time.Now().After(deadline) { + t.Fatal("actual producer did not publish copy recovery manifest") + } + runtime.Gosched() + } + srv := newTestServer(t) + srv.sessions[sid] = session + srv.activeJobs.Store(1) + var fenced atomic.Bool + srv.drain = workerdrain.New(func(context.Context) (int, string, string, error) { + if fenced.Load() { + return 7, "durable-fence", "synthetic-realm", nil + } + return 7, "", "synthetic-realm", nil + }, srv.drainActiveJobs, nil) + listener := httptest.NewServer(srv.Handler()) + defer listener.Close() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + request, err := http.NewRequestWithContext(ctx, http.MethodGet, listener.URL+"/transcode/"+sid+"/segment/seg_00010.m4s", nil) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Authorization", "Bearer "+testSecret) + returned := make(chan error, 1) + go func() { + response, err := listener.Client().Do(request) + if response != nil { + _ = response.Body.Close() + } + returned <- err + }() + select { + case err := <-started: + if err != nil { + t.Fatal(err) + } + case <-time.After(10 * time.Second): + t.Fatal("actual routed copy recovery did not start detached FFmpeg") + } + cancel() + select { + case err := <-returned: + if err == nil { + t.Fatal("canceled recovery did not return its cancellation") + } + case <-time.After(5 * time.Second): + t.Fatal("copy recovery request did not detach") + } + fenced.Store(true) + private := func(method, path string) *http.Response { + t.Helper() + request, err := http.NewRequest(method, listener.URL+path, nil) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Authorization", "Bearer "+testSecret) + response, err := listener.Client().Do(request) + if err != nil { + t.Fatal(err) + } + return response + } + stopped := private(http.MethodDelete, "/transcode/"+sid) + _ = stopped.Body.Close() + if stopped.StatusCode != http.StatusNoContent || srv.activeJobs.Load() != 0 { + t.Fatalf("original session stop did not finish: %d jobs=%d", stopped.StatusCode, srv.activeJobs.Load()) + } + observe := func() workerdrain.Status { + t.Helper() + response := private(http.MethodGet, "/admin/drain") + defer func() { _ = response.Body.Close() }() + var status workerdrain.Status + if response.StatusCode != http.StatusOK || json.NewDecoder(response.Body).Decode(&status) != nil { + t.Fatalf("copy recovery drain: %d", response.StatusCode) + } + return status + } + busy := observe() + if busy.Drained || busy.ActiveJobs < 1 || busy.ActiveRequests != 0 || busy.ActiveReservations != 0 { + t.Fatalf("stopped session hid detached recovery: %+v", busy) + } + unblock() + deadline = time.Now().Add(5 * time.Second) + for playback.CopySeekAnchorsInFlight() != 0 { + if time.Now().After(deadline) { + t.Fatal("copy recovery child did not finish") + } + runtime.Gosched() + } + if zero := observe(); !zero.Drained || zero.ActiveJobs != 0 { + t.Fatalf("finished recovery retained drain work: %+v", zero) + } +} diff --git a/internal/transcodenode/drain_detached_test.go b/internal/transcodenode/drain_detached_test.go new file mode 100644 index 0000000000..a5b2af03a7 --- /dev/null +++ b/internal/transcodenode/drain_detached_test.go @@ -0,0 +1,122 @@ +//go:build unix + +package transcodenode + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + "golang.org/x/sys/unix" + + "github.com/Silo-Server/silo-server/internal/tonemap" + "github.com/Silo-Server/silo-server/internal/workerdrain" +) + +// The production tone-map cache runs a real child command on its shared +// background context. A FIFO makes its lifetime observable and deterministic; +// canceling its HTTP caller cannot be mistaken for the process exiting. +func TestTranscodeDrainCountsCanceledToneMapProbe(t *testing.T) { + dir := t.TempDir() + startedPath := filepath.Join(dir, "started.fifo") + releasePath := filepath.Join(dir, "release.fifo") + for _, path := range []string{startedPath, releasePath} { + if err := unix.Mkfifo(path, 0600); err != nil { + t.Fatal(err) + } + } + startedPipe, err := os.OpenFile(startedPath, os.O_RDWR, 0600) + if err != nil { + t.Fatal(err) + } + defer func() { _ = startedPipe.Close() }() + releasePipe, err := os.OpenFile(releasePath, os.O_RDWR, 0600) + if err != nil { + t.Fatal(err) + } + defer func() { _ = releasePipe.Close() }() + quote := func(value string) string { return "'" + strings.ReplaceAll(value, "'", "'\\''") + "'" } + ffmpeg := filepath.Join(dir, "ffmpeg") + script := fmt.Sprintf("#!/bin/sh\nif mkdir %s 2>/dev/null; then\n printf 'started\\n' > %s\n read ignored < %s\nfi\nexit 1\n", quote(filepath.Join(dir, "first")), quote(startedPath), quote(releasePath)) + if err = os.WriteFile(ffmpeg, []byte(script), 0700); err != nil { + t.Fatal(err) + } + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { _, _ = releasePipe.WriteString("release\n") }) } + defer unblock() + started := make(chan error, 1) + go func() { _, err := bufio.NewReader(startedPipe).ReadString('\n'); started <- err }() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + returned := make(chan error, 1) + go func() { _, err := tonemap.Probe(ctx, ffmpeg, tonemap.BackendSoftware, ""); returned <- err }() + select { + case err := <-started: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("actual probe command did not start") + } + cancel() + select { + case err := <-returned: + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled probe returned: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("caller did not detach from probe") + } + if tonemap.ProbesInFlight() < 1 { + t.Fatal("actual child work disappeared with caller") + } + srv := newTestServer(t) + srv.drain = workerdrain.New(func(context.Context) (int, string, string, error) { return 7, "durable-fence", "synthetic-realm", nil }, srv.drainActiveJobs, nil) + listener := httptest.NewServer(srv.Handler()) + defer listener.Close() + status := func() workerdrain.Status { + t.Helper() + request, err := http.NewRequest(http.MethodGet, listener.URL+"/admin/drain", nil) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Authorization", "Bearer "+testSecret) + response, err := listener.Client().Do(request) + if err != nil { + t.Fatal(err) + } + defer func() { _ = response.Body.Close() }() + var status workerdrain.Status + if response.StatusCode != 200 || json.NewDecoder(response.Body).Decode(&status) != nil { + t.Fatalf("drain status: %d", response.StatusCode) + } + return status + } + busy := status() + if busy.Drained || busy.ActiveJobs < 1 || busy.ActiveRequests != 0 { + t.Fatalf("detached execution falsely retired: %+v", busy) + } + unblock() + deadline := time.Now().Add(5 * time.Second) + for tonemap.ProbesInFlight() != 0 { + if time.Now().After(deadline) { + t.Fatal("released child work did not exit") + } + runtime.Gosched() + } + zero := status() + if !zero.Drained || zero.ActiveJobs != 0 { + t.Fatalf("finished child work retained: %+v", zero) + } +} diff --git a/internal/transcodenode/drain_test.go b/internal/transcodenode/drain_test.go new file mode 100644 index 0000000000..23695f7bcb --- /dev/null +++ b/internal/transcodenode/drain_test.go @@ -0,0 +1,77 @@ +package transcodenode + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Silo-Server/silo-server/internal/workerdrain" +) + +func TestTranscodeDrainListenerCountsTeardownAndRefusesLateExecution(t *testing.T) { + srv := newTestServer(t) + srv.drain = workerdrain.New(func(context.Context) (int, string, string, error) { return 7, "durable-fence", "synthetic-realm", nil }, srv.drainActiveJobs, nil) + srv.gpu.holdWork() // activeJobs has already dropped, but Close is still running. + listener := httptest.NewServer(srv.Handler()) + defer listener.Close() + request := func(method, path, body string) (int, workerdrain.Status) { + t.Helper() + req, err := http.NewRequest(method, listener.URL+path, strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer "+testSecret) + resp, err := listener.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + var status workerdrain.Status + if path == "/admin/drain" && resp.StatusCode == http.StatusOK { + if err = json.NewDecoder(resp.Body).Decode(&status); err != nil { + t.Fatal(err) + } + } + return resp.StatusCode, status + } + if code, status := request(http.MethodGet, "/admin/drain", ""); code != http.StatusOK || status.Drained || status.ActiveJobs != 1 || status.NativeServerID != "synthetic-realm" { + t.Fatalf("teardown omitted: %d %+v", code, status) + } + if code, _ := request(http.MethodPost, "/transcode/start", `{"session_id":"late","input_path":"/irrelevant"}`); code != http.StatusServiceUnavailable { + t.Fatalf("late start: %d", code) + } + for _, path := range []string{"/remux/late", "/transcode/late/master.m3u8", "/transcode/late/segment/0.ts", "/hw-capabilities"} { + if code, _ := request(http.MethodGet, path, ""); code != http.StatusServiceUnavailable { + t.Fatalf("late %s: %d", path, code) + } + } + srv.gpu.endWork() + if code, status := request(http.MethodPost, "/admin/drain", ""); code != http.StatusOK || !status.Drained || status.ActiveRequests != 0 || status.ActiveReservations != 0 { + t.Fatalf("post-teardown proof: %d %+v", code, status) + } +} + +func TestTranscodeHealthBindsExactPrivateWorkerIdentity(t *testing.T) { + srv := newTestServer(t) + srv.drain = workerdrain.New(func(context.Context) (int, string, string, error) { return 7, "", "synthetic-realm", nil }, srv.drainActiveJobs, nil) + health := httptest.NewRecorder() + srv.Handler().ServeHTTP(health, httptest.NewRequest(http.MethodGet, "/api/v1/health", nil)) + var public HealthResponse + if health.Code != http.StatusOK || json.Unmarshal(health.Body.Bytes(), &public) != nil || public.NativeServerID != "synthetic-realm" || public.WorkerInstanceID == "" || health.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("public worker identity: %d %s", health.Code, health.Body.String()) + } + proof, err := srv.drain.Observe(t.Context()) + if err != nil || proof.NativeServerID != public.NativeServerID || proof.WorkerInstanceID != public.WorkerInstanceID { + t.Fatalf("public/private identity mismatch: %+v %v", proof, err) + } + request := httptest.NewRequest(http.MethodGet, "/admin/drain", nil) + request.Header.Set("Authorization", "Bearer synthetic-at-rest-master-key") + response := httptest.NewRecorder() + srv.Handler().ServeHTTP(response, request) + if response.Code != http.StatusUnauthorized { + t.Fatalf("at-rest master key admitted as control bearer: %d", response.Code) + } +} diff --git a/internal/transcodenode/protocol_registry.go b/internal/transcodenode/protocol_registry.go index 4e157866de..73a317a011 100644 --- a/internal/transcodenode/protocol_registry.go +++ b/internal/transcodenode/protocol_registry.go @@ -5,6 +5,7 @@ import ( "github.com/Silo-Server/silo-server/internal/nodemetrics" "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/workerdrain" "github.com/Silo-Server/silo-server/internal/workerprotocol" "github.com/danielgtaylor/huma/v2" ) @@ -25,11 +26,16 @@ func ProtocolReads(schemas huma.Registry) []workerprotocol.Operation { return []workerprotocol.Operation{ workerprotocol.JSONRead[playback.HWAccelInfo](schemas, "transcode_node", "/hw-capabilities", "(*internal/transcodenode.Server).handleHWCapabilities", 401, 503), workerprotocol.JSONRead[statusResponse](schemas, "transcode_node", "/status", "(*internal/transcodenode.Server).handleStatus", 401, 503), + workerprotocol.JSONRead[workerdrain.Status](schemas, "transcode_node", "/admin/drain", "(*internal/transcodenode.Server).handleDrain", 401, 503), } } // ProtocolControls describes the existing worker admin commands, not native API aliases. func ProtocolControls(schemas huma.Registry) []workerprotocol.Operation { + drain := workerprotocol.JSONRead[workerdrain.Status](schemas, "transcode_node", "/admin/drain", "(*internal/transcodenode.Server).handleDrain", 401, 409, 503) + drain.Method = "POST" + drain.RetrySafety = idempotent + drain.Description = "Confirm the durable database admission fence; existing permits continue, and zero execution, request and permit reservations seals admission. A missing durable fence returns 409." reprobe := workerprotocol.JSONRead[reprobeCapabilitiesResponse](schemas, "transcode_node", "/admin/reprobe-capabilities", "(*internal/transcodenode.Server).handleReprobeCapabilities", 401, 409, 503) reprobe.Method = "POST" reprobe.RetrySafety = "non_retryable" @@ -38,5 +44,6 @@ func ProtocolControls(schemas huma.Registry) []workerprotocol.Operation { workerprotocol.EmptyCommand("transcode_node", "/admin/force-reload", "(*internal/transcodenode.Server).handleForceReload", "Reload configuration and tear down active sessions and delivery authority. Failure can follow partial effects.", 503), workerprotocol.EmptyCommand("transcode_node", "/admin/reload-config", "(*internal/transcodenode.Server).handleReloadConfig", "Reload configuration without tearing down active sessions. No durable replay receipt.", 503), reprobe, + drain, } } diff --git a/internal/transcodenode/reprobe.go b/internal/transcodenode/reprobe.go index ded05c74bd..4d56bec6a2 100644 --- a/internal/transcodenode/reprobe.go +++ b/internal/transcodenode/reprobe.go @@ -62,7 +62,7 @@ func (s *Server) handleReprobeCapabilities(w http.ResponseWriter, r *http.Reques // same exclusion the transcode-start path consults, so from here until the // deferred release no new GPU work is admitted. busy, ok := s.gpu.beginReprobe(func() int { - return int(s.activeJobs.Load()) + playback.HWProbesInFlight() + tonemap.ProbesInFlight() + return int(s.activeJobs.Load()) + playback.HWProbesInFlight() + playback.SubtitleFillsInFlight() + playback.CopySeekAnchorsInFlight() + tonemap.ProbesInFlight() + tonemap.PreflightsInFlight() + mediasample.CapabilitiesInFlight() }) if !ok { slog.InfoContext(r.Context(), "transcode node capability re-probe refused while busy", diff --git a/internal/transcodenode/segment_ack_protocol.go b/internal/transcodenode/segment_ack_protocol.go index ed1deba02c..71ebd7f3f3 100644 --- a/internal/transcodenode/segment_ack_protocol.go +++ b/internal/transcodenode/segment_ack_protocol.go @@ -9,6 +9,8 @@ import ( "github.com/danielgtaylor/huma/v2" ) +const idempotent = "natural_idempotent" + // ProtocolSegmentAcknowledgement retains the private downstream-completion hop. func ProtocolSegmentAcknowledgement() workerprotocol.Operation { const ( @@ -19,7 +21,6 @@ func ProtocolSegmentAcknowledgement() workerprotocol.Operation { bearerClass = "node_bearer" sessionParameter = "session_id" textMedia = "text/plain" - idempotent = "natural_idempotent" ) parameters := []*huma.Param{ {Name: sessionParameter, In: pathParameter, Required: true, Schema: &huma.Schema{Type: huma.TypeString}, Description: "Existing transport session."}, diff --git a/internal/transcodenode/server.go b/internal/transcodenode/server.go index 2177dadc34..9ee3a1e987 100644 --- a/internal/transcodenode/server.go +++ b/internal/transcodenode/server.go @@ -38,6 +38,7 @@ import ( "github.com/Silo-Server/silo-server/internal/themesongs" "github.com/Silo-Server/silo-server/internal/tonemap" "github.com/Silo-Server/silo-server/internal/transcodeproxy" + "github.com/Silo-Server/silo-server/internal/workerdrain" ) // TranscodeStartRequest is the JSON body for POST /transcode/start. @@ -185,6 +186,7 @@ func ValidateThrottleAttestation(req TranscodeStartRequest, response TranscodeSt // HealthResponse is the JSON response for GET /api/v1/health. type HealthResponse struct { + workerdrain.Identity Status string `json:"status"` ActiveJobs int32 `json:"active_jobs"` // CapabilitiesHash identifies this node's last computed hardware capability @@ -259,6 +261,7 @@ type progressiveRemuxRequest struct { // Server is the HTTP handler for transcode mode. type Server struct { + drain *workerdrain.Gate watcher *nodeconfig.Watcher streamDeny *playback.StreamDeny nodeRowID func() (int, bool) @@ -519,6 +522,11 @@ func NewServer(watcher *nodeconfig.Watcher, tracker *nodesessions.Tracker) *Serv if watcher != nil { s.nodeRowID = watcher.NodeRowID s.registeredNodeURL = watcher.NodeRegisteredURL + if watcher.HasDrainAuthority() { + s.drain = workerdrain.New(watcher.ReadDrainFence, s.drainActiveJobs, func(ctx context.Context, key string) bool { + return s.streamDeny != nil && s.streamDeny.Denied(ctx, key) + }) + } } return s } @@ -574,6 +582,11 @@ func (s *Server) StartHardwareEncoderWarmup(ctx context.Context) <-chan struct{} return done } playbackCfg := cfg.Playback + endDrainWork, err := s.drain.Begin(ctx, "", time.Time{}) + if err != nil { + close(done) + return done + } // Claimed here, before the goroutine exists. Warmup is a real smoke encode // and the listener opens while it may still be running, so an admin re-probe // arriving in a node's first seconds must not see an idle gate — and a claim @@ -583,6 +596,7 @@ func (s *Server) StartHardwareEncoderWarmup(ctx context.Context) <-chan struct{} s.gpu.holdWork() go func() { defer close(done) + defer endDrainWork() defer s.gpu.endWork() if err := playback.WarmHardwareEncoder(ctx, playbackCfg.FFmpegPath, playbackCfg.HWAccel, playbackCfg.HWDevice); err != nil { slog.DebugContext(ctx, "transcode node hardware encoder warmup failed", "component", "transcodenode", "error", err) @@ -889,6 +903,7 @@ func (s *Server) router() chi.Router { declareTranscodeNodeMediaRoutes() s.startIdleReaper() r := chi.NewRouter() + r.Use(s.drainMiddleware) r.Get("/api/v1/health", s.handleHealth) // Unauthenticated, matching the API listener's own /metrics posture: a // scrape target that needs a credential is a scrape target that goes @@ -917,6 +932,8 @@ func (s *Server) router() chi.Router { r.Post("/admin/reload-config", s.handleReloadConfig) r.Post("/admin/reprobe-capabilities", s.handleReprobeCapabilities) r.Get("/status", s.handleStatus) + r.Get("/admin/drain", s.handleDrain) + r.Post("/admin/drain", s.handleDrain) }) return r } @@ -1255,10 +1272,15 @@ func (s *Server) trackDownloadPrepare(ctx context.Context, info nodesessions.Ses } } -func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { +func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { + identityCtx, cancel := context.WithTimeout(r.Context(), 2*time.Second) + defer cancel() + identity := s.drain.PublicIdentity(identityCtx) + w.Header().Set("Cache-Control", "no-store") snapshot := s.metrics.Snapshot().RedactPaths() w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(HealthResponse{ + Identity: identity, Status: "ok", ActiveJobs: s.activeJobs.Load(), CapabilitiesHash: s.storedCapabilityHash(), @@ -1306,6 +1328,11 @@ func (s *Server) StartMetricsSampler(ctx context.Context) { var ErrCapabilityBuildBusy = errors.New("capability build refused while the node is re-probing") func (s *Server) buildCapabilitySnapshot(ctx context.Context) (playback.HWAccelInfo, error) { + end, err := s.drain.Begin(ctx, "", time.Time{}) + if err != nil { + return playback.HWAccelInfo{}, err + } + defer end() // A snapshot runs ffmpeg on the GPU whenever the probe caches are cold, so // it registers as GPU work. That is what stops a manual re-probe from // claiming an apparently idle encoder and running its own smoke matrix @@ -2510,6 +2537,7 @@ func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) { http.Error(w, "session not found", http.StatusNotFound) return } + s.drain.Revoke(sessionID) w.WriteHeader(http.StatusNoContent) } diff --git a/internal/workerdrain/gate.go b/internal/workerdrain/gate.go new file mode 100644 index 0000000000..3b5f163cd0 --- /dev/null +++ b/internal/workerdrain/gate.go @@ -0,0 +1,191 @@ +// Package workerdrain owns the private worker's retirement admission fence. +// Redis observations are deliberately not used as proof that work is absent. +package workerdrain + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/google/uuid" +) + +var ErrUnavailable = errors.New("worker drain authority unavailable") +var ErrFenced = errors.New("worker admission fenced") + +// Source resolves the worker's stable registered identity and current durable +// fence. Calls are serialized with admission and status inside this process. +type Source func(context.Context) (nodeID int, fenceID, nativeServerID string, err error) + +// Identity correlates public health with an authenticated drain receipt from +// the same process. The public identity alone is never authorization. +type Identity struct { + NativeServerID string `json:"native_server_id,omitempty"` + WorkerInstanceID string `json:"worker_instance_id,omitempty"` +} + +type Status struct { + NodeID int `json:"node_id"` + FenceID string `json:"fence_id"` + WorkerInstanceID string `json:"worker_instance_id"` + NativeServerID string `json:"native_server_id"` + Fenced bool `json:"fenced"` + Drained bool `json:"drained"` + ActiveJobs int `json:"active_jobs"` + ActiveRequests int `json:"active_requests"` + ActiveReservations int `json:"active_reservations"` + ObservedAt time.Time `json:"observed_at"` +} + +type Gate struct { + mu sync.Mutex + source Source + jobs func() int + revoked func(context.Context, string) bool + now func() time.Time + instanceID string + serverID string + nodeID int + fenceID string + sealed bool + active int + permits map[string]time.Time + lastPruned time.Time +} + +func New(source Source, jobs func() int, revoked func(context.Context, string) bool) *Gate { + return &Gate{source: source, jobs: jobs, revoked: revoked, now: time.Now, instanceID: uuid.NewString(), permits: make(map[string]time.Time)} +} + +func (g *Gate) syncLocked(ctx context.Context) error { + if g.source == nil { + return ErrUnavailable + } + id, fence, serverID, err := g.source(ctx) + if err != nil { + return err + } + if id <= 0 || serverID == "" { + return ErrUnavailable + } + if g.nodeID != 0 && (g.nodeID != id || g.serverID != serverID) { + return ErrUnavailable + } + g.nodeID = id + g.serverID = serverID + if fence != g.fenceID { + // Only an explicit durable cancellation can reopen a sealed worker. + g.fenceID = fence + g.sealed = false + } + return nil +} + +// PublicIdentity returns the last verified process/realm binding. Its first +// read resolves durable authority without minting an identity. Later reads stay +// cheap liveness observations; authenticated drain always verifies it freshly. +func (g *Gate) PublicIdentity(ctx context.Context) Identity { + if g == nil { + return Identity{} + } + g.mu.Lock() + defer g.mu.Unlock() + if g.serverID == "" { + if err := g.syncLocked(ctx); err != nil { + return Identity{} + } + } + return Identity{NativeServerID: g.serverID, WorkerInstanceID: g.instanceID} +} + +func (g *Gate) pruneLocked(ctx context.Context, force bool) { + now := g.now() + if !force && now.Sub(g.lastPruned) < time.Minute { + return + } + g.lastPruned = now + for key, expiry := range g.permits { + if !expiry.After(now) || (force && g.revoked != nil && g.revoked(ctx, key)) { + delete(g.permits, key) + } + } +} + +// Begin reserves a request before any execution or egress can start. Existing +// identities retain their bounded media permit while retiring; an unknown +// identity cannot start, reconstruct, or resume after the worker sees a fence. +// Empty identities are one-shot background/extraction work and never retained. +func (g *Gate) Begin(ctx context.Context, key string, expiry time.Time) (func(), error) { + if g == nil { + return func() {}, nil + } + g.mu.Lock() + defer g.mu.Unlock() + authorityConfirmed := true + if err := g.syncLocked(ctx); err != nil { + // Existing playback keeps its original bounded permit through a + // temporary authority outage. No new identity, renewal, background work + // or retirement receipt is allowed without current durable authority. + expires, known := g.permits[key] + if g.sealed || key == "" || !known || !expires.After(g.now()) { + return nil, err + } + authorityConfirmed = false + } + // Expiry is checked for this identity on every admission. Full expiry + // sweeps are bounded, and potentially remote deny lookups belong to the + // explicit retirement observation rather than every media segment. + if prior, ok := g.permits[key]; ok && !prior.After(g.now()) { + delete(g.permits, key) + } + g.pruneLocked(ctx, false) + _, known := g.permits[key] + if g.fenceID != "" && (g.sealed || key == "" || !known) { + return nil, ErrFenced + } + if authorityConfirmed && key != "" && expiry.After(g.now()) && expiry.After(g.permits[key]) { + g.permits[key] = expiry + } + g.active++ + var once sync.Once + return func() { once.Do(func() { g.mu.Lock(); g.active--; g.mu.Unlock() }) }, nil +} + +// Observe samples actual worker execution plus request and permit reservations. +// A zero receipt seals admission under the same mutex, so a delayed dispatch +// cannot turn that receipt into a false zero. Process restart keeps the DB fence +// and starts with no permits, refusing resurrection from old client tokens. +func (g *Gate) Observe(ctx context.Context) (Status, error) { + if g == nil { + return Status{}, ErrUnavailable + } + g.mu.Lock() + defer g.mu.Unlock() + if err := g.syncLocked(ctx); err != nil { + return Status{}, err + } + g.pruneLocked(ctx, true) + jobs := 0 + if g.jobs != nil { + jobs = g.jobs() + } + if jobs < 0 { + return Status{}, ErrUnavailable + } + if g.fenceID != "" && jobs == 0 && g.active == 0 && len(g.permits) == 0 { + g.sealed = true + } + return Status{NodeID: g.nodeID, FenceID: g.fenceID, WorkerInstanceID: g.instanceID, NativeServerID: g.serverID, Fenced: g.fenceID != "", Drained: g.fenceID != "" && g.sealed && jobs == 0 && g.active == 0 && len(g.permits) == 0, ActiveJobs: jobs, ActiveRequests: g.active, ActiveReservations: len(g.permits), ObservedAt: g.now().UTC()}, nil +} + +// Revoke forgets a positively completed transport. In-flight requests remain +// reserved until their release; a fenced worker cannot admit the identity again. +func (g *Gate) Revoke(key string) { + if g == nil { + return + } + g.mu.Lock() + delete(g.permits, key) + g.mu.Unlock() +} diff --git a/internal/workerdrain/gate_test.go b/internal/workerdrain/gate_test.go new file mode 100644 index 0000000000..b4eac7eec9 --- /dev/null +++ b/internal/workerdrain/gate_test.go @@ -0,0 +1,177 @@ +package workerdrain + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" +) + +func TestDrainGatePreservesAdmittedWorkAndSealsLateAdmission(t *testing.T) { + var fence atomic.Value + fence.Store("") + var busy atomic.Int64 + var denied atomic.Bool + source := func(context.Context) (int, string, string, error) { + return 7, fence.Load().(string), "synthetic-realm", nil + } + gate := New(source, func() int { return int(busy.Load()) }, func(context.Context, string) bool { return denied.Load() }) + expiry := time.Now().Add(time.Hour) + end, err := gate.Begin(t.Context(), "existing", expiry) + if err != nil { + t.Fatal(err) + } + fence.Store("retirement-1") + status, err := gate.Observe(t.Context()) + if err != nil || !status.Fenced || status.Drained || status.ActiveRequests != 1 || status.ActiveReservations != 1 { + t.Fatalf("in-flight admission lost: %+v %v", status, err) + } + if _, err = gate.Begin(t.Context(), "late-dispatch", expiry); !errors.Is(err, ErrFenced) { + t.Fatalf("late admission: %v", err) + } + continued, err := gate.Begin(t.Context(), "existing", expiry) + if err != nil { + t.Fatalf("existing stream refused: %v", err) + } + continued() + end() + end() + busy.Store(1) + denied.Store(true) + status, err = gate.Observe(t.Context()) + if err != nil || status.Drained || status.ActiveJobs != 1 || status.ActiveRequests != 0 || status.ActiveReservations != 0 { + t.Fatalf("teardown execution lost: %+v %v", status, err) + } + busy.Store(0) + status, err = gate.Observe(t.Context()) + if err != nil || !status.Drained || status.WorkerInstanceID == "" || status.ObservedAt.IsZero() { + t.Fatalf("zero proof: %+v %v", status, err) + } + for _, key := range []string{"existing", "late-dispatch", ""} { + if _, err = gate.Begin(t.Context(), key, expiry); !errors.Is(err, ErrFenced) { + t.Fatalf("sealed worker accepted %q: %v", key, err) + } + } + // Explicit durable cancellation is the only path that restores admission. + fence.Store("") + reopened, err := gate.Begin(t.Context(), "new", expiry) + if err != nil { + t.Fatal(err) + } + reopened() + status, err = gate.Observe(t.Context()) + if err != nil || status.Fenced || status.Drained { + t.Fatalf("cancellation: %+v %v", status, err) + } +} + +func TestDrainGateRestartAndUncertainAuthority(t *testing.T) { + var unavailable atomic.Bool + source := func(context.Context) (int, string, string, error) { + if unavailable.Load() { + return 0, "", "", ErrUnavailable + } + return 7, "durable-fence", "synthetic-realm", nil + } + first := New(source, nil, nil) + before, err := first.Observe(t.Context()) + if err != nil || !before.Drained { + t.Fatalf("initial zero: %+v %v", before, err) + } + restarted := New(source, nil, nil) + after, err := restarted.Observe(t.Context()) + if err != nil || !after.Drained || after.WorkerInstanceID == before.WorkerInstanceID { + t.Fatalf("restart fence: %+v %v", after, err) + } + if _, err = restarted.Begin(t.Context(), "old-token", time.Now().Add(time.Hour)); !errors.Is(err, ErrFenced) { + t.Fatalf("resurrection: %v", err) + } + unavailable.Store(true) + if _, err = restarted.Observe(t.Context()); !errors.Is(err, ErrUnavailable) { + t.Fatalf("unavailable source became proof: %v", err) + } + if _, err = restarted.Begin(t.Context(), "anything", time.Now().Add(time.Hour)); !errors.Is(err, ErrUnavailable) { + t.Fatalf("unavailable source admitted: %v", err) + } +} + +func TestDrainGatePermitExpiryIsConservative(t *testing.T) { + var fence string + gate := New(func(context.Context) (int, string, string, error) { return 7, fence, "synthetic-realm", nil }, nil, nil) + now := time.Now() + gate.now = func() time.Time { return now } + end, err := gate.Begin(t.Context(), "token", now.Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + end() + fence = "retirement" + status, err := gate.Observe(t.Context()) + if err != nil || status.Drained || status.ActiveReservations != 1 { + t.Fatalf("quiet token incorrectly retired: %+v %v", status, err) + } + now = now.Add(time.Hour) + status, err = gate.Observe(t.Context()) + if err != nil || !status.Drained { + t.Fatalf("expired permit: %+v %v", status, err) + } +} + +func TestDrainGateAuthorityOutagePreservesOnlyBoundedExistingPlayback(t *testing.T) { + var unavailable bool + gate := New(func(context.Context) (int, string, string, error) { + if unavailable { + return 0, "", "", ErrUnavailable + } + return 7, "", "synthetic-realm", nil + }, nil, nil) + now := time.Now() + gate.now = func() time.Time { return now } + end, err := gate.Begin(t.Context(), "existing", now.Add(time.Minute)) + if err != nil { + t.Fatal(err) + } + end() + unavailable = true + continued, err := gate.Begin(t.Context(), "existing", now.Add(time.Hour)) + if err != nil { + t.Fatalf("authority outage interrupted admitted playback: %v", err) + } + continued() + if _, err = gate.Observe(t.Context()); !errors.Is(err, ErrUnavailable) { + t.Fatalf("outage yielded retirement proof: %v", err) + } + for _, key := range []string{"new", ""} { + if _, err = gate.Begin(t.Context(), key, now.Add(time.Hour)); !errors.Is(err, ErrUnavailable) { + t.Fatalf("outage admitted %q: %v", key, err) + } + } + now = now.Add(time.Minute) + if _, err = gate.Begin(t.Context(), "existing", now.Add(time.Hour)); !errors.Is(err, ErrUnavailable) { + t.Fatalf("outage extended original permit: %v", err) + } +} + +func TestDrainGateRejectsMissingAndChangedRealm(t *testing.T) { + for _, initial := range []string{"", "synthetic-realm"} { + t.Run(initial, func(t *testing.T) { + realm := initial + gate := New(func(context.Context) (int, string, string, error) { return 7, "durable-fence", realm, nil }, nil, nil) + before, err := gate.Observe(t.Context()) + if realm == "" { + if !errors.Is(err, ErrUnavailable) || before.Drained || gate.PublicIdentity(t.Context()).NativeServerID != "" { + t.Fatalf("missing realm accepted: %+v %v", before, err) + } + return + } + if err != nil || !before.Drained || before.NativeServerID != realm { + t.Fatalf("bound realm rejected: %+v %v", before, err) + } + realm = "foreign-realm" + if after, err := gate.Observe(t.Context()); !errors.Is(err, ErrUnavailable) || after.Drained { + t.Fatalf("changed realm accepted: %+v %v", after, err) + } + }) + } +} diff --git a/migrations/sql/20261008031132_add_worker_drain_fences.sql b/migrations/sql/20261008031132_add_worker_drain_fences.sql new file mode 100644 index 0000000000..704e3e3a49 --- /dev/null +++ b/migrations/sql/20261008031132_add_worker_drain_fences.sql @@ -0,0 +1,47 @@ +-- +goose Up +ALTER TABLE stream_nodes ADD COLUMN drain_revision bigint NOT NULL DEFAULT 0; +CREATE TABLE stream_node_drain_fences ( + node_id integer PRIMARY KEY REFERENCES stream_nodes(id) ON DELETE CASCADE, + fence_id text NOT NULL UNIQUE, + requested_at timestamptz NOT NULL DEFAULT now() +); +-- +goose StatementBegin +CREATE FUNCTION enforce_stream_node_drain_fence() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NEW.enabled AND EXISTS (SELECT 1 FROM stream_node_drain_fences WHERE node_id=NEW.id) THEN + RAISE EXCEPTION 'node retirement must be canceled before enabling' + USING ERRCODE = '23514', CONSTRAINT = 'stream_node_drain_requires_disabled'; + END IF; + IF ROW(NEW.url,NEW.type,NEW.name) IS DISTINCT FROM ROW(OLD.url,OLD.type,OLD.name) + AND EXISTS (SELECT 1 FROM stream_node_drain_fences WHERE node_id=NEW.id) THEN + RAISE EXCEPTION 'node retirement must be canceled before changing worker identity' + USING ERRCODE = '23514', CONSTRAINT = 'stream_node_drain_target_fenced'; + END IF; + RETURN NEW; +END; +$$; +-- +goose StatementEnd +CREATE TRIGGER stream_node_drain_requires_disabled BEFORE UPDATE ON stream_nodes +FOR EACH ROW EXECUTE FUNCTION enforce_stream_node_drain_fence(); + +-- +goose Down +-- Serialize with the production node-row -> fence write order before checking +-- emptiness. Otherwise a BeginDrain can publish after the check but before DDL. +LOCK TABLE stream_nodes IN ACCESS EXCLUSIVE MODE; +-- +goose StatementBegin +DO $$ BEGIN + IF EXISTS (SELECT 1 FROM stream_node_drain_fences) THEN + RAISE EXCEPTION 'cancel worker drains before rolling back this migration'; + END IF; +END $$; +-- +goose StatementEnd +-- Keep the last effective validator when the older binary resumes reading +-- admin_revision alone. Its normal no-change trigger would otherwise overwrite +-- this promotion; the exclusive lock prevents any writer during the transfer. +ALTER TABLE stream_nodes DISABLE TRIGGER stream_node_configuration_revision; +UPDATE stream_nodes SET admin_revision=GREATEST(admin_revision,drain_revision); +ALTER TABLE stream_nodes ENABLE TRIGGER stream_node_configuration_revision; +DROP TRIGGER stream_node_drain_requires_disabled ON stream_nodes; +DROP FUNCTION enforce_stream_node_drain_fence(); +DROP TABLE stream_node_drain_fences; +ALTER TABLE stream_nodes DROP COLUMN drain_revision; diff --git a/scripts/ci/db-contracts.txt b/scripts/ci/db-contracts.txt index c6a94e2e66..e48d94ee6d 100644 --- a/scripts/ci/db-contracts.txt +++ b/scripts/ci/db-contracts.txt @@ -111,6 +111,17 @@ # Intro analysis state timestamps and config-hash isolation. ./internal/intromarkers TestSeasonStateIsKeyedByAnalysisHashPostgres +# Guarded worker retirement, realm binding, and disabled commissioning. +./internal/nodepool TestAdminNodeDrainTransactions +./internal/nodepool TestAdminNodeDrainPublicationFaultRollsBack +./internal/nodepool TestAdminNodeDrainConcurrentMigrationRollbackPreservesFence +./internal/nodepool TestAdminNodeDrainMigrationRollbackPreservesConfigurationRevision +./internal/apiv2 TestAdminNodeDrainPostgresProductionPath +./internal/apiv2 TestAdminNodeDisabledCreationPostgresProductionPath +./internal/apiv2 TestAdminNodeDisabledPublicationFaultsPostgresProductionPath +./internal/apiv2 TestAdminNodeLateDisabledCreationPostgresProductionPath +./internal/apiv2 TestAdminNodeCreationPreservesEnabledDefaultsPostgresProductionPath + # Login sessions record the device that signed in. ./internal/auth TestLoginSessionRecordsReportedDeviceDB ./internal/auth TestDeviceLoginSessionRecordsPollingDeviceDB diff --git a/web/src/api/v2/operations.ts b/web/src/api/v2/operations.ts index eb497dace5..7e1e48dcb8 100644 --- a/web/src/api/v2/operations.ts +++ b/web/src/api/v2/operations.ts @@ -25,6 +25,7 @@ export const v2Operations = { "DELETE /api/v2/admin/invite-codes/{id}": "deleteAdminInviteCode", "DELETE /api/v2/admin/literary-works/{work_id}/items/{content_id}": "unlinkAdminLiteraryItem", "DELETE /api/v2/admin/nodes/{id}": "deleteAdminNode", + "DELETE /api/v2/admin/nodes/{id}/drain": "cancelAdminNodeDrain", "DELETE /api/v2/admin/notifications/push/relay": "clearAdminNotificationRelay", "DELETE /api/v2/admin/notifications/server-channels/{id}": "deleteAdminNotificationServerChannel", "DELETE /api/v2/admin/plugins/installations/{id}": "deleteAdminPluginInstallation", @@ -174,6 +175,8 @@ export const v2Operations = { "GET /api/v2/admin/network-access/{provider}/status": "getAdminNetworkAccessStatus", "GET /api/v2/admin/node-sessions": "listAdminNodeSessions", "GET /api/v2/admin/nodes": "listAdminNodes", + "GET /api/v2/admin/nodes/capabilities": "getAdminNodeCapabilities", + "GET /api/v2/admin/nodes/{id}/drain": "getAdminNodeDrain", "GET /api/v2/admin/notifications/server-channels": "listAdminNotificationServerChannels", "GET /api/v2/admin/playback-history": "listAdminPlaybackHistory", "GET /api/v2/admin/playback-routing/capabilities": "getAdminPlaybackRoutingCapabilities", @@ -805,6 +808,7 @@ export const v2Operations = { "reorderAdminCollectionGroups", "PUT /api/v2/admin/markers/providers/{provider}": "updateAdminMarkerProvider", "PUT /api/v2/admin/nodes/{id}": "updateAdminNode", + "PUT /api/v2/admin/nodes/{id}/drain": "beginAdminNodeDrain", "PUT /api/v2/admin/notifications/server-channels/{id}": "updateAdminNotificationServerChannel", "PUT /api/v2/admin/plugins/catalog-settings": "updateAdminPluginCatalogSettings", "PUT /api/v2/admin/plugins/installations/{id}": "updateAdminPluginInstallation", diff --git a/web/src/api/v2/schema.ts b/web/src/api/v2/schema.ts index ce6cab53f2..66a3e4a096 100644 --- a/web/src/api/v2/schema.ts +++ b/web/src/api/v2/schema.ts @@ -2614,7 +2614,7 @@ export interface paths { /** Page configured nodes and their last stored observations; no worker probe is performed. */ get: operations["listAdminNodes"]; put?: never; - /** Persist node configuration and durable pool invalidation. An identical natural URL configuration resolves a repeated create; conflicting configuration returns 409. No worker probe or replica completion acknowledgement. */ + /** Persist node configuration and durable pool invalidation. Omitted enabled defaults to true; enabled=false commissions without placement until a separate guarded activation. An identical natural URL configuration, including enabled, resolves a repeated create; conflicting configuration returns 409. No worker probe or replica completion acknowledgement. */ post: operations["createAdminNode"]; delete?: never; options?: never; @@ -2657,6 +2657,25 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v2/admin/nodes/{id}/drain": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Freshly confirm worker admission fencing and execution, egress and media-permit reservations. Zero seals admission, including late dispatch and reconstruction. A changed configuration or unavailable worker cannot yield a drained receipt. */ + get: operations["getAdminNodeDrain"]; + /** Disable placement and durably fence new worker admission under the original If-Match configuration; existing admitted work continues. Then confirm the private worker. An unavailable confirmation can follow a committed fence; inspect configuration and GET drain before any explicit retry. Stored disable alone is insufficient. */ + put: operations["beginAdminNodeDrain"]; + post?: never; + /** Explicitly cancel the durable worker admission fence under the original If-Match validator, invalidating earlier drained receipts. Placement stays disabled until a separate guarded configuration update. Workers synchronize cancellation on the next admission. */ + delete: operations["cancelAdminNodeDrain"]; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v2/admin/nodes/{id}/force-reload": { parameters: { query?: never; @@ -2691,6 +2710,23 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v2/admin/nodes/capabilities": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Discover native worker retirement fencing and disabled commissioning support; each configured worker must also confirm its private drain protocol. */ + get: operations["getAdminNodeCapabilities"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v2/admin/nodes/force-reload": { parameters: { query?: never; @@ -14962,6 +14998,11 @@ export interface components { url: string; }; AdminNodeCreateBody: { + /** + * @description Create disabled for worker commissioning; activate separately with a guarded configuration update. + * @default true + */ + enabled: boolean; group?: string; /** Format: int64 */ max_bandwidth_kbps?: number | null; @@ -14973,6 +15014,46 @@ export interface components { type: "proxy" | "transcode"; url: string; }; + AdminNodeDrain: { + /** Format: int64 */ + active_jobs: number; + /** Format: int64 */ + active_requests: number; + /** + * Format: int64 + * @description Conservative existing media permits retained to expiry or a positive session deny; absence from Redis is never retirement proof. + */ + active_reservations: number; + config_etag: string; + drained: boolean; + fence_id: string; + fenced: boolean; + native_server_id: string; + /** + * @description Opaque identifier + * @example 1 + */ + node_id: string; + /** + * Format: date-time + * @description RFC 3339 instant in UTC with millisecond precision + */ + observed_at: string; + worker_instance_id: string; + }; + AdminNodeDrainCapabilities: { + /** @description Whether the current principal may use the capability */ + allowed: boolean; + disabled_creation: boolean; + /** @description Opaque revision of this document */ + revision: string; + /** + * @description Support and configuration state, not health + * @enum {string} + */ + state: "available" | "disabled" | "not_configured" | "unsupported"; + worker_drain: boolean; + }; AdminNodeHealth: { /** Format: int64 */ active_jobs: number; @@ -53517,18 +53598,481 @@ export interface operations { }; }; }; - listAdminMarkerHistory: { + listAdminMarkerHistory: { + parameters: { + query?: { + limit?: number; + }; + header?: { + /** @description Optional. When present, it must name the authenticated account's primary profile; an absent header is accepted. */ + "X-Profile-Id"?: string; + /** @description Verification proof for a PIN-locked profile, issued by POST /api/v2/profiles/{id}/verify-pin; required only when the declared profile is locked */ + "X-Profile-Token"?: string; + }; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AdminMarkerHistory"]; + }; + }; + /** @description Bad Request */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Unauthorized */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Not Acceptable */ + 406: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Too Many Requests */ + 429: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Service Unavailable */ + 503: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + }; + }; + listAdminItemMarkerHistory: { + parameters: { + query?: { + limit?: number; + }; + header?: { + /** @description Optional. When present, it must name the authenticated account's primary profile; an absent header is accepted. */ + "X-Profile-Id"?: string; + /** @description Verification proof for a PIN-locked profile, issued by POST /api/v2/profiles/{id}/verify-pin; required only when the declared profile is locked */ + "X-Profile-Token"?: string; + }; + path: { + id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AdminMarkerHistory"]; + }; + }; + /** @description Bad Request */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Unauthorized */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Not Acceptable */ + 406: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Too Many Requests */ + 429: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Service Unavailable */ + 503: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + }; + }; + listAdminMarkerProviders: { + parameters: { + query?: never; + header?: { + /** @description Optional. When present, it must name the authenticated account's primary profile; an absent header is accepted. */ + "X-Profile-Id"?: string; + /** @description Verification proof for a PIN-locked profile, issued by POST /api/v2/profiles/{id}/verify-pin; required only when the declared profile is locked */ + "X-Profile-Token"?: string; + }; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AdminMarkerProviders"]; + }; + }; + /** @description Bad Request */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Unauthorized */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Not Acceptable */ + 406: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Too Many Requests */ + 429: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Service Unavailable */ + 503: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + }; + }; + updateAdminMarkerProvider: { + parameters: { + query?: never; + header?: { + /** @description Optional. When present, it must name the authenticated account's primary profile; an absent header is accepted. */ + "X-Profile-Id"?: string; + /** @description Verification proof for a PIN-locked profile, issued by POST /api/v2/profiles/{id}/verify-pin; required only when the declared profile is locked */ + "X-Profile-Token"?: string; + }; + path: { + provider: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["AdminMarkerProviderUpdate"]; + }; + }; + responses: { + /** @description OK */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AdminMarkerProvider"]; + }; + }; + /** @description Bad Request */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Unauthorized */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Forbidden */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Not Found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Not Acceptable */ + 406: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Request Timeout */ + 408: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Request Entity Too Large */ + 413: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Unsupported Media Type */ + 415: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Unprocessable Entity */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Too Many Requests */ + 429: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Internal Server Error */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Service Unavailable */ + 503: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + }; + }; + validateAdminMarkerProvider: { parameters: { - query?: { - limit?: number; - }; + query?: never; header?: { /** @description Optional. When present, it must name the authenticated account's primary profile; an absent header is accepted. */ "X-Profile-Id"?: string; /** @description Verification proof for a PIN-locked profile, issued by POST /api/v2/profiles/{id}/verify-pin; required only when the declared profile is locked */ "X-Profile-Token"?: string; }; - path?: never; + path: { + provider: string; + }; cookie?: never; }; requestBody?: never; @@ -53539,7 +54083,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["AdminMarkerHistory"]; + "application/json": components["schemas"]["AdminMarkerProviderValidation"]; }; }; /** @description Bad Request */ @@ -53625,11 +54169,9 @@ export interface operations { }; }; }; - listAdminItemMarkerHistory: { + connectNetworkAccess: { parameters: { - query?: { - limit?: number; - }; + query?: never; header?: { /** @description Optional. When present, it must name the authenticated account's primary profile; an absent header is accepted. */ "X-Profile-Id"?: string; @@ -53637,19 +54179,23 @@ export interface operations { "X-Profile-Token"?: string; }; path: { - id: string; + provider: string; }; cookie?: never; }; - requestBody?: never; + requestBody?: { + content: { + "application/json": components["schemas"]["NetworkAccessCommand"]; + }; + }; responses: { - /** @description OK */ - 200: { + /** @description Accepted */ + 202: { headers: { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["AdminMarkerHistory"]; + "application/json": components["schemas"]["NetworkAccessStatus"]; }; }; /** @description Bad Request */ @@ -53697,6 +54243,33 @@ export interface operations { "application/problem+json": components["schemas"]["Problem"]; }; }; + /** @description Request Timeout */ + 408: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Request Entity Too Large */ + 413: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Unsupported Media Type */ + 415: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; /** @description Unprocessable Entity */ 422: { headers: { @@ -53735,7 +54308,7 @@ export interface operations { }; }; }; - listAdminMarkerProviders: { + disconnectNetworkAccess: { parameters: { query?: never; header?: { @@ -53744,18 +54317,24 @@ export interface operations { /** @description Verification proof for a PIN-locked profile, issued by POST /api/v2/profiles/{id}/verify-pin; required only when the declared profile is locked */ "X-Profile-Token"?: string; }; - path?: never; + path: { + provider: string; + }; cookie?: never; }; - requestBody?: never; + requestBody?: { + content: { + "application/json": components["schemas"]["NetworkAccessCommand"]; + }; + }; responses: { - /** @description OK */ - 200: { + /** @description Accepted */ + 202: { headers: { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["AdminMarkerProviders"]; + "application/json": components["schemas"]["NetworkAccessStatus"]; }; }; /** @description Bad Request */ @@ -53803,6 +54382,33 @@ export interface operations { "application/problem+json": components["schemas"]["Problem"]; }; }; + /** @description Request Timeout */ + 408: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Request Entity Too Large */ + 413: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Unsupported Media Type */ + 415: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; /** @description Unprocessable Entity */ 422: { headers: { @@ -53841,7 +54447,7 @@ export interface operations { }; }; }; - updateAdminMarkerProvider: { + getAdminNetworkAccessStatus: { parameters: { query?: never; header?: { @@ -53855,11 +54461,7 @@ export interface operations { }; cookie?: never; }; - requestBody: { - content: { - "application/json": components["schemas"]["AdminMarkerProviderUpdate"]; - }; - }; + requestBody?: never; responses: { /** @description OK */ 200: { @@ -53867,7 +54469,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["AdminMarkerProvider"]; + "application/json": components["schemas"]["NetworkAccessStatus"]; }; }; /** @description Bad Request */ @@ -53915,33 +54517,6 @@ export interface operations { "application/problem+json": components["schemas"]["Problem"]; }; }; - /** @description Request Timeout */ - 408: { - headers: { - [name: string]: unknown; - }; - content: { - "application/problem+json": components["schemas"]["Problem"]; - }; - }; - /** @description Request Entity Too Large */ - 413: { - headers: { - [name: string]: unknown; - }; - content: { - "application/problem+json": components["schemas"]["Problem"]; - }; - }; - /** @description Unsupported Media Type */ - 415: { - headers: { - [name: string]: unknown; - }; - content: { - "application/problem+json": components["schemas"]["Problem"]; - }; - }; /** @description Unprocessable Entity */ 422: { headers: { @@ -53980,18 +54555,22 @@ export interface operations { }; }; }; - validateAdminMarkerProvider: { + listAdminNodeSessions: { parameters: { - query?: never; + query?: { + cursor?: string; + /** @description Page size; default 50, maximum 200 */ + limit?: number; + /** @description Opaque identifier */ + node_id?: string; + }; header?: { /** @description Optional. When present, it must name the authenticated account's primary profile; an absent header is accepted. */ "X-Profile-Id"?: string; /** @description Verification proof for a PIN-locked profile, issued by POST /api/v2/profiles/{id}/verify-pin; required only when the declared profile is locked */ "X-Profile-Token"?: string; }; - path: { - provider: string; - }; + path?: never; cookie?: never; }; requestBody?: never; @@ -54002,7 +54581,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["AdminMarkerProviderValidation"]; + "application/json": components["schemas"]["AdminNodeSessionsOutputBody"]; }; }; /** @description Bad Request */ @@ -54088,33 +54667,31 @@ export interface operations { }; }; }; - connectNetworkAccess: { + listAdminNodes: { parameters: { - query?: never; + query?: { + cursor?: string; + /** @description Page size; default 50, maximum 200 */ + limit?: number; + }; header?: { /** @description Optional. When present, it must name the authenticated account's primary profile; an absent header is accepted. */ "X-Profile-Id"?: string; /** @description Verification proof for a PIN-locked profile, issued by POST /api/v2/profiles/{id}/verify-pin; required only when the declared profile is locked */ "X-Profile-Token"?: string; }; - path: { - provider: string; - }; + path?: never; cookie?: never; }; - requestBody?: { - content: { - "application/json": components["schemas"]["NetworkAccessCommand"]; - }; - }; + requestBody?: never; responses: { - /** @description Accepted */ - 202: { + /** @description OK */ + 200: { headers: { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["NetworkAccessStatus"]; + "application/json": components["schemas"]["CollectionAdminNode"]; }; }; /** @description Bad Request */ @@ -54162,33 +54739,6 @@ export interface operations { "application/problem+json": components["schemas"]["Problem"]; }; }; - /** @description Request Timeout */ - 408: { - headers: { - [name: string]: unknown; - }; - content: { - "application/problem+json": components["schemas"]["Problem"]; - }; - }; - /** @description Request Entity Too Large */ - 413: { - headers: { - [name: string]: unknown; - }; - content: { - "application/problem+json": components["schemas"]["Problem"]; - }; - }; - /** @description Unsupported Media Type */ - 415: { - headers: { - [name: string]: unknown; - }; - content: { - "application/problem+json": components["schemas"]["Problem"]; - }; - }; /** @description Unprocessable Entity */ 422: { headers: { @@ -54227,7 +54777,7 @@ export interface operations { }; }; }; - disconnectNetworkAccess: { + createAdminNode: { parameters: { query?: never; header?: { @@ -54236,24 +54786,23 @@ export interface operations { /** @description Verification proof for a PIN-locked profile, issued by POST /api/v2/profiles/{id}/verify-pin; required only when the declared profile is locked */ "X-Profile-Token"?: string; }; - path: { - provider: string; - }; + path?: never; cookie?: never; }; - requestBody?: { + requestBody: { content: { - "application/json": components["schemas"]["NetworkAccessCommand"]; + "application/json": components["schemas"]["AdminNodeCreateBody"]; }; }; responses: { - /** @description Accepted */ - 202: { + /** @description Created */ + 201: { headers: { + ETag?: string; [name: string]: unknown; }; content: { - "application/json": components["schemas"]["NetworkAccessStatus"]; + "application/json": components["schemas"]["AdminNode"]; }; }; /** @description Bad Request */ @@ -54310,6 +54859,15 @@ export interface operations { "application/problem+json": components["schemas"]["Problem"]; }; }; + /** @description Conflict */ + 409: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; /** @description Request Entity Too Large */ 413: { headers: { @@ -54366,29 +54924,39 @@ export interface operations { }; }; }; - getAdminNetworkAccessStatus: { + updateAdminNode: { parameters: { query?: never; - header?: { + header: { + /** @description The resource's current ETag, or "*" to overwrite deliberately. A missing field is 428 precondition_required; a stale tag is 412 precondition_failed with the current ETag. */ + "If-Match": string; + /** @description Optional second precondition, evaluated after If-Match succeeds: "*" or any tag matching the current representation is 412 precondition_failed with the current ETag. */ + "If-None-Match"?: string; /** @description Optional. When present, it must name the authenticated account's primary profile; an absent header is accepted. */ "X-Profile-Id"?: string; /** @description Verification proof for a PIN-locked profile, issued by POST /api/v2/profiles/{id}/verify-pin; required only when the declared profile is locked */ "X-Profile-Token"?: string; }; path: { - provider: string; + id: string; }; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + "application/json": components["schemas"]["AdminNodeUpdateBody"]; + }; + }; responses: { /** @description OK */ 200: { headers: { + /** @description The strong, opaque validator of the representation; send it back in If-Match on a guarded mutation or If-None-Match on a conditional read. */ + ETag?: string; [name: string]: unknown; }; content: { - "application/json": components["schemas"]["NetworkAccessStatus"]; + "application/json": components["schemas"]["AdminNode"]; }; }; /** @description Bad Request */ @@ -54436,6 +55004,53 @@ export interface operations { "application/problem+json": components["schemas"]["Problem"]; }; }; + /** @description Request Timeout */ + 408: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Conflict */ + 409: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Precondition Failed */ + 412: { + headers: { + /** @description The strong, opaque validator of the representation; send it back in If-Match on a guarded mutation or If-None-Match on a conditional read. */ + ETag?: string; + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Request Entity Too Large */ + 413: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; + /** @description Unsupported Media Type */ + 415: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; /** @description Unprocessable Entity */ 422: { headers: { @@ -54445,6 +55060,15 @@ export interface operations { "application/problem+json": components["schemas"]["Problem"]; }; }; + /** @description Precondition Required */ + 428: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; /** @description Too Many Requests */ 429: { headers: { @@ -54474,34 +55098,32 @@ export interface operations { }; }; }; - listAdminNodeSessions: { + deleteAdminNode: { parameters: { - query?: { - cursor?: string; - /** @description Page size; default 50, maximum 200 */ - limit?: number; - /** @description Opaque identifier */ - node_id?: string; - }; - header?: { + query?: never; + header: { + /** @description The resource's current ETag, or "*" to overwrite deliberately. A missing field is 428 precondition_required; a stale tag is 412 precondition_failed with the current ETag. */ + "If-Match": string; + /** @description Optional second precondition, evaluated after If-Match succeeds: "*" or any tag matching the current representation is 412 precondition_failed with the current ETag. */ + "If-None-Match"?: string; /** @description Optional. When present, it must name the authenticated account's primary profile; an absent header is accepted. */ "X-Profile-Id"?: string; /** @description Verification proof for a PIN-locked profile, issued by POST /api/v2/profiles/{id}/verify-pin; required only when the declared profile is locked */ "X-Profile-Token"?: string; }; - path?: never; + path: { + id: string; + }; cookie?: never; }; requestBody?: never; responses: { - /** @description OK */ - 200: { + /** @description No Content */ + 204: { headers: { [name: string]: unknown; }; - content: { - "application/json": components["schemas"]["AdminNodeSessionsOutputBody"]; - }; + content?: never; }; /** @description Bad Request */ 400: { @@ -54548,6 +55170,17 @@ export interface operations { "application/problem+json": components["schemas"]["Problem"]; }; }; + /** @description Precondition Failed */ + 412: { + headers: { + /** @description The strong, opaque validator of the representation; send it back in If-Match on a guarded mutation or If-None-Match on a conditional read. */ + ETag?: string; + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; /** @description Unprocessable Entity */ 422: { headers: { @@ -54557,6 +55190,15 @@ export interface operations { "application/problem+json": components["schemas"]["Problem"]; }; }; + /** @description Precondition Required */ + 428: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; /** @description Too Many Requests */ 429: { headers: { @@ -54586,20 +55228,18 @@ export interface operations { }; }; }; - listAdminNodes: { + checkAdminNode: { parameters: { - query?: { - cursor?: string; - /** @description Page size; default 50, maximum 200 */ - limit?: number; - }; + query?: never; header?: { /** @description Optional. When present, it must name the authenticated account's primary profile; an absent header is accepted. */ "X-Profile-Id"?: string; /** @description Verification proof for a PIN-locked profile, issued by POST /api/v2/profiles/{id}/verify-pin; required only when the declared profile is locked */ "X-Profile-Token"?: string; }; - path?: never; + path: { + id: string; + }; cookie?: never; }; requestBody?: never; @@ -54610,7 +55250,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["CollectionAdminNode"]; + "application/json": components["schemas"]["AdminNodeHealth"]; }; }; /** @description Bad Request */ @@ -54696,7 +55336,7 @@ export interface operations { }; }; }; - createAdminNode: { + getAdminNodeDrain: { parameters: { query?: never; header?: { @@ -54705,23 +55345,22 @@ export interface operations { /** @description Verification proof for a PIN-locked profile, issued by POST /api/v2/profiles/{id}/verify-pin; required only when the declared profile is locked */ "X-Profile-Token"?: string; }; - path?: never; - cookie?: never; - }; - requestBody: { - content: { - "application/json": components["schemas"]["AdminNodeCreateBody"]; + path: { + id: string; }; + cookie?: never; }; + requestBody?: never; responses: { - /** @description Created */ - 201: { + /** @description OK */ + 200: { headers: { + "Cache-Control"?: string; ETag?: string; [name: string]: unknown; }; content: { - "application/json": components["schemas"]["AdminNode"]; + "application/json": components["schemas"]["AdminNodeDrain"]; }; }; /** @description Bad Request */ @@ -54769,15 +55408,6 @@ export interface operations { "application/problem+json": components["schemas"]["Problem"]; }; }; - /** @description Request Timeout */ - 408: { - headers: { - [name: string]: unknown; - }; - content: { - "application/problem+json": components["schemas"]["Problem"]; - }; - }; /** @description Conflict */ 409: { headers: { @@ -54787,24 +55417,6 @@ export interface operations { "application/problem+json": components["schemas"]["Problem"]; }; }; - /** @description Request Entity Too Large */ - 413: { - headers: { - [name: string]: unknown; - }; - content: { - "application/problem+json": components["schemas"]["Problem"]; - }; - }; - /** @description Unsupported Media Type */ - 415: { - headers: { - [name: string]: unknown; - }; - content: { - "application/problem+json": components["schemas"]["Problem"]; - }; - }; /** @description Unprocessable Entity */ 422: { headers: { @@ -54843,7 +55455,7 @@ export interface operations { }; }; }; - updateAdminNode: { + beginAdminNodeDrain: { parameters: { query?: never; header: { @@ -54861,21 +55473,18 @@ export interface operations { }; cookie?: never; }; - requestBody: { - content: { - "application/json": components["schemas"]["AdminNodeUpdateBody"]; - }; - }; + requestBody?: never; responses: { /** @description OK */ 200: { headers: { + "Cache-Control"?: string; /** @description The strong, opaque validator of the representation; send it back in If-Match on a guarded mutation or If-None-Match on a conditional read. */ ETag?: string; [name: string]: unknown; }; content: { - "application/json": components["schemas"]["AdminNode"]; + "application/json": components["schemas"]["AdminNodeDrain"]; }; }; /** @description Bad Request */ @@ -54923,15 +55532,6 @@ export interface operations { "application/problem+json": components["schemas"]["Problem"]; }; }; - /** @description Request Timeout */ - 408: { - headers: { - [name: string]: unknown; - }; - content: { - "application/problem+json": components["schemas"]["Problem"]; - }; - }; /** @description Conflict */ 409: { headers: { @@ -54952,24 +55552,6 @@ export interface operations { "application/problem+json": components["schemas"]["Problem"]; }; }; - /** @description Request Entity Too Large */ - 413: { - headers: { - [name: string]: unknown; - }; - content: { - "application/problem+json": components["schemas"]["Problem"]; - }; - }; - /** @description Unsupported Media Type */ - 415: { - headers: { - [name: string]: unknown; - }; - content: { - "application/problem+json": components["schemas"]["Problem"]; - }; - }; /** @description Unprocessable Entity */ 422: { headers: { @@ -55017,7 +55599,7 @@ export interface operations { }; }; }; - deleteAdminNode: { + cancelAdminNodeDrain: { parameters: { query?: never; header: { @@ -55147,7 +55729,7 @@ export interface operations { }; }; }; - checkAdminNode: { + forceReloadAdminNode: { parameters: { query?: never; header?: { @@ -55169,7 +55751,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["AdminNodeHealth"]; + "application/json": components["schemas"]["AdminNodeReloadOutputBody"]; }; }; /** @description Bad Request */ @@ -55255,7 +55837,7 @@ export interface operations { }; }; }; - forceReloadAdminNode: { + reprobeAdminNode: { parameters: { query?: never; header?: { @@ -55277,7 +55859,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["AdminNodeReloadOutputBody"]; + "application/json": components["schemas"]["AdminNodeReprobe"]; }; }; /** @description Bad Request */ @@ -55363,18 +55945,19 @@ export interface operations { }; }; }; - reprobeAdminNode: { + getAdminNodeCapabilities: { parameters: { query?: never; header?: { + /** @description Optional first precondition, evaluated before If-None-Match: a tag that does not match the current representation is 412 precondition_failed. */ + "If-Match"?: string; + "If-None-Match"?: string; /** @description Optional. When present, it must name the authenticated account's primary profile; an absent header is accepted. */ "X-Profile-Id"?: string; /** @description Verification proof for a PIN-locked profile, issued by POST /api/v2/profiles/{id}/verify-pin; required only when the declared profile is locked */ "X-Profile-Token"?: string; }; - path: { - id: string; - }; + path?: never; cookie?: never; }; requestBody?: never; @@ -55382,11 +55965,23 @@ export interface operations { /** @description OK */ 200: { headers: { + "Cache-Control"?: string; + /** @description The strong, opaque validator of the representation; send it back in If-Match on a guarded mutation or If-None-Match on a conditional read. */ + ETag?: string; [name: string]: unknown; }; content: { - "application/json": components["schemas"]["AdminNodeReprobe"]; + "application/json": components["schemas"]["AdminNodeDrainCapabilities"]; + }; + }; + /** @description The representation named by If-None-Match is current; no body. */ + 304: { + headers: { + /** @description The strong, opaque validator of the representation; send it back in If-Match on a guarded mutation or If-None-Match on a conditional read. */ + ETag?: string; + [name: string]: unknown; }; + content?: never; }; /** @description Bad Request */ 400: { @@ -55433,6 +56028,17 @@ export interface operations { "application/problem+json": components["schemas"]["Problem"]; }; }; + /** @description Precondition Failed */ + 412: { + headers: { + /** @description The strong, opaque validator of the representation; send it back in If-Match on a guarded mutation or If-None-Match on a conditional read. */ + ETag?: string; + [name: string]: unknown; + }; + content: { + "application/problem+json": components["schemas"]["Problem"]; + }; + }; /** @description Unprocessable Entity */ 422: { headers: { diff --git a/web/src/hooks/queries/admin/nodeConfiguration.test.tsx b/web/src/hooks/queries/admin/nodeConfiguration.test.tsx index 510cb15623..2a1bc9d8c7 100644 --- a/web/src/hooks/queries/admin/nodeConfiguration.test.tsx +++ b/web/src/hooks/queries/admin/nodeConfiguration.test.tsx @@ -226,6 +226,12 @@ it("setup create preserves authenticated profile absence", async () => { act(() => result.current.mutate({ name: "Setup", type: "proxy", url: node.url })); await waitFor(() => expect(result.current.isSuccess).toBe(true)); const headers = new Headers(fetchMock.mock.calls[0]![1].headers); + expect(JSON.parse(fetchMock.mock.calls[0]![1].body)).toEqual({ + name: "Setup", + type: "proxy", + url: node.url, + enabled: true, + }); expect(headers.get("Authorization")).toBe("Bearer synthetic"); expect(headers.get("X-Profile-Id")).toBe(""); expect(result.current.isAuthorityActive()).toBe(true); diff --git a/web/src/hooks/queries/admin/nodes.ts b/web/src/hooks/queries/admin/nodes.ts index 3c61d5cabc..f2bd3f81b5 100644 --- a/web/src/hooks/queries/admin/nodes.ts +++ b/web/src/hooks/queries/admin/nodes.ts @@ -144,7 +144,7 @@ function useNodeWrite() { if (intent.action === "create") { result = await v2("POST /api/v2/admin/nodes", { ...common, - body: intent.body as CreateNodeRequest, + body: { ...(intent.body as CreateNodeRequest), enabled: true }, onResponse, }); } else { From b5c7c0d97a8387fd2083a6f5416364e864f1e309 Mon Sep 17 00:00:00 2001 From: Silo Server Developer Date: Thu, 8 Oct 2026 08:55:16 +0200 Subject: [PATCH 2/3] fix(nodes): correct artifact admission and drain lookup races --- docs/architecture/worker-http-protocol.md | 6 + docs/transcode-node-api.md | 26 +- internal/api/testdata/media_routes.txt | 8 + internal/apiv2/admin_node_drain_db_test.go | 84 ++- internal/apiv2/worker_protocols_test.go | 17 +- internal/apiv2/worker_streaming_test.go | 2 +- internal/nodeconfig/drain.go | 12 +- internal/proxy/testdata/media_routes.txt | 4 + internal/transcodenode/drain.go | 23 +- .../drain_artifact_preparation_test.go | 105 ++++ internal/transcodenode/drain_artifact_test.go | 325 ++++++++++++ internal/transcodenode/server.go | 24 +- .../transcodenode/testdata/media_routes.txt | 4 + internal/workerdrain/gate.go | 180 +++++-- internal/workerdrain/gate_concurrency_test.go | 478 ++++++++++++++++++ scripts/ci/db-contracts.txt | 1 + 16 files changed, 1238 insertions(+), 61 deletions(-) create mode 100644 internal/transcodenode/drain_artifact_preparation_test.go create mode 100644 internal/transcodenode/drain_artifact_test.go create mode 100644 internal/workerdrain/gate_concurrency_test.go diff --git a/docs/architecture/worker-http-protocol.md b/docs/architecture/worker-http-protocol.md index e41c79d94a..2ec9cfe810 100644 --- a/docs/architecture/worker-http-protocol.md +++ b/docs/architecture/worker-http-protocol.md @@ -161,6 +161,12 @@ attest that its recipe matches the new request. Requested execution attestations require matching receipt contents. Missing or mismatched receipts can cause re-execution and replacement, and receipt invalidation/publication can fail. The GPU gate applies to new work, not successful existing-artifact reuse. +Worker retirement separately refuses new preparation and known-ID reuse after +a durable fence. Preparation admitted before the fence may finish; successful +publication or reuse retains an artifact permit while its lifecycle lock remains +held. Authenticated GET and HEAD of that admitted artifact may continue until +successful exact deletion or bounded permit expiry. Unknown artifacts and files +left by a preceding worker process cannot acquire a permit after fencing. Preparation follows request cancellation, but a lost response or cancellation does not establish that no bytes or receipt were published. This command is classified non-retryable; it supplies no durable cross-node admission or replay diff --git a/docs/transcode-node-api.md b/docs/transcode-node-api.md index b789184be2..90bb4a54d5 100644 --- a/docs/transcode-node-api.md +++ b/docs/transcode-node-api.md @@ -80,8 +80,8 @@ work, and subtitle-cache fills through child exit and publication/discard cleanu These cover the interval before job registration and after the ordinary job count drops. Request reservations cover in-flight execution and egress; bounded session permits cover gaps between media requests. -Permits remain until signed expiry, a positive session-deny marker, or confirmed -transport teardown. Header-authenticated media and internal transport permits +Permits remain until signed expiry, a positive session-deny marker, confirmed +transport teardown, or successful deletion of the exact prepared artifact. Header-authenticated media and internal transport permits use the maximum token lifetime, currently 24 hours. Absence from Redis and quiet bandwidth are never proof that a permit ended. @@ -103,6 +103,28 @@ retirement observations. An admitted, unsealed media identity may continue only until its original permit expiry; an outage cannot extend that permit or reopen a sealed worker. Ordinary token playback therefore keeps its bounded continuity without treating unavailable database authority as a drain acknowledgement. +A positively missing or changed registration or server realm refuses even a +known, unsealed permit; this is distinct from a temporary source I/O failure. + +Authority reads and explicit remote deny lookups run outside the gate's state +mutex, so slow database or Redis calls cannot block request release or independent +lookups. An internal epoch changes on binding, fence changes, every positive +identity loss, and zero sealing. Older successful reads and errors are discarded; +three consecutive stale reads refuse admission and proof. Positive deny replies +apply only to the observed permit generation, so revoke followed by re-admission +cannot lose its new permit to an older response. Final counters and timestamps +are sampled after those checks, under the sealing mutex. + +Prepared-download admission uses an artifact namespace separate from transport +identities. A successful preparation or reuse retains the artifact permit before +its request reservation and artifact lifecycle lock are released. An already +admitted artifact's authenticated GET and HEAD may continue during retirement. +Preparation, including a known artifact ID, refuses after fencing; a restarted +worker cannot adopt a permit merely because the file exists. A definitive missing +file removes its permit, and successful exact deletion revokes only that artifact. +Failed preparation retains no new permit; uncertain deletion retains its existing +permit. A preparation begun before fencing may finish and transfer its reservation, +which still prevents a zero receipt until deletion or bounded expiry. Only zero execution, request, and permit reservations sets `drained=true`. That observation seals admission under the same worker mutex, so a late dispatch diff --git a/internal/api/testdata/media_routes.txt b/internal/api/testdata/media_routes.txt index 3c1aa8306a..66f86d4370 100644 --- a/internal/api/testdata/media_routes.txt +++ b/internal/api/testdata/media_routes.txt @@ -199,10 +199,14 @@ GET /api/v2/admin/network-access/{provider}/status non-media GET /api/v2/admin/node-sessions non-media GET /api/v2/admin/nodes non-media POST /api/v2/admin/nodes non-media +GET /api/v2/admin/nodes/capabilities non-media POST /api/v2/admin/nodes/force-reload non-media DELETE /api/v2/admin/nodes/{id} non-media PUT /api/v2/admin/nodes/{id} non-media POST /api/v2/admin/nodes/{id}/check non-media +DELETE /api/v2/admin/nodes/{id}/drain non-media +GET /api/v2/admin/nodes/{id}/drain non-media +PUT /api/v2/admin/nodes/{id}/drain non-media POST /api/v2/admin/nodes/{id}/force-reload non-media POST /api/v2/admin/nodes/{id}/reprobe non-media POST /api/v2/admin/notifications/discord/test non-media @@ -1444,10 +1448,14 @@ GET /api/v2/admin/network-access/{provider}/status non-media GET /api/v2/admin/node-sessions non-media GET /api/v2/admin/nodes non-media POST /api/v2/admin/nodes non-media +GET /api/v2/admin/nodes/capabilities non-media POST /api/v2/admin/nodes/force-reload non-media DELETE /api/v2/admin/nodes/{id} non-media PUT /api/v2/admin/nodes/{id} non-media POST /api/v2/admin/nodes/{id}/check non-media +DELETE /api/v2/admin/nodes/{id}/drain non-media +GET /api/v2/admin/nodes/{id}/drain non-media +PUT /api/v2/admin/nodes/{id}/drain non-media POST /api/v2/admin/nodes/{id}/force-reload non-media POST /api/v2/admin/nodes/{id}/reprobe non-media POST /api/v2/admin/notifications/discord/test non-media diff --git a/internal/apiv2/admin_node_drain_db_test.go b/internal/apiv2/admin_node_drain_db_test.go index 785f5eb296..a903a7d1c9 100644 --- a/internal/apiv2/admin_node_drain_db_test.go +++ b/internal/apiv2/admin_node_drain_db_test.go @@ -7,6 +7,8 @@ import ( "net/http" "net/http/httptest" "os" + "path/filepath" + "strconv" "sync/atomic" "testing" "time" @@ -19,8 +21,10 @@ import ( "github.com/Silo-Server/silo-server/internal/config" "github.com/Silo-Server/silo-server/internal/nodeconfig" "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/nodesessions" "github.com/Silo-Server/silo-server/internal/proxy" "github.com/Silo-Server/silo-server/internal/streamtoken" + "github.com/Silo-Server/silo-server/internal/workerdrain" ) // The registered native routes, durable production stores, config watcher and @@ -53,7 +57,7 @@ func TestAdminNodeDrainPostgresProductionPath(t *testing.T) { listener.Close() t.Fatal(err) } - worker := proxy.NewServer(watcher, nil).Handler() + worker := proxy.NewServer(watcher, nodesessions.NewTracker(nil, url, name, "proxy")).Handler() unregistered := httptest.NewRecorder() request := httptest.NewRequest(http.MethodGet, "/admin/drain", nil) request.Header.Set("Authorization", "Bearer "+secret) @@ -301,3 +305,81 @@ func TestAdminNodeDrainPostgresProductionPath(t *testing.T) { t.Fatalf("old process reopened admission after URL reuse: %d %s", response.Code, response.Body.String()) } } + +// A known unsealed token must not reinterpret positively deleted PG authority +// as the bounded-continuity path for a transient connection failure. +func TestWorkerDrainKnownPermitMissingAuthorityPostgresProductionPath(t *testing.T) { + pool, _, native, url := nativeNodeCreationFixture(t) + if _, err := pool.Exec(t.Context(), `INSERT INTO server_settings(key,value) VALUES ('server.identity_id',$1) ON CONFLICT(key) DO NOTHING`, uuid.NewString()); err != nil { + t.Fatal(err) + } + created := do(t, native, http.MethodPost, Prefix+"/admin/nodes", fmt.Sprintf(`{"name":"authority-worker","type":"proxy","url":%q,"enabled":false}`, url), bearer(adminToken)) + var node AdminNode + if created.Code != http.StatusCreated || json.Unmarshal(created.Body.Bytes(), &node) != nil { + t.Fatalf("disabled registration: %d %s", created.Code, created.Body.String()) + } + id, err := strconv.Atoi(string(node.ID)) + if err != nil { + t.Fatal(err) + } + secret := "synthetic-known-permit-secret" + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + watcher := nodeconfig.NewWatcher(pool, nil, &cache.NoopEventBus{}, nodeconfig.BootstrapOverrides{Mode: "proxy", NodeURL: url, NodeName: "authority-worker"}) + dir := t.TempDir() + watcher.OnLoad(func(cfg *config.Config) { cfg.Auth.JWTSecret = secret; cfg.Playback.TranscodeDir = dir }) + if err := watcher.Start(ctx); err != nil { + t.Fatal(err) + } + if got, ok := watcher.NodeRowID(); !ok || got != id { + t.Fatalf("fresh registration binding: %d %t", got, ok) + } + worker := proxy.NewServer(watcher, nodesessions.NewTracker(nil, url, "authority-worker", "proxy")).Handler() + media := filepath.Join(dir, "sample.mp4") + if err := os.WriteFile(media, []byte("owned fixture media"), 0600); err != nil { + t.Fatal(err) + } + token, err := streamtoken.Sign(streamtoken.Claims{SessionID: "known-unsealed", MediaPath: media, PlayMethod: "direct"}, secret, time.Hour) + if err != nil { + t.Fatal(err) + } + request := func(path string, authenticated bool) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodGet, path, nil) + if authenticated { + req.Header.Set("Authorization", "Bearer "+secret) + } + response := httptest.NewRecorder() + worker.ServeHTTP(response, req) + return response + } + if response := request("/stream/direct/"+token, false); response.Code != http.StatusOK { + t.Fatalf("pre-loss actual media: %d %s", response.Code, response.Body.String()) + } + before := request("/admin/drain", true) + var status workerdrain.Status + if before.Code != http.StatusOK || json.Unmarshal(before.Body.Bytes(), &status) != nil || status.Fenced || status.ActiveRequests != 0 || status.ActiveReservations != 1 { + t.Fatalf("known unsealed permit not retained: %d %s", before.Code, before.Body.String()) + } + // Keep the watcher's remembered ID, but positively remove that exact row. + removed := do(t, native, http.MethodDelete, Prefix+"/admin/nodes/"+string(node.ID), "", with(bearer(adminToken), "If-Match", node.ConfigETag)) + if removed.Code != http.StatusNoContent { + t.Fatalf("guarded authority removal: %d %s", removed.Code, removed.Body.String()) + } + if got, ok := watcher.NodeRowID(); !ok || got != id { + t.Fatalf("fixture lost remembered identity before fresh lookup: %d %t", got, ok) + } + if response := request("/stream/direct/"+token, false); response.Code != http.StatusServiceUnavailable { + t.Fatalf("known permit survived positive registration loss: %d %s", response.Code, response.Body.String()) + } + if response := request("/admin/drain", true); response.Code != http.StatusServiceUnavailable { + t.Fatalf("missing registration produced proof: %d %s", response.Code, response.Body.String()) + } + var remaining int + if err := pool.QueryRow(t.Context(), `SELECT COUNT(*) FROM stream_nodes WHERE id=$1`, id).Scan(&remaining); err != nil || remaining != 0 { + t.Fatalf("exact registration cleanup: %d %v", remaining, err) + } + if data, err := os.ReadFile(media); err != nil || string(data) != "owned fixture media" { + t.Fatalf("authority loss changed media: %q %v", data, err) + } +} diff --git a/internal/apiv2/worker_protocols_test.go b/internal/apiv2/worker_protocols_test.go index 1b6c322cda..51546b7efc 100644 --- a/internal/apiv2/worker_protocols_test.go +++ b/internal/apiv2/worker_protocols_test.go @@ -145,13 +145,18 @@ func TestWorkerControlsRetainProtocolAndBearerGate(t *testing.T) { "proxy": proxy.NewServer(watcher, nil).Handler(), "transcode_node": transcodenode.NewServer(watcher, nil).Handler(), } - count := 0 + count, drainControls := 0, 0 for _, op := range describeWorkerProtocols().Operations { if op.Method != http.MethodPost || !strings.HasPrefix(op.Path, "/admin/") { continue } count++ - if op.RetrySafety != "non_retryable" || op.Description == "" { + expectedRetry := "non_retryable" + if op.Path == "/admin/drain" { + drainControls++ + expectedRetry = "natural_idempotent" + } + if op.RetrySafety != expectedRetry || op.Description == "" { t.Fatalf("command lacks uncertainty semantics: %+v", op) } if op.Responses["401"].Content["text/plain"] == nil { @@ -163,16 +168,16 @@ func TestWorkerControlsRetainProtocolAndBearerGate(t *testing.T) { if response.Code != http.StatusUnauthorized || response.Header().Get("Content-Type") != "text/plain; charset=utf-8" { t.Fatalf("%s %s bearer refusal: %d %s", op.Listener, op.Path, response.Code, response.Body) } - if op.Path == "/admin/reprobe-capabilities" { + if op.Path == "/admin/reprobe-capabilities" || op.Path == "/admin/drain" { if op.Responses["409"] == nil || op.Responses["503"] == nil || op.Responses["200"].Content["application/json"] == nil { - t.Fatal("incomplete reprobe protocol", op.Listener) + t.Fatal("incomplete worker control protocol", op.Listener) } } else if op.Responses["204"] == nil || len(op.Responses["204"].Content) != 0 || op.Responses["500"] == nil { t.Fatal("reload must preserve empty success and possible failure", op.Listener, op.Path) } } - if count != 6 { - t.Fatalf("got %d worker controls, want 6", count) + if count != 8 || drainControls != 2 || count-drainControls != 6 { + t.Fatalf("got %d worker controls (%d drain), want 6 legacy and 2 drain", count, drainControls) } } diff --git a/internal/apiv2/worker_streaming_test.go b/internal/apiv2/worker_streaming_test.go index d67943f29b..d0a231cbe9 100644 --- a/internal/apiv2/worker_streaming_test.go +++ b/internal/apiv2/worker_streaming_test.go @@ -36,7 +36,7 @@ func TestWorkerDeliveryInventoryDescriptionCoverage(t *testing.T) { t.Fatal("undescribed worker registration", route.Listener, route.Method, route.Path) } } - if count != 49 || len(described) != 49 { + if count != 53 || len(described) != 53 { t.Fatal("retained inventory drift", count, len(described)) } } diff --git a/internal/nodeconfig/drain.go b/internal/nodeconfig/drain.go index 2c074dcce2..124d67a0bf 100644 --- a/internal/nodeconfig/drain.go +++ b/internal/nodeconfig/drain.go @@ -21,14 +21,20 @@ func (w *Watcher) ReadDrainFence(ctx context.Context) (int, string, string, erro } id, ok := w.NodeRowID() if !ok { - return 0, "", "", workerdrain.ErrUnavailable + return 0, "", "", workerdrain.ErrIdentityUnavailable } var fence, realm string err := w.pool.QueryRow(ctx, `SELECT COALESCE(f.fence_id,''), s.value FROM stream_nodes n LEFT JOIN stream_node_drain_fences f ON f.node_id=n.id JOIN server_settings s ON s.key=$2 WHERE n.id=$1`, id, serveridentity.Key).Scan(&fence, &realm) - if errors.Is(err, pgx.ErrNoRows) || strings.TrimSpace(realm) == "" { - return 0, "", "", workerdrain.ErrUnavailable + if errors.Is(err, pgx.ErrNoRows) { + return 0, "", "", workerdrain.ErrIdentityUnavailable + } + if err != nil { + return 0, "", "", err + } + if strings.TrimSpace(realm) == "" { + return 0, "", "", workerdrain.ErrIdentityUnavailable } return id, fence, strings.TrimSpace(realm), err } diff --git a/internal/proxy/testdata/media_routes.txt b/internal/proxy/testdata/media_routes.txt index b45a50c1ee..b8dda2203a 100644 --- a/internal/proxy/testdata/media_routes.txt +++ b/internal/proxy/testdata/media_routes.txt @@ -1,4 +1,6 @@ # fixture 1 +GET /admin/drain non-media +POST /admin/drain non-media POST /admin/force-reload non-media POST /admin/reload-config non-media POST /admin/reprobe-capabilities non-media @@ -31,6 +33,8 @@ GET /stream/v3/{session_id}/master.m3u8 media manifest viewer_egress true true HEAD /stream/v3/{session_id}/master.m3u8 media manifest viewer_egress true true GET /stream/v3/{session_id}/segment/{name} media playback viewer_egress true true # fixture 2 +GET /admin/drain non-media +POST /admin/drain non-media POST /admin/force-reload non-media POST /admin/reload-config non-media POST /admin/reprobe-capabilities non-media diff --git a/internal/transcodenode/drain.go b/internal/transcodenode/drain.go index 19dfb2086d..9fa4d3faa0 100644 --- a/internal/transcodenode/drain.go +++ b/internal/transcodenode/drain.go @@ -8,14 +8,26 @@ import ( "strings" "time" + "github.com/Silo-Server/silo-server/internal/downloadprepare" "github.com/Silo-Server/silo-server/internal/mediasample" "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/tonemap" ) +func transportDrainKey(id string) string { + if id == "" { + return "" + } + return "transport:" + id +} + +func artifactDrainKey(id string) string { return "artifact:" + id } + const ( drainStartSegment = "start" drainTranscodeSegment = "transcode" + downloadPreparePath = "/downloads/prepare" + transcodeStartPath = "/transcode/start" ) func (s *Server) drainActiveJobs() int { @@ -31,7 +43,7 @@ func (s *Server) drainActiveJobs() int { func (s *Server) drainMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { path := r.URL.Path - if s.drain == nil || path == "/admin/drain" || path == "/api/v1/health" || path == "/metrics" || path == "/status" || path == "/admin/reload-config" || path == "/admin/force-reload" || r.Method == http.MethodDelete || strings.HasSuffix(path, "/progress") || strings.HasSuffix(path, "/downloaded") { + if s.drain == nil || path == "/admin/drain" || path == "/api/v1/health" || path == "/metrics" || path == "/status" || path == "/admin/reload-config" || path == "/admin/force-reload" || path == downloadPreparePath || r.Method == http.MethodDelete || strings.HasSuffix(path, "/progress") || strings.HasSuffix(path, "/downloaded") { next.ServeHTTP(w, r) return } @@ -44,9 +56,12 @@ func (s *Server) drainMiddleware(next http.Handler) http.Handler { key := "" parts := strings.Split(strings.Trim(path, "/"), "/") if len(parts) >= 2 && (parts[0] == drainTranscodeSegment || parts[0] == "remux") && parts[1] != drainStartSegment { - key = parts[1] + key = transportDrainKey(parts[1]) + } + if len(parts) == 3 && parts[0] == "downloads" && parts[1] == "artifacts" && downloadprepare.ValidArtifactID(parts[2]) && (r.Method == http.MethodGet || r.Method == http.MethodHead) { + key = artifactDrainKey(parts[2]) } - if path == "/transcode/start" { + if path == transcodeStartPath { body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 64<<10)) if err != nil { http.Error(w, "invalid request body", http.StatusBadRequest) @@ -57,7 +72,7 @@ func (s *Server) drainMiddleware(next http.Handler) http.Handler { SessionID string `json:"session_id"` } if json.Unmarshal(body, &input) == nil { - key = input.SessionID + key = transportDrainKey(input.SessionID) } } end, err := s.drain.Begin(r.Context(), key, time.Now().Add(playback.MaxTokenTTL)) diff --git a/internal/transcodenode/drain_artifact_preparation_test.go b/internal/transcodenode/drain_artifact_preparation_test.go new file mode 100644 index 0000000000..67c1810d87 --- /dev/null +++ b/internal/transcodenode/drain_artifact_preparation_test.go @@ -0,0 +1,105 @@ +//go:build unix + +package transcodenode + +import ( + "bufio" + "context" + "fmt" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/downloadprepare" + "github.com/Silo-Server/silo-server/internal/workerdrain" + "golang.org/x/sys/unix" +) + +func TestTranscodeDrainRetainsPreparationPublishedAfterFence(t *testing.T) { + dir := t.TempDir() + startedPath, releasePath := filepath.Join(dir, "started.fifo"), filepath.Join(dir, "release.fifo") + for _, path := range []string{startedPath, releasePath} { + if err := unix.Mkfifo(path, 0600); err != nil { + t.Fatal(err) + } + } + startedPipe, err := os.OpenFile(startedPath, os.O_RDWR, 0600) + if err != nil { + t.Fatal(err) + } + defer func() { _ = startedPipe.Close() }() + releasePipe, err := os.OpenFile(releasePath, os.O_RDWR, 0600) + if err != nil { + t.Fatal(err) + } + defer func() { _ = releasePipe.Close() }() + quote := func(value string) string { return "'" + strings.ReplaceAll(value, "'", "'\\''") + "'" } + ffmpeg := filepath.Join(dir, "ffmpeg.sh") + program := fmt.Sprintf("#!/bin/sh\nfor last; do :; done\nprintf 'started\\n' > %s\nread ignored < %s\nprintf artifact > \"$last\"\n", quote(startedPath), quote(releasePath)) + if err := os.WriteFile(ffmpeg, []byte(program), 0700); err != nil { + t.Fatal(err) + } + server := newTestServer(t) + server.watcher.Config().Playback.FFmpegPath = ffmpeg + server.watcher.Config().Playback.HWAccel = "none" + var fence atomic.Value + fence.Store("") + server.drain = workerdrain.New(func(context.Context) (int, string, string, error) { + return 7, fence.Load().(string), "synthetic-realm", nil + }, server.drainActiveJobs, nil) + listener := httptest.NewServer(server.Handler()) + defer listener.Close() + var once sync.Once + unblock := func() { once.Do(func() { _, _ = releasePipe.WriteString("release\n") }) } + started := make(chan error, 1) + go func() { _, err := bufio.NewReader(startedPipe).ReadString('\n'); started <- err }() + completed := make(chan error, 1) + preparer := downloadprepare.HTTPPreparer{} + request := downloadprepare.Request{ArtifactID: "prepared", InputPath: "/media/movie.mkv", TargetCodecVideo: "copy", TargetCodecAudio: "copy", AudioTrackIndex: -1} + go func() { _, err := preparer.Prepare(t.Context(), listener.URL, testSecret, request); completed <- err }() + defer func() { unblock(); <-completed }() + select { + case err := <-started: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("actual preparation child did not start") + } + fence.Store("durable-fence") + proof, err := server.drain.Observe(t.Context()) + if err != nil || proof.Drained || proof.ActiveJobs < 1 || proof.ActiveRequests != 1 || proof.ActiveReservations != 0 { + t.Fatalf("pre-fence preparation was not reserved: %+v %v", proof, err) + } + unblock() + select { + case err = <-completed: + completed <- err + case <-time.After(5 * time.Second): + t.Fatal("actual preparation did not finish") + } + if err != nil { + t.Fatal(err) + } + proof, err = server.drain.Observe(t.Context()) + if err != nil || proof.Drained || proof.ActiveJobs != 0 || proof.ActiveRequests != 0 || proof.ActiveReservations != 1 { + t.Fatalf("publication lost artifact reservation: %+v %v", proof, err) + } + if err := preparer.Delete(t.Context(), listener.URL, testSecret, request.ArtifactID); err != nil { + t.Fatal(err) + } + proof, err = server.drain.Observe(t.Context()) + if err != nil || !proof.Drained || proof.ActiveReservations != 0 { + t.Fatalf("publication cleanup: %+v %v", proof, err) + } + for _, suffix := range []string{".mp4", ".mp4.part"} { + if _, err := os.Stat(filepath.Join(server.artifactRoot, request.ArtifactID+suffix)); !os.IsNotExist(err) { + t.Fatalf("artifact file cleanup: %v", err) + } + } +} diff --git a/internal/transcodenode/drain_artifact_test.go b/internal/transcodenode/drain_artifact_test.go new file mode 100644 index 0000000000..270f33936d --- /dev/null +++ b/internal/transcodenode/drain_artifact_test.go @@ -0,0 +1,325 @@ +package transcodenode + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/downloadprepare" + "github.com/Silo-Server/silo-server/internal/workerdrain" +) + +func TestTranscodeDrainRetainsPreparedArtifactDelivery(t *testing.T) { + server := newTestServer(t) + ffmpeg := filepath.Join(t.TempDir(), "ffmpeg.sh") + if err := os.WriteFile(ffmpeg, []byte("#!/bin/sh\nfor last; do :; done\nprintf artifact > \"$last\"\n"), 0700); err != nil { + t.Fatal(err) + } + server.watcher.Config().Playback.FFmpegPath = ffmpeg + server.watcher.Config().Playback.HWAccel = "none" + var fence atomic.Value + fence.Store("") + source := func(context.Context) (int, string, string, error) { + return 7, fence.Load().(string), "synthetic-realm", nil + } + server.drain = workerdrain.New(source, server.drainActiveJobs, nil) + listener := httptest.NewServer(server.Handler()) + defer listener.Close() + preparer := downloadprepare.HTTPPreparer{} + prepare := downloadprepare.Request{ArtifactID: "prepared", InputPath: "/media/movie.mkv", TargetCodecVideo: "copy", TargetCodecAudio: "copy", AudioTrackIndex: -1} + result, err := preparer.Prepare(t.Context(), listener.URL, testSecret, prepare) + if err != nil || result.FileSize != int64(len("artifact")) { + t.Fatalf("production preparation: %+v %v", result, err) + } + fence.Store("durable-fence") + for _, method := range []string{http.MethodHead, http.MethodGet} { + response, err := preparer.Open(t.Context(), listener.URL, testSecret, prepare.ArtifactID, method, nil) + if err != nil { + t.Fatalf("existing %s artifact refused during drain: %v", method, err) + } + body, readErr := io.ReadAll(response.Body) + closeErr := response.Body.Close() + if readErr != nil || closeErr != nil || response.StatusCode != http.StatusOK { + t.Fatalf("delivery %s: %d %v %v", method, response.StatusCode, readErr, closeErr) + } + if method == http.MethodGet && string(body) != "artifact" { + t.Fatalf("artifact delivery: %q", body) + } + } + proof, err := server.drain.Observe(t.Context()) + if err != nil || proof.Drained || proof.ActiveRequests != 0 || proof.ActiveReservations != 1 { + t.Fatalf("quiet artifact lost: %+v %v", proof, err) + } + if _, err := preparer.Prepare(t.Context(), listener.URL, testSecret, prepare); err == nil { + t.Fatal("known artifact preparation admitted after fence") + } + for _, method := range []string{http.MethodGet, http.MethodHead} { + response, err := preparer.Open(t.Context(), listener.URL, testSecret, "late", method, nil) + if err != nil { + t.Fatal(err) + } + closeErr := response.Body.Close() + if response.StatusCode != http.StatusServiceUnavailable || closeErr != nil { + t.Fatalf("unknown artifact admitted after fence: %s status=%d close=%v", method, response.StatusCode, closeErr) + } + } + // Raw transport IDs cannot inherit a permit from the artifact namespace. + for _, route := range []struct{ method, path, body string }{ + {http.MethodGet, "/transcode/artifact:prepared/master.m3u8", ""}, + {http.MethodGet, "/remux/artifact:prepared", ""}, + {http.MethodPost, "/transcode/start", `{"session_id":"artifact:prepared"}`}, + } { + request, err := http.NewRequestWithContext(t.Context(), route.method, listener.URL+route.path, strings.NewReader(route.body)) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Authorization", "Bearer "+testSecret) + response, err := listener.Client().Do(request) + if err != nil { + t.Fatal(err) + } + _ = response.Body.Close() + if response.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("transport inherited artifact permit: %s %s %d", route.method, route.path, response.StatusCode) + } + } + original := server.drain + server.drain = workerdrain.New(source, server.drainActiveJobs, nil) + response, restartErr := preparer.Open(t.Context(), listener.URL, testSecret, prepare.ArtifactID, http.MethodGet, nil) + server.drain = original + if restartErr != nil { + t.Fatal(restartErr) + } + if closeErr := response.Body.Close(); closeErr != nil || response.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("restart resurrected an artifact permit from disk: status=%d close=%v", response.StatusCode, closeErr) + } + if err := preparer.Delete(t.Context(), listener.URL, testSecret, prepare.ArtifactID); err != nil { + t.Fatal(err) + } + proof, err = server.drain.Observe(t.Context()) + if err != nil || !proof.Drained || proof.ActiveReservations != 0 { + t.Fatalf("artifact deletion did not retire permit: %+v %v", proof, err) + } + if _, err := os.Stat(filepath.Join(server.artifactRoot, prepare.ArtifactID+".mp4")); !os.IsNotExist(err) { + t.Fatalf("artifact cleanup: %v", err) + } +} + +func TestTranscodeDrainArtifactFailureAndExactCleanup(t *testing.T) { + for _, scenario := range []string{"unauthorized admission", "quiet missing artifact", "failed preparation", "failed deletion", "exact deletion"} { + t.Run(scenario, func(t *testing.T) { + server := newTestServer(t) + ffmpeg := filepath.Join(t.TempDir(), "ffmpeg.sh") + program := "#!/bin/sh\nfor last; do :; done\nprintf artifact > \"$last\"\n" + if scenario == "failed preparation" { + program = "#!/bin/sh\nexit 1\n" + } + if err := os.WriteFile(ffmpeg, []byte(program), 0700); err != nil { + t.Fatal(err) + } + server.watcher.Config().Playback.FFmpegPath = ffmpeg + server.watcher.Config().Playback.HWAccel = "none" + var fence atomic.Value + fence.Store("") + server.drain = workerdrain.New(func(context.Context) (int, string, string, error) { + return 7, fence.Load().(string), "synthetic-realm", nil + }, server.drainActiveJobs, nil) + listener := httptest.NewServer(server.Handler()) + defer listener.Close() + preparer := downloadprepare.HTTPPreparer{} + request := downloadprepare.Request{ArtifactID: "prepared", InputPath: "/media/movie.mkv", TargetCodecVideo: "copy", TargetCodecAudio: "copy", AudioTrackIndex: -1} + switch scenario { + case "unauthorized admission": + payload, err := json.Marshal(request) + if err != nil { + t.Fatal(err) + } + for _, token := range []string{"", "wrong-node-bearer"} { + for _, method := range []string{http.MethodPost, http.MethodGet, http.MethodHead} { + path := "/downloads/artifacts/prepared" + if method == http.MethodPost { + path = "/downloads/prepare" + } + request, err := http.NewRequestWithContext(t.Context(), method, listener.URL+path, bytes.NewReader(payload)) + if err != nil { + t.Fatal(err) + } + if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + response, err := listener.Client().Do(request) + if err != nil { + t.Fatal(err) + } + _ = response.Body.Close() + if response.StatusCode != http.StatusUnauthorized { + t.Fatalf("unauthenticated artifact admission: %s %d", method, response.StatusCode) + } + } + } + case "quiet missing artifact": + for _, method := range []string{http.MethodGet, http.MethodHead} { + response, err := preparer.Open(t.Context(), listener.URL, testSecret, "missing", method, nil) + if err != nil { + t.Fatal(err) + } + if closeErr := response.Body.Close(); closeErr != nil || response.StatusCode != http.StatusNotFound { + t.Fatalf("missing artifact: %s status=%d close=%v", method, response.StatusCode, closeErr) + } + } + case "failed preparation": + if _, err := preparer.Prepare(t.Context(), listener.URL, testSecret, request); err == nil { + t.Fatal("faulting command succeeded") + } + case "failed deletion", "exact deletion": + if _, err := preparer.Prepare(t.Context(), listener.URL, testSecret, request); err != nil { + t.Fatal(err) + } + if scenario == "exact deletion" { + request.ArtifactID = "survivor" + if _, err := preparer.Prepare(t.Context(), listener.URL, testSecret, request); err != nil { + t.Fatal(err) + } + } + fence.Store("durable-fence") + if scenario == "failed deletion" { + part := filepath.Join(server.artifactRoot, "prepared.mp4.part") + if err := os.Mkdir(part, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(part, "owned-fault"), []byte("fault"), 0600); err != nil { + t.Fatal(err) + } + if err := preparer.Delete(t.Context(), listener.URL, testSecret, "prepared"); err == nil { + t.Fatal("faulting deletion succeeded") + } + } else if err := preparer.Delete(t.Context(), listener.URL, testSecret, "prepared"); err != nil { + t.Fatal(err) + } + } + fence.Store("durable-fence") + proof, err := server.drain.Observe(t.Context()) + wantReservations := 0 + if scenario == "failed deletion" || scenario == "exact deletion" { + wantReservations = 1 + } + if err != nil || proof.ActiveJobs != 0 || proof.ActiveRequests != 0 || proof.ActiveReservations != wantReservations || proof.Drained != (wantReservations == 0) { + t.Fatalf("fault/cleanup proof: %+v %v", proof, err) + } + if scenario == "exact deletion" { + if err := preparer.Delete(t.Context(), listener.URL, testSecret, "survivor"); err != nil { + t.Fatal(err) + } + proof, err = server.drain.Observe(t.Context()) + if err != nil || !proof.Drained { + t.Fatalf("survivor cleanup: %+v %v", proof, err) + } + } + if scenario == "failed preparation" { + for _, suffix := range []string{".mp4", ".mp4.part"} { + if _, err := os.Stat(filepath.Join(server.artifactRoot, "prepared"+suffix)); !os.IsNotExist(err) { + t.Fatalf("failed preparation cleanup: %v", err) + } + } + } + }) + } +} + +type blockedArtifactResponse struct { + *httptest.ResponseRecorder + entered chan struct{} + release <-chan struct{} + once sync.Once +} + +func (w *blockedArtifactResponse) Write(data []byte) (int, error) { + w.once.Do(func() { close(w.entered); <-w.release }) + return w.ResponseRecorder.Write(data) +} + +func TestTranscodeDrainCachedArtifactDeleteCannotRacePermitTransfer(t *testing.T) { + server := newTestServer(t) + ffmpeg := filepath.Join(t.TempDir(), "ffmpeg.sh") + if err := os.WriteFile(ffmpeg, []byte("#!/bin/sh\nfor last; do :; done\nprintf artifact > \"$last\"\n"), 0700); err != nil { + t.Fatal(err) + } + server.watcher.Config().Playback.FFmpegPath = ffmpeg + server.watcher.Config().Playback.HWAccel = "none" + var fence atomic.Value + fence.Store("") + server.drain = workerdrain.New(func(context.Context) (int, string, string, error) { + return 7, fence.Load().(string), "synthetic-realm", nil + }, server.drainActiveJobs, nil) + listener := httptest.NewServer(server.Handler()) + defer listener.Close() + preparer := downloadprepare.HTTPPreparer{} + input := downloadprepare.Request{ArtifactID: "prepared", InputPath: "/media/movie.mkv", TargetCodecVideo: "copy", TargetCodecAudio: "copy", AudioTrackIndex: -1} + if _, err := preparer.Prepare(t.Context(), listener.URL, testSecret, input); err != nil { + t.Fatal(err) + } + payload, err := json.Marshal(input) + if err != nil { + t.Fatal(err) + } + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + request := httptest.NewRequest(http.MethodPost, "/downloads/prepare", bytes.NewReader(payload)) + request.Header.Set("Authorization", "Bearer "+testSecret) + writer := &blockedArtifactResponse{ResponseRecorder: httptest.NewRecorder(), entered: entered, release: release} + finished := make(chan struct{}) + go func() { server.Handler().ServeHTTP(writer, request); close(finished) }() + defer func() { unblock(); <-finished }() + select { + case <-entered: + case <-time.After(3 * time.Second): + t.Fatal("routed cached publication did not write result") + } + fence.Store("durable-fence") + deleted := make(chan error, 1) + go func() { deleted <- preparer.Delete(t.Context(), listener.URL, testSecret, input.ArtifactID) }() + defer func() { unblock(); <-deleted }() + // Give deletion the opportunity to complete while the result writer is + // stalled. A broken read-lock scope makes that ordering deterministic. + select { + case err := <-deleted: + deleted <- err + case <-time.After(250 * time.Millisecond): + } + unblock() + select { + case <-finished: + case <-time.After(3 * time.Second): + t.Fatal("cached publication did not finish") + } + select { + case err := <-deleted: + deleted <- err + if err != nil { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + t.Fatal("exact deletion did not finish") + } + if writer.Code != http.StatusOK { + t.Fatalf("cached reuse: %d %s", writer.Code, writer.Body) + } + proof, err := server.drain.Observe(t.Context()) + if err != nil || !proof.Drained || proof.ActiveReservations != 0 || proof.ActiveRequests != 0 { + t.Fatalf("successful deletion retained a phantom permit: %+v %v", proof, err) + } + if _, err := os.Stat(filepath.Join(server.artifactRoot, input.ArtifactID+".mp4")); !os.IsNotExist(err) { + t.Fatalf("exact artifact cleanup: %v", err) + } +} diff --git a/internal/transcodenode/server.go b/internal/transcodenode/server.go index 9ee3a1e987..1319f2a746 100644 --- a/internal/transcodenode/server.go +++ b/internal/transcodenode/server.go @@ -524,7 +524,8 @@ func NewServer(watcher *nodeconfig.Watcher, tracker *nodesessions.Tracker) *Serv s.registeredNodeURL = watcher.NodeRegisteredURL if watcher.HasDrainAuthority() { s.drain = workerdrain.New(watcher.ReadDrainFence, s.drainActiveJobs, func(ctx context.Context, key string) bool { - return s.streamDeny != nil && s.streamDeny.Denied(ctx, key) + transportID, ok := strings.CutPrefix(key, "transport:") + return ok && s.streamDeny != nil && s.streamDeny.Denied(ctx, transportID) }) } } @@ -966,6 +967,13 @@ func (s *Server) handleDownloadPrepare(w http.ResponseWriter, r *http.Request) { if !s.requireApprovedInputPath(w, r, req.InputPath) { return } + finishPreparation, err := s.drain.BeginPreparation(r.Context(), artifactDrainKey(req.ArtifactID), time.Now().Add(playback.MaxTokenTTL)) + if err != nil { + http.Error(w, "worker retiring or drain authority unavailable", http.StatusServiceUnavailable) + return + } + published := false + defer func() { finishPreparation(published) }() opts := req.TranscodeOpts(cfg.Playback.FFmpegPath, cfg.Playback.HWAccel, cfg.Playback.HWDevice, s.ffmpegSink) artifactRoot := s.artifactRoot if err := os.MkdirAll(artifactRoot, 0o755); err != nil { @@ -979,11 +987,15 @@ func (s *Server) handleDownloadPrepare(w http.ResponseWriter, r *http.Request) { if _, reusable := existingDownloadPrepareResult(outputPath, req); reusable { readUnlock := s.lockSessionLifecycleRead("download-artifact-" + req.ArtifactID) result, stillReusable := existingDownloadPrepareResult(outputPath, req) - readUnlock() if stillReusable { + // Transfer the reservation before deletion can acquire the same + // artifact lock and positively revoke its exact permit. + defer func() { finishPreparation(published); readUnlock() }() + published = true writeDownloadPrepareResult(w, result) return } + readUnlock() } // Claimed only once this request is committed to producing something: a // prepared download encodes on the GPU like any transcode, and the tone-map @@ -1002,8 +1014,9 @@ func (s *Server) handleDownloadPrepare(w http.ResponseWriter, r *http.Request) { } } unlock := s.lockSessionLifecycle("download-artifact-" + req.ArtifactID) - defer unlock() + defer func() { finishPreparation(published); unlock() }() if result, ok := existingDownloadPrepareResult(outputPath, req); ok { + published = true writeDownloadPrepareResult(w, result) return } @@ -1061,6 +1074,7 @@ func (s *Server) handleDownloadPrepare(w http.ResponseWriter, r *http.Request) { return } } + published = true writeDownloadPrepareResult(w, result) } @@ -1200,6 +1214,7 @@ func (s *Server) handleDownloadArtifact(w http.ResponseWriter, r *http.Request) f, err := os.Open(path) if err != nil { if os.IsNotExist(err) { + s.drain.Revoke(artifactDrainKey(artifactID)) http.NotFound(w, r) return } @@ -1246,6 +1261,7 @@ func (s *Server) handleDeleteDownloadArtifact(w http.ResponseWriter, r *http.Req return } } + s.drain.Revoke(artifactDrainKey(artifactID)) w.WriteHeader(http.StatusNoContent) } @@ -2537,7 +2553,7 @@ func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) { http.Error(w, "session not found", http.StatusNotFound) return } - s.drain.Revoke(sessionID) + s.drain.Revoke(transportDrainKey(sessionID)) w.WriteHeader(http.StatusNoContent) } diff --git a/internal/transcodenode/testdata/media_routes.txt b/internal/transcodenode/testdata/media_routes.txt index 75225c7859..37fd32e31c 100644 --- a/internal/transcodenode/testdata/media_routes.txt +++ b/internal/transcodenode/testdata/media_routes.txt @@ -1,4 +1,6 @@ # fixture 1 +GET /admin/drain non-media +POST /admin/drain non-media POST /admin/force-reload non-media POST /admin/reload-config non-media POST /admin/reprobe-capabilities non-media @@ -22,6 +24,8 @@ GET /transcode/{session_id}/segment/{name} media playback internal_relay false t POST /transcode/{session_id}/segment/{name}/downloaded non-media POST /trickplay/extract non-media # fixture 2 +GET /admin/drain non-media +POST /admin/drain non-media POST /admin/force-reload non-media POST /admin/reload-config non-media POST /admin/reprobe-capabilities non-media diff --git a/internal/workerdrain/gate.go b/internal/workerdrain/gate.go index 3b5f163cd0..47cae29105 100644 --- a/internal/workerdrain/gate.go +++ b/internal/workerdrain/gate.go @@ -14,8 +14,14 @@ import ( var ErrUnavailable = errors.New("worker drain authority unavailable") var ErrFenced = errors.New("worker admission fenced") +// ErrIdentityUnavailable distinguishes positive registration/realm loss from +// a transient source I/O failure; known permits cannot use outage continuity. +var ErrIdentityUnavailable = errors.Join(ErrUnavailable, errors.New("worker identity missing or changed")) + +var errAuthorityStale = errors.Join(ErrUnavailable, errors.New("worker authority changed during lookup")) + // Source resolves the worker's stable registered identity and current durable -// fence. Calls are serialized with admission and status inside this process. +// fence. It may be called concurrently; each read must resolve fresh authority. type Source func(context.Context) (nodeID int, fenceID, nativeServerID string, err error) // Identity correlates public health with an authenticated drain receipt from @@ -39,47 +45,90 @@ type Status struct { } type Gate struct { - mu sync.Mutex - source Source - jobs func() int - revoked func(context.Context, string) bool - now func() time.Time - instanceID string - serverID string - nodeID int - fenceID string - sealed bool - active int - permits map[string]time.Time - lastPruned time.Time + mu sync.Mutex + source Source + jobs func() int + revoked func(context.Context, string) bool + now func() time.Time + instanceID string + serverID string + nodeID int + fenceID string + sealed bool + active int + permits map[string]time.Time + lastPruned time.Time + authorityEpoch uint64 + identityLost bool + permitVersions map[string]uint64 + nextPermitVersion uint64 } func New(source Source, jobs func() int, revoked func(context.Context, string) bool) *Gate { - return &Gate{source: source, jobs: jobs, revoked: revoked, now: time.Now, instanceID: uuid.NewString(), permits: make(map[string]time.Time)} + return &Gate{source: source, jobs: jobs, revoked: revoked, now: time.Now, instanceID: uuid.NewString(), permits: make(map[string]time.Time), permitVersions: make(map[string]uint64)} } +// syncLocked enters and returns with mu held, releasing it around source I/O. +// An epoch change invalidates both successes and errors from an older read. func (g *Gate) syncLocked(ctx context.Context) error { if g.source == nil { return ErrUnavailable } - id, fence, serverID, err := g.source(ctx) - if err != nil { - return err - } - if id <= 0 || serverID == "" { - return ErrUnavailable + for attempt := 0; attempt < 3; attempt++ { + epoch := g.authorityEpoch + g.mu.Unlock() + id, fence, serverID, err := g.source(ctx) + g.mu.Lock() + if epoch != g.authorityEpoch { + continue + } + if ctx.Err() != nil { + return ctx.Err() + } + if err != nil { + if errors.Is(err, ErrIdentityUnavailable) { + g.loseIdentityLocked() + } + return err + } + if id <= 0 || serverID == "" || g.nodeID != 0 && (g.nodeID != id || g.serverID != serverID) { + g.loseIdentityLocked() + return ErrIdentityUnavailable + } + if g.nodeID == 0 || g.identityLost || fence != g.fenceID { + g.authorityEpoch++ + } + g.nodeID, g.serverID, g.identityLost = id, serverID, false + if fence != g.fenceID { + // Only a fresh explicit durable cancellation can reopen sealing. + g.fenceID, g.sealed = fence, false + } + return nil } - if g.nodeID != 0 && (g.nodeID != id || g.serverID != serverID) { - return ErrUnavailable + return errAuthorityStale +} + +func (g *Gate) loseIdentityLocked() { + // Every positive loss invalidates reads already in flight, including reads + // that began after an earlier loss but before another foreign observation. + g.authorityEpoch++ + g.identityLost = true +} + +func (g *Gate) retainLocked(key string, expiry time.Time) { + if key == "" || !expiry.After(g.now()) { + return } - g.nodeID = id - g.serverID = serverID - if fence != g.fenceID { - // Only an explicit durable cancellation can reopen a sealed worker. - g.fenceID = fence - g.sealed = false + if expiry.After(g.permits[key]) { + g.permits[key] = expiry } - return nil + g.nextPermitVersion++ + g.permitVersions[key] = g.nextPermitVersion +} + +func (g *Gate) revokeLocked(key string) { + delete(g.permits, key) + delete(g.permitVersions, key) } // PublicIdentity returns the last verified process/realm binding. Its first @@ -99,17 +148,44 @@ func (g *Gate) PublicIdentity(ctx context.Context) Identity { return Identity{NativeServerID: g.serverID, WorkerInstanceID: g.instanceID} } -func (g *Gate) pruneLocked(ctx context.Context, force bool) { +// pruneLocked keeps expiry checks local. Explicit deny observations run without +// mu and apply only to the exact permit generation that was observed. +func (g *Gate) pruneLocked(ctx context.Context, force bool) bool { now := g.now() if !force && now.Sub(g.lastPruned) < time.Minute { - return + return true } g.lastPruned = now for key, expiry := range g.permits { - if !expiry.After(now) || (force && g.revoked != nil && g.revoked(ctx, key)) { - delete(g.permits, key) + if !expiry.After(now) { + g.revokeLocked(key) + } + } + if !force || g.revoked == nil || len(g.permits) == 0 { + return true + } + versions := make(map[string]uint64, len(g.permits)) + for key := range g.permits { + versions[key] = g.permitVersions[key] + } + epoch := g.authorityEpoch + g.mu.Unlock() + denied := make([]string, 0, len(versions)) + for key := range versions { + if g.revoked(ctx, key) { + denied = append(denied, key) } } + g.mu.Lock() + if epoch != g.authorityEpoch || ctx.Err() != nil { + return false + } + for _, key := range denied { + if current, exists := g.permitVersions[key]; exists && current == versions[key] { + g.revokeLocked(key) + } + } + return true } // Begin reserves a request before any execution or egress can start. Existing @@ -128,7 +204,7 @@ func (g *Gate) Begin(ctx context.Context, key string, expiry time.Time) (func(), // temporary authority outage. No new identity, renewal, background work // or retirement receipt is allowed without current durable authority. expires, known := g.permits[key] - if g.sealed || key == "" || !known || !expires.After(g.now()) { + if g.identityLost || errors.Is(err, ErrIdentityUnavailable) || errors.Is(err, errAuthorityStale) || ctx.Err() != nil || g.sealed || key == "" || !known || !expires.After(g.now()) { return nil, err } authorityConfirmed = false @@ -137,7 +213,7 @@ func (g *Gate) Begin(ctx context.Context, key string, expiry time.Time) (func(), // sweeps are bounded, and potentially remote deny lookups belong to the // explicit retirement observation rather than every media segment. if prior, ok := g.permits[key]; ok && !prior.After(g.now()) { - delete(g.permits, key) + g.revokeLocked(key) } g.pruneLocked(ctx, false) _, known := g.permits[key] @@ -145,13 +221,34 @@ func (g *Gate) Begin(ctx context.Context, key string, expiry time.Time) (func(), return nil, ErrFenced } if authorityConfirmed && key != "" && expiry.After(g.now()) && expiry.After(g.permits[key]) { - g.permits[key] = expiry + g.retainLocked(key, expiry) } g.active++ var once sync.Once return func() { once.Do(func() { g.mu.Lock(); g.active--; g.mu.Unlock() }) }, nil } +// BeginPreparation admits new one-shot preparation, never a known-ID retry +// after fencing. Successful publication transfers its reservation to a retained +// artifact permit before request release. Failed preparation retains nothing. +func (g *Gate) BeginPreparation(ctx context.Context, key string, expiry time.Time) (func(bool), error) { + end, err := g.Begin(ctx, "", time.Time{}) + if err != nil { + return nil, err + } + var once sync.Once + return func(published bool) { + once.Do(func() { + if g != nil && published { + g.mu.Lock() + g.retainLocked(key, expiry) + g.mu.Unlock() + } + end() + }) + }, nil +} + // Observe samples actual worker execution plus request and permit reservations. // A zero receipt seals admission under the same mutex, so a delayed dispatch // cannot turn that receipt into a false zero. Process restart keeps the DB fence @@ -165,7 +262,9 @@ func (g *Gate) Observe(ctx context.Context) (Status, error) { if err := g.syncLocked(ctx); err != nil { return Status{}, err } - g.pruneLocked(ctx, true) + if !g.pruneLocked(ctx, true) { + return Status{}, errAuthorityStale + } jobs := 0 if g.jobs != nil { jobs = g.jobs() @@ -173,8 +272,9 @@ func (g *Gate) Observe(ctx context.Context) (Status, error) { if jobs < 0 { return Status{}, ErrUnavailable } - if g.fenceID != "" && jobs == 0 && g.active == 0 && len(g.permits) == 0 { + if g.fenceID != "" && jobs == 0 && g.active == 0 && len(g.permits) == 0 && !g.sealed { g.sealed = true + g.authorityEpoch++ } return Status{NodeID: g.nodeID, FenceID: g.fenceID, WorkerInstanceID: g.instanceID, NativeServerID: g.serverID, Fenced: g.fenceID != "", Drained: g.fenceID != "" && g.sealed && jobs == 0 && g.active == 0 && len(g.permits) == 0, ActiveJobs: jobs, ActiveRequests: g.active, ActiveReservations: len(g.permits), ObservedAt: g.now().UTC()}, nil } @@ -186,6 +286,6 @@ func (g *Gate) Revoke(key string) { return } g.mu.Lock() - delete(g.permits, key) + g.revokeLocked(key) g.mu.Unlock() } diff --git a/internal/workerdrain/gate_concurrency_test.go b/internal/workerdrain/gate_concurrency_test.go new file mode 100644 index 0000000000..607050e11c --- /dev/null +++ b/internal/workerdrain/gate_concurrency_test.go @@ -0,0 +1,478 @@ +package workerdrain + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestDrainGateAuthorityLookupDoesNotBlockStateOrOtherLookups(t *testing.T) { + for _, operation := range []string{"release", "admission", "observation"} { + t.Run(operation, func(t *testing.T) { + var blocked atomic.Bool + entered := make(chan struct{}, 4) + release := make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + gate := New(func(ctx context.Context) (int, string, string, error) { + if blocked.Load() { + entered <- struct{}{} + select { + case <-release: + case <-ctx.Done(): + return 0, "", "", ctx.Err() + } + } + return 7, "", "synthetic-realm", nil + }, nil, nil) + expiry := time.Now().Add(time.Hour) + end, err := gate.Begin(t.Context(), "existing", expiry) + if err != nil { + t.Fatal(err) + } + blocked.Store(true) + firstDone := make(chan error, 1) + go func() { + finish, err := gate.Begin(t.Context(), "existing", expiry) + if err == nil { + finish() + } + firstDone <- err + }() + select { + case <-entered: + case <-time.After(3 * time.Second): + t.Fatal("first lookup never entered") + } + secondDone := make(chan error, 1) + defer func() { + unblock() + end() + if err := <-firstDone; err != nil { + t.Errorf("first lookup cleanup: %v", err) + } + if err := <-secondDone; err != nil { + t.Errorf("second lookup cleanup: %v", err) + } + if proof, err := gate.Observe(t.Context()); err != nil || proof.ActiveRequests != 0 { + t.Errorf("request cleanup: %+v %v", proof, err) + } + }() + switch operation { + case "release": + go func() { end(); secondDone <- nil }() + select { + case err := <-secondDone: + secondDone <- err + case <-time.After(3 * time.Second): + t.Fatal("authority I/O blocked request release") + } + case "admission": + go func() { + finish, err := gate.Begin(t.Context(), "another", expiry) + if err == nil { + finish() + } + secondDone <- err + }() + select { + case <-entered: + case <-time.After(3 * time.Second): + t.Fatal("authority I/O serialized independent admission") + } + case "observation": + go func() { _, err := gate.Observe(t.Context()); secondDone <- err }() + select { + case <-entered: + case <-time.After(3 * time.Second): + t.Fatal("authority I/O blocked fresh drain observation") + } + } + }) + } +} + +func TestDrainGateLateAuthoritySnapshotCannotReopenSealedAdmission(t *testing.T) { + var fence atomic.Value + fence.Store("") + var delay atomic.Bool + entered := make(chan struct{}) + release := make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + gate := New(func(ctx context.Context) (int, string, string, error) { + current := fence.Load().(string) + if delay.CompareAndSwap(true, false) { + close(entered) + select { + case <-release: + case <-ctx.Done(): + return 0, "", "", ctx.Err() + } + } + return 7, current, "synthetic-realm", nil + }, nil, nil) + gate.PublicIdentity(t.Context()) + delay.Store(true) + lateDone := make(chan error, 1) + go func() { + finish, err := gate.Begin(t.Context(), "late", time.Now().Add(time.Hour)) + if err == nil { + finish() + } + lateDone <- err + }() + defer func() { unblock(); <-lateDone }() + select { + case <-entered: + case <-time.After(3 * time.Second): + t.Fatal("old unfenced source snapshot not captured") + } + fence.Store("durable-fence") + proofDone := make(chan struct{}) + var proof Status + var proofErr error + go func() { proof, proofErr = gate.Observe(t.Context()); close(proofDone) }() + select { + case <-proofDone: + case <-time.After(3 * time.Second): + unblock() + <-proofDone + t.Fatal("old snapshot blocked sealed observation") + } + if proofErr != nil || !proof.Drained { + t.Fatalf("fresh zero proof: %+v %v", proof, proofErr) + } + unblock() + err := <-lateDone + lateDone <- err + if !errors.Is(err, ErrFenced) { + t.Fatalf("stale unfenced snapshot admitted after sealed proof: %v", err) + } + after, err := gate.Observe(t.Context()) + if err != nil || !after.Drained || after.ActiveRequests != 0 || after.ActiveReservations != 0 { + t.Fatalf("sealed receipt invalidated: %+v %v", after, err) + } +} + +func TestDrainGatePositiveIdentityLossRefusesExistingUnsealedPermit(t *testing.T) { + for _, variant := range []string{"different node", "different realm", "missing node", "missing realm"} { + t.Run(variant, func(t *testing.T) { + id, realm := 7, "synthetic-realm" + gate := New(func(context.Context) (int, string, string, error) { return id, "", realm, nil }, nil, nil) + end, err := gate.Begin(t.Context(), "existing", time.Now().Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + end() + switch variant { + case "different node": + id = 8 + case "different realm": + realm = "foreign-realm" + case "missing node": + id = 0 + case "missing realm": + realm = "" + } + finish, err := gate.Begin(t.Context(), "existing", time.Now().Add(time.Hour)) + if finish != nil { + finish() + } + if !errors.Is(err, ErrUnavailable) { + t.Fatalf("positive identity loss allowed existing playback: %v", err) + } + }) + } +} + +func TestDrainGateRepeatedIdentityLossInvalidatesDelayedRestoration(t *testing.T) { + var foreign atomic.Bool + var delay atomic.Bool + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + gate := New(func(context.Context) (int, string, string, error) { + realm := "synthetic-realm" + if foreign.Load() { + realm = "foreign-realm" + } + if delay.CompareAndSwap(true, false) { + close(entered) + <-release + } + return 7, "", realm, nil + }, nil, nil) + end, err := gate.Begin(t.Context(), "existing", time.Now().Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + end() + foreign.Store(true) + if _, err := gate.Observe(t.Context()); !errors.Is(err, ErrIdentityUnavailable) { + t.Fatal(err) + } + foreign.Store(false) + delay.Store(true) + done := make(chan error, 1) + go func() { + finish, err := gate.Begin(t.Context(), "existing", time.Now().Add(time.Hour)) + if finish != nil { + finish() + } + done <- err + }() + defer func() { unblock(); <-done }() + select { + case <-entered: + case <-time.After(3 * time.Second): + t.Fatal("restoration did not enter source") + } + foreign.Store(true) + if _, err := gate.Observe(t.Context()); !errors.Is(err, ErrIdentityUnavailable) { + t.Fatal(err) + } + unblock() + err = <-done + done <- err + if !errors.Is(err, ErrIdentityUnavailable) { + t.Fatalf("stale restoration reopened foreign identity: %v", err) + } +} + +func TestDrainGateStaleAuthorityErrorCannotUseOutageContinuity(t *testing.T) { + var foreign atomic.Bool + var delay atomic.Bool + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + gate := New(func(context.Context) (int, string, string, error) { + if delay.CompareAndSwap(true, false) { + close(entered) + <-release + return 0, "", "", ErrUnavailable + } + if foreign.Load() { + return 7, "", "foreign-realm", nil + } + return 7, "", "synthetic-realm", nil + }, nil, nil) + end, err := gate.Begin(t.Context(), "existing", time.Now().Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + end() + delay.Store(true) + done := make(chan error, 1) + go func() { + finish, err := gate.Begin(t.Context(), "existing", time.Now().Add(time.Hour)) + if finish != nil { + finish() + } + done <- err + }() + defer func() { unblock(); <-done }() + select { + case <-entered: + case <-time.After(3 * time.Second): + t.Fatal("old error did not enter source") + } + foreign.Store(true) + if _, err := gate.Observe(t.Context()); !errors.Is(err, ErrIdentityUnavailable) { + t.Fatal(err) + } + unblock() + err = <-done + done <- err + if !errors.Is(err, ErrIdentityUnavailable) { + t.Fatalf("old outage admitted after identity loss: %v", err) + } +} + +func TestDrainGateRepeatedStaleAuthorityRemainsClosed(t *testing.T) { + gate := New(nil, nil, nil) + gate.source = func(context.Context) (int, string, string, error) { + // A newer positive authority observation wins each time this read runs. + gate.mu.Lock() + gate.authorityEpoch++ + gate.mu.Unlock() + return 7, "", "synthetic-realm", nil + } + gate.mu.Lock() + gate.retainLocked("existing", time.Now().Add(time.Hour)) + gate.mu.Unlock() + finish, err := gate.Begin(t.Context(), "existing", time.Now().Add(time.Hour)) + if finish != nil { + finish() + } + if !errors.Is(err, errAuthorityStale) { + t.Fatalf("stale reads used bounded permit fallback: %v", err) + } +} + +func TestDrainGateRevocationLookupDoesNotBlockOrRevokeNewGeneration(t *testing.T) { + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + var deny atomic.Bool + gate := New(func(context.Context) (int, string, string, error) { return 7, "", "synthetic-realm", nil }, nil, func(context.Context, string) bool { + if deny.CompareAndSwap(true, false) { + close(entered) + <-release + return true + } + return false + }) + expiry := time.Now().Add(time.Hour) + end, err := gate.Begin(t.Context(), "existing", expiry) + if err != nil { + t.Fatal(err) + } + deny.Store(true) + done := make(chan error, 1) + var status Status + go func() { var err error; status, err = gate.Observe(t.Context()); done <- err }() + defer func() { unblock(); end(); <-done }() + select { + case <-entered: + case <-time.After(3 * time.Second): + t.Fatal("deny lookup not entered") + } + released := make(chan error, 1) + go func() { + end() + gate.Revoke("existing") + renewed, err := gate.Begin(t.Context(), "existing", expiry) + if renewed != nil { + renewed() + } + released <- err + }() + select { + case err := <-released: + if err != nil { + t.Fatal(err) + } + case <-time.After(3 * time.Second): + unblock() + <-released + t.Fatal("remote deny lookup blocked release/revoke/admission") + } + unblock() + err = <-done + done <- err + if err != nil || status.ActiveRequests != 0 || status.ActiveReservations != 1 { + t.Fatalf("old positive deny revoked new generation: %+v %v", status, err) + } +} + +func TestDrainGatePreparationPublishesBeforeReservationRelease(t *testing.T) { + var fence atomic.Value + fence.Store("") + gate := New(func(context.Context) (int, string, string, error) { + return 7, fence.Load().(string), "synthetic-realm", nil + }, nil, nil) + finish, err := gate.BeginPreparation(t.Context(), "artifact:prepared", time.Now().Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + fence.Store("durable-fence") + before, err := gate.Observe(t.Context()) + if err != nil || before.Drained || before.ActiveRequests != 1 { + t.Fatalf("preparation reservation: %+v %v", before, err) + } + finish(true) + after, err := gate.Observe(t.Context()) + if err != nil || after.Drained || after.ActiveRequests != 0 || after.ActiveReservations != 1 { + t.Fatalf("publication handoff: %+v %v", after, err) + } + if _, err := gate.BeginPreparation(t.Context(), "artifact:prepared", time.Now().Add(time.Hour)); !errors.Is(err, ErrFenced) { + t.Fatalf("known preparation passed fence: %v", err) + } + gate.Revoke("artifact:prepared") + after, err = gate.Observe(t.Context()) + if err != nil || !after.Drained { + t.Fatalf("published artifact cleanup: %+v %v", after, err) + } +} + +func TestDrainGateColdRegistrationBindsOnlyFreshAuthority(t *testing.T) { + var registered atomic.Bool + gate := New(func(context.Context) (int, string, string, error) { + if !registered.Load() { + return 0, "", "", ErrIdentityUnavailable + } + return 7, "", "synthetic-realm", nil + }, nil, nil) + if identity := gate.PublicIdentity(t.Context()); identity.NativeServerID != "" || identity.WorkerInstanceID != "" { + t.Fatalf("cold unregistered identity: %+v", identity) + } + if _, err := gate.Begin(t.Context(), "early", time.Now().Add(time.Hour)); !errors.Is(err, ErrIdentityUnavailable) { + t.Fatalf("cold worker admitted before registration: %v", err) + } + registered.Store(true) + end, err := gate.Begin(t.Context(), "first", time.Now().Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + end() + if identity := gate.PublicIdentity(t.Context()); identity.NativeServerID != "synthetic-realm" || identity.WorkerInstanceID == "" { + t.Fatalf("first fresh registration not bound: %+v", identity) + } + after, err := gate.Observe(t.Context()) + if err != nil || after.NodeID != 7 || after.ActiveRequests != 0 || after.ActiveReservations != 1 { + t.Fatalf("registered membership: %+v %v", after, err) + } +} + +func TestDrainGateLateFencedSnapshotCannotUndoExplicitCancellation(t *testing.T) { + for _, variant := range []string{"fenced snapshot", "source error"} { + t.Run(variant, func(t *testing.T) { + var fence atomic.Value + fence.Store("durable-fence") + var delay atomic.Bool + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + gate := New(func(context.Context) (int, string, string, error) { + current := fence.Load().(string) + if delay.CompareAndSwap(true, false) { + close(entered) + <-release + if variant == "source error" { + return 0, "", "", ErrUnavailable + } + } + return 7, current, "synthetic-realm", nil + }, nil, nil) + if proof, err := gate.Observe(t.Context()); err != nil || !proof.Drained { + t.Fatalf("initial sealed fence: %+v %v", proof, err) + } + delay.Store(true) + done := make(chan error, 1) + var late Status + go func() { var err error; late, err = gate.Observe(t.Context()); done <- err }() + defer func() { unblock(); <-done }() + select { + case <-entered: + case <-time.After(3 * time.Second): + t.Fatal("old fenced observation did not enter source") + } + fence.Store("") + finish, err := gate.Begin(t.Context(), "after-cancellation", time.Now().Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + finish() + unblock() + err = <-done + done <- err + if err != nil || late.Fenced || late.Drained || late.ActiveRequests != 0 || late.ActiveReservations != 1 { + t.Fatalf("old observation undid cancellation: %+v %v", late, err) + } + }) + } +} diff --git a/scripts/ci/db-contracts.txt b/scripts/ci/db-contracts.txt index e48d94ee6d..7508a10c12 100644 --- a/scripts/ci/db-contracts.txt +++ b/scripts/ci/db-contracts.txt @@ -117,6 +117,7 @@ ./internal/nodepool TestAdminNodeDrainConcurrentMigrationRollbackPreservesFence ./internal/nodepool TestAdminNodeDrainMigrationRollbackPreservesConfigurationRevision ./internal/apiv2 TestAdminNodeDrainPostgresProductionPath +./internal/apiv2 TestWorkerDrainKnownPermitMissingAuthorityPostgresProductionPath ./internal/apiv2 TestAdminNodeDisabledCreationPostgresProductionPath ./internal/apiv2 TestAdminNodeDisabledPublicationFaultsPostgresProductionPath ./internal/apiv2 TestAdminNodeLateDisabledCreationPostgresProductionPath From af169fbb753cc4f7b6371f212a25eb9aba7d883f Mon Sep 17 00:00:00 2001 From: Silo Server Developer Date: Thu, 8 Oct 2026 09:34:56 +0200 Subject: [PATCH 3/3] fix(nodes): bound permit expiry during worker retirement --- docs/architecture/worker-http-protocol.md | 6 +- docs/transcode-node-api.md | 8 +- internal/transcodenode/drain.go | 5 +- .../drain_artifact_expiry_test.go | 242 ++++++++++++++++++ internal/transcodenode/server.go | 2 +- internal/workerdrain/gate.go | 33 ++- internal/workerdrain/gate_test.go | 50 ++++ 7 files changed, 337 insertions(+), 9 deletions(-) create mode 100644 internal/transcodenode/drain_artifact_expiry_test.go diff --git a/docs/architecture/worker-http-protocol.md b/docs/architecture/worker-http-protocol.md index 2ec9cfe810..6c2df1020b 100644 --- a/docs/architecture/worker-http-protocol.md +++ b/docs/architecture/worker-http-protocol.md @@ -165,8 +165,10 @@ Worker retirement separately refuses new preparation and known-ID reuse after a durable fence. Preparation admitted before the fence may finish; successful publication or reuse retains an artifact permit while its lifecycle lock remains held. Authenticated GET and HEAD of that admitted artifact may continue until -successful exact deletion or bounded permit expiry. Unknown artifacts and files -left by a preceding worker process cannot acquire a permit after fencing. +successful exact deletion or bounded permit expiry. Delivery does not renew an +existing artifact permit; successful preparation/reuse before fencing owns that +renewal. Fenced media requests cannot extend their last admitted permit expiry. +Unknown artifacts and files left by a preceding worker process cannot acquire a permit after fencing. Preparation follows request cancellation, but a lost response or cancellation does not establish that no bytes or receipt were published. This command is classified non-retryable; it supplies no durable cross-node admission or replay diff --git a/docs/transcode-node-api.md b/docs/transcode-node-api.md index 90bb4a54d5..29a789d44c 100644 --- a/docs/transcode-node-api.md +++ b/docs/transcode-node-api.md @@ -82,8 +82,9 @@ drops. Request reservations cover in-flight execution and egress; bounded session permits cover gaps between media requests. Permits remain until signed expiry, a positive session-deny marker, confirmed transport teardown, or successful deletion of the exact prepared artifact. Header-authenticated media and internal transport permits -use the maximum token lifetime, currently 24 hours. Absence from Redis and quiet -bandwidth are never proof that a permit ended. +use the maximum token lifetime, currently 24 hours. Once fenced, requests keep +the last admitted expiry and cannot renew it, even when they carry a later TTL. +Absence from Redis and quiet bandwidth are never proof that a permit ended. `native_server_id` is the read-only `server.identity_id` in the worker's shared database, matching native server discovery. The worker never initializes a @@ -119,6 +120,9 @@ Prepared-download admission uses an artifact namespace separate from transport identities. A successful preparation or reuse retains the artifact permit before its request reservation and artifact lifecycle lock are released. An already admitted artifact's authenticated GET and HEAD may continue during retirement. +Delivery never extends an existing artifact permit, including before fencing. +Successful unfenced preparation or reuse may establish a later bounded expiry; +a preparation admitted before fencing may finish that original reservation. Preparation, including a known artifact ID, refuses after fencing; a restarted worker cannot adopt a permit merely because the file exists. A definitive missing file removes its permit, and successful exact deletion revokes only that artifact. diff --git a/internal/transcodenode/drain.go b/internal/transcodenode/drain.go index 9fa4d3faa0..3539bdb953 100644 --- a/internal/transcodenode/drain.go +++ b/internal/transcodenode/drain.go @@ -6,7 +6,6 @@ import ( "io" "net/http" "strings" - "time" "github.com/Silo-Server/silo-server/internal/downloadprepare" "github.com/Silo-Server/silo-server/internal/mediasample" @@ -54,12 +53,14 @@ func (s *Server) drainMiddleware(next http.Handler) http.Handler { return } key := "" + begin := s.drain.Begin parts := strings.Split(strings.Trim(path, "/"), "/") if len(parts) >= 2 && (parts[0] == drainTranscodeSegment || parts[0] == "remux") && parts[1] != drainStartSegment { key = transportDrainKey(parts[1]) } if len(parts) == 3 && parts[0] == "downloads" && parts[1] == "artifacts" && downloadprepare.ValidArtifactID(parts[2]) && (r.Method == http.MethodGet || r.Method == http.MethodHead) { key = artifactDrainKey(parts[2]) + begin = s.drain.BeginDelivery } if path == transcodeStartPath { body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 64<<10)) @@ -75,7 +76,7 @@ func (s *Server) drainMiddleware(next http.Handler) http.Handler { key = transportDrainKey(input.SessionID) } } - end, err := s.drain.Begin(r.Context(), key, time.Now().Add(playback.MaxTokenTTL)) + end, err := begin(r.Context(), key, s.drain.ExpiryAfter(playback.MaxTokenTTL)) if err != nil { http.Error(w, "worker retiring or drain authority unavailable", http.StatusServiceUnavailable) return diff --git a/internal/transcodenode/drain_artifact_expiry_test.go b/internal/transcodenode/drain_artifact_expiry_test.go new file mode 100644 index 0000000000..5abfac36fc --- /dev/null +++ b/internal/transcodenode/drain_artifact_expiry_test.go @@ -0,0 +1,242 @@ +package transcodenode + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/downloadprepare" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/workerdrain" +) + +type artifactExpiryFixture struct { + server *Server + listener *httptest.Server + preparer downloadprepare.HTTPPreparer + request downloadprepare.Request + initial time.Time + clock atomic.Int64 + fence atomic.Value +} + +func newArtifactExpiryFixture(t *testing.T) *artifactExpiryFixture { + t.Helper() + fixture := &artifactExpiryFixture{server: newTestServer(t), initial: time.Now().UTC(), request: downloadprepare.Request{ArtifactID: "bounded", InputPath: "/media/movie.mkv", TargetCodecVideo: "copy", TargetCodecAudio: "copy", AudioTrackIndex: -1}} + fixture.clock.Store(fixture.initial.UnixNano()) + fixture.fence.Store("") + ffmpeg := filepath.Join(t.TempDir(), "ffmpeg.sh") + if err := os.WriteFile(ffmpeg, []byte("#!/bin/sh\nfor last; do :; done\nprintf artifact > \"$last\"\n"), 0700); err != nil { + t.Fatal(err) + } + fixture.server.watcher.Config().Playback.FFmpegPath = ffmpeg + fixture.server.watcher.Config().Playback.HWAccel = "none" + fixture.server.drain = workerdrain.NewWithClock(func(context.Context) (int, string, string, error) { + return 7, fixture.fence.Load().(string), "synthetic-realm", nil + }, fixture.server.drainActiveJobs, nil, func() time.Time { return time.Unix(0, fixture.clock.Load()).UTC() }) + fixture.listener = httptest.NewServer(fixture.server.Handler()) + t.Cleanup(fixture.listener.Close) + // Cleanup remains independent of the expiry observer, including mutation RED. + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.WithoutCancel(t.Context()), 3*time.Second) + defer cancel() + if err := fixture.preparer.Delete(ctx, fixture.listener.URL, testSecret, fixture.request.ArtifactID); err != nil { + t.Errorf("exact fixture deletion: %v", err) + } + path := filepath.Join(fixture.server.artifactRoot, fixture.request.ArtifactID+".mp4") + for _, candidate := range []string{path, path + ".part"} { + if _, err := os.Stat(candidate); !os.IsNotExist(err) { + t.Errorf("fixture file cleanup: %v", err) + } + } + status, err := fixture.server.drain.Observe(ctx) + if err != nil || status.ActiveRequests != 0 || status.ActiveJobs != 0 || status.ActiveReservations != 0 { + t.Errorf("fixture reservation cleanup: %+v %v", status, err) + } + }) + return fixture +} + +func (f *artifactExpiryFixture) advance(elapsed time.Duration) { + f.clock.Store(f.initial.Add(elapsed).UnixNano()) +} + +func (f *artifactExpiryFixture) open(t *testing.T, method string, expected int) { + t.Helper() + response, err := f.preparer.Open(t.Context(), f.listener.URL, testSecret, f.request.ArtifactID, method, nil) + if err != nil { + t.Fatal(err) + } + body, readErr := io.ReadAll(response.Body) + closeErr := response.Body.Close() + if readErr != nil || closeErr != nil || response.StatusCode != expected { + t.Fatalf("actual %s: status=%d body=%q read=%v close=%v", method, response.StatusCode, body, readErr, closeErr) + } + if method == http.MethodGet && expected == http.StatusOK && string(body) != "artifact" { + t.Fatalf("actual artifact changed: %q", body) + } +} + +func (f *artifactExpiryFixture) proof(t *testing.T) workerdrain.Status { + t.Helper() + request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, f.listener.URL+"/admin/drain", nil) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Authorization", "Bearer "+testSecret) + response, err := f.listener.Client().Do(request) + if err != nil { + t.Fatal(err) + } + defer func() { _ = response.Body.Close() }() + var status workerdrain.Status + if response.StatusCode != http.StatusOK { + t.Fatalf("actual drain status: %d", response.StatusCode) + } + if err := json.NewDecoder(response.Body).Decode(&status); err != nil { + t.Fatal(err) + } + return status +} + +func TestTranscodeDrainArtifactPollingCannotExtendBoundedPermit(t *testing.T) { + for _, admission := range []string{"prepared artifact", "initial delivery", "cached reuse"} { + t.Run(admission, func(t *testing.T) { + fixture := newArtifactExpiryFixture(t) + path := filepath.Join(fixture.server.artifactRoot, fixture.request.ArtifactID+".mp4") + if admission == "initial delivery" { + if err := os.MkdirAll(fixture.server.artifactRoot, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("artifact"), 0600); err != nil { + t.Fatal(err) + } + fixture.open(t, http.MethodHead, http.StatusOK) + } else { + result, err := fixture.preparer.Prepare(t.Context(), fixture.listener.URL, testSecret, fixture.request) + if err != nil || result.FileSize != 8 { + t.Fatalf("actual preparation: %+v %v", result, err) + } + } + original, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + deadline := playback.MaxTokenTTL + // Even before fencing, delivery must not renew existing ownership. + fixture.advance(time.Hour) + fixture.open(t, http.MethodHead, http.StatusOK) + fixture.advance(5 * time.Hour) + fixture.open(t, http.MethodGet, http.StatusOK) + if admission == "cached reuse" { + fixture.advance(12 * time.Hour) + result, err := fixture.preparer.Prepare(t.Context(), fixture.listener.URL, testSecret, fixture.request) + if err != nil || result.FileSize != 8 { + t.Fatalf("actual cached reuse: %+v %v", result, err) + } + deadline += 12 * time.Hour + fixture.advance(13 * time.Hour) + fixture.open(t, http.MethodHead, http.StatusOK) + fixture.open(t, http.MethodGet, http.StatusOK) + } + fixture.fence.Store("durable-fence") + for _, elapsed := range []time.Duration{deadline - time.Hour, deadline - time.Second} { + fixture.advance(elapsed) + fixture.open(t, http.MethodHead, http.StatusOK) + fixture.open(t, http.MethodGet, http.StatusOK) + status := fixture.proof(t) + if !status.Fenced || status.Drained || status.ActiveReservations != 1 || status.ActiveRequests != 0 || status.ActiveJobs != 0 { + t.Fatalf("live bounded artifact: %+v", status) + } + } + fixture.advance(deadline) + // Ask the actual HEAD route first: it must not renew/resurrect at expiry. + fixture.open(t, http.MethodHead, http.StatusServiceUnavailable) + fixture.open(t, http.MethodGet, http.StatusServiceUnavailable) + status := fixture.proof(t) + if !status.Drained || status.ActiveReservations != 0 || status.ActiveRequests != 0 || status.ActiveJobs != 0 { + t.Fatalf("polling defeated bounded retirement: %+v", status) + } + data, err := os.ReadFile(path) + after, statErr := os.Stat(path) + if err != nil || statErr != nil || string(data) != "artifact" || !after.ModTime().Equal(original.ModTime()) { + t.Fatalf("delivery/expiry mutated artifact: bytes=%q read=%v stat=%v", data, err, statErr) + } + if err := fixture.preparer.Delete(t.Context(), fixture.listener.URL, testSecret, fixture.request.ArtifactID); err != nil { + t.Fatal(err) + } + for _, candidate := range []string{path, path + ".part"} { + if _, err := os.Stat(candidate); !os.IsNotExist(err) { + t.Fatalf("exact artifact cleanup: %v", err) + } + } + if status := fixture.proof(t); !status.Drained || status.ActiveReservations != 0 || status.ActiveJobs != 0 || status.ActiveRequests != 0 { + t.Fatalf("post-delete cleanup: %+v", status) + } + }) + } +} + +func TestTranscodeDrainPreparationKeepsAdmissionDeadlineAcrossFence(t *testing.T) { + fixture := newArtifactExpiryFixture(t) + payload, err := json.Marshal(fixture.request) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequestWithContext(t.Context(), http.MethodPost, downloadPreparePath, bytes.NewReader(payload)) + request.Header.Set("Authorization", "Bearer "+testSecret) + entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + writer := &blockedArtifactResponse{ResponseRecorder: httptest.NewRecorder(), entered: entered, release: release} + handler := fixture.server.Handler() + go func() { defer close(done); handler.ServeHTTP(writer, request) }() + t.Cleanup(func() { + unblock() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Error("preparation handler did not finish during cleanup") + } + }) + select { + case <-entered: + case <-time.After(3 * time.Second): + t.Fatal("actual preparation did not reach publication") + } + fixture.advance(12 * time.Hour) + fixture.fence.Store("durable-fence") + status := fixture.proof(t) + if !status.Fenced || status.Drained || status.ActiveRequests != 1 || status.ActiveJobs < 1 || status.ActiveReservations != 0 { + t.Fatalf("pre-fence publication lost its active reservation: %+v", status) + } + unblock() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("actual preparation did not finish publication") + } + var result downloadprepare.Result + if err := json.Unmarshal(writer.Body.Bytes(), &result); err != nil || writer.Code != http.StatusOK || result.FileSize != 8 { + t.Fatalf("actual preparation result: status=%d result=%+v decode=%v", writer.Code, result, err) + } + fixture.advance(playback.MaxTokenTTL - time.Second) + fixture.open(t, http.MethodHead, http.StatusOK) + fixture.open(t, http.MethodGet, http.StatusOK) + fixture.advance(playback.MaxTokenTTL) + fixture.open(t, http.MethodHead, http.StatusServiceUnavailable) + fixture.open(t, http.MethodGet, http.StatusServiceUnavailable) + status = fixture.proof(t) + if !status.Drained || status.ActiveRequests != 0 || status.ActiveJobs != 0 || status.ActiveReservations != 0 { + t.Fatalf("preparation retained a completion-time deadline: %+v", status) + } +} diff --git a/internal/transcodenode/server.go b/internal/transcodenode/server.go index 1319f2a746..df5281c401 100644 --- a/internal/transcodenode/server.go +++ b/internal/transcodenode/server.go @@ -967,7 +967,7 @@ func (s *Server) handleDownloadPrepare(w http.ResponseWriter, r *http.Request) { if !s.requireApprovedInputPath(w, r, req.InputPath) { return } - finishPreparation, err := s.drain.BeginPreparation(r.Context(), artifactDrainKey(req.ArtifactID), time.Now().Add(playback.MaxTokenTTL)) + finishPreparation, err := s.drain.BeginPreparation(r.Context(), artifactDrainKey(req.ArtifactID), s.drain.ExpiryAfter(playback.MaxTokenTTL)) if err != nil { http.Error(w, "worker retiring or drain authority unavailable", http.StatusServiceUnavailable) return diff --git a/internal/workerdrain/gate.go b/internal/workerdrain/gate.go index 47cae29105..ce48ab62fe 100644 --- a/internal/workerdrain/gate.go +++ b/internal/workerdrain/gate.go @@ -65,7 +65,25 @@ type Gate struct { } func New(source Source, jobs func() int, revoked func(context.Context, string) bool) *Gate { - return &Gate{source: source, jobs: jobs, revoked: revoked, now: time.Now, instanceID: uuid.NewString(), permits: make(map[string]time.Time), permitVersions: make(map[string]uint64)} + return NewWithClock(source, jobs, revoked, time.Now) +} + +// NewWithClock uses an immutable accounting clock. The clock must be safe for +// concurrent callers; nil selects the production wall/monotonic clock. +func NewWithClock(source Source, jobs func() int, revoked func(context.Context, string) bool, clock func() time.Time) *Gate { + if clock == nil { + clock = time.Now + } + return &Gate{source: source, jobs: jobs, revoked: revoked, now: clock, instanceID: uuid.NewString(), permits: make(map[string]time.Time), permitVersions: make(map[string]uint64)} +} + +// ExpiryAfter uses the same clock as admission and expiry accounting. A nil +// gate retains legacy callers' ordinary time-based deadlines. +func (g *Gate) ExpiryAfter(lifetime time.Duration) time.Time { + if g == nil { + return time.Now().Add(lifetime) + } + return g.now().Add(lifetime) } // syncLocked enters and returns with mu held, releasing it around source I/O. @@ -191,8 +209,19 @@ func (g *Gate) pruneLocked(ctx context.Context, force bool) bool { // Begin reserves a request before any execution or egress can start. Existing // identities retain their bounded media permit while retiring; an unknown // identity cannot start, reconstruct, or resume after the worker sees a fence. +// Only unfenced admission may renew an existing transport permit. // Empty identities are one-shot background/extraction work and never retained. func (g *Gate) Begin(ctx context.Context, key string, expiry time.Time) (func(), error) { + return g.begin(ctx, key, expiry, true) +} + +// BeginDelivery admits a present artifact without renewing an existing permit. +// Successful preparation/reuse owns renewal; delivery keeps that bounded term. +func (g *Gate) BeginDelivery(ctx context.Context, key string, expiry time.Time) (func(), error) { + return g.begin(ctx, key, expiry, false) +} + +func (g *Gate) begin(ctx context.Context, key string, expiry time.Time, renewExisting bool) (func(), error) { if g == nil { return func() {}, nil } @@ -220,7 +249,7 @@ func (g *Gate) Begin(ctx context.Context, key string, expiry time.Time) (func(), if g.fenceID != "" && (g.sealed || key == "" || !known) { return nil, ErrFenced } - if authorityConfirmed && key != "" && expiry.After(g.now()) && expiry.After(g.permits[key]) { + if authorityConfirmed && g.fenceID == "" && key != "" && (!known || renewExisting) && expiry.After(g.now()) && expiry.After(g.permits[key]) { g.retainLocked(key, expiry) } g.active++ diff --git a/internal/workerdrain/gate_test.go b/internal/workerdrain/gate_test.go index b4eac7eec9..8ed7ec4850 100644 --- a/internal/workerdrain/gate_test.go +++ b/internal/workerdrain/gate_test.go @@ -175,3 +175,53 @@ func TestDrainGateRejectsMissingAndChangedRealm(t *testing.T) { }) } } + +func TestDrainGateFencedPollingCannotRenewBoundedPermit(t *testing.T) { + var fence atomic.Value + fence.Store("") + initial := time.Now().UTC() + var clock atomic.Int64 + clock.Store(initial.UnixNano()) + gate := NewWithClock(func(context.Context) (int, string, string, error) { + return 7, fence.Load().(string), "synthetic-realm", nil + }, nil, nil, func() time.Time { return time.Unix(0, clock.Load()).UTC() }) + end, err := gate.Begin(t.Context(), "existing", gate.ExpiryAfter(time.Hour)) + if err != nil { + t.Fatal(err) + } + end() + clock.Store(initial.Add(20 * time.Minute).UnixNano()) + end, err = gate.Begin(t.Context(), "existing", gate.ExpiryAfter(time.Hour)) + if err != nil { + t.Fatal(err) + } + end() + fence.Store("durable-fence") + for _, elapsed := range []time.Duration{time.Hour, 79 * time.Minute} { + clock.Store(initial.Add(elapsed).UnixNano()) + end, err = gate.Begin(t.Context(), "existing", gate.ExpiryAfter(time.Hour)) + if err != nil { + t.Fatalf("known request before last unfenced expiry: %v", err) + } + end() + } + clock.Store(initial.Add(80 * time.Minute).UnixNano()) + status, err := gate.Observe(t.Context()) + if err != nil || !status.Drained || status.ActiveJobs != 0 || status.ActiveRequests != 0 || status.ActiveReservations != 0 { + t.Fatalf("fenced polling extended last unfenced permit: %+v %v", status, err) + } + if _, err := gate.Begin(t.Context(), "existing", gate.ExpiryAfter(time.Hour)); !errors.Is(err, ErrFenced) { + t.Fatalf("expired permit resumed after fence: %v", err) + } + fence.Store("") + end, err = gate.Begin(t.Context(), "existing", gate.ExpiryAfter(time.Hour)) + if err != nil { + t.Fatalf("explicit cancellation did not restore bounded admission: %v", err) + } + end() + gate.Revoke("existing") + status, err = gate.Observe(t.Context()) + if err != nil || status.Fenced || status.ActiveReservations != 0 || status.ActiveRequests != 0 { + t.Fatalf("cancellation cleanup: %+v %v", status, err) + } +}