diff --git a/.fallout/build.schema.json b/.fallout/build.schema.json index e3d4d33..86ca2ac 100644 --- a/.fallout/build.schema.json +++ b/.fallout/build.schema.json @@ -29,8 +29,8 @@ "GitHubRelease", "Images", "Push", + "PushDevelop", "PushDockerHub", - "PushEdge", "PushGhcr", "ReleaseBundle", "Test", @@ -115,6 +115,10 @@ "allOf": [ { "properties": { + "DockerHubNamespace": { + "type": "string", + "description": "Docker Hub namespace — the account images are mirrored under" + }, "ImageTag": { "type": "string", "description": "Image tag. Defaults to the git tag on a tag build, otherwise 'dev'" diff --git a/.github/workflows/publish-edge.yml b/.github/workflows/publish-develop.yml similarity index 91% rename from .github/workflows/publish-edge.yml rename to .github/workflows/publish-develop.yml index 5e0a3dc..c445b96 100644 --- a/.github/workflows/publish-edge.yml +++ b/.github/workflows/publish-develop.yml @@ -9,12 +9,12 @@ # # - To trigger manual generation invoke: # -# fallout --generate-configuration GitHubActions_publish-edge --host GitHubActions +# fallout --generate-configuration GitHubActions_publish-develop --host GitHubActions # # # ------------------------------------------------------------------------------ -name: publish-edge +name: publish-develop on: push: @@ -49,8 +49,8 @@ jobs: global-json-file: global.json - name: 'Restore: dotnet tools' run: dotnet tool restore - - name: 'Run: PushEdge' - run: dotnet fallout PushEdge + - name: 'Run: PushDevelop' + run: dotnet fallout PushDevelop env: RegistryUser: ${{ secrets.REGISTRY_USER }} RegistryPassword: ${{ secrets.REGISTRY_PASSWORD }} diff --git a/CLAUDE.md b/CLAUDE.md index 67f82a0..02c8c6c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -190,7 +190,7 @@ dotnet fallout Test # same thing via the tool, which is what CI invokes > Change the attribute, then regenerate — once per workflow: > ```bash > dotnet fallout --generate-configuration GitHubActions_build --host GitHubActions -> # …and publish-edge · publish-ghcr · publish-release · publish-dockerhub +> # …and publish-develop · publish-ghcr · publish-release · publish-dockerhub > ``` ### Branching — GitFlow (2026-08-16) @@ -200,7 +200,7 @@ tagged `v*`. Work goes `feat|fix|chore|docs/*` → PR into `develop`. A release `release/*` window) **fast-forwarded** into `main`, then tagged. `hotfix/*` is cut from `main` and **must** be ported back to `develop`. -- Every non-docs push to `develop` republishes the images as `:edge` (`PushEdge`). +- Every non-docs push to `develop` republishes the images as `:develop` (`PushDevelop`). - `GitHubRelease` **refuses** a tag that is not reachable from `main` or `support/*` — the trunk is never tagged for release. - **Never merge a release PR with GitHub's button.** It rewrites the commits, which severs the diff --git a/Directory.Packages.props b/Directory.Packages.props index f7ecd83..545c34c 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -92,6 +92,8 @@ + + diff --git a/README.md b/README.md index 1fdb48e..aa3c96b 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,10 @@ Every response carries ``, so a client knows whe stop. Item GUIDs are the stable episode id (`{broadcaster}:{native-id}`), unchanged by a re-crawl — which is what stops Sonarr grabbing the same episode twice. +Krautwatch can also create that instance for you on first boot — set `SONARR_URL` / `SONARR_API_KEY` +(and the `RADARR_*` pair) in the environment and they are inserted while no instance is configured. +After that the UI owns them and the variables are ignored, so a restart never undoes an edit. + **As a download client** — add a *SABnzbd* client pointing at the same host. Supported modes: `version`, `get_config`, `addurl`, `addfile`, `queue`, `history`. @@ -236,8 +240,13 @@ error: ZDF API rejected our Api-Auth key (401) — it has most likely been rotat Set Zdf:ApiAuthKey to the current value. ZDF crawling produces nothing until then. ``` -The ZDF agent's `/health` also goes **degraded** (amber in the Aspire dashboard; still HTTP 200, so it -won't restart-loop your container). Recovery is a config change and a restart, not a rebuild: +**It usually fixes itself.** ZDF's own web player publishes the current key, so on a rejection +Krautwatch reads it, checks it against the API, and carries on with it — logging loudly, and holding +`/health` at **degraded** until you make it permanent, because the recovered key lives in memory only +and a restart goes back to what is configured. Turn it off with `Zdf__KeyDiscovery__Enabled=false`. + +To make it permanent — or if discovery is off or fails — it is a config change and a restart, not a +rebuild: ``` Zdf__ApiAuthKey= # environment variable @@ -448,7 +457,7 @@ pinned as a local dotnet tool, so run `dotnet tool restore` once on a fresh clon **The whole CI/CD pipeline is Fallout.** Every workflow under `.github/workflows/` is *generated* from the `[GitHubActions]` attributes in [`build/Build.CI.GitHubActions.cs`](build/Build.CI.GitHubActions.cs), and each one only provisions a runner and invokes a target — the gate runs `Test`, the trunk runs -`PushEdge`, a tag runs `PushGhcr` and `GitHubRelease`. Edit the attribute, not the YAML, or your +`PushDevelop`, a tag runs `PushGhcr` and `GitHubRelease`. Edit the attribute, not the YAML, or your change is overwritten: ```bash @@ -500,8 +509,8 @@ gh pr create --base develop --label enhancement # one category label — th ### Running the trunk -Every push to `develop` republishes the images as `:edge` (multi-arch, GHCR). Point an existing -deployment's `.env` at that tag to follow along — it moves under you, migrations included, and +Every push to `develop` republishes the images as `:develop` (multi-arch, GHCR and Docker Hub). +Point an existing deployment's `.env` at that tag to follow along — it moves under you, migrations included, and downgrading back to a release is not supported, so back up first. --- diff --git a/build/Build.CI.GitHubActions.cs b/build/Build.CI.GitHubActions.cs index 06511e4..69bae1a 100644 --- a/build/Build.CI.GitHubActions.cs +++ b/build/Build.CI.GitHubActions.cs @@ -43,9 +43,10 @@ }, InvokedTargets = new[] { nameof(Test) })] -// ── The edge channel (GitFlow's preview channel) ────────────────────────────── +// ── The develop channel (GitFlow's preview channel) ─────────────────────────── // -// Every push to the trunk republishes the six images under `:edge`, so a tester can run the next +// Every push to the trunk republishes the six images under `:develop` — named after the branch +// they are built from — so a tester can run the next // release before it is a release. Mirrors the extension repo's rolling `preview` VSIX; the shape // differs only because our artefact is a registry tag rather than a GitHub release asset. // @@ -53,16 +54,16 @@ // and rebuilding six multi-arch images because a markdown file changed is pure waste. // // Concurrency QUEUES rather than cancels (ConcurrencyCancelInProgress is left at its default -// false): cancelling a push mid-way can leave `:edge` pointing at a half-written manifest list, -// which is worse than an edge build running a few minutes behind. +// false): cancelling a push mid-way can leave `:develop` pointing at a half-written manifest +// list, which is worse than the channel running a few minutes behind the branch. [GitHubActions( - "publish-edge", + "publish-develop", GitHubActionsImage.UbuntuLatest, FetchDepth = 0, OnPushBranches = new[] { DevelopBranch }, OnPushExcludePaths = new[] { "**/*.md", "docs/**" }, ConcurrencyGroup = "${{ github.workflow }}", - InvokedTargets = new[] { nameof(PushEdge) }, + InvokedTargets = new[] { nameof(PushDevelop) }, EnvironmentName = "ghcr", ImportSecrets = new[] { nameof(RegistryUser), nameof(RegistryPassword) })] @@ -100,7 +101,7 @@ // // It also refuses to go out from the wrong branch: GitHubRelease asserts the tag is reachable from // main or a support line (see Build.Release.cs). Under GitFlow the trunk is never tagged for -// release — it ships through the edge channel instead. +// release — it ships through the develop channel instead. [GitHubActions( "publish-release", GitHubActionsImage.UbuntuLatest, diff --git a/build/Build.Publish.cs b/build/Build.Publish.cs index fa9ce05..c6f514e 100644 --- a/build/Build.Publish.cs +++ b/build/Build.Publish.cs @@ -30,11 +30,28 @@ partial class Build /// Set by the per-registry targets; falls back to the --registry parameter. string _targetRegistry; + /// Set by the per-registry targets; falls back to --registry-namespace. + string _targetNamespace; + string EffectiveRegistry => _targetRegistry ?? Registry; + string EffectiveNamespace => _targetNamespace ?? RegistryNamespace; + [Parameter("Registry namespace — the owner or organisation the images live under")] readonly string RegistryNamespace = "chrison-dev"; + /// + /// The Docker Hub account images are mirrored under. + /// + /// + /// Separate from because the two registries genuinely disagree: the + /// GitHub org is chrison-dev, and no such namespace exists on Docker Hub (its API answers 404) + /// — pushes there have to go to the account, chrison. Sharing one value meant the mirror could + /// never have worked, which went unnoticed because the channel had no credentials to fail with (#115). + /// + [Parameter("Docker Hub namespace — the account images are mirrored under")] + readonly string DockerHubNamespace = "chrison"; + [Parameter("Registry username for the push")] readonly string RegistryUser; @@ -56,7 +73,7 @@ partial class Build AbsolutePath ComposeDirectory => RootDirectory / ".artifacts" / "compose"; - /// Set by ; wins over every other tag resolution. + /// Set by ; wins over every other tag resolution. string _tagOverride; /// @@ -77,7 +94,7 @@ string EffectiveTag { get { - // An edge build runs off a branch, where the resolutions below would answer "dev" — + // A trunk build runs off a branch, where the resolutions below would answer "dev" — // the point of the channel is that it has its own, stable, name. if (!string.IsNullOrWhiteSpace(_tagOverride)) return _tagOverride; @@ -127,7 +144,7 @@ static readonly (string Service, string Project, string Assembly, bool Ffmpeg)[] static string LocalImage(string service) => $"krautwatch-{service}"; string RemoteImage(string service) => - $"{EffectiveRegistry}/{RegistryNamespace}/{LocalImage(service)}"; + $"{EffectiveRegistry}/{EffectiveNamespace}/{LocalImage(service)}"; Target Compose => _ => _ .Description("Generate docker-compose.yaml + .env from the Aspire AppHost") @@ -191,16 +208,16 @@ string RemoteImage(string service) => PushImages(); }); - /// The rolling tag the trunk publishes under. + /// The rolling tag the trunk publishes under — named after the branch it comes from. /// /// Not "latest": that name is conventionally the newest stable image, and a compose file - /// left on the default tag would silently follow the trunk. Anyone running :edge has - /// typed the word. + /// left on the default tag would silently follow the trunk. Anyone running :develop has + /// typed the name of a branch, which says exactly what they are getting. /// - const string EdgeTag = "edge"; + const string DevelopTag = "develop"; - Target PushEdge => _ => _ - .Description($"Push the rolling ':{EdgeTag}' images to GHCR — CI target for pushes to the trunk") + Target PushDevelop => _ => _ + .Description($"Push the rolling ':{DevelopTag}' images to GHCR — CI target for pushes to the trunk") .DependsOn(Compile) .Requires(() => RegistryUser) .Requires(() => RegistryPassword) @@ -208,9 +225,9 @@ string RemoteImage(string service) => { // The images are the whole channel — there is no separate release artefact to attach, // as there would be for a versioned release. A tester points their existing .env at - // :edge (docs/releasing.md#the-edge-channel) and pulls. + // :develop (docs/releasing.md#the-develop-channel) and pulls. _targetRegistry = "ghcr.io"; - _tagOverride = EdgeTag; + _tagOverride = DevelopTag; PushImages(); }); @@ -222,6 +239,7 @@ string RemoteImage(string service) => .Executes(() => { _targetRegistry = "docker.io"; + _targetNamespace = DockerHubNamespace; PushImages(); }); diff --git a/build/Build.Release.cs b/build/Build.Release.cs index 20ab2fc..82ec89d 100644 --- a/build/Build.Release.cs +++ b/build/Build.Release.cs @@ -110,7 +110,7 @@ void AssertTagged() => /// /// /// - /// Under GitFlow the trunk is never tagged for release — develop ships through the edge + /// Under GitFlow the trunk is never tagged for release — develop ships through its own /// channel, and a release comes from main after a stabilisation window /// (docs/branching-and-release.md). Without this check that rule is documentation only, and /// breaking it is silent: a v* tag anywhere publishes real images and a real release @@ -149,7 +149,7 @@ void AssertReleasableRef() Assert.True(releasable, $"{ReleaseTag} is not reachable from origin/{MainBranch} or a support line — it is on " + $"[{string.Join(", ", branches)}]. Releases are cut from {MainBranch}; the trunk ships " + - "through the edge channel. See docs/branching-and-release.md."); + "through the develop channel. See docs/branching-and-release.md."); Log.Information("{Tag} is reachable from {Branches}", ReleaseTag, string.Join(", ", branches)); } diff --git a/docs/branching-and-release.md b/docs/branching-and-release.md index d080fd1..79c7dc0 100644 --- a/docs/branching-and-release.md +++ b/docs/branching-and-release.md @@ -13,7 +13,7 @@ runs. | Branch | Purpose | Lifetime | Tagged? | |---|---|---|---| -| `develop` | **Integration trunk. Default branch.** All finished work lands here first. Every push republishes the [`:edge` images](releasing.md#the-edge-channel). | Permanent | No | +| `develop` | **Integration trunk. Default branch.** All finished work lands here first. Every push republishes the [`:develop` images](releasing.md#the-develop-channel). | Permanent | No | | `main` | **Production.** Only receives merges from `release/*` and `hotfix/*`, and every one of those is tagged. Never committed to directly. | Permanent | **Yes** | | `release/*` | **Stabilisation window** for a release being prepared. Cut from `develop`; takes only fixes and release prep. Merges to `main` *and back to* `develop`, then deleted. | Short-lived | No (the merge into `main` is) | | `hotfix/*` | **Urgent production fix.** Cut from `main`. Merges to `main` *and* `develop`, then deleted. | Short-lived | No (the merge into `main` is) | @@ -89,7 +89,7 @@ flowchart TD A["A change needs to ship"] --> B{"Is a released version
broken right now?"} B -->|No| C["feat/* or fix/*
off develop"] C --> D["PR → develop"] - D --> E["Ships on the next release
(and on :edge immediately)"] + D --> E["Ships on the next release
(and on :develop immediately)"] B -->|Yes| F{"Does it affect the
version main is on?"} F -->|Yes| G["hotfix/* off main"] G --> H["PR → main, tag,
then merge back to develop"] diff --git a/docs/ci.md b/docs/ci.md index 224d975..23e4d99 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -18,7 +18,7 @@ it. | File | Trigger | Invokes | Environment | |---|---|---|---| | `build.yml` | push to `develop`/`main`; PR into `develop`, `main`, `release/*`, `hotfix/*`, `support/*` | `Test` — the required check | — | -| `publish-edge.yml` | push to `develop` (non-docs) | `PushEdge` — rolling `:edge` images | `ghcr` | +| `publish-develop.yml` | push to `develop` (non-docs) | `PushDevelop` — rolling `:develop` images | `ghcr` | | `publish-ghcr.yml` | `v*` tag | `PushGhcr` — versioned multi-arch images | `ghcr` | | `publish-release.yml` | `v*` tag | `GitHubRelease` — compose bundle + notes | `github-release` | | `publish-dockerhub.yml` | dispatch only | `PushDockerHub` — mirror | `dockerhub` | @@ -29,8 +29,8 @@ flowchart LR B --> C(["ubuntu-latest ✓"]) D["Push to develop"] --> B - D --> E["publish-edge.yml
PushEdge"] - E --> ET["ghcr.io/…:edge"] + D --> E["publish-develop.yml
PushDevelop"] + E --> ET["ghcr.io/…:develop"] T["Tag v* on main"] --> G["publish-ghcr.yml
PushGhcr"] T --> R["publish-release.yml
GitHubRelease"] @@ -63,7 +63,7 @@ The generator names a job after its runner image, so **all five workflows produc In practice it only shows up in one place. A PR from a working branch has a head SHA that only the gate ran on, so its check list is clean. A **release PR from `develop`** has a head SHA that -`publish-edge` also ran on, so the PR lists the gate's check *and* the edge publish's, both under +`publish-develop` also ran on, so the PR lists the gate's check *and* the edge publish's, both under the same name — and GitHub requires every check with that name to pass. That is a defensible thing to be blocked by (don't cut a release from a commit whose images @@ -82,23 +82,24 @@ complements forever. The extension repo pays that price because its gate packages a VSIX. Ours is a five-minute `dotnet test`. We pay the five minutes. -`publish-edge.yml` *does* filter paths, and safely: it is not a required check, so a skipped run +`publish-develop.yml` *does* filter paths, and safely: it is not a required check, so a skipped run blocks nothing. -## publish-edge.yml — the trunk channel +## publish-develop.yml — the trunk channel Every non-docs push to `develop` rebuilds the six service images for `linux/amd64` and -`linux/arm64` and pushes them to GHCR under `:edge`, replacing what was there. See -[releasing.md](releasing.md#the-edge-channel) for how to run it. +`linux/arm64` and pushes them to GHCR under `:develop`, replacing what was there. See +[releasing.md](releasing.md#the-develop-channel) for how to run it. Two deliberate choices worth knowing when reading the attribute: - **Concurrency queues, never cancels.** `ConcurrencyCancelInProgress` stays at its default - `false`. A cancelled push can leave `:edge` pointing at a half-written manifest list, which is - worse than an edge image running a few minutes behind the trunk. -- **`edge`, not `latest`.** `latest` is conventionally the newest *stable* image, and it is what a - compose file falls back to when a tag is omitted — so naming the trunk channel `latest` would - silently upgrade people who never asked for it. Anyone on `:edge` typed the word. + `false`. A cancelled push can leave `:develop` pointing at a half-written manifest list, which is + worse than the channel running a few minutes behind the branch. +- **`develop`, not `latest`.** `latest` is conventionally the newest *stable* image, and it is what + a compose file falls back to when a tag is omitted — so naming the trunk channel `latest` would + silently upgrade people who never asked for it. Anyone on `:develop` typed the name of a branch, + which says exactly what they are getting. ## The tag pipeline @@ -155,8 +156,8 @@ leaking into the build definition. Adding a registry is one more attribute plus an environment holding `REGISTRY_USER` and `REGISTRY_PASSWORD`. No target changes. -Neither `ghcr` nor `github-release` currently has an approval rule, so a tag ships without a human -in the loop. If that ever stops being the right trade, add required reviewers to the environment — +None of `ghcr`, `github-release` or `dockerhub` currently has an approval rule, so a tag ships without +a human in the loop. If that ever stops being the right trade, add required reviewers to the environment — not a condition in the build. ## Regenerating @@ -165,7 +166,7 @@ After changing any `[GitHubActions]` attribute: ```bash ./build.sh --generate-configuration GitHubActions_build --host GitHubActions -./build.sh --generate-configuration GitHubActions_publish-edge --host GitHubActions +./build.sh --generate-configuration GitHubActions_publish-develop --host GitHubActions ./build.sh --generate-configuration GitHubActions_publish-ghcr --host GitHubActions ./build.sh --generate-configuration GitHubActions_publish-release --host GitHubActions ./build.sh --generate-configuration GitHubActions_publish-dockerhub --host GitHubActions diff --git a/docs/releasing.md b/docs/releasing.md index 5b0b38c..c8f4bd7 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -8,7 +8,7 @@ Channels, versioning, and the runbook for every kind of release. The branch mode ```mermaid flowchart LR - DEV["develop"] -->|every push| EDGE["ghcr.io/…:edge
rolling"] + DEV["develop"] -->|every push| EDGE["…:develop
rolling"] MAIN["main / support/*"] -->|"v* tag"| GHCR["ghcr.io/…:0.3.0"] MAIN -->|"v* tag"| REL["GitHub release
compose + env template"] GHCR -.->|"manual dispatch"| DH["docker.io mirror"] @@ -21,7 +21,7 @@ flowchart LR | Channel | Trigger | Gating | |---|---|---| -| `:edge` images | every non-docs push to `develop` | none | +| `:develop` images | every non-docs push to `develop` | none | | versioned images + GitHub release | any `v*` tag reachable from `main`/`support/*` | the [release guard](ci.md#the-release-guard) | | Docker Hub mirror | manual dispatch | manual by definition | @@ -36,24 +36,25 @@ same string with the `v` stripped (`v0.3.0` → `:0.3.0`). Pre-1.0, so: There is deliberately **no Nerdbank.GitVersioning here**, unlike the Fallout repos. They need a computed monotonic version because every preview build publishes a numbered package to a registry -that will not accept the same number twice. Our preview channel is a single rolling `:edge` tag with +that will not accept the same number twice. Our preview channel is a single rolling `:develop` tag with no number in it, so the machinery would buy nothing and cost a `version.json` to keep honest. -## The edge channel +## The develop channel -Every non-docs push to `develop` republishes all six images under `:edge`. That is the whole +Every non-docs push to `develop` republishes all six images under `:develop`, named after the branch +they are built from. That is the whole channel: no release object, no version, just the trunk in runnable form. -To follow it, point an existing deployment's image tags at `edge`: +To follow it, point an existing deployment's image tags at `develop`: ```bash -sed -i '' 's/:[0-9]\+\.[0-9]\+\.[0-9]\+$/:edge/' .env # in a release bundle's .env +sed -i '' 's/:[0-9]\+\.[0-9]\+\.[0-9]\+$/:develop/' .env # in a release bundle's .env docker compose pull && docker compose up -d ``` -`:edge` moves under you — that is the point, and it is why the tag is not called `latest`. Expect a +`:develop` moves under you — that is the point, and it is why the tag is not called `latest`. Expect a schema migration to land there before it lands in a release; the Migrator runs to completion on -every start, so an edge deployment upgrades itself, and **downgrading back to a release is not +every start, so a develop deployment upgrades itself, and **downgrading back to a release is not supported**. Take a database backup before following the trunk with data you care about. ## Cutting a release @@ -195,8 +196,12 @@ radius of a bad release for no benefit. gh workflow run publish-dockerhub.yml -f ImageTag=0.3.0 ``` -It needs a `dockerhub` environment holding `REGISTRY_USER` and `REGISTRY_PASSWORD`; there isn't one -yet, so the run fails fast on the missing parameter until it's created. +It reads `REGISTRY_USER` and `REGISTRY_PASSWORD` from the `dockerhub` environment (a Docker Hub access +token, not the account password). + +Note the namespace differs by registry: GHCR publishes under the GitHub org `chrison-dev`, Docker Hub +under the account `chrison` — there is no `chrison-dev` on Docker Hub. `--docker-hub-namespace` +overrides it. ## If a publish fails partway diff --git a/docs/self-hosting.md b/docs/self-hosting.md index 8ae7590..216e114 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -135,6 +135,20 @@ Once an administrator exists, `/setup` never reopens. Point both at the **`newznab`** service on port `5055` — not the web UI. +> **Skip the clicking.** Put your instance in `.env` before the first `docker compose up` and it is +> created for you: +> +> ```dotenv +> SONARR_URL=http://sonarr:8989 +> SONARR_API_KEY= +> RADARR_URL= # optional, same shape +> RADARR_API_KEY= +> ``` +> +> This runs **only while no instance is configured**. Once one exists — created this way or in the UI — +> the UI owns them: editing a key or a URL is never undone by a restart, and these variables are +> ignored. Both halves of a pair are required; a URL without a key is treated as a typo and skipped. + **As an indexer** (Prowlarr, or Sonarr → Settings → Indexers → *Newznab*): | Field | Value | @@ -410,8 +424,13 @@ error: ZDF API rejected our Api-Auth key (401) — it has most likely been rotat and its `/health` reports **degraded** (still HTTP 200 — a restart cannot fix a rotated key, so it deliberately does not fail the container's health probe). ARD and KiKA are unaffected. -Fix it without waiting for a release: set `Zdf__ApiAuthKey` to the current value — the one ZDF's own -web player sends — on the ZDF agent and the Newznab API, then restart both. +In most cases crawling then resumes on its own: Krautwatch reads the current key off ZDF's player, +verifies it against the API, and uses it. Health stays **degraded** while it is running on a +discovered key, because that key is not persisted — a restart goes back to your configuration. + +Make it permanent by setting `Zdf__ApiAuthKey` to the value in the log, on the ZDF agent and the +Newznab API, then restarting both. To disable the automatic recovery entirely, set +`Zdf__KeyDiscovery__Enabled=false`. ### A settings row shows an API-key problem diff --git a/src/Application/ApplicationServiceExtensions.cs b/src/Application/ApplicationServiceExtensions.cs index 2e987e4..7f5ffcb 100644 --- a/src/Application/ApplicationServiceExtensions.cs +++ b/src/Application/ApplicationServiceExtensions.cs @@ -50,6 +50,7 @@ public static IServiceCollection AddApplication(this IServiceCollection services // Instance CRUD only needs IArrInstanceRepository, which every host gets from AddInfrastructure, // so these are safe here. services.AddScoped(); + services.AddScoped(); // first-run env bootstrap (#5) services.AddScoped(); services.AddScoped(); diff --git a/src/Application/Settings/BootstrapArrInstances.cs b/src/Application/Settings/BootstrapArrInstances.cs new file mode 100644 index 0000000..25f9ad9 --- /dev/null +++ b/src/Application/Settings/BootstrapArrInstances.cs @@ -0,0 +1,116 @@ +using Krautwatch.Domain.Entities; +using Krautwatch.Domain.Enums; +using Krautwatch.Domain.Interfaces; +using Microsoft.Extensions.Logging; + +namespace Krautwatch.Application.Settings; + +// ============================================================ +// Message +// ============================================================ + +/// One *arr instance described by the environment. +public sealed record ArrInstanceSeed(ArrKind Kind, string Name, string BaseUrl, string ApiKey); + +/// +/// The environment variables that can describe a first *arr instance (#5), and how to read them. +/// +/// +/// Takes a lookup rather than IConfiguration so the Application layer keeps its dependencies and +/// this stays trivially testable. Flat SCREAMING_CASE names because these are meant to be typed into a +/// compose .env, next to KRAUTWATCH_APIKEY and friends, rather than into appsettings. +/// +public static class ArrBootstrapEnvironment +{ + public const string SonarrUrl = "SONARR_URL"; + public const string SonarrApiKey = "SONARR_API_KEY"; + public const string RadarrUrl = "RADARR_URL"; + public const string RadarrApiKey = "RADARR_API_KEY"; + + public static IReadOnlyList ReadSeeds(Func configuration) + { + var seeds = new List(); + + Add(ArrKind.Sonarr, "Sonarr", SonarrUrl, SonarrApiKey); + Add(ArrKind.Radarr, "Radarr", RadarrUrl, RadarrApiKey); + + return seeds; + + void Add(ArrKind kind, string name, string urlKey, string keyKey) + { + var url = configuration(urlKey)?.Trim(); + var apiKey = configuration(keyKey)?.Trim(); + + // Half a pair is a typo, not a configuration: an instance with no key cannot be contacted, + // and one with no URL has nothing to contact. Skipped rather than half-created. + if (string.IsNullOrWhiteSpace(url) || string.IsNullOrWhiteSpace(apiKey)) return; + + seeds.Add(new ArrInstanceSeed(kind, name, url, apiKey)); + } + } +} + +// ============================================================ +// Command +// ============================================================ + +/// +/// Creates the first *arr instances from the environment, so a compose deployment can arrive +/// already wired up instead of requiring a trip through Settings (#5). +/// +/// +/// +/// It runs only while no instance exists at all. That is what keeps the UI the source of truth +/// after first boot: once anything is configured, this never touches it again — no overwriting an +/// edited API key, no second copy when someone corrects the URL, no re-applying on every restart. +/// +/// +/// The consequence to know: adding these variables to an instance that already has one configured does +/// nothing, and removing every instance in the UI makes the next restart look like a first boot again. +/// Both follow from "first instance" being the actual feature — a convenience for arriving at a +/// working deployment, not a second, competing configuration surface. +/// +/// +public class BootstrapArrInstancesHandler( + IArrInstanceRepository instances, + ILogger logger) +{ + /// How many instances were created. + public async Task HandleAsync( + IReadOnlyList seeds, + CancellationToken ct = default) + { + if (seeds.Count == 0) return 0; + + var existing = await instances.GetAllAsync(ct); + if (existing.Count > 0) + { + logger.LogDebug( + "{Count} *arr instance(s) already configured — leaving them alone and ignoring the " + + "environment.", existing.Count); + return 0; + } + + var created = 0; + + foreach (var seed in seeds) + { + await instances.AddAsync(new ArrInstance + { + Name = seed.Name, + Kind = seed.Kind, + BaseUrl = seed.BaseUrl, + ApiKey = seed.ApiKey, + Enabled = true, + }, ct); + + created++; + + // The key may be a secret reference (env:/file:), so it is never logged — only its source. + logger.LogInformation("Created {Kind} instance '{Name}' at {BaseUrl} from the environment.", + seed.Kind, seed.Name, seed.BaseUrl); + } + + return created; + } +} diff --git a/src/Infrastructure/Crawling/Zdf/ZdfAuthHealthCheck.cs b/src/Infrastructure/Crawling/Zdf/ZdfAuthHealthCheck.cs index f56d29c..b0170ab 100644 --- a/src/Infrastructure/Crawling/Zdf/ZdfAuthHealthCheck.cs +++ b/src/Infrastructure/Crawling/Zdf/ZdfAuthHealthCheck.cs @@ -31,7 +31,16 @@ public Task CheckHealthAsync( var snapshot = state.Snapshot(); if (!snapshot.IsRejected) - return Task.FromResult(HealthCheckResult.Healthy("ZDF is accepting our Api-Auth key.")); + { + // Working, but not on the configured key — so it still needs saying. A restart goes back to + // whatever is in configuration, which is the value that was just rejected. + return Task.FromResult(snapshot.UsingDiscoveredKey + ? HealthCheckResult.Degraded( + "ZDF rotated its Api-Auth key; we recovered by reading the current one from ZDF's " + + $"player and are running on that. Set {ZdfOptions.SectionName}:" + + $"{nameof(ZdfOptions.ApiAuthKey)} to it — the recovered key is not persisted.") + : HealthCheckResult.Healthy("ZDF is accepting our Api-Auth key.")); + } var since = snapshot.FirstRejectionAt is { } first ? _time.GetUtcNow() - first diff --git a/src/Infrastructure/Crawling/Zdf/ZdfAuthState.cs b/src/Infrastructure/Crawling/Zdf/ZdfAuthState.cs index be58510..ef33e48 100644 --- a/src/Infrastructure/Crawling/Zdf/ZdfAuthState.cs +++ b/src/Infrastructure/Crawling/Zdf/ZdfAuthState.cs @@ -36,6 +36,32 @@ public sealed class ZdfAuthState private int _consecutiveRejections; private DateTimeOffset? _firstRejectionAt; private HttpStatusCode? _lastStatusCode; + private string? _discoveredKey; + + /// + /// A key read off ZDF's player and verified against the API, in force for this process (#112). + /// + /// + /// Deliberately not persisted. It is a recovery measure, not a second place to configure the key — + /// the operator is told to put the value in Zdf:ApiAuthKey, and a restart should go back to + /// asking what they configured rather than quietly running on something we scraped weeks ago. + /// + public string? DiscoveredKey + { + get { lock (_gate) return _discoveredKey; } + } + + /// Adopts a key that has already been verified against the API. + public void AdoptDiscoveredKey(string key) + { + lock (_gate) + { + _discoveredKey = key; + _consecutiveRejections = 0; + _firstRejectionAt = null; + _lastStatusCode = null; + } + } /// A request was answered — the key is good. public void RecordSuccess() @@ -64,7 +90,8 @@ public ZdfAuthSnapshot Snapshot() { lock (_gate) { - return new ZdfAuthSnapshot(_consecutiveRejections, _firstRejectionAt, _lastStatusCode); + return new ZdfAuthSnapshot( + _consecutiveRejections, _firstRejectionAt, _lastStatusCode, _discoveredKey is not null); } } } @@ -72,10 +99,15 @@ public ZdfAuthSnapshot Snapshot() /// Rejections since the last success. Zero means healthy. /// When the current run of rejections began. /// The status the API last rejected us with. +/// +/// True when the configured key was rejected and a key read off ZDF's player is carrying us instead — +/// working, but the operator still needs to update configuration. +/// public readonly record struct ZdfAuthSnapshot( int ConsecutiveRejections, DateTimeOffset? FirstRejectionAt, - HttpStatusCode? LastStatusCode) + HttpStatusCode? LastStatusCode, + bool UsingDiscoveredKey = false) { public bool IsRejected => ConsecutiveRejections > 0; } diff --git a/src/Infrastructure/Crawling/Zdf/ZdfCatalogClient.cs b/src/Infrastructure/Crawling/Zdf/ZdfCatalogClient.cs index 3ddbf90..964e0df 100644 --- a/src/Infrastructure/Crawling/Zdf/ZdfCatalogClient.cs +++ b/src/Infrastructure/Crawling/Zdf/ZdfCatalogClient.cs @@ -24,7 +24,8 @@ public sealed class ZdfCatalogClient( HttpClient http, ZdfOptions? options = null, ZdfAuthState? authState = null, - ILogger? logger = null) + ILogger? logger = null, + ZdfKeyDiscovery? keyDiscovery = null) { public const string ApiBase = "https://api.zdf.de"; private const string PlayerId = "android_native_6"; @@ -214,10 +215,16 @@ void Walk(JsonElement e) /// private async Task GetJsonAsync(string url, CancellationToken ct) { + var triedDiscovery = false; + for (var attempt = 1; ; attempt++) { using var req = new HttpRequestMessage(HttpMethod.Get, url); - req.Headers.TryAddWithoutValidation("Api-Auth", $"Bearer {_options.ApiAuthKey}"); + + // A key recovered from the player outranks the configured one for this process — it is the + // one that currently works, and the operator has been told to make it permanent. + req.Headers.TryAddWithoutValidation( + "Api-Auth", $"Bearer {_authState.DiscoveredKey ?? _options.ApiAuthKey}"); using var resp = await http.SendAsync(req, ct); if (resp.IsSuccessStatusCode) @@ -228,6 +235,20 @@ void Walk(JsonElement e) if (resp.StatusCode is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden) { + // Before giving up: ZDF's own player knows the current key, and a rejection is exactly + // the moment to go and read it (#112). Only a key that has already answered 200 comes + // back from here, so this either fixes the crawl or changes nothing. + if (keyDiscovery is not null && !triedDiscovery) + { + triedDiscovery = true; + + if (await keyDiscovery.TryDiscoverAsync(ct) is { } recovered) + { + _authState.AdoptDiscoveredKey(recovered); + continue; // same attempt number: recovery is not a retry of a failure + } + } + _authState.RecordRejection(resp.StatusCode, DateTimeOffset.UtcNow); _logger.LogError( diff --git a/src/Infrastructure/Crawling/Zdf/ZdfKeyDiscovery.cs b/src/Infrastructure/Crawling/Zdf/ZdfKeyDiscovery.cs new file mode 100644 index 0000000..4c95608 --- /dev/null +++ b/src/Infrastructure/Crawling/Zdf/ZdfKeyDiscovery.cs @@ -0,0 +1,141 @@ +using System.Net.Http.Headers; +using System.Text.RegularExpressions; +using Microsoft.Extensions.Logging; + +namespace Krautwatch.Infrastructure.Crawling.Zdf; + +/// +/// Recovers a rotated ZDF Api-Auth key by reading the one ZDF's own web player uses (#112). +/// +/// +/// +/// The player's page embeds it as "apiAuthToken":"…" in its server-rendered data. That is a +/// stable place — a documented-by-observation property name at a fixed URL — rather than the +/// hash-named JS chunk the issue feared, which is what makes this worth doing at all. +/// +/// +/// A candidate is never adopted until it has answered 200. A scrape of an undocumented page +/// fails by returning something plausible, and sending a garbage bearer would turn "your key is +/// stale" into an unexplained failure inside the recovery path. Verified first, adopted second; a +/// failed discovery leaves exactly the behaviour of a deployment without it. +/// +/// +public sealed partial class ZdfKeyDiscovery( + HttpClient http, + ZdfOptions options, + TimeProvider? timeProvider = null, + ILogger? logger = null) +{ + private readonly TimeProvider _time = timeProvider ?? TimeProvider.System; + private readonly ILogger _logger = logger ?? Microsoft.Extensions.Logging.Abstractions + .NullLogger.Instance; + + // One flight at a time, and not too often: the source page is several megabytes, and a fleet whose + // key has just rotated would otherwise pull it once per failed request. + private readonly SemaphoreSlim _gate = new(1, 1); + private DateTimeOffset _lastAttempt = DateTimeOffset.MinValue; + + /// The page publishes two tokens; only this property name is the one the API expects. + [GeneratedRegex(@"\\?""apiAuthToken\\?""\s*:\s*\\?""(?[A-Za-z0-9._-]{16,128})\\?""")] + private static partial Regex TokenPattern(); + + /// A key that has just authenticated, or null when discovery did not produce one. + public async Task TryDiscoverAsync(CancellationToken ct = default) + { + if (!options.KeyDiscovery.Enabled) return null; + + if (!await _gate.WaitAsync(TimeSpan.Zero, ct)) + { + // Another request is already doing this. Its result lands in ZdfAuthState either way. + return null; + } + + try + { + var now = _time.GetUtcNow(); + if (now - _lastAttempt < options.KeyDiscovery.MinimumInterval) return null; + _lastAttempt = now; + + var candidate = await ReadKeyFromPlayerAsync(ct); + if (candidate is null) return null; + + if (candidate == options.ApiAuthKey) + { + // The player is using the key we already have, so the rejection is about something else + // — saying so beats reporting a "recovery" that changes nothing. + _logger.LogWarning( + "ZDF's player is using the same Api-Auth key we already send, so the rejection is " + + "not a rotation. Something else is refusing us."); + return null; + } + + if (!await VerifyAsync(candidate, ct)) + { + _logger.LogWarning("Read a candidate Api-Auth key from ZDF's player, but it was rejected " + + "too — not adopting it."); + return null; + } + + _logger.LogWarning( + "Recovered from a rotated ZDF Api-Auth key by reading the current one from {Source}. " + + "It is in use for this process only — set {Section}:{Setting} to make it permanent.", + options.KeyDiscovery.SourceUrl, ZdfOptions.SectionName, nameof(ZdfOptions.ApiAuthKey)); + + return candidate; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, "Could not read the current Api-Auth key from ZDF's player."); + return null; + } + finally + { + _gate.Release(); + } + } + + private async Task ReadKeyFromPlayerAsync(CancellationToken ct) + { + using var request = new HttpRequestMessage(HttpMethod.Get, options.KeyDiscovery.SourceUrl); + + // Asking as a browser, because that is what the page is for; a default .NET agent is the kind of + // thing a CDN answers differently. + request.Headers.UserAgent.ParseAdd( + "Mozilla/5.0 (compatible; Krautwatch/1.0; +https://github.com/Chrison-dev/Krautwatch)"); + + using var response = await http.SendAsync(request, ct); + if (!response.IsSuccessStatusCode) + { + _logger.LogWarning("{Source} answered {Status} — cannot read the current key.", + options.KeyDiscovery.SourceUrl, (int)response.StatusCode); + return null; + } + + var html = await response.Content.ReadAsStringAsync(ct); + var match = TokenPattern().Match(html); + + if (!match.Success) + { + // The shape changed. Worth a distinct line: this is the failure that means the scrape needs + // revisiting, as opposed to the API simply being down. + _logger.LogWarning("No apiAuthToken found in {Source}. The page's shape has probably changed.", + options.KeyDiscovery.SourceUrl); + return null; + } + + return match.Groups["key"].Value; + } + + /// Proves a candidate against the API before anything starts using it. + private async Task VerifyAsync(string candidate, CancellationToken ct) + { + using var request = new HttpRequestMessage( + HttpMethod.Get, $"{ZdfCatalogClient.ApiBase}/search/documents?q=heute&hasVideo=true&page=1"); + + request.Headers.TryAddWithoutValidation("Api-Auth", $"Bearer {candidate}"); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + + using var response = await http.SendAsync(request, ct); + return response.IsSuccessStatusCode; + } +} diff --git a/src/Infrastructure/Crawling/Zdf/ZdfOptions.cs b/src/Infrastructure/Crawling/Zdf/ZdfOptions.cs index cb79e4e..109807e 100644 --- a/src/Infrastructure/Crawling/Zdf/ZdfOptions.cs +++ b/src/Infrastructure/Crawling/Zdf/ZdfOptions.cs @@ -19,4 +19,38 @@ public sealed class ZdfOptions /// Zdf__ApiAuthKey (or Zdf:ApiAuthKey) when ZDF rotates it. /// public string ApiAuthKey { get; set; } = DefaultApiAuthKey; + + /// Recovering a rotated key without human intervention (#112). + public ZdfKeyDiscoveryOptions KeyDiscovery { get; set; } = new(); +} + +/// +/// Reading the current Api-Auth key off ZDF's own web player when ours is rejected (#112). +/// +public sealed class ZdfKeyDiscoveryOptions +{ + /// + /// On by default. + /// + /// + /// The issue that proposed this expected it to be opt-in, on the assumption that the key would be + /// buried in a hash-named JS chunk and the scrape would rot faster than the key rotates. It is not: + /// the page publishes it as apiAuthToken in its own embedded data, at a stable URL. Combined + /// with never adopting a key that has not just answered 200, a failed discovery lands exactly where + /// today's behaviour already lands — a loud error and a degraded health check — so there is nothing + /// left for opting in to protect against. + /// + public bool Enabled { get; set; } = true; + + /// The page carrying the token. Only the homepage does; the sub-pages tested did not. + public string SourceUrl { get; set; } = "https://www.zdf.de/"; + + /// + /// How long to wait before trying discovery again after an attempt. + /// + /// + /// The page is ~5 MB. Without this, every crawl in a fleet hitting a rotated key at once would each + /// pull it — and re-pull it on every subsequent failure. + /// + public TimeSpan MinimumInterval { get; set; } = TimeSpan.FromMinutes(15); } diff --git a/src/Infrastructure/InfrastructureServiceExtensions.cs b/src/Infrastructure/InfrastructureServiceExtensions.cs index 13c3f55..04d7798 100644 --- a/src/Infrastructure/InfrastructureServiceExtensions.cs +++ b/src/Infrastructure/InfrastructureServiceExtensions.cs @@ -153,6 +153,9 @@ public static IServiceCollection AddZdfCrawler( services.AddSingleton(options); services.AddSingleton(); services.AddHttpClient(); + // Its own client: the source page is several megabytes and unrelated to the API, so it should + // not share timeouts or handlers with the crawl path (#112). + services.AddHttpClient(http => http.Timeout = TimeSpan.FromSeconds(30)); services.AddScoped(sp => new ZdfBroadcasterCrawler(sp.GetRequiredService())); diff --git a/src/Presentation/AppHost/Program.cs b/src/Presentation/AppHost/Program.cs index b1217e1..cff5d2a 100644 --- a/src/Presentation/AppHost/Program.cs +++ b/src/Presentation/AppHost/Program.cs @@ -72,6 +72,14 @@ // Optional: TheTVDB matching. Absent is fine — matching degrades to titles rather than failing. var tvdbApiKey = builder.AddParameter("tvdb-apikey", secret: true); +// Optional: a first Sonarr/Radarr instance, so a compose deployment can arrive already wired up +// instead of requiring a trip through Settings (#5). Applied by the Web host only while no instance +// exists — after that the UI owns them, and these are ignored. +var sonarrUrl = builder.AddParameter("sonarr-url", secret: false); +var sonarrApiKey = builder.AddParameter("sonarr-apikey", secret: true); +var radarrUrl = builder.AddParameter("radarr-url", secret: false); +var radarrApiKey = builder.AddParameter("radarr-apikey", secret: true); + // Newznab + SABnzbd — the public *arr-facing surface (indexer + download client). builder.AddProject("newznab") .WithHttpEndpoint(port: 5055, targetPort: 8080, name: "http") @@ -90,6 +98,10 @@ .WaitFor(db) .WaitForCompletion(migrator) .WithEnvironment("TvdbConfiguration__ApiKey", tvdbApiKey) + .WithEnvironment("SONARR_URL", sonarrUrl) // ArrBootstrapEnvironment in the Web host + .WithEnvironment("SONARR_API_KEY", sonarrApiKey) + .WithEnvironment("RADARR_URL", radarrUrl) + .WithEnvironment("RADARR_API_KEY", radarrApiKey) .WithExternalHttpEndpoints(); // ────────────────────────────────────────────────────────────── diff --git a/src/Presentation/Web/Program.cs b/src/Presentation/Web/Program.cs index 0675775..851753f 100644 --- a/src/Presentation/Web/Program.cs +++ b/src/Presentation/Web/Program.cs @@ -148,10 +148,42 @@ // First-run: print the gated setup link. The token lives in memory for this process only, so it rotates // on restart, and whoever can read the log (the operator) is the only one able to claim the instance. if (authProvider is "local") - await LogSetupLinkIfRequiredAsync(app); + await BootstrapArrInstancesAsync(app); +await LogSetupLinkIfRequiredAsync(app); app.Run(); +/// +/// Creates the first Sonarr/Radarr instances from SONARR_URL / SONARR_API_KEY / RADARR_* (#5), so a +/// compose deployment can arrive already wired up. +/// +/// +/// Here rather than in an agent because this host owns setup and the UI that supersedes it, and +/// because exactly one process must do it. It runs only while no instance exists — see +/// for why that is the whole guard. +/// +static async Task BootstrapArrInstancesAsync(WebApplication app) +{ + var logger = app.Services.GetRequiredService().CreateLogger("Krautwatch.Setup"); + + var seeds = ArrBootstrapEnvironment.ReadSeeds(key => app.Configuration[key]); + if (seeds.Count == 0) return; + + try + { + using var scope = app.Services.CreateScope(); + var bootstrap = scope.ServiceProvider.GetRequiredService(); + + await bootstrap.HandleAsync(seeds); + } + catch (Exception ex) + { + // Never block startup on this — on a cold deploy the schema may not exist yet, and a UI that + // starts is what lets an operator fix whatever is wrong. + logger.LogWarning(ex, "Could not apply the *arr instances described by the environment."); + } +} + static async Task LogSetupLinkIfRequiredAsync(WebApplication app) { var logger = app.Services.GetRequiredService().CreateLogger("Krautwatch.Setup"); diff --git a/tests/Application.Tests/BootstrapArrInstancesTests.cs b/tests/Application.Tests/BootstrapArrInstancesTests.cs new file mode 100644 index 0000000..b791638 --- /dev/null +++ b/tests/Application.Tests/BootstrapArrInstancesTests.cs @@ -0,0 +1,136 @@ +using Krautwatch.Application.Settings; +using Krautwatch.Domain.Entities; +using Krautwatch.Domain.Enums; +using Krautwatch.Domain.Interfaces; +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; +using Shouldly; +using Xunit; + +namespace Krautwatch.Application.Tests; + +/// +/// Creating the first Sonarr/Radarr instance from the environment (#5). The interesting behaviour is +/// all in what it refuses to do on the second boot: the UI is the source of truth once anything is +/// configured. +/// +public class BootstrapArrInstancesTests +{ + private readonly IArrInstanceRepository _instances = Substitute.For(); + + private BootstrapArrInstancesHandler Sut => + new(_instances, NullLogger.Instance); + + [Fact] + public async Task Both_instances_are_created_on_an_empty_deployment() + { + GivenNoInstances(); + + var created = await Sut.HandleAsync( + [Seed(ArrKind.Sonarr, "http://sonarr:8989"), Seed(ArrKind.Radarr, "http://radarr:7878")], + TestContext.Current.CancellationToken); + + created.ShouldBe(2); + await _instances.Received(2).AddAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task Nothing_is_touched_when_an_instance_already_exists() + { + // The whole guard. An operator who edited the key, renamed the instance, or corrected the URL + // must not have any of it undone — or duplicated — by the next restart. + _instances.GetAllAsync(Arg.Any()) + .Returns([Existing("http://sonarr.local:8989")]); + + var created = await Sut.HandleAsync( + [Seed(ArrKind.Sonarr, "http://sonarr:8989")], TestContext.Current.CancellationToken); + + created.ShouldBe(0); + await _instances.DidNotReceive().AddAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task An_empty_environment_asks_the_database_nothing() + { + var created = await Sut.HandleAsync([], TestContext.Current.CancellationToken); + + created.ShouldBe(0); + await _instances.DidNotReceive().GetAllAsync(Arg.Any()); + } + + [Fact] + public async Task The_created_instance_carries_what_the_environment_said() + { + GivenNoInstances(); + ArrInstance? saved = null; + await _instances.AddAsync(Arg.Do(i => saved = i), Arg.Any()); + + await Sut.HandleAsync([Seed(ArrKind.Sonarr, "http://sonarr:8989")], + TestContext.Current.CancellationToken); + + saved.ShouldNotBeNull(); + saved.Kind.ShouldBe(ArrKind.Sonarr); + saved.BaseUrl.ShouldBe("http://sonarr:8989"); + saved.ApiKey.ShouldBe("key-abc"); + saved.Enabled.ShouldBeTrue(); + } + + // ── reading the environment ─────────────────────────────── + + [Fact] + public void Both_pairs_are_read_when_both_are_set() + { + var seeds = ArrBootstrapEnvironment.ReadSeeds(new Dictionary + { + ["SONARR_URL"] = "http://sonarr:8989", + ["SONARR_API_KEY"] = "s-key", + ["RADARR_URL"] = "http://radarr:7878", + ["RADARR_API_KEY"] = "r-key", + }.GetValueOrDefault); + + seeds.Select(s => s.Kind).ShouldBe([ArrKind.Sonarr, ArrKind.Radarr]); + } + + [Theory] + [InlineData("http://sonarr:8989", null)] + [InlineData(null, "s-key")] + [InlineData("http://sonarr:8989", " ")] + public void Half_a_pair_is_a_typo_and_is_skipped(string? url, string? apiKey) + { + // An instance with no key cannot be contacted and one with no URL has nothing to contact, so a + // half-configured pair is dropped rather than half-created. + var seeds = ArrBootstrapEnvironment.ReadSeeds(new Dictionary + { + ["SONARR_URL"] = url, + ["SONARR_API_KEY"] = apiKey, + }.GetValueOrDefault); + + seeds.ShouldBeEmpty(); + } + + [Fact] + public void Surrounding_whitespace_is_trimmed() + { + // Copy-paste out of a compose file picks up trailing spaces, and a URL with one does not connect. + var seeds = ArrBootstrapEnvironment.ReadSeeds(new Dictionary + { + ["SONARR_URL"] = " http://sonarr:8989 ", + ["SONARR_API_KEY"] = " s-key ", + }.GetValueOrDefault); + + var seed = seeds.ShouldHaveSingleItem(); + seed.BaseUrl.ShouldBe("http://sonarr:8989"); + seed.ApiKey.ShouldBe("s-key"); + } + + private void GivenNoInstances() => + _instances.GetAllAsync(Arg.Any()).Returns([]); + + private static ArrInstanceSeed Seed(ArrKind kind, string url) => + new(kind, kind.ToString(), url, "key-abc"); + + private static ArrInstance Existing(string url) => new() + { + Name = "Sonarr", Kind = ArrKind.Sonarr, BaseUrl = url, ApiKey = "edited-in-the-ui", + }; +} diff --git a/tests/Infrastructure.Tests/Krautwatch.Infrastructure.Tests.csproj b/tests/Infrastructure.Tests/Krautwatch.Infrastructure.Tests.csproj index 0ea2a9a..42c2d68 100644 --- a/tests/Infrastructure.Tests/Krautwatch.Infrastructure.Tests.csproj +++ b/tests/Infrastructure.Tests/Krautwatch.Infrastructure.Tests.csproj @@ -16,6 +16,7 @@ + diff --git a/tests/Infrastructure.Tests/ZdfKeyDiscoveryTests.cs b/tests/Infrastructure.Tests/ZdfKeyDiscoveryTests.cs new file mode 100644 index 0000000..f5dc661 --- /dev/null +++ b/tests/Infrastructure.Tests/ZdfKeyDiscoveryTests.cs @@ -0,0 +1,184 @@ +using System.Net; +using System.Text; +using Krautwatch.Infrastructure.Crawling.Zdf; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using Shouldly; +using Xunit; + +namespace Krautwatch.Infrastructure.Tests; + +/// +/// Recovering from a rotated ZDF key by reading the current one off ZDF's player (#112). The spike +/// that preceded this found the key published as apiAuthToken in the page's own embedded data, +/// which is what makes it worth scraping; these cover the guards around doing so. +/// +public class ZdfKeyDiscoveryTests +{ + private const string Current = "aa3noh4ohz9eeboo8shiesheec9ciequ9Quah7el"; + private const string Rotated = "ahBaeMeekaiy5ohsai4bee4ki6Oopoi5quailieb"; + + [Fact] + public async Task The_key_is_read_from_the_page_and_verified_before_being_returned() + { + var handler = new PlayerStub(pageKey: Rotated, apiAccepts: Rotated); + + var discovered = await Discovery(handler).TryDiscoverAsync(TestContext.Current.CancellationToken); + + discovered.ShouldBe(Rotated); + handler.VerifiedWith.ShouldBe(Rotated); // proved against the API, not just parsed + } + + [Fact] + public async Task A_candidate_the_api_also_rejects_is_not_adopted() + { + // The failure mode that makes an undocumented scrape dangerous: it returns something + // plausible. Adopting it would turn "your key is stale" into an unexplained failure. + var handler = new PlayerStub(pageKey: "not-a-real-key", apiAccepts: Rotated); + + var discovered = await Discovery(handler).TryDiscoverAsync(TestContext.Current.CancellationToken); + + discovered.ShouldBeNull(); + } + + [Fact] + public async Task A_page_whose_shape_changed_yields_nothing_rather_than_nonsense() + { + var handler = new PlayerStub(page: "redesigned", apiAccepts: Rotated); + + var discovered = await Discovery(handler).TryDiscoverAsync(TestContext.Current.CancellationToken); + + discovered.ShouldBeNull(); + } + + [Fact] + public async Task Finding_the_key_we_already_send_is_not_a_recovery() + { + // The player agreeing with us means the rejection is about something else entirely, and + // "recovering" to the same value would just loop. + var handler = new PlayerStub(pageKey: Current, apiAccepts: Current); + + var discovered = await Discovery(handler).TryDiscoverAsync(TestContext.Current.CancellationToken); + + discovered.ShouldBeNull(); + } + + [Fact] + public async Task Discovery_is_rate_limited_so_a_fleet_cannot_stampede_a_five_megabyte_page() + { + var handler = new PlayerStub(pageKey: Rotated, apiAccepts: Rotated); + var time = new FakeTimeProvider(); + var sut = Discovery(handler, time); + + await sut.TryDiscoverAsync(TestContext.Current.CancellationToken); + var second = await sut.TryDiscoverAsync(TestContext.Current.CancellationToken); + + second.ShouldBeNull(); + handler.PageFetches.ShouldBe(1); + + time.Advance(TimeSpan.FromMinutes(16)); + await sut.TryDiscoverAsync(TestContext.Current.CancellationToken); + + handler.PageFetches.ShouldBe(2); + } + + [Fact] + public async Task Nothing_happens_when_it_is_switched_off() + { + var handler = new PlayerStub(pageKey: Rotated, apiAccepts: Rotated); + var options = new ZdfOptions { KeyDiscovery = { Enabled = false } }; + + var discovered = await new ZdfKeyDiscovery(new HttpClient(handler), options) + .TryDiscoverAsync(TestContext.Current.CancellationToken); + + discovered.ShouldBeNull(); + handler.PageFetches.ShouldBe(0); + } + + // ── the client's side of it ─────────────────────────────── + + [Fact] + public async Task A_rejected_crawl_recovers_and_completes_instead_of_throwing() + { + var handler = new PlayerStub(pageKey: Rotated, apiAccepts: Rotated); + var state = new ZdfAuthState(); + var client = new ZdfCatalogClient(new HttpClient(handler), Options(), state, + keyDiscovery: Discovery(handler)); + + // Before #112 this threw ZdfAuthRejectedException and the crawl produced nothing. + var episodes = await client.SearchEpisodesAsync("heute-show", TestContext.Current.CancellationToken); + + episodes.ShouldBeEmpty(); // the stub's payload is empty, but it answered 200 + state.DiscoveredKey.ShouldBe(Rotated); + state.Snapshot().IsRejected.ShouldBeFalse(); + } + + [Fact] + public async Task Health_says_recovered_rather_than_healthy_so_the_operator_still_updates_config() + { + var state = new ZdfAuthState(); + state.AdoptDiscoveredKey(Rotated); + + var check = new ZdfAuthHealthCheck(state); + var result = await check.CheckHealthAsync( + new HealthCheckContext + { + Registration = new HealthCheckRegistration("zdf-auth", check, HealthStatus.Degraded, null), + }, + TestContext.Current.CancellationToken); + + // The recovered key is not persisted, so a restart goes back to the rejected one. + result.Status.ShouldBe(HealthStatus.Degraded); + result.Description.ShouldContain("not persisted"); + } + + private static ZdfOptions Options() => new() { ApiAuthKey = Current }; + + private static ZdfKeyDiscovery Discovery(PlayerStub handler, TimeProvider? time = null) => + new(new HttpClient(handler), Options(), time, NullLogger.Instance); + + /// Serves ZDF's player page and the API, so the whole read-verify-adopt path is exercised. + private sealed class PlayerStub(string? pageKey = null, string? apiAccepts = null, string? page = null) + : HttpMessageHandler + { + public int PageFetches { get; private set; } + public string? VerifiedWith { get; private set; } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var url = request.RequestUri!.ToString(); + + if (url.Contains("zdf.de/", StringComparison.Ordinal) && !url.Contains("api.zdf.de")) + { + PageFetches++; + + // The real page embeds it inside a JS string, so the quotes arrive backslash-escaped — + // and it carries a second, different token under "apiToken" that must not be picked up. + var body = page ?? $$""" + + """; + + return Task.FromResult(Json(body)); + } + + var sent = request.Headers.TryGetValues("Api-Auth", out var values) + ? values.First().Replace("Bearer ", "") + : ""; + + if (url.Contains("q=heute&", StringComparison.Ordinal)) + VerifiedWith = sent; + + return Task.FromResult(sent == apiAccepts + ? Json("""{"http://zdf.de/rels/search/results":[]}""") + : new HttpResponseMessage(HttpStatusCode.Unauthorized)); + } + + private static HttpResponseMessage Json(string body) => new(HttpStatusCode.OK) + { + Content = new StringContent(body, Encoding.UTF8, "application/json"), + }; + } +} diff --git a/tests/Live.Tests/ZdfLiveTests.cs b/tests/Live.Tests/ZdfLiveTests.cs index 9d4f4e7..d90d04d 100644 --- a/tests/Live.Tests/ZdfLiveTests.cs +++ b/tests/Live.Tests/ZdfLiveTests.cs @@ -18,6 +18,29 @@ public class ZdfLiveTests static ZdfLiveTests() => Http.DefaultRequestHeaders.UserAgent.ParseAdd("Krautwatch/1.0 (+https://github.com/Chrison-dev/Krautwatch)"); + [Fact] + public async Task The_current_Api_Auth_key_can_be_read_from_ZDFs_own_player() + { + // #112's premise, checked against the live page rather than assumed: the key is published as + // apiAuthToken in the page's embedded data, and what comes back authenticates. If ZDF + // redesigns, this fails here rather than during someone's outage. + var discovery = new ZdfKeyDiscovery( + Http, + // A key we will never send, so discovery treats the page's value as a rotation and + // verifies it end to end instead of short-circuiting on "same as configured". + new ZdfOptions { ApiAuthKey = "not-the-current-key" }); + + var key = await discovery.TryDiscoverAsync(TestContext.Current.CancellationToken); + + key.ShouldNotBeNullOrWhiteSpace(); + + // Verified inside TryDiscoverAsync, but assert independently that it really is usable. + var client = new ZdfCatalogClient(Http, new ZdfOptions { ApiAuthKey = key! }); + var episodes = await client.SearchEpisodesAsync("heute-show", TestContext.Current.CancellationToken); + + episodes.ShouldNotBeEmpty(); + } + [Fact] public async Task Search_finds_HeuteShow_episodes() {