diff --git a/.github/ISSUE_TEMPLATE/integrations.yml b/.github/ISSUE_TEMPLATE/integrations.yml
new file mode 100644
index 0000000..9adc3df
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/integrations.yml
@@ -0,0 +1,58 @@
+name: Integrations (experimental)
+description: A bug, a rough edge or an idea for the Integrations tab.
+title: "[integrations] "
+labels: ["integrations"]
+body:
+ - type: markdown
+ attributes:
+ value: |
+ Integrations are new and experimental, so every report helps shape them. Thanks for testing.
+
+ - type: input
+ id: version
+ attributes:
+ label: InstantClone version
+ description: Top-right of the dashboard, or the `v0.1.15` in the .exe name.
+ placeholder: "v0.1.15"
+ validations:
+ required: true
+
+ - type: dropdown
+ id: kind
+ attributes:
+ label: What is it?
+ options:
+ - Something doesn't work
+ - Something is confusing
+ - An idea or a missing feature
+ validations:
+ required: true
+
+ - type: input
+ id: integration
+ attributes:
+ label: Which integration
+ description: Its name on the card, like "Smart platform alerts" or one you built.
+ placeholder: "Highlight button"
+
+ - type: textarea
+ id: what-happened
+ attributes:
+ label: What happened, or what you'd like
+ description: In your words. Screenshots help a lot.
+ validations:
+ required: true
+
+ - type: textarea
+ id: run
+ attributes:
+ label: The run log (if something failed)
+ description: Open the integration and look at its last runs. Paste the steps and what each one answered. Check it first for links, keys or tokens and remove them.
+ render: text
+
+ - type: textarea
+ id: recipe
+ attributes:
+ label: The integration as a recipe (optional)
+ description: On the card, use Share and paste it here. Recipes never contain your webhooks, keys or tokens.
+ render: text
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index bb04c69..38a587e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -111,9 +111,10 @@ jobs:
- uses: Swatinem/rust-cache@v2
# native-tls links system OpenSSL on Linux (schannel on Windows), so the
- # headers must be present for the build to link.
+ # headers must be present for the build to link. ffmpeg lets the
+ # reconnect screen's lossless round-trip test run instead of skipping.
- name: install build deps
- run: sudo apt-get update && sudo apt-get install -y pkg-config libssl-dev
+ run: sudo apt-get update && sudo apt-get install -y pkg-config libssl-dev ffmpeg
- name: clippy
run: cargo clippy --all-targets -- -D warnings
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 39d6ce3..fca2c82 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -11,6 +11,12 @@ on:
permissions:
contents: write # needed to create releases
+# The Twitch app id the Integrations tab logs in with (Device Code flow, a
+# public client: the id is not a secret). Baked in at compile time through
+# option_env!; set it as a repository variable named TWITCH_CLIENT_ID.
+env:
+ INSTANTCLONE_TWITCH_CLIENT_ID: ${{ vars.TWITCH_CLIENT_ID }}
+
jobs:
# Gate 1: tests must pass on the exact tagged commit before we ship.
# This guards against the "tag a commit that was never on main" case
@@ -48,8 +54,10 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
save-if: false
+ # ffmpeg: the reconnect screen's decode tests fail on Linux CI without it
+ # (same deps as ci.yml's test-linux).
- name: install build deps
- run: sudo apt-get update && sudo apt-get install -y pkg-config libssl-dev
+ run: sudo apt-get update && sudo apt-get install -y pkg-config libssl-dev ffmpeg
- name: clippy
run: cargo clippy --all-targets -- -D warnings
- name: test
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e4342a2..89272f1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,7 +4,792 @@ All notable changes will land here. Format loosely follows
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), versions follow
[SemVer](https://semver.org/spec/v2.0.0.html).
-## [Unreleased]
+## [0.1.15] - Crash protection, Integrations (experimental), vertical without Twitch, and Enhanced Broadcasting on your GPU
+
+### Crash protection: OBS crashes, the stream stays up
+
+When OBS crashed, froze or lost its connection, every destination went
+offline with it, and viewers were gone by the time OBS was back. Turn on
+**Settings -> System -> Crash protection** and they stay live on a
+reconnect screen instead, for up to the time you pick (30 s to 5 min).
+
+- **A reconnect screen, not a black frame.** Five styles: **Whisper**
+ (pulsing dots), **Beacon** (radar rings), **Orbit** (a spinner),
+ **Studio** (a lower third, lifted clear of TikTok's buttons on vertical
+ canvases) and **Arcade** (pixel art). Pick your own accent, background,
+ headline and subline. Settings shows every style as a card with your
+ colours and text on it, plus an animated preview in 16:9 and 9:16,
+ drawn by the same code that streams it. Every style animates in small
+ steps over a small area, so the loop stays light (under 1 Mbps at
+ 1080p). It plays at the resolution each destination receives, with
+ silent audio, after any buffered delay has aired, and timestamps carry
+ on from the last real frame, so platforms see one unbroken stream.
+- **OBS coming back resumes on the same connection.** No reconnect on the
+ platform side: the stream picks up at OBS's first keyframe. With a delay
+ armed, the screen stays up until the delay has rebuilt, so viewers never
+ see live.
+- **Frozen OBS counts too.** OBS still connected but sending no video for
+ 3 seconds opens the same screen, and its next frame closes it.
+- **Enhanced Broadcasting is covered.** Every track of a Twitch multitrack
+ stream gets its own screen at its own resolution, the VOD audio track
+ gets its own silence, and OBS reconnecting is handed the same Twitch
+ session, so the destination carries on instead of restarting. Twitch
+ VOD-audio sessions are kept the same way.
+- **HEVC and AV1 hold the last frame.** The screen is H.264, so those
+ tracks re-send their last keyframe once a second instead.
+- **Stopping in OBS never opens the screen,** even while it is up for a
+ frozen OBS: the stream ends once your delay has aired. Only a drop
+ without a goodbye opens it. **Protect every disconnect** (under
+ Advanced) covers encoders that fail cleanly, like an NVENC error.
+- **End it early from anywhere.** A banner on the dashboard and a strip in
+ the OBS dock show the countdown with **End now** (click twice), the tray
+ menu has **End crash protection** while it is on air, and it can be
+ bound to a global hotkey or a MIDI pad. `POST /crash-protection/end`
+ does the same for a Stream Deck.
+- **You hear about it.** The destination card reads **Reconnect screen**,
+ the tray status shows the time left, and the "Stream alerts" integration
+ posts on Discord when the screen goes on air, when OBS is back, and when
+ it ends.
+- **Good to know.** After a crash, OBS asks whether to run in Safe Mode:
+ pick **Run in Normal Mode**, or your scripts and plugins (the VOD track
+ script too) stay off. The banner says so while it waits. The hold time
+ counts from the crash, so a delay still airing uses part of it. A delay
+ as long as the hold time or longer airs in full, and then the stream
+ ends.
+
+### Integrations (experimental): InstantClone talks to Discord, Twitch chat, your phone and your tools
+
+A new **Integrations** tab. Each card previews its Discord message, chat
+reply, phone push or VOD marker before you switch it on.
+
+Integrations are **experimental**: they work, but how they look and how
+you set them up (mostly the screens) may still change between versions.
+Please try them and tell us what breaks or confuses you: the tab's
+**Report it** link opens an integrations issue form, and a recipe from
+**Share** shows what you built without any of your keys.
+
+- **A catalog of ready-made integrations**, one click to add: a crash
+ alert on Discord, a `!delay` command that always answers with your
+ current delay, "tell chat" when OBS drops (only after 20 s, so short
+ blips never reach chat), VOD markers at every crash and cut, a phone
+ push, a destination-down alert, mod controls (`!cut`, `!setdelay 30`),
+ a webhook for n8n or Home Assistant, a crash counter an OBS text source
+ can show, and more. Starter packs add a few at once.
+- **An editor that is the preview.** Pick a moment (OBS crashes, OBS is
+ back, the hold runs out), write on the message itself, and click a
+ variable to insert it. Every option (where it posts, when it sends,
+ who can use a command, cooldowns) is a chip that opens its own panel.
+- **Build your own, with no limits.** A step editor: any trigger (an
+ InstantClone event, a chat command or message, a timer, or a secret web
+ address a Stream Deck or script can call), checks with then/otherwise,
+ waits, web requests whose answer later steps can use, delay actions,
+ clips, programs, files and counters. Plus the one only InstantClone can
+ do: **wait until the moment has aired for your viewers**, so a `!clip`
+ captures what they actually saw.
+- **Test runs** send the real messages marked `[TEST]`, skip the waits
+ and never touch the stream, then show every step and what it answered.
+ A **dry run** sends nothing at all and shows what each step would have
+ sent, and you can test with your own values ("a Kick drop", "a mod
+ typing !chapter Intro") instead of the samples. A test of a "Kick only"
+ alert reads Kick, not YouTube.
+- **Chat commands that call any API.** Add **Command from a website** and
+ paste the command a site gives you for Nightbot, StreamElements,
+ Fossabot or Streamlabs: the address, the reply and the command name fill
+ in, their variables translated (`$(user)`, `${1}`, `$(touser)`). Every
+ web request has **Try it**: send it for real, see the answer as a tree
+ and click the value you want in the reply. What viewers type is encoded into
+ its own spot of the request, so `!rank` can't be bent into calling
+ something else. Or start from a ready-made one: `!uptime`,
+ `!accountage`, `!followers` and `!game` work in one click, and the
+ preview shows the real answer right away.
+- **One Discord message per drop.** The crash alert posts "OBS dropped"
+ with a live countdown to the end of the hold, then edits that same
+ message to "back after 38 s" instead of adding another. Any Discord step
+ can "edit the last one".
+- **Start an integration from a hotkey or a MIDI pad**, even in game.
+ The new **Clip button** clips the moment (once it has aired, so the delay
+ never makes it clip the wrong seconds), drops the link in chat and says
+ so on stream, from one key.
+- **Show on stream.** Add InstantClone's alerts page to OBS once as a
+ browser source; "Show on stream" steps put a card on it, queued so two
+ never overlap. Tests never put anything on stream.
+- **Edit text** shapes an answer before it goes out: its first line, the
+ part between two texts, find and replace, round a number, group digits,
+ or pick one line at random (an 8-ball in two steps).
+- **React to every step of the delay**: you arm one, it's ready to go on
+ air, it goes on, changes, goes off, or you cancel it before it went on.
+ The new **Delay status** alert keeps one Discord message in step with
+ all of it, and a delay step can now also **disarm** the delay.
+- **Quiet hours** keep an integration silent overnight, `{uses}` counts
+ how often one ran, and every card shows its last runs as a row of dots.
+- **Everything can be taken back.** Every card has a menu (edit,
+ duplicate, share, delete), adding and deleting come with **Undo**, and
+ views opened from the catalog or the simple editor have a back button.
+- **Nothing private on screen by default.** Your phone topic, webhook
+ links, API headers and secret addresses are hidden until you click
+ Show, and addresses elsewhere only show their site.
+- **Twitch: log in once and forget it.** You approve InstantClone on
+ twitch.tv with a code; no password goes through the app. The login
+ refreshes itself, so as long as you open InstantClone at least once a
+ month you never log in again. An optional bot account can post instead
+ of you. Prefer your own Twitch app? Paste its Client ID in **System >
+ Twitch & OBS > Twitch login app**; the steps to make one are right there.
+- **Share integrations as recipes**: a line of text you can paste in
+ Discord. Recipes never include your connections, keys or secret links,
+ arrive switched off, and anything that runs a program or writes a file
+ needs your explicit yes.
+- **Nothing an integration does can touch the stream.** They run on their
+ own thread; a slow webhook or a busy chat never delays a frame, and a
+ failing integration shows up in the tab with the reason instead of
+ failing quietly.
+- **Discord cards.** Any Discord step can send a card: a colored edge, a
+ title, fields side by side, a footer with the time, images, and a line
+ above it where a clip link unfurls into a player. You write on the card
+ itself in the editor, pick its color from swatches, and add or resize
+ fields in place. The catalog's Discord alerts are cards now.
+- **Smart platform alerts.** Quiet on blips: one card when a platform
+ is really down (still down after 60 s), turning green when it's back;
+ one card when it keeps dropping (3 drops in 10 min), updating its count
+ live and turning green once it's steady again; and a health report
+ when the stream ends. Every threshold is yours to change.
+- **Stream timeline.** One Discord card per stream that writes itself:
+ crashes, delay changes, platform outages, highlights and chapters, each
+ with its place in the VOD (the delay is taken into account). Any
+ integration can add a line with the new **Timeline** step.
+- **Highlight button.** A hotkey, MIDI pad or Stream Deck button clips the
+ moment once it has aired, marks the VOD, adds it to the timeline and
+ posts it to Discord. When Twitch can't clip, the moment is still marked
+ and the stream shows "Marked, but Twitch couldn't clip it". A
+ `!highlight` for mods is one switch away, and answers the mod either way.
+ Presses during a long delay queue up instead of being turned away.
+- **Hype clip.** Chat decides when to clip. The new **Chat gets busy**
+ trigger takes rules you stack (busier than normal, how many people,
+ the words they use), all or any of them, over a window you pick.
+ "Busier than normal" is measured against your own chat, so one setting
+ fits a small and a huge channel. A live meter shows where chat is
+ against each rule while you tune it. It fires once per burst, then waits
+ for chat to calm down, so one hype moment is one clip.
+- **Chapters.** Mods type `!chapter Boss fight` to mark the VOD; when the
+ stream ends you get a YouTube chapter list ready to paste, with
+ YouTube's rules applied (the first at 0:00, each at least 10 s, three or
+ more). An optional "Technical difficulties" chapter for crashes goes
+ back to the chapter before once OBS is back.
+- **On-air light.** A light by your door: red when live, amber with the
+ delay on, flashing if OBS crashed, off at the end. Pick Home Assistant,
+ WLED or Philips Hue and paste its address. The light hears the current
+ state as soon as the app starts or the integration is switched on.
+- **Scene delay.** Your OBS scene runs the delay: on in game, back to live
+ in the lobby. It only acts once a scene has stayed on air (3 s by
+ default), so flicking through scenes does nothing, and never during the
+ reconnect screen. Scenes come straight from OBS's own WebSocket server,
+ whose settings InstantClone reads itself: switch the server on in OBS
+ and pick from your scene list. The "back to live" rule is "any other
+ scene", which never counts the game scene, and starting the stream
+ already in game puts the delay on too. Scene and button triggers can be
+ limited to while you stream.
+- **OBS steps.** Switch to a scene, show or hide a source, or change what
+ a text source says, from any integration. Scenes and sources are picked
+ from OBS's own lists, and chat can never choose them.
+- **A builder for real flows.** Steps move into and out of checks with
+ the arrow buttons, any step can be switched off without deleting it,
+ removing a step can be undone, and new blocks land right after the step
+ you picked. Each step only offers the values it can really use, grouped
+ by where they come from, and a step that misses something says what.
+ Saving keeps the builder open.
+- **Times read like times.** Every wait, cooldown and timer takes "10s",
+ "2m" or "1m 30s".
+- **Buttons work everywhere.** A hotkey or MIDI pad trigger is now a
+ **button**: it also gets its own address a Stream Deck, Bitfocus
+ Companion or any program can press. That works on any system, not just
+ Windows.
+- **New moments to react to:** a stream starts or ends (a crash OBS comes
+ back from is the same stream), the on-air state changes, a destination
+ comes back, stays down, keeps dropping or is steady again, and a line
+ is added to the timeline. New variables: `{uptime}`, `{vod_time}`,
+ `{onair}`, `{timeline}` and `{chapters}`. A Discord message can now also
+ be **finished**: edited one last time, and the next alert starts fresh.
+- **New packs:** Stream pro, Highlights and Studio. Every catalog entry and
+ pack shows everything it needs (Discord, Twitch, OBS…) and what's
+ already connected.
+- **Recipes say what they do before you add them:** who can start each
+ integration (anyone in chat, only mods…) and everything it reaches. A
+ program or file on your PC, or a delay anyone in chat can change, shows
+ in red; the delay and OBS in amber. **See every step** lists each step
+ in plain words, checks and their branches included, and **Raw settings**
+ shows exactly what gets added. A recipe can never choose a program, its
+ arguments, a file, a web address or an OBS scene: you pick those
+ yourself after adding it, and the steps say so. The delay action that
+ turns the delay off now says so ("Turn the delay off"); it used to read
+ as if it only cancelled one still getting ready.
+- **Shared Chat is your choice.** While you share chat with other
+ streamers, their viewers can't use your commands or count toward "chat
+ gets busy" unless you switch on "listen to partner channels" in
+ Connections › Twitch.
+- **Clearer editors.** Fields show what they will say (sample values
+ filled in, links and bold drawn like Discord) and turn back into what
+ you wrote while you edit them. Insert buttons say what they hold ("how
+ long it was down") instead of variable names, options read as plain
+ sentences ("Repeat at most: once every 10 s", "Each time: updates the
+ same message"), a card that misses something says what under its name,
+ and the editor's switch says On or Off (draft).
+
+Fixes in the new integrations, found before release:
+
+- A network blip with crash protection off no longer ends the stream:
+ OBS has a minute to come back before the health report, the chapters
+ and a new timeline go out.
+- Stopping on purpose with "hold on every disconnect" is no longer counted
+ as a crash, and doesn't make the on-air light flash.
+- A platform that keeps reconnecting no longer floods the timeline, and a
+ full timeline drops plain lines before chapters and highlights.
+- A "down" or "keeps dropping" card left open when a stream ended is no
+ longer edited from the next stream, far up the channel.
+- Timeline card updates arrive in order, wait out Discord's rate limit
+ instead of failing, and a highlight that finished after the stream
+ ended no longer adds a `0:00` line.
+- A card with only a title or fields can be switched on; the "Ping" option
+ shows when it can actually ping; each words rule counts its own words;
+ OBS's crash reasons are only offered where they can match, and filters
+ take lists and wildcards (`YouTube, Kick`, `*timed out*`).
+- Shared recipes keep the on-air light's states and give buttons a fresh
+ Stream Deck address, and a switched-off trigger or step no longer stops
+ the rest from being switched on.
+- Recipes can never pick an OBS scene or source, or keep a web request's
+ body by claiming to be a light; a card's link and images can't come
+ from chat; a wait filled in from chat lasts a minute at most.
+- An OBS request that timed out is dropped instead of being done late, and
+ a busy chat (a raid) is judged a few times a second instead of on every
+ message. Counters are saved with the rest every few seconds.
+- An OBS drop without crash protection shows in the health report and
+ lights the on-air light, and a moment's reconnect of OBS's WebSocket no
+ longer re-runs the scene delay.
+- Adding a pack before any Discord channel opens the Discord setup right
+ away, and the pack's integrations switch on once it's saved.
+- In the builder, Alt+Up and Alt+Down move the picked step, Ctrl+D copies
+ it and Delete removes it.
+- On a smaller window, the builder's steps keep their text readable: only
+ the picked step shows its move, copy and delete buttons, on a row of
+ their own. Chat-activity rules fit the side panel too.
+- A switched-on integration that pushes to your phone says "Needs your
+ phone" on its card until a phone is connected.
+- `!setdelay 0` turns the delay off. It used to set your default delay
+ instead, while chat said "Delay set to 0s". A "Set the delay" step left
+ blank still uses your default, and a new Mod controls answers
+ `!setdelay 9999` with the 600 s limit instead of claiming it.
+- `!setdelay` takes whole seconds only. `!setdelay 0.5` used to set 1 s
+ and `!setdelay 0.0004` your default delay, while chat said otherwise; a
+ new Mod controls now answers with how to use it, and checks have a new
+ "is a whole number".
+- Removing a Discord channel that couldn't be removed says so, instead of
+ "Channel removed".
+- A card's message box no longer shrinks and cuts its text when the
+ editor redraws while you type in it (most visible on a phone).
+
+Also fixed before release:
+
+- **InstantClone no longer crashes in the first hour after Windows
+ starts.** The Twitch login check worked out "an hour ago" from the
+ time since boot, and that is negative right after a restart. Launching
+ at startup hit it every time.
+- **The Twitch login can't get stuck.** A Twitch request that never
+ answered could stop token refreshes for the rest of the session. Twitch
+ requests now time out. Logging out during a refresh no longer logs you
+ back in. A clip or marker refused for a revoked login checks the login
+ right away. Chat reconnects with the current login instead of the one
+ it started with.
+- **Hotkeys and MIDI pads, sorted out:**
+ - A delay hotkey or pad can't silently take one an integration uses.
+ - An integration can't be switched on with a key the delay already
+ uses.
+ - A key another app holds shows "Key taken" on its card.
+ - A knob or fader fires once per turn, not dozens of times.
+ - An integration's pad works on any controller, not just the one
+ picked in Controls, and learning one can use any controller.
+ - Learning a delay pad in Controls refuses a pad an integration uses,
+ and says which one.
+- **A chat message Twitch refuses fails its step**, with Twitch's reason
+ (slow mode, a duplicate, a ban), instead of showing as sent.
+- **A destination that drops again during the alert cooldown still gets
+ its "dropped" alert** once the 5 minutes are up, so "live" is never the
+ last word on a dead one.
+- **Duplicate makes a copy that starts switched off**, with its own web
+ call link and no hotkey, so one press never runs both.
+- **Cooldowns only start when something ran.** A command skipped during
+ quiet hours, or while the integration was busy, no longer makes the
+ viewer wait. "Busy" runs no longer hide a failing integration, and a
+ flood of them is one line in the log.
+- **A Discord ping is kept when the message it edits was deleted** and
+ has to be posted again.
+- **`{uses}` survives a restart or quit**, and the integrations data file
+ is written safely: a file damaged by a power cut is kept aside as
+ `.bad` instead of being overwritten.
+- **Phones:** the editor's Save button and the builder's close button no
+ longer fall off the screen. The builder scrolls as one page, and the
+ dashboard scrolls normally instead of in a small box.
+- **Modals stay put.** The Connections tabs and opening a chip no longer
+ shift the window, and Esc closes an open chip before the whole editor.
+- **The builder is easier to follow:**
+ - Each trigger's on/off switch and Remove sit right under its name,
+ instead of at the end of the panel, where the confirm overflowed it.
+ - Two triggers alike are numbered, so their tabs never read the same.
+ - Settings for the whole integration (wait between runs, quiet hours)
+ are grouped apart from the trigger's own.
+ - Every block says what it does, and an empty trigger says how to
+ start.
+ - The catalog's tiles no longer cut off their Add button.
+ - Integrations with many moments show them as chips that wrap, each
+ with its switch, instead of a bar running off the edge.
+ - Choices that don't fit a phone's width wrap instead of sticking out.
+- **A delay changed by an integration says "Integration"** in its toast,
+ not "Hotkey".
+- **Smaller fixes:**
+ - A timer switched back on waits a full period.
+ - "Wait for the delay" can't crash on an unlucky tick.
+ - A chat reply starting with `/` stays within Twitch's 500 characters.
+ - Logins in web addresses (`user:pass@`) are hidden like the rest.
+ - Counters named after viewers stop at 5,000.
+- When the Twitch account you logged in with isn't the channel your
+ Twitch destination streams to, Connections says so, and markers and
+ clips show "Other channel" instead of failing with Twitch's "offline".
+
+Your existing Discord webhook moves over by itself: it becomes a Discord
+connection plus a "Stream alerts" integration that sends the same
+messages as before.
+
+Going back to 0.1.14 afterwards: 0.1.14 doesn't know integrations, so it
+stops posting to Discord and drops your integrations the first time it
+saves its settings. Copy `instantclone.config.json` first if you might go
+back.
+
+### Vertical (9:16) no longer needs a Twitch destination
+
+Sending a vertical feed to TikTok, YouTube Shorts or Kick mobile used to
+require a Twitch destination, because the only 9:16 canvas InstantClone knew
+about was Twitch's Dual Format one. Streamers who only go to TikTok (through
+Restream, for example) could not use it at all, and the Vertical option was
+greyed out for them (thanks **fashion** on Discord for the report and the
+logs that tracked it down).
+
+- **OBS's Additional canvas feeds vertical destinations.** With no Twitch
+ destination, InstantClone now builds OBS's Enhanced Broadcasting config
+ itself from what OBS reports, and asks for your Additional canvas (for
+ example Aitum Vertical) when you picked one. Set it once in OBS:
+ Settings -> Stream -> Enhanced Broadcasting, then Additional canvas.
+- **Vertical is always selectable.** The lock that disabled it without a
+ Twitch destination is gone.
+- **A waiting destination says why.** Its card shows **EB off** or
+ **No 9:16 canvas** with the fix written out underneath, the Destinations
+ header counts how many are waiting, and the log says it once per stream
+ instead of the destination sitting silent. With a Twitch destination
+ whose channel has no Dual Format, the log names the vertical
+ destinations that will get nothing.
+- **Twitch 2K channels feed vertical destinations too.** Twitch streams
+ Partners and Affiliates at 1440p in HEVC, vertical track included, even
+ with H.264 set in OBS. InstantClone could only find a vertical track in
+ H.264, so a vertical destination showed "No 9:16 canvas" and got nothing.
+ It now reads the vertical track from Twitch's config, in any codec, and
+ forwards it as Twitch sent it. Restream takes HEVC and converts it to
+ H.264 for TikTok and the rest.
+- **A horizontal destination besides Twitch gets an H.264 main track.**
+ While one is on, InstantClone asks Twitch for H.264, since Kick and most
+ ingests can't play HEVC. Twitch then streams at 1080p instead of 2K.
+
+### Enhanced Broadcasting without Twitch now uses your OBS settings
+
+With Enhanced Broadcasting on, OBS asks InstantClone which video tracks to
+encode. Where that answer comes from depends on your destinations:
+
+- **A Twitch destination is on when you start streaming:** nothing changes.
+ InstantClone asks Twitch, and Twitch's own Enhanced Broadcasting config is
+ used, exactly as when OBS streams straight to Twitch. Twitch picks the
+ codec, resolution and bitrate, and that's the best quality you can get
+ there.
+- **No Twitch destination is on:** there is no Twitch to ask, so
+ InstantClone answers itself. This is the fallback, and it is what changed
+ in this version.
+
+Twitch is only asked at the moment you press Start Streaming. Switching a
+Twitch destination on mid-stream keeps the fallback until you stop and start
+streaming in OBS again.
+
+**What the fallback does now.** It used to make up its own settings: first
+three x264 encodes at once (enough to overload a gaming PC's CPU), then a
+fixed 6000 kbps at your full canvas size plus a vertical track nobody asked
+for. That could look worse than streaming with Enhanced Broadcasting off.
+Now it copies you instead:
+
+- **Your stream settings, as with Enhanced Broadcasting off.** The main
+ track uses what you set in OBS under Settings > Output. In Advanced mode
+ that is your encoder with all its settings, your bitrate and your Rescale
+ Output. In Simple mode it is your bitrate. To change how the fallback
+ looks, change it in OBS as you always would.
+- **A vertical track only when you stream one.** OBS's Additional canvas is
+ encoded only while a destination is set to Vertical, so it costs no GPU
+ otherwise. It gets the same quality per pixel as your main track. It uses
+ your encoder too when every vertical destination plays its codec (HEVC to
+ YouTube or Restream, for example), and H.264 otherwise, which plays
+ everywhere.
+- **If your OBS settings can't be read**, the old defaults are used: 6000
+ kbps for the main track (or the Maximum Streaming Bandwidth you set for
+ Enhanced Broadcasting), on your GPU's H.264 encoder (NVENC, AMF or QSV)
+ when OBS lists it as available, and x264 otherwise.
+
+The event log says which one you got, for example `main track uses your OBS
+stream settings: h264_texture_amf, 6000 kbps`.
+
+### Security
+
+- **Websites can't reach the dashboard any more.** With no dashboard
+ password, a web page could reach InstantClone through DNS rebinding: a
+ domain that re-resolves to your own PC. It could then save an
+ integration that runs a program. The dashboard now answers only when
+ it's opened by its address (`127.0.0.1`, `localhost` or a LAN IP).
+ Opening it by a name needs a password, and a reverse proxy should
+ always have one. Overlays, the alerts page and web call links keep working from
+ anywhere.
+- **Your settings can't be read by other websites.** Every page used to
+ let any website read it. Now only the overlay, alerts and delay-status
+ pages can be read from another site. The config (your ingest key and webhooks),
+ destinations, integrations and logs can't.
+- **Programs and files need a password once the dashboard is on your
+ network.** With the dashboard open to your network and no password,
+ anyone on the network could add a step that runs a program and start it
+ from chat, directly or through an overlay page the streaming PC loads.
+ While the dashboard is open to the network, or reached through a
+ reverse proxy or a port forwarder, Run a program and Write a file can
+ only be added, changed or switched on once it has a password; they show
+ as locked and say why. A web request step aimed at the dashboard itself
+ counts as coming from outside, so it can't set the first password
+ either. With the dashboard only on this PC (the default), nothing
+ changes.
+- **Overlay pages run sandboxed.** An overlay is a page from the overlays
+ folder; it now runs with an origin of its own, so it still draws but
+ can't act as the dashboard. An overlay file opened in the Studio is
+ cleaned first, so one made to break out of its styles can't run code
+ in the dashboard either.
+- **Chat can't run commands through a program.** A value typed in chat
+ stays one argument, and keeps only plain letters, numbers, spaces and
+ `_ . , + = / -`: a step like `cmd /c echo {arg1}` can't be talked into
+ running something else, look-alike letters can't turn into quotes, and
+ no word of it can pass itself off as an option (`--config`, `/s`).
+- **Web request addresses are hidden** until you click Show, like webhook
+ links: they can hold keys (`?key=`, a Hue bridge username).
+- **A stream key under six characters is no longer logged in the clear**
+ when a destination drops. Platform keys are never that short, but a
+ custom RTMP destination takes whatever you paste.
+- **Recipes leave out more**: request bodies and program arguments too.
+ An import is now cleaned the same way, so a hand-made recipe can't
+ bring in a web address, a program or a hotkey.
+- **Web requests don't follow redirects.** A redirect could have pointed
+ a request at the dashboard itself and posted its answer to chat. The
+ step now says where the server moved instead.
+
+### Fixes
+
+- **Stopping in OBS no longer cuts the end of a delayed stream.** With a
+ delay on and no crash protection hold (protection off, or a stop it
+ lets through), the last seconds of the delay used to vanish: viewers
+ never saw your goodbye. Now the rest of the delay airs, then the stream
+ ends. A crash without protection airs what OBS sent before it, too. The
+ dashboard and dock show "Your delay is still airing" with a countdown,
+ and **End now** or **Cut delay** cuts it short. Starting OBS again before
+ it finishes reconnects the destinations at your delay, as before.
+- **Restarting OBS no longer leaves a destination offline for up to 30 s.**
+ After a platform had refused a few connections, the wait between
+ retries stayed long even once it worked, so a quick OBS restart had to
+ sit it out. A stream that ended because of OBS now resets that wait,
+ and one that ends the moment it starts backs off instead of redialling
+ the platform every second.
+- **Nothing airs twice after a freeze you ended.** When you ended the
+ reconnect screen of a frozen OBS (or it ran out) and OBS then recovered,
+ a destination starting afterwards could join on video from before the
+ freeze and replay it, sometimes over and over. It now joins OBS's new
+ video at your delay.
+- **A 9:16 destination stays on when OBS comes back from a crash.** It was
+ ended if its vertical canvas arrived a moment after the main one; it now
+ gets a few seconds' grace.
+- **End now when there's nothing left to end** says so, instead of "Stream
+ ended on every destination".
+- **Discord webhooks work.** The HTTPS client checked certificates
+ against a bundled list instead of Windows' own store, and refused
+ Discord's certificate chain, so every Discord alert failed silently. It
+ now uses the system store.
+- **The "Twitch · mobile risk" chip no longer flickers.** It read the
+ total bitrate one second at a time, so a stream at 8 Mbps bounced
+ across the line with every keyframe, and audio plus the VOD track
+ pushed a 7.8 Mbps video over it. It now reads the video alone,
+ averaged over 10 s, and clears only once it drops back under 7.7 Mbps.
+- **A warning when OBS ignores Enhanced Broadcasting.** OBS only applies
+ Stream settings while nothing is running: tick EB with the Replay
+ Buffer on and the next stream still goes out single-track, with no
+ vertical canvas. When OBS's saved settings have EB on but the stream
+ arrives single-track, the dashboard says so and how to fix it.
+- **Crash protection keeps a new Twitch session when the hold ends first.**
+ If OBS came back and asked for a fresh Enhanced Broadcasting session just
+ as the hold ran out (or you pressed End now), the session was thrown away
+ and Twitch restarted without it.
+- **An OBS that reconnects but never sends a picture** no longer keeps the
+ reconnect screen up forever: after 10 seconds it counts as a freeze and
+ gets the normal time limit.
+- **No backwards timestamps when the reconnect screen starts.** Its first
+ frames are stamped after the newest audio or video already sent, not the
+ last one, which can be a few milliseconds older.
+- **A second Twitch account on a 2K channel** gets an H.264 main track like
+ Kick does, since it streams outside the Enhanced Broadcasting session.
+- **The vertical track survives a stray config request** made with the wrong
+ InstantClone key, and the header's Vertical pill now also recognises an
+ HEVC vertical canvas.
+- **Arcade style draws accents and curly quotes.** "¡Volvé!" and "We’ll" show
+ as VOLVE and WE'LL instead of dropping the letters.
+- **Dashboard:** unsaved System edits survive saving a destination or the
+ password; saving a destination no longer replays the tab's animation;
+ arrow-key style changes show the save bar; Space on a focused button or
+ inside a dialog no longer arms or stops the delay; a log source filter
+ that empties resets to all sources; End now reports failures in the
+ dashboard and the dock; the panel refits when the crash-protection banner
+ appears.
+- **Twitch no longer loops on "connection aborted" after OBS reconnects
+ during crash protection.** When OBS dropped with no Enhanced Broadcasting
+ session to keep and then started a fresh one, the reconnect threw that new
+ session away as stale. Twitch then got the multi-track stream on its plain
+ ingest and refused it on every retry. The same happened when OBS took
+ back the kept session and the hold ran out, or was ended, before OBS
+ started publishing on it.
+- **A frozen OBS that recovers just after crash protection's deadline is
+ reported as ended, not as back in time.** Its first frame could beat the
+ deadline check and post "OBS is back, live again" for destinations that
+ had already ended.
+- **A delay always goes on air, and never shorter than you set.** A 1 s
+ delay, or a 2 s delay with a 4 s keyframe interval, counted as "close
+ enough" to live, so viewers kept watching live while the dashboard said
+ delayed. A delay now lands on the newest keyframe at least that old, so it
+ can run up to one keyframe interval longer than set, never shorter.
+- **Raising the delay on air jumps back once.** It could jump back twice
+ and show viewers the same stretch again.
+- **An OBS restart mid-delay never puts viewers on live.** With crash
+ protection off, destinations came back live and then jumped back once the
+ buffer refilled. They now wait for the buffer to reach the delay.
+- **A destination added mid-delay starts delayed** with long keyframe
+ intervals too, instead of airing live.
+- **A delay longer than the buffer holds no longer gets stuck.** Only the
+ dashboard checked a delay against the buffer size; a hotkey, a MIDI pad,
+ auto-arm or a bitrate that rose after arming could ask for more than it
+ holds, and then the delay never filled: destinations waited forever and
+ ones already on air stalled. A full buffer now counts as ready, the delay
+ runs at what it holds, and the log says so.
+- **A delay no longer replays the same moment over and over** when the
+ encoder's keyframes turn out further apart than first measured (OBS's
+ "auto" keyframe interval, right after a restart).
+- **Crash protection plays out what's airing.** When its time runs out while
+ the delay is still airing, the rest airs in full before the stream ends,
+ as promised. **End now** ends at once. A frozen OBS that outlasts the
+ hold ends the stream too; it used to replay the delay's last seconds in a
+ loop.
+- **Coming back from a freeze keeps the full delay.** The delay rebuilds from
+ OBS's first keyframe, not its first frame, so it no longer rejoins short
+ of the delay.
+- **A destination switched on during crash protection joins the reconnect
+ screen** instead of waiting for OBS.
+- **"Track 2" destinations get only the clean track.** With OBS sending both
+ tracks, a YouTube or Kick destination set to the VOD / clean track also
+ got the live one mixed in, copyrighted music included (since 0.1.13).
+- **The VOD unlocker script works without Enhanced Broadcasting.** With EB
+ off, OBS's own stream setup freed the script's VOD audio encoder right
+ before going live, so OBS sent one audio track and the Twitch VOD had no
+ separate audio. Re-download the script from **System -> Twitch & OBS ->
+ Twitch VOD audio** and replace the old one in OBS. It now also logs a
+ warning when the VOD track goes missing, without needing verbose logging
+ (thanks **adTexaz** on Discord for the report and the logs that tracked
+ it down).
+- **Editing a live destination restarts it cleanly.** A new URL, stream
+ format or audio track takes effect right away with a proper goodbye to
+ the platform. Switching to Vertical mid-stream used to corrupt the picture
+ until the next cut, and audio track changes waited for the next restart.
+- **A platform that accepts and then drops the stream** is retried with a
+ growing backoff instead of every second, and a platform that never answers
+ a connect is retried after 20 s instead of showing "connecting" forever.
+ The reconnect counter now counts only real drops.
+- **Restarting a hung OBS works.** A publisher that stopped sending video
+ hands over to a new one instead of locking it out until its connection
+ dies.
+- **A stopped destination no longer holds the buffer back**, which let the
+ buffer grow to its full size.
+- **Hardened ingest.** Oversized commands, half-sent messages and idle
+ connections that never publish are cut off before they can use lots of
+ memory, and refused publishes are logged at most every 10 s. The local
+ test sink now listens on this computer only.
+- **Keyframes are read more carefully:** an encoder's end-of-stream marker is
+ no longer taken for a keyframe, and streams with short NAL lengths get cut
+ points. Custom RTMP servers that reject part of the publish setup
+ InstantClone sends (some answer it with an error) no longer fail the
+ connection.
+- **No backward timestamp when OBS reconnects within half a second**, and a
+ new stream header mid-stream now goes out before the frame that needs it.
+- **Your settings survive a settings file that can't be read.** A file
+ saved from Notepad as ANSI with an accent in it, or one briefly locked by
+ antivirus or OneDrive, used to start InstantClone on defaults and save
+ them over every destination and stream key. Now a stray byte only costs
+ that character, a locked file is retried, and a file that still can't be
+ read is copied to `.unreadable` and never overwritten at startup.
+- **A deleted destination stays deleted.** Removing your last destination
+ brought back an enabled "Main" one on the next launch.
+- **A malformed video frame can no longer close InstantClone.** A frame
+ ending in an empty NAL unit crashed the app mid-stream. The RTMP ingest
+ also no longer reserves memory for data a client only announces.
+- **Enhanced Broadcasting destinations start on a clean picture.** A
+ destination joining a multitrack stream could start on another track's
+ keyframe and show blocky video until the next one.
+- **A broken control request no longer disarms the delay.** A `POST /arm`
+ with an unreadable body, or one cut off before its end, used to run as if
+ it had no arguments. It now gets an error and changes nothing.
+- **The OBS dock and Stream Deck token no longer see custom RTMP URLs**,
+ which often hold a stream key. The dashboard still shows them for editing.
+- **OBS's service list stays valid.** Registering InstantClone into an OBS
+ with no services wrote invalid JSON, a failed refresh could list
+ InstantClone twice, and every refresh added a blank line.
+- **The OBS VOD track setting sticks** when OBS's settings file has the same
+ section twice.
+- **Update checks see every newer version**, including `beta.10` after
+ `beta.9` and versions tagged with build info.
+- **Custom RTMP destinations accept IPv6 addresses** like
+ `rtmp://[::1]:1935/app/key`, and a URL with no stream key is reported
+ instead of publishing with an empty one.
+- **Servers that send AMF0 dates or long strings** in their replies no
+ longer drop the connection.
+- **No false "1920p" warning for Kick** when the main canvas is portrait.
+- **Ending a stream now reaches the platform as a clean stop.** InstantClone
+ sent the platform its goodbye and closed the connection straight away,
+ and the closing could throw that goodbye away before the platform read
+ it, so the platform saw a dropped stream instead. It now closes only
+ after the platform has read it (or after 1 second).
+- **The old "VOD + EB" desktop shortcut no longer edits OBS 32.2+.** On
+ those versions it can't turn on Enhanced Broadcasting for Custom RTMP, so
+ it now only opens OBS and the log points to the InstantClone service and
+ the VOD unlocker script.
+- **Editing a Twitch destination could reset its pinned ingest to Auto.**
+ The first time the editor opened after loading the dashboard, the region
+ list arrived after the saved value was applied, so saving wrote Auto.
+- **The Twitch mobile-risk pill never appeared.** It looked for a platform
+ field the live state doesn't carry.
+- **The VOD + EB setup button reset a destination's audio track** to Auto.
+- **The header's Vertical pill stayed on after OBS stopped.**
+- **Destination save errors are shown as plain text,** and a save that
+ gets no answer from the app says so instead of doing nothing.
+- **Hovering a blue button turned it grey** with near-invisible text, on
+ every primary button in the dashboard (Save, Arm, Add destination...).
+ Amber warning buttons lost their colour on hover the same way.
+- **Buttons hovered green while a delay was arming.** The hover colour
+ blended the accent with the stream state's colour, and cyan plus amber
+ came out green. Each state now hovers in one clean colour: amber while
+ arming, green while active, the accent otherwise.
+- **The bitrate stuck at its last value after OBS stopped** (the header and
+ graph kept showing, say, 17.40 Mbps with OBS offline). It now drops to 0
+ a couple of seconds after data stops, and a destination's rate does the
+ same.
+- **"… Applying" lingered for 2.5 s after Cut or Disarm** even when the app
+ answered at once. It now clears as soon as the new state arrives, and
+ reads "Applying" with the dots used elsewhere. The OBS dock no longer
+ flashes Arm for a moment after Cut delay (the delay stays armed, so it
+ goes straight to Activate).
+- **Idle destination cards squeezed their message** into a narrow column
+ beside the counters. The counters now drop to their own line.
+
+### Polish
+
+- **System is organized by topic.** General, Crash protection, Controls
+ (hotkeys and MIDI), Twitch & OBS, Access (ingest key, password, ports,
+ LAN), Alerts & logs, and About. Behavior used to hold six unrelated
+ things while three tabs held one each.
+- **Find a setting.** Type in System's search (or press **/**) and it
+ opens the right tab and highlights the section. It knows the words
+ people use, like "keybind", "brb" or "port".
+- **A save bar that follows you.** It stays on screen while there are
+ unsaved changes, with **Discard** to undo them, and the page asks before
+ you close it with changes unsaved.
+- Each tab's scroll area now ends inside the window instead of under it.
+- **Destination cards say what they're doing.** "Goes live when OBS does",
+ "Connecting…" or "Off. Switch it on to stream here." instead of a dash,
+ frames, cuts and reconnects in words once there is something to count,
+ and one pulse in the platform's colour when a destination goes live.
+- **Platform logos.** Twitch, YouTube and Kick show their own marks in their
+ colours on destination cards, the platform picker and the setup wizard;
+ Restream, Custom and the test sink get a clear symbol each.
+- **A roomier destination editor.** Two columns on wide screens (name beside
+ platform, server beside stream key, format beside audio), one on narrow
+ ones, and a full-screen sheet on phones. The header shows the platform's
+ logo and an Enabled switch, and Save stays in view while the rest scrolls.
+- **Livelier fields and buttons.** Inputs glow softly when focused and their
+ label lights up with them; buttons lift a touch on hover. Both pick up the
+ stream state's colour: cyan when idle, shifting toward amber or green.
+ Dropdowns open as a styled list that fades in, with a turning arrow and a
+ check on the current choice (older browsers keep the native list).
+- **The aurora, reworked.** Five soft pools of light circle slowly behind the
+ whole delay panel, each at its own size, speed and direction, so the
+ colours keep folding into each other. They take the stream state's colour
+ (grey offline, amber buffering, cyan armed, green delayed), and while
+ buffering the light grows as the buffer fills. The page top glows faintly
+ with it, the panel's top edge catches the light, and each state change
+ blooms from the number. Buttons and cards pick up a subtle lit edge too.
+ It holds still with the system's reduced-motion setting.
+- **Clearer at a glance.** Larger section titles, labels and stats numbers,
+ bigger tab icons with the open tab in the accent colour, and screens that
+ explain themselves when there's nothing yet (Stats before OBS streams, an
+ empty Profiles list).
+- **A log you can read.** Every event is a row with its time and source;
+ errors are red, warnings amber, good news green. Filter to Problems or
+ Errors (with counts), one source, or any text, and jump back to the latest
+ line after scrolling up.
+- **A friendly start.** With no destinations yet, the Destinations tab asks
+ where you stream and opens the form with that platform picked.
+- **Motion that guides the eye.** Cards and sections ease in one after
+ another when a tab opens, and buttons give a small press. All of it
+ switches off with the system's reduced-motion setting.
+- **A delay panel that answers back.**
+ - The main button eases between Arm, Activate, Adjust and Cut instead of
+ snapping, and when only the number changes, only the number flips.
+ - The delay field rolls its digits up or down as you step through values.
+ - − at 0, + at 600 or a typed 700 light up the range, and a typed value
+ past 600 settles on 600.
+ - A profile too big for the buffer says why when clicked, instead of
+ hiding it in a tooltip.
+ - "Waiting for OBS" visibly listens, and the button blooms once when OBS
+ connects.
+ - Clicking a profile sends a spark into the button it arms, and the armed
+ profile fills in.
+
+ All of it plays only on a real change, never on the dashboard's 4× a
+ second refresh, which now writes nothing when nothing changed. Reduced
+ motion keeps the colour cues and drops the movement.
+- **The status pill glides.** Its words change in step with its colour, and
+ it eases to its new width instead of jumping. The delay readout also stops
+ rewriting itself on every refresh when nothing changed.
+- **The OBS dock, reworked for mid-stream use.**
+ - While the buffer fills, the main button counts down ("Ready in 0:23")
+ and fills toward Activate, instead of a spinner that restarted four
+ times a second.
+ - During a crash hold the dock says **Holding** (not Active and Offline
+ at once), and the hold card shows the countdown, how many destinations
+ are on the reconnect screen, and **End now**. It takes the place of the
+ delay controls, which step aside until OBS is back. A dock without
+ them, like a destinations-only dock, shows the card on top.
+ - Destinations show when they are on but not streaming yet ("Connecting…"
+ in amber) and when they are on the reconnect screen, instead of the
+ same grey dot as off. They follow the live state instead of a 4 s
+ refresh, so a "Turn off?" no longer vanishes before you can confirm.
+ - The status chip glides between states, the number pops only on a change
+ you made (not while the buffer counts up), and the idle "0 s" sits
+ together as "0s" without moving the +/− buttons.
+ - Like the dashboard, the dock now writes nothing when a refresh changes
+ nothing.
+- **Add destination** is a tinted button with a real plus icon.
+- The log wraps long lines instead of cutting them off, and stops jumping
+ to the bottom while you are reading older lines.
+- Destination cards in a row line up, the frame, cut and reconnect counters
+ have a tooltip, and the "+ Add" tile keeps keyboard focus.
+- The destination editor shows the vertical setup as two steps, only when
+ Vertical is picked, and folds the audio track details away.
+- The OBS tab recommends a GPU encoder and notes that Custom RTMP can't use
+ Enhanced Broadcasting on OBS 32.2 and newer.
## [0.1.14] - Hotkeys and MIDI for the delay, a dashboard password, and Linux builds
@@ -1703,7 +2488,8 @@ See `0.1.0` below for the full feature list.
port pre-flight, and process / RSS sampler have Windows-specific paths.
- No automated release pipeline yet. Build from source per the README.
-[Unreleased]: https://github.com/Soulhackzlol/InstantClone/compare/v0.1.14...HEAD
+[Unreleased]: https://github.com/Soulhackzlol/InstantClone/compare/v0.1.15...HEAD
+[0.1.15]: https://github.com/Soulhackzlol/InstantClone/compare/v0.1.14...v0.1.15
[0.1.14]: https://github.com/Soulhackzlol/InstantClone/compare/v0.1.13...v0.1.14
[0.1.13]: https://github.com/Soulhackzlol/InstantClone/compare/v0.1.12...v0.1.13
[0.1.12]: https://github.com/Soulhackzlol/InstantClone/compare/v0.1.11...v0.1.12
diff --git a/Cargo.lock b/Cargo.lock
index add05c3..03ed2d8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -219,7 +219,7 @@ dependencies = [
[[package]]
name = "instantclone"
-version = "0.1.14"
+version = "0.1.15"
dependencies = [
"bytes",
"flate2",
diff --git a/Cargo.toml b/Cargo.toml
index c838b3e..e280172 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "instantclone"
-version = "0.1.14"
+version = "0.1.15"
edition = "2021"
authors = ["s1moscs"]
license = "GPL-3.0-only"
@@ -61,6 +61,7 @@ windows-sys = { version = "0.61", features = [
"Win32_System_LibraryLoader",
"Win32_Security_Cryptography",
"Win32_System_Threading",
+ "Win32_System_SystemInformation",
"Win32_System_Diagnostics_ToolHelp",
"Win32_System_DataExchange",
"Win32_System_Memory",
diff --git a/README.es.md b/README.es.md
index b6ea29d..b9b0fb1 100644
--- a/README.es.md
+++ b/README.es.md
@@ -149,7 +149,7 @@ Haz simulcast de una señal de OBS a Twitch, YouTube, Kick y RTMP personalizado
**📱 Vertical (9:16) gratis**
-Activa **Formato Dual** de Twitch (Enhanced Broadcasting) y pon el formato de cualquier destino no-Twitch en **Vertical**. InstantClone reutiliza el lienzo 9:16 que OBS ya crea para Twitch y lo envía a YouTube Shorts, Kick móvil o TikTok, sin codificación extra.
+En OBS, activa **Enhanced Broadcasting** y elige tu lienzo vertical (por ejemplo Aitum Vertical) en **Lienzo adicional**, y pon el formato de cualquier destino no-Twitch en **Vertical**. InstantClone envía ese lienzo 9:16 a TikTok, YouTube Shorts o Kick móvil, con o sin un destino de Twitch. Activa el destino Vertical antes de empezar a emitir: sin Twitch, la pista vertical solo se codifica mientras haya uno activo. ¿También en Twitch? El **Formato Dual** de Twitch aporta el mismo lienzo.
@@ -191,13 +191,29 @@ Rearma o ajusta el delay arriba/abajo sin desarmar primero, expuesto como un con
**⌨ Atajos de teclado globales**
-Asigna delay on/off, armar, activar, cortar y **cortar cuando esto salga** a una combinación que funciona con un juego en pantalla completa por delante. Toda combinación necesita un modificador para que nada se dispare a media partida, una que otra app ya ocupa se marca en su fila en vez de fallar en silencio, y una acción rechazada te llega como globo en la bandeja.
+Asigna delay on/off, armar, activar, cortar, **cortar cuando esto salga** y terminar la protección contra crasheos a una combinación que funciona con un juego en pantalla completa por delante. Toda combinación necesita un modificador para que nada se dispare a media partida, una que otra app ya ocupa se marca en su fila en vez de fallar en silencio, y una acción rechazada te llega como globo en la bandeja.
**🎹 Pads y controladoras MIDI**
-Mapea las mismas cinco acciones a un pad o un knob, aprendidos pulsando el control en vez de escribir un número de nota. Cada mapeo recuerda de qué dispositivo vino, así dos controladoras pueden mover acciones distintas aunque manden la misma nota, y puedes acotar a qué dispositivo escucha InstantClone.
+Mapea las mismas acciones a un pad o un knob, aprendidos pulsando el control en vez de escribir un número de nota. Cada mapeo recuerda de qué dispositivo vino, así dos controladoras pueden mover acciones distintas aunque manden la misma nota, y puedes acotar a qué dispositivo escucha InstantClone.
+
+
+
+
+
+
+**🛡 Protección contra crasheos**
+Si OBS se cuelga o se congela, todos los destinos siguen en directo con una pantalla de reconexión (cinco estilos, con tus colores y tu texto) y audio en silencio, hasta 5 minutos. Cuando OBS vuelve, retoma en la misma conexión desde su primer keyframe, Enhanced Broadcasting incluido: cada pista recibe su propia pantalla y OBS recibe la misma sesión de Twitch. Las pistas HEVC y AV1 mantienen su último fotograma. Termínala antes desde el panel, el dock de OBS, la bandeja, un atajo o un pad MIDI.
+
+
+
+
+
+
+**🧩 Integraciones (experimental)**
+InstantClone habla con Discord, el chat de Twitch, tu móvil y tus propias herramientas. Con un clic añades un aviso de crasheo, un comando `!delay`, controles para mods, un botón de highlight que clipea lo que los espectadores vieron de verdad, una tarjeta de Discord que escribe la línea de tiempo de tu stream, capítulos de YouTube, una luz de "en directo" y más, y un editor por pasos cubre el resto. Son nuevas y aún pueden cambiar: cuéntanos lo que falle o confunda con el enlace **Report it** de la pestaña.
@@ -216,7 +232,7 @@ Mapea las mismas cinco acciones a un pad o un knob, aprendidos pulsando el contr
-**Dos fases por diseño.** **Armas** un buffer (un tamaño objetivo en segundos). InstantClone lo prellena desde la señal de OBS sin tocar lo que sale. Cuando se llena pulsas **Activar**, y el cambio a diferido es instantáneo en pantalla: el reproductor solo salta del borde en directo a un punto N segundos atrás.
+**Dos fases por diseño.** **Armas** un buffer (un tamaño objetivo en segundos). InstantClone lo prellena desde la señal de OBS sin tocar lo que sale. Cuando se llena pulsas **Activar**, y el cambio a diferido es instantáneo en pantalla: el reproductor solo salta del borde en directo al fotograma clave de al menos N segundos atrás, así que el público nunca queda más cerca del directo de lo que pusiste.
@@ -249,6 +265,7 @@ Mapea las mismas cinco acciones a un pad o un knob, aprendidos pulsando el contr
| POST | `/cut-after` | | Marca el borde en directo; autocorta cuando sale en todos. |
| POST | `/cut-after/cancel` | | Descarta un corte programado pendiente. |
| POST | `/delay` | `ms=NNN` | De un tiro: arma y autoactiva en cuanto esté listo. |
+| POST | `/crash-protection/end` | | Termina ya la pantalla de reconexión: todos los destinos terminan. |
| GET | `/state` | | Instantánea JSON puntual. |
| GET | `/events` | | Flujo de estado JSON por server-sent events. Solo push. |
@@ -306,7 +323,7 @@ Armado de un botón. Añade `/activate` y `/stop` a otros dos botones para contr
- **Handshake RTMP con paridad total con OBS.** `connect` lleva la misma bolsa de capacidades de códec que envía librtmp (`audioCodecs=3191`, `videoCodecs=252`, `videoFunction=1`), el `fourCcList` de Enhanced-RTMP (AVC / HEVC / AV1 / VP9 / Opus / AC-3 / FLAC), `Set Chunk Size` antes de connect, `FCUnpublish → deleteStream` al cerrar, y RTMP Acknowledgement (BYTES_READ_REPORT) al umbral ventana/10 declarado por el par, en entrada y salida.
- **Passthrough de Enhanced Broadcasting a Twitch.** Cuando OBS pasa a multipista "Auto" hacemos de proxy de `GetClientConfiguration` de Twitch, ruteamos la salida al endpoint IVS asignado a la sesión y reenviamos cada SPS/PPS por pista fielmente para que se encienda la escalera de transcodificado sin importar el nivel de cuenta. Los destinos no-Twitch reciben la pista primaria horizontal por defecto; las etiquetas de escalera con `TrackId != 0` se descartan para evitar la tormenta de varios frames por PTS que hace caer el decodificador de YouTube. Los cortes de EB caen en el IDR de la pista primaria (no en el keyframe del peldaño que gane el `partition_point`) para que el decodificador del destino siempre tenga su ancla.
-- **Selección del lienzo vertical (9:16).** El lienzo vertical se identifica decodificando el SPS de cada pista por orientación (retrato, mayor área) en vez de confiar en el JSON privado de sesión de Twitch, y se autocorrige según se activa/desactiva Formato Dual.
+- **Selección del lienzo vertical (9:16).** El lienzo vertical se identifica decodificando el SPS de cada pista por orientación (retrato, mayor área) en vez de confiar en el JSON privado de sesión de Twitch, y se autocorrige según aparece o desaparece el lienzo vertical.
- **Audio de VOD de Twitch, desbloqueado en el servicio InstantClone.** OBS ata su pista de VOD al servicio llamado literalmente "Twitch" (`ServiceSupportsVodTrack == {"Twitch"}`), así que está bloqueada en el servicio InstantClone. Un pequeño script de OBS incluido (`optional-vod-unlocker.lua`, descargado desde el panel) engancha el mismo segundo codificador de audio que usaría la propia pista de VOD de OBS, sin la restricción. Su lector de formato coincide byte a byte con el `flv_packet_audio_ex` de OBS (`AudioPacketType` en el byte 0, `TrackId` en el byte 6). OBS 32.2+ necesita el script; OBS anterior puede usar la casilla de VOD Track integrada (escribimos `EnableCustomServerVodTrack` en el `user.ini` de OBS 32, con `global.ini` como respaldo).
- **Ruteo de audio por destino.** Las pistas no seleccionadas se descartan y la elegida se aplana a una etiqueta de una sola pista estándar (AAC reescrito al `0xAF` legado), espejando el `flatten_multitrack_video` del lado de vídeo. Si la pista elegida no se está enviando, cae a la pista en directo en vez de quedarse en silencio.
@@ -336,7 +353,7 @@ El HTML del panel se minifica + gzipea en tiempo de compilación con `build.rs`
**E/S de disco síncrona en la ruta caliente de escritura al anillo, por elección.** La escritura con buffer aterriza en la caché de páginas del SO en microsegundos y el kernel vacía en segundo plano, así que la caché de páginas ya es el buffer asíncrono; el índice y los bytes avanzan bajo un solo lock para que un lector nunca vea una etiqueta cuyos bytes aún no están en disco.
-**Tests.** `cargo test --release` cubre la máquina de estados (`arm → preparing → ready → active → cut`), detección de IDR de AVC + Enhanced-RTMP, AMF0 (incluido Strict Array + guardia de recursión), round-trip de settings, expulsión del buffer en anillo con protección de lecturas en vuelo, parseo HTTP, política CSRF, pre-flight de puerto, negociación de contenido, caché de cabeceras de secuencia por pista de Enhanced Broadcasting + selección de etiquetas por TrackId, audio multipista + ruteo por destino, parseo de orientación SPS para la selección vertical, el parcheador de `services.json`, el parser del check de actualizaciones, el SHA-256 hecho a mano (vectores NIST), el lector/escritor de chunk-stream RTMP, la máquina del corte programado, las tablas de atajos y de mapeos MIDI (incluido el dispositivo que distingue dos controladoras), y la descarga de autoactualización + verificación de checksum + intercambio del exe. **392 tests, todos en verde.**
+**Tests.** `cargo test --release` cubre la máquina de estados (`arm → preparing → ready → active → cut`), detección de IDR de AVC + Enhanced-RTMP, AMF0 (incluido Strict Array + guardia de recursión), round-trip de settings, expulsión del buffer en anillo con protección de lecturas en vuelo, parseo HTTP, política CSRF, pre-flight de puerto, negociación de contenido, caché de cabeceras de secuencia por pista de Enhanced Broadcasting + selección de etiquetas por TrackId, audio multipista + ruteo por destino, parseo de orientación SPS para la selección vertical, la configuración de Enhanced Broadcasting construida a partir de los lienzos y codificadores GPU de OBS, el parcheador de `services.json`, el parser del check de actualizaciones, el SHA-256 hecho a mano (vectores NIST), el lector/escritor de chunk-stream RTMP, la máquina del corte programado, las tablas de atajos y de mapeos MIDI (incluido el dispositivo que distingue dos controladoras), la protección contra crasheos (distinguir crasheo, parada y congelado, el codificador sin pérdidas de la pantalla de reconexión comprobado con ffmpeg, el empaquetado por pista de Enhanced Broadcasting, el AAC en silencio), el motor de integraciones (pasos, plantillas, recetas, chat, tarjetas de Discord, la línea de tiempo del stream), y la descarga de autoactualización + verificación de checksum + intercambio del exe. **843 tests, todos en verde.**
@@ -346,7 +363,7 @@ El HTML del panel se minifica + gzipea en tiempo de compilación con `build.rs`
## Estado
-**Listo para uso diario en Windows.** Lo uso en mis propios streams, y un grupo creciente de streamers lo corre a diario también. CI ejecuta fmt + clippy (`-D warnings`) + 392 tests en cada push, y un commit etiquetado compila y publica una release con un `SHA256SUMS.txt` al lado (todavía sin certificado de firma de código, así que el SO puede avisar al primer arranque).
+**Listo para uso diario en Windows.** Lo uso en mis propios streams, y un grupo creciente de streamers lo corre a diario también. CI ejecuta fmt + clippy (`-D warnings`) + 843 tests en cada push, y un commit etiquetado compila y publica una release con un `SHA256SUMS.txt` al lado (todavía sin certificado de firma de código, así que el SO puede avisar al primer arranque).
**Lo áspero, con honestidad**
@@ -415,7 +432,7 @@ En la pestaña **Destinos** de InstantClone, nunca en OBS. OBS solo apunta a Ins
-Sí, ambas cosas. Cinco acciones - delay on/off, armar, activar, cortar a directo y **cortar cuando esto salga** - se asignan a un atajo de teclado global, a un pad o knob MIDI, o a los dos a la vez, en **Ajustes**. Los atajos funcionan con un juego en pantalla completa por delante, así no haces alt-tab a media partida, y toda combinación necesita un modificador (Ctrl, Alt, Shift o Win) para que una tecla suelta no dispare una acción de delay. Los mapeos MIDI se aprenden pulsando el control en vez de escribir un número de nota, y cada uno recuerda de qué dispositivo vino, así dos controladoras pueden mover acciones distintas. Por ahora solo en Windows.
+Sí, ambas cosas. Seis acciones - delay on/off, armar, activar, cortar a directo, **cortar cuando esto salga** y terminar la protección contra crasheos - se asignan a un atajo de teclado global, a un pad o knob MIDI, o a los dos a la vez, en **Ajustes**. Los atajos funcionan con un juego en pantalla completa por delante, así no haces alt-tab a media partida, y toda combinación necesita un modificador (Ctrl, Alt, Shift o Win) para que una tecla suelta no dispare una acción de delay. Los mapeos MIDI se aprenden pulsando el control en vez de escribir un número de nota, y cada uno recuerda de qué dispositivo vino, así dos controladoras pueden mover acciones distintas. Por ahora solo en Windows.
diff --git a/README.md b/README.md
index 55d45e9..1a0cd90 100644
--- a/README.md
+++ b/README.md
@@ -149,7 +149,7 @@ Simulcast one OBS feed to Twitch, YouTube, Kick, and custom RTMP at once, a free
**📱 Vertical (9:16) for free**
-Turn on Twitch **Dual Format** (Enhanced Broadcasting) and set any non-Twitch destination's format to **Vertical**. InstantClone reuses the 9:16 canvas OBS already makes for Twitch and sends it to YouTube Shorts, Kick mobile, or TikTok, with no extra encoding.
+In OBS, turn on **Enhanced Broadcasting** and pick your vertical canvas (for example Aitum Vertical) under **Additional canvas**, then set any non-Twitch destination's format to **Vertical**. InstantClone sends that 9:16 canvas to TikTok, YouTube Shorts, or Kick mobile, with or without a Twitch destination. Switch the Vertical destination on before you start streaming: without Twitch, the vertical track is only encoded while one is on. Streaming to Twitch too? Twitch **Dual Format** provides the same canvas.
@@ -191,13 +191,29 @@ Re-arm or nudge the delay up/down without disarming first, exposed as a single t
**⌨ Global hotkeys**
-Bind delay on/off, arm, activate, cut, and **cut after this airs** to a key combo that fires while a fullscreen game holds focus. Every binding needs a modifier so nothing trips mid-match, a combo another app already owns is flagged on the row instead of failing silently, and a refused action reaches you as a tray balloon.
+Bind delay on/off, arm, activate, cut, **cut after this airs**, and ending crash protection to a key combo that fires while a fullscreen game holds focus. Every binding needs a modifier so nothing trips mid-match, a combo another app already owns is flagged on the row instead of failing silently, and a refused action reaches you as a tray balloon.
**🎹 MIDI pads and decks**
-Map the same five actions to a pad or knob, learned by pressing the control rather than typing a note number. Each mapping remembers which device it came from, so two controllers can drive different actions even when they send the same note, and you can narrow which device InstantClone listens to.
+Map the same actions to a pad or knob, learned by pressing the control rather than typing a note number. Each mapping remembers which device it came from, so two controllers can drive different actions even when they send the same note, and you can narrow which device InstantClone listens to.
+
+
+
+
+
+
+**🛡 Crash protection**
+If OBS crashes or freezes, every destination stays live on a reconnect screen (five styles, your colours and text) with silent audio, for up to 5 minutes. OBS coming back resumes on the same connection at its first keyframe, Enhanced Broadcasting included: each track gets its own screen and OBS is handed the same Twitch session. HEVC and AV1 tracks hold their last frame. End it early from the dashboard, the OBS dock, the tray, a hotkey or a MIDI pad.
+
+
+
+
+
+
+**🧩 Integrations (experimental)**
+InstantClone talks to Discord, Twitch chat, your phone and your own tools. One click adds a crash alert, a `!delay` command, mod controls, a highlight button that clips what viewers actually saw, a Discord card that writes your stream's timeline, YouTube chapters, an on-air light and more, and a step builder covers the rest. They're new and may still change: please report what breaks or confuses you with the tab's **Report it** link.
@@ -216,7 +232,7 @@ Map the same five actions to a pad or knob, learned by pressing the control rath
-**Two-phase by design.** You **arm** a buffer (a target size in seconds). InstantClone pre-fills it from the live OBS feed without touching what's going out. Once it's full you hit **Activate**, and the switch to delayed is instant on screen: the player just jumps from the live edge to a point N seconds back.
+**Two-phase by design.** You **arm** a buffer (a target size in seconds). InstantClone pre-fills it from the live OBS feed without touching what's going out. Once it's full you hit **Activate**, and the switch to delayed is instant on screen: the player just jumps from the live edge to the keyframe at least N seconds back, so viewers are never closer to live than you set.
@@ -249,6 +265,7 @@ Map the same five actions to a pad or knob, learned by pressing the control rath
| POST | `/cut-after` | | Mark the live edge; auto-cut once it airs everywhere. |
| POST | `/cut-after/cancel` | | Drop a pending scheduled cut. |
| POST | `/delay` | `ms=NNN` | One-shot: arm, auto-activate as soon as ready. |
+| POST | `/crash-protection/end` | | End the reconnect screen now: every destination ends. |
| GET | `/state` | | One-shot JSON snapshot. |
| GET | `/events` | | Server-sent stream of state JSON. Push-only. |
@@ -306,7 +323,7 @@ One-button arming. Add `/activate` and `/stop` to two more buttons for full dela
- **Full OBS-parity RTMP handshake.** `connect` carries the same codec-capability bag librtmp ships (`audioCodecs=3191`, `videoCodecs=252`, `videoFunction=1`), the Enhanced-RTMP `fourCcList` (AVC / HEVC / AV1 / VP9 / Opus / AC-3 / FLAC), `Set Chunk Size` before connect, `FCUnpublish → deleteStream` on shutdown, and RTMP Acknowledgement (BYTES_READ_REPORT) at the peer-declared window/10 threshold on both ingest and egress.
- **Enhanced Broadcasting passthrough to Twitch.** When OBS hits multi-track "Auto" we proxy Twitch's `GetClientConfiguration`, route egress to the session-allocated IVS endpoint, and forward every per-track SPS/PPS bit-faithfully so the transcoded ladder lights up regardless of account tier. Non-Twitch destinations get the horizontal primary track by default; ladder tags with `TrackId != 0` are dropped to avoid the multi-frame-per-PTS storm that crashes YouTube's decoder. EB cuts land on the primary track's IDR (not whichever ladder rung's keyframe wins the `partition_point`) so the destination decoder always has its anchor.
-- **Vertical (9:16) canvas selection.** The vertical canvas is identified by decoding each track's SPS for orientation (portrait, largest area) rather than trusting Twitch's private session JSON, and it self-heals as Dual Format toggles on/off.
+- **Vertical (9:16) canvas selection.** The vertical canvas is identified by decoding each track's SPS for orientation (portrait, largest area) rather than trusting Twitch's private session JSON, and it self-heals as the vertical canvas comes and goes.
- **Twitch VOD audio, unlocked on the InstantClone service.** OBS hardcodes its VOD Track to the service literally named "Twitch" (`ServiceSupportsVodTrack == {"Twitch"}`), so it's locked on the InstantClone service. A tiny bundled OBS script (`optional-vod-unlocker.lua`, downloaded from the dashboard) attaches the same second audio encoder OBS's own VOD Track would, without the gate. Its wire-format reader matches OBS's `flv_packet_audio_ex` byte-for-byte (`AudioPacketType` in byte 0, `TrackId` at byte 6). OBS 32.2+ needs the script; older OBS can still use the built-in VOD Track checkbox (we write `EnableCustomServerVodTrack` to OBS 32's `user.ini`, falling back to `global.ini`).
- **Per-destination audio routing.** Non-selected tracks are dropped and the chosen one is flattened to a standard single-track tag (AAC rewritten to legacy `0xAF`), mirroring the video-side `flatten_multitrack_video`. If the chosen track isn't being sent, it falls back to the live track rather than going silent.
@@ -336,7 +353,7 @@ The dashboard HTML is minified + gzipped at build time by `build.rs` (`flate2`,
**Sync disk I/O on the ring-append hot path, by choice.** The buffered write lands in the OS page cache in microseconds and the kernel flushes in the background, so the page cache is already the async buffer; the index and the bytes advance under one lock so a reader never sees a tag whose bytes aren't on disk yet.
-**Tests.** `cargo test --release` covers the state machine (`arm → preparing → ready → active → cut`), AVC + Enhanced-RTMP IDR detection, AMF0 (including Strict Array + recursion guard), settings round-trip, ring-buffer eviction with in-flight-read protection, HTTP parsing, CSRF policy, port pre-flight, content negotiation, Enhanced Broadcasting per-track seq-header cache + TrackId-aware tag selection, multi-track audio + per-destination routing, SPS orientation parsing for vertical selection, the OBS `services.json` patcher, the update-check parser, the hand-rolled SHA-256 (NIST vectors), the RTMP chunk-stream reader/writer, the scheduled safe-cut state machine, the hotkey and MIDI binding tables (including the device that tells two controllers apart), and the self-update download + checksum-verify + exe swap. **392 tests, all green.**
+**Tests.** `cargo test --release` covers the state machine (`arm → preparing → ready → active → cut`), AVC + Enhanced-RTMP IDR detection, AMF0 (including Strict Array + recursion guard), settings round-trip, ring-buffer eviction with in-flight-read protection, HTTP parsing, CSRF policy, port pre-flight, content negotiation, Enhanced Broadcasting per-track seq-header cache + TrackId-aware tag selection, multi-track audio + per-destination routing, SPS orientation parsing for vertical selection, the Enhanced Broadcasting config built from OBS's canvases and GPU encoders, the OBS `services.json` patcher, the update-check parser, the hand-rolled SHA-256 (NIST vectors), the RTMP chunk-stream reader/writer, the scheduled safe-cut state machine, the hotkey and MIDI binding tables (including the device that tells two controllers apart), crash protection (crash vs. stop vs. freeze detection, the lossless reconnect-screen encoder checked against ffmpeg, per-track framing for Enhanced Broadcasting, silent AAC), the integrations engine (steps, templates, recipes, chat, Discord cards, the stream timeline), and the self-update download + checksum-verify + exe swap. **843 tests, all green.**
@@ -346,7 +363,7 @@ The dashboard HTML is minified + gzipped at build time by `build.rs` (`flate2`,
## Status
-**Daily-driver ready on Windows.** I use it on my own streams, and a growing group of streamers now run it daily too. CI runs fmt + clippy (`-D warnings`) + 392 tests on every push, and a tagged commit auto-builds and publishes a release with a `SHA256SUMS.txt` alongside (no code-signing certificate yet, so the OS may warn on first launch).
+**Daily-driver ready on Windows.** I use it on my own streams, and a growing group of streamers now run it daily too. CI runs fmt + clippy (`-D warnings`) + 843 tests on every push, and a tagged commit auto-builds and publishes a release with a `SHA256SUMS.txt` alongside (no code-signing certificate yet, so the OS may warn on first launch).
**What's rough, honestly**
@@ -415,7 +432,7 @@ In InstantClone's **Destinations** tab, never in OBS. OBS only ever points at In
-Yes, both. Five actions - delay on/off, arm, activate, cut to live, and **cut after this airs** - bind to a global keyboard shortcut, a MIDI pad or knob, or both at once, in **Settings**. Hotkeys fire while a fullscreen game holds focus, so you never alt-tab mid-match, and every binding needs a modifier (Ctrl, Alt, Shift or Win) so a stray keypress can't trip a delay action. MIDI mappings are learned by pressing the control rather than typing a note number, and each one remembers which device it came from, so two controllers can drive different actions. Windows only for now.
+Yes, both. Six actions - delay on/off, arm, activate, cut to live, **cut after this airs**, and ending crash protection - bind to a global keyboard shortcut, a MIDI pad or knob, or both at once, in **Settings**. Hotkeys fire while a fullscreen game holds focus, so you never alt-tab mid-match, and every binding needs a modifier (Ctrl, Alt, Shift or Win) so a stray keypress can't trip a delay action. MIDI mappings are learned by pressing the control rather than typing a note number, and each one remembers which device it came from, so two controllers can drive different actions. Windows only for now.
diff --git a/assets/fonts/Inter-Regular-slate.ttf b/assets/fonts/Inter-Regular-slate.ttf
new file mode 100644
index 0000000..bea36c7
Binary files /dev/null and b/assets/fonts/Inter-Regular-slate.ttf differ
diff --git a/assets/fonts/Inter-SemiBold-slate.ttf b/assets/fonts/Inter-SemiBold-slate.ttf
new file mode 100644
index 0000000..d24ee49
Binary files /dev/null and b/assets/fonts/Inter-SemiBold-slate.ttf differ
diff --git a/assets/fonts/OFL.txt b/assets/fonts/OFL.txt
new file mode 100644
index 0000000..9b2ca37
--- /dev/null
+++ b/assets/fonts/OFL.txt
@@ -0,0 +1,92 @@
+Copyright (c) 2016 The Inter Project Authors (https://github.com/rsms/inter)
+
+This Font Software is licensed under the SIL Open Font License, Version 1.1.
+This license is copied below, and is also available with a FAQ at:
+http://scripts.sil.org/OFL
+
+-----------------------------------------------------------
+SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
+-----------------------------------------------------------
+
+PREAMBLE
+The goals of the Open Font License (OFL) are to stimulate worldwide
+development of collaborative font projects, to support the font creation
+efforts of academic and linguistic communities, and to provide a free and
+open framework in which fonts may be shared and improved in partnership
+with others.
+
+The OFL allows the licensed fonts to be used, studied, modified and
+redistributed freely as long as they are not sold by themselves. The
+fonts, including any derivative works, can be bundled, embedded,
+redistributed and/or sold with any software provided that any reserved
+names are not used by derivative works. The fonts and derivatives,
+however, cannot be released under any other type of license. The
+requirement for fonts to remain under this license does not apply
+to any document created using the fonts or their derivatives.
+
+DEFINITIONS
+"Font Software" refers to the set of files released by the Copyright
+Holder(s) under this license and clearly marked as such. This may
+include source files, build scripts and documentation.
+
+"Reserved Font Name" refers to any names specified as such after the
+copyright statement(s).
+
+"Original Version" refers to the collection of Font Software components as
+distributed by the Copyright Holder(s).
+
+"Modified Version" refers to any derivative made by adding to, deleting,
+or substituting -- in part or in whole -- any of the components of the
+Original Version, by changing formats or by porting the Font Software to a
+new environment.
+
+"Author" refers to any designer, engineer, programmer, technical
+writer or other person who contributed to the Font Software.
+
+PERMISSION AND CONDITIONS
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Font Software, to use, study, copy, merge, embed, modify,
+redistribute, and sell modified and unmodified copies of the Font
+Software, subject to the following conditions:
+
+1) Neither the Font Software nor any of its individual components,
+in Original or Modified Versions, may be sold by itself.
+
+2) Original or Modified Versions of the Font Software may be bundled,
+redistributed and/or sold with any software, provided that each copy
+contains the above copyright notice and this license. These can be
+included either as stand-alone text files, human-readable headers or
+in the appropriate machine-readable metadata fields within text or
+binary files as long as those fields can be easily viewed by the user.
+
+3) No Modified Version of the Font Software may use the Reserved Font
+Name(s) unless explicit written permission is granted by the corresponding
+Copyright Holder. This restriction only applies to the primary font name as
+presented to the users.
+
+4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
+Software shall not be used to promote, endorse or advertise any
+Modified Version, except to acknowledge the contribution(s) of the
+Copyright Holder(s) and the Author(s) or with their explicit written
+permission.
+
+5) The Font Software, modified or unmodified, in part or in whole,
+must be distributed entirely under this license, and must not be
+distributed under any other license. The requirement for fonts to
+remain under this license does not apply to any document created
+using the Font Software.
+
+TERMINATION
+This license becomes null and void if any of the above conditions are
+not met.
+
+DISCLAIMER
+THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
+COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
+DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
+OTHER DEALINGS IN THE FONT SOFTWARE.
diff --git a/build.rs b/build.rs
index 570f4bc..6010b44 100644
--- a/build.rs
+++ b/build.rs
@@ -29,6 +29,8 @@ fn main() {
// overlay runtime), so skip the line minifier and let gzip do it.
("dock.js", false),
("overlay-runtime.js", false),
+ // Integrations UI: template-literal heavy like the dock.
+ ("integrations.js", false),
] {
let src_path = web_dir.join(name);
println!("cargo:rerun-if-changed={}", src_path.display());
diff --git a/obs/instantclone-vod-track.lua b/obs/instantclone-vod-track.lua
index 517f3cc..0347dc8 100644
--- a/obs/instantclone-vod-track.lua
+++ b/obs/instantclone-vod-track.lua
@@ -25,18 +25,38 @@
-- next start, on unload) is exactly what raced OBS and crashed
-- (c0000005 in receive_audio / obs_encoder_release). So: no encoder state
-- kept, no deferred releases, nothing to clean up.
+--
+-- ONE EXCEPTION (OBS bug, every version 28-32): on OBS's own plain stream
+-- outputs, StartStreaming -> SetupVodTrack -> clear_archive_encoder calls
+-- obs_encoder_release() on the index-1 encoder it only BORROWED from
+-- obs_output_get_audio_encoder. That stray release destroys our encoder
+-- (and obs_encoder_destroy detaches it) before the stream starts, so OBS
+-- sends one audio track. On those outputs we hand our create-reference to
+-- OBS instead of releasing it: the stray release consumes it and the
+-- output is again the sole owner. Enhanced Broadcasting streams through a
+-- separate multitrack output that clear_archive_encoder never touches, so
+-- there we release normally. (With OBS's native Twitch VOD enabled, OBS
+-- replaces our encoder instead; the one idle encoder left over is inert.)
local obs = obslua
local ENCODER_ID = "ffmpeg_aac" -- always-present AAC encoder
local ENCODER_NAME = "InstantClone VOD audio" -- how we recognise our own track
+-- OBS's plain stream outputs (Advanced / Simple output mode), the ones
+-- clear_archive_encoder stray-releases on. See the OWNERSHIP exception.
+local PLAIN_STREAM_OUTPUTS = { adv_stream = true, simple_stream = true }
+
-- ── settings (mirrored from the script properties UI) ────────────────
local enabled = true
local vod_track = 2 -- OBS mixer track (2..6) that carries the VOD audio
local vod_bitrate = 160 -- kbps
local verbose = false -- extra diagnostic logging (off for clean logs)
+-- Whether this stream start attached our track, so the post-start check
+-- can warn when it went missing. A flag only - never an encoder handle.
+local attached = false
+
local function log(msg)
obs.script_log(obs.LOG_INFO, "[InstantClone VOD] " .. msg)
end
@@ -60,6 +80,7 @@ end
-- Called on STREAMING_STARTING - the pre-start window OBS uses for its own
-- native VOD setup, so the encoder is picked up when the output starts.
local function on_streaming_starting()
+ attached = false
local output = obs.obs_frontend_get_streaming_output()
if output == nil then
log("no streaming output at start - cannot attach VOD track")
@@ -85,10 +106,11 @@ local function on_streaming_starting()
return
end
- -- Create a fresh encoder, attach it, and immediately release our
- -- create-reference. The output now owns it (and releases the previous
- -- index-1 encoder as part of the set). We keep no reference and never
- -- release it again - see the OWNERSHIP note.
+ -- Create a fresh encoder, attach it, and give up our create-reference:
+ -- released here, or on a plain output left for OBS's stray release.
+ -- The output now owns it (and releases the previous index-1 encoder as
+ -- part of the set). We keep no reference and never release it again -
+ -- see the OWNERSHIP note.
local settings = obs.obs_data_create()
obs.obs_data_set_int(settings, "bitrate", vod_bitrate)
local enc = obs.obs_audio_encoder_create(ENCODER_ID, ENCODER_NAME, settings, vod_track - 1, nil)
@@ -100,14 +122,19 @@ local function on_streaming_starting()
end
obs.obs_encoder_set_audio(enc, obs.obs_get_audio())
obs.obs_output_set_audio_encoder(output, enc, 1)
- obs.obs_encoder_release(enc) -- balance our create-ref; the output owns it now
+ if not PLAIN_STREAM_OUTPUTS[obs.obs_output_get_name(output)] then
+ obs.obs_encoder_release(enc) -- balance our create-ref; the output owns it now
+ end
+ attached = true
log(string.format("VOD track attached: OBS track %d -> stream audio 2 @ %d kbps", vod_track, vod_bitrate))
obs.obs_output_release(output)
end
--- Diagnostic only (verbose): confirm our track is at index 1 after start.
+-- Diagnostic: confirm our track is still at index 1 after start. Always
+-- warns when an attached track went missing (the stream then carries one
+-- audio track and Twitch has no VOD audio); the full line is verbose-only.
local function verify_after_start()
- if not verbose then
+ if not (attached or verbose) then
return
end
local output = obs.obs_frontend_get_streaming_output()
@@ -116,6 +143,11 @@ local function verify_after_start()
end
local present, ours = inspect_index_1(output)
vlog(string.format("post-start check: idx1 present=%s ours=%s", tostring(present), tostring(ours)))
+ if attached and not present then
+ log("WARNING: VOD track missing after stream start - this stream has no VOD audio")
+ elseif attached and not ours then
+ log("OBS's native VOD track replaced ours - using OBS's VOD track")
+ end
obs.obs_output_release(output)
end
diff --git a/scripts/subset_slate_fonts.py b/scripts/subset_slate_fonts.py
new file mode 100644
index 0000000..976a3bb
--- /dev/null
+++ b/scripts/subset_slate_fonts.py
@@ -0,0 +1,73 @@
+"""Build the Inter subsets the crash-protection reconnect screen draws with.
+
+Keeps Basic Latin, Latin-1 and common punctuation, drops hinting and
+OpenType layout, and decomposes composite glyphs so the built-in
+rasterizer (src/slate/font.rs) only ever sees simple quadratic outlines.
+
+Usage:
+ pip install fonttools
+ python scripts/subset_slate_fonts.py path/to/Inter-4.1/extras/ttf
+
+Inter is by Rasmus Andersson, SIL Open Font License 1.1
+(assets/fonts/OFL.txt). Download: https://github.com/rsms/inter/releases
+"""
+
+import sys
+from pathlib import Path
+
+from fontTools import subset
+from fontTools.pens.recordingPen import DecomposingRecordingPen
+from fontTools.pens.ttGlyphPen import TTGlyphPen
+from fontTools.ttLib import TTFont
+
+WEIGHTS = ("Regular", "SemiBold")
+UNICODES = (
+ list(range(0x20, 0x7F))
+ + list(range(0xA0, 0x100))
+ + [0x2013, 0x2014, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2026]
+)
+OUT_DIR = Path(__file__).resolve().parent.parent / "assets" / "fonts"
+
+
+def subset_font(source: Path) -> TTFont:
+ options = subset.Options()
+ options.hinting = False
+ options.layout_features = []
+ options.name_IDs = [0, 1, 2, 3, 4, 5, 6, 13, 14]
+ options.notdef_outline = True
+ font = TTFont(source)
+ subsetter = subset.Subsetter(options)
+ subsetter.populate(unicodes=UNICODES)
+ subsetter.subset(font)
+ return font
+
+
+def decompose_composites(font: TTFont) -> None:
+ glyph_set = font.getGlyphSet()
+ glyf = font["glyf"]
+ for name in font.getGlyphOrder():
+ if not glyf[name].isComposite():
+ continue
+ recording = DecomposingRecordingPen(glyph_set)
+ glyph_set[name].draw(recording)
+ pen = TTGlyphPen(None)
+ recording.replay(pen)
+ glyf[name] = pen.glyph()
+ glyf[name].recalcBounds(glyf)
+
+
+def main() -> None:
+ if len(sys.argv) != 2:
+ sys.exit(__doc__)
+ source_dir = Path(sys.argv[1])
+ OUT_DIR.mkdir(parents=True, exist_ok=True)
+ for weight in WEIGHTS:
+ font = subset_font(source_dir / f"Inter-{weight}.ttf")
+ decompose_composites(font)
+ target = OUT_DIR / f"Inter-{weight}-slate.ttf"
+ font.save(target)
+ print(f"{target.name}: {target.stat().st_size} bytes")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/auth.rs b/src/auth.rs
index c566f2c..cc6ed9b 100644
--- a/src/auth.rs
+++ b/src/auth.rs
@@ -254,23 +254,105 @@ mod tests {
assert!(a.begin_login_attempt(ip).is_ok());
}
+ /// The lockout `client` is currently serving, read back through `check`.
+ /// Measured just after the failure that armed it, so it sits a hair under
+ /// the armed value; `assert_lockout` allows for that.
+ fn remaining_lockout(r: &RateLimiter, client: &str) -> Duration {
+ r.check(client).expect_err("client should be locked out")
+ }
+
+ fn assert_lockout(r: &RateLimiter, client: &str, want_secs: u64) {
+ let left = remaining_lockout(r, client);
+ let want = Duration::from_secs(want_secs);
+ assert!(
+ left <= want && left > want - Duration::from_secs(1),
+ "expected a ~{want_secs}s lockout, got {left:?}"
+ );
+ }
+
+ /// Each failure past the threshold doubles the lockout until it reaches
+ /// `max_lockout`, where it stays. Read back without sleeping: `check`
+ /// reports how long is left, which right after arming is the armed value.
#[test]
- fn check_and_count_counts_at_check_time() {
- // The gate must count the attempt itself, so concurrent callers can't
- // all pass before any failure lands (the TOCTOU the login await opened).
+ fn lockout_doubles_past_the_threshold_up_to_the_cap() {
let r = RateLimiter::new(
- 3,
- Duration::from_secs(30),
- Duration::from_secs(600),
- Duration::from_secs(600),
+ 2,
+ Duration::from_secs(10),
+ Duration::from_secs(45),
+ Duration::from_secs(3600),
);
let c = "peer";
- assert!(r.check_and_count(c).is_ok()); // 1
- assert!(r.check_and_count(c).is_ok()); // 2
- assert!(r.check_and_count(c).is_ok()); // 3 -> arms the lock
- assert!(r.check_and_count(c).is_err()); // 4 -> locked, no verify happens
- r.record_success(c);
- assert!(r.check_and_count(c).is_ok()); // cleared
+ r.record_failure(c);
+ assert!(r.check(c).is_ok(), "one failure is under the threshold");
+ for want in [10, 20, 40, 45, 45] {
+ r.record_failure(c);
+ assert_lockout(&r, c, want);
+ }
+ }
+
+ /// A lockout never outlasts the forgiveness window. `check` forgets an
+ /// entry once it is older than the window, so a longer lock would be
+ /// wiped early and the two settings would disagree about when a client
+ /// may try again.
+ #[test]
+ fn lockout_is_clamped_to_the_forgiveness_window() {
+ let r = RateLimiter::new(
+ 1,
+ Duration::from_secs(10),
+ Duration::from_secs(3600),
+ Duration::from_secs(25),
+ );
+ let c = "peer";
+ for want in [10, 20, 25, 25] {
+ r.record_failure(c);
+ assert_lockout(&r, c, want);
+ }
+ }
+
+ /// A client that keeps failing must not overflow the doubling. Without the
+ /// cap on the shift, the 33rd failure past the threshold is `1u32 << 32`:
+ /// a panic in a debug build, and in release a shift that wraps back to a
+ /// short lockout, handing the most persistent guesser the fastest retries.
+ #[test]
+ fn endless_failures_do_not_overflow_the_doubling() {
+ let day = Duration::from_secs(24 * 3600);
+ let r = RateLimiter::new(1, Duration::from_secs(1), day, day);
+ let c = "peer";
+ for _ in 0..100 {
+ r.record_failure(c);
+ }
+ // The shift is capped at 6: base * 64.
+ assert_lockout(&r, c, 64);
+ }
+
+ /// The tracked-client map is bounded, and when a flood fills it the
+ /// oldest entry is the one dropped. `MAX_TRACKED` is private to
+ /// `bump_failure`; the 8192 here mirrors it.
+ #[test]
+ fn a_full_table_evicts_the_oldest_client() {
+ const MAX_TRACKED: usize = 8192;
+ let r = RateLimiter::new(
+ 1,
+ Duration::from_secs(60),
+ Duration::from_secs(60),
+ Duration::from_secs(3600),
+ );
+ r.record_failure("oldest");
+ // Make every later entry strictly newer than "oldest" so it is the
+ // unique minimum. A busy-wait on the clock, well under a microsecond,
+ // rather than a sleep.
+ let armed = Instant::now();
+ while Instant::now() <= armed {}
+ for i in 1..MAX_TRACKED {
+ r.record_failure(&format!("client-{i}"));
+ }
+ assert_eq!(r.attempts.lock().len(), MAX_TRACKED, "table is full");
+
+ r.record_failure("newcomer");
+ assert_eq!(r.attempts.lock().len(), MAX_TRACKED, "table stays bounded");
+ assert!(r.check("oldest").is_ok(), "the oldest entry was evicted");
+ assert!(r.check("client-1").is_err(), "newer entries are kept");
+ assert!(r.check("newcomer").is_err(), "the newcomer is tracked");
}
#[test]
diff --git a/src/autostart.rs b/src/autostart.rs
index 8f70bd9..339f373 100644
--- a/src/autostart.rs
+++ b/src/autostart.rs
@@ -43,8 +43,15 @@ fn run_command() -> io::Result {
/// means the same thing to the user either way.
#[cfg(windows)]
pub fn is_enabled() -> bool {
+ is_enabled_under(RUN_KEY)
+}
+
+/// `is_enabled` against any HKCU subkey. Split out so the round-trip test
+/// can use a private key instead of the developer's real Run key.
+#[cfg(windows)]
+fn is_enabled_under(subkey: &str) -> bool {
use windows_sys::Win32::System::Registry::{RegGetValueW, HKEY_CURRENT_USER, RRF_RT_REG_SZ};
- let subkey = wide(RUN_KEY);
+ let subkey = wide(subkey);
let name = wide(VALUE_NAME);
// Null output buffer + null size: asks only "does this value exist,
// and is it a string?" without reading the data back.
@@ -69,10 +76,17 @@ pub fn is_enabled() -> bool {
/// of pointing at an executable that no longer exists.
#[cfg(windows)]
pub fn set(enabled: bool) -> io::Result<()> {
+ set_under(RUN_KEY, enabled)
+}
+
+/// `set` against any HKCU subkey. Split out so the round-trip test can use
+/// a private key instead of the developer's real Run key.
+#[cfg(windows)]
+fn set_under(subkey: &str, enabled: bool) -> io::Result<()> {
use windows_sys::Win32::System::Registry::{
RegDeleteKeyValueW, RegSetKeyValueW, HKEY_CURRENT_USER, REG_SZ,
};
- let subkey = wide(RUN_KEY);
+ let subkey = wide(subkey);
let name = wide(VALUE_NAME);
let rc = if enabled {
@@ -297,29 +311,47 @@ mod tests {
);
}
- /// Round-trip against the real registry. Writes under HKCU, which
- /// needs no elevation, and restores whatever state the machine was in
- /// so running the suite never changes the developer's own startup.
+ /// Private HKCU key the round-trip test owns outright. A single level so
+ /// deleting it leaves nothing behind, and never the real Run key: a test
+ /// that touched that could rewrite or drop the developer's own startup
+ /// entry, and a panic midway would leave it that way.
+ const TEST_KEY: &str = r"Software\InstantCloneAutostartTest";
+
+ /// Deletes `TEST_KEY` on drop, so a failed assertion still cleans up.
+ struct TestKeyGuard;
+
+ impl Drop for TestKeyGuard {
+ fn drop(&mut self) {
+ use windows_sys::Win32::System::Registry::{RegDeleteTreeW, HKEY_CURRENT_USER};
+ let subkey = wide(TEST_KEY);
+ unsafe { RegDeleteTreeW(HKEY_CURRENT_USER, subkey.as_ptr()) };
+ }
+ }
+
+ /// Round-trip through the same code path `set`/`is_enabled` use, but
+ /// against a private key, so running the suite never changes the
+ /// developer's own startup.
#[test]
fn enable_disable_round_trip() {
- let original = is_enabled();
+ let _cleanup = TestKeyGuard;
+ assert!(
+ !is_enabled_under(TEST_KEY),
+ "stale test key from an earlier run"
+ );
- set(true).expect("enable");
- assert!(is_enabled(), "value must exist after enabling");
+ set_under(TEST_KEY, true).expect("enable");
+ assert!(
+ is_enabled_under(TEST_KEY),
+ "value must exist after enabling"
+ );
- set(false).expect("disable");
- assert!(!is_enabled(), "value must be gone after disabling");
+ set_under(TEST_KEY, false).expect("disable");
+ assert!(
+ !is_enabled_under(TEST_KEY),
+ "value must be gone after disabling"
+ );
// Disabling an already-absent value is a no-op, not an error.
- set(false).expect("disable twice");
-
- if original {
- set(true).expect("restore");
- }
- assert_eq!(
- is_enabled(),
- original,
- "test must leave startup state as found"
- );
+ set_under(TEST_KEY, false).expect("disable twice");
}
}
diff --git a/src/buffer.rs b/src/buffer.rs
index 5b2a4c2..6b63b06 100644
--- a/src/buffer.rs
+++ b/src/buffer.rs
@@ -83,6 +83,11 @@ struct RingInner {
/// `index`: every IDR push_back also push_backs here; every eviction
/// also pops the IDR-front if it matches by seq.
idr_index: VecDeque,
+ /// Payload bytes of every tag in `index`, kept in step with it. Near
+ /// `capacity` means the ring is full: new tags are overwriting the
+ /// oldest ones, so the buffer reaches back no further (see
+ /// `is_saturated`).
+ indexed_bytes: u64,
}
impl DiskRing {
@@ -122,6 +127,7 @@ impl DiskRing {
next_seq: 0,
index: VecDeque::with_capacity(65_536),
idr_index: VecDeque::with_capacity(2_048),
+ indexed_bytes: 0,
}),
video_seq_headers: Mutex::new(std::collections::BTreeMap::new()),
audio_seq_headers: Mutex::new(std::collections::BTreeMap::new()),
@@ -194,6 +200,7 @@ impl DiskRing {
while let Some(front) = inner.index.front().copied() {
if write_overlaps(offset, len, self.capacity, front.offset, front.len as u64) {
inner.index.pop_front();
+ inner.indexed_bytes -= u64::from(front.len);
if front.is_idr {
// Front of idr_index MUST be this same IDR - both
// queues are time-ordered and we only ever push at
@@ -237,6 +244,7 @@ impl DiskRing {
is_idr,
};
inner.index.push_back(meta);
+ inner.indexed_bytes += len;
if is_idr && crate::h264::is_primary_video_idr(payload) {
// Only the PRIMARY track's IDRs become cut candidates. Each
// EB ladder rung has its own IDR cadence; OBS aligns them
@@ -385,39 +393,46 @@ impl DiskRing {
}
}
- /// Return the most recent IDR (used to seed the egress state cleanly
- /// without walking the entire index entry-by-entry).
- pub fn newest_idr(&self) -> Option {
+ /// The newest primary-track IDR stamped at or before `ts`, or None when
+ /// every IDR in the ring is newer. A delay lands here so viewers are
+ /// never closer to live than asked: the nearest IDR can be the newer one.
+ pub fn newest_idr_at_or_before(&self, ts: u64) -> Option {
let inner = self.inner.lock();
- inner.index.iter().rev().find(|m| m.is_idr).copied()
+ let pos = inner.idr_index.partition_point(|m| m.ts_ms <= ts);
+ pos.checked_sub(1)
+ .and_then(|below| inner.idr_index.get(below).copied())
+ }
+
+ /// Return the most recent primary-track IDR (used to seed the egress
+ /// state cleanly without walking the entire index entry-by-entry).
+ ///
+ /// This and the two lookups below read `idr_index`, not `index`: a
+ /// tag's `is_idr` is also true for an Enhanced Broadcasting rung's
+ /// keyframe, which OBS sends right after the primary's for the same
+ /// frame. Seeding on that one starts a horizontal destination one tag
+ /// past its own keyframe, on P-frames with no reference.
+ pub fn newest_idr(&self) -> Option {
+ self.inner.lock().idr_index.back().copied()
}
- /// Most recent IDR whose seq is strictly greater than `min_seq`.
- /// Used after a publisher reconnect to skip stale IDRs from the
- /// previous session that still happen to live in the ring.
+ /// Most recent primary IDR whose seq is strictly greater than
+ /// `min_seq`. Used after a publisher reconnect to skip stale IDRs from
+ /// the previous session that still happen to live in the ring.
pub fn newest_idr_after(&self, min_seq: u64) -> Option {
let inner = self.inner.lock();
- inner
- .index
- .iter()
- .rev()
- .find(|m| m.is_idr && m.seq > min_seq)
- .copied()
+ inner.idr_index.back().filter(|m| m.seq > min_seq).copied()
}
- /// OLDEST IDR whose seq is >= `min_seq`. Used by the egress pump
- /// after eviction skip-ahead - landing on a random P-frame would
+ /// OLDEST primary IDR whose seq is >= `min_seq`. Used by the egress
+ /// pump after eviction skip-ahead - landing on a random P-frame would
/// stream P-frames that reference absent reference frames and the
/// player would show macroblocking until the next IDR. Returning
/// the *earliest* IDR at or after the skip target loses the least
/// content while keeping the decode chain valid.
pub fn oldest_idr_at_or_after(&self, min_seq: u64) -> Option {
let inner = self.inner.lock();
- inner
- .index
- .iter()
- .find(|m| m.is_idr && m.seq >= min_seq)
- .copied()
+ let pos = inner.idr_index.partition_point(|m| m.seq < min_seq);
+ inner.idr_index.get(pos).copied()
}
/// Seq of the most recently appended tag, or None if the ring is empty.
@@ -446,6 +461,7 @@ impl DiskRing {
}
if front.ts_ms < cutoff {
inner.index.pop_front();
+ inner.indexed_bytes -= u64::from(front.len);
// Keep the IDR-only index in sync - same defensive front
// check as the byte-overlap eviction path in `append`.
if front.is_idr && inner.idr_index.front().map(|m| m.seq) == Some(front.seq) {
@@ -477,6 +493,15 @@ impl DiskRing {
let mut inner = self.inner.lock();
inner.index.clear();
inner.idr_index.clear();
+ inner.indexed_bytes = 0;
+ }
+
+ /// Whether the ring is full: the tags it holds fill all but the last
+ /// 1/16 of it, so every new tag overwrites an old one and the buffer
+ /// reaches back no further at this bitrate. A delay longer than that
+ /// can never fill; the controller then delays by what the ring holds.
+ pub fn is_saturated(&self) -> bool {
+ self.inner.lock().indexed_bytes >= self.capacity - self.capacity / 16
}
}
@@ -683,41 +708,6 @@ mod tests {
assert_eq!(*t.0.metadata.lock(), Some(b"onMetaData".to_vec()));
}
- #[test]
- fn oversized_tag_is_rejected_silently() {
- // Capacity 4096, tag of 3000 bytes (> cap/2) - must be dropped.
- let t = tmp(4096);
- let big = vec![0u8; 3000];
- let r = t.0.append(9, 0, &big, false, false).unwrap();
- assert!(r.is_none());
- assert_eq!(t.0.front_seq(), None, "ring must remain empty");
- }
-
- #[test]
- fn wrapping_write_evicts_oldest_and_reads_correctly() {
- // Capacity 256. Write 6 × 80 = 480 bytes total - wraps the cursor
- // twice. Oldest tags get evicted; we read the latest one back.
- let t = tmp(256);
- let mut last_seq = 0;
- for i in 0..6 {
- let payload = vec![(i + 1) as u8; 80];
- last_seq =
- t.0.append(9, i as u64, &payload, false, false)
- .unwrap()
- .unwrap();
- }
- // Front is no longer seq=0
- let front = t.0.front_seq().unwrap();
- assert!(front > 0, "oldest tags must have been evicted");
-
- let mut buf = Vec::new();
- assert!(t.0.try_read_seq(last_seq, &mut buf).unwrap().is_some());
- assert_eq!(buf, vec![6u8; 80]);
-
- // The evicted oldest seq must read as None (it's gone).
- assert!(t.0.try_read_seq(0, &mut buf).unwrap().is_none());
- }
-
#[test]
fn find_idr_near_picks_closest() {
let t = tmp(8192);
@@ -818,45 +808,6 @@ mod tests {
assert!(t.0.try_read_seq(u64::MAX, &mut buf).unwrap().is_none());
}
- #[test]
- fn tag_exactly_at_half_capacity_is_accepted() {
- // The rejection threshold is `> cap/2`. A tag exactly at cap/2
- // bytes must be accepted (boundary check, not off-by-one).
- let t = tmp(2048);
- let payload = vec![0u8; 1024]; // exactly cap/2
- let r = t.0.append(9, 100, &payload, false, false).unwrap();
- assert!(
- r.is_some(),
- "tag at cap/2 must be accepted, not the rejection branch"
- );
- }
-
- #[test]
- fn ring_survives_multiple_full_wraps() {
- // Cap 512. Write 30 tags of 80 bytes = 2400 bytes total, ~4.7×
- // the capacity. The ring must keep the most recent tag readable
- // and the index must not corrupt itself across multiple wraps.
- let t = tmp(512);
- let mut last_seq = 0;
- for i in 0..30u32 {
- let payload = vec![i as u8; 80];
- last_seq =
- t.0.append(9, (i * 100) as u64, &payload, false, false)
- .unwrap()
- .unwrap();
- }
- let mut buf = Vec::new();
- let r = t.0.try_read_seq(last_seq, &mut buf).unwrap();
- assert!(r.is_some());
- assert_eq!(
- buf.first(),
- Some(&29u8),
- "latest tag's bytes must be intact"
- );
- // Front seq is well past 0 (many evictions happened)
- assert!(t.0.front_seq().unwrap() > 20);
- }
-
#[test]
fn all_idr_queries_return_none_on_empty_ring() {
// Every IDR-lookup variant must early-return None instead of
@@ -880,6 +831,62 @@ mod tests {
assert_eq!(t.0.find_idr_near(1000, 0).unwrap().ts_ms, 1000);
}
+ /// A Twitch Enhanced Broadcasting keyframe for a NON-primary ladder
+ /// rung: OneTrack multitrack, FrameType 1, TrackId `track`. Ingest
+ /// stores it with `is_idr = true` (classify_video_tag reads the outer
+ /// FrameType), but it is not a keyframe of the horizontal primary.
+ fn rung_idr_payload(track: u8) -> Vec {
+ let mut v = vec![0x96, 0x01, b'a', b'v', b'c', b'1', track];
+ v.resize(40, 0);
+ v
+ }
+
+ #[test]
+ fn idr_seeds_are_primary_track_keyframes_only() {
+ // With EB, OBS sends the primary's IDR and then each rung's IDR for
+ // the same frame. Seeding a Track(0) destination on a rung's IDR
+ // starts it one tag PAST its own keyframe, so it opens on primary
+ // P-frames with no reference: exactly the glitch the primary-only
+ // `idr_index` exists to prevent, reintroduced by the seed lookups
+ // that walked `index` for any `is_idr`.
+ let t = tmp(8192);
+ let p_frame = [0x27u8; 40];
+ let first =
+ t.0.append(9, 1000, &primary_idr_payload(40), true, false)
+ .unwrap()
+ .unwrap();
+ let rung =
+ t.0.append(9, 1000, &rung_idr_payload(1), true, false)
+ .unwrap()
+ .unwrap();
+ t.0.append(9, 1033, &p_frame, false, false).unwrap();
+ let second =
+ t.0.append(9, 2000, &primary_idr_payload(40), true, false)
+ .unwrap()
+ .unwrap();
+ t.0.append(9, 2000, &rung_idr_payload(2), true, false)
+ .unwrap();
+ t.0.append(9, 2033, &p_frame, false, false).unwrap();
+
+ assert_eq!(t.0.newest_idr().map(|m| m.seq), Some(second));
+ assert_eq!(t.0.newest_idr_after(first).map(|m| m.seq), Some(second));
+ assert!(
+ t.0.newest_idr_after(second).is_none(),
+ "only rung IDRs follow"
+ );
+ assert_eq!(
+ t.0.oldest_idr_at_or_after(rung).map(|m| m.seq),
+ Some(second)
+ );
+ assert_eq!(
+ t.0.oldest_idr_at_or_after(first).map(|m| m.seq),
+ Some(first)
+ );
+ // The rung's own flag is kept: the vertical-canvas pump reads
+ // `meta.is_idr` to find its own canvas's first keyframe.
+ assert!(t.0.find_by_seq(rung).unwrap().1.is_idr);
+ }
+
#[test]
fn newest_idr_after_skips_stale_publisher_idrs() {
// Models the publisher-reconnect case: there are old IDRs in the
@@ -897,16 +904,58 @@ mod tests {
#[cfg(test)]
mod edge_tests {
use super::*;
+ use std::sync::atomic::{AtomicU32, Ordering};
+
+ static UNIQ: AtomicU32 = AtomicU32::new(0);
+
+ /// A ring in its own temp file, deleted on drop - including when an
+ /// assertion fails mid-test. The name is process id plus a counter, so
+ /// tests running in parallel (or two `cargo test` runs) never share a
+ /// file the way a wall-clock name could.
+ struct Ring(DiskRing, std::path::PathBuf);
- fn ring(capacity: u64) -> (DiskRing, std::path::PathBuf) {
- let nanos = std::time::SystemTime::now()
- .duration_since(std::time::UNIX_EPOCH)
- .map(|d| d.as_nanos())
- .unwrap_or(0);
- let path = std::env::temp_dir().join(format!("ic-edge-ring-{nanos}-{capacity}.buf"));
+ impl Drop for Ring {
+ fn drop(&mut self) {
+ let _ = std::fs::remove_file(&self.1);
+ }
+ }
+
+ impl std::ops::Deref for Ring {
+ type Target = DiskRing;
+ fn deref(&self) -> &DiskRing {
+ &self.0
+ }
+ }
+
+ fn ring(capacity: u64) -> Ring {
+ let n = UNIQ.fetch_add(1, Ordering::SeqCst);
+ let path =
+ std::env::temp_dir().join(format!("ic-edge-ring-{}-{}.buf", std::process::id(), n));
let _ = std::fs::remove_file(&path);
- let r = DiskRing::create(&path, capacity).expect("ring create");
- (r, path)
+ Ring(
+ DiskRing::create(&path, capacity).expect("ring create"),
+ path,
+ )
+ }
+
+ /// Legacy AVC keyframe shape, so it passes the primary-IDR gate.
+ fn idr(len: usize) -> Vec {
+ let mut v = vec![0x17];
+ v.resize(len, 0);
+ v
+ }
+
+ /// Payload whose every byte depends on both the tag and the position,
+ /// so a read that mixes two tags or rotates the split cannot pass.
+ fn distinct(tag: u64, len: usize) -> Vec {
+ (0..len).map(|j| (tag * 7 + j as u64 * 13) as u8).collect()
+ }
+
+ fn read(r: &DiskRing, seq: u64) -> Option> {
+ let mut buf = Vec::new();
+ r.try_read_seq(seq, &mut buf)
+ .expect("no io error")
+ .map(|()| buf)
}
/// A tag too big for the ring to hold half of can never be stored
@@ -914,7 +963,7 @@ mod edge_tests {
/// the index. The ring has to stay usable afterwards.
#[test]
fn an_oversized_tag_is_refused_and_leaves_the_ring_usable() {
- let (r, path) = ring(64 * 1024);
+ let r = ring(64 * 1024);
let half = 32 * 1024;
assert_eq!(
@@ -931,31 +980,23 @@ mod edge_tests {
.expect("no io error")
.is_some());
assert_eq!(r.latest_ts(), Some(2_000));
-
- let _ = std::fs::remove_file(&path);
}
/// An empty payload is a degenerate tag, not a reason to panic or to
/// desynchronise the write cursor from the index.
#[test]
fn an_empty_payload_does_not_break_the_cursor() {
- let (r, path) = ring(64 * 1024);
+ let r = ring(64 * 1024);
r.append(9, 1_000, &[], false, false).expect("no io error");
r.append(9, 2_000, &[0x27; 100], false, false)
.expect("no io error");
assert_eq!(r.latest_ts(), Some(2_000));
- let mut buf = Vec::new();
let seq = r.latest_seq().expect("a populated ring");
- assert!(r
- .try_read_seq(seq, &mut buf)
- .expect("no io error")
- .is_some());
assert_eq!(
- buf.len(),
- 100,
+ read(&r, seq).map(|b| b.len()),
+ Some(100),
"the tag after an empty one reads back whole"
);
- let _ = std::fs::remove_file(&path);
}
/// Trim takes a cutoff derived from the newest timestamp. When every tag
@@ -963,7 +1004,7 @@ mod edge_tests {
/// clock jump from emptying the buffer.
#[test]
fn trim_keeps_everything_newer_than_its_cutoff() {
- let (r, path) = ring(64 * 1024);
+ let r = ring(64 * 1024);
for i in 1..=10u64 {
r.append(9, i * 100, &[0x27; 64], false, false)
.expect("no io error");
@@ -975,31 +1016,152 @@ mod edge_tests {
// future timestamp used to trigger.
r.trim_older_than(0, 1_000_000, u64::MAX);
assert_eq!(r.oldest_ts(), None);
- let _ = std::fs::remove_file(&path);
+ }
+
+ /// The one tag whose bytes are split across the end of the file: the
+ /// write and the read each take two syscalls, and both halves have to
+ /// land in the right order at the right offsets.
+ #[test]
+ fn a_tag_straddling_the_end_of_the_file_reads_back_exactly() {
+ let r = ring(256);
+ for tag in 0..4u64 {
+ r.append(9, tag * 10, &distinct(tag, 80), false, false)
+ .expect("no io error");
+ }
+ // Offsets 0, 80, 160, then 240..256 + 0..64, which overwrote tag 0.
+ assert_eq!(read(&r, 0), None, "tag 0's bytes were overwritten");
+ for tag in 1..4u64 {
+ assert_eq!(read(&r, tag), Some(distinct(tag, 80)), "tag {tag}");
+ }
}
/// Writes wrap the file. The index has to keep pointing at bytes that
- /// still belong to the tag it names, across a full lap of the ring.
+ /// still belong to the tag it names, across several laps of the ring.
+ /// 300-byte tags do not divide the capacity, so the split point moves
+ /// on every lap, and every live tag is checked after every append.
#[test]
- fn tags_read_back_correctly_after_the_ring_laps() {
- let (r, path) = ring(16 * 1024);
- let mut last_seq = 0;
- for i in 1..=200u64 {
- let payload = vec![(i % 251) as u8; 300];
- if let Some(seq) = r.append(9, i * 10, &payload, false, false).expect("no io") {
- last_seq = seq;
+ fn every_live_tag_reads_back_exactly_across_several_laps() {
+ let r = ring(16 * 1024);
+ for tag in 0..120u64 {
+ let seq = r
+ .append(9, tag * 10, &distinct(tag, 300), false, false)
+ .expect("no io error")
+ .expect("fits");
+ assert_eq!(seq, tag);
+ let front = r.front_seq().expect("a populated ring");
+ assert!(
+ (seq + 1 - front) * 300 <= 16 * 1024,
+ "live tags fit the file"
+ );
+ for live in front..=seq {
+ assert_eq!(read(&r, live), Some(distinct(live, 300)), "tag {live}");
}
}
- let mut buf = Vec::new();
- assert!(r
- .try_read_seq(last_seq, &mut buf)
- .expect("no io error")
- .is_some());
- assert_eq!(buf.len(), 300);
+ let front = r.front_seq().expect("a populated ring");
+ assert!(front > 60, "two laps evicted the early tags");
+ assert_eq!(read(&r, front - 1), None, "an evicted tag reads as gone");
+ }
+
+ /// Both ways a keyframe leaves the index - overwritten by a wrap, or
+ /// aged out by trim - must also take it out of the cut-point index, or
+ /// a cut would seek to bytes that now belong to another tag.
+ #[test]
+ fn evicted_keyframes_are_never_returned_as_cut_points() {
+ let r = ring(256);
+ r.append(9, 1_000, &idr(80), true, false)
+ .expect("no io error");
+ for i in 1..4u64 {
+ r.append(9, 1_000 + i * 33, &[0x27; 80], false, false)
+ .expect("no io error");
+ }
+ assert!(r.find_idr_near(1_000, 10_000).is_none(), "wrapped away");
+ assert!(r.newest_idr().is_none());
+ assert!(r.oldest_idr_at_or_after(0).is_none());
+
+ let r = ring(64 * 1024);
+ r.append(9, 100, &idr(40), true, false)
+ .expect("no io error");
+ r.append(9, 200, &[0x27; 40], false, false)
+ .expect("no io error");
+ r.append(9, 5_000, &idr(40), true, false)
+ .expect("no io error");
+ r.append(9, 5_100, &[0x27; 40], false, false)
+ .expect("no io error");
+ r.trim_older_than(1_000, 5_100, u64::MAX);
+ assert_eq!(r.oldest_ts(), Some(5_000));
+ assert!(r.find_idr_near(100, 1_000).is_none(), "trimmed away");
+ assert_eq!(r.find_idr_near(100, 10_000).map(|m| m.ts_ms), Some(5_000));
+ assert_eq!(r.oldest_idr_at_or_after(0).map(|m| m.ts_ms), Some(5_000));
+ }
+
+ /// `clear` runs when a new publisher takes over. Seqs keep counting
+ /// from the old high-water mark so a destination's saved position can
+ /// never alias a new tag, and nothing from the old session - bytes or
+ /// cut points - may surface again.
+ #[test]
+ fn clear_forgets_the_old_session_but_keeps_counting_seqs() {
+ let r = ring(64 * 1024);
+ r.append(9, 90_000, &idr(40), true, false)
+ .expect("no io error");
+ r.append(9, 90_033, &[0x27; 40], false, false)
+ .expect("no io error");
+ r.clear();
+
+ assert_eq!(r.latest_seq(), None);
+ assert!(r.newest_idr().is_none());
+ assert!(r.find_idr_near(90_000, 1_000).is_none());
+ assert_eq!(read(&r, 0), None);
+
+ let seq = r.append(9, 10, &[0x27; 40], false, false).unwrap().unwrap();
+ assert_eq!(seq, 2, "continues above the old high-water mark");
+ assert_eq!(read(&r, 1), None, "old seqs stay gone");
assert!(
- buf.iter().all(|b| *b == buf[0]),
- "the newest tag read back as a mix of two writes"
+ r.newest_idr_after(1).is_none(),
+ "no keyframe in this session yet"
);
- let _ = std::fs::remove_file(&path);
+ let key = r.append(9, 20, &idr(40), true, false).unwrap().unwrap();
+ assert_eq!(r.newest_idr_after(1).map(|m| m.seq), Some(key));
+ }
+
+ #[test]
+ fn find_idr_near_before_the_first_keyframe_and_on_a_tie() {
+ let r = ring(64 * 1024);
+ r.append(9, 1_000, &idr(40), true, false)
+ .expect("no io error");
+ r.append(9, 2_000, &idr(40), true, false)
+ .expect("no io error");
+ // Before every keyframe: only the one above can match.
+ assert_eq!(r.find_idr_near(800, 300).map(|m| m.ts_ms), Some(1_000));
+ assert!(r.find_idr_near(800, 100).is_none());
+ // Equidistant: the earlier one wins, so the delay is never less
+ // than asked for when the two are otherwise equal.
+ assert_eq!(r.find_idr_near(1_500, 500).map(|m| m.ts_ms), Some(1_000));
+ assert!(r.find_idr_near(1_500, 499).is_none());
+ }
+
+ /// `is_saturated` follows what the ring holds: filling it past a lap
+ /// trips it (every new tag now overwrites the oldest), and trimming or
+ /// clearing releases it.
+ #[test]
+ fn saturation_follows_what_the_ring_holds() {
+ let r = ring(1_600);
+ let append = |from: u64, to: u64| {
+ for i in from..to {
+ r.append(8, i * 20, &[0u8; 100], false, false).unwrap();
+ }
+ };
+ append(0, 10);
+ assert!(!r.is_saturated(), "1000 of 1600 bytes held");
+ append(10, 40);
+ assert!(
+ r.is_saturated(),
+ "a full lap: each tag overwrites the oldest"
+ );
+ r.trim_older_than(100, 39 * 20, u64::MAX);
+ assert!(!r.is_saturated(), "trimmed to the last 100 ms");
+ append(40, 80);
+ assert!(r.is_saturated(), "full again");
+ r.clear();
+ assert!(!r.is_saturated(), "cleared");
}
}
diff --git a/src/compat.rs b/src/compat.rs
index f4a6f5c..fc1311f 100644
--- a/src/compat.rs
+++ b/src/compat.rs
@@ -95,10 +95,19 @@ fn keyframe_issue(params: &StreamParams, live: &[&Destination]) -> Option Option {
- if params.height == 0 || params.height <= KICK_MAX_HEIGHT {
+ let short_side = if params.width == 0 {
+ params.height
+ } else {
+ params.width.min(params.height)
+ };
+ if short_side == 0 || short_side <= KICK_MAX_HEIGHT {
return None;
}
if !live.iter().any(|d| d.platform == "kick") {
@@ -106,7 +115,7 @@ fn resolution_issue(params: &StreamParams, live: &[&Destination]) -> Option=` line. See `crate::integrations`.
+ pub integrations: Vec,
+ /// Discord webhooks the integrations can post to.
+ pub discord_channels: Vec,
+ /// Phone pushes (ntfy).
+ pub phone: PhoneConnection,
+ /// Whether the old single `discord_webhook_url` has been turned into a
+ /// Discord connection plus a "Stream alerts" integration. False only in
+ /// configs written before integrations existed.
+ pub integrations_migrated: bool,
+ /// The user's own Twitch app (Client ID from dev.twitch.tv), used to log
+ /// in to Twitch instead of the one built into releases. Empty uses the
+ /// built-in app. Not a secret: public apps have no client secret.
+ pub twitch_client_id: String,
+ /// During Twitch Shared Chat, messages from the partner channels'
+ /// viewers trigger chat integrations too. Off: only this channel's.
+ pub twitch_shared_chat: bool,
}
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq)]
pub struct DelayProfile {
pub name: String,
pub delay_ms: u32,
@@ -155,11 +179,12 @@ pub const HK_MOD_CONTROL: u32 = 0x0002;
pub const HK_MOD_SHIFT: u32 = 0x0004;
pub const HK_MOD_WIN: u32 = 0x0008;
-/// The five delay actions a hotkey or MIDI control can drive, in the order
-/// everything else keys off: `Hotkeys::entries`, `MidiBindings::entries`, and
-/// the tray's `RegisterHotKey` ids. Adding an action means adding it here and
-/// to the two structs below, and nothing else has to agree by hand.
-pub const ACTIONS: [&str; 5] = ["toggle", "arm", "activate", "cut", "cut_after"];
+/// The actions a hotkey or MIDI control can drive, in the order everything
+/// else keys off: `Hotkeys::entries`, `MidiBindings::entries`, and the
+/// tray's `RegisterHotKey` ids. Adding an action means adding it here and to
+/// the two structs below, and nothing else has to agree by hand. New actions
+/// go at the end so existing tray ids stay put.
+pub const ACTIONS: [&str; 6] = ["toggle", "arm", "activate", "cut", "cut_after", "end_hold"];
/// Global hotkey bindings, one canonical combo string per delay action
/// (e.g. "Ctrl+Alt+D"; empty means unbound). Windows binds them via
@@ -180,19 +205,22 @@ pub struct Hotkeys {
pub cut: String,
/// Schedule a cut for the moment the current live edge airs.
pub cut_after: String,
+ /// End a crash-protection hold now (every destination ends).
+ pub end_hold: String,
}
impl Hotkeys {
/// Stable (action-name, binding) pairs. One source of truth for the
/// save writer, the JSON serializer, and the tray registrar, so a new
/// action is added in exactly one place.
- pub fn entries(&self) -> [(&'static str, &str); 5] {
+ pub fn entries(&self) -> [(&'static str, &str); 6] {
[
("toggle", &self.toggle),
("arm", &self.arm),
("activate", &self.activate),
("cut", &self.cut),
("cut_after", &self.cut_after),
+ ("end_hold", &self.end_hold),
]
}
@@ -203,6 +231,7 @@ impl Hotkeys {
"activate" => &mut self.activate,
"cut" => &mut self.cut,
"cut_after" => &mut self.cut_after,
+ "end_hold" => &mut self.end_hold,
_ => return None,
})
}
@@ -242,10 +271,10 @@ impl Hotkeys {
}
}
-/// MIDI controller bindings, one signature string per delay action (empty
+/// MIDI controller bindings, one signature string per action (empty
/// means unbound). A signature is `note::` for a note-on pad
/// or `cc::` for a control-change knob/button, channel
-/// 1-16 and data 0-127. Same five actions as `Hotkeys`; the MIDI listener
+/// 1-16 and data 0-127. Same actions as `Hotkeys`; the MIDI listener
/// thread matches an incoming message against these and fires the shared
/// controller action. Windows-only in effect (winmm), round-tripped
/// everywhere.
@@ -256,17 +285,19 @@ pub struct MidiBindings {
pub activate: String,
pub cut: String,
pub cut_after: String,
+ pub end_hold: String,
}
impl MidiBindings {
/// Stable (action-name, signature) pairs, mirroring `Hotkeys::entries`.
- pub fn entries(&self) -> [(&'static str, &str); 5] {
+ pub fn entries(&self) -> [(&'static str, &str); 6] {
[
("toggle", &self.toggle),
("arm", &self.arm),
("activate", &self.activate),
("cut", &self.cut),
("cut_after", &self.cut_after),
+ ("end_hold", &self.end_hold),
]
}
@@ -277,6 +308,7 @@ impl MidiBindings {
"activate" => &mut self.activate,
"cut" => &mut self.cut,
"cut_after" => &mut self.cut_after,
+ "end_hold" => &mut self.end_hold,
_ => return None,
})
}
@@ -533,7 +565,7 @@ fn vk_to_key(vk: u32) -> Option {
/// One streaming destination - Twitch, YouTube, a private server, etc.
/// Each destination runs its own RTMP egress and is paced/cut/timestamp-
/// rewritten independently from the same shared DiskRing.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq)]
pub struct Destination {
pub id: String, // stable across renames; UI key
pub name: String, // user label ("Twitch", "Backup", etc.)
@@ -585,15 +617,15 @@ pub struct Destination {
/// non-Twitch platforms (Twitch gets the native dual-canvas
/// passthrough). One of:
/// "horizontal" (default) - the primary 16:9 canvas (TrackId 0)
- /// "vertical" - the 9:16 canvas Twitch Dual Format /
- /// Enhanced Broadcasting produces, reused
- /// for YouTube Shorts / Kick mobile / etc.
+ /// "vertical" - the 9:16 canvas OBS sends with Enhanced
+ /// Broadcasting (its Additional canvas, or
+ /// Twitch Dual Format), for TikTok /
+ /// YouTube Shorts / Kick mobile.
///
- /// Vertical only has data on the wire while Twitch Dual Format (EB)
- /// is active in OBS. With EB off there is no vertical canvas, so a
- /// vertical destination waits (sends no video) until one appears -
- /// see `h264::detect_vertical_primary_track`. Unknown / empty values
- /// fall back to "horizontal".
+ /// Vertical only has data on the wire while OBS sends that canvas.
+ /// Without it a vertical destination waits (sends no video) until one
+ /// appears - see `h264::detect_vertical_primary_track`. Unknown / empty
+ /// values fall back to "horizontal".
pub stream_format: String,
/// Which OBS audio track(s) this destination receives. Only meaningful
/// once OBS sends a second audio track (the VOD-unlocker script, or
@@ -700,9 +732,121 @@ impl Destination {
pub fn wants_vertical(&self) -> bool {
self.platform != "twitch" && self.stream_format == "vertical"
}
+
+ /// Whether this destination's ingest plays `codec` over Enhanced RTMP.
+ /// H.264 plays everywhere; HEVC and AV1 only where known to work
+ /// (YouTube takes both; Restream took an HEVC vertical live). Kick,
+ /// Trovo and custom servers get H.264, since a wrong guess is a stream
+ /// that never starts.
+ pub fn plays(&self, codec: crate::h264::VideoCodec) -> bool {
+ use crate::h264::VideoCodec;
+ match codec {
+ VideoCodec::Avc => true,
+ VideoCodec::Hevc => matches!(self.platform.as_str(), "youtube" | "restream"),
+ VideoCodec::Av1 => self.platform == "youtube",
+ VideoCodec::Vp9 | VideoCodec::Unknown => false,
+ }
+ }
}
impl Settings {
+ /// `integration.=`, `connection.discord.=` and
+ /// `connection.phone=` lines. Compact JSON has no newline, so each
+ /// stays one line of the key=value format.
+ fn write_integration_lines(&self, f: &mut impl Write) -> io::Result<()> {
+ if self.integrations_migrated {
+ writeln!(f, "integrations_migrated=true")?;
+ }
+ for c in &self.discord_channels {
+ let v = json::obj([("name", json::str(&c.name)), ("url", json::str(&c.url))]);
+ writeln!(
+ f,
+ "connection.discord.{}={}",
+ one_line(&c.id),
+ one_line(&v.to_json())
+ )?;
+ }
+ if self.phone != PhoneConnection::default() {
+ let v = json::obj([
+ ("server", json::str(&self.phone.server)),
+ ("topic", json::str(&self.phone.topic)),
+ ]);
+ writeln!(f, "connection.phone={}", one_line(&v.to_json()))?;
+ }
+ for i in &self.integrations {
+ writeln!(
+ f,
+ "integration.{}={}",
+ one_line(&i.id),
+ one_line(&i.to_json().to_json())
+ )?;
+ }
+ Ok(())
+ }
+
+ /// Load one integration line. A line that no longer parses (a hand edit,
+ /// a newer version's step type) is skipped rather than failing the
+ /// whole config: the other integrations and settings still load.
+ fn load_integration(&mut self, value: &str) {
+ if self.integrations.len() >= MAX_INTEGRATIONS {
+ return;
+ }
+ let Ok(parsed) = json::parse(value) else {
+ return;
+ };
+ let Ok(integration) = Integration::from_json(&parsed) else {
+ return;
+ };
+ if crate::integrations::model::valid_id(&integration.id)
+ && !self.integrations.iter().any(|i| i.id == integration.id)
+ {
+ self.integrations.push(integration);
+ }
+ }
+
+ fn load_discord_channel(&mut self, id: &str, value: &str) {
+ if !crate::integrations::model::valid_id(id)
+ || self.discord_channels.len() >= 32
+ || self.discord_channels.iter().any(|c| c.id == id)
+ {
+ return;
+ }
+ if let Ok(v) = json::parse(value) {
+ self.discord_channels.push(DiscordChannel {
+ id: id.to_string(),
+ name: v.str_or("name", "Discord").to_string(),
+ url: v.str_or("url", "").to_string(),
+ });
+ }
+ }
+
+ /// One-time move of the old single Discord webhook into integrations:
+ /// a Discord connection plus a "Stream alerts" integration that posts
+ /// the same messages as before. Runs once per config.
+ fn migrate_discord_webhook(&mut self) {
+ if self.integrations_migrated {
+ return;
+ }
+ self.integrations_migrated = true;
+ let url = std::mem::take(&mut self.discord_webhook_url);
+ if url.trim().is_empty() {
+ return;
+ }
+ let channel_id = "discord".to_string();
+ if !self.discord_channels.iter().any(|c| c.id == channel_id) {
+ self.discord_channels.push(DiscordChannel {
+ id: channel_id.clone(),
+ name: "Discord".to_string(),
+ url,
+ });
+ }
+ self.integrations
+ .push(crate::integrations::presets::legacy_alerts(
+ "stream-alerts".to_string(),
+ &channel_id,
+ ));
+ }
+
pub fn defaults() -> Self {
Self {
ingest_port: 1935,
@@ -755,7 +899,7 @@ impl Settings {
tracing_enabled: false,
// Behaviour toggles default off so the two-phase
// arm/activate ceremony stays the canonical flow. Streamers
- // who always want delay opt in once via System -> Behavior.
+ // who always want delay opt in once via System -> General.
auto_arm_on_connect: false,
auto_activate_when_ready: false,
// 15 s is the same default the wizard suggests and matches
@@ -778,6 +922,15 @@ impl Settings {
// Likewise no MIDI bindings until the user maps a controller.
midi: MidiBindings::default(),
midi_device: String::new(),
+ crash_protection: CrashProtection::default(),
+ integrations: Vec::new(),
+ discord_channels: Vec::new(),
+ phone: PhoneConnection::default(),
+ // A fresh install has no old webhook to migrate; `load` sets
+ // this after its one-time pass either way.
+ integrations_migrated: false,
+ twitch_client_id: String::new(),
+ twitch_shared_chat: false,
}
}
@@ -787,7 +940,11 @@ impl Settings {
// must see them stay deleted across restarts.
s.profiles.clear();
s.destinations.clear();
- let text = fs::read_to_string(path)?;
+ // Lossy on purpose: a hand edit saved as ANSI is not valid UTF-8,
+ // and failing the whole read over one accented character would
+ // cost the user every other setting. Only the damaged line changes.
+ let bytes = fs::read(path)?;
+ let text = String::from_utf8_lossy(&bytes);
for line in text.lines() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
@@ -824,6 +981,7 @@ impl Settings {
audio_track: "auto".into(),
});
}
+ s.migrate_discord_webhook();
// Clamp / sanitize on load - hand-edited values can otherwise
// hit divide-by-zero (buffer_mb=0 → `% capacity` in DiskRing) or
// bind two services to the same port (one will silently fail).
@@ -868,11 +1026,57 @@ impl Settings {
}
}
- pub fn load_or_default(path: &Path) -> Self {
- match Self::load(path) {
- Ok(s) => s,
- Err(_) => Self::defaults().with_smart_defaults(),
- }
+ /// Load the config for process startup. Only a missing file means
+ /// "fresh install": that case gets smart defaults, saved at once so the
+ /// file appears for the user to find.
+ ///
+ /// Any other read error (antivirus or a sync client holding the file,
+ /// a permissions problem) is retried briefly. If it persists, the file
+ /// is copied aside to `.unreadable` and startup runs on defaults
+ /// with `may_save == false`, so startup itself never writes over it.
+ /// A later save from the dashboard does replace it, which is safe only
+ /// because the copy exists; when the copy cannot be made either, this
+ /// returns the error and the caller must not start.
+ pub fn load_for_startup(path: &Path) -> io::Result {
+ Self::load_for_startup_with(path, Self::load)
+ }
+
+ /// `load_for_startup` with the loader injected, so tests can simulate a
+ /// locked file without one.
+ fn load_for_startup_with(
+ path: &Path,
+ load: impl Fn(&Path) -> io::Result,
+ ) -> io::Result {
+ let mut attempt = 1;
+ let error = loop {
+ match load(path) {
+ Ok(settings) => return Ok(StartupConfig::loaded(settings)),
+ Err(e) if e.kind() == io::ErrorKind::NotFound => {
+ let settings = Self::defaults().with_smart_defaults();
+ let _ = settings.save(path);
+ return Ok(StartupConfig::loaded(settings));
+ }
+ Err(e) if attempt >= STARTUP_READ_ATTEMPTS => break e,
+ Err(_) => {
+ attempt += 1;
+ std::thread::sleep(STARTUP_READ_RETRY_DELAY);
+ }
+ }
+ };
+ let backup = copy_aside(path, "unreadable")?;
+ Ok(StartupConfig {
+ settings: Self::defaults().with_smart_defaults(),
+ may_save: false,
+ notice: Some(format!(
+ "InstantClone couldn't read its settings file, so it started with \
+ default settings.\n\nFile: {}\nError: {error}\n\nA copy of the file \
+ was saved as:\n{}\n\nClose whatever is holding the file and restart \
+ InstantClone to use your settings again. Until then, any settings \
+ change replaces the file; the copy stays either way.",
+ path.display(),
+ backup.display()
+ )),
+ })
}
/// On a brand-new install, pick smarter starting values than the
@@ -931,6 +1135,11 @@ impl Settings {
// Legacy single-destination fields: kept for backward compat with
// older binaries that don't know about `destination.*`. We mirror
// destinations[0] into them so a downgrade still works.
+ //
+ // With no destinations the key and URL are written empty, never
+ // from the loaded legacy fields: those still hold whatever the
+ // file had, and `load` would migrate them back into an enabled
+ // destination, undoing the user's delete of the last one.
let mirror = self.destinations.first();
writeln!(
f,
@@ -944,20 +1153,12 @@ impl Settings {
writeln!(
f,
"stream_key={}",
- one_line(
- mirror
- .map(|d| d.stream_key.as_str())
- .unwrap_or(&self.stream_key)
- )
+ one_line(mirror.map(|d| d.stream_key.as_str()).unwrap_or(""))
)?;
writeln!(
f,
"custom_egress_url={}",
- one_line(
- mirror
- .map(|d| d.custom_egress_url.as_str())
- .unwrap_or(&self.custom_egress_url)
- )
+ one_line(mirror.map(|d| d.custom_egress_url.as_str()).unwrap_or(""))
)?;
writeln!(f, "ingest_port={}", self.ingest_port)?;
writeln!(f, "ingest_bind_all={}", self.ingest_bind_all)?;
@@ -1037,6 +1238,12 @@ impl Settings {
writeln!(f, "hotkey.{}={}", action, combo)?;
}
}
+ if !self.twitch_client_id.is_empty() {
+ writeln!(f, "twitch_client_id={}", one_line(&self.twitch_client_id))?;
+ }
+ if self.twitch_shared_chat {
+ writeln!(f, "twitch_shared_chat=true")?;
+ }
// MIDI bindings, same only-when-bound rule.
if !self.midi_device.is_empty() {
writeln!(f, "midi_device={}", one_line(&self.midi_device))?;
@@ -1046,6 +1253,9 @@ impl Settings {
writeln!(f, "midi.{}={}", action, sig)?;
}
}
+ // Crash protection: only fields that differ from the default.
+ self.crash_protection.write_lines(&mut f)?;
+ self.write_integration_lines(&mut f)?;
for (i, p) in self.profiles.iter().enumerate() {
writeln!(f, "profile.{}.name={}", i, one_line(&p.name))?;
writeln!(f, "profile.{}.delay_ms={}", i, p.delay_ms)?;
@@ -1187,11 +1397,30 @@ impl Settings {
self.hotkeys.set(&k["hotkey.".len()..], value);
}
"midi_device" => self.midi_device = sanitize_device_name(value),
+ "twitch_client_id" => self.twitch_client_id = value.trim().to_string(),
+ "twitch_shared_chat" => self.twitch_shared_chat = value == "true",
// midi.=. `set` validates and drops anything
// malformed, so a hand-edited config can't load a bad signature.
k if k.starts_with("midi.") => {
self.midi.set(&k["midi.".len()..], value);
}
+ k if k.starts_with(crash_protection::KEY_PREFIX) => {
+ self.crash_protection
+ .set(&k[crash_protection::KEY_PREFIX.len()..], value);
+ }
+ "integrations_migrated" => self.integrations_migrated = value == "true",
+ k if k.starts_with("integration.") => self.load_integration(value),
+ k if k.starts_with("connection.discord.") => {
+ self.load_discord_channel(&k["connection.discord.".len()..], value)
+ }
+ "connection.phone" => {
+ if let Ok(v) = json::parse(value) {
+ self.phone = PhoneConnection {
+ server: v.str_or("server", "").to_string(),
+ topic: v.str_or("topic", "").to_string(),
+ };
+ }
+ }
k if k.starts_with("profile.") => {
let rest = &k[8..];
if let Some(dot) = rest.find('.') {
@@ -1431,20 +1660,22 @@ impl Settings {
// Hotkey bindings. Not secrets, so always emitted; the dashboard
// only surfaces the editor on Windows (where they take effect).
let hotkeys = format!(
- r#"{{"toggle":{t},"arm":{a},"activate":{ac},"cut":{c},"cut_after":{ca}}}"#,
+ r#"{{"toggle":{t},"arm":{a},"activate":{ac},"cut":{c},"cut_after":{ca},"end_hold":{eh}}}"#,
t = json_str(&self.hotkeys.toggle),
a = json_str(&self.hotkeys.arm),
ac = json_str(&self.hotkeys.activate),
c = json_str(&self.hotkeys.cut),
ca = json_str(&self.hotkeys.cut_after),
+ eh = json_str(&self.hotkeys.end_hold),
);
let midi = format!(
- r#"{{"toggle":{t},"arm":{a},"activate":{ac},"cut":{c},"cut_after":{ca}}}"#,
+ r#"{{"toggle":{t},"arm":{a},"activate":{ac},"cut":{c},"cut_after":{ca},"end_hold":{eh}}}"#,
t = json_str(&self.midi.toggle),
a = json_str(&self.midi.arm),
ac = json_str(&self.midi.activate),
c = json_str(&self.midi.cut),
ca = json_str(&self.midi.cut_after),
+ eh = json_str(&self.midi.end_hold),
);
// The two raw credentials are emitted only for a full session; a
// dock-token caller gets them blanked (see the doc comment above).
@@ -1459,7 +1690,7 @@ impl Settings {
""
};
format!(
- r#"{{"configured":{c},"ingest_port":{ip},"ingest_bind_all":{iba},"web_port":{wp},"web_bind_all":{wba},"buffer_mb":{bm},"buffer_path":{bp},"target_delay_ms":{td},"obs_url":{ou},"discord_webhook_url":{dw},"webhook_set":{ws},"overlays_dir":{ov},"tracing_enabled":{te},"auto_arm_on_connect":{aaoc},"auto_activate_when_ready":{aawr},"auto_arm_delay_ms":{aadm},"overlays_seeded":{os},"start_with_windows":{sww},"update_check_enabled":{uce},"open_dashboard_on_launch":{odol},"ingest_key":{ik},"auth_enabled":{ae},"dock_token":{dt},"os":{osname},"version":{ver},"hotkeys":{hotkeys},"midi":{midi},"midi_device":{midid},"destinations":{dests}}}"#,
+ r#"{{"configured":{c},"ingest_port":{ip},"ingest_bind_all":{iba},"web_port":{wp},"web_bind_all":{wba},"buffer_mb":{bm},"buffer_path":{bp},"target_delay_ms":{td},"obs_url":{ou},"discord_webhook_url":{dw},"webhook_set":{ws},"overlays_dir":{ov},"tracing_enabled":{te},"auto_arm_on_connect":{aaoc},"auto_activate_when_ready":{aawr},"auto_arm_delay_ms":{aadm},"overlays_seeded":{os},"start_with_windows":{sww},"update_check_enabled":{uce},"open_dashboard_on_launch":{odol},"ingest_key":{ik},"auth_enabled":{ae},"dock_token":{dt},"os":{osname},"version":{ver},"hotkeys":{hotkeys},"midi":{midi},"midi_device":{midid},"crash_protection":{cp},"twitch_client_id":{tci},"twitch_app_built_in":{tbi},"destinations":{dests}}}"#,
c = self.configured,
sww = start_with_windows,
ik = json_str(ik_shown),
@@ -1488,6 +1719,9 @@ impl Settings {
hotkeys = hotkeys,
midi = midi,
midid = json_str(&self.midi_device),
+ tci = json_str(&self.twitch_client_id),
+ tbi = !crate::integrations::twitch::default_client_id().is_empty(),
+ cp = self.crash_protection.to_json(),
dests = dests,
)
}
@@ -1522,6 +1756,12 @@ impl Settings {
.into(),
);
}
+ if !is_twitch_client_id(&self.twitch_client_id) {
+ errs.push(
+ "That Twitch Client ID doesn't look right: copy the Client ID (letters and numbers) from your app at dev.twitch.tv/console"
+ .into(),
+ );
+ }
if self.destinations.len() > MAX_DESTINATIONS {
errs.push(format!("too many destinations (max {})", MAX_DESTINATIONS));
}
@@ -1532,6 +1772,12 @@ impl Settings {
errs.push("destination is missing a name".into());
continue;
}
+ // A switched-off destination is never streamed, so a half-filled
+ // one (no server URL or key yet) must not block saving anything
+ // else. Switching it on runs the checks below.
+ if !d.enabled {
+ continue;
+ }
if d.platform == "custom" || d.platform == "kick" {
if d.custom_egress_url.is_empty() {
let need = if d.platform == "kick" {
@@ -1793,6 +2039,14 @@ fn os_name() -> &'static str {
}
}
+/// Empty (use the built-in app) or a Twitch Client ID: Twitch issues 30
+/// lowercase letters and digits; a little slack either way in case that
+/// ever changes, but never spaces, quotes or anything else.
+fn is_twitch_client_id(id: &str) -> bool {
+ id.is_empty()
+ || ((20..=64).contains(&id.len()) && id.chars().all(|c| c.is_ascii_alphanumeric()))
+}
+
/// Strip control characters (notably CR/LF) from a value bound for the
/// line-based config file. The format is `key=value` per line, so a newline in
/// a value would inject a spurious second setting on the next load - e.g. a
@@ -1808,7 +2062,7 @@ fn one_line(s: &str) -> std::borrow::Cow<'_, str> {
}
}
-fn json_str(s: &str) -> String {
+pub(crate) fn json_str(s: &str) -> String {
let mut out = String::with_capacity(s.len() + 2);
out.push('"');
for c in s.chars() {
@@ -1826,6 +2080,216 @@ fn json_str(s: &str) -> String {
out
}
+/// Attempts at reading an existing config before startup treats it as
+/// unreadable. Antivirus scans and sync clients hold a file for well under
+/// a second, so a short retry rides those out without delaying a real
+/// failure noticeably.
+const STARTUP_READ_ATTEMPTS: u32 = 4;
+const STARTUP_READ_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(100);
+
+/// What `Settings::load_for_startup` hands to `main`.
+pub struct StartupConfig {
+ pub settings: Settings,
+ /// False when the settings are stand-in defaults for a file that exists
+ /// but could not be read: startup must not save them over it.
+ pub may_save: bool,
+ /// Message the user must see before the app runs, if any.
+ pub notice: Option,
+}
+
+impl StartupConfig {
+ fn loaded(settings: Settings) -> Self {
+ Self {
+ settings,
+ may_save: true,
+ notice: None,
+ }
+ }
+}
+
+/// Copy `path` to the first free `.`, `..2`, ...
+/// so an earlier copy is never overwritten. Returns where it went.
+fn copy_aside(path: &Path, suffix: &str) -> io::Result {
+ let name = path
+ .file_name()
+ .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "config path has no file name"))?
+ .to_string_lossy()
+ .into_owned();
+ for n in 1..=99 {
+ let candidate = if n == 1 {
+ path.with_file_name(format!("{name}.{suffix}"))
+ } else {
+ path.with_file_name(format!("{name}.{suffix}.{n}"))
+ };
+ if !candidate.exists() {
+ fs::copy(path, &candidate)?;
+ return Ok(candidate);
+ }
+ }
+ Err(io::Error::new(
+ io::ErrorKind::AlreadyExists,
+ format!("too many {name}.{suffix} copies already; move some away"),
+ ))
+}
+
+/// Strict JSON validity check (RFC 8259, one value, nothing trailing). The
+/// crate hand-writes every JSON document it emits, and a substring assert
+/// cannot catch a stray comma or an unescaped newline, so tests parse what
+/// they build with this. It also vets JSON read from OBS's files before it
+/// goes back to OBS, which refuses to stream on a malformed config. No
+/// dependency: a JSON crate would be pulled in for a yes or no.
+pub(crate) fn is_valid_json(s: &str) -> bool {
+ let mut parser = json_check::Parser {
+ bytes: s.as_bytes(),
+ pos: 0,
+ };
+ parser.skip_ws();
+ let ok = parser.value(0);
+ parser.skip_ws();
+ ok && parser.pos == parser.bytes.len()
+}
+
+mod json_check {
+ /// Deep enough for anything we emit, shallow enough that a runaway
+ /// nesting bug fails the test instead of overflowing the stack.
+ const MAX_DEPTH: usize = 64;
+
+ pub(super) struct Parser<'a> {
+ pub(super) bytes: &'a [u8],
+ pub(super) pos: usize,
+ }
+
+ impl Parser<'_> {
+ fn peek(&self) -> Option {
+ self.bytes.get(self.pos).copied()
+ }
+
+ fn eat(&mut self, b: u8) -> bool {
+ let hit = self.peek() == Some(b);
+ if hit {
+ self.pos += 1;
+ }
+ hit
+ }
+
+ pub(super) fn skip_ws(&mut self) {
+ while matches!(self.peek(), Some(b' ' | b'\t' | b'\n' | b'\r')) {
+ self.pos += 1;
+ }
+ }
+
+ pub(super) fn value(&mut self, depth: usize) -> bool {
+ if depth > MAX_DEPTH {
+ return false;
+ }
+ match self.peek() {
+ Some(b'{') => self.container(depth, b'}', true),
+ Some(b'[') => self.container(depth, b']', false),
+ Some(b'"') => self.string(),
+ Some(b't') => self.literal(b"true"),
+ Some(b'f') => self.literal(b"false"),
+ Some(b'n') => self.literal(b"null"),
+ _ => self.number(),
+ }
+ }
+
+ /// Object (`keyed`) or array body, after the opening bracket.
+ fn container(&mut self, depth: usize, close: u8, keyed: bool) -> bool {
+ self.pos += 1;
+ self.skip_ws();
+ if self.eat(close) {
+ return true;
+ }
+ loop {
+ self.skip_ws();
+ if keyed {
+ if !self.string() {
+ return false;
+ }
+ self.skip_ws();
+ if !self.eat(b':') {
+ return false;
+ }
+ self.skip_ws();
+ }
+ if !self.value(depth + 1) {
+ return false;
+ }
+ self.skip_ws();
+ if self.eat(close) {
+ return true;
+ }
+ if !self.eat(b',') {
+ return false;
+ }
+ }
+ }
+
+ fn string(&mut self) -> bool {
+ if !self.eat(b'"') {
+ return false;
+ }
+ while let Some(b) = self.peek() {
+ self.pos += 1;
+ match b {
+ b'"' => return true,
+ b'\\' => {
+ let Some(esc) = self.peek() else { return false };
+ self.pos += 1;
+ if esc == b'u' {
+ let hex = self.bytes.get(self.pos..self.pos + 4);
+ if !hex.is_some_and(|h| h.iter().all(u8::is_ascii_hexdigit)) {
+ return false;
+ }
+ self.pos += 4;
+ } else if !b"\"\\/bfnrt".contains(&esc) {
+ return false;
+ }
+ }
+ 0x00..=0x1f => return false,
+ _ => {}
+ }
+ }
+ false
+ }
+
+ fn literal(&mut self, word: &[u8]) -> bool {
+ let hit = self.bytes[self.pos..].starts_with(word);
+ if hit {
+ self.pos += word.len();
+ }
+ hit
+ }
+
+ fn digits(&mut self) -> usize {
+ let start = self.pos;
+ while self.peek().is_some_and(|b| b.is_ascii_digit()) {
+ self.pos += 1;
+ }
+ self.pos - start
+ }
+
+ fn number(&mut self) -> bool {
+ self.eat(b'-');
+ if self.eat(b'0') {
+ // A leading zero stands alone: `01` is not JSON.
+ } else if self.digits() == 0 {
+ return false;
+ }
+ if self.eat(b'.') && self.digits() == 0 {
+ return false;
+ }
+ if self.eat(b'e') || self.eat(b'E') {
+ let _ = self.eat(b'+') || self.eat(b'-');
+ if self.digits() == 0 {
+ return false;
+ }
+ }
+ true
+ }
+ }
+}
+
/// Best-effort free space query. Returns None if the OS doesn't tell us
/// (in which case the caller keeps the default).
fn free_space_bytes(path: &Path) -> Option {
@@ -1955,6 +2419,22 @@ mod tests {
use super::*;
use std::path::PathBuf;
+ /// HEVC or AV1 goes only where it is known to play; a wrong guess is a
+ /// vertical stream that never starts, so the rest get H.264.
+ #[test]
+ fn only_known_platforms_are_sent_hevc_or_av1() {
+ use crate::h264::VideoCodec::{Av1, Avc, Hevc, Unknown};
+ let on = |platform: &str| test_destination("d", platform, "key");
+ for platform in ["youtube", "restream", "kick", "trovo", "custom", "twitch"] {
+ assert!(on(platform).plays(Avc), "{platform} plays H.264");
+ assert!(!on(platform).plays(Unknown), "{platform}");
+ }
+ assert!(on("youtube").plays(Hevc) && on("restream").plays(Hevc));
+ assert!(!on("kick").plays(Hevc) && !on("custom").plays(Hevc));
+ assert!(on("youtube").plays(Av1));
+ assert!(!on("restream").plays(Av1) && !on("kick").plays(Av1));
+ }
+
/// Both legs, IPv4 first, in both bind modes. Order is contractual:
/// `spawn_ingest_legs` treats only the first address as required, so a
/// reordering here would silently make the IPv4 listener give up after
@@ -2405,6 +2885,31 @@ mod tests {
);
}
+ /// A switched-off, half-filled destination must not block saving other
+ /// settings (turning on crash protection failed on one); switching it
+ /// on still needs its fields.
+ #[test]
+ fn a_disabled_incomplete_destination_does_not_block_saving() {
+ let mut s = Settings::defaults();
+ s.destinations.push(Destination {
+ id: "k".into(),
+ name: "test3".into(),
+ enabled: false,
+ platform: "kick".into(),
+ stream_key: String::new(),
+ custom_egress_url: String::new(),
+ twitch_ingest: String::new(),
+ youtube_ingest: String::new(),
+ vod_audio: false,
+ vod_audio_inject_eb: false,
+ stream_format: "horizontal".into(),
+ audio_track: "auto".into(),
+ });
+ assert!(s.validate().is_empty(), "{:?}", s.validate());
+ s.destinations[0].enabled = true;
+ assert!(s.validate().iter().any(|e| e.contains("Kick Server URL")));
+ }
+
#[test]
fn destination_custom_rtmps_url_is_accepted() {
// The Custom platform is the escape hatch for any TLS ingest
@@ -2477,238 +2982,6 @@ mod tests {
);
}
- #[test]
- fn settings_save_load_roundtrip_preserves_destinations_and_profiles() {
- // Build a config that exercises every field the on-disk format
- // touches: ports, paths, multiple destinations including a
- // custom-URL one, multiple profiles, and a webhook URL.
- let mut s = Settings::defaults();
- s.ingest_port = 1936;
- s.web_port = 7898;
- s.buffer_mb = 200;
- s.discord_webhook_url = "https://discord.com/api/webhooks/123/abc".into();
- s.destinations.clear();
- s.destinations.push(Destination {
- id: "tw1".into(),
- name: "Twitch".into(),
- enabled: true,
- platform: "twitch".into(),
- stream_key: "live_secret".into(),
- custom_egress_url: String::new(),
- twitch_ingest: "fra".into(),
- youtube_ingest: String::new(),
- vod_audio: false,
- vod_audio_inject_eb: false,
- stream_format: "horizontal".into(),
- audio_track: "auto".into(),
- });
- s.destinations.push(Destination {
- id: "yt1".into(),
- name: "YT Backup".into(),
- enabled: false,
- platform: "youtube".into(),
- stream_key: "yt_secret".into(),
- custom_egress_url: String::new(),
- twitch_ingest: String::new(),
- youtube_ingest: "backup".into(),
- vod_audio: false,
- vod_audio_inject_eb: false,
- // Non-default value so the round-trip proves stream_format
- // both writes (only when != "horizontal") and reads back.
- stream_format: "vertical".into(),
- // Non-default: the clean track routed to YouTube (copyright).
- // Proves audio_track writes only when != "auto" and reads back.
- audio_track: "2".into(),
- });
- s.profiles = vec![
- DelayProfile {
- name: "Quick".into(),
- delay_ms: 10_000,
- },
- DelayProfile {
- name: "Long".into(),
- delay_ms: 120_000,
- },
- ];
-
- let path = std::env::temp_dir().join(format!("ic-test-cfg-{}.ini", std::process::id()));
- let _ = std::fs::remove_file(&path);
- s.save(&path).expect("save");
- let loaded = Settings::load(&path).expect("load");
- let _ = std::fs::remove_file(&path);
-
- assert_eq!(loaded.ingest_port, 1936);
- assert_eq!(loaded.web_port, 7898);
- assert_eq!(loaded.buffer_mb, 200);
- assert_eq!(
- loaded.discord_webhook_url,
- "https://discord.com/api/webhooks/123/abc"
- );
- assert_eq!(loaded.destinations.len(), 2);
- assert_eq!(loaded.destinations[0].id, "tw1");
- assert_eq!(loaded.destinations[0].name, "Twitch");
- assert_eq!(loaded.destinations[0].stream_key, "live_secret");
- assert_eq!(loaded.destinations[0].twitch_ingest, "fra");
- assert!(loaded.destinations[0].enabled);
- assert!(!loaded.destinations[1].enabled);
- assert_eq!(loaded.destinations[1].youtube_ingest, "backup");
- assert_eq!(loaded.destinations[0].stream_format, "horizontal");
- assert_eq!(loaded.destinations[1].stream_format, "vertical");
- // audio_track: default stays "auto", the non-default "2" survives.
- assert_eq!(loaded.destinations[0].audio_track, "auto");
- assert_eq!(loaded.destinations[1].audio_track, "2");
- assert_eq!(loaded.profiles.len(), 2);
- assert_eq!(loaded.profiles[0].delay_ms, 10_000);
- assert_eq!(loaded.profiles[1].name, "Long");
- }
-
- #[test]
- fn dock_layouts_round_trip_through_save_and_load() {
- // Dock layouts are opaque single-line JSON blobs stored as
- // `dock.=`. Prove a value containing `=`, `:`, and quotes
- // survives the line parser (it splits on the first `=` only, so the
- // JSON body is preserved verbatim).
- let mut s = Settings::defaults();
- s.docks.insert(
- "default".into(),
- r#"{"v":1,"preset":"minimal","w":{"number":{"on":true,"step":5}}}"#.into(),
- );
- s.docks
- .insert("gaming".into(), r#"{"preset":"delay","eq":"a=b"}"#.into());
-
- let path = std::env::temp_dir().join(format!("ic-test-docks-{}.ini", std::process::id()));
- let _ = std::fs::remove_file(&path);
- s.save(&path).expect("save");
- let loaded = Settings::load(&path).expect("load");
- let _ = std::fs::remove_file(&path);
-
- assert_eq!(loaded.docks.len(), 2);
- assert_eq!(
- loaded.docks.get("default").map(String::as_str),
- Some(r#"{"v":1,"preset":"minimal","w":{"number":{"on":true,"step":5}}}"#)
- );
- assert_eq!(
- loaded.docks.get("gaming").map(String::as_str),
- Some(r#"{"preset":"delay","eq":"a=b"}"#),
- "value after the first = (including embedded =) must survive"
- );
- }
-
- #[test]
- fn behavior_toggles_round_trip_through_save_and_load() {
- // v0.1.4 added three Settings fields for the auto-arm /
- // auto-activate behaviour. Save them, load them back, assert
- // every field survives. The format::Display writer is
- // conditional (only emits non-default values) so this also
- // verifies the apply_field reader sees those conditional keys.
- let path = std::env::temp_dir().join(format!(
- "ic-test-behavior-roundtrip-{}-{}.ini",
- std::process::id(),
- std::time::SystemTime::now()
- .duration_since(std::time::UNIX_EPOCH)
- .unwrap()
- .as_nanos()
- ));
- let _ = std::fs::remove_file(&path);
-
- let mut s = Settings::defaults();
- s.auto_arm_on_connect = true;
- s.auto_activate_when_ready = true;
- s.auto_arm_delay_ms = 30_000;
- s.overlays_seeded = true;
- // Need a destination so save() doesn't bail on the
- // configured-but-empty path.
- s.destinations.push(Destination {
- id: "test".into(),
- name: "Test".into(),
- enabled: true,
- platform: "twitch".into(),
- stream_key: "test".into(),
- custom_egress_url: String::new(),
- twitch_ingest: String::new(),
- youtube_ingest: String::new(),
- vod_audio: false,
- vod_audio_inject_eb: false,
- stream_format: "horizontal".into(),
- audio_track: "auto".into(),
- });
- s.save(&path).expect("save");
-
- let loaded = Settings::load(&path).expect("load");
- assert!(
- loaded.auto_arm_on_connect,
- "auto_arm_on_connect must survive round-trip"
- );
- assert!(
- loaded.auto_activate_when_ready,
- "auto_activate_when_ready must survive round-trip"
- );
- assert_eq!(
- loaded.auto_arm_delay_ms, 30_000,
- "auto_arm_delay_ms must survive round-trip"
- );
- assert!(
- loaded.overlays_seeded,
- "overlays_seeded must survive round-trip"
- );
-
- let _ = std::fs::remove_file(&path);
- }
-
- #[test]
- fn default_true_toggles_round_trip_when_disabled() {
- // update_check_enabled and open_dashboard_on_launch both default
- // true and are written only when turned off. Flipping them off must
- // survive a save/load, or a privacy opt-out would silently revert.
- let path = std::env::temp_dir().join(format!(
- "ic-test-optout-roundtrip-{}-{}.ini",
- std::process::id(),
- std::time::SystemTime::now()
- .duration_since(std::time::UNIX_EPOCH)
- .unwrap()
- .as_nanos()
- ));
- let _ = std::fs::remove_file(&path);
-
- let mut s = Settings::defaults();
- assert!(s.update_check_enabled, "defaults on");
- assert!(s.open_dashboard_on_launch, "defaults on");
- s.update_check_enabled = false;
- s.open_dashboard_on_launch = false;
- s.destinations.push(Destination {
- id: "test".into(),
- name: "Test".into(),
- enabled: true,
- platform: "twitch".into(),
- stream_key: "test".into(),
- custom_egress_url: String::new(),
- twitch_ingest: String::new(),
- youtube_ingest: String::new(),
- vod_audio: false,
- vod_audio_inject_eb: false,
- stream_format: "horizontal".into(),
- audio_track: "auto".into(),
- });
- s.save(&path).expect("save");
-
- let loaded = Settings::load(&path).expect("load");
- assert!(
- !loaded.update_check_enabled,
- "update_check_enabled opt-out must survive round-trip"
- );
- assert!(
- !loaded.open_dashboard_on_launch,
- "open_dashboard_on_launch opt-out must survive round-trip"
- );
-
- // A fresh config with neither key present must read as the on
- // default, not accidentally inherit the false above.
- let fresh = Settings::defaults();
- assert!(fresh.update_check_enabled && fresh.open_dashboard_on_launch);
-
- let _ = std::fs::remove_file(&path);
- }
-
#[test]
fn behavior_defaults_omitted_from_save_to_keep_config_lean() {
// The save writer skips default values for the behaviour keys
@@ -2833,16 +3106,119 @@ mod tests {
let _ = std::fs::remove_file(&path);
}
+ /// The actual atomicity guarantee: when the write cannot complete, the
+ /// config on disk is the old one, byte for byte. A directory squatting
+ /// on the `.tmp` name makes the temp write fail the way a full disk or
+ /// an antivirus lock would. A save that wrote in place (truncate, then
+ /// write) would already have destroyed the original at that point.
+ #[test]
+ fn a_failed_save_leaves_the_existing_file_untouched() {
+ let path = scratch_config_path("atomic");
+ let mut s = Settings::defaults();
+ s.web_port = 9001;
+ s.save(&path).expect("first save");
+ let before = std::fs::read(&path).unwrap();
+
+ let mut tmp_name = path.file_name().unwrap().to_os_string();
+ tmp_name.push(".tmp");
+ std::fs::create_dir(path.with_file_name(tmp_name)).unwrap();
+ s.web_port = 9002;
+ let result = s.save(&path);
+
+ let after = std::fs::read(&path).unwrap();
+ let _ = std::fs::remove_dir_all(path.parent().unwrap());
+ assert!(result.is_err(), "the blocked temp write must surface");
+ assert_eq!(after, before, "the original config must be untouched");
+ }
+
#[test]
fn missing_config_file_falls_back_to_defaults() {
- // load_or_default must not panic on a non-existent path - that's
- // the cold-start case where there's nothing to read yet.
- let path = std::env::temp_dir().join(format!("ic-no-such-file-{}.ini", std::process::id()));
- let _ = std::fs::remove_file(&path);
- let s = Settings::load_or_default(&path);
+ // The cold-start case: nothing to read yet. Defaults, written out
+ // at once so the user can find the file, and normal saving after.
+ let path = scratch_config_path("missing");
+ let startup = Settings::load_for_startup(&path).expect("fresh install");
+ let written = path.exists();
+ let _ = std::fs::remove_dir_all(path.parent().unwrap());
+ let s = startup.settings;
assert_eq!(s.ingest_port, 1935);
assert_eq!(s.web_port, 7799);
assert!(!s.configured);
+ assert!(startup.may_save && startup.notice.is_none());
+ assert!(written, "a fresh install writes its config file");
+ }
+
+ /// Stand-in for a file another process holds open: every read fails
+ /// with something other than NotFound.
+ fn locked_read(_: &Path) -> io::Result {
+ Err(io::Error::from(io::ErrorKind::PermissionDenied))
+ }
+
+ /// The data-loss path: a config that exists but cannot be read must
+ /// never be replaced by defaults at startup. Its bytes are copied
+ /// aside first, the original is left alone, and startup is told not
+ /// to save. Before this, a single failed read meant every destination
+ /// and stream key was overwritten with defaults on launch.
+ #[test]
+ fn an_unreadable_config_is_copied_aside_and_not_overwritten() {
+ let path = scratch_config_path("locked");
+ let original = b"configured=true\ndestination.0.stream_key=live_abc\n".to_vec();
+ std::fs::write(&path, &original).unwrap();
+ let earlier_copy = path.with_file_name("instantclone.cfg.unreadable");
+ std::fs::write(&earlier_copy, b"an older copy").unwrap();
+
+ let startup = Settings::load_for_startup_with(&path, locked_read).expect("copy made");
+
+ let on_disk = std::fs::read(&path).unwrap();
+ let first_copy = std::fs::read(&earlier_copy).unwrap();
+ let new_copy = std::fs::read(path.with_file_name("instantclone.cfg.unreadable.2"));
+ let _ = std::fs::remove_dir_all(path.parent().unwrap());
+ assert_eq!(on_disk, original, "the original must be untouched");
+ assert_eq!(
+ first_copy, b"an older copy",
+ "an earlier copy is not clobbered"
+ );
+ assert_eq!(new_copy.expect("a second copy"), original);
+ assert!(!startup.may_save, "startup must not save defaults over it");
+ assert!(!startup.settings.configured, "runs on defaults meanwhile");
+ assert!(startup.notice.is_some(), "and the user is told why");
+ }
+
+ /// No copy, no start: if the file cannot even be copied, running on
+ /// defaults would let the first settings change destroy the only copy.
+ #[test]
+ fn an_unreadable_config_that_cannot_be_copied_stops_startup() {
+ // A directory at the config path reads and copies as an error on
+ // every OS, without needing a real lock.
+ let path = scratch_config_path("dir");
+ std::fs::create_dir(&path).unwrap();
+ let result = Settings::load_for_startup_with(&path, locked_read);
+ let still_there = path.is_dir();
+ let _ = std::fs::remove_dir_all(path.parent().unwrap());
+ assert!(result.is_err());
+ assert!(still_there);
+ }
+
+ /// A lock that clears during the retry window loads normally, with no
+ /// copy and no notice: a brief antivirus scan is not worth a dialog.
+ #[test]
+ fn a_briefly_locked_config_loads_after_a_retry() {
+ let path = scratch_config_path("brief");
+ std::fs::write(&path, "configured=true\nweb_port=9124\n").unwrap();
+ let reads = std::cell::Cell::new(0);
+ let flaky = |p: &Path| {
+ reads.set(reads.get() + 1);
+ if reads.get() == 1 {
+ locked_read(p)
+ } else {
+ Settings::load(p)
+ }
+ };
+ let startup = Settings::load_for_startup_with(&path, flaky).expect("load");
+ let copied = path.with_file_name("instantclone.cfg.unreadable").exists();
+ let _ = std::fs::remove_dir_all(path.parent().unwrap());
+ assert_eq!(startup.settings.web_port, 9124);
+ assert!(startup.may_save && startup.notice.is_none());
+ assert!(!copied);
}
#[test]
@@ -2878,28 +3254,6 @@ mod tests {
assert_eq!(parse_hotkey("Ctrl+Home").expect("valid").1, 0x24);
}
- #[test]
- fn parse_hotkey_rejects_bare_keys_and_multi_key_chords() {
- // A bare key (no modifier) is the misclick risk we refuse, and a
- // two-letter chord is not expressible via RegisterHotKey.
- assert!(parse_hotkey("D").is_none(), "bare key must be rejected");
- assert!(
- parse_hotkey("F8").is_none(),
- "bare function key must be rejected"
- );
- assert!(
- parse_hotkey("Ctrl+D+L").is_none(),
- "multi-key chord must be rejected"
- );
- assert!(
- parse_hotkey("Ctrl+Alt").is_none(),
- "modifiers with no main key"
- );
- assert!(parse_hotkey("Ctrl+Q1").is_none(), "unknown key token");
- assert!(parse_hotkey("Ctrl+F25").is_none(), "F-key out of range");
- assert!(parse_hotkey("").is_none());
- }
-
/// `ACTIONS` is what the tray derives its RegisterHotKey ids from and
/// what both `unbind_elsewhere` walks, so it has to stay in step with
/// the two structs. A reorder here without one there would file a
@@ -3048,7 +3402,7 @@ mod tests {
}
// And nothing it injected survives a reload as a real setting.
- let back = Settings::load_or_default(&path);
+ let back = Settings::load(&path).expect("load");
assert!(back.dashboard_password_hash.is_empty());
assert!(!back.web_bind_all);
assert_eq!(back.web_port, s.web_port);
@@ -3084,7 +3438,7 @@ mod tests {
)
.expect("write");
- let s = Settings::load_or_default(&path);
+ let s = Settings::load(&path).expect("load");
assert_eq!(s.web_port, 8080, "a good line after bad ones still loads");
assert_eq!(s.hotkeys.toggle, "Ctrl+Alt+D", "and so does a good binding");
assert!(
@@ -3189,30 +3543,42 @@ mod tests {
/// Combos are typed by users and hand-edited in the config file.
#[test]
fn hotkey_parsing_rejects_the_awkward_shapes() {
- // A modifier is mandatory: that requirement is the misclick guard.
- assert!(canonicalize_hotkey("D").is_none(), "a bare key is refused");
- assert!(canonicalize_hotkey("Ctrl").is_none(), "modifiers alone too");
- assert!(canonicalize_hotkey("Ctrl+Alt").is_none());
- assert!(canonicalize_hotkey("").is_none());
- assert!(canonicalize_hotkey("+++").is_none());
- assert!(canonicalize_hotkey("Ctrl+NotAKey").is_none());
- // Two keys is not a combo we can register.
- assert!(canonicalize_hotkey("Ctrl+A+B").is_none());
- // Order and case do not matter, and the output is canonical.
- assert_eq!(
- canonicalize_hotkey("alt+CTRL+d").as_deref(),
- canonicalize_hotkey("Ctrl+Alt+D").as_deref()
- );
- // A repeated modifier is the same combo, not a different one.
- assert_eq!(
- canonicalize_hotkey("Ctrl+Ctrl+Alt+D").as_deref(),
- canonicalize_hotkey("Ctrl+Alt+D").as_deref()
- );
- // Whitespace around the tokens survives a hand edit.
- assert_eq!(
- canonicalize_hotkey(" Ctrl + Alt + D ").as_deref(),
- canonicalize_hotkey("Ctrl+Alt+D").as_deref()
- );
+ let refused = [
+ // A modifier is mandatory: that requirement is the misclick guard.
+ ("D", "a bare key"),
+ ("F8", "a bare function key"),
+ ("Ctrl", "a modifier alone"),
+ ("Ctrl+Alt", "modifiers with no main key"),
+ ("", "nothing"),
+ ("+++", "only separators"),
+ ("Ctrl+NotAKey", "an unknown key name"),
+ ("Ctrl+Q1", "an unknown key token"),
+ ("Ctrl+F25", "an F-key out of range"),
+ // Two main keys is a chord RegisterHotKey cannot express.
+ ("Ctrl+A+B", "two main keys"),
+ ];
+ for (combo, why) in refused {
+ assert!(
+ parse_hotkey(combo).is_none(),
+ "{why} must be refused: {combo:?}"
+ );
+ assert!(canonicalize_hotkey(combo).is_none(), "{why}: {combo:?}");
+ }
+ // Order, case, repeats and stray whitespace all land on the one
+ // canonical spelling. Compared against the literal, so a parser
+ // that refused both sides cannot pass as None == None.
+ for combo in [
+ "Ctrl+Alt+D",
+ "alt+CTRL+d",
+ "Ctrl+Ctrl+Alt+D",
+ " Ctrl + Alt + D ",
+ ] {
+ assert_eq!(
+ canonicalize_hotkey(combo).as_deref(),
+ Some("Ctrl+Alt+D"),
+ "{combo:?}"
+ );
+ }
}
#[test]
@@ -3350,6 +3716,530 @@ Name@x"
// An empty signature must never match an unbound action.
assert_eq!(m.action_for(""), None);
}
+
+ /// Unique scratch path per call, so parallel tests never share a file
+ /// and none of them can reach the user's real settings.
+ fn scratch_config_path(tag: &str) -> PathBuf {
+ use std::sync::atomic::{AtomicU32, Ordering};
+ static NEXT: AtomicU32 = AtomicU32::new(0);
+ let dir = std::env::temp_dir().join(format!(
+ "ic-cfg-{tag}-{}-{}",
+ std::process::id(),
+ NEXT.fetch_add(1, Ordering::Relaxed)
+ ));
+ let _ = std::fs::remove_dir_all(&dir);
+ std::fs::create_dir_all(&dir).unwrap();
+ dir.join("instantclone.cfg")
+ }
+
+ fn test_destination(id: &str, platform: &str, key: &str) -> Destination {
+ Destination {
+ id: id.into(),
+ name: id.into(),
+ enabled: true,
+ platform: platform.into(),
+ stream_key: key.into(),
+ custom_egress_url: String::new(),
+ twitch_ingest: String::new(),
+ youtube_ingest: String::new(),
+ vod_audio: false,
+ vod_audio_inject_eb: false,
+ stream_format: "horizontal".into(),
+ audio_track: "auto".into(),
+ }
+ }
+
+ /// A `Settings` with EVERY field away from its default, built as a
+ /// struct literal with no `..Default` so adding a field to `Settings`,
+ /// `Destination`, `DelayProfile`, `Hotkeys` or `MidiBindings` does not
+ /// compile until it is given a non-default value here. That is the
+ /// point: the round-trip test below then fails until `save` writes the
+ /// new field and `apply_field` reads it back.
+ ///
+ /// The legacy `platform`/`stream_key`/`custom_egress_url` fields mirror
+ /// `destinations[0]`, as `save` writes them.
+ fn every_field_non_default() -> Settings {
+ let first = Destination {
+ id: "d-main".into(),
+ name: "Main custom".into(),
+ enabled: false,
+ platform: "custom".into(),
+ stream_key: "key-one".into(),
+ custom_egress_url: "rtmps://ingest.example.com/app".into(),
+ twitch_ingest: "fra".into(),
+ youtube_ingest: "backup".into(),
+ vod_audio: true,
+ vod_audio_inject_eb: true,
+ stream_format: "vertical".into(),
+ audio_track: "2".into(),
+ };
+ let second = Destination {
+ id: "d-yt".into(),
+ name: "YouTube".into(),
+ enabled: true,
+ platform: "youtube".into(),
+ stream_key: "key-two".into(),
+ custom_egress_url: String::new(),
+ twitch_ingest: String::new(),
+ youtube_ingest: String::new(),
+ vod_audio: false,
+ vod_audio_inject_eb: false,
+ stream_format: "horizontal".into(),
+ audio_track: "both".into(),
+ };
+ Settings {
+ ingest_port: 1940,
+ ingest_bind_all: true,
+ ingest_key: "ingest-secret".into(),
+ platform: first.platform.clone(),
+ stream_key: first.stream_key.clone(),
+ custom_egress_url: first.custom_egress_url.clone(),
+ web_port: 7810,
+ web_bind_all: true,
+ dashboard_password_hash: "pbkdf2$1000$c2FsdA$aGFzaA".into(),
+ dock_token: "0123abcd".into(),
+ buffer_mb: 777,
+ buffer_path: PathBuf::from("D:/buffers/ic.buf"),
+ target_delay_ms: 45_000,
+ armed_delay_ms: 60_000,
+ configured: true,
+ profiles: vec![
+ DelayProfile {
+ name: "Scrim".into(),
+ delay_ms: 20_000,
+ },
+ DelayProfile {
+ name: "Finals".into(),
+ delay_ms: 180_000,
+ },
+ ],
+ destinations: vec![first, second],
+ // Migrated away on load (see `migrate_discord_webhook`), so a
+ // round-trip only holds for the empty value.
+ discord_webhook_url: String::new(),
+ overlays_dir: PathBuf::from("D:/overlays"),
+ tracing_enabled: true,
+ auto_arm_on_connect: true,
+ auto_activate_when_ready: true,
+ auto_arm_delay_ms: 25_000,
+ update_check_enabled: false,
+ open_dashboard_on_launch: false,
+ overlays_seeded: true,
+ // A layout value holding its own `=`: the line parser must split
+ // on the first `=` only.
+ docks: BTreeMap::from([
+ ("main".to_string(), r#"{"w":["delay","cut"]}"#.to_string()),
+ (
+ "gaming".to_string(),
+ r#"{"preset":"delay","eq":"a=b"}"#.to_string(),
+ ),
+ ]),
+ hotkeys: Hotkeys {
+ toggle: "Ctrl+Alt+D".into(),
+ arm: "Ctrl+Alt+A".into(),
+ activate: "Ctrl+Shift+F9".into(),
+ cut: "Alt+Win+C".into(),
+ cut_after: "Ctrl+PageUp".into(),
+ end_hold: "Ctrl+Alt+E".into(),
+ },
+ midi: MidiBindings {
+ toggle: "note:1:36".into(),
+ arm: "note:2:37".into(),
+ activate: "cc:1:20".into(),
+ cut: "cc:16:21".into(),
+ cut_after: "note:1:38".into(),
+ end_hold: "note:3:40".into(),
+ },
+ midi_device: "Launchpad Mini".into(),
+ crash_protection: crate::crash_protection::CrashProtection {
+ enabled: true,
+ hold_secs: 120,
+ theme: crate::crash_protection::SlateTheme::Orbit,
+ accent: crate::slate::Rgb {
+ r: 250,
+ g: 60,
+ b: 90,
+ },
+ background: Some(crate::slate::Rgb {
+ r: 10,
+ g: 20,
+ b: 30,
+ }),
+ headline: "Back in a sec".into(),
+ subline: "OBS is restarting".into(),
+ every_disconnect: true,
+ },
+ integrations: vec![
+ crate::integrations::presets::build("crash_alert", "alerts1".into(), "mods")
+ .unwrap(),
+ crate::integrations::presets::build("mod_controls", "mods2".into(), "").unwrap(),
+ ],
+ discord_channels: vec![DiscordChannel {
+ id: "mods".into(),
+ name: "Mods = team".into(),
+ url: "https://discord.com/api/webhooks/1/abc".into(),
+ }],
+ phone: PhoneConnection {
+ server: "https://ntfy.example".into(),
+ topic: "texaz-alerts".into(),
+ },
+ integrations_migrated: true,
+ twitch_client_id: "abcdefghij0123456789klmnopqrst".into(),
+ twitch_shared_chat: true,
+ }
+ }
+
+ /// A pasted Client ID is checked before it is saved, so a stray space
+ /// or the client secret by mistake never reaches Twitch.
+ #[test]
+ fn a_twitch_client_id_must_look_like_one() {
+ let twitch_errors = |id: &str| {
+ let mut s = Settings::defaults();
+ s.twitch_client_id = id.into();
+ s.validate()
+ .iter()
+ .filter(|e| e.contains("Twitch Client ID"))
+ .count()
+ };
+ assert_eq!(twitch_errors(""), 0);
+ assert_eq!(twitch_errors("abcdefghij0123456789klmnopqrst"), 0);
+ for bad in [
+ "short",
+ "has space in it 0123456789abc",
+ "abc\"defghij0123456789klmnopq",
+ ] {
+ assert_eq!(twitch_errors(bad), 1, "{bad}");
+ }
+ }
+
+ /// The old single webhook becomes a Discord connection plus a "Stream
+ /// alerts" integration, once, and a config saved afterwards keeps them.
+ #[test]
+ fn old_discord_webhook_is_migrated_once() {
+ let path = scratch_config_path("webhook-migration");
+ std::fs::write(
+ &path,
+ "discord_webhook_url=https://discord.com/api/webhooks/9/tok
+",
+ )
+ .unwrap();
+ let loaded = Settings::load(&path).unwrap();
+ assert!(loaded.discord_webhook_url.is_empty());
+ assert!(loaded.integrations_migrated);
+ assert_eq!(loaded.discord_channels.len(), 1);
+ assert_eq!(
+ loaded.discord_channels[0].url,
+ "https://discord.com/api/webhooks/9/tok"
+ );
+ assert_eq!(loaded.integrations.len(), 1);
+ assert_eq!(loaded.integrations[0].preset, "stream_alerts");
+ assert!(loaded.integrations[0].enabled);
+
+ loaded.save(&path).unwrap();
+ let again = Settings::load(&path).unwrap();
+ assert_eq!(again.integrations.len(), 1, "migrated exactly once");
+ assert_eq!(again.discord_channels, loaded.discord_channels);
+ let _ = std::fs::remove_file(&path);
+ }
+
+ /// A damaged integration line is skipped; everything else still loads.
+ #[test]
+ fn a_broken_integration_line_does_not_cost_the_rest() {
+ let path = scratch_config_path("broken-integration");
+ let good = crate::integrations::presets::build("delay_command", "good".into(), "")
+ .unwrap()
+ .to_json()
+ .to_json();
+ std::fs::write(
+ &path,
+ format!(
+ "web_port=7811
+integration.bad={{not json
+integration.good={good}
+integrations_migrated=true
+"
+ ),
+ )
+ .unwrap();
+ let loaded = Settings::load(&path).unwrap();
+ assert_eq!(loaded.web_port, 7811);
+ assert_eq!(loaded.integrations.len(), 1);
+ assert_eq!(loaded.integrations[0].id, "good");
+ let _ = std::fs::remove_file(&path);
+ }
+
+ /// Every setting survives save then load unchanged. The one test that
+ /// notices a field `save` forgets to write or `apply_field` forgets to
+ /// read, which otherwise shows up as a setting that resets on restart.
+ #[test]
+ fn every_setting_round_trips_through_save_and_load() {
+ let path = scratch_config_path("full-roundtrip");
+ let original = every_field_non_default();
+ original.save(&path).unwrap();
+ let loaded = Settings::load(&path);
+ let _ = std::fs::remove_dir_all(path.parent().unwrap());
+ assert_eq!(loaded.unwrap(), original);
+ }
+
+ /// Guards the guard: if `every_field_non_default` drifted back to a
+ /// default somewhere, the round trip would stop covering that field.
+ #[test]
+ fn the_round_trip_fixture_really_is_non_default() {
+ let s = every_field_non_default();
+ let d = Settings::defaults();
+ assert_ne!(s.ingest_port, d.ingest_port);
+ assert_ne!(s.web_port, d.web_port);
+ assert_ne!(s.buffer_mb, d.buffer_mb);
+ assert_ne!(s.buffer_path, d.buffer_path);
+ assert_ne!(s.overlays_dir, d.overlays_dir);
+ assert_ne!(s.auto_arm_delay_ms, d.auto_arm_delay_ms);
+ assert_ne!(s.profiles, d.profiles);
+ assert_ne!(s.hotkeys, d.hotkeys);
+ assert_ne!(s.midi, d.midi);
+ for (action, combo) in s.hotkeys.entries() {
+ assert_eq!(
+ canonicalize_hotkey(combo).as_deref(),
+ Some(combo),
+ "{action}"
+ );
+ }
+ for (action, sig) in s.midi.entries() {
+ assert_eq!(canonicalize_midi(sig).as_deref(), Some(sig), "{action}");
+ }
+ }
+
+ /// A config exactly as 0.1.14 wrote it (same `save` order and keys).
+ /// Upgrading must keep every value, and anything added since must load
+ /// at its default.
+ #[test]
+ fn a_0_1_14_config_loads_intact() {
+ let path = scratch_config_path("v0114");
+ std::fs::write(
+ &path,
+ "# InstantClone - written by the app. Hand edits OK.\n\
+ configured=true\n\
+ platform=twitch\n\
+ stream_key=live_123\n\
+ custom_egress_url=\n\
+ ingest_port=1935\n\
+ ingest_bind_all=false\n\
+ web_port=7799\n\
+ web_bind_all=false\n\
+ buffer_mb=500\n\
+ buffer_path=./instantclone.buf\n\
+ target_delay_ms=0\n\
+ armed_delay_ms=30000\n\
+ discord_webhook_url=\n\
+ overlays_dir=./overlays\n\
+ tracing_enabled=false\n\
+ overlays_seeded=true\n\
+ hotkey.toggle=Ctrl+Alt+D\n\
+ profile.0.name=Quick\n\
+ profile.0.delay_ms=15000\n\
+ destination.0.id=main\n\
+ destination.0.name=Main\n\
+ destination.0.enabled=true\n\
+ destination.0.platform=twitch\n\
+ destination.0.stream_key=live_123\n\
+ destination.0.custom_egress_url=\n\
+ destination.0.twitch_ingest=\n\
+ destination.0.youtube_ingest=\n\
+ destination.0.vod_audio=true\n",
+ )
+ .unwrap();
+ let loaded = Settings::load(&path);
+ let _ = std::fs::remove_dir_all(path.parent().unwrap());
+
+ let mut expected = Settings::defaults();
+ expected.configured = true;
+ expected.stream_key = "live_123".into();
+ expected.armed_delay_ms = 30_000;
+ expected.overlays_seeded = true;
+ expected.hotkeys.toggle = "Ctrl+Alt+D".into();
+ expected.profiles.truncate(1);
+ let mut main = test_destination("main", "twitch", "live_123");
+ main.name = "Main".into();
+ main.vod_audio = true;
+ expected.destinations = vec![main];
+ // Loading always completes the one-time webhook migration.
+ expected.integrations_migrated = true;
+ assert_eq!(loaded.unwrap(), expected);
+ }
+
+ /// Rolling back to this build after a newer one wrote the file: keys we
+ /// do not know are skipped without disturbing the ones we do.
+ #[test]
+ fn unknown_keys_from_a_newer_build_do_not_disturb_known_ones() {
+ let path = scratch_config_path("newer");
+ let original = every_field_non_default();
+ original.save(&path).unwrap();
+ let mut text = std::fs::read_to_string(&path).unwrap();
+ text.insert_str(0, "future=1\ncrash_protection.future_mode=on\n");
+ text.push_str("destination.0.newfield=x\ndestination.1.another=y\nfuture_tail=2\n");
+ std::fs::write(&path, &text).unwrap();
+
+ let loaded = Settings::load(&path).unwrap();
+ loaded.save(&path).unwrap();
+ let resaved = std::fs::read_to_string(&path).unwrap();
+ let _ = std::fs::remove_dir_all(path.parent().unwrap());
+ assert_eq!(loaded, original);
+ // Pinned behavior: unknown lines are dropped on the next save, so a
+ // rollback followed by any settings change loses the newer build's
+ // extra settings. Change this deliberately if that ever changes.
+ for unknown in [
+ "future=",
+ "future_mode",
+ "newfield",
+ "another=",
+ "future_tail",
+ ] {
+ assert!(!resaved.contains(unknown), "{unknown} survived:\n{resaved}");
+ }
+ }
+
+ /// A config hand-edited in Notepad and saved as ANSI turns an accented
+ /// character into a byte that is not UTF-8. That used to fail the whole
+ /// read, fall back to defaults, and have startup save the defaults over
+ /// every stream key. The damaged character may be lost; nothing else is.
+ #[test]
+ fn a_config_with_non_utf8_bytes_still_loads_its_other_fields() {
+ let path = scratch_config_path("ansi");
+ let mut bytes = b"configured=true\nweb_port=9123\ndestination.0.id=d1\n\
+ destination.0.enabled=true\ndestination.0.platform=twitch\n\
+ destination.0.stream_key=live_abc\ndestination.0.name=Espa"
+ .to_vec();
+ bytes.push(0xF1); // "ñ" in Windows-1252
+ bytes.extend_from_slice(b"a\n");
+ std::fs::write(&path, &bytes).unwrap();
+
+ let s = Settings::load(&path);
+ let _ = std::fs::remove_dir_all(path.parent().unwrap());
+ let s = s.expect("a non-UTF-8 byte must not fail the whole load");
+ assert!(s.configured);
+ assert_eq!(s.web_port, 9123);
+ assert_eq!(s.destinations.len(), 1);
+ assert_eq!(s.destinations[0].stream_key, "live_abc");
+ assert!(s.destinations[0].name.starts_with("Espa"));
+ }
+
+ /// Deleting the last destination must stick. The legacy `stream_key=`
+ /// line used to keep the old key after the delete, and the next load's
+ /// single-destination migration rebuilt an ENABLED "Main" from it: a
+ /// stream the user removed would go live again after a restart.
+ #[test]
+ fn deleting_the_last_destination_survives_a_restart() {
+ let path = scratch_config_path("last-dest");
+ let mut s = Settings::defaults();
+ s.destinations
+ .push(test_destination("d1", "youtube", "secret-key"));
+ s.save(&path).unwrap();
+
+ let mut loaded = Settings::load(&path).unwrap();
+ assert_eq!(loaded.destinations.len(), 1);
+ loaded.destinations.clear();
+ loaded.save(&path).unwrap();
+
+ let reloaded = Settings::load(&path).unwrap();
+ let _ = std::fs::remove_dir_all(path.parent().unwrap());
+ assert!(
+ reloaded.destinations.is_empty(),
+ "deleted destination came back: {:?}",
+ reloaded.destinations
+ );
+ }
+
+ /// A config from before destinations existed (only the legacy
+ /// single-destination lines) still becomes one enabled destination.
+ #[test]
+ fn a_pre_destinations_config_still_migrates() {
+ let path = scratch_config_path("legacy");
+ std::fs::write(&path, "configured=true\nplatform=youtube\nstream_key=abc\n").unwrap();
+ let s = Settings::load(&path).unwrap();
+ let _ = std::fs::remove_dir_all(path.parent().unwrap());
+ assert_eq!(s.destinations.len(), 1);
+ let d = &s.destinations[0];
+ assert_eq!(
+ (d.platform.as_str(), d.stream_key.as_str()),
+ ("youtube", "abc")
+ );
+ assert!(d.enabled);
+ assert_eq!(
+ d.egress_url().as_deref(),
+ Some("rtmp://a.rtmp.youtube.com/live2/abc")
+ );
+ }
+
+ /// The custom-URL-only variant: a key embedded in the URL and no
+ /// `stream_key=` line must migrate too, and keep the URL intact.
+ #[test]
+ fn a_pre_destinations_custom_url_config_still_migrates() {
+ let path = scratch_config_path("legacy-custom");
+ let url = "rtmp://ingest.example.com/app/embedded-key";
+ std::fs::write(&path, format!("platform=custom\ncustom_egress_url={url}\n")).unwrap();
+ let s = Settings::load(&path).unwrap();
+ let _ = std::fs::remove_dir_all(path.parent().unwrap());
+ assert_eq!(s.destinations.len(), 1);
+ assert_eq!(s.destinations[0].platform, "custom");
+ assert_eq!(s.destinations[0].custom_egress_url, url);
+ assert_eq!(s.destinations[0].egress_url().as_deref(), Some(url));
+ }
+
+ /// Other tests trust `is_valid_json` to catch broken hand-written JSON,
+ /// so it has to reject exactly the mistakes those writers can make.
+ #[test]
+ fn json_validator_accepts_json_and_rejects_the_usual_breakage() {
+ let valid = [
+ r#"{}"#,
+ r#"[]"#,
+ r#" {"a":[1,-2.5e3,true,false,null,"x\né"],"b":{}} "#,
+ r#""plain string""#,
+ ];
+ for doc in valid {
+ assert!(is_valid_json(doc), "should accept: {doc}");
+ }
+ let invalid = [
+ "",
+ r#"{"a":1,}"#,
+ r#"[1,,2]"#,
+ r#"[,1]"#,
+ "{\"a\":\"raw\nnewline\"}",
+ r#"{"a":1}{"b":2}"#,
+ r#"{"a" 1}"#,
+ r#"{a:1}"#,
+ r#"[01]"#,
+ r#"["\x"]"#,
+ r#"[1"#,
+ ];
+ for doc in invalid {
+ assert!(!is_valid_json(doc), "should reject: {doc}");
+ }
+ }
+
+ /// The dashboard parses this on every poll; a stray comma or unescaped
+ /// character in any field blanks the whole page.
+ #[test]
+ fn to_json_is_valid_json_with_awkward_values() {
+ let mut s = Settings::defaults();
+ s.midi_device = "Pad \"8\"\\\ttab".into();
+ s.destinations.push(Destination {
+ id: "d1".into(),
+ name: "Main \"quoted\"".into(),
+ enabled: true,
+ platform: "custom".into(),
+ stream_key: "k\\ey".into(),
+ custom_egress_url: "rtmp://host/app".into(),
+ twitch_ingest: String::new(),
+ youtube_ingest: String::new(),
+ vod_audio: false,
+ vod_audio_inject_eb: false,
+ stream_format: String::new(),
+ audio_track: String::new(),
+ });
+ for secrets in [false, true] {
+ let json = s.to_json(false, secrets);
+ assert!(is_valid_json(&json), "invalid JSON: {json}");
+ }
+ }
}
// Tiny libc shim for Unix free-space - avoids adding the `libc` crate.
diff --git a/src/controller.rs b/src/controller.rs
index e25e296..ac93c83 100644
--- a/src/controller.rs
+++ b/src/controller.rs
@@ -11,6 +11,8 @@
use crate::buffer::{DiskRing, TagMeta};
use crate::compat::StreamParams;
use crate::h264::{AudioCodec, VideoCodec};
+use crate::integrations::host::fmt_duration;
+use crate::integrations::{Event, EventKind};
use crate::rtmp::client::{EgressClient, EgressSink, EgressUrl};
use std::collections::HashMap;
use std::io;
@@ -32,6 +34,62 @@ fn process_now_ms() -> u64 {
Instant::now().saturating_duration_since(*start).as_millis() as u64
}
+/// Rolling bitrate, lock-free: bytes are summed over windows of at least
+/// `RATE_WINDOW_MS` and the rate is the last full window. A window that
+/// hasn't rolled for `RATE_STALE_MS` means nothing has arrived since (the
+/// stream stopped or froze), so the rate reads 0 rather than holding the
+/// last second's value forever.
+#[derive(Default)]
+pub struct RateMeter {
+ window_bytes: AtomicU64,
+ /// 0 = no window yet.
+ window_start_ms: AtomicU64,
+ kbps: AtomicU32,
+}
+
+const RATE_WINDOW_MS: u64 = 1_000;
+const RATE_STALE_MS: u64 = 2_500;
+
+impl RateMeter {
+ pub fn note(&self, bytes: usize) {
+ self.note_at(bytes, process_now_ms());
+ }
+
+ fn note_at(&self, bytes: usize, now_ms: u64) {
+ let now_ms = now_ms.max(1);
+ let start = self.window_start_ms.load(Ordering::Relaxed);
+ // First bytes, or the first after a gap: they mark the start of a
+ // fresh window (so they count toward the time before it), instead
+ // of the silence being averaged into the rate.
+ if start == 0 || now_ms.saturating_sub(start) > RATE_STALE_MS {
+ self.window_bytes.store(0, Ordering::Relaxed);
+ self.window_start_ms.store(now_ms, Ordering::Relaxed);
+ return;
+ }
+ let total = self.window_bytes.fetch_add(bytes as u64, Ordering::Relaxed) + bytes as u64;
+ let elapsed = now_ms.saturating_sub(start);
+ if elapsed >= RATE_WINDOW_MS {
+ // bytes * 8 / ms = kbps
+ self.kbps
+ .store(((total * 8) / elapsed) as u32, Ordering::Relaxed);
+ self.window_bytes.store(0, Ordering::Relaxed);
+ self.window_start_ms.store(now_ms, Ordering::Relaxed);
+ }
+ }
+
+ pub fn kbps(&self) -> u32 {
+ self.kbps_at(process_now_ms())
+ }
+
+ fn kbps_at(&self, now_ms: u64) -> u32 {
+ let start = self.window_start_ms.load(Ordering::Relaxed);
+ if start == 0 || now_ms.saturating_sub(start) > RATE_STALE_MS {
+ return 0;
+ }
+ self.kbps.load(Ordering::Relaxed)
+ }
+}
+
/// Encode VideoCodec as a u8 for atomic storage.
fn enc_vcodec(c: VideoCodec) -> u8 {
match c {
@@ -110,6 +168,9 @@ impl ActivateError {
/// because a 24/7 relay logs for months: the oldest line is dropped rather
/// than letting the buffer track uptime.
const LOG_LINES_MAX: usize = 1_500;
+/// How long OBS gets to send its first video frame after coming back from
+/// a crash-protection hold before a missing picture counts as a freeze.
+const FIRST_VIDEO_GRACE: Duration = Duration::from_secs(10);
pub const NO_INGEST: &str = "OBS isn't sending anything yet - start streaming first";
@@ -152,7 +213,7 @@ pub struct DestinationState {
pub bytes_sent: AtomicU64,
pub cuts_performed: AtomicU32,
pub reconnects: AtomicU32,
- pub bitrate_kbps_out: AtomicU32,
+ pub rate_out: RateMeter,
/// Set by the supervisor when this destination is being removed or
/// the app is shutting down. The egress pump checks it once per loop
/// and tears down the upstream session politely (deleteStream) before
@@ -166,8 +227,6 @@ pub struct DestinationState {
/// bytes don't match and the upstream decoder silently rejects every
/// subsequent frame.
last_seq_header_gen: AtomicU32,
- rate_window_bytes: AtomicU64,
- rate_window_start_ms: AtomicU64,
/// True if this destination accepts Enhanced Broadcasting multi-track
/// video on the wire. Set by the supervisor to `true` when the
/// destination's platform is `twitch` and to `false` for everything
@@ -204,9 +263,9 @@ pub struct DestinationState {
/// The OneTrack TrackId of the vertical-canvas primary, discovered by
/// `h264::detect_vertical_primary_track` from the per-track seq-header
/// cache and refreshed whenever that cache changes. `0xFF` means
- /// "not resolved yet" (Twitch Dual Format isn't active, or no portrait
- /// track has been seen): a vertical destination then sends no video
- /// and surfaces a "waiting for Dual Format" status, while every other
+ /// "not resolved yet" (OBS sends no 9:16 canvas, from its Additional
+ /// canvas or Twitch Dual Format): a vertical destination then sends no
+ /// video and the dashboard says what's missing, while every other
/// destination is unaffected.
pub vertical_primary_track: AtomicU8,
/// Twitch only: when our /obs/multitrack-config proxy successfully
@@ -251,6 +310,19 @@ pub struct DestinationState {
pub session_epoch: AtomicU64,
}
+/// The Enhanced Broadcasting config last handed to OBS, and the Twitch
+/// session it opened on one destination.
+#[derive(Debug, Clone)]
+pub struct EbSession {
+ /// The rewritten GetClientConfiguration response OBS received.
+ pub config: String,
+ /// The session tokens OBS may publish with.
+ pub auths: Vec,
+ pub dest_id: String,
+ /// The IVS ingest URL the destination streams to for this session.
+ pub ivs_url: String,
+}
+
/// Outcome of finishing a VOD-session fetch, returned by
/// `complete_vod_fetch` so the supervisor can log it without re-deriving
/// what happened.
@@ -281,12 +353,10 @@ impl DestinationState {
bytes_sent: AtomicU64::new(0),
cuts_performed: AtomicU32::new(0),
reconnects: AtomicU32::new(0),
- bitrate_kbps_out: AtomicU32::new(0),
+ rate_out: RateMeter::default(),
shutdown_requested: AtomicBool::new(false),
last_seq_header_gen: AtomicU32::new(0),
eb_override_url: crate::sync::Mutex::new(None),
- rate_window_bytes: AtomicU64::new(0),
- rate_window_start_ms: AtomicU64::new(0),
// Default false: every newly-spawned destination flattens
// multi-track until the supervisor decides otherwise. This
// preserves beta.6 behaviour for any code path that creates
@@ -308,9 +378,9 @@ impl DestinationState {
/// The video egress policy for this destination right now. Read by
/// both the live send path and the seq-header replay so they always
/// agree on which canvas to forward. Returns `None` when a vertical
- /// destination has no resolved canvas yet (Twitch Dual Format isn't
- /// active): the caller drops all video and the dest waits, leaving
- /// every other destination untouched.
+ /// destination has no resolved canvas yet (no 9:16 canvas on the wire):
+ /// the caller drops all video and the dest waits, leaving every other
+ /// destination untouched.
pub fn video_egress(&self) -> Option {
use crate::h264::VideoEgress;
if self.pass_through_multitrack_video.load(Ordering::Relaxed) {
@@ -423,27 +493,20 @@ impl DestinationState {
outcome
}
- fn note_outbound_bytes(&self, n: usize) {
- let now = process_now_ms();
- let total = self
- .rate_window_bytes
- .fetch_add(n as u64, Ordering::Relaxed)
- + n as u64;
- let start = self.rate_window_start_ms.load(Ordering::Relaxed);
- if start == 0 {
- self.rate_window_start_ms.store(now, Ordering::Relaxed);
- return;
- }
- let elapsed = now.saturating_sub(start);
- if elapsed >= 1_000 {
- let kbps = ((total * 8) / elapsed.max(1)) as u32;
- self.bitrate_kbps_out.store(kbps, Ordering::Relaxed);
- self.rate_window_bytes.store(0, Ordering::Relaxed);
- self.rate_window_start_ms.store(now, Ordering::Relaxed);
- }
+ pub(crate) fn note_outbound_bytes(&self, n: usize) {
+ self.rate_out.note(n);
}
}
+/// `Controller::resumed_after_seq` when the ring held nothing.
+const NO_SEQ: u64 = u64::MAX;
+/// `Controller::eb_vertical_track` when the config has no vertical track.
+const NO_TRACK: u8 = 0xFF;
+
+/// `Controller::last_hold_end` values.
+const HOLD_END_RESUMED: u8 = 1;
+const HOLD_END_ENDED: u8 = 2;
+
/// What the main loop should do after a graceful shutdown: exit for good, or
/// relaunch a fresh process in place.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
@@ -494,6 +557,68 @@ pub struct Controller {
ingest_alive: AtomicBool,
buffer_building: AtomicBool,
publisher_token: AtomicU64,
+ /// Set when the live publisher sends FCUnpublish or deleteStream, read
+ /// (and cleared) when its connection closes: a disconnect after that
+ /// goodbye is a deliberate stop, not a crash.
+ unpublish_received: AtomicBool,
+ /// When the last video tag arrived (process ms), 0 before the first
+ /// one of a session or while crash protection is off. Used to spot a
+ /// frozen OBS.
+ last_video_tag_ms: AtomicU64,
+ /// OBS froze (a freeze hold opened) and has sent no video since. The
+ /// next video tag clears it and resumes; until then freeze detection
+ /// stays quiet, so a freeze hold that ran out or was ended doesn't
+ /// reopen itself. Kept apart from `hold` to stay lock-free per tag.
+ ingest_frozen: AtomicBool,
+ /// `hold` is Some. Lets the per-tag checks skip the lock.
+ hold_open: AtomicBool,
+ /// The newest ring seq when the last hold opened (`NO_SEQ`: none, or
+ /// OBS is back): the delay tail ends there. A frozen OBS can keep
+ /// sending audio, so the ring's own newest seq would keep moving.
+ hold_tail_end_seq: AtomicU64,
+ /// OBS is gone with no hold covering it (crash protection off, OBS
+ /// stopped on purpose, or the hold ran out): a destination still airing
+ /// the delay tail finishes it, then ends, as if OBS had sent its end
+ /// through the delay too. Not when the streamer ended it.
+ tail_plays_out: AtomicBool,
+ /// "The buffer holds only N s" was logged for the armed delay; cleared
+ /// on the next arm so each new value is checked once.
+ capacity_capped_logged: AtomicBool,
+ /// Process ms of the last "rejected publisher" log line (see
+ /// `log_rejected_publish`).
+ last_rejection_log_ms: AtomicU64,
+ /// The newest ring seq when the last hold ended because OBS came back
+ /// (`NO_SEQ` when the ring was empty): the video before it is from
+ /// before the hold, and a rejoin never starts there.
+ resumed_after_seq: AtomicU64,
+ /// The newest ring seq when a frozen OBS recovered after its hold had
+ /// already ended (`NO_SEQ`: none this session). The destinations ended
+ /// with the hold, so the video before it is over for viewers: a pump
+ /// that starts later never seeds there (see `fresh_from_seq`).
+ replay_floor_seq: AtomicU64,
+ /// How the last hold closed: 0 none yet, 1 resumed, 2 ended.
+ last_hold_end: AtomicU8,
+ /// Crash protection settings, mirrored from Settings by the supervisor.
+ crash_protection: crate::sync::Mutex,
+ /// `crash_protection.enabled`, readable without a lock: the ingest path
+ /// does its crash-protection bookkeeping only while this is set.
+ crash_protection_on: AtomicBool,
+ /// The open crash-protection hold: OBS dropped and destinations are
+ /// being kept live on the reconnect screen. See `crate::crash_hold`.
+ hold: crate::sync::Mutex>,
+ /// Encoded reconnect-screen loops, shared by every destination.
+ pub slate_cache: Arc,
+ /// The latest keyframe of each track the reconnect screen can't
+ /// replace (HEVC, AV1), keyed like `video_seq_headers`: a hold re-sends
+ /// it instead. See `crash_hold::is_held_keyframe`.
+ held_keyframes: crate::sync::Mutex>>,
+ /// The Enhanced Broadcasting config last handed to OBS and the Twitch
+ /// session it opened, so OBS coming back during a hold continues it.
+ eb_session: crate::sync::Mutex>,
+ /// OBS asked for an Enhanced Broadcasting config (the kept one or a new
+ /// one) since the hold opened: it is coming back on that session.
+ /// Cleared when a hold opens.
+ eb_session_claimed: AtomicBool,
// --- Per-destination state, keyed by Destination.id -----------------
// RwLock not Mutex: every `on_tag` (~150-300/s per active stream)
@@ -506,12 +631,17 @@ pub struct Controller {
// Ingest-side stats
ingest_disconnects: AtomicU32,
- bitrate_kbps: AtomicU32, // inbound (from OBS)
- rate_window_bytes: AtomicU64,
- rate_window_start_ms: AtomicU64,
-
- // Discord webhook URL. Empty = disabled. Updated live via update_webhook.
- webhook_url: crate::sync::Mutex,
+ rate_in: RateMeter, // inbound (from OBS)
+ // Video tags only. Viewers decode the video track, so the Twitch
+ // mobile-decoder check compares this, not the total with audio.
+ rate_video_in: RateMeter,
+ // The current publisher connected over loopback, so the OBS profile on
+ // this machine is the one streaming. Set per publisher in `begin_publish`.
+ publisher_local: AtomicBool,
+
+ // The integrations engine, attached once at startup. Events reach it
+ // through `emit`, which never blocks and works from any thread.
+ integrations: std::sync::OnceLock>,
// Required RTMP stream key. Empty = accept any publisher (local default).
// Mirrored from Settings via update_ingest_key so `begin_publish` can
// enforce it without the ingest task needing a settings handle.
@@ -525,10 +655,6 @@ pub struct Controller {
// token out, so `begin_publish` trusts it alongside the ingest key. Bounded
// and TTL'd (see remember_eb_key) so tokens never accumulate.
eb_keys: crate::sync::Mutex>,
- // Wall-clock ms (since UNIX epoch) of last webhook fire. Throttles
- // rapid event sequences (e.g. reconnect flapping) so we never spawn
- // more than one curl every ~2 s.
- webhook_last_fire_ms: AtomicU64,
// Coordination
publish_lock: Mutex<()>,
@@ -539,8 +665,16 @@ pub struct Controller {
// whether Enhanced Broadcasting was caught + flattened.
video_codec: AtomicU8,
audio_codec: AtomicU8,
+ /// The track the last Enhanced Broadcasting config OBS got puts the
+ /// vertical canvas on, `NO_TRACK` when it has none. Finds the vertical
+ /// track when its SPS can't be read (Twitch's HEVC for 2K channels).
+ eb_vertical_track: AtomicU8,
multitrack_video: AtomicBool,
multitrack_audio: AtomicBool,
+ /// OBS is sending a 2nd audio track (TrackId 1, the VOD / clean track)
+ /// in this stream. A destination set to that track then gets only it
+ /// (see `h264::select_audio_bytes`). Reset with the publisher.
+ second_audio_track: AtomicBool,
// --- Measured encoder parameters (per publisher session) ---
// Feed `compat::compat_warning`, which compares them against the
@@ -676,12 +810,27 @@ impl Controller {
ingest_alive: AtomicBool::new(false),
buffer_building: AtomicBool::new(false),
publisher_token: AtomicU64::new(0),
+ unpublish_received: AtomicBool::new(false),
+ last_video_tag_ms: AtomicU64::new(0),
+ ingest_frozen: AtomicBool::new(false),
+ hold_open: AtomicBool::new(false),
+ hold_tail_end_seq: AtomicU64::new(NO_SEQ),
+ tail_plays_out: AtomicBool::new(false),
+ capacity_capped_logged: AtomicBool::new(false),
+ last_rejection_log_ms: AtomicU64::new(0),
+ resumed_after_seq: AtomicU64::new(NO_SEQ),
+ replay_floor_seq: AtomicU64::new(NO_SEQ),
+ last_hold_end: AtomicU8::new(0),
+ crash_protection: crate::sync::Mutex::new(Default::default()),
+ crash_protection_on: AtomicBool::new(false),
+ hold: crate::sync::Mutex::new(None),
+ slate_cache: Arc::default(),
destinations: crate::sync::RwLock::new(HashMap::new()),
ingest_disconnects: AtomicU32::new(0),
- bitrate_kbps: AtomicU32::new(0),
- rate_window_bytes: AtomicU64::new(0),
- rate_window_start_ms: AtomicU64::new(0),
- webhook_url: crate::sync::Mutex::new(String::new()),
+ rate_in: RateMeter::default(),
+ rate_video_in: RateMeter::default(),
+ publisher_local: AtomicBool::new(false),
+ integrations: std::sync::OnceLock::new(),
ingest_key: crate::sync::Mutex::new(String::new()),
// 5 wrong keys then a short exponential lockout. A legit OBS uses
// the right key and clears its record on the first accept, so this
@@ -693,10 +842,13 @@ impl Controller {
std::time::Duration::from_secs(10 * 60),
),
eb_keys: crate::sync::Mutex::new(Vec::new()),
- webhook_last_fire_ms: AtomicU64::new(0),
+ held_keyframes: crate::sync::Mutex::new(std::collections::BTreeMap::new()),
+ eb_session: crate::sync::Mutex::new(None),
+ eb_session_claimed: AtomicBool::new(false),
publish_lock: Mutex::new(()),
video_codec: AtomicU8::new(0),
audio_codec: AtomicU8::new(0),
+ eb_vertical_track: AtomicU8::new(NO_TRACK),
video_dims: AtomicU64::new(0),
idr_window_open: AtomicBool::new(false),
first_idr_ts_ms: AtomicU64::new(0),
@@ -705,6 +857,7 @@ impl Controller {
keyframe_interval_cached: AtomicU32::new(0),
multitrack_video: AtomicBool::new(false),
multitrack_audio: AtomicBool::new(false),
+ second_audio_track: AtomicBool::new(false),
seq_header_gen: AtomicU32::new(0),
last_input_ts_u32: AtomicU32::new(0),
input_ts_wrap_high: AtomicU32::new(0),
@@ -767,6 +920,26 @@ impl Controller {
pub fn video_codec(&self) -> VideoCodec {
dec_vcodec(self.video_codec.load(Ordering::Relaxed))
}
+ /// Remember which track the config just handed to OBS puts the
+ /// vertical canvas on.
+ pub fn note_eb_config(&self, config: &str) {
+ let track = crate::local_eb_config::vertical_track(config).unwrap_or(NO_TRACK);
+ self.eb_vertical_track.store(track, Ordering::Relaxed);
+ }
+ /// The vertical track of the last config handed to OBS.
+ pub fn eb_vertical_track(&self) -> Option {
+ Some(self.eb_vertical_track.load(Ordering::Relaxed)).filter(|&t| t != NO_TRACK)
+ }
+ /// The portrait (9:16) track OBS is sending now. Only H.264 headers can
+ /// be measured; for any other codec, the vertical track the Enhanced
+ /// Broadcasting config named counts once OBS is actually sending it.
+ pub fn vertical_track_on_wire(&self) -> Option {
+ let headers = self.ring.video_seq_headers.lock();
+ crate::h264::detect_vertical_primary_track(&headers).or_else(|| {
+ self.eb_vertical_track()
+ .filter(|track| headers.contains_key(track))
+ })
+ }
pub fn audio_codec(&self) -> AudioCodec {
dec_acodec(self.audio_codec.load(Ordering::Relaxed))
}
@@ -899,16 +1072,28 @@ impl Controller {
forwarding raw to Twitch destinations, flattening to the \
primary resolution for any other platform.",
);
- self.fire_webhook(
- "🎚️",
- "Enhanced Broadcasting detected - multi-track forwarding active.",
- );
+ self.emit(Event::new(EventKind::EbDetected));
}
}
- pub fn note_multitrack_audio(&self) {
+ pub fn note_multitrack_audio(&self, payload: &[u8]) {
if !self.multitrack_audio.swap(true, Ordering::Relaxed) {
self.log("ingest: multi-track audio detected (VOD audio track) - forwarding as-is.");
}
+ if !self.second_audio_track.load(Ordering::Relaxed)
+ && crate::h264::multitrack_audio_has_track(payload, 1)
+ {
+ self.second_audio_track.store(true, Ordering::Relaxed);
+ }
+ }
+
+ /// Whether the audio track `egress` asks for is in this stream (the live
+ /// track always is). See `h264::select_audio_bytes`.
+ pub fn audio_target_on_wire(&self, egress: crate::h264::AudioEgress) -> bool {
+ match egress {
+ crate::h264::AudioEgress::Passthrough | crate::h264::AudioEgress::Track(0) => true,
+ crate::h264::AudioEgress::Track(1) => self.second_audio_track.load(Ordering::Relaxed),
+ crate::h264::AudioEgress::Track(_) => false,
+ }
}
/// Wipe codec/multitrack state when the publisher disconnects so a
/// fresh OBS connect with a different codec starts from a clean slate.
@@ -929,6 +1114,7 @@ impl Controller {
self.keyframe_interval_cached.store(0, Ordering::Relaxed);
self.multitrack_video.store(false, Ordering::Relaxed);
self.multitrack_audio.store(false, Ordering::Relaxed);
+ self.second_audio_track.store(false, Ordering::Relaxed);
self.last_multitrack_video_ms.store(0, Ordering::Relaxed);
self.backpressure_since_ms.store(0, Ordering::Relaxed);
self.last_input_ts_u32.store(0, Ordering::Relaxed);
@@ -999,6 +1185,14 @@ impl Controller {
return last as u64;
}
wrap_high -= 1;
+ } else if last - wire_ts > MAX_TS_STEP_BACK_MS {
+ // Far behind the timeline, and not a wrap: no interleaving is
+ // seconds long. OBS ends an Enhanced Broadcasting stream with
+ // SequenceEnd tags stamped 0, and adopting that stamp made the
+ // buffer's newest moment read 0: the delay still to air looked
+ // like none, and the destination was restarted mid-tail. Pin it
+ // to where we are, as for a stamp from before the session.
+ return ((wrap_high as u64) << 32) | (last as u64);
}
self.input_ts_wrap_high.store(wrap_high, Ordering::Relaxed);
self.last_input_ts_u32.store(wire_ts, Ordering::Relaxed);
@@ -1044,7 +1238,7 @@ impl Controller {
d.id.clone(),
d.egress_alive.load(Ordering::Relaxed),
d.consumer_seq.load(Ordering::Relaxed),
- d.bitrate_kbps_out.load(Ordering::Relaxed),
+ d.rate_out.kbps(),
d.tags_sent.load(Ordering::Relaxed),
d.bytes_sent.load(Ordering::Relaxed),
d.cuts_performed.load(Ordering::Relaxed),
@@ -1090,6 +1284,7 @@ impl Controller {
pub fn arm_delay(&self, ms: u32) {
let ms = ms.min(600_000);
let previous_target = self.target_delay_ms.load(Ordering::Relaxed);
+ self.capacity_capped_logged.store(false, Ordering::Relaxed);
self.armed_delay_ms.store(ms, Ordering::Relaxed);
if ms == 0 {
// Disarm wipes target as well.
@@ -1133,9 +1328,8 @@ impl Controller {
if !self.ingest_alive() {
return Err(ActivateError::NoIngest);
}
- let fill = self.buffer_fill_ms();
- if fill + 500 < armed {
- let remaining_ms = armed.saturating_sub(fill);
+ if !self.buffer_holds(armed) {
+ let remaining_ms = armed.saturating_sub(self.buffer_fill_ms());
return Err(ActivateError::BufferShort { remaining_ms });
}
self.target_delay_ms.store(armed, Ordering::Relaxed);
@@ -1163,6 +1357,12 @@ impl Controller {
// A manual cut supersedes any scheduled "cut after this airs" -
// the streamer chose "now" over "when the mark airs".
self.safe_cut_input_ts.store(0, Ordering::Relaxed);
+ // OBS already left and its delay is still airing: there is no live
+ // to cut to, so the cut ends the stream. Jumping to the newest
+ // keyframe would still air the stream's last moments.
+ if self.tail_left().is_some() {
+ self.end_tail_now();
+ }
}
// --- "Cut after this airs" (scheduled safe cut) -------------------
@@ -1326,12 +1526,19 @@ impl Controller {
if armed == 0 {
return "idle";
}
- if self.buffer_fill_ms() + 500 < armed {
+ if !self.buffer_holds(armed) {
return "preparing";
}
"ready"
}
+ /// Whether the buffer holds a delay of `delay_ms`: it spans that much,
+ /// or it is full and spans all it ever will at this bitrate (the delay
+ /// then runs at what it holds, see `delayed_idr`).
+ fn buffer_holds(&self, delay_ms: u32) -> bool {
+ self.buffer_fill_ms() + 500 >= delay_ms || self.ring.is_saturated()
+ }
+
pub fn ingest_alive(&self) -> bool {
self.ingest_alive.load(Ordering::Relaxed)
}
@@ -1389,32 +1596,22 @@ impl Controller {
self.ingest_disconnects.load(Ordering::Relaxed)
}
pub fn bitrate_kbps(&self) -> u32 {
- self.bitrate_kbps.load(Ordering::Relaxed)
+ self.rate_in.kbps()
+ }
+ /// Inbound video bitrate alone (no audio tracks, no sequence headers).
+ pub fn video_bitrate_kbps(&self) -> u32 {
+ self.rate_video_in.kbps()
+ }
+ /// The current publisher connected from this machine.
+ pub fn publisher_local(&self) -> bool {
+ self.publisher_local.load(Ordering::Relaxed)
}
// ---- Internal: ingest counters ----
- /// Called from the ingest path on every audio/video tag. Maintains a
- /// 1-second rolling bitrate average (kbps) - cheap, lock-free.
+ /// Called from the ingest path on every audio/video tag.
pub fn note_inbound_bytes(&self, n: usize) {
- let now = process_now_ms();
- let total = self
- .rate_window_bytes
- .fetch_add(n as u64, Ordering::Relaxed)
- + n as u64;
- let start = self.rate_window_start_ms.load(Ordering::Relaxed);
- if start == 0 {
- self.rate_window_start_ms.store(now, Ordering::Relaxed);
- return;
- }
- let elapsed = now.saturating_sub(start);
- if elapsed >= 1_000 {
- // Convert bytes/ms → kbps: bytes * 8 / ms
- let kbps = ((total * 8) / elapsed.max(1)) as u32;
- self.bitrate_kbps.store(kbps, Ordering::Relaxed);
- self.rate_window_bytes.store(0, Ordering::Relaxed);
- self.rate_window_start_ms.store(now, Ordering::Relaxed);
- }
+ self.rate_in.note(n);
}
pub fn note_ingest_disconnect(&self) {
@@ -1450,6 +1647,11 @@ impl Controller {
// stream-key identity, so strip it here - otherwise the exact ingest key
// arrives as `mykey?clientConfigId=...` and gets rejected as a wrong key.
let stream_key = stream_key.split('?').next().unwrap_or(stream_key);
+ // An unparseable IP is treated as remote (fail-safe).
+ let local = peer_ip
+ .parse::()
+ .map(|a| a.is_loopback())
+ .unwrap_or(false);
// Ingest auth: when a key is configured, only a publisher using that
// exact key gets in. Empty key (the default) accepts anyone, which is
// the right behaviour on a local machine. Checked before the slot lock
@@ -1462,14 +1664,11 @@ impl Controller {
// behind an HTTP reverse proxy that could mask its IP - so it is
// no brute-force threat and must not be locked out of its own
// machine for a mistyped key. Apply the limiter to remote peers
- // only; an unparseable IP is treated as remote (fail-safe).
- let remote = !peer_ip
- .parse::()
- .map(|a| a.is_loopback())
- .unwrap_or(false);
+ // only.
+ let remote = !local;
// Throttle first so a locked-out guesser burns no work.
if remote && self.ingest_limiter.check(peer_ip).is_err() {
- self.log("ingest: rejected publisher (rate limited)");
+ self.log_rejected_publish("ingest: rejected publisher (rate limited)");
return Err(io::Error::new(
io::ErrorKind::PermissionDenied,
"too many attempts",
@@ -1489,7 +1688,7 @@ impl Controller {
if remote {
self.ingest_limiter.record_failure(peer_ip);
}
- self.log("ingest: rejected publisher (wrong stream key)");
+ self.log_rejected_publish("ingest: rejected publisher (wrong stream key)");
return Err(io::Error::new(
io::ErrorKind::PermissionDenied,
"invalid stream key",
@@ -1505,21 +1704,34 @@ impl Controller {
// interleave its tags into the buffer with its own timestamp
// origin and guarantee a viewer-visible glitch.
if self.ingest_alive.load(Ordering::Relaxed) {
- self.log("ingest: rejected second publisher (slot in use)");
- return Err(io::Error::new(
- io::ErrorKind::AlreadyExists,
- "another publisher is already active",
- ));
+ if !self.video_frozen_at(process_now_ms()) {
+ self.log_rejected_publish("ingest: rejected second publisher (slot in use)");
+ return Err(io::Error::new(
+ io::ErrorKind::AlreadyExists,
+ "another publisher is already active",
+ ));
+ }
+ // The publisher holding the slot has sent no video for
+ // FREEZE_AFTER: a hung OBS whose connection the OS keeps alive
+ // would otherwise lock a restarted OBS out indefinitely. Hand the
+ // slot over; the new token shuts the old connection out (see
+ // `end_publish`). A freeze hold, if one is up, resumes below as
+ // for any returning publisher.
+ self.log("ingest: the connected publisher stopped sending video - handing over to the new one");
+ self.ingest_alive.store(false, Ordering::Relaxed);
+ self.note_ingest_disconnect();
+ self.reset_codec_state();
}
// Wipe every per-session cache that survives mark_ingest_dead.
// The seq-header caches deliberately persist across an in-session
- // egress restart (see `eb_seq_headers_survive_egress_restart`) so
+ // egress restart (see `every_track_config_leads_each_twitch_connection`) so
// we clear them only when a new publisher takes the slot -
// otherwise stale multi-track SPS/PPS leak into a non-EB
// session and freeze the destination's decoder.
self.ring.video_seq_headers.lock().clear();
self.ring.audio_seq_headers.lock().clear();
+ self.held_keyframes.lock().clear();
// onMetaData leaks the same way: the prior publisher's
// resolution / fps / encoder fields would be replayed at every
// pump start until the new publisher's first onMetaData arrives.
@@ -1545,11 +1757,39 @@ impl Controller {
// fire hours later (or never) - clear it with the session.
self.safe_cut_input_ts.store(0, Ordering::Relaxed);
+ // A goodbye from an earlier connection must not mark this session's
+ // end as a clean stop.
+ self.unpublish_received.store(false, Ordering::Relaxed);
+ self.last_video_tag_ms.store(0, Ordering::Relaxed);
+ self.ingest_frozen.store(false, Ordering::Relaxed);
+ // The ring is cleared above: nothing in it is from before a freeze.
+ self.replay_floor_seq.store(NO_SEQ, Ordering::Relaxed);
+ // A hold past its deadline already ended for the pumps. Close it as
+ // ended while ingest still reads as dead, so the sessions it kept
+ // are forgotten rather than carried into this stream.
+ self.expire_hold();
+ let resuming = self.hold_active();
+ if resuming {
+ self.forget_unclaimed_eb_session();
+ }
// Bump token so any prior egress reader knows it's stale.
let token = self.publisher_token.fetch_add(1, Ordering::SeqCst) + 1;
+ self.publisher_local.store(local, Ordering::Relaxed);
self.ingest_alive.store(true, Ordering::Relaxed);
+ self.resume_from_hold();
+ if resuming {
+ // Freeze detection only starts at a session's first video
+ // frame, so an OBS that reconnects and never sends any would
+ // keep the screen up with no deadline. Start its clock now,
+ // with room for a slow encoder start before it counts.
+ let grace = FIRST_VIDEO_GRACE.saturating_sub(crate::crash_hold::FREEZE_AFTER);
+ self.last_video_tag_ms.store(
+ process_now_ms() + grace.as_millis() as u64,
+ Ordering::Relaxed,
+ );
+ }
self.log("ingest: publisher connected");
- self.fire_webhook("✅", "OBS publisher connected - going live.");
+ self.emit(Event::new(EventKind::ObsConnected));
Ok(token)
}
@@ -1563,11 +1803,9 @@ impl Controller {
// which are tiny and one-shot).
if !is_seq {
self.note_inbound_bytes(payload.len());
- }
- // Bump the generation on every seq header so each egress pump
- // knows to re-emit its cached config bytes on the next iteration.
- if is_seq {
- self.seq_header_gen.fetch_add(1, Ordering::Relaxed);
+ if kind == 9 {
+ self.rate_video_in.note(payload.len());
+ }
}
// Sample the encoder parameters the compatibility check compares
// against the enabled destinations. Both are cheap: the dimension
@@ -1576,11 +1814,44 @@ impl Controller {
if kind == 9 {
if is_seq {
self.note_video_dimensions(payload);
- } else if is_idr {
- self.sample_keyframe_interval(ts_ms);
+ } else {
+ if self.crash_protection_on.load(Ordering::Relaxed) {
+ self.note_video_for_crash_protection(payload);
+ }
+ if self.ingest_frozen.load(Ordering::Relaxed) {
+ self.ingest_frozen.store(false, Ordering::Relaxed);
+ // Back in time, the pumps rejoin past the freeze on
+ // their own and a short freeze's tail may still be
+ // airing. Everything before the freeze is behind viewers
+ // once the hold is over (ended or run out), or once the
+ // delay has moved past it: a freeze longer than the
+ // delay aired its whole tail, and the screen took over.
+ let tail_aired = self.ring.latest_ts().is_some_and(|last| {
+ ts_ms.saturating_sub(u64::from(self.target_delay_ms())) > last
+ });
+ let hold_was_over = !self.hold_active() || tail_aired;
+ // A hold past its deadline already ended for the pumps:
+ // report it as ended, not as OBS being back in time.
+ self.expire_hold();
+ self.resume_from_hold();
+ if hold_was_over {
+ self.mark_replay_floor();
+ }
+ }
+ if is_idr {
+ self.sample_keyframe_interval(ts_ms);
+ }
}
}
let _ = self.ring.append(kind, ts_ms, payload, is_idr, is_seq);
+ // Bump the generation on every seq header so each egress pump
+ // knows to re-emit its cached config bytes on the next iteration.
+ // After the append: a pump that sees the new generation must find
+ // the new header in the cache, or it would resend the old one and
+ // never learn of the change.
+ if is_seq {
+ self.seq_header_gen.fetch_add(1, Ordering::Relaxed);
+ }
// Cap the buffer to the user's armed delay (plus a small slack for
// IDR alignment). This keeps the on-screen "Buffer N/N s" exactly
// what the user asked for, instead of growing to the full disk cap.
@@ -1588,9 +1859,38 @@ impl Controller {
// so we never evict a tag any pump is still about to read.
if !is_seq {
let target = self.effective_target_buffer_ms();
- let min_consumer = self.min_consumer_seq();
- self.ring.trim_older_than(target, ts_ms, min_consumer);
+ // Never trim the keyframe a delay of the whole buffer lands on
+ // (see `delayed_idr`): with a GOP longer than the slack, it is
+ // older than the cutoff, and without it the delay can't join.
+ let delay_keyframe = self
+ .ring
+ .newest_idr_at_or_before(ts_ms.saturating_sub(u64::from(self.target_buffer_ms())))
+ .map_or(u64::MAX, |idr| idr.seq);
+ let keep_from = self.min_consumer_seq().min(delay_keyframe);
+ self.ring.trim_older_than(target, ts_ms, keep_from);
+ self.note_capacity_cap();
+ }
+ }
+
+ /// Say once per armed value when the buffer is full before it spans the
+ /// armed delay: the delay then runs at what the buffer holds.
+ fn note_capacity_cap(&self) {
+ let armed = self.armed_delay_ms();
+ let fill = self.buffer_fill_ms();
+ if armed == 0
+ || self.capacity_capped_logged.load(Ordering::Relaxed)
+ || fill + 500 >= armed
+ || !self.ring.is_saturated()
+ {
+ return;
}
+ self.capacity_capped_logged.store(true, Ordering::Relaxed);
+ self.log(format!(
+ "delay: the buffer is full at {} s at this bitrate, short of the {} s armed, so the delay runs at {} s. Raise the buffer size in System for the full delay.",
+ fill / 1000,
+ armed / 1000,
+ fill / 1000
+ ));
}
/// Slowest consumer across all destinations. Used by trim to ensure
@@ -1701,28 +2001,110 @@ impl Controller {
self.target_buffer_ms() + BUFFER_SLACK_MS
}
+ /// Whether a delayed destination can join right now: no delay is on,
+ /// or the buffer reaches back to it (see `delayed_idr`).
+ pub fn delay_reachable(&self) -> bool {
+ let target = u64::from(self.target_delay_ms());
+ target == 0 || delayed_idr(self, target).is_some()
+ }
+
pub fn on_metadata(&self, payload: Vec) {
*self.ring.metadata.lock() = Some(payload);
}
+ /// A publishing connection closed. Only the current publisher's closing
+ /// counts: one that was handed over (`begin_publish`) has a stale token
+ /// and must not end the session that replaced it.
+ pub fn end_publish(&self, token: u64) {
+ if token == self.publisher_token() {
+ self.mark_ingest_dead();
+ }
+ }
+
+ /// Log a refused publish at most once every 10 s: a peer reconnecting
+ /// in a loop would otherwise push everything else out of the bounded
+ /// log.
+ fn log_rejected_publish(&self, line: &str) {
+ // Stamped one ms late, so 0 only ever means "never" (the process
+ // clock itself starts at 0).
+ let stamp = process_now_ms() + 1;
+ let last = self.last_rejection_log_ms.load(Ordering::Relaxed);
+ if last != 0 && stamp.saturating_sub(last) < 10_000 {
+ return;
+ }
+ self.last_rejection_log_ms.store(stamp, Ordering::Relaxed);
+ self.log(line.to_string());
+ }
+
pub fn mark_ingest_dead(&self) {
+ // Only the live publisher's own disconnect gets here (its
+ // PublishGuard), once per session.
+ if !self.ingest_alive() {
+ return;
+ }
+ // Decide the hold before ingest reads as dead: a pump that sees
+ // ingest gone with no hold open ends its destination.
+ let stopped = self.unpublish_received.swap(false, Ordering::Relaxed);
+ if !stopped {
+ self.start_hold(crate::crash_hold::HoldReason::Crash);
+ } else if self.crash_protection.lock().every_disconnect {
+ self.start_hold(crate::crash_hold::HoldReason::Stopped);
+ } else if self.close_hold(HOLD_END_ENDED).is_some() {
+ // Stopped on purpose while a freeze hold was up: that ends it.
+ self.log("crash protection: OBS stopped the stream - destinations ended");
+ }
+ // No hold took over: what the delay still holds airs, then the
+ // destinations end. Viewers see the stream end when they would have
+ // seen everything else, not the delay's length early.
+ if !self.hold_active() {
+ self.hold_tail_end_seq
+ .store(self.ring.latest_seq().unwrap_or(NO_SEQ), Ordering::Relaxed);
+ self.tail_plays_out.store(true, Ordering::Relaxed);
+ let behind = self.current_delay_ms();
+ if behind >= 1_000 {
+ self.log(format!(
+ "delay: airing the last {} s before the destinations end (End now cuts it)",
+ behind / 1000
+ ));
+ }
+ }
// Only count when transitioning alive → dead, so a stray call
// doesn't inflate the counter.
if self.ingest_alive.swap(false, Ordering::Relaxed) {
self.note_ingest_disconnect();
self.reset_codec_state();
+ if stopped {
+ self.log("ingest: publisher stopped the stream");
+ } else {
+ self.log(
+ "ingest: publisher dropped without stopping (crash, kill or network loss)",
+ );
+ }
+ // `protected`: a hold opened and has its own, more useful event.
+ self.emit(
+ Event::new(EventKind::ObsDisconnected)
+ .with("stopped", if stopped { "yes" } else { "no" })
+ .with("protected", if self.hold_active() { "yes" } else { "no" }),
+ );
// Clear any Enhanced Broadcasting URL overrides on the
// way out - the next stream may or may not be EB, and a
// stale override would force a non-EB stream onto an IVS
// endpoint that has no allocated session. The
// /obs/multitrack-config proxy sets a fresh override on
// every new EB session anyway.
+ //
+ // A destination a crash-protection hold keeps live keeps its
+ // session: OBS coming back continues it (see `held_eb_config`),
+ // and the hold ending without OBS forgets it (`close_hold`).
for (_id, state) in self.all_destination_states() {
- // Clear the override AND bump the session epoch atomically,
- // so a VOD-session fetch still in flight discards its stale
- // result instead of writing a dead-session IVS URL into the
- // next stream (the late-completion race).
- state.invalidate_session_override();
+ if !self.hold_keeps(&state) {
+ // Clear the override AND bump the session epoch
+ // atomically, so a VOD-session fetch still in flight
+ // discards its stale result instead of writing a
+ // dead-session IVS URL into the next stream (the
+ // late-completion race).
+ state.invalidate_session_override();
+ }
// Note: we deliberately do NOT touch `vod_fetch_pending`
// here. It's owned solely by the fetch lifecycle (claim sets
// it, the task clears it on completion within ~15 s). If a
@@ -1731,15 +2113,413 @@ impl Controller {
// being spawned for the same destination - clearing it here
// would reintroduce the multi-session bug on a fast restart.
}
- self.log("ingest: publisher disconnected");
- self.fire_webhook("⚠️", "OBS publisher disconnected.");
}
}
- /// Update the Discord webhook URL - call when settings change. Empty
- /// string disables webhook delivery entirely.
- pub fn update_webhook(&self, url: String) {
- *self.webhook_url.lock() = url;
+ /// Whether the open hold keeps `dest` live on the reconnect screen.
+ fn hold_keeps(&self, dest: &DestinationState) -> bool {
+ self.hold_active()
+ && dest.egress_alive.load(Ordering::Relaxed)
+ && crate::crash_hold::covers(self, dest)
+ }
+
+ /// Remember the Enhanced Broadcasting config just handed to OBS and
+ /// the Twitch session it opened.
+ pub fn remember_eb_session(&self, session: EbSession) {
+ *self.eb_session.lock() = Some(session);
+ // OBS asked for this session, so it is Enhanced Broadcasting's own
+ // even when a hold is open (one that had no session to hand back):
+ // OBS publishing next must continue it, not drop it as stale.
+ self.eb_session_claimed.store(true, Ordering::Relaxed);
+ }
+
+ /// The config to hand OBS again when it asks during a hold that is
+ /// keeping its Twitch session live: the same tracks and session token,
+ /// so its new publish continues the session instead of opening another
+ /// one (which would restart the destination once OBS is back).
+ pub fn held_eb_config(&self) -> Option {
+ if !self.hold_active() {
+ return None;
+ }
+ let session = self.eb_session.lock().clone()?;
+ let dest = self.destinations.read().get(&session.dest_id).cloned()?;
+ if dest.eb_override_url.lock().as_deref() != Some(session.ivs_url.as_str()) {
+ return None;
+ }
+ // The tokens were remembered when the session opened, possibly
+ // longer ago than their TTL.
+ for auth in &session.auths {
+ self.remember_eb_key(auth.clone());
+ }
+ self.eb_session_claimed.store(true, Ordering::Relaxed);
+ Some(session.config)
+ }
+
+ /// OBS is publishing again during a hold without having asked for an
+ /// Enhanced Broadcasting config (held or fresh) since the hold opened:
+ /// Enhanced Broadcasting is off now, and that multitrack session can't
+ /// take a single-track stream, so the destination starts a fresh one.
+ fn forget_unclaimed_eb_session(&self) {
+ if self.eb_session_claimed.load(Ordering::Relaxed) {
+ return;
+ }
+ let Some(session) = self.eb_session.lock().take() else {
+ return;
+ };
+ if let Some(dest) = self.destinations.read().get(&session.dest_id) {
+ if dest.eb_override_url.lock().as_deref() == Some(session.ivs_url.as_str()) {
+ dest.invalidate_session_override();
+ self.log(format!(
+ "[{}] OBS came back without Enhanced Broadcasting - starting a new \
+ Twitch session",
+ session.dest_id
+ ));
+ }
+ }
+ }
+
+ /// The newest ring seq when OBS came back from the last hold, or None
+ /// when the ring was empty (a new publisher starts it afresh).
+ pub fn resumed_after_seq(&self) -> Option {
+ Some(self.resumed_after_seq.load(Ordering::Relaxed)).filter(|seq| *seq != NO_SEQ)
+ }
+
+ /// The first ring seq a destination may start from. When a frozen OBS
+ /// recovers after its hold ended, everything it sent before the freeze
+ /// is over for viewers (aired as the delay tail, or cut by End now), so
+ /// a pump that starts later, or a cut back to restore the delay, must
+ /// never land there and replay it. 0 otherwise.
+ fn fresh_from_seq(&self) -> u64 {
+ match self.replay_floor_seq.load(Ordering::Relaxed) {
+ NO_SEQ => 0,
+ seq => seq + 1,
+ }
+ }
+
+ /// Everything in the ring now is over for viewers (see `fresh_from_seq`).
+ fn mark_replay_floor(&self) {
+ if let Some(latest) = self.ring.latest_seq() {
+ self.replay_floor_seq.store(latest, Ordering::Relaxed);
+ }
+ }
+
+ /// The latest keyframe of a track the hold re-sends instead of the
+ /// reconnect screen.
+ pub fn held_keyframe(&self, track: u8) -> Option> {
+ self.held_keyframes.lock().get(&track).cloned()
+ }
+
+ /// The live publisher sent FCUnpublish or deleteStream: it is ending
+ /// the stream on purpose, so its disconnect is a stop, not a crash.
+ pub fn note_unpublish(&self) {
+ self.unpublish_received.store(true, Ordering::Relaxed);
+ }
+
+ /// Mirror the crash-protection settings (the supervisor calls this).
+ pub fn update_crash_protection(&self, settings: crate::crash_protection::CrashProtection) {
+ self.crash_protection_on
+ .store(settings.enabled, Ordering::Relaxed);
+ if !settings.enabled {
+ // Off means no bookkeeping, and nothing stale left behind: an
+ // old video stamp would read as a freeze and hold back egress.
+ self.last_video_tag_ms.store(0, Ordering::Relaxed);
+ // Switched off mid-freeze: no reconnect screen any more, so the
+ // freeze ends the stream like any OBS drop without protection,
+ // and when OBS is back the destinations start again past it.
+ if self.ingest_frozen.swap(false, Ordering::Relaxed) {
+ self.mark_replay_floor();
+ self.end_hold_now();
+ }
+ self.held_keyframes.lock().clear();
+ self.slate_cache.clear();
+ }
+ *self.crash_protection.lock() = settings;
+ }
+
+ /// Per video frame while crash protection is on: when video last
+ /// arrived (for freeze detection), and the latest keyframe of any track
+ /// the reconnect screen can't replace.
+ fn note_video_for_crash_protection(&self, payload: &[u8]) {
+ // Never 0: that means "no video yet this session".
+ self.last_video_tag_ms
+ .store(process_now_ms().max(1), Ordering::Relaxed);
+ if crate::crash_hold::is_held_keyframe(payload) {
+ let track = crate::h264::seq_header_track_id(payload);
+ self.held_keyframes.lock().insert(track, Arc::from(payload));
+ }
+ }
+
+ pub fn crash_protection(&self) -> crate::crash_protection::CrashProtection {
+ self.crash_protection.lock().clone()
+ }
+
+ /// Whether destinations are being held on the reconnect screen now.
+ pub fn hold_active(&self) -> bool {
+ self.hold_open.load(Ordering::Relaxed)
+ && self.hold_state() == crate::crash_hold::HoldState::Holding
+ }
+
+ /// Where the hold stands. A hold past its deadline already counts as
+ /// ended, before `expire_hold` gets round to reporting it.
+ pub fn hold_state(&self) -> crate::crash_hold::HoldState {
+ use crate::crash_hold::HoldState;
+ match *self.hold.lock() {
+ Some(hold) if Instant::now() < hold.deadline => HoldState::Holding,
+ Some(_) => HoldState::Ended,
+ None if self.last_hold_end.load(Ordering::Relaxed) == HOLD_END_RESUMED => {
+ HoldState::Resumed
+ }
+ None => HoldState::Ended,
+ }
+ }
+
+ /// The hold while it is on air, for the dashboard, dock and tray.
+ pub fn hold_status(&self) -> Option {
+ let hold = (*self.hold.lock())?;
+ Some(crate::crash_hold::HoldStatus {
+ reason: hold.reason,
+ remaining: hold.deadline.checked_duration_since(Instant::now())?,
+ total: hold.deadline - hold.started,
+ })
+ }
+
+ /// Open a hold when crash protection is on and a destination is live
+ /// to protect.
+ fn start_hold(&self, reason: crate::crash_hold::HoldReason) {
+ let settings = self.crash_protection();
+ let any_live = self
+ .all_destination_states()
+ .iter()
+ .any(|(_, state)| state.egress_alive.load(Ordering::Relaxed));
+ if !settings.enabled || !any_live {
+ return;
+ }
+ let hold_for = Duration::from_secs(u64::from(settings.hold_secs));
+ let now = Instant::now();
+ {
+ let mut hold = self.hold.lock();
+ if hold.is_some() {
+ return;
+ }
+ *hold = Some(crate::crash_hold::Hold {
+ reason,
+ started: now,
+ deadline: now + hold_for,
+ });
+ }
+ self.hold_open.store(true, Ordering::Relaxed);
+ self.hold_tail_end_seq
+ .store(self.ring.latest_seq().unwrap_or(NO_SEQ), Ordering::Relaxed);
+ self.tail_plays_out.store(false, Ordering::Relaxed);
+ self.eb_session_claimed.store(false, Ordering::Relaxed);
+ if reason == crate::crash_hold::HoldReason::Freeze {
+ self.ingest_frozen.store(true, Ordering::Relaxed);
+ }
+ let window = crate::crash_hold::minutes_seconds(hold_for);
+ let what = match reason {
+ crate::crash_hold::HoldReason::Crash => "OBS dropped",
+ crate::crash_hold::HoldReason::Freeze => "OBS stopped sending video",
+ crate::crash_hold::HoldReason::Stopped => "OBS stopped the stream",
+ };
+ self.log(format!(
+ "crash protection: {what} - destinations stay live on the reconnect screen for up to {window}"
+ ));
+ self.emit(
+ Event::new(EventKind::HoldOpened)
+ .with(
+ "reason",
+ match reason {
+ crate::crash_hold::HoldReason::Crash => "crash",
+ crate::crash_hold::HoldReason::Freeze => "freeze",
+ crate::crash_hold::HoldReason::Stopped => "stopped",
+ },
+ )
+ .with("hold", fmt_duration(hold_for.as_millis() as u64))
+ .with(
+ "hold_ends_at",
+ (crate::integrations::twitch::unix_ms() / 1000 + hold_for.as_secs())
+ .to_string(),
+ ),
+ );
+ }
+
+ /// Close the open hold, remembering how it ended for the pumps.
+ fn close_hold(&self, end: u8) -> Option {
+ // How it ended is stored before the lock is released, so
+ // `hold_state` never reads "no hold" with the previous hold's end.
+ let hold = {
+ let mut open = self.hold.lock();
+ let hold = open.take()?;
+ self.last_hold_end.store(end, Ordering::Relaxed);
+ hold
+ };
+ self.hold_open.store(false, Ordering::Relaxed);
+ // OBS never came back: forget the sessions the hold kept for it
+ // (see `mark_ingest_dead`); ending the hold ended them on Twitch. A
+ // frozen OBS is still connected, so its session is still the
+ // current one, and a config OBS asked for during the hold (the kept
+ // session or a new one) is the one it is about to publish on.
+ let obs_is_coming_back = self.eb_session_claimed.load(Ordering::Relaxed);
+ if end == HOLD_END_ENDED && !self.ingest_alive() && !obs_is_coming_back {
+ for (_id, state) in self.all_destination_states() {
+ state.invalidate_session_override();
+ }
+ }
+ Some(hold)
+ }
+
+ /// OBS is sending again (a new publisher, or video after a freeze).
+ /// Runs before the tag that brought it back reaches the ring.
+ fn resume_from_hold(&self) {
+ let after = self.ring.latest_seq().unwrap_or(NO_SEQ);
+ self.resumed_after_seq.store(after, Ordering::Relaxed);
+ self.tail_plays_out.store(false, Ordering::Relaxed);
+ self.hold_tail_end_seq.store(NO_SEQ, Ordering::Relaxed);
+ let Some(hold) = self.close_hold(HOLD_END_RESUMED) else {
+ return;
+ };
+ let lasted = crate::crash_hold::minutes_seconds(hold.started.elapsed());
+ self.log(format!(
+ "crash protection: OBS is back after {lasted} - resuming live"
+ ));
+ self.emit(Event::new(EventKind::ObsBack).with(
+ "down_for",
+ fmt_duration(hold.started.elapsed().as_millis() as u64),
+ ));
+ }
+
+ /// Whether the publisher has sent no video for `FREEZE_AFTER` as of
+ /// `now_ms` (process ms). Never true before its first video frame.
+ fn video_frozen_at(&self, now_ms: u64) -> bool {
+ let last = self.last_video_tag_ms.load(Ordering::Relaxed);
+ last != 0
+ && now_ms.saturating_sub(last) >= crate::crash_hold::FREEZE_AFTER.as_millis() as u64
+ }
+
+ /// A connected OBS that has sent no video for `FREEZE_AFTER` is frozen:
+ /// open a hold for it (the supervisor calls this every tick).
+ pub fn check_ingest_freeze(&self) {
+ self.check_ingest_freeze_at(process_now_ms());
+ }
+
+ fn check_ingest_freeze_at(&self, now_ms: u64) {
+ let latched = self.ingest_frozen.load(Ordering::Relaxed);
+ if !latched
+ && self.ingest_alive()
+ && self.video_frozen_at(now_ms)
+ && self.hold.lock().is_none()
+ {
+ self.start_hold(crate::crash_hold::HoldReason::Freeze);
+ }
+ }
+
+ /// Whether OBS is connected and actually sending video, as opposed to
+ /// connected but frozen. Egress waits for this before connecting.
+ pub fn ingest_sending(&self) -> bool {
+ self.ingest_alive() && !self.video_frozen_at(process_now_ms())
+ }
+
+ /// OBS is gone: disconnected, or frozen with a freeze hold opened for
+ /// it and no video since. Nothing new reaches the buffer either way.
+ pub fn obs_gone(&self) -> bool {
+ !self.ingest_alive() || self.ingest_frozen.load(Ordering::Relaxed)
+ }
+
+ /// Whether a destination still airing the delay tail of an OBS that is
+ /// gone finishes it before ending: yes once crash protection's time ran
+ /// out (the tail airs in full), not when the streamer ended the hold.
+ fn tail_plays_out(&self) -> bool {
+ self.tail_plays_out.load(Ordering::Relaxed)
+ || matches!(*self.hold.lock(), Some(open) if Instant::now() >= open.deadline)
+ }
+
+ /// The last tag of the delay tail: the newest when the hold opened, or
+ /// the newest now when no hold marked it.
+ fn delay_tail_end_seq(&self) -> Option {
+ match self.hold_tail_end_seq.load(Ordering::Relaxed) {
+ NO_SEQ => self.ring.latest_seq(),
+ seq => Some(seq),
+ }
+ }
+
+ /// Close a hold whose time ran out. The pumps end their sessions at the
+ /// deadline on their own; this reports it once (the supervisor calls it
+ /// every tick).
+ pub fn expire_hold(&self) {
+ let expired = matches!(*self.hold.lock(), Some(open) if Instant::now() >= open.deadline);
+ let Some(hold) = expired.then(|| self.close_hold(HOLD_END_ENDED)).flatten() else {
+ return;
+ };
+ self.tail_plays_out
+ .store(self.obs_gone(), Ordering::Relaxed);
+ let window = crate::crash_hold::minutes_seconds(hold.deadline - hold.started);
+ self.log(format!(
+ "crash protection: OBS didn't come back within {window} - destinations ended"
+ ));
+ self.emit(Event::new(EventKind::HoldExpired).with(
+ "hold",
+ fmt_duration((hold.deadline - hold.started).as_millis() as u64),
+ ));
+ }
+
+ /// End the hold now (dashboard, dock, tray or hotkey), or the delay
+ /// tail still airing after OBS left. True when either was on.
+ pub fn end_hold_now(&self) -> bool {
+ let Some(hold) = self.close_hold(HOLD_END_ENDED) else {
+ return self.end_tail_now();
+ };
+ self.tail_plays_out.store(false, Ordering::Relaxed);
+ let lasted = crate::crash_hold::minutes_seconds(hold.started.elapsed());
+ self.log(format!(
+ "crash protection: ended after {lasted} - destinations ended"
+ ));
+ self.emit(Event::new(EventKind::HoldEnded).with(
+ "down_for",
+ fmt_duration(hold.started.elapsed().as_millis() as u64),
+ ));
+ true
+ }
+
+ /// Stop airing the delay tail: the destinations end at once.
+ fn end_tail_now(&self) -> bool {
+ if self.tail_left().is_none() {
+ return false;
+ }
+ self.tail_plays_out.store(false, Ordering::Relaxed);
+ self.log("delay: ended now - the rest of the delay didn't air");
+ true
+ }
+
+ /// How much of the delay is still to air after OBS left with no hold
+ /// covering it: `None` once no destination is airing it.
+ pub fn tail_left(&self) -> Option {
+ if !self.obs_gone() || self.hold_active() || !self.tail_plays_out() {
+ return None;
+ }
+ let airing = self
+ .all_destination_states()
+ .iter()
+ .any(|(_, state)| state.egress_alive.load(Ordering::Relaxed));
+ let left = self.current_delay_ms();
+ (airing && left > 0).then(|| Duration::from_millis(u64::from(left)))
+ }
+
+ /// Attach the integrations engine. Called once at startup; events
+ /// emitted before it is attached are dropped.
+ pub fn attach_integrations(&self, handle: Arc) {
+ let _ = self.integrations.set(handle);
+ }
+
+ pub fn integrations(&self) -> Option<&Arc> {
+ self.integrations.get()
+ }
+
+ /// Tell the integrations something happened. Never blocks and needs no
+ /// runtime, so it is safe from the tray, hotkey and MIDI threads.
+ pub fn emit(&self, event: Event) {
+ if let Some(handle) = self.integrations.get() {
+ handle.emit(event);
+ }
}
/// Mirror the required ingest stream key from Settings. Empty disables the
@@ -1781,60 +2561,26 @@ impl Controller {
&& crate::crypto::constant_time_eq(k.as_bytes(), key.as_bytes())
})
}
+}
- /// Snapshot the current webhook URL. Used by the test endpoint so it
- /// can route the request with verbose error reporting instead of
- /// going through `fire_webhook` (which is fire-and-forget and
- /// silently swallows everything from empty-URL to TLS failures).
- pub fn webhook_url_snapshot(&self) -> String {
- self.webhook_url.lock().clone()
- }
-
- /// Fire-and-forget Discord post. Skips silently when no webhook is
- /// configured, OR when the last fire was less than 2 s ago (rate
- /// limit - prevents subprocess spam if a destination flaps).
- ///
- /// Uses `ureq` (tiny blocking HTTPS client, ~150 KB) wrapped in
- /// `spawn_blocking` so the actual TCP+TLS work doesn't park the
- /// current-thread runtime. Previously shelled out to `curl`, which
- /// (a) silently failed when `curl.exe` wasn't on PATH and (b) made
- /// "runtime deps" technically include the system curl binary.
- pub fn fire_webhook(&self, emoji: &str, message: &str) {
- let url = self.webhook_url.lock().clone();
- if url.is_empty() {
- return;
- }
-
- // Throttle: skip if we fired less than 2 s ago.
- let now = process_now_ms();
- let last = self.webhook_last_fire_ms.load(Ordering::Relaxed);
- if now.saturating_sub(last) < 2_000 {
- return;
- }
- self.webhook_last_fire_ms.store(now, Ordering::Relaxed);
-
- let content = format!("{emoji} **InstantClone**: {message}");
- let body = format!(r#"{{"content":"{}"}}"#, json_escape_inline(&content));
-
- tokio::spawn(async move {
- // Spawn-blocking + 10 s outer timeout so a wedged Discord
- // edge can't keep a thread tied up forever. Ignore the
- // result: webhooks are fire-and-forget by design.
- let _ = tokio::time::timeout(
- Duration::from_secs(10),
- tokio::task::spawn_blocking(move || {
- let _ = crate::https::https_agent()
- .post(&url)
- .config()
- .timeout_connect(Some(Duration::from_secs(5)))
- .timeout_global(Some(Duration::from_secs(8)))
- .build()
- .header("Content-Type", "application/json")
- .send(&body);
- }),
- )
- .await;
- });
+/// The platform a destination's ingest host belongs to, for the `{platform}`
+/// variable of destination events.
+fn platform_of_host(host: &str) -> &'static str {
+ let host = host.to_ascii_lowercase();
+ if host.contains("twitch") {
+ "twitch"
+ } else if host.contains("youtube") || host.contains("google") {
+ "youtube"
+ } else if host.contains("kick") || host.contains("live-video.net") {
+ "kick"
+ } else if host.contains("trovo") {
+ "trovo"
+ } else if host.contains("restream") {
+ "restream"
+ } else if host == "127.0.0.1" || host == "localhost" {
+ "sink"
+ } else {
+ "custom"
}
}
@@ -1909,6 +2655,27 @@ impl Controller {
problem
}
+ /// Set the delay to `ms` (an integration's "set the delay"): arms it
+ /// when nothing is armed, and changes it in place when a delay is armed
+ /// or on air. Unlike the `arm` hotkey action, which toggles, this never
+ /// disarms, so `!setdelay 30` twice leaves a 30 s delay.
+ pub fn set_delay_to(&self, ms: u32, source: &str) {
+ let ms = ms.clamp(1000, 600_000);
+ self.arm_delay(ms);
+ self.log(format!("[{source}] delay set to {} s", ms / 1000));
+ self.record_fired_action("arm", source, None);
+ self.state_dirty.notify_one();
+ }
+
+ /// Back to no delay at all (an integration's "Disarm"): live, and
+ /// nothing armed. What the dashboard's "Cancel arming" does.
+ pub fn disarm(&self, source: &str) {
+ self.arm_delay(0);
+ self.log(format!("[{source}] delay disarmed"));
+ self.record_fired_action("disarm", source, None);
+ self.state_dirty.notify_one();
+ }
+
fn dispatch_named_action(&self, action: &str, default_ms: u32, source: &str) -> Option {
match action {
"toggle" => self.action_toggle(default_ms, source),
@@ -1920,6 +2687,15 @@ impl Controller {
None
}
"cut_after" => self.action_cut_after(source),
+ "end_hold" => {
+ if self.end_hold_now() {
+ return None;
+ }
+ self.log(format!(
+ "[{source}] end crash protection ignored - nothing is on hold"
+ ));
+ Some("Crash protection isn't holding anything right now".to_string())
+ }
_ => None,
}
}
@@ -2043,31 +2819,6 @@ impl Controller {
}
}
-/// JSON-string escape that handles every C0 control char that would
-/// otherwise produce an invalid Discord payload (the previous
-/// `replace('\\', ..).replace('"', ..).replace('\n', ..)` chain missed
-/// `\r`, `\t`, `\u{0008}` and friends - any destination name with a
-/// stray control character could nuke the webhook body).
-fn json_escape_inline(s: &str) -> String {
- let mut out = String::with_capacity(s.len() + 8);
- for c in s.chars() {
- match c {
- '"' => out.push_str(r#"\""#),
- '\\' => out.push_str(r"\\"),
- '\n' => out.push_str(r"\n"),
- '\r' => out.push_str(r"\r"),
- '\t' => out.push_str(r"\t"),
- '\u{0008}' => out.push_str(r"\b"),
- '\u{000C}' => out.push_str(r"\f"),
- c if (c as u32) < 0x20 => {
- out.push_str(&format!("\\u{:04x}", c as u32));
- }
- c => out.push(c),
- }
- }
- out
-}
-
// ---------------------------------------------------------------------------
// Egress driver - the timing & cut-alignment core.
// ---------------------------------------------------------------------------
@@ -2095,6 +2846,7 @@ pub async fn run_egress(
let mut backoff = Duration::from_secs(1);
let max_backoff = Duration::from_secs(30);
+ let mut waiting_for_delay = false;
loop {
// Cooperative shutdown - check BEFORE attempting another
@@ -2119,28 +2871,79 @@ pub async fn run_egress(
// spam visible in /logs. Wait for ingest to come back instead,
// polling cheaply every 500 ms so we react quickly to OBS
// resuming a publish session.
- if !ctrl.ingest_alive() {
+ // During a crash-protection hold a destination that lost its
+ // platform connection reconnects and goes straight back onto the
+ // reconnect screen - if the screen can cover it at all.
+ let holding = ctrl.hold_active() && crate::crash_hold::covers(&ctrl, &dest);
+ if !ctrl.ingest_sending() && !holding {
tokio::time::sleep(Duration::from_millis(500)).await;
continue;
}
- eprintln!(
- "[egress {}] connecting to {}:{}/{}",
- label, parsed.host, parsed.port, parsed.app
- );
+ // With a delay on, join only once the buffer reaches back to it (an
+ // OBS restart empties it; a new destination may find it short).
+ // Joining sooner would put viewers on live video, then jump back.
+ if !holding && !ctrl.delay_reachable() {
+ if !waiting_for_delay {
+ waiting_for_delay = true;
+ ctrl.log(format!(
+ "[{label}] waiting for the buffer to reach the delay before going live"
+ ));
+ }
+ tokio::time::sleep(Duration::from_millis(250)).await;
+ continue;
+ }
+ waiting_for_delay = false;
+ eprintln!(
+ "[egress {}] connecting to {}:{}/{}",
+ label, parsed.host, parsed.port, parsed.app
+ );
ctrl.log(format!(
"[{}] connecting to {}:{}",
label, parsed.host, parsed.port
));
- match EgressClient::connect(&parsed).await {
+ // Bounded: an edge that accepts TCP and then stalls in the TLS or
+ // RTMP handshake would otherwise hold this destination on
+ // "connecting" forever, never retrying.
+ let connected = tokio::time::timeout(CONNECT_TIMEOUT, EgressClient::connect(&parsed))
+ .await
+ .unwrap_or_else(|_| {
+ Err(io::Error::new(
+ io::ErrorKind::TimedOut,
+ format!("no answer within {} s", CONNECT_TIMEOUT.as_secs()),
+ ))
+ });
+ match connected {
Ok(client) => {
- backoff = Duration::from_secs(1);
+ let connected_at = Instant::now();
let was_alive = dest.egress_alive.swap(true, Ordering::Relaxed);
if !was_alive {
- ctrl.fire_webhook("🟢", &format!("**{}** is now live.", label));
+ ctrl.emit(
+ Event::new(EventKind::DestinationLive)
+ .with("destination", label.clone())
+ .with("platform", platform_of_host(&parsed.host)),
+ );
}
let sink = client.spawn_reader_drain();
let pump_result = pump_dest(&ctrl, &dest, sink).await;
dest.egress_alive.store(false, Ordering::Relaxed);
+ // Not reading any more: its position must not hold back the
+ // buffer's trim while it reconnects or waits.
+ dest.consumer_seq.store(u64::MAX, Ordering::Relaxed);
+ // Only a platform connection that failed counts as a
+ // reconnect. The pump also ends cleanly when OBS goes away
+ // or the destination is switched off; neither is one.
+ let dropped = pump_result.is_err();
+ // A session that stayed up, or ended because of OBS rather
+ // than the platform, proves the endpoint works: an OBS
+ // restart must not wait out backoff left from old failures.
+ // A platform that accepts the publish and drops it at once
+ // (a bad key, a refusing edge), or a session that ends
+ // cleanly the moment it starts, backs off like a failed
+ // connect instead of being redialled every second forever.
+ let lasted = connected_at.elapsed();
+ if (!dropped && lasted >= MIN_CLEAN_SESSION) || lasted >= STABLE_SESSION {
+ backoff = Duration::from_secs(1);
+ }
if let Err(e) = pump_result {
// Twitch/etc. sometimes echo the stream key in error
// descriptions ("Authentication failed for live_…").
@@ -2150,15 +2953,23 @@ pub async fn run_egress(
let safe = scrub_secret(&e.to_string(), &parsed.stream_key);
eprintln!("[egress {}] pump error: {}", label, safe);
ctrl.log(format!("[{}] disconnected ({})", label, safe));
- ctrl.fire_webhook("🔴", &format!("**{}** disconnected: {}", label, safe));
+ ctrl.emit(
+ Event::new(EventKind::DestinationDropped)
+ .with("destination", label.clone())
+ .with("platform", platform_of_host(&parsed.host))
+ .with("reason", safe.clone()),
+ );
+ if ctrl.destination_alive_summary().0 == 0 {
+ ctrl.emit(Event::new(EventKind::AllDestinationsDown));
+ }
+ }
+ // Counted here (instead of at the loop tail) so a fresh
+ // connect or a connect failure never bumps it: "Egress
+ // reconnects: 1" must not be the resting state of a healthy
+ // fresh stream.
+ if dropped {
+ dest.reconnects.fetch_add(1, Ordering::Relaxed);
}
- // Count this as a reconnect: the next loop iteration WILL
- // re-establish a connection. Incrementing here (instead of
- // at the loop tail) avoids the old bug where every initial
- // connect AND every connect-failure bumped the counter,
- // making "Egress reconnects: 1" the resting state of a
- // healthy fresh stream.
- dest.reconnects.fetch_add(1, Ordering::Relaxed);
}
Err(e) => {
let safe = scrub_secret(&e.to_string(), &parsed.stream_key);
@@ -2246,6 +3057,28 @@ async fn pump_dest(
state.last_publisher_token = ctrl.publisher_token();
let mut io_buf: Vec = Vec::with_capacity(64 * 1024);
+ // Connected to the platform in the middle of a crash-protection hold
+ // (the platform connection itself dropped and came back, or this
+ // destination was just switched on): go straight onto the reconnect
+ // screen, whether OBS crashed or froze, then rejoin on its new video.
+ let mut resume_after_ts: Option = None;
+ let mut rejoin_from_seq: Option = None;
+ if ctrl.hold_active() {
+ send_sequence_headers(ctrl, dest, &mut sink, 0).await?;
+ let outcome = crate::crash_hold::play(ctrl, dest, &mut sink, 0).await?;
+ let crate::crash_hold::HoldExit::Resumed {
+ rejoin_from_seq: from_seq,
+ } = outcome.exit
+ else {
+ let _ = sink.send_delete_stream().await;
+ return Ok(());
+ };
+ resume_after_ts = Some(outcome.last_ts);
+ rejoin_from_seq = Some(from_seq);
+ // The seed below is on the publisher OBS came back as.
+ state.last_publisher_token = ctrl.publisher_token();
+ }
+
// Initial seed: if a delay is ALREADY active when this pump spawns
// (multi-destination case - a second destination added mid-stream
// while the first is on a 5 s delay), join at the right delayed
@@ -2255,16 +3088,21 @@ async fn pump_dest(
// No seed means the publisher went away (or this destination was
// disabled) before a keyframe arrived. Leave rather than hold an open
// session to the platform with nothing to send down it.
- let Some(first_idr) = seed_idr(ctrl, dest).await else {
+ let seed = match rejoin_from_seq {
+ Some(from_seq) => rejoin_idr(ctrl, dest, from_seq).await,
+ None => seed_idr(ctrl, dest).await,
+ };
+ let Some(first_idr) = seed else {
ctrl.log(format!(
"[{}] nothing to send - the publisher went away before a keyframe",
dest.id
));
+ let _ = sink.send_delete_stream().await;
return Ok(());
};
state.consumer_seq = first_idr.seq;
state.input_ts_anchor = first_idr.ts_ms;
- state.output_ts_base = 0;
+ state.output_ts_base = resume_after_ts.map_or(0, |ts| ts.wrapping_add(1));
state.wall_anchor = Instant::now();
state.wall_anchor_input_ts = first_idr.ts_ms;
state.last_sent_input_ts = first_idr.ts_ms;
@@ -2289,6 +3127,8 @@ async fn pump_dest(
// Always lead with sequence headers + the IDR itself.
send_sequence_headers(ctrl, dest, &mut sink, state.output_ts_base).await?;
+ // Airing what the delay held when OBS left with no hold covering it.
+ let mut airing_tail = false;
loop {
// Cooperative shutdown: when the supervisor flips this, we end
// the session cleanly (sending deleteStream) instead of dropping
@@ -2311,13 +3151,52 @@ async fn pump_dest(
// YouTube long after OBS dropped. The supervisor's gate on
// ingest_alive prevents an immediate respawn here, so the
// destination stays cleanly disconnected until OBS comes back.
- if !ctrl.ingest_alive() {
- ctrl.log(format!(
- "[{}] ingest gone - closing destination session",
- dest.id
- ));
- let _ = sink.send_delete_stream().await;
- return Ok(());
+ //
+ // Crash protection changes that: during a hold the pump first plays
+ // out whatever delay is still buffered, then loops the reconnect
+ // screen until OBS returns (the reconnect branch below takes over)
+ // or the hold ends.
+ // A frozen OBS (connected, no video) gets the same treatment.
+ //
+ // With no hold covering OBS being gone (crash protection off, its
+ // time ran out, or the streamer ended it) the session ends. When the
+ // time ran out mid-tail, the tail airs in full first: the pump keeps
+ // sending until it is out (a delay as long as the hold or longer).
+ if ctrl.obs_gone() || ctrl.hold_active() {
+ if !ctrl.hold_active() {
+ if !ctrl.tail_plays_out() || delay_tail_sent(ctrl, &state) {
+ ctrl.log(format!(
+ "[{}] OBS is gone - closing destination session",
+ dest.id
+ ));
+ let _ = sink.send_delete_stream().await;
+ return Ok(());
+ }
+ airing_tail = true;
+ } else if delay_tail_sent(ctrl, &state) {
+ let outcome =
+ crate::crash_hold::play(ctrl, dest, &mut sink, last_output_ts(&state)).await?;
+ // Continue the timeline from the last slate frame, so the
+ // reconnect re-anchor stays strictly after it.
+ state.output_ts_base = outcome.last_ts;
+ state.input_ts_anchor = state.last_sent_input_ts;
+ let crate::crash_hold::HoldExit::Resumed { rejoin_from_seq } = outcome.exit else {
+ let _ = sink.send_delete_stream().await;
+ return Ok(());
+ };
+ // OBS is back (a new publisher, or the same one after a
+ // freeze): rejoin on its new video, a delay back when one is
+ // armed. The destination's decoder holds the screen's
+ // pictures, so it has to start at a keyframe.
+ ctrl.log(format!("[{}] OBS is back - rejoining", dest.id));
+ let Some(idr) = rejoin_idr(ctrl, dest, rejoin_from_seq).await else {
+ let _ = sink.send_delete_stream().await;
+ return Ok(());
+ };
+ state.last_publisher_token = ctrl.publisher_token();
+ rejoin_at(ctrl, dest, &mut sink, &mut state, idr).await?;
+ continue;
+ }
}
// Detect publisher reconnect (OBS stopped and re-started, new
@@ -2327,6 +3206,18 @@ async fn pump_dest(
// tag - the upstream stream would freeze forever even though
// ingest is happily receiving bytes.
let current_token = ctrl.publisher_token();
+ if current_token != state.last_publisher_token && airing_tail {
+ // OBS started again before the old stream's delay finished
+ // airing. The new session wiped the buffer, and rejoining here
+ // would air its video live: end, and the supervisor reconnects
+ // at the delay, as it would after a stop.
+ ctrl.log(format!(
+ "[{}] OBS started again while the delay was airing - reconnecting",
+ dest.id
+ ));
+ let _ = sink.send_delete_stream().await;
+ return Ok(());
+ }
if current_token != state.last_publisher_token {
ctrl.log(format!("[{}] publisher reconnect - re-anchoring", dest.id));
let watermark = ctrl.ring.latest_seq().unwrap_or(0);
@@ -2335,17 +3226,33 @@ async fn pump_dest(
"[{}] publisher went away again before re-anchoring",
dest.id
));
+ let _ = sink.send_delete_stream().await;
return Ok(());
};
- reseed_after_publisher_change(&mut state, new_idr);
state.last_publisher_token = current_token;
- send_sequence_headers(ctrl, dest, &mut sink, state.output_ts_base).await?;
- dest.consumer_seq
- .store(state.consumer_seq, Ordering::Relaxed);
- dest.last_seq_header_gen.store(
- ctrl.seq_header_gen.load(Ordering::Relaxed),
- Ordering::Relaxed,
- );
+ // OBS came back from a crash while the delay tail was still
+ // airing: the rest of the tail went with the old session (a new
+ // publisher starts the buffer afresh), and rejoining here would
+ // put viewers on live. Cover the gap with the screen until the
+ // delay has rebuilt from the new video, then rejoin delayed.
+ if resumed_mid_delay(ctrl, dest) {
+ let outcome =
+ crate::crash_hold::play(ctrl, dest, &mut sink, last_output_ts(&state)).await?;
+ state.output_ts_base = outcome.last_ts;
+ state.input_ts_anchor = state.last_sent_input_ts;
+ let crate::crash_hold::HoldExit::Resumed { rejoin_from_seq } = outcome.exit else {
+ let _ = sink.send_delete_stream().await;
+ return Ok(());
+ };
+ let Some(idr) = rejoin_idr(ctrl, dest, rejoin_from_seq).await else {
+ let _ = sink.send_delete_stream().await;
+ return Ok(());
+ };
+ state.last_publisher_token = ctrl.publisher_token();
+ rejoin_at(ctrl, dest, &mut sink, &mut state, idr).await?;
+ continue;
+ }
+ rejoin_at(ctrl, dest, &mut sink, &mut state, new_idr).await?;
continue;
}
@@ -2356,19 +3263,39 @@ async fn pump_dest(
let cur_gen = ctrl.seq_header_gen.load(Ordering::Relaxed);
if cur_gen != dest.last_seq_header_gen.load(Ordering::Relaxed) {
ctrl.log(format!("[{}] sequence header changed - resending", dest.id));
- // Use last_sent_input_ts so the resent header lands AFTER
- // anything we've already sent (same trick as apply_cut).
- let delta_u32 = state
- .last_sent_input_ts
- .saturating_sub(state.input_ts_anchor) as u32;
- let resend_ts = state.output_ts_base.wrapping_add(delta_u32);
- send_sequence_headers(ctrl, dest, &mut sink, resend_ts).await?;
+ // On the last output timestamp, so the resent header lands
+ // AFTER anything we've already sent (same trick as apply_cut).
+ send_sequence_headers(ctrl, dest, &mut sink, last_output_ts(&state)).await?;
dest.last_seq_header_gen.store(cur_gen, Ordering::Relaxed);
}
let next_real = next_or_wait(&ctrl.ring, state.consumer_seq, 500).await;
+ // A new publisher or sequence header can arrive while we wait, and
+ // the tag that woke us belongs to it: re-anchor or resend the header
+ // above first, or that tag airs on the old timeline or config.
+ let header_changed = ctrl.seq_header_gen.load(Ordering::Relaxed)
+ != dest.last_seq_header_gen.load(Ordering::Relaxed);
+ if header_changed || ctrl.publisher_token() != state.last_publisher_token {
+ continue;
+ }
if let Some(meta) = next_real {
+ // Fell off the back of the ring (eviction passed this pump), so
+ // `next_or_wait` jumped ahead to the oldest keyframe. Take it as
+ // a cut: the timeline re-anchors instead of pausing for the
+ // skipped stretch, and a vertical destination waits for its
+ // own keyframe.
+ if meta.seq > state.consumer_seq {
+ apply_cut(
+ ctrl,
+ dest,
+ &mut sink,
+ &mut state,
+ PendingCut { target: meta },
+ )
+ .await?;
+ continue;
+ }
let target_now = ctrl.target_delay_ms();
let due = target_now != state.last_seen_target
|| state.last_cut_check.elapsed() >= Duration::from_millis(500);
@@ -2393,6 +3320,74 @@ async fn pump_dest(
}
}
+/// Output timestamp of the last tag this pump sent.
+fn last_output_ts(state: &EgressState) -> u32 {
+ let delta = state
+ .last_sent_input_ts
+ .saturating_sub(state.input_ts_anchor) as u32;
+ state.output_ts_base.wrapping_add(delta)
+}
+
+/// OBS just came back from a crash-protection hold, with a delay armed
+/// that the screen can hold viewers behind while it rebuilds.
+fn resumed_mid_delay(ctrl: &Controller, dest: &DestinationState) -> bool {
+ ctrl.crash_protection_on.load(Ordering::Relaxed)
+ && ctrl.target_delay_ms() > 0
+ && ctrl.hold_state() == crate::crash_hold::HoldState::Resumed
+ && crate::crash_hold::covers(ctrl, dest)
+}
+
+/// Re-anchor on `idr` and lead with the sequence headers: how every
+/// rejoin starts (a new publisher, or OBS back after a hold).
+async fn rejoin_at(
+ ctrl: &Arc,
+ dest: &Arc,
+ sink: &mut EgressSink,
+ state: &mut EgressState,
+ idr: TagMeta,
+) -> io::Result<()> {
+ reseed_after_publisher_change(state, idr);
+ send_sequence_headers(ctrl, dest, sink, state.output_ts_base).await?;
+ dest.consumer_seq
+ .store(state.consumer_seq, Ordering::Relaxed);
+ dest.last_seq_header_gen.store(
+ ctrl.seq_header_gen.load(Ordering::Relaxed),
+ Ordering::Relaxed,
+ );
+ Ok(())
+}
+
+/// The keyframe to rejoin at once OBS is back: the newest at least a delay
+/// back from the live edge, like every delayed join (`delayed_idr`), and
+/// never one from before `from_seq`, where OBS's new video starts. Without
+/// a delay that is the newest keyframe.
+async fn rejoin_idr(
+ ctrl: &Arc,
+ dest: &Arc,
+ from_seq: u64,
+) -> Option {
+ let delay = ctrl.target_delay_ms() as u64;
+ let delayed = ctrl
+ .ring
+ .latest_ts()
+ .and_then(|latest| {
+ ctrl.ring
+ .newest_idr_at_or_before(latest.saturating_sub(delay))
+ })
+ .filter(|idr| idr.seq >= from_seq);
+ match delayed.or_else(|| ctrl.ring.oldest_idr_at_or_after(from_seq)) {
+ Some(idr) => Some(idr),
+ None => wait_for_idr(ctrl, dest, Some(from_seq.saturating_sub(1))).await,
+ }
+}
+
+/// Whether every tag the ended publisher left in the buffer has been sent
+/// (the delay tail), so the reconnect screen can take over.
+fn delay_tail_sent(ctrl: &Controller, state: &EgressState) -> bool {
+ ctrl.delay_tail_end_seq()
+ .is_none_or(|end| state.consumer_seq > end)
+}
+
/// Per-egress-session state. Lost on reconnect; re-anchored from scratch.
struct EgressState {
consumer_seq: u64,
@@ -2400,7 +3395,8 @@ struct EgressState {
output_ts_base: u32, // output ts assigned to the most recent cut target (RTMP wire is u32)
wall_anchor: Instant, // wall clock at the most recent cut
wall_anchor_input_ts: u64, // input ts that pairs with wall_anchor
- last_sent_input_ts: u64, // input ts of the last tag we actually emitted -
+ last_sent_input_ts: u64, // highest input ts we actually emitted (audio and
+ // video interleave a few ms out of order) -
// required so apply_cut can re-anchor the
// output timeline *after* the last sent frame
// (instead of after the last cut, which would
@@ -2470,11 +3466,11 @@ async fn pace_and_send(
return Ok(());
}
let raw_delta_u64 = meta.ts_ms - state.input_ts_anchor;
- // The wire send carries a u32. Truncation is fine here because the
- // delta is bounded by buffer_target_ms + slack (~10 min absolute
- // max), which fits in u32. wrapping_add against output_ts_base
- // handles the (rare) out_ts wrap at the 49-day mark; downstream
- // players accept the wrap because rtmp ts is defined as wrap-modulo.
+ // The wire send carries a u32. The delta is the time since the last
+ // cut or reseed, which can run for days, so the truncation relies on
+ // RTMP timestamps being modulo 2^32: wrapping_add against
+ // output_ts_base gives the same wire value as the untruncated sum, and
+ // players accept the wrap at the 49-day mark.
let raw_delta = raw_delta_u64 as u32;
let logical_offset_ms = meta.ts_ms.saturating_sub(state.wall_anchor_input_ts);
@@ -2511,7 +3507,7 @@ async fn pace_and_send(
match meta.kind {
8 => {
// A vertical destination whose 9:16 canvas isn't on the wire yet
- // (Dual Format off) has `video_egress() == None`. Drop its AUDIO
+ // has `video_egress() == None`. Drop its AUDIO
// too - otherwise we'd feed the platform an audio-only stream
// with no video, which reads as a broken/black broadcast. It
// should send nothing until the canvas appears.
@@ -2527,7 +3523,9 @@ async fn pace_and_send(
// (`audio_egress`), flattened, so a simulcast YouTube / Kick
// gets exactly one audio track it can decode. Single-track
// audio borrows through unchanged.
- let Some(selected) = crate::h264::select_audio_bytes(io_buf, dest.audio_egress())
+ let egress = dest.audio_egress();
+ let Some(selected) =
+ crate::h264::select_audio_bytes(io_buf, egress, ctrl.audio_target_on_wire(egress))
else {
state.consumer_seq = meta.seq + 1;
dest.consumer_seq
@@ -2536,7 +3534,7 @@ async fn pace_and_send(
};
let bytes_out: &[u8] = &selected;
let tags_so_far = dest.tags_sent.load(Ordering::Relaxed);
- if tags_so_far < 20 || tags_so_far.is_multiple_of(200) {
+ if crate::trace::is_enabled() && (tags_so_far < 20 || tags_so_far.is_multiple_of(200)) {
crate::trace::log(
"TAG_AUDIO",
&format!(
@@ -2644,7 +3642,7 @@ async fn pace_and_send(
.fetch_add(bytes_out.len() as u64, Ordering::Relaxed);
dest.note_outbound_bytes(bytes_out.len());
state.consumer_seq = meta.seq + 1;
- state.last_sent_input_ts = meta.ts_ms;
+ state.last_sent_input_ts = state.last_sent_input_ts.max(meta.ts_ms);
dest.consumer_seq
.store(state.consumer_seq, Ordering::Relaxed);
return Ok(());
@@ -2656,7 +3654,7 @@ async fn pace_and_send(
.fetch_add(io_buf.len() as u64, Ordering::Relaxed);
dest.note_outbound_bytes(io_buf.len());
state.consumer_seq = meta.seq + 1;
- state.last_sent_input_ts = meta.ts_ms;
+ state.last_sent_input_ts = state.last_sent_input_ts.max(meta.ts_ms);
// Tell the ingest-side trimmer how far we've read. The trimmer takes
// the MIN across all destinations, so a slow consumer protects all
// others from over-aggressive eviction.
@@ -2701,48 +3699,148 @@ fn idr_search_tolerance_ms(keyframe_interval_ms: u32) -> u32 {
IDR_SEARCH_FLOOR_MS.max(keyframe_interval_ms / 2)
}
+/// The IDR a delay of `target_ms` lands on: the newest one at least that
+/// old, so viewers are never closer to live than asked. None until the
+/// buffer reaches back that far.
+fn delayed_idr(ctrl: &Controller, target_ms: u64) -> Option {
+ let latest = ctrl.ring.latest_ts()?;
+ let fresh = ctrl.fresh_from_seq();
+ let at_delay = latest
+ .checked_sub(target_ms)
+ .and_then(|desired| ctrl.ring.newest_idr_at_or_before(desired))
+ .filter(|idr| idr.seq >= fresh);
+ // A full ring reaches back no further at this bitrate, so a longer
+ // delay could never fill: it delays by as much as the ring holds
+ // instead of waiting forever (`is_saturated`). Not from its very oldest
+ // keyframe: each new tag overwrites the oldest, and a pump parked there
+ // would be overwritten as it reads and keep falling off the back.
+ at_delay.or_else(|| {
+ if !ctrl.ring.is_saturated() {
+ return None;
+ }
+ let oldest = ctrl.ring.oldest_ts()?;
+ let margin = ctrl
+ .ring
+ .newest_idr_at_or_before(oldest + FULL_RING_MARGIN_MS);
+ // After a freeze the keyframe there can be from before it: wait for
+ // the ring to roll past the freeze rather than join on OBS's newest
+ // video, far short of the delay.
+ if fresh > 0 {
+ return margin.filter(|idr| idr.seq >= fresh);
+ }
+ margin.or_else(|| ctrl.ring.oldest_idr_at_or_after(0))
+ })
+}
+
+/// How far in from its oldest tag a delay capped by a full ring starts
+/// (see `delayed_idr`): room for the reads to stay ahead of the overwrites.
+const FULL_RING_MARGIN_MS: u64 = 3_000;
+
+/// How long a connect to a platform (TCP, TLS and the RTMP handshake) may
+/// take before it counts as failed and is retried.
+const CONNECT_TIMEOUT: Duration = Duration::from_secs(20);
+
+/// How long a platform session has to stay up before a reconnect starts
+/// from the shortest backoff again.
+const STABLE_SESSION: Duration = Duration::from_secs(30);
+
+/// A session that ends cleanly (OBS stopped, the destination was switched
+/// off) resets the backoff only after this long. One that ends the moment
+/// it starts would otherwise redial the platform every second.
+const MIN_CLEAN_SESSION: Duration = Duration::from_secs(5);
+
+/// How far a tag may be stamped behind the last one and still be read as
+/// audio and video interleaving (a few ms in practice). Anything further
+/// back is pinned to the current moment (see `Controller::expand_ts`).
+const MAX_TS_STEP_BACK_MS: u32 = 10_000;
+
+/// A delay lands at or above its target (see `delayed_idr`); this much
+/// below it still counts as on target, for send jitter.
+const DELAY_JITTER_MS: u64 = 500;
+
+/// Keyframe interval assumed until one is measured (OBS's default).
+const DEFAULT_KEYFRAME_INTERVAL_MS: u64 = 2_000;
+
fn compute_delay_cut(ctrl: &Arc, current: &TagMeta) -> Option {
let target_delay = ctrl.target_delay_ms.load(Ordering::Relaxed) as u64;
let latest = ctrl.ring.latest_ts()?;
- let oldest = ctrl.ring.oldest_ts()?;
let current_delay = latest.saturating_sub(current.ts_ms);
-
- // Dead band scales with the measured GOP (see recut_dead_band_ms). At the
- // default 2 s cadence this is the tuned 1500 ms; the trade-off is the
- // delivered delay may sit up to one dead-band away from the requested
- // value, which is exactly the price of not re-cutting to the same IDR.
- let keyframe_interval = ctrl.keyframe_interval_ms();
- let dead_band = recut_dead_band_ms(keyframe_interval);
- let diff = (current_delay as i64) - (target_delay as i64);
- if diff.abs() < dead_band as i64 {
- ctrl.buffer_building.store(false, Ordering::Relaxed);
- return None;
+ if target_delay > 0 {
+ return compute_delayed_cut(ctrl, current, current_delay, target_delay);
}
+ ctrl.buffer_building.store(false, Ordering::Relaxed);
- // "Build buffer first" - if the user asked for a delay deeper than
- // the buffer currently extends, we can't honor it yet. Mark the state
- // and hold our position; the buffer keeps filling at real time, and
- // once the requested delay becomes reachable, the next iteration cuts.
- // Same dead band as the re-cut gate so the two agree on "close enough".
- let have_seconds_back = latest.saturating_sub(oldest);
- if target_delay > have_seconds_back.saturating_add(dead_band) {
- ctrl.buffer_building.store(true, Ordering::Relaxed);
+ // Back to live: the keyframe nearest the live edge. Dead band scales
+ // with the measured GOP (see recut_dead_band_ms), so a pump already
+ // parked on the best keyframe doesn't re-cut to it every tick.
+ let keyframe_interval = ctrl.keyframe_interval_ms();
+ if current_delay < recut_dead_band_ms(keyframe_interval) {
return None;
}
- ctrl.buffer_building.store(false, Ordering::Relaxed);
-
// Binary-search on the IDR-only secondary index (~log n over just
// the keyframes) rather than the old O(n) walk over every tag.
- let desired_input_ts = latest.saturating_sub(target_delay);
let target = ctrl
.ring
- .find_idr_near(desired_input_ts, idr_search_tolerance_ms(keyframe_interval))?;
+ .find_idr_near(latest, idr_search_tolerance_ms(keyframe_interval))?;
if target.seq == current.seq {
return None;
}
Some(PendingCut { target })
}
+/// The cut a delay of `target_delay` needs, if any. A delay lands on the
+/// newest keyframe at least that old (see `delayed_idr`), so it sits
+/// between the target and one keyframe interval above it. Re-cutting only
+/// outside that window means a landed delay never re-trips the check, and
+/// viewers are never closer to live than asked, even on long GOPs.
+fn compute_delayed_cut(
+ ctrl: &Controller,
+ current: &TagMeta,
+ current_delay: u64,
+ target_delay: u64,
+) -> Option {
+ let keyframe_interval = ctrl.keyframe_interval_ms();
+ let gop = match keyframe_interval {
+ 0 => DEFAULT_KEYFRAME_INTERVAL_MS,
+ measured => u64::from(measured),
+ };
+ let too_little = current_delay + DELAY_JITTER_MS < target_delay;
+ let upper_band = recut_dead_band_ms(keyframe_interval).max(gop + DELAY_JITTER_MS);
+ let too_much = current_delay > target_delay + upper_band;
+ if !too_little && !too_much {
+ ctrl.buffer_building.store(false, Ordering::Relaxed);
+ return None;
+ }
+ // While OBS is gone (crashed, or frozen and held) the buffer can't
+ // grow, so the delay only shrinks as the delay tail plays out. Jumping
+ // back to restore it would replay the tail instead of finishing it.
+ if too_little && (ctrl.hold_active() || ctrl.obs_gone()) {
+ return None;
+ }
+ // "Build buffer first": until the buffer reaches back to the delay,
+ // hold position; it fills at real time and the next check cuts.
+ let Some(target) = delayed_idr(ctrl, target_delay) else {
+ ctrl.buffer_building.store(true, Ordering::Relaxed);
+ return None;
+ };
+ ctrl.buffer_building.store(false, Ordering::Relaxed);
+ // A cut has to fix what it was called for. Too much delay moves
+ // forward; with a keyframe gap longer than measured, the keyframe at
+ // the delay can be the one this pump is already past, and cutting back
+ // to it would replay the same stretch on every check. Too little delay
+ // moves back, by more than the jitter: on a full ring the oldest
+ // keyframe is barely behind this pump, and re-cutting to it would
+ // replay a moment of video every check without adding any delay.
+ let latest = current_delay + current.ts_ms;
+ let worth_it = if too_much {
+ target.seq > current.seq
+ } else {
+ target.seq < current.seq
+ && latest.saturating_sub(target.ts_ms) > current_delay + DELAY_JITTER_MS
+ };
+ worth_it.then_some(PendingCut { target })
+}
+
async fn apply_cut(
ctrl: &Arc,
dest: &Arc,
@@ -2913,10 +4011,14 @@ async fn send_sequence_headers(
} else {
v_headers.iter().find(|(k, _)| *k == target)
};
- if let Some((_, h)) = pick {
- let selected =
- crate::h264::select_video_bytes(h, crate::h264::VideoEgress::Track(target))
- .unwrap_or(std::borrow::Cow::Borrowed(h.as_slice()));
+ // A header that can't be flattened for this destination (only
+ // another track's has arrived so far) is not sent at all: a raw
+ // multi-track header means nothing to a single-track platform. The
+ // right one goes out when it arrives (new generation).
+ let selected = pick.and_then(|(_, h)| {
+ crate::h264::select_video_bytes(h, crate::h264::VideoEgress::Track(target))
+ });
+ if let Some(selected) = selected {
let bytes_out: &[u8] = &selected;
crate::trace::log(
"VIDEO_SEQ_HDR_SENT",
@@ -2976,7 +4078,9 @@ async fn send_sequence_headers(
.or_else(|| a_headers.first())
.map(|(_, v)| v);
if let Some(h) = header {
- if let Some(bytes) = crate::h264::select_audio_bytes(h, egress) {
+ if let Some(bytes) =
+ crate::h264::select_audio_bytes(h, egress, ctrl.audio_target_on_wire(egress))
+ {
crate::trace::log(
"AUDIO_SEQ_HDR_SENT",
&format!(
@@ -3024,7 +4128,10 @@ async fn wait_for_idr(
let notified = ctrl.ring.on_append.notified();
let found = match min_seq {
Some(seq) => ctrl.ring.newest_idr_after(seq),
- None => ctrl.ring.newest_idr(),
+ None => ctrl
+ .ring
+ .newest_idr()
+ .filter(|idr| idr.seq >= ctrl.fresh_from_seq()),
};
if let Some(m) = found {
return Some(m);
@@ -3039,26 +4146,28 @@ async fn wait_for_idr(
}
}
-/// Pick the seed IDR for a freshly spawned egress pump. If a delay is
-/// already active and the ring has enough history, seed at the delayed
-/// position so the new pump joins mid-stream cleanly; otherwise fall
-/// back to the newest IDR (live edge).
+/// Pick the seed IDR for a freshly spawned egress pump. With a delay on,
+/// join at the delayed position, waiting for the buffer to reach back that
+/// far (`run_egress` normally waits before connecting); a live start would
+/// air live video, then jump back. With no delay, the newest IDR.
async fn seed_idr(ctrl: &Arc, dest: &Arc) -> Option {
- let target = ctrl.target_delay_ms() as u64;
- if target > 0 {
- if let (Some(latest), Some(oldest)) = (ctrl.ring.latest_ts(), ctrl.ring.oldest_ts()) {
- // Only attempt the delayed seed if the buffer actually spans
- // far enough back - otherwise find_idr_near may give us an
- // IDR much closer to live than the user asked for.
- if latest.saturating_sub(oldest) + 1_500 >= target {
- let desired = latest.saturating_sub(target);
- if let Some(idr) = ctrl.ring.find_idr_near(desired, 2_000) {
- return Some(idr);
- }
- }
+ loop {
+ let target = u64::from(ctrl.target_delay_ms());
+ if target == 0 {
+ return wait_for_idr(ctrl, dest, None).await;
+ }
+ let notified = ctrl.ring.on_append.notified();
+ if let Some(idr) = delayed_idr(ctrl, target) {
+ return Some(idr);
+ }
+ if !ctrl.ingest_alive() || dest.shutdown_requested.load(Ordering::Relaxed) {
+ return None;
+ }
+ tokio::select! {
+ _ = notified => {}
+ _ = tokio::time::sleep(Duration::from_millis(250)) => {}
}
}
- wait_for_idr(ctrl, dest, None).await
}
/// Re-anchor egress state after the publisher changed identity. Mirrors
@@ -3115,9 +4224,40 @@ async fn next_or_wait(ring: &Arc, seq: u64, wait_ms: u64) -> Option 0, "mark is ahead of the consumer");
+ // More stream arrives, all of it after the mark.
+ feed_seconds(&h.ctrl, 4_000, 2, 30);
+ let still_pending = |why: &str| {
+ h.ctrl.maybe_fire_safe_cut();
+ assert!(h.ctrl.safe_cut_pending(), "{why}");
+ assert_eq!(h.ctrl.target_delay_ms(), 2_000, "{why}");
+ };
+ let dest = |id: &str, alive: bool, seq: u64| {
+ let st = h.ctrl.destination_state(id);
+ st.egress_alive.store(alive, Ordering::Relaxed);
+ st.consumer_seq.store(seq, Ordering::Relaxed);
+ };
- // Consumer hasn't aired the mark yet → must NOT fire.
- h.ctrl.maybe_fire_safe_cut();
- assert!(h.ctrl.safe_cut_pending());
- assert_eq!(h.ctrl.target_delay_ms(), 2_000);
+ dest("slow", false, 10);
+ dest("fast", false, mark_seq + 5);
+ dest("dead", false, 0);
+ still_pending("no live destination to measure against");
- // More stream arrives, and the consumer advances past the mark
- // (the newest tag's ts is beyond the mark by construction).
- feed_seconds(&h.ctrl, 4_000, 2, 30);
- let latest_seq = h.ctrl.ring.latest_seq().expect("ring has tags");
- st.consumer_seq.store(latest_seq, Ordering::Relaxed);
+ dest("slow", true, 10);
+ dest("fast", true, mark_seq + 5);
+ still_pending("the slower live destination has not aired the mark");
+
+ dest("slow", true, mark_seq);
+ still_pending("the mark's own frame has not gone out yet");
+ dest("slow", true, mark_seq + 1);
h.ctrl.maybe_fire_safe_cut();
- assert!(!h.ctrl.safe_cut_pending(), "mark aired - must fire");
+ assert!(!h.ctrl.safe_cut_pending(), "aired everywhere live - fires");
assert_eq!(h.ctrl.target_delay_ms(), 0, "fire runs the normal cut");
// The armed value survives, same as a manual Cut - the next
// activate is instant.
@@ -3488,32 +4639,17 @@ mod tests {
);
}
- #[test]
- fn activate_consumes_auto_activate_pending() {
- let h = harness(0);
- h.ctrl.arm_delay(2_000);
- feed_seconds(&h.ctrl, 0, 3, 30);
- assert!(h.ctrl.auto_activate_pending());
- h.ctrl.activate_delay().expect("buffer is full");
- assert!(
- !h.ctrl.auto_activate_pending(),
- "successful activate must consume the pending slot"
- );
- }
-
#[test]
fn cut_consumes_auto_activate_pending_so_it_sticks() {
// The bug-of-record: without this clear, the supervisor sees
// phase revert to "ready" after cut and re-fires activate_delay.
+ // Cut while the buffer is still preparing: the slot is still set
+ // (nothing has activated yet), and the streamer's "stay live" has
+ // to win over the auto-activate the buffer filling would trigger.
let h = harness(0);
h.ctrl.arm_delay(2_000);
- feed_seconds(&h.ctrl, 0, 3, 30);
- h.ctrl.activate_delay().unwrap();
- // Slot was already cleared by activate. Re-arm to set it again,
- // then exercise the cut path explicitly. (arm_delay live-update
- // path - target > 0, so this DOESN'T set pending; we have to
- // simulate the post-cut state.)
- h.ctrl.stop_delay(); // cut: target -> 0, pending -> false
+ assert!(h.ctrl.auto_activate_pending(), "armed, not yet activated");
+ h.ctrl.stop_delay();
assert!(
!h.ctrl.auto_activate_pending(),
"cut must keep pending false so the supervisor doesn't re-activate"
@@ -3628,6 +4764,27 @@ mod tests {
assert_eq!(h.ctrl.phase(), "idle");
}
+ #[test]
+ fn setting_the_delay_twice_never_disarms_and_updates_a_live_delay() {
+ let h = harness(0);
+ h.ctrl.mark_ingest_alive_for_test();
+ h.ctrl.set_delay_to(30_000, "integration");
+ h.ctrl.set_delay_to(30_000, "integration");
+ assert_eq!(h.ctrl.armed_delay_ms(), 30_000, "a repeat keeps it armed");
+
+ let h = harness(0);
+ h.ctrl.arm_delay(3_000);
+ feed_seconds(&h.ctrl, 0, 5, 30);
+ h.ctrl.activate_delay().unwrap();
+ h.ctrl.set_delay_to(2_000, "integration");
+ assert_eq!(
+ h.ctrl.target_delay_ms(),
+ 2_000,
+ "changed on air, not refused"
+ );
+ assert_eq!(h.ctrl.phase(), "active");
+ }
+
#[test]
fn named_action_arm_never_disarms_a_delay_on_air() {
// Disarming wipes the target as well, so an arm press while the
@@ -4058,6 +5215,38 @@ mod tests {
/// actually stamped in: promoting it puts one tag 49.7 days ahead, and
/// `on_tag` feeds that to the trim, which then measures the whole ring
/// against a cutoff past every frame in it and evicts the lot.
+ /// OBS stops an Enhanced Broadcasting stream with SequenceEnd tags
+ /// stamped 0. They must not rewind the timeline: the buffer's newest
+ /// moment is what the delay still to air is measured against.
+ #[test]
+ fn an_end_of_stream_tag_stamped_zero_does_not_rewind_the_timeline() {
+ let h = harness(0);
+ h.ctrl.ingest_alive.store(true, Ordering::Relaxed);
+ let v = [0x27u8; 64];
+ h.ctrl.on_tag(9, 200_000, &v, false, false);
+ h.ctrl.on_tag(9, 200_033, &v, false, false);
+
+ h.ctrl.on_tag(9, 0, &v, false, false);
+ assert_eq!(
+ h.ctrl.ring.latest_ts(),
+ Some(200_033),
+ "pinned, not rewound"
+ );
+
+ h.ctrl.on_tag(8, 200_020, &v, false, false);
+ assert_eq!(
+ h.ctrl.ring.latest_ts(),
+ Some(200_020),
+ "ordinary interleaving still reads as stamped"
+ );
+ h.ctrl.on_tag(9, 200_066, &v, false, false);
+ assert_eq!(
+ h.ctrl.ring.latest_ts(),
+ Some(200_066),
+ "and the timeline goes on"
+ );
+ }
+
#[test]
fn a_late_tag_at_the_wrap_stays_in_the_epoch_it_came_from() {
let h = harness(0);
@@ -4335,111 +5524,556 @@ mod tests {
}
}
- // ── Ring + index correctness ─────────────────────────────────────
-
- #[test]
- fn on_tag_appends_to_ring_with_monotonic_ts() {
- // The wrap-expansion path is the load-bearing piece of timestamp
- // logic. Even a small wrap should produce strictly increasing
- // u64 timestamps in the ring.
+ /// Harness with crash protection on and one destination streaming.
+ fn protected_harness(every_disconnect: bool) -> Harness {
let h = harness(0);
- // Two tags well below any wrap point - strictly increasing.
- h.ctrl.on_tag(9, 100, &[0u8; 10], true, false);
- h.ctrl.on_tag(9, 200, &[0u8; 10], false, false);
- h.ctrl.on_tag(9, 300, &[0u8; 10], false, false);
-
- let oldest = h.ctrl.ring.oldest_ts().unwrap();
- let latest = h.ctrl.ring.latest_ts().unwrap();
- assert_eq!(oldest, 100);
- assert_eq!(latest, 300);
- assert!(latest > oldest);
+ h.ctrl
+ .update_crash_protection(crate::crash_protection::CrashProtection {
+ enabled: true,
+ every_disconnect,
+ ..Default::default()
+ });
+ h.ctrl
+ .destination_state("live")
+ .egress_alive
+ .store(true, Ordering::Relaxed);
+ h
}
- #[test]
- fn idr_index_seek_finds_the_right_frame() {
- let h = harness(0);
- feed_seconds(&h.ctrl, 0, 5, 30); // IDR at 0, 1000, 2000, 3000, 4000
- // Target 2500, tolerance 600 → closest IDR is 2000 (distance 500)
- let m = h
- .ctrl
- .ring
- .find_idr_near(2500, 600)
- .expect("should find IDR near 2500");
- assert_eq!(m.ts_ms, 2000);
- assert!(m.is_idr);
+ /// OBS sends FCUnpublish + deleteStream from `RTMP_Close` on every
+ /// deliberate stop; a crash closes the socket without them. Only a
+ /// crash opens a hold, and a goodbye never leaks into the next session.
+ #[tokio::test]
+ async fn only_a_crash_opens_a_hold() {
+ let h = protected_harness(false);
+ h.ctrl.begin_publish("k", "127.0.0.1").await.unwrap();
+ h.ctrl.note_unpublish();
+ h.ctrl.mark_ingest_dead();
+ assert!(!h.ctrl.hold_active(), "a deliberate stop ends the stream");
+
+ h.ctrl.note_unpublish(); // stray goodbye between sessions
+ h.ctrl.begin_publish("k", "127.0.0.1").await.unwrap();
+ h.ctrl.mark_ingest_dead();
+ assert!(h.ctrl.hold_active(), "a drop without goodbye is a crash");
+
+ h.ctrl.begin_publish("k", "127.0.0.1").await.unwrap();
+ assert_eq!(h.ctrl.hold_state(), crate::crash_hold::HoldState::Resumed);
}
- // ── Timestamp wrap (the load-bearing one) ──────────────────────
+ #[tokio::test]
+ async fn end_hold_hotkey_ends_a_hold_and_explains_when_there_is_none() {
+ let h = protected_harness(false);
+ assert!(h.ctrl.run_named_action("end_hold", 0, "hotkey").is_some());
+ h.ctrl.begin_publish("k", "127.0.0.1").await.unwrap();
+ h.ctrl.mark_ingest_dead();
+ assert!(h.ctrl.hold_active());
+ assert_eq!(h.ctrl.run_named_action("end_hold", 0, "hotkey"), None);
+ assert_eq!(h.ctrl.hold_state(), crate::crash_hold::HoldState::Ended);
+ assert!(
+ h.ctrl.run_named_action("end_hold", 0, "hotkey").is_some(),
+ "nothing left to end"
+ );
+ }
- #[test]
- fn wire_ts_wrap_promotes_to_monotonic_u64() {
- // RTMP wire timestamps are u32 ms, so they wrap every ~49.7 days.
- // The Controller's expand_ts() promotes them to u64 by detecting
- // a wrap (current u32 << previous u32) and bumping a "high"
- // counter. Without this, a wrap would make `latest_ts - oldest_ts`
- // negative and the whole delay accounting would explode.
- //
- // We can't call expand_ts directly (private), so we exercise it
- // through on_tag() and observe the result via ring.latest_ts().
+ #[tokio::test]
+ async fn no_hold_when_protection_is_off_or_nothing_is_live() {
let h = harness(0);
- // Walk forward toward the wrap boundary, then over it.
- let near_max = u32::MAX - 1000;
- h.ctrl.on_tag(9, near_max, &[0u8; 20], true, false);
- h.ctrl.on_tag(9, near_max + 500, &[0u8; 20], false, false);
- // Cross the boundary: wire_ts drops near 0 (this LOOKS LIKE
- // going backwards if you only had u32 math).
- h.ctrl.on_tag(9, 200, &[0u8; 20], false, false);
-
- let latest = h.ctrl.ring.latest_ts().unwrap();
- let oldest = h.ctrl.ring.oldest_ts().unwrap();
+ h.ctrl.begin_publish("k", "127.0.0.1").await.unwrap();
+ h.ctrl.mark_ingest_dead();
+ assert!(!h.ctrl.hold_active(), "protection is off by default");
+
+ let h = protected_harness(false);
+ h.ctrl
+ .destination_state("live")
+ .egress_alive
+ .store(false, Ordering::Relaxed);
+ h.ctrl.begin_publish("k", "127.0.0.1").await.unwrap();
+ h.ctrl.mark_ingest_dead();
+ assert!(
+ !h.ctrl.hold_active(),
+ "no live destination, nothing to protect"
+ );
+ }
+
+ #[tokio::test]
+ async fn a_frozen_publisher_opens_a_hold_that_its_next_frame_closes() {
+ let h = protected_harness(false);
+ h.ctrl.begin_publish("k", "127.0.0.1").await.unwrap();
+ h.ctrl.on_tag(9, 0, &[0x27, 1, 0, 0, 0], false, false);
+ assert!(h.ctrl.ingest_sending());
+ let last = h.ctrl.last_video_tag_ms.load(Ordering::Relaxed);
+ let freeze_ms = crate::crash_hold::FREEZE_AFTER.as_millis() as u64;
+ assert!(!h.ctrl.video_frozen_at(last + freeze_ms - 1));
+ assert!(
+ h.ctrl.video_frozen_at(last + freeze_ms),
+ "connected but frozen"
+ );
+ h.ctrl.check_ingest_freeze_at(last + 500);
+ assert!(!h.ctrl.hold_active(), "a short gap is not a freeze");
+ h.ctrl.check_ingest_freeze_at(last + freeze_ms);
+ assert!(h.ctrl.hold_active());
+ h.ctrl.on_tag(9, 40, &[0x27, 1, 0, 0, 0], false, false);
+ assert_eq!(h.ctrl.hold_state(), crate::crash_hold::HoldState::Resumed);
+ assert!(h.ctrl.ingest_sending());
+ }
+
+ /// OBS crashes with a 10 s delay on air: the buffer stops growing, so
+ /// the delay only shrinks while the tail plays out. Live, that gap
+ /// would re-cut back; during the hold it must not, or the tail loops.
+ #[tokio::test]
+ async fn a_crash_mid_delay_never_jumps_back_into_the_tail() {
+ let h = protected_harness(false);
+ h.ctrl.begin_publish("k", "127.0.0.1").await.unwrap();
+ h.ctrl.arm_delay(10_000);
+ feed_seconds(&h.ctrl, 0, 12, 30);
+ h.ctrl.activate_delay().expect("the buffer holds the delay");
+ // The pump is 4 s from the frozen live edge: 6 s short of the delay.
+ let current = h.ctrl.ring.find_idr_near(8_000, 500).unwrap();
assert!(
- latest > oldest,
- "post-wrap latest ({latest}) must be > oldest ({oldest}); \
- wrap promotion is broken"
+ compute_delay_cut(&h.ctrl, ¤t).is_some(),
+ "live, it re-cuts"
);
- // The expand_ts machinery should have lifted the post-wrap value
- // above u32::MAX (high bit promoted).
+ h.ctrl.mark_ingest_dead();
+ assert!(h.ctrl.hold_active());
assert!(
- latest > u32::MAX as u64,
- "expected post-wrap ts above u32::MAX, got {latest}"
+ compute_delay_cut(&h.ctrl, ¤t).is_none(),
+ "holding, the tail plays out once"
);
}
- // ── Enhanced Broadcasting stability ──────────────────────────────
+ /// The tray, global hotkeys and MIDI run on plain threads with no Tokio
+ /// runtime. Ending a hold from them emits an integrations event, which
+ /// must not need one: a panic there aborts the whole process.
+ #[tokio::test]
+ async fn ending_a_hold_off_the_runtime_does_not_panic() {
+ let h = protected_harness(false);
+ h.ctrl.begin_publish("k", "127.0.0.1").await.unwrap();
+ h.ctrl.mark_ingest_dead();
+ assert!(h.ctrl.hold_active());
+ let ctrl = h.ctrl.clone();
+ let hotkey = std::thread::spawn(move || ctrl.run_named_action("end_hold", 0, "hotkey"));
+ assert_eq!(hotkey.join().expect("no panic off the runtime"), None);
+ assert!(!h.ctrl.hold_active());
+ }
- /// EB seq headers must survive across the `pace_and_send` resync
- /// path: the supervisor restarts egress without disturbing the
- /// publisher, the per-track BTreeMap stays populated, and the next
- /// `send_sequence_headers` for a Twitch destination re-emits every
- /// track from the cache. The bug we shipped + reverted was the
- /// pre-BTreeMap single-Option cache stomping all tracks down to
- /// just the last one received - verify via the ring's cache
- /// directly so we'd catch a regression to that storage shape.
- #[test]
- fn eb_seq_headers_survive_egress_restart() {
+ /// Opens a freeze hold for `h` and returns the last video stamp.
+ fn freeze(h: &Harness) -> u64 {
+ h.ctrl.on_tag(9, 0, &[0x27, 1, 0, 0, 0], false, false);
+ let last = h.ctrl.last_video_tag_ms.load(Ordering::Relaxed);
+ let freeze_ms = crate::crash_hold::FREEZE_AFTER.as_millis() as u64;
+ h.ctrl.check_ingest_freeze_at(last + freeze_ms);
+ assert!(h.ctrl.hold_active());
+ last
+ }
+
+ #[tokio::test]
+ async fn stopping_in_obs_ends_a_freeze_hold() {
+ let h = protected_harness(false);
+ h.ctrl.begin_publish("k", "127.0.0.1").await.unwrap();
+ freeze(&h);
+ h.ctrl.note_unpublish();
+ h.ctrl.mark_ingest_dead();
+ assert!(!h.ctrl.hold_active(), "a deliberate stop ends it");
+ assert_eq!(h.ctrl.hold_state(), crate::crash_hold::HoldState::Ended);
+ }
+
+ #[tokio::test]
+ async fn a_freeze_hold_that_ended_does_not_reopen_until_video_returns() {
+ let h = protected_harness(false);
+ h.ctrl.begin_publish("k", "127.0.0.1").await.unwrap();
+ let last = freeze(&h);
+ assert!(h.ctrl.end_hold_now());
+ h.ctrl.check_ingest_freeze_at(last + 60_000);
+ assert!(!h.ctrl.hold_active(), "still the same freeze");
+ // Video comes back, then OBS freezes again: that one is protected.
+ let last = freeze(&h);
+ assert!(last > 0);
+ }
+
+ /// With a delay armed, the screen stays up after OBS is back until its
+ /// new video spans the delay, and the pump rejoins from that video.
+ #[tokio::test]
+ async fn the_screen_waits_for_the_delay_to_rebuild_from_new_video() {
let h = harness(0);
- // Simulate OBS sending OneTrack-format seq headers for tracks
- // 0..=3 (a typical Twitch EB four-rung ladder).
- for track in 0u8..=3 {
- let mut tag = vec![0x96, 0x00, 0x61, 0x76, 0x63, 0x31, track];
- tag.extend_from_slice(&[0xde, 0xad, 0xbe, 0xef]);
- h.ctrl
- .ring
- .append(9, 0, &tag, false, true)
- .expect("seq-header append");
+ h.ctrl.arm_delay(10_000);
+ feed_seconds(&h.ctrl, 0, 2, 10);
+ // OBS comes back (after a freeze: the old video stays in the ring).
+ h.ctrl.resume_from_hold();
+ let mut rebuild = None;
+ let rebuilt =
+ |rebuild: &mut Option<_>| crate::crash_hold::delay_rebuilt(&h.ctrl, rebuild, 3_000);
+ assert_eq!(rebuilt(&mut rebuild), None, "no new video yet");
+ let first_new = h.ctrl.ring.latest_seq().unwrap() + 1;
+ feed_seconds(&h.ctrl, 10_000, 2, 10);
+ assert_eq!(rebuilt(&mut rebuild), None, "1.9 s of new video");
+ feed_seconds(&h.ctrl, 12_000, 2, 10);
+ assert_eq!(rebuilt(&mut rebuild), Some(first_new), "3.9 s: rejoin");
+ }
+
+ const KEPT_IVS: &str = "rtmps://ivs/app/kept";
+ const KEPT_TOKEN: &str = "kept-token";
+ const FRESH_IVS: &str = "rtmps://ivs/app/fresh";
+ const FRESH_TOKEN: &str = "fresh-token";
+
+ /// How OBS left the stream.
+ #[derive(Debug, Clone, Copy, PartialEq)]
+ enum ObsLeaves {
+ Crashes,
+ /// Stopped on purpose, with "protect every disconnect" on.
+ Stops,
+ /// Still connected, but sending no video.
+ Freezes,
+ }
+
+ /// When OBS asked for its Enhanced Broadcasting config again.
+ #[derive(Debug, Clone, Copy, PartialEq)]
+ enum ObsAsks {
+ Never,
+ DuringHold,
+ AfterHold,
+ }
+
+ /// How the hold closed.
+ #[derive(Debug, Clone, Copy, PartialEq)]
+ enum HoldCloses {
+ ObsReturns,
+ /// `ticked`: the supervisor reported the deadline before OBS came back.
+ Deadline {
+ ticked: bool,
+ obs_returns: bool,
+ },
+ EndNow {
+ obs_returns: bool,
+ },
+ }
+
+ #[derive(Debug)]
+ struct HoldCase {
+ had_eb_session: bool,
+ leaves: ObsLeaves,
+ asks: ObsAsks,
+ closes: HoldCloses,
+ }
+
+ impl HoldCase {
+ fn obs_returns(&self) -> bool {
+ match self.closes {
+ HoldCloses::ObsReturns => true,
+ HoldCloses::Deadline { obs_returns, .. } | HoldCloses::EndNow { obs_returns } => {
+ obs_returns
+ }
+ }
}
- // mark_ingest_dead clears codec state and EB overrides, but
- // the seq-header cache itself belongs to the ring and only
- // gets wiped by a fresh publisher session - verify it stays.
- let dest = h.ctrl.destination_state("d1");
- *dest.eb_override_url.lock() = Some("rtmps://stale".into());
- h.ctrl.ingest_alive.store(true, Ordering::Relaxed);
+
+ fn is_possible(&self) -> bool {
+ match self.asks {
+ ObsAsks::Never => true,
+ // A frozen OBS is still streaming: it asks for nothing.
+ ObsAsks::DuringHold => self.leaves != ObsLeaves::Freezes,
+ ObsAsks::AfterHold => {
+ self.leaves != ObsLeaves::Freezes
+ && self.obs_returns()
+ && self.closes != HoldCloses::ObsReturns
+ }
+ }
+ }
+
+ /// The Twitch session the destination must end on: the one OBS
+ /// asked for and publishes on, or none for a single-track stream.
+ fn expected_session(&self) -> Option<&'static str> {
+ let kept = self.had_eb_session.then_some(KEPT_IVS);
+ if self.leaves == ObsLeaves::Freezes {
+ return kept;
+ }
+ match self.asks {
+ ObsAsks::Never => None,
+ ObsAsks::DuringHold => Some(kept.unwrap_or(FRESH_IVS)),
+ ObsAsks::AfterHold => Some(FRESH_IVS),
+ }
+ }
+
+ fn expected_state(&self) -> crate::crash_hold::HoldState {
+ if self.closes == HoldCloses::ObsReturns {
+ crate::crash_hold::HoldState::Resumed
+ } else {
+ crate::crash_hold::HoldState::Ended
+ }
+ }
+ }
+
+ fn all_hold_cases() -> Vec {
+ let mut closings = vec![HoldCloses::ObsReturns];
+ for obs_returns in [true, false] {
+ closings.push(HoldCloses::EndNow { obs_returns });
+ for ticked in [true, false] {
+ closings.push(HoldCloses::Deadline {
+ ticked,
+ obs_returns,
+ });
+ }
+ }
+ let mut cases = Vec::new();
+ for had_eb_session in [true, false] {
+ for leaves in [ObsLeaves::Crashes, ObsLeaves::Stops, ObsLeaves::Freezes] {
+ for asks in [ObsAsks::Never, ObsAsks::DuringHold, ObsAsks::AfterHold] {
+ for &closes in &closings {
+ let case = HoldCase {
+ had_eb_session,
+ leaves,
+ asks,
+ closes,
+ };
+ if case.is_possible() {
+ cases.push(case);
+ }
+ }
+ }
+ }
+ }
+ cases
+ }
+
+ /// The fresh-session path of /obs/multitrack-config (web.rs): accept the
+ /// token, point the Twitch destination at the session, keep it for a hold.
+ fn open_eb_session(h: &Harness, ivs: &str, token: &str) {
+ h.ctrl.remember_eb_key(token.into());
+ *h.ctrl.destination_state("live").eb_override_url.lock() = Some(ivs.into());
+ h.ctrl.remember_eb_session(EbSession {
+ config: format!("{{{token}}}"),
+ auths: vec![token.into()],
+ dest_id: "live".into(),
+ ivs_url: ivs.into(),
+ });
+ }
+
+ /// OBS asks /obs/multitrack-config for its config, which hands back the
+ /// session a hold kept or opens a new one. Returns OBS's stream key.
+ fn obs_asks_for_eb_config(h: &Harness) -> &'static str {
+ if h.ctrl.held_eb_config().is_some() {
+ return KEPT_TOKEN;
+ }
+ open_eb_session(h, FRESH_IVS, FRESH_TOKEN);
+ FRESH_TOKEN
+ }
+
+ /// Plays `case` out and returns what went wrong, if anything.
+ async fn run_hold_case(case: &HoldCase) -> Vec {
+ let h = protected_harness(case.leaves == ObsLeaves::Stops);
+ h.ctrl.update_ingest_key("k".into());
+ let mut stream_key = "k";
+ if case.had_eb_session {
+ open_eb_session(&h, KEPT_IVS, KEPT_TOKEN);
+ stream_key = KEPT_TOKEN;
+ }
+ h.ctrl.begin_publish(stream_key, "127.0.0.1").await.unwrap();
+ h.ctrl.on_tag(9, 0, &[0x17, 0, 0, 0, 0, 1], false, true);
+ match case.leaves {
+ ObsLeaves::Crashes => h.ctrl.mark_ingest_dead(),
+ ObsLeaves::Stops => {
+ h.ctrl.note_unpublish();
+ h.ctrl.mark_ingest_dead();
+ }
+ ObsLeaves::Freezes => {
+ freeze(&h);
+ }
+ }
+ assert!(h.ctrl.hold_active(), "{case:?}: no hold opened");
+
+ stream_key = "k";
+ if case.asks == ObsAsks::DuringHold {
+ stream_key = obs_asks_for_eb_config(&h);
+ }
+ match case.closes {
+ HoldCloses::ObsReturns => {}
+ HoldCloses::Deadline { ticked, .. } => {
+ if let Some(hold) = h.ctrl.hold.lock().as_mut() {
+ hold.deadline = Instant::now() - Duration::from_millis(1);
+ }
+ if ticked {
+ h.ctrl.expire_hold();
+ }
+ }
+ HoldCloses::EndNow { .. } => {
+ assert!(h.ctrl.end_hold_now(), "{case:?}: End now found no hold");
+ }
+ }
+ if case.obs_returns() {
+ if case.asks == ObsAsks::AfterHold {
+ stream_key = obs_asks_for_eb_config(&h);
+ }
+ if case.leaves == ObsLeaves::Freezes {
+ h.ctrl.on_tag(9, 40, &[0x27, 1, 0, 0, 0], false, false);
+ } else if let Err(e) = h.ctrl.begin_publish(stream_key, "127.0.0.1").await {
+ return vec![format!("{case:?}: OBS's publish was refused: {e}")];
+ }
+ }
+ // The supervisor's next tick.
+ h.ctrl.expire_hold();
+
+ let mut failures = Vec::new();
+ let session = h
+ .ctrl
+ .destination_state("live")
+ .eb_override_url
+ .lock()
+ .clone();
+ if session.as_deref() != case.expected_session() {
+ failures.push(format!(
+ "{case:?}: Twitch session {session:?}, expected {:?}",
+ case.expected_session()
+ ));
+ }
+ let state = h.ctrl.hold_state();
+ if state != case.expected_state() {
+ failures.push(format!(
+ "{case:?}: hold {state:?}, expected {:?}",
+ case.expected_state()
+ ));
+ }
+ failures
+ }
+
+ /// OBS asking for its config again during a hold gets the kept session
+ /// back, and that session's token is brokered again: it was accepted
+ /// when the session opened, possibly longer ago than its TTL, and OBS's
+ /// publish with it must still get past the ingest key. (The matrix below
+ /// pre-brokers every token, so it can't see this.) Outside a hold there
+ /// is nothing to hand back.
+ #[tokio::test]
+ async fn a_kept_session_is_handed_back_only_during_a_hold_and_rebrokers_its_token() {
+ let h = protected_harness(false);
+ h.ctrl.update_ingest_key("k".into());
+ h.ctrl.begin_publish("k", "127.0.0.1").await.unwrap();
+ h.ctrl.on_tag(9, 0, &[0x17, 0, 0, 0, 0, 1], false, true);
+ let live = h.ctrl.destination_state("live");
+ *live.eb_override_url.lock() = Some(KEPT_IVS.into());
+ // Remembered without brokering its token: as if the token expired.
+ h.ctrl.remember_eb_session(EbSession {
+ config: "{config}".into(),
+ auths: vec![KEPT_TOKEN.into()],
+ dest_id: "live".into(),
+ ivs_url: KEPT_IVS.into(),
+ });
+ assert_eq!(h.ctrl.held_eb_config(), None, "only during a hold");
+
h.ctrl.mark_ingest_dead();
- let cache = h.ctrl.ring.video_seq_headers.lock();
- assert_eq!(cache.len(), 4, "all 4 tracks must persist across cut");
- for track in 0u8..=3 {
- assert!(cache.contains_key(&track), "track {track} dropped");
+ assert!(!h.ctrl.is_brokered_eb_key(KEPT_TOKEN));
+ assert_eq!(h.ctrl.held_eb_config().as_deref(), Some("{config}"));
+ h.ctrl
+ .begin_publish(KEPT_TOKEN, "127.0.0.1")
+ .await
+ .expect("the token is let in again");
+ assert_eq!(live.eb_override_url.lock().as_deref(), Some(KEPT_IVS));
+ assert_eq!(h.ctrl.held_eb_config(), None, "the hold is over");
+ }
+
+ /// Every way a hold can play out with Enhanced Broadcasting in the mix.
+ /// Twitch refuses a stream on the wrong session (the multitrack ladder on
+ /// plain ingest, or one track on a multitrack session), so the
+ /// destination must end on exactly the session OBS publishes on.
+ #[tokio::test]
+ async fn every_hold_path_leaves_twitch_on_the_session_obs_publishes_on() {
+ let mut failures = Vec::new();
+ for case in all_hold_cases() {
+ failures.extend(run_hold_case(&case).await);
}
+ assert!(
+ failures.is_empty(),
+ "{} hold paths went wrong:\n{}",
+ failures.len(),
+ failures.join("\n")
+ );
+ }
+
+ /// OBS reconnects after a crash but never sends video: once the grace
+ /// runs out that counts as a freeze, so the screen gets a deadline.
+ #[tokio::test]
+ async fn a_returning_obs_that_sends_no_video_is_a_freeze() {
+ let h = protected_harness(false);
+ h.ctrl.begin_publish("k", "127.0.0.1").await.unwrap();
+ h.ctrl.on_tag(9, 0, &[0x17, 0, 0, 0, 0, 1], false, true);
+ h.ctrl.mark_ingest_dead();
+ h.ctrl.begin_publish("k", "127.0.0.1").await.unwrap();
+ assert!(!h.ctrl.hold_active(), "resumed");
+ let now = process_now_ms();
+ let grace = FIRST_VIDEO_GRACE.as_millis() as u64;
+ h.ctrl.check_ingest_freeze_at(now + grace - 1_000);
+ assert!(!h.ctrl.hold_active(), "still within the grace");
+ h.ctrl.check_ingest_freeze_at(now + grace + 1_000);
+ assert!(h.ctrl.hold_active(), "no video after the grace is a freeze");
+ }
+
+ #[tokio::test]
+ async fn protection_off_keeps_no_per_frame_state() {
+ let h = harness(0);
+ h.ctrl.begin_publish("k", "127.0.0.1").await.unwrap();
+ let hevc_keyframe = [0x91, b'h', b'v', b'c', b'1', 0, 0, 0, 7];
+ h.ctrl.on_tag(9, 0, &hevc_keyframe, true, false);
+ assert_eq!(h.ctrl.last_video_tag_ms.load(Ordering::Relaxed), 0);
+ assert!(h.ctrl.held_keyframe(0).is_none());
+
+ // Switched off mid-stream: nothing stale can later read as a freeze.
+ let h = protected_harness(false);
+ h.ctrl.begin_publish("k", "127.0.0.1").await.unwrap();
+ h.ctrl.on_tag(9, 0, &hevc_keyframe, true, false);
+ assert!(h.ctrl.held_keyframe(0).is_some());
+ h.ctrl
+ .update_crash_protection(crate::crash_protection::CrashProtection::default());
+ assert_eq!(h.ctrl.last_video_tag_ms.load(Ordering::Relaxed), 0);
+ assert!(h.ctrl.held_keyframe(0).is_none());
+ assert!(h.ctrl.ingest_sending());
+ }
+
+ #[tokio::test]
+ async fn an_hevc_stream_is_covered_by_holding_its_last_keyframe() {
+ let h = protected_harness(false);
+ h.ctrl.begin_publish("k", "127.0.0.1").await.unwrap();
+ let live = h.ctrl.destination_state("live");
+ // Enhanced RTMP HEVC: SequenceStart, then a coded keyframe.
+ h.ctrl
+ .on_tag(9, 0, &[0x90, b'h', b'v', b'c', b'1', 1], false, true);
+ assert!(
+ !crate::crash_hold::covers(&h.ctrl, &live),
+ "nothing to hold yet"
+ );
+ let keyframe = [0x91, b'h', b'v', b'c', b'1', 0, 0, 0, 7];
+ h.ctrl.on_tag(9, 33, &keyframe, true, false);
+ assert!(crate::crash_hold::covers(&h.ctrl, &live));
+ assert_eq!(h.ctrl.held_keyframe(0).as_deref(), Some(&keyframe[..]));
+ }
+
+ #[tokio::test]
+ async fn an_enhanced_broadcasting_destination_is_covered_track_by_track() {
+ let h = protected_harness(false);
+ h.ctrl.begin_publish("k", "127.0.0.1").await.unwrap();
+ let live = h.ctrl.destination_state("live");
+ live.pass_through_multitrack_video
+ .store(true, Ordering::Relaxed);
+ // Track 0 legacy H.264, track 1 an H.264 rung, track 2 an HEVC rung.
+ let one_track = |fourcc: &[u8; 4], track: u8, first: u8, packet: u8| {
+ let mut tag = vec![first, packet];
+ tag.extend_from_slice(fourcc);
+ tag.extend_from_slice(&[track, 0, 0, 0, 7]);
+ tag
+ };
+ h.ctrl.on_tag(9, 0, &[0x17, 0, 0, 0, 0, 1], false, true);
+ h.ctrl
+ .on_tag(9, 0, &one_track(b"avc1", 1, 0x96, 0x00), false, true);
+ assert!(crate::crash_hold::covers(&h.ctrl, &live));
+ h.ctrl
+ .on_tag(9, 0, &one_track(b"hvc1", 2, 0x96, 0x00), false, true);
+ assert!(
+ !crate::crash_hold::covers(&h.ctrl, &live),
+ "every track needs a picture"
+ );
+ h.ctrl
+ .on_tag(9, 33, &one_track(b"hvc1", 2, 0x96, 0x01), false, false);
+ assert!(crate::crash_hold::covers(&h.ctrl, &live));
+ assert!(h.ctrl.held_keyframe(2).is_some());
+ assert!(
+ h.ctrl.held_keyframe(1).is_none(),
+ "H.264 rungs get the screen"
+ );
}
/// `mark_ingest_dead` must clear every destination's
@@ -4472,6 +6106,44 @@ mod tests {
/// Twitch Inspector. The latch is released by the fetch task on
/// completion (modelled here by the explicit store), after which the
/// next tick may claim again (e.g. to retry a failed fetch).
+ /// A hold keeps a destination's Twitch session only when it keeps that
+ /// destination on air: live, and coverable by the screen. One that is
+ /// down, or a vertical one whose canvas isn't known yet (the screen
+ /// can't cover it), ends with OBS, so its session goes with it, and a
+ /// session fetch still in flight for it is discarded.
+ #[tokio::test]
+ async fn a_hold_keeps_the_session_only_of_destinations_it_keeps_on_air() {
+ let h = protected_harness(false);
+ h.ctrl.begin_publish("k", "127.0.0.1").await.unwrap();
+ h.ctrl.on_tag(9, 0, &[0x17, 0, 0, 0, 0, 1], false, true);
+ let live = h.ctrl.destination_state("live");
+ let down = h.ctrl.destination_state("down");
+ let vertical = h.ctrl.destination_state("vertical");
+ vertical.egress_alive.store(true, Ordering::Relaxed);
+ vertical.egress_vertical.store(true, Ordering::Relaxed);
+ assert!(crate::crash_hold::covers(&h.ctrl, &live));
+ assert!(
+ !crate::crash_hold::covers(&h.ctrl, &vertical),
+ "no canvas to cover yet"
+ );
+ for dest in [&live, &down, &vertical] {
+ *dest.eb_override_url.lock() = Some(format!("rtmps://ivs/app/{}", dest.id));
+ }
+ let epochs = [&down, &vertical].map(|dest| dest.session_epoch());
+
+ h.ctrl.mark_ingest_dead();
+ assert!(h.ctrl.hold_active());
+ assert_eq!(
+ live.eb_override_url.lock().as_deref(),
+ Some("rtmps://ivs/app/live"),
+ "kept on air, so its session is kept"
+ );
+ for (dest, epoch) in [&down, &vertical].into_iter().zip(epochs) {
+ assert!(dest.eb_override_url.lock().is_none(), "{} ended", dest.id);
+ assert!(dest.session_epoch() > epoch, "{}: fetch discarded", dest.id);
+ }
+ }
+
#[test]
fn try_claim_vod_fetch_admits_one_claimant() {
let s = DestinationState::new("main".into());
@@ -4738,7 +6410,7 @@ mod tests {
/// A fresh publisher session must clear out all cached audio and multi-track
/// video sequence headers left over from a previous stream. While those
/// headers are required to survive mid-stream egress supervisor restarts
- /// (tested via `eb_seq_headers_survive_egress_restart`), allowing them to
+ /// (tested via `every_track_config_leads_each_twitch_connection`), allowing them to
/// leak into a subsequent session causes severe pipeline pollution. If a
/// publisher reconnects without Enhanced Broadcasting, a failure to clear
/// this state causes the egress engine to inject stale multi-track headers
@@ -5010,31 +6682,32 @@ mod tests {
}
/// A reconnect may bring a completely different OBS profile, so the
- /// previous session's measurement must not leak into the new one.
+ /// previous session's measurements must not leak into the new one.
#[test]
fn reset_codec_state_clears_the_measurement() {
let h = harness(0);
feed_idrs(&h.ctrl, 0, 4_000, 4);
- assert_eq!(h.ctrl.keyframe_interval_ms(), 4_000);
+ let header = crate::slate::test_sequence_header(64, 36);
+ h.ctrl.on_tag(9, 12_000, &header, false, true);
+ let before = h.ctrl.stream_params();
+ assert_eq!(before.keyframe_interval_ms, 4_000);
+ assert_eq!((before.width, before.height), (64, 36));
h.ctrl.reset_codec_state();
- assert_eq!(h.ctrl.keyframe_interval_ms(), 0);
- assert_eq!(h.ctrl.stream_params().width, 0);
- assert_eq!(h.ctrl.stream_params().height, 0);
+ let after = h.ctrl.stream_params();
+ assert_eq!(after.keyframe_interval_ms, 0);
+ assert_eq!((after.width, after.height), (0, 0));
}
- /// The packed dims atomic must round-trip a real resolution and never
- /// bleed width into height or vice-versa.
+ /// OBS's sequence header decodes to the resolution the compatibility
+ /// check sees, width and height each in their own place.
#[test]
- fn stream_params_reports_packed_dimensions() {
+ fn stream_params_reports_the_resolution_obs_sends() {
let h = harness(0);
- // 0x11 = AVC keyframe; sps_dimensions decodes 1920x1080 from a real
- // SPS, but the unit here only needs the packing path exercised via a
- // known resolution, so drive it through the public snapshot instead.
- h.ctrl
- .video_dims
- .store(((1920u64) << 32) | 1080, Ordering::Relaxed);
+ assert_eq!(h.ctrl.stream_params().width, 0, "nothing sent yet");
+ let header = crate::slate::test_sequence_header(64, 36);
+ h.ctrl.on_tag(9, 0, &header, false, true);
let p = h.ctrl.stream_params();
- assert_eq!((p.width, p.height), (1920, 1080));
+ assert_eq!((p.width, p.height), (64, 36));
}
/// Dead band stays at the tuned 1500 ms for a 2 s GOP (and while
@@ -5055,6 +6728,25 @@ mod tests {
}
}
+ /// Going delayed from live has to jump back even when the delay is no
+ /// longer than the re-cut dead band. Live sits a whole delay short of
+ /// it, which the dead band (hysteresis for a delay already in place)
+ /// used to read as close enough: 1 s against its 1.5 s floor, so the
+ /// dashboard said the delay was on while viewers stayed on live.
+ #[test]
+ fn a_one_second_delay_jumps_back_from_live() {
+ let h = harness(0);
+ h.ctrl.arm_delay(1_000);
+ feed_seconds(&h.ctrl, 0, 3, 30);
+ h.ctrl.activate_delay().expect("the buffer holds the delay");
+ let latest = h.ctrl.ring.latest_seq().expect("a populated ring");
+ let (_, live_edge) = h.ctrl.ring.find_by_seq(latest).unwrap();
+ assert!(
+ compute_delay_cut(&h.ctrl, &live_edge).is_some(),
+ "live is a whole second short of the delay"
+ );
+ }
+
#[test]
fn idr_search_tolerance_never_below_floor_and_scales_up() {
assert_eq!(idr_search_tolerance_ms(0), 2_000);
@@ -5066,4 +6758,64 @@ mod tests {
"6 s GOP needs 3 s reach"
);
}
+
+ /// OBS hung without its connection closing (the OS keeps ACKing), and
+ /// the streamer restarts it: the new publisher takes the slot once the
+ /// old one has sent no video for the freeze threshold, instead of being
+ /// refused until the old socket dies. The old connection closing later
+ /// must not end the new session.
+ #[tokio::test]
+ async fn a_new_publisher_takes_over_from_one_that_stopped_sending_video() {
+ let h = harness(0);
+ let first = h.ctrl.begin_publish("k", "127.0.0.1").await.unwrap();
+ h.ctrl
+ .on_tag(9, 0, &[0x17, 1, 0, 0, 0, 0, 0, 0, 1, 0x65], true, false);
+ assert!(
+ h.ctrl.begin_publish("k", "127.0.0.1").await.is_err(),
+ "a publisher that is sending keeps the slot"
+ );
+
+ // The process clock starts at 0: let it pass the freeze threshold,
+ // so "the last video frame" can sit that far back.
+ let freeze_ms = crate::crash_hold::FREEZE_AFTER.as_millis() as u64;
+ while process_now_ms() <= freeze_ms + 10 {
+ tokio::time::sleep(Duration::from_millis(100)).await;
+ }
+ h.ctrl.last_video_tag_ms.store(1, Ordering::Relaxed);
+ let second = h
+ .ctrl
+ .begin_publish("k", "127.0.0.1")
+ .await
+ .expect("a publisher that stopped sending video hands over");
+ assert_ne!(first, second);
+
+ h.ctrl.end_publish(first);
+ assert!(
+ h.ctrl.ingest_alive(),
+ "the old connection closing is ignored"
+ );
+ h.ctrl.end_publish(second);
+ assert!(!h.ctrl.ingest_alive(), "the current one closing ends it");
+ }
+
+ /// A peer retrying a refused publish in a loop logs one line per 10 s,
+ /// not one per attempt, so the bounded log keeps everything else.
+ #[tokio::test]
+ async fn refused_publishes_are_logged_at_most_once_per_ten_seconds() {
+ let h = harness(0);
+ h.ctrl.begin_publish("k", "127.0.0.1").await.unwrap();
+ h.ctrl
+ .on_tag(9, 0, &[0x17, 1, 0, 0, 0, 0, 0, 0, 1, 0x65], true, false);
+ for _ in 0..50 {
+ let _ = h.ctrl.begin_publish("k", "127.0.0.1").await;
+ }
+ let refusals = h
+ .ctrl
+ .logs
+ .lock()
+ .iter()
+ .filter(|line| line.contains("slot in use"))
+ .count();
+ assert_eq!(refusals, 1);
+ }
}
diff --git a/src/controller/delay_sim.rs b/src/controller/delay_sim.rs
new file mode 100644
index 0000000..83a6e85
--- /dev/null
+++ b/src/controller/delay_sim.rs
@@ -0,0 +1,843 @@
+//! Delay simulations (see `sim` for the harness): the normal lifecycle of a
+//! delay, played in real time through a real pump. OBS streams at 10 fps
+//! with a keyframe every 500 ms; the streamer arms a delay, goes delayed,
+//! cuts back to live, schedules "cut after this airs", and changes the
+//! delay on air. The assertions are about what viewers would see: where the
+//! picture jumps, whether it lands on a keyframe, how far behind OBS it
+//! runs, that nothing airs more often than the delay itself explains, and
+//! that timestamps never go back.
+
+use super::sim::{assert_strictly_increasing, wait, Frame, Sim, FRAME_MS, GOP_FRAMES};
+use super::*;
+use std::collections::HashSet;
+
+/// The delay most scenarios arm.
+const DELAY_MS: u32 = 2_000;
+const GOP_MS: u32 = FRAME_MS * GOP_FRAMES;
+/// Room for scheduling: the pump and OBS each sleep to a schedule, and the
+/// test binary runs many of these at once.
+const JITTER_MS: u32 = 150;
+/// Each frame's audio is stamped this far behind its video, so a cut lands
+/// on a keyframe whose own audio is older than it and has to be dropped.
+const AUDIO_LAG_MS: u32 = 30;
+
+async fn delay_sim() -> Sim {
+ let mut sim = Sim::new(false).await;
+ sim.audio_lag_ms = AUDIO_LAG_MS;
+ sim
+}
+
+/// How long after OBS made it `frame` reached the platform.
+fn delay_ms(sim: &Sim, frame: &Frame) -> u32 {
+ frame
+ .at
+ .duration_since(sim.produced[&frame.number])
+ .as_millis() as u32
+}
+
+/// Every frame in `frames` ran between `min_ms` and `max_ms` behind OBS.
+fn assert_delays(sim: &Sim, frames: &[Frame], min_ms: u32, max_ms: u32, what: &str) {
+ let delays: Vec<(u32, u32)> = frames
+ .iter()
+ .map(|f| (f.number, delay_ms(sim, f)))
+ .collect();
+ let off = delays
+ .iter()
+ .find(|(_, ms)| !(min_ms..=max_ms).contains(ms));
+ assert_eq!(
+ off, None,
+ "{what}: a frame ran outside {min_ms}..={max_ms} ms behind OBS: {delays:?}"
+ );
+}
+
+/// Sleeps can wake a few ms early on a loaded test runner.
+const TIMER_SLOP_MS: u32 = 50;
+
+/// A delay is honored: every frame in `frames` ran at least `delay_ms`
+/// behind OBS (never closer to live than asked) and at most one keyframe
+/// interval more, which is how far back the newest keyframe that old sits.
+fn assert_delay_honored(sim: &Sim, frames: &[Frame], delay_ms: u32, gop_ms: u32, what: &str) {
+ let most = delay_ms + gop_ms + JITTER_MS;
+ assert_delays(sim, frames, delay_ms - TIMER_SLOP_MS, most, what);
+}
+
+/// Where the picture jumps back to an earlier frame.
+fn backward_jumps(frames: &[Frame]) -> Vec {
+ (1..frames.len())
+ .filter(|&i| frames[i].number < frames[i - 1].number)
+ .collect()
+}
+
+/// Where the picture skips ahead past frames it never showed.
+fn forward_jumps(frames: &[Frame]) -> Vec {
+ (1..frames.len())
+ .filter(|&i| frames[i].number > frames[i - 1].number + 1)
+ .collect()
+}
+
+/// A jump back replays what the delay holds once; nothing airs a third time.
+fn assert_airs_at_most_twice(frames: &[Frame]) {
+ let mut seen: HashMap = HashMap::new();
+ for frame in frames {
+ *seen.entry(frame.number).or_default() += 1;
+ }
+ let mut thrice: Vec = seen
+ .into_iter()
+ .filter(|(_, n)| *n > 2)
+ .map(|(number, _)| number)
+ .collect();
+ thrice.sort_unstable();
+ assert!(thrice.is_empty(), "frames aired three times: {thrice:?}");
+}
+
+/// The pump re-sent the video and audio sequence headers between
+/// `frames[i - 1]` and `frames[i]`, on `frames[i]`'s own timestamp, so the
+/// decoder has its config before the frame it jumped to.
+fn assert_headers_lead(sim: &mut Sim, conn: usize, frames: &[Frame], i: usize) {
+ let (from, to, ts) = (frames[i - 1].index, frames[i].index, frames[i].ts);
+ let between = &sim.received()[from + 1..to];
+ let is_header = |kind: u8| {
+ between
+ .iter()
+ .any(|m| m.conn == conn && m.kind == kind && m.payload.get(1) == Some(&0) && m.ts == ts)
+ };
+ assert!(
+ is_header(9),
+ "no video sequence header led frame {}",
+ frames[i].number
+ );
+ assert!(
+ is_header(8),
+ "no audio sequence header led frame {}",
+ frames[i].number
+ );
+}
+
+/// Audio kept up with the video: every frame that aired had its audio air
+/// too, except the one audio frame each join or cut drops because it is
+/// stamped just before the keyframe it lands on. `joins` counts the pump's
+/// seed plus its cuts.
+fn assert_audio_kept_up(sim: &mut Sim, conn: usize, joins: usize) {
+ let video = sim.frames(conn);
+ let Some(last) = video.last().map(|f| f.number) else {
+ return;
+ };
+ let mut owed: HashMap = HashMap::new();
+ for frame in &video {
+ *owed.entry(frame.number).or_default() += 1;
+ }
+ for number in sim.audio_numbers(conn) {
+ *owed.entry(number).or_default() -= 1;
+ }
+ // The newest frame's audio may still be on its way.
+ owed.remove(&last);
+ let mut missing: Vec = owed
+ .iter()
+ .filter(|(_, n)| **n > 0)
+ .map(|(number, _)| *number)
+ .collect();
+ missing.sort_unstable();
+ assert!(
+ missing.len() <= joins,
+ "audio went missing for frames {missing:?} ({joins} joins and cuts)"
+ );
+}
+
+fn cuts(sim: &Sim, id: &str) -> u32 {
+ sim.ctrl
+ .destination_state(id)
+ .cuts_performed
+ .load(Ordering::Relaxed)
+}
+
+/// Live (no delay): a destination joining mid-stream starts on OBS's newest
+/// keyframe, then every frame goes out once, in order, as soon as it can.
+#[tokio::test]
+async fn live_frames_go_out_once_in_order_as_obs_makes_them() {
+ let mut sim = delay_sim().await;
+ sim.obs_connects().await;
+ sim.obs_sends(700).await;
+ let newest_keyframe = (sim.next_frame - 1) / GOP_FRAMES * GOP_FRAMES;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(2_000).await;
+
+ let frames = sim.frames(0);
+ assert!(frames.len() >= 18, "{} frames aired", frames.len());
+ assert_eq!(
+ frames[0].number, newest_keyframe,
+ "joins on the newest keyframe"
+ );
+ assert!(frames[0].keyframe);
+ assert_strictly_increasing(&frames, "live");
+ assert_delays(&sim, &frames, 0, GOP_MS + JITTER_MS, "live");
+ assert_eq!(cuts(&sim, "platform"), 0);
+ sim.assert_timestamps_monotonic(0);
+ assert_audio_kept_up(&mut sim, 0, 1);
+}
+
+/// Going delayed: one jump back to the keyframe nearest the delay, led by
+/// the sequence headers on its own timestamp, then the stream runs that far
+/// behind OBS without ever jumping again.
+#[tokio::test]
+async fn activating_jumps_back_once_to_a_keyframe_the_delay_ago() {
+ let mut sim = delay_sim().await;
+ sim.ctrl.arm_delay(DELAY_MS);
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(2_100).await;
+ let activated = Instant::now();
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ sim.obs_sends(3_400).await;
+
+ let frames = sim.frames(0);
+ let back = backward_jumps(&frames);
+ assert_eq!(back.len(), 1, "exactly one jump back");
+ let i = back[0];
+ let landed = frames[i];
+ assert!(landed.keyframe, "lands on a keyframe");
+ let late = landed.at.duration_since(activated).as_millis() as u32;
+ assert!(
+ late <= FRAME_MS + JITTER_MS,
+ "jumped {late} ms after activating"
+ );
+ assert_headers_lead(&mut sim, 0, &frames, i);
+ let delayed = &frames[i..];
+ assert_strictly_increasing(delayed, "delayed");
+ let span = delayed.last().unwrap().at.duration_since(landed.at);
+ assert!(
+ span >= Duration::from_millis(2_500),
+ "only {span:?} delayed"
+ );
+ assert_delay_honored(&sim, delayed, DELAY_MS, GOP_MS, "delayed");
+ assert_eq!(cuts(&sim, "platform"), 1, "no re-cut once on the delay");
+ assert_airs_at_most_twice(&frames);
+ sim.assert_timestamps_monotonic(0);
+ assert_audio_kept_up(&mut sim, 0, 2);
+}
+
+/// Cutting back to live: one jump forward to the newest keyframe, with
+/// nothing that already aired shown again, and the delay gone.
+#[tokio::test]
+async fn cutting_to_live_jumps_forward_to_the_newest_keyframe() {
+ let mut sim = delay_sim().await;
+ sim.ctrl.arm_delay(DELAY_MS);
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(2_100).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ sim.obs_sends(2_000).await;
+ let cut_at = Instant::now();
+ sim.ctrl.stop_delay();
+ sim.obs_sends(1_500).await;
+
+ let frames = sim.frames(0);
+ let ahead = forward_jumps(&frames);
+ assert_eq!(ahead.len(), 1, "exactly one jump forward");
+ let i = ahead[0];
+ let landed = frames[i];
+ assert!(landed.at > cut_at, "the jump is the cut");
+ let late = landed.at.duration_since(cut_at).as_millis() as u32;
+ assert!(late <= FRAME_MS + JITTER_MS, "cut {late} ms after asking");
+ assert!(landed.keyframe, "lands on a keyframe");
+ let aired_before = frames[..i].iter().map(|f| f.number).max().unwrap();
+ assert!(landed.number > aired_before, "nothing is replayed");
+ assert_headers_lead(&mut sim, 0, &frames, i);
+ assert_strictly_increasing(&frames[i..], "live again");
+ assert_delays(&sim, &frames[i..], 0, GOP_MS + JITTER_MS, "live again");
+ assert_eq!(cuts(&sim, "platform"), 2, "activate, then cut");
+ sim.assert_timestamps_monotonic(0);
+ assert_audio_kept_up(&mut sim, 0, 3);
+}
+
+/// "Cut after this airs": everything up to the moment the streamer marked
+/// airs first, delayed, and only then does the stream jump to live.
+#[tokio::test]
+async fn cut_after_this_airs_waits_for_the_marked_frame_then_goes_live() {
+ let mut sim = delay_sim().await;
+ sim.ctrl.arm_delay(DELAY_MS);
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(2_100).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ sim.obs_sends(600).await;
+ let marked = sim.next_frame - 1;
+ sim.ctrl.schedule_safe_cut().expect("a delay is on air");
+ sim.obs_sends(3_000).await;
+
+ let frames = sim.frames(0);
+ let back = backward_jumps(&frames);
+ assert_eq!(back.len(), 1, "the activation is the only jump back");
+ let ahead = forward_jumps(&frames);
+ assert_eq!(ahead.len(), 1, "exactly one jump forward");
+ let (from, to) = (back[0], ahead[0]);
+ let delayed = &frames[from..to];
+ assert!(
+ delayed.windows(2).all(|p| p[1].number == p[0].number + 1),
+ "every delayed frame aired, in order"
+ );
+ let aired = delayed
+ .iter()
+ .find(|f| f.number == marked)
+ .expect("the marked frame aired before the cut");
+ let waited = frames[to].at.duration_since(aired.at).as_millis() as u32;
+ // The pump looks at the mark every 500 ms, on its next frame.
+ assert!(
+ waited <= 500 + FRAME_MS + JITTER_MS,
+ "cut {waited} ms after the mark aired"
+ );
+ assert!(frames[to].keyframe, "lands on a keyframe");
+ assert_delays(&sim, &frames[to..], 0, GOP_MS + JITTER_MS, "live again");
+ assert!(!sim.ctrl.safe_cut_pending());
+ assert_eq!(sim.ctrl.target_delay_ms(), 0);
+ assert_eq!(sim.ctrl.armed_delay_ms(), DELAY_MS, "armed for next time");
+ assert_eq!(cuts(&sim, "platform"), 2);
+ sim.assert_timestamps_monotonic(0);
+ assert_audio_kept_up(&mut sim, 0, 3);
+}
+
+/// Raising the delay on air past what the buffer holds: the pump holds its
+/// place while the buffer grows, then jumps back once and plays on. It
+/// used to jump as soon as the buffer spanned the new delay less the re-cut
+/// dead band, landing short enough that the next check jumped back to the
+/// same keyframe and viewers saw the same stretch twice in a row. Now it
+/// waits for a keyframe at least the new delay old.
+///
+/// Raised here to the most the buffer can reach at a moment its oldest
+/// keyframe sits 400 ms in, which is where the old build-up cut short.
+#[tokio::test]
+async fn raising_the_delay_on_air_jumps_back_exactly_once() {
+ let mut sim = delay_sim().await;
+ sim.ctrl.arm_delay(DELAY_MS);
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(2_500).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ sim.obs_sends(1_700).await;
+ let ring = &sim.ctrl.ring;
+ let oldest = ring.oldest_ts().unwrap();
+ let oldest_keyframe = ring.oldest_idr_at_or_after(ring.front_seq().unwrap());
+ let gap = oldest_keyframe.unwrap().ts_ms - oldest;
+ assert!(gap >= 300, "the oldest keyframe sits only {gap} ms in");
+ let dead_band = recut_dead_band_ms(sim.ctrl.keyframe_interval_ms()) as u32;
+ let raised = (ring.latest_ts().unwrap() - oldest) as u32 + dead_band;
+ let raised_at = Instant::now();
+ sim.ctrl.arm_delay(raised);
+ sim.obs_sends(3_000).await;
+
+ assert_eq!(cuts(&sim, "platform"), 2, "activate, then one jump back");
+ let after: Vec = sim
+ .frames(0)
+ .into_iter()
+ .filter(|f| f.at > raised_at)
+ .collect();
+ let mut seen = HashSet::new();
+ let again = after.iter().find(|f| !seen.insert(f.number));
+ assert!(
+ again.is_none(),
+ "{again:?} aired twice after raising the delay"
+ );
+ let reached = delay_ms(&sim, after.last().unwrap());
+ assert!(
+ reached + TIMER_SLOP_MS >= raised,
+ "ran {reached} ms behind, asked for {raised}"
+ );
+ sim.assert_timestamps_monotonic(0);
+}
+
+/// Lowering the delay on air: one jump forward to the new delay, with
+/// nothing that already aired shown again.
+#[tokio::test]
+async fn lowering_the_delay_on_air_jumps_forward_once() {
+ const LOWERED_MS: u32 = 1_000;
+ let mut sim = delay_sim().await;
+ sim.ctrl.arm_delay(3_000);
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(3_100).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ sim.obs_sends(900).await;
+ let lowered_at = Instant::now();
+ sim.ctrl.arm_delay(LOWERED_MS);
+ sim.obs_sends(1_500).await;
+
+ let frames = sim.frames(0);
+ let back = backward_jumps(&frames);
+ assert_eq!(back.len(), 1, "the activation is the only jump back");
+ let ahead = forward_jumps(&frames);
+ assert_eq!(ahead.len(), 1, "exactly one jump forward");
+ let i = ahead[0];
+ assert!(frames[i].at > lowered_at, "the jump is the change");
+ assert!(frames[i].keyframe, "lands on a keyframe");
+ // What aired live before going delayed airs again by nature; nothing
+ // from the delayed stretch may.
+ let aired_delayed = frames[back[0]..i].iter().map(|f| f.number).max().unwrap();
+ assert!(frames[i].number > aired_delayed, "nothing is replayed");
+ assert_airs_at_most_twice(&frames);
+ assert_strictly_increasing(&frames[i..], "after lowering");
+ assert_delay_honored(&sim, &frames[i..], LOWERED_MS, GOP_MS, "after lowering");
+ assert_eq!(cuts(&sim, "platform"), 2, "activate, then lower");
+ sim.assert_timestamps_monotonic(0);
+ assert_audio_kept_up(&mut sim, 0, 3);
+}
+
+/// A keyframe every 6 s, in frames.
+const GOP_6S: u32 = 60;
+
+/// A destination is switched on 2.6 s into a 2 s delay, with a keyframe
+/// every 6 s: the only keyframe in the stream so far is OBS's first one.
+async fn destination_added_mid_delay_on_a_long_gop(sim: &mut Sim) {
+ sim.gop_frames = GOP_6S;
+ sim.ctrl.arm_delay(DELAY_MS);
+ sim.obs_connects().await;
+ sim.destination_connects("first").await;
+ sim.obs_sends(1_700).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ sim.obs_sends(2_600).await;
+ sim.destination_connects("added").await;
+ sim.obs_sends(2_400).await;
+}
+
+/// The added destination joins on a keyframe and plays on from there,
+/// without the jump back a live start followed by a cut to the delay would
+/// show, at least the delay behind and within a keyframe interval of it.
+#[tokio::test]
+async fn a_destination_added_mid_delay_joins_without_a_jump_on_a_long_gop() {
+ let mut sim = delay_sim().await;
+ destination_added_mid_delay_on_a_long_gop(&mut sim).await;
+
+ let frames = sim.frames(1);
+ let first = frames.first().expect("the added destination is on air");
+ assert!(first.keyframe, "joins on a keyframe");
+ assert_delay_honored(&sim, &frames, DELAY_MS, GOP_6S * FRAME_MS, "added");
+ assert_strictly_increasing(&frames, "the added destination");
+ assert_eq!(cuts(&sim, "added"), 0, "no jump after joining");
+ sim.assert_timestamps_monotonic(1);
+}
+
+/// The keyframe nearest the delay when the destination is added is OBS's
+/// first one, 4.3 s back, but the buffer keeps only the armed delay plus
+/// 2 s, so it was trimmed as soon as the first destination played past it.
+/// With no keyframe left to join on, the seed waits for OBS's next one and
+/// starts there, at the live edge: the added destination's viewers watch
+/// live while the other destination and the dashboard are on the delay.
+#[tokio::test]
+async fn a_destination_added_mid_delay_on_a_long_gop_never_airs_live() {
+ let mut sim = delay_sim().await;
+ destination_added_mid_delay_on_a_long_gop(&mut sim).await;
+
+ let frames = sim.frames(1);
+ let first = frames.first().expect("the added destination is on air");
+ let joined = delay_ms(&sim, first);
+ assert!(
+ joined + TIMER_SLOP_MS >= DELAY_MS,
+ "joined {joined} ms behind OBS with a {DELAY_MS} ms delay on"
+ );
+}
+
+/// Going delayed with a 4 s keyframe interval: a 2 s delay can only land
+/// on a keyframe about 0 s or 4 s back, and it takes the one at least the
+/// delay old, then stays put instead of chasing the exact delay.
+///
+/// The re-cut dead band (2.5 s here) used to apply to going delayed too:
+/// live sat 2 s from the 2 s delay, inside the band, so the pump never cut
+/// and viewers kept watching live while the dashboard said delayed.
+#[tokio::test]
+async fn activating_with_a_long_gop_never_lands_short_of_the_delay() {
+ const GOP_4S: u32 = 40;
+ let mut sim = delay_sim().await;
+ sim.gop_frames = GOP_4S;
+ sim.ctrl.arm_delay(DELAY_MS);
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(4_500).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ sim.obs_sends(1_500).await;
+
+ let frames = sim.frames(0);
+ let back = backward_jumps(&frames);
+ assert_eq!(back.len(), 1, "exactly one jump back");
+ let i = back[0];
+ assert!(frames[i].keyframe, "lands on a keyframe");
+ assert_delay_honored(&sim, &frames[i..], DELAY_MS, GOP_4S * FRAME_MS, "delayed");
+ assert_strictly_increasing(&frames[i..], "delayed");
+ assert_eq!(cuts(&sim, "platform"), 1, "no re-cut once on the delay");
+ sim.assert_timestamps_monotonic(0);
+}
+
+/// OBS's 32-bit millisecond clock rolls over every 49.7 days. Going
+/// delayed across the rollover (live is past it, the delay before it) and
+/// cutting back across it again must behave exactly as anywhere else.
+#[tokio::test]
+async fn going_delayed_and_back_across_the_timestamp_wrap() {
+ let mut sim = delay_sim().await;
+ sim.ctrl.arm_delay(DELAY_MS);
+ // The clock rolls over 1.5 s in.
+ sim.obs_connects_at(0u32.wrapping_sub(1_500)).await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(2_100).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ sim.obs_sends(1_000).await;
+ let cut_at = Instant::now();
+ sim.ctrl.stop_delay();
+ sim.obs_sends(1_500).await;
+
+ let frames = sim.frames(0);
+ let back = backward_jumps(&frames);
+ assert_eq!(back.len(), 1, "exactly one jump back");
+ let ahead = forward_jumps(&frames);
+ assert_eq!(ahead.len(), 1, "exactly one jump forward");
+ let (from, to) = (back[0], ahead[0]);
+ assert!(frames[from].keyframe && frames[to].keyframe);
+ assert!(frames[to].at > cut_at);
+ assert_delay_honored(&sim, &frames[from..to], DELAY_MS, GOP_MS, "delayed");
+ let aired_before = frames[..to].iter().map(|f| f.number).max().unwrap();
+ assert!(frames[to].number > aired_before, "nothing is replayed");
+ assert_delays(&sim, &frames[to..], 0, GOP_MS + JITTER_MS, "live again");
+ assert_eq!(cuts(&sim, "platform"), 2);
+ sim.assert_timestamps_monotonic(0);
+ assert_audio_kept_up(&mut sim, 0, 3);
+}
+
+/// OBS drops and reconnects inside the half second the pump waits for its
+/// next frame, with crash protection off: the pump never sees it gone,
+/// only a new publisher. It carries on on the same connection, from a
+/// keyframe of the new session, with its timestamps still going forward.
+async fn obs_reconnects_inside_the_pumps_wait(sim: &mut Sim) -> Instant {
+ sim.obs_sends(1_000).await;
+ wait(100).await;
+ sim.obs_crashes();
+ wait(100).await;
+ sim.obs_connects().await;
+ let back = Instant::now();
+ sim.obs_sends(1_500).await;
+ back
+}
+
+fn assert_rejoined_the_new_session(sim: &mut Sim, back: Instant) {
+ assert!(
+ !sim.commands(0).contains(&"deleteStream".into()),
+ "the destination never ended"
+ );
+ assert!(sim.received().iter().all(|m| m.conn == 0), "one connection");
+ let frames = sim.frames(0);
+ let rejoined = frames
+ .iter()
+ .find(|f| f.at > back)
+ .expect("the stream came back");
+ assert!(rejoined.keyframe, "rejoins at a keyframe");
+ let paused = rejoined.at.duration_since(back).as_millis() as u32;
+ assert!(
+ paused <= GOP_MS + FRAME_MS + JITTER_MS,
+ "back {paused} ms after OBS"
+ );
+ assert_strictly_increasing(&frames, "across the reconnect");
+ sim.assert_timestamps_monotonic(0);
+}
+
+/// A destination that joined mid-stream, on a keyframe after OBS's first.
+#[tokio::test]
+async fn a_fast_reconnect_rejoins_the_new_session_on_a_keyframe() {
+ let mut sim = delay_sim().await;
+ sim.obs_connects().await;
+ // The newest frame is a keyframe: the pump joins on it and runs level
+ // with OBS, parked on the next frame when OBS drops.
+ sim.obs_sends(600).await;
+ sim.destination_connects("platform").await;
+ let back = obs_reconnects_inside_the_pumps_wait(&mut sim).await;
+ assert_rejoined_the_new_session(&mut sim, back);
+}
+
+/// The same, for a destination that joined on OBS's very first keyframe
+/// (the usual case: it connects as OBS starts). Its timeline is anchored at
+/// input ts 0, so the new session's first keyframe, also at ts 0, passes
+/// the "older than the anchor" guard. It used to go out before the pump
+/// noticed the new publisher, on output timestamp 0, back at the start of
+/// the stream.
+#[tokio::test]
+async fn a_fast_reconnect_keeps_timestamps_forward_when_the_pump_joined_at_the_start() {
+ let mut sim = delay_sim().await;
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ let back = obs_reconnects_inside_the_pumps_wait(&mut sim).await;
+ assert_rejoined_the_new_session(&mut sim, back);
+}
+
+/// OBS's encoder restarts mid-session with a new resolution: new config,
+/// then a keyframe. Returns the header and the first frame made with it.
+async fn obs_changes_resolution_mid_stream(sim: &mut Sim) -> (Vec, u32) {
+ // Audio level with the video, the order OBS's interleaver sends in. With
+ // audio stamped behind, the audio config resent on the video's newer
+ // timestamp would run ahead of the audio still to go out.
+ sim.audio_lag_ms = 0;
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(1_000).await;
+ let header = sim.obs_changes_resolution(128, 72);
+ let first_new = sim.next_frame;
+ sim.obs_sends(1_000).await;
+ (header, first_new)
+}
+
+/// Where connection 0 received `header`, and the last timestamp of media
+/// it had received by then.
+fn header_arrival(sim: &mut Sim, header: &[u8]) -> (usize, u32, u32) {
+ let received = sim.received();
+ let at = received
+ .iter()
+ .position(|m| m.conn == 0 && m.kind == 9 && m.payload == header)
+ .expect("the new sequence header was sent");
+ let last_media_ts = received[..at]
+ .iter()
+ .filter(|m| m.conn == 0 && (m.kind == 8 || m.kind == 9))
+ .map(|m| m.ts)
+ .max()
+ .unwrap();
+ (at, received[at].ts, last_media_ts)
+}
+
+/// A new sequence header mid-stream is resent once, on the last output
+/// timestamp (never behind what already went out), with the audio config
+/// alongside, and the stream carries on.
+#[tokio::test]
+async fn a_new_sequence_header_is_resent_mid_stream_on_the_last_timestamp() {
+ let mut sim = delay_sim().await;
+ let (header, first_new) = obs_changes_resolution_mid_stream(&mut sim).await;
+
+ let (at, ts, last_media_ts) = header_arrival(&mut sim, &header);
+ assert_eq!(ts, last_media_ts, "on the last output timestamp");
+ let sent = sim
+ .received()
+ .iter()
+ .filter(|m| m.conn == 0 && m.payload == header)
+ .count();
+ assert_eq!(sent, 1, "resent once");
+ let audio_config = sim.received()[at..]
+ .iter()
+ .find(|m| m.conn == 0 && m.kind == 8 && m.payload.get(1) == Some(&0));
+ assert_eq!(
+ audio_config.map(|m| m.ts),
+ Some(ts),
+ "audio config alongside"
+ );
+ let frames = sim.frames(0);
+ let second_new = frames
+ .iter()
+ .find(|f| f.number == first_new + 1)
+ .expect("the stream carried on");
+ assert!(at < second_new.index, "before the stream moved on");
+ sim.assert_timestamps_monotonic(0);
+}
+
+/// The header has to reach the decoder before the frame encoded with it.
+/// The pump compares the header generation at the top of its loop, then
+/// parks waiting for the next frame; the frame that wakes it used to go
+/// out before the loop came round again, decoded with the old config.
+#[tokio::test]
+async fn a_new_sequence_header_goes_out_before_the_frame_that_needs_it() {
+ let mut sim = delay_sim().await;
+ let (header, first_new) = obs_changes_resolution_mid_stream(&mut sim).await;
+
+ let (at, _, _) = header_arrival(&mut sim, &header);
+ let frames = sim.frames(0);
+ let needs_it = frames
+ .iter()
+ .find(|f| f.number == first_new)
+ .expect("the new keyframe aired");
+ assert!(
+ at < needs_it.index,
+ "frame {first_new} went out before its sequence header"
+ );
+}
+
+/// OBS stops and starts again while a delay is on air, crash protection
+/// off. The destination ends with OBS, and the supervisor reconnects it
+/// once OBS is back. Returns the first frame OBS made after the restart.
+async fn obs_restarts_mid_delay(sim: &mut Sim) -> u32 {
+ sim.ctrl.arm_delay(DELAY_MS);
+ sim.obs_connects().await;
+ sim.destination_runs("platform");
+ sim.obs_sends(1_700).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ sim.obs_sends(500).await;
+ sim.obs_stops();
+ wait(700).await;
+ sim.obs_connects().await;
+ let first_new = sim.next_frame;
+ sim.obs_sends(2_500).await;
+ first_new
+}
+
+#[tokio::test]
+async fn obs_restarting_mid_delay_brings_the_destination_back_on_a_keyframe() {
+ let mut sim = delay_sim().await;
+ let first_new = obs_restarts_mid_delay(&mut sim).await;
+
+ assert_eq!(sim.ctrl.target_delay_ms(), DELAY_MS, "the delay stays on");
+ assert!(sim.commands(0).contains(&"deleteStream".into()));
+ let frames = sim.frames(1);
+ let first = frames.first().expect("the destination came back");
+ assert!(first.keyframe, "starts on a keyframe");
+ assert!(first.number >= first_new, "on the new session's video");
+ sim.assert_timestamps_monotonic(1);
+}
+
+/// The delay stays on through the restart, so viewers must come back
+/// delayed: never on live video, and never jumping back once the buffer
+/// has rebuilt. The restart empties the buffer, so the destination waits
+/// for it to reach back to the delay before it goes on air again.
+#[tokio::test]
+async fn obs_restarting_mid_delay_never_puts_viewers_on_live() {
+ let mut sim = delay_sim().await;
+ obs_restarts_mid_delay(&mut sim).await;
+
+ let frames = sim.frames(1);
+ let first = frames.first().expect("the destination came back");
+ let joined = delay_ms(&sim, first);
+ assert!(
+ joined + TIMER_SLOP_MS >= DELAY_MS,
+ "came back {joined} ms behind OBS with a {DELAY_MS} ms delay on"
+ );
+ assert_strictly_increasing(&frames, "after the restart");
+}
+
+/// Twitch's Enhanced Broadcasting binds each track's decoder config to its
+/// session, so every connection has to lead with every track's config,
+/// including the one made when the supervisor restarts egress while OBS
+/// keeps publishing. A cache that kept only the last track (the shape that
+/// shipped once) leaves the others with none, and Twitch drops the stream.
+#[tokio::test]
+async fn every_track_config_leads_each_twitch_connection() {
+ let mut sim = delay_sim().await;
+ sim.obs_connects().await;
+ for track in 1u8..=3 {
+ let mut rung = vec![0x96, 0x00, b'a', b'v', b'c', b'1', track];
+ rung.extend_from_slice(&[0xde, 0xad, 0xbe, 0xef]);
+ sim.ctrl.on_tag(9, 0, &rung, false, true);
+ }
+ let twitch = sim.ctrl.destination_state("twitch");
+ twitch
+ .pass_through_multitrack_video
+ .store(true, Ordering::Relaxed);
+ sim.destination_connects("twitch").await;
+ sim.obs_sends(600).await;
+ twitch.shutdown_requested.store(true, Ordering::Relaxed);
+ sim.obs_sends(300).await;
+ assert!(sim.pumps[0].is_finished(), "the first session ended");
+ twitch.shutdown_requested.store(false, Ordering::Relaxed);
+ sim.destination_connects("twitch").await;
+ sim.obs_sends(600).await;
+
+ for conn in [0, 1] {
+ let first = sim.frames(conn).first().expect("frames aired").index;
+ let configs: HashSet<&[u8]> = sim.received()[..first]
+ .iter()
+ .filter(|m| m.conn == conn && m.kind == 9)
+ .map(|m| m.payload.as_slice())
+ .collect();
+ assert_eq!(configs.len(), 4, "connection {conn} led with every track");
+ }
+}
+
+/// A delay longer than the buffer can hold at this bitrate (a hotkey,
+/// MIDI or auto-arm skip the dashboard's capacity check, and the bitrate
+/// can rise after arming) must not wait forever. Once the buffer is full
+/// it counts as ready, the destination joins as far back as the buffer
+/// safely reaches, and the stream runs there without stalling or
+/// replaying. The log says the delay is capped.
+#[tokio::test]
+async fn a_delay_longer_than_the_buffer_runs_at_what_it_holds() {
+ // About 7.5 s of stream (24 bytes a frame at 10 fps).
+ let mut sim = Sim::with_ring_bytes(false, 1_800).await;
+ sim.ctrl.arm_delay(30_000);
+ sim.obs_connects().await;
+ sim.obs_sends(9_000).await;
+ assert_eq!(
+ sim.ctrl.phase(),
+ "ready",
+ "a full buffer holds all it ever will"
+ );
+ sim.ctrl
+ .activate_delay()
+ .expect("activates at what the buffer holds");
+ sim.destination_runs("platform");
+ sim.obs_sends(6_000).await;
+
+ let frames = sim.frames(0);
+ assert!(
+ frames.len() >= 30,
+ "the destination joined and kept streaming: {} frames",
+ frames.len()
+ );
+ assert_strictly_increasing(&frames, "a delay capped by the buffer");
+ let behind = delay_ms(&sim, frames.last().unwrap());
+ assert!(
+ (2_500..=7_500).contains(&behind),
+ "delayed by about what the buffer holds, not {behind} ms"
+ );
+ let gaps: Vec = frames
+ .windows(2)
+ .map(|w| w[1].at.duration_since(w[0].at).as_millis() as u32)
+ .collect();
+ assert!(
+ gaps.iter().all(|gap| *gap < 1_000),
+ "no stall while capped: {gaps:?}"
+ );
+ assert!(
+ sim.ctrl
+ .logs
+ .lock()
+ .iter()
+ .any(|line| line.contains("the buffer is full")),
+ "the log says the delay is capped"
+ );
+ sim.assert_timestamps_monotonic(0);
+}
+
+/// The keyframe interval is measured from the first few gaps only. When
+/// the encoder's real interval turns out much longer (OBS's "auto" after a
+/// restart, or early scene-cut keyframes), a landed delay can sit more than
+/// the expected band above its target. Cutting to fix that must move
+/// forward: the keyframe at the delay is the one the pump already passed,
+/// and cutting back to it replayed the same stretch on every check.
+#[tokio::test]
+async fn a_longer_keyframe_interval_than_measured_never_replays() {
+ let mut sim = delay_sim().await;
+ sim.ctrl.arm_delay(DELAY_MS);
+ sim.obs_connects().await;
+ // The measurement: keyframes every 500 ms (frames 0..29).
+ sim.obs_sends(3_000).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ // Then the real interval: a keyframe every 4 s, at frames 40 and 80.
+ sim.gop_frames = 40;
+ // Join 1 s after keyframe 80: it isn't the delay old yet, so the join
+ // lands on keyframe 40, about 5 s back - past the expected band.
+ sim.obs_sends(6_000).await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(10_000).await;
+
+ let frames = sim.frames(0);
+ let back = backward_jumps(&frames);
+ assert!(back.is_empty(), "the picture jumped back at {back:?}");
+ assert_airs_at_most_twice(&frames);
+ sim.assert_timestamps_monotonic(0);
+}
diff --git a/src/controller/hold_sim.rs b/src/controller/hold_sim.rs
new file mode 100644
index 0000000..669f8fe
--- /dev/null
+++ b/src/controller/hold_sim.rs
@@ -0,0 +1,666 @@
+//! Crash-protection simulations (see `sim` for the harness). A real
+//! destination pump streams to an in-process platform while the test plays
+//! OBS in real time: live, crash, freeze, stop, back again. The assertions
+//! are about what viewers would see: the tail airs once, the screen covers
+//! the gap, nothing from before the hold is replayed, and timestamps never
+//! go back.
+
+use super::sim::{assert_strictly_increasing, wait, Frame, Sim};
+use std::sync::atomic::Ordering;
+use std::time::{Duration, Instant};
+
+/// OBS crashes with no delay: the screen covers the gap on the same
+/// connection, and OBS coming back rejoins at its first keyframe with
+/// nothing replayed.
+#[tokio::test]
+async fn crash_screen_then_obs_back_on_the_same_connection() {
+ let mut sim = Sim::new(true).await;
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(1_500).await;
+ sim.obs_crashes();
+ let crashed = Instant::now();
+ wait(1_500).await;
+ sim.obs_connects().await;
+ let back = Instant::now();
+ sim.obs_sends(1_500).await;
+
+ let frames = sim.frames(0);
+ assert!(
+ frames.iter().any(|f| f.at < crashed),
+ "live before the crash"
+ );
+ assert!(
+ sim.screen_tags(0, crashed, back) >= 5,
+ "the screen covered the gap"
+ );
+ let after: Vec = frames.iter().copied().filter(|f| f.at > back).collect();
+ assert!(
+ after.first().is_some_and(|f| f.keyframe),
+ "rejoins at a keyframe"
+ );
+ assert_strictly_increasing(&frames, "crash and back");
+ assert!(
+ !sim.commands(0).contains(&"deleteStream".into()),
+ "never ended"
+ );
+ assert!(sim.received().iter().all(|m| m.conn == 0), "one connection");
+ sim.assert_timestamps_monotonic(0);
+}
+
+/// OBS crashes with a 1.5 s delay on air: the tail airs once (no loop),
+/// then the screen; when OBS is back the screen stays up until the delay
+/// has rebuilt, so the stream comes back still delayed.
+#[tokio::test]
+async fn a_crash_mid_delay_airs_the_tail_once_and_the_delay_survives_the_return() {
+ const DELAY_MS: u32 = 1_500;
+ let mut sim = Sim::new(true).await;
+ sim.ctrl.arm_delay(DELAY_MS);
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(2_500).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ sim.obs_sends(2_000).await;
+ let last_live = sim.next_frame - 1;
+ sim.obs_crashes();
+ let crashed = Instant::now();
+ wait(3_000).await;
+
+ let tail: Vec = sim
+ .frames(0)
+ .into_iter()
+ .filter(|f| f.at > crashed)
+ .collect();
+ assert_strictly_increasing(&tail, "the tail");
+ assert_eq!(
+ tail.last().map(|f| f.number),
+ Some(last_live),
+ "the whole tail aired"
+ );
+ let tail_ms = tail.last().unwrap().at.duration_since(crashed).as_millis();
+ assert!(
+ tail_ms >= 1_000,
+ "the tail took the delay to air, not {tail_ms} ms"
+ );
+ let screen_from = tail.last().unwrap().at;
+ assert!(
+ sim.screen_tags(0, screen_from, Instant::now()) >= 5,
+ "then the screen"
+ );
+
+ sim.obs_connects().await;
+ let back = Instant::now();
+ sim.obs_sends(3_000).await;
+ let after: Vec = sim.frames(0).into_iter().filter(|f| f.at > back).collect();
+ let rejoined = after.first().expect("the stream came back");
+ assert!(rejoined.keyframe, "rejoins at a keyframe");
+ let delayed_by = rejoined.at.duration_since(sim.produced[&rejoined.number]);
+ assert!(
+ delayed_by >= Duration::from_millis(1_000),
+ "still delayed after OBS came back, not live ({delayed_by:?})"
+ );
+ assert_strictly_increasing(&after, "after the return");
+ sim.assert_timestamps_monotonic(0);
+}
+
+/// OBS comes back from a crash while the delay tail is still airing. The
+/// rest of the tail went with the old session; the stream must not jump to
+/// live: the screen covers the gap until the delay has rebuilt.
+#[tokio::test]
+async fn obs_back_mid_tail_never_puts_viewers_on_live() {
+ const DELAY_MS: u32 = 3_000;
+ let mut sim = Sim::new(true).await;
+ sim.ctrl.arm_delay(DELAY_MS);
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(3_500).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ sim.obs_sends(2_000).await;
+ sim.obs_crashes();
+ wait(1_000).await;
+ sim.obs_connects().await;
+ let back = Instant::now();
+ sim.obs_sends(5_000).await;
+
+ let first_new = sim
+ .frames(0)
+ .into_iter()
+ .find(|f| f.at > back && sim.produced[&f.number] > back)
+ .expect("the stream came back");
+ assert!(first_new.keyframe, "rejoins at a keyframe");
+ let delayed_by = first_new.at.duration_since(sim.produced[&first_new.number]);
+ assert!(
+ delayed_by >= Duration::from_millis(2_000),
+ "OBS's new video must air delayed, not live ({delayed_by:?})"
+ );
+ sim.assert_timestamps_monotonic(0);
+}
+
+/// End now from the tray, a hotkey or MIDI (threads with no runtime): the
+/// destination ends with deleteStream, and nothing panics.
+#[tokio::test]
+async fn end_now_from_a_plain_thread_ends_the_destination_cleanly() {
+ let mut sim = Sim::new(true).await;
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(1_000).await;
+ sim.obs_crashes();
+ wait(800).await;
+ let ctrl = sim.ctrl.clone();
+ let hotkey = std::thread::spawn(move || ctrl.run_named_action("end_hold", 0, "hotkey"));
+ assert_eq!(hotkey.join().expect("no panic off the runtime"), None);
+ wait(800).await;
+ assert!(sim.commands(0).contains(&"deleteStream".into()));
+ assert!(sim.pumps[0].is_finished(), "the pump ended");
+}
+
+/// OBS freezes (connected, no video): the screen covers it, a destination
+/// that (re)connects mid-freeze goes straight to the screen instead of
+/// replaying old video, and both rejoin at OBS's first keyframe after it.
+#[tokio::test]
+async fn a_freeze_is_covered_and_a_reconnecting_destination_goes_straight_to_the_screen() {
+ let mut sim = Sim::new(true).await;
+ sim.obs_connects().await;
+ sim.destination_connects("first").await;
+ sim.obs_sends(1_000).await;
+ let last_before = sim.next_frame - 1;
+ sim.obs_stalls(300).await;
+ sim.freeze_detected();
+ assert!(sim.ctrl.hold_active());
+ sim.obs_stalls(800).await;
+ sim.destination_connects("second").await;
+ sim.obs_stalls(800).await;
+ let back = Instant::now();
+ sim.obs_sends(1_500).await;
+
+ let second = sim.frames(1);
+ assert!(second.iter().all(|f| f.at > back), "no old video replayed");
+ assert!(sim.screen_tags(1, back - Duration::from_millis(700), back) >= 3);
+ for conn in [0, 1] {
+ let after: Vec = sim
+ .frames(conn)
+ .into_iter()
+ .filter(|f| f.at > back)
+ .collect();
+ let first = after.first().expect("the stream came back");
+ assert!(first.keyframe, "conn {conn} rejoins at a keyframe");
+ assert!(
+ first.number > last_before,
+ "conn {conn} rejoins on new video"
+ );
+ sim.assert_timestamps_monotonic(conn);
+ }
+ assert_strictly_increasing(&sim.frames(0), "the first destination");
+}
+
+/// Stopping in OBS while the screen covers a freeze ends the stream.
+#[tokio::test]
+async fn stopping_during_a_freeze_ends_the_destination() {
+ let mut sim = Sim::new(true).await;
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(1_000).await;
+ sim.obs_stalls(300).await;
+ sim.freeze_detected();
+ sim.obs_stalls(500).await;
+ sim.obs_stops();
+ wait(800).await;
+ assert!(sim.commands(0).contains(&"deleteStream".into()));
+}
+
+/// Crash protection off: OBS dropping ends the destination at once, as
+/// before crash protection existed.
+#[tokio::test]
+async fn protection_off_ends_the_destination_when_obs_drops() {
+ let mut sim = Sim::new(false).await;
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(1_000).await;
+ sim.obs_crashes();
+ let crashed = Instant::now();
+ wait(800).await;
+ assert!(sim.commands(0).contains(&"deleteStream".into()));
+ assert_eq!(sim.screen_tags(0, crashed, Instant::now()), 0, "no screen");
+}
+
+/// OBS comes back with a delay armed, so the screen stays up while the
+/// delay rebuilds from its new video, then OBS stops on purpose before it
+/// has. There is no hold left to end, so the screen has to notice OBS is
+/// gone and end the stream at once, not wait out the rebuild grace.
+#[tokio::test]
+async fn obs_stopping_while_the_delay_rebuilds_ends_the_destination() {
+ const DELAY_MS: u32 = 1_000;
+ let mut sim = Sim::new(true).await;
+ sim.ctrl.arm_delay(DELAY_MS);
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(1_000).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ sim.obs_crashes();
+ wait(1_500).await;
+ sim.obs_connects().await;
+ let back = Instant::now();
+ sim.obs_sends(300).await;
+ assert_eq!(sim.ctrl.hold_state(), crate::crash_hold::HoldState::Resumed);
+ sim.obs_stops();
+ wait(600).await;
+
+ let back_on_air = sim.frames(0).into_iter().filter(|f| f.at > back).count();
+ assert_eq!(back_on_air, 0, "OBS's short new video never aired");
+ assert!(
+ sim.commands(0).contains(&"deleteStream".into()),
+ "the stream ended when OBS stopped"
+ );
+ assert!(sim.pumps[0].is_finished(), "the pump ended");
+ sim.assert_timestamps_monotonic(0);
+}
+
+/// Crash protection with a short hold, for the scenarios where it runs out.
+fn one_second_hold(sim: &Sim) {
+ sim.ctrl
+ .update_crash_protection(crate::crash_protection::CrashProtection {
+ enabled: true,
+ hold_secs: 1,
+ ..Default::default()
+ });
+}
+
+/// OBS crashes with a delay longer than the hold time. The hold runs out
+/// while the tail is still airing, and the tail airs in full before the
+/// stream ends: viewers see everything OBS sent before the crash, then a
+/// clean end. It used to end at the deadline, dropping the rest.
+#[tokio::test]
+async fn a_hold_that_runs_out_mid_tail_airs_the_tail_in_full_then_ends() {
+ const DELAY_MS: u32 = 3_000;
+ let mut sim = Sim::new(true).await;
+ one_second_hold(&sim);
+ sim.ctrl.arm_delay(DELAY_MS);
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(3_500).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ sim.obs_sends(2_000).await;
+ let last_live = sim.next_frame - 1;
+ sim.obs_crashes();
+ let crashed = Instant::now();
+ wait(1_500).await;
+ // What the supervisor does every tick.
+ sim.ctrl.expire_hold();
+ wait(3_000).await;
+
+ let tail: Vec = sim
+ .frames(0)
+ .into_iter()
+ .filter(|f| f.at > crashed)
+ .collect();
+ assert_eq!(
+ tail.last().map(|f| f.number),
+ Some(last_live),
+ "the whole tail aired"
+ );
+ assert_strictly_increasing(&tail, "the tail");
+ assert!(
+ sim.commands(0).contains(&"deleteStream".into()),
+ "then it ended"
+ );
+ assert!(sim.pumps[0].is_finished(), "the pump ended");
+ sim.assert_timestamps_monotonic(0);
+}
+
+/// OBS freezes with a delay on, and stays frozen past the hold. The
+/// destination ends; it used to fall back into the normal delay logic,
+/// which saw too little delay and cut back into the tail again and again.
+#[tokio::test]
+async fn a_freeze_hold_that_runs_out_never_replays_the_tail() {
+ const DELAY_MS: u32 = 1_500;
+ let mut sim = Sim::new(true).await;
+ one_second_hold(&sim);
+ sim.ctrl.arm_delay(DELAY_MS);
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(2_500).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ sim.obs_sends(2_000).await;
+ sim.obs_stalls(300).await;
+ let froze = Instant::now();
+ sim.freeze_detected();
+ assert!(sim.ctrl.hold_active(), "the freeze is held");
+ sim.obs_stalls(1_500).await;
+ sim.ctrl.expire_hold();
+ sim.obs_stalls(3_000).await;
+
+ // From the last frame before the freeze on (activating the delay
+ // replayed its own stretch once, as it should).
+ let frames = sim.frames(0);
+ let from = frames.iter().rposition(|f| f.at < froze).unwrap_or(0);
+ assert_strictly_increasing(&frames[from..], "a frozen OBS past the hold");
+ assert!(
+ sim.commands(0).contains(&"deleteStream".into()),
+ "the destination ended"
+ );
+ sim.assert_timestamps_monotonic(0);
+}
+
+/// End now while the delay tail is airing ends the stream at once: the
+/// streamer asked for it, so the rest of the tail doesn't air first.
+#[tokio::test]
+async fn end_now_mid_tail_ends_the_stream_at_once() {
+ const DELAY_MS: u32 = 3_000;
+ let mut sim = Sim::new(true).await;
+ sim.ctrl.arm_delay(DELAY_MS);
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(3_500).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ sim.obs_sends(2_000).await;
+ let last_live = sim.next_frame - 1;
+ sim.obs_crashes();
+ wait(500).await;
+ assert!(sim.ctrl.end_hold_now(), "a hold was on");
+ wait(1_000).await;
+
+ assert!(sim.commands(0).contains(&"deleteStream".into()), "it ended");
+ assert!(
+ sim.frames(0).last().is_some_and(|f| f.number < last_live),
+ "without airing the rest of the tail"
+ );
+}
+
+/// OBS leaves with a 2 s delay on air and no hold to cover it. Returns the
+/// last frame OBS made and when it left.
+async fn obs_leaves_mid_delay(sim: &mut Sim, stopped: bool) -> (u32, Instant) {
+ sim.ctrl.arm_delay(2_000);
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(2_500).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ sim.obs_sends(1_500).await;
+ let last_live = sim.next_frame - 1;
+ if stopped {
+ sim.obs_stops();
+ } else {
+ sim.obs_crashes();
+ }
+ (last_live, Instant::now())
+}
+
+/// What viewers see once OBS left: everything it sent, over the delay's
+/// length, then a clean end with no reconnect screen.
+fn assert_tail_aired_then_ended(sim: &mut Sim, last_live: u32, left: Instant) {
+ let tail: Vec = sim.frames(0).into_iter().filter(|f| f.at > left).collect();
+ assert_eq!(
+ tail.last().map(|f| f.number),
+ Some(last_live),
+ "the whole delay aired"
+ );
+ assert_strictly_increasing(&tail, "the tail");
+ let took = tail.last().unwrap().at.duration_since(left).as_millis();
+ assert!(
+ took >= 1_500,
+ "the tail took the delay to air, not {took} ms"
+ );
+ assert!(
+ sim.commands(0).contains(&"deleteStream".into()),
+ "then it ended"
+ );
+ assert_eq!(sim.screen_tags(0, left, Instant::now()), 0, "no screen");
+ assert!(sim.pumps[0].is_finished(), "the pump ended");
+ sim.assert_timestamps_monotonic(0);
+}
+
+/// Crash protection off, OBS stops on purpose with a delay on: the end of
+/// the stream reaches viewers through the delay like everything else. It
+/// used to cut the last 2 s off.
+#[tokio::test]
+async fn protection_off_a_stop_airs_the_rest_of_the_delay() {
+ let mut sim = Sim::new(false).await;
+ let (last_live, left) = obs_leaves_mid_delay(&mut sim, true).await;
+ wait(1_000).await;
+ assert!(
+ sim.ctrl
+ .tail_left()
+ .is_some_and(|left| left > Duration::ZERO),
+ "the dashboard shows the delay still airing"
+ );
+ wait(2_000).await;
+ assert_tail_aired_then_ended(&mut sim, last_live, left);
+ assert_eq!(sim.ctrl.tail_left(), None, "nothing left to air");
+}
+
+/// Crash protection off, OBS crashes with a delay on: what it sent before
+/// the crash airs too.
+#[tokio::test]
+async fn protection_off_a_crash_airs_the_rest_of_the_delay() {
+ let mut sim = Sim::new(false).await;
+ let (last_live, left) = obs_leaves_mid_delay(&mut sim, false).await;
+ wait(3_000).await;
+ assert_tail_aired_then_ended(&mut sim, last_live, left);
+}
+
+/// Crash protection on but only for crashes: a stop on purpose opens no
+/// hold, and the delay still airs in full before the end.
+#[tokio::test]
+async fn a_stop_crash_protection_lets_through_airs_the_rest_of_the_delay() {
+ let mut sim = Sim::new(true).await;
+ let (last_live, left) = obs_leaves_mid_delay(&mut sim, true).await;
+ assert!(!sim.ctrl.hold_active(), "no hold for a stop");
+ wait(3_000).await;
+ assert_tail_aired_then_ended(&mut sim, last_live, left);
+}
+
+/// End now while the delay airs after a stop: the stream ends at once.
+#[tokio::test]
+async fn end_now_after_a_stop_cuts_the_rest_of_the_delay() {
+ let mut sim = Sim::new(false).await;
+ let (last_live, _) = obs_leaves_mid_delay(&mut sim, true).await;
+ wait(500).await;
+ assert!(sim.ctrl.end_hold_now(), "the delay was airing");
+ wait(1_000).await;
+
+ assert!(sim.commands(0).contains(&"deleteStream".into()), "it ended");
+ assert!(
+ sim.frames(0).last().is_some_and(|f| f.number < last_live),
+ "without airing the rest of the delay"
+ );
+ assert!(!sim.ctrl.end_hold_now(), "nothing left to end");
+}
+
+/// OBS comes back from a freeze on a P-frame, mid-GOP, with a delay on.
+/// The rejoin lands at least the delay back: the rebuild counts from OBS's
+/// first keyframe, not its first frame, and the rejoin takes the newest
+/// keyframe at least that old, not the nearest one.
+#[tokio::test]
+async fn a_freeze_rejoin_lands_at_least_the_delay_back() {
+ const DELAY_MS: u32 = 1_000;
+ let mut sim = Sim::new(true).await;
+ sim.ctrl.arm_delay(DELAY_MS);
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(2_000).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ // Frames 20..41: the freeze starts two frames into a GOP.
+ sim.obs_sends(2_200).await;
+ sim.obs_stalls(300).await;
+ sim.freeze_detected();
+ sim.obs_stalls(1_500).await;
+ let back = Instant::now();
+ sim.obs_sends(4_000).await;
+
+ let rejoined = sim
+ .frames(0)
+ .into_iter()
+ .find(|f| f.at > back && sim.produced[&f.number] > back)
+ .expect("the stream came back");
+ assert!(rejoined.keyframe, "rejoins at a keyframe");
+ let delayed_by = rejoined
+ .at
+ .duration_since(sim.produced[&rejoined.number])
+ .as_millis() as u32;
+ assert!(
+ delayed_by + 50 >= DELAY_MS,
+ "rejoined {delayed_by} ms behind, short of the {DELAY_MS} ms delay"
+ );
+ sim.assert_timestamps_monotonic(0);
+}
+
+/// The streamer ends a freeze hold, then OBS recovers on the same
+/// connection. A destination that starts afterwards joins OBS's new video
+/// at the delay; it used to seed on a keyframe from before the freeze and
+/// air the delay tail a second time.
+#[tokio::test]
+async fn a_destination_after_an_ended_freeze_never_replays_what_already_aired() {
+ const DELAY_MS: u32 = 1_500;
+ let mut sim = Sim::new(true).await;
+ sim.ctrl.arm_delay(DELAY_MS);
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(2_500).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ sim.obs_sends(2_000).await;
+ let last_before_freeze = sim.next_frame - 1;
+ sim.obs_stalls(300).await;
+ sim.freeze_detected();
+ assert!(sim.ctrl.end_hold_now(), "a hold was on");
+ sim.obs_stalls(2_000).await;
+ let back = Instant::now();
+ sim.obs_sends(500).await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(3_000).await;
+
+ let frames = sim.frames(1);
+ let first = frames.first().expect("the new session airs");
+ assert!(first.keyframe, "starts at a keyframe");
+ assert!(
+ frames.iter().all(|f| f.number > last_before_freeze),
+ "nothing from before the freeze airs again"
+ );
+ let delayed_by = first
+ .at
+ .duration_since(sim.produced[&first.number])
+ .as_millis() as u32;
+ assert!(
+ sim.produced[&first.number] > back && delayed_by + 50 >= DELAY_MS,
+ "joins OBS's new video {delayed_by} ms behind, at least the delay"
+ );
+ assert_strictly_increasing(&frames, "after an ended freeze");
+}
+
+/// OBS freezes for a moment and is back while the hold is still up. What it
+/// sent before the freeze is still the delay viewers are watching, so a
+/// destination that starts right after joins at the delay at once. The
+/// replay guard above used to apply here too, and it waited out the whole
+/// delay again before going live.
+#[tokio::test]
+async fn a_destination_after_a_short_freeze_joins_at_once() {
+ const DELAY_MS: u32 = 3_000;
+ let mut sim = Sim::new(true).await;
+ sim.ctrl.arm_delay(DELAY_MS);
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(3_500).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ sim.obs_sends(1_000).await;
+ sim.obs_stalls(300).await;
+ sim.freeze_detected();
+ assert!(sim.ctrl.hold_active(), "the freeze is held");
+ sim.obs_stalls(300).await;
+ sim.obs_sends(500).await;
+ assert!(!sim.ctrl.hold_active(), "OBS is back in time");
+ let joined = Instant::now();
+ sim.destination_connects("second").await;
+ sim.obs_sends(1_500).await;
+
+ let frames = sim.frames(1);
+ let first = frames.first().expect("the new destination went live");
+ assert!(
+ first.at.duration_since(joined) < Duration::from_millis(1_000),
+ "it waited {:?} to go live",
+ first.at.duration_since(joined)
+ );
+ let delayed_by = first
+ .at
+ .duration_since(sim.produced[&first.number])
+ .as_millis() as u32;
+ assert!(
+ delayed_by + 50 >= DELAY_MS,
+ "joined {delayed_by} ms behind, short of the {DELAY_MS} ms delay"
+ );
+}
+
+/// OBS freezes for longer than the delay and is back while the hold is still
+/// up: the whole tail aired and the screen took over. A destination that
+/// reconnects then must not seed on the last keyframe before the freeze
+/// (it would replay it, then sit silent across the freeze's gap).
+#[tokio::test]
+async fn a_destination_after_a_freeze_longer_than_the_delay_never_replays() {
+ const DELAY_MS: u32 = 1_500;
+ let mut sim = Sim::new(true).await;
+ sim.ctrl.arm_delay(DELAY_MS);
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(2_500).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ sim.obs_sends(2_000).await;
+ let last_before_freeze = sim.next_frame - 1;
+ sim.obs_stalls(300).await;
+ sim.freeze_detected();
+ sim.obs_stalls(2_500).await;
+ assert!(sim.ctrl.hold_active(), "the hold is still up");
+ sim.obs_sends(300).await;
+ sim.destination_connects("second").await;
+ sim.obs_sends(3_000).await;
+
+ let frames = sim.frames(1);
+ assert!(!frames.is_empty(), "the new destination went live");
+ assert!(
+ frames.iter().all(|f| f.number > last_before_freeze),
+ "nothing from before the freeze airs again"
+ );
+ assert_strictly_increasing(&frames, "after a long freeze");
+}
+
+/// A pump that ends (OBS gone, protection off) stops holding its place in
+/// the buffer. A stale place pinned the buffer's trim, so it grew to the
+/// whole disk allowance while the destination waited to reconnect.
+#[tokio::test]
+async fn a_pump_that_ends_releases_its_place_in_the_buffer() {
+ let mut sim = Sim::new(false).await;
+ sim.obs_connects().await;
+ sim.destination_runs("platform");
+ sim.obs_sends(1_500).await;
+ let dest = sim.ctrl.destination_state("platform");
+ assert_ne!(
+ dest.consumer_seq.load(Ordering::Relaxed),
+ u64::MAX,
+ "reading"
+ );
+ sim.obs_crashes();
+ wait(1_000).await;
+ assert_eq!(
+ dest.consumer_seq.load(Ordering::Relaxed),
+ u64::MAX,
+ "the ended pump released its place"
+ );
+}
diff --git a/src/controller/sim.rs b/src/controller/sim.rs
new file mode 100644
index 0000000..421e5ae
--- /dev/null
+++ b/src/controller/sim.rs
@@ -0,0 +1,324 @@
+//! The harness the pump simulations share. A real destination pump streams
+//! to an in-process platform (`sink::record`) that keeps everything it
+//! receives, while the test plays OBS by feeding tags to the controller in
+//! real time. Every video frame and its audio carry a number that never
+//! repeats across OBS sessions, so what viewers saw can be read back: a
+//! replay is a number seen twice, and the delay is when a frame arrived
+//! minus when OBS made it.
+
+use super::*;
+use crate::rtmp::client::{EgressClient, EgressUrl};
+use crate::sink::{record, Received};
+use std::path::PathBuf;
+use tokio::sync::mpsc;
+
+/// OBS's frame interval (10 fps) and default keyframe cadence (every 500 ms).
+pub(super) const FRAME_MS: u32 = 100;
+pub(super) const GOP_FRAMES: u32 = 5;
+/// The stream's resolution: small, so the screen encodes in no time.
+const WIDTH: usize = 64;
+const HEIGHT: usize = 36;
+
+pub(super) struct Sim {
+ pub(super) ctrl: Arc,
+ ring_path: PathBuf,
+ port: u16,
+ rx: mpsc::UnboundedReceiver,
+ got: Vec,
+ /// Frames get a number that never repeats, across OBS sessions, so a
+ /// replay shows up as a number seen twice.
+ pub(super) next_frame: u32,
+ pub(super) produced: HashMap,
+ /// This OBS session's wire clock.
+ wire_ts: u32,
+ /// When the wire clock read `clock_wire_ts`. Frames go out on that
+ /// schedule rather than one sleep after another, so the wire clock
+ /// never drifts from the wall clock the pump paces by.
+ clock_start: Instant,
+ clock_wire_ts: u32,
+ /// A keyframe every this many frames.
+ pub(super) gop_frames: u32,
+ /// How far each frame's audio is stamped behind its video. A real
+ /// muxer's audio trails the video it arrives after by a few ms.
+ pub(super) audio_lag_ms: u32,
+ pub(super) pumps: Vec>>,
+}
+
+impl Sim {
+ pub(super) async fn new(protected: bool) -> Sim {
+ Sim::with_ring_bytes(protected, 16 * 1024 * 1024).await
+ }
+
+ /// A simulation whose buffer holds only `ring_bytes`. A frame and its
+ /// audio take 24 bytes, so at 10 fps a buffer of 240 bytes a second
+ /// of stream fills up.
+ pub(super) async fn with_ring_bytes(protected: bool, ring_bytes: u64) -> Sim {
+ static UNIQ: AtomicU32 = AtomicU32::new(0);
+ let n = UNIQ.fetch_add(1, Ordering::Relaxed);
+ let ring_path =
+ std::env::temp_dir().join(format!("ic-pump-sim-{}-{n}.buf", std::process::id()));
+ let _ = std::fs::remove_file(&ring_path);
+ let ring = Arc::new(DiskRing::create(&ring_path, ring_bytes).unwrap());
+ let ctrl = Arc::new(Controller::new(ring, 0));
+ ctrl.update_crash_protection(crate::crash_protection::CrashProtection {
+ enabled: protected,
+ ..Default::default()
+ });
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let port = listener.local_addr().unwrap().port();
+ let (tx, rx) = mpsc::unbounded_channel();
+ tokio::spawn(record(listener, tx));
+ Sim {
+ ctrl,
+ ring_path,
+ port,
+ rx,
+ got: Vec::new(),
+ next_frame: 0,
+ produced: HashMap::new(),
+ wire_ts: 0,
+ clock_start: Instant::now(),
+ clock_wire_ts: 0,
+ gop_frames: GOP_FRAMES,
+ audio_lag_ms: 0,
+ pumps: Vec::new(),
+ }
+ }
+
+ /// OBS starts publishing: a new session, its clock from 0, the
+ /// sequence headers, and a keyframe first.
+ pub(super) async fn obs_connects(&mut self) {
+ self.obs_connects_at(0).await;
+ }
+
+ /// OBS starts publishing with its wire clock at `wire_ts`.
+ pub(super) async fn obs_connects_at(&mut self, wire_ts: u32) {
+ self.ctrl.begin_publish("key", "127.0.0.1").await.unwrap();
+ self.wire_ts = wire_ts;
+ self.clock_start = Instant::now();
+ self.clock_wire_ts = wire_ts;
+ let video = crate::slate::test_sequence_header(WIDTH, HEIGHT);
+ self.ctrl.on_tag(9, wire_ts, &video, false, true);
+ // AAC-LC, 48 kHz, stereo.
+ self.ctrl
+ .on_tag(8, wire_ts, &[0xAF, 0, 0x11, 0x90], false, true);
+ self.next_frame = self.next_frame.div_ceil(self.gop_frames) * self.gop_frames;
+ }
+
+ /// OBS sends `ms` of video and audio in real time.
+ pub(super) async fn obs_sends(&mut self, ms: u32) {
+ for _ in 0..ms / FRAME_MS {
+ let frame = self.next_frame;
+ let keyframe = frame.is_multiple_of(self.gop_frames);
+ self.ctrl.on_tag(
+ 9,
+ self.wire_ts,
+ &frame_tag(frame, keyframe),
+ keyframe,
+ false,
+ );
+ let audio_ts = self.wire_ts.wrapping_sub(self.audio_lag_ms);
+ self.ctrl
+ .on_tag(8, audio_ts, &audio_tag(frame), false, false);
+ self.produced.insert(frame, Instant::now());
+ self.next_frame += 1;
+ self.advance_clock(FRAME_MS).await;
+ }
+ }
+
+ /// OBS is connected but sends nothing for `ms` (its clock runs on).
+ pub(super) async fn obs_stalls(&mut self, ms: u32) {
+ self.advance_clock(ms).await;
+ }
+
+ async fn advance_clock(&mut self, ms: u32) {
+ self.wire_ts = self.wire_ts.wrapping_add(ms);
+ let elapsed = self.wire_ts.wrapping_sub(self.clock_wire_ts);
+ let due = self.clock_start + Duration::from_millis(u64::from(elapsed));
+ tokio::time::sleep_until(tokio::time::Instant::from_std(due)).await;
+ }
+
+ /// The encoder restarts mid-session with a new resolution: a new video
+ /// sequence header, and the next frame is a keyframe.
+ pub(super) fn obs_changes_resolution(&mut self, width: usize, height: usize) -> Vec {
+ let header = crate::slate::test_sequence_header(width, height);
+ self.ctrl.on_tag(9, self.wire_ts, &header, false, true);
+ self.next_frame = self.next_frame.div_ceil(self.gop_frames) * self.gop_frames;
+ header
+ }
+
+ pub(super) fn obs_crashes(&self) {
+ self.ctrl.mark_ingest_dead();
+ }
+
+ pub(super) fn obs_stops(&self) {
+ self.ctrl.note_unpublish();
+ self.ctrl.mark_ingest_dead();
+ }
+
+ /// What the supervisor does once OBS has sent no video for a while.
+ pub(super) fn freeze_detected(&self) {
+ let frozen_at = process_now_ms() + crate::crash_hold::FREEZE_AFTER.as_millis() as u64;
+ self.ctrl.check_ingest_freeze_at(frozen_at);
+ }
+
+ fn platform_url(&self) -> String {
+ format!("rtmp://127.0.0.1:{}/live/key", self.port)
+ }
+
+ /// A destination connects to the platform and its pump starts.
+ pub(super) async fn destination_connects(&mut self, id: &str) {
+ let dest = self.ctrl.destination_state(id);
+ dest.egress_alive.store(true, Ordering::Relaxed);
+ let url = EgressUrl::parse(&self.platform_url()).unwrap();
+ let sink = EgressClient::connect(&url)
+ .await
+ .unwrap()
+ .spawn_reader_drain();
+ let ctrl = self.ctrl.clone();
+ self.pumps.push(tokio::spawn(
+ async move { pump_dest(&ctrl, &dest, sink).await },
+ ));
+ }
+
+ /// A destination the supervisor runs: it connects once OBS is sending,
+ /// and reconnects (with its backoff) after every session ends.
+ pub(super) fn destination_runs(&mut self, id: &str) {
+ let dest = self.ctrl.destination_state(id);
+ let ctrl = self.ctrl.clone();
+ let url = self.platform_url();
+ let label = id.to_string();
+ self.pumps.push(tokio::spawn(async move {
+ run_egress(ctrl, label, url, dest).await
+ }));
+ }
+
+ /// Everything the platform has received so far.
+ pub(super) fn received(&mut self) -> &[Received] {
+ while let Ok(message) = self.rx.try_recv() {
+ self.got.push(message);
+ }
+ &self.got
+ }
+
+ /// OBS's frames as connection `conn` received them.
+ pub(super) fn frames(&mut self, conn: usize) -> Vec {
+ self.received()
+ .iter()
+ .enumerate()
+ .filter(|(_, m)| m.conn == conn && m.kind == 9)
+ .filter_map(|(index, m)| {
+ let number = frame_number(&m.payload)?;
+ Some(Frame {
+ number,
+ keyframe: m.payload[0] == 0x17,
+ ts: m.ts,
+ at: m.at,
+ index,
+ })
+ })
+ .collect()
+ }
+
+ /// The numbers of the frames whose audio connection `conn` received.
+ pub(super) fn audio_numbers(&mut self, conn: usize) -> Vec {
+ self.received()
+ .iter()
+ .filter(|m| m.conn == conn && m.kind == 8)
+ .filter_map(|m| audio_number(&m.payload))
+ .collect()
+ }
+
+ /// Reconnect-screen video tags connection `conn` received in `window`.
+ pub(super) fn screen_tags(&mut self, conn: usize, from: Instant, to: Instant) -> usize {
+ self.received()
+ .iter()
+ .filter(|m| m.conn == conn && m.kind == 9 && (from..to).contains(&m.at))
+ .filter(|m| m.payload.get(1) == Some(&1) && frame_number(&m.payload).is_none())
+ .count()
+ }
+
+ pub(super) fn commands(&mut self, conn: usize) -> Vec {
+ self.received()
+ .iter()
+ .filter(|m| m.conn == conn)
+ .filter_map(|m| m.command.clone())
+ .collect()
+ }
+
+ /// Video and audio timestamps each never go backwards.
+ pub(super) fn assert_timestamps_monotonic(&mut self, conn: usize) {
+ for kind in [8, 9] {
+ let stamps: Vec = self
+ .received()
+ .iter()
+ .filter(|m| m.conn == conn && m.kind == kind)
+ .map(|m| m.ts)
+ .collect();
+ let back = stamps.windows(2).position(|pair| pair[1] < pair[0]);
+ assert_eq!(
+ back, None,
+ "type {kind} timestamps went backwards: {stamps:?}"
+ );
+ }
+ }
+}
+
+impl Drop for Sim {
+ fn drop(&mut self) {
+ for pump in &self.pumps {
+ pump.abort();
+ }
+ let _ = std::fs::remove_file(&self.ring_path);
+ }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub(super) struct Frame {
+ pub(super) number: u32,
+ pub(super) keyframe: bool,
+ /// Output timestamp on the wire.
+ pub(super) ts: u32,
+ pub(super) at: Instant,
+ /// Where it sits in everything the platform received.
+ pub(super) index: usize,
+}
+
+/// A legacy AVC video tag carrying frame `number` in its NAL unit.
+fn frame_tag(number: u32, keyframe: bool) -> Vec {
+ let (flags, nal_type) = if keyframe { (0x17, 0x65) } else { (0x27, 0x41) };
+ let mut tag = vec![flags, 1, 0, 0, 0, 0, 0, 0, 7, nal_type, b'I', b'C'];
+ tag.extend_from_slice(&number.to_be_bytes());
+ tag
+}
+
+pub(super) fn frame_number(payload: &[u8]) -> Option {
+ if payload.len() != 16 || payload[1] != 1 || &payload[10..12] != b"IC" {
+ return None;
+ }
+ Some(u32::from_be_bytes(payload[12..16].try_into().ok()?))
+}
+
+/// A legacy AAC tag carrying the number of the frame it belongs to.
+fn audio_tag(number: u32) -> Vec {
+ let mut tag = vec![0xAF, 1, b'I', b'C'];
+ tag.extend_from_slice(&number.to_be_bytes());
+ tag
+}
+
+fn audio_number(payload: &[u8]) -> Option {
+ if payload.len() != 8 || payload[1] != 1 || &payload[2..4] != b"IC" {
+ return None;
+ }
+ Some(u32::from_be_bytes(payload[4..8].try_into().ok()?))
+}
+
+pub(super) async fn wait(ms: u32) {
+ tokio::time::sleep(Duration::from_millis(u64::from(ms))).await;
+}
+
+pub(super) fn assert_strictly_increasing(frames: &[Frame], what: &str) {
+ let numbers: Vec = frames.iter().map(|f| f.number).collect();
+ let replay = numbers.windows(2).position(|pair| pair[1] <= pair[0]);
+ assert_eq!(replay, None, "{what}: a frame was replayed: {numbers:?}");
+}
diff --git a/src/controller/user_sim.rs b/src/controller/user_sim.rs
new file mode 100644
index 0000000..42dafdb
--- /dev/null
+++ b/src/controller/user_sim.rs
@@ -0,0 +1,291 @@
+//! Streamer profiles played end to end (see `sim` for the harness): how
+//! different people end, restart and cut a delayed stream, and what their
+//! viewers see. The rules every profile keeps: nothing airs twice, nothing
+//! from a new OBS session airs undelayed, timestamps never go back, and a
+//! stream that ends sends its goodbye (`deleteStream`).
+
+use super::sim::{assert_strictly_increasing, wait, Frame, Sim};
+use std::collections::HashSet;
+use std::time::{Duration, Instant};
+
+const DELAY_MS: u32 = 2_000;
+
+/// OBS live with the delay on air, then it sends `ms` more.
+async fn delayed_stream(sim: &mut Sim, ms: u32) {
+ sim.ctrl.arm_delay(DELAY_MS);
+ sim.obs_connects().await;
+ sim.obs_sends(2_500).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ sim.obs_sends(ms).await;
+}
+
+fn frames_after(sim: &mut Sim, conn: usize, at: Instant) -> Vec {
+ sim.frames(conn).into_iter().filter(|f| f.at > at).collect()
+}
+
+fn ended(sim: &mut Sim, conn: usize) -> bool {
+ sim.commands(conn).contains(&"deleteStream".into())
+}
+
+/// How long after OBS made it a frame reached viewers.
+fn delay_of(sim: &Sim, frame: &Frame) -> Duration {
+ frame.at.duration_since(sim.produced[&frame.number])
+}
+
+/// A multistreamer (Twitch and YouTube), no crash protection, stops in
+/// OBS: both platforms air the whole delay, then both end.
+#[tokio::test]
+async fn multistreamer_stop_airs_the_delay_on_every_platform() {
+ let mut sim = Sim::new(false).await;
+ sim.ctrl.arm_delay(DELAY_MS);
+ sim.obs_connects().await;
+ sim.destination_connects("twitch").await;
+ sim.destination_connects("youtube").await;
+ sim.obs_sends(2_500).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ sim.obs_sends(1_500).await;
+ let last_live = sim.next_frame - 1;
+ sim.obs_stops();
+ let stopped = Instant::now();
+ wait(3_000).await;
+
+ for conn in 0..2 {
+ let tail = frames_after(&mut sim, conn, stopped);
+ assert_eq!(
+ tail.last().map(|f| f.number),
+ Some(last_live),
+ "platform {conn} aired the whole delay"
+ );
+ assert_strictly_increasing(&tail, "the tail");
+ assert!(ended(&mut sim, conn), "platform {conn} ended");
+ sim.assert_timestamps_monotonic(conn);
+ }
+ assert_eq!(sim.ctrl.tail_left(), None);
+}
+
+/// A streamer armed a delay but stopped before it went on air: nothing is
+/// held back, so the stream ends at once.
+#[tokio::test]
+async fn armed_but_not_on_yet_a_stop_ends_at_once() {
+ let mut sim = Sim::new(false).await;
+ sim.ctrl.arm_delay(5_000);
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(1_500).await;
+ sim.obs_stops();
+ wait(800).await;
+
+ assert!(ended(&mut sim, 0), "ended within the moment");
+ assert_eq!(sim.ctrl.tail_left(), None, "nothing to count down");
+}
+
+/// Stops, then starts OBS again a moment later while the delay of the
+/// first stream is still airing: the new stream reaches viewers delayed,
+/// never live.
+#[tokio::test]
+async fn restarting_mid_tail_never_airs_the_new_stream_live() {
+ restart_mid_tail(true).await;
+}
+
+/// Same, when OBS crashed instead (no crash protection) and came back.
+#[tokio::test]
+async fn coming_back_from_a_crash_mid_tail_never_airs_live() {
+ restart_mid_tail(false).await;
+}
+
+async fn restart_mid_tail(stopped: bool) {
+ let mut sim = Sim::new(false).await;
+ sim.destination_runs("platform");
+ delayed_stream(&mut sim, 1_000).await;
+ if stopped {
+ sim.obs_stops();
+ } else {
+ sim.obs_crashes();
+ }
+ wait(600).await;
+ sim.obs_connects().await;
+ let first_new = sim.next_frame;
+ sim.obs_sends(4_500).await;
+
+ assert!(ended(&mut sim, 0), "the first stream ended");
+ let new: Vec = sim
+ .frames(1)
+ .into_iter()
+ .filter(|f| f.number >= first_new)
+ .collect();
+ let first = new.first().expect("the destination came back");
+ assert!(first.keyframe, "starts on a keyframe");
+ let delayed_by = delay_of(&sim, first);
+ assert!(
+ delayed_by >= Duration::from_millis(1_500),
+ "the new stream airs delayed, not live ({delayed_by:?})"
+ );
+ sim.assert_timestamps_monotonic(1);
+}
+
+/// Stops, then panics and hits Cut to live while the delay still airs:
+/// the cut wins and the stream ends without the rest.
+#[tokio::test]
+async fn cut_to_live_during_the_tail_ends_the_stream() {
+ let mut sim = Sim::new(false).await;
+ sim.destination_connects("platform").await;
+ delayed_stream(&mut sim, 1_500).await;
+ let last_live = sim.next_frame - 1;
+ sim.obs_stops();
+ wait(300).await;
+ sim.ctrl.stop_delay();
+ wait(1_200).await;
+
+ assert!(ended(&mut sim, 0), "it ended");
+ assert!(
+ sim.frames(0).last().is_some_and(|f| f.number < last_live),
+ "without airing the rest"
+ );
+ sim.assert_timestamps_monotonic(0);
+}
+
+/// Marks "cut after this airs", then stops before the mark has aired:
+/// the mark airs, nothing airs twice, and the stream ends.
+#[tokio::test]
+async fn cut_after_this_airs_then_a_stop() {
+ let mut sim = Sim::new(false).await;
+ sim.destination_connects("platform").await;
+ delayed_stream(&mut sim, 1_000).await;
+ let mark = sim.next_frame - 1;
+ assert_eq!(
+ sim.ctrl.run_named_action("cut_after", DELAY_MS, "sim"),
+ None,
+ "the cut is scheduled"
+ );
+ sim.obs_sends(500).await;
+ sim.obs_stops();
+ let stopped = Instant::now();
+ wait(3_000).await;
+
+ let frames = sim.frames(0);
+ assert!(
+ frames.iter().any(|f| f.number == mark),
+ "the marked moment aired"
+ );
+ assert_strictly_increasing(&frames_after(&mut sim, 0, stopped), "after the stop");
+ assert!(ended(&mut sim, 0), "it ended");
+ sim.assert_timestamps_monotonic(0);
+}
+
+/// Stops, hits End now, then goes live again: the new stream starts
+/// delayed like any other.
+#[tokio::test]
+async fn end_now_then_a_new_stream_starts_delayed() {
+ let mut sim = Sim::new(false).await;
+ sim.destination_runs("platform");
+ delayed_stream(&mut sim, 1_000).await;
+ sim.obs_stops();
+ wait(300).await;
+ assert!(sim.ctrl.end_hold_now(), "the delay was airing");
+ wait(500).await;
+ assert!(ended(&mut sim, 0), "ended at once");
+ sim.obs_connects().await;
+ let first_new = sim.next_frame;
+ sim.obs_sends(4_500).await;
+
+ let first = sim
+ .frames(1)
+ .into_iter()
+ .find(|f| f.number >= first_new)
+ .expect("the new stream aired");
+ let delayed_by = delay_of(&sim, &first);
+ assert!(
+ delayed_by >= Duration::from_millis(1_500),
+ "delayed, not live ({delayed_by:?})"
+ );
+}
+
+/// The dashboard's countdown while the delay airs goes down, then away.
+#[tokio::test]
+async fn the_countdown_goes_down_then_away() {
+ let mut sim = Sim::new(false).await;
+ sim.destination_connects("platform").await;
+ delayed_stream(&mut sim, 1_000).await;
+ sim.obs_stops();
+ wait(300).await;
+ let early = sim.ctrl.tail_left().expect("counting down");
+ wait(800).await;
+ let later = sim.ctrl.tail_left().expect("still counting down");
+ assert!(later < early, "{later:?} after {early:?}");
+ assert!(
+ early <= Duration::from_millis(u64::from(DELAY_MS) + 600),
+ "never more than the delay ({early:?})"
+ );
+ wait(2_000).await;
+ assert_eq!(sim.ctrl.tail_left(), None, "gone once it aired");
+}
+
+/// A streamer who stops and starts three times in a row (scene setup,
+/// a crash of their own, a restart): every stream airs delayed and whole,
+/// and nothing airs twice across all of them.
+#[tokio::test]
+async fn three_stops_and_starts_in_a_row() {
+ const SHORT_DELAY_MS: u32 = 1_000;
+ let mut sim = Sim::new(false).await;
+ sim.destination_runs("platform");
+ sim.ctrl.arm_delay(SHORT_DELAY_MS);
+ sim.obs_connects().await;
+ sim.obs_sends(1_500).await;
+ sim.ctrl
+ .activate_delay()
+ .expect("the buffer holds the delay");
+ // Turning the delay on replays its own stretch once, by design: count
+ // replays from then on.
+ let activated = Instant::now();
+ let mut last_lives = Vec::new();
+ for round in 0..3 {
+ sim.obs_sends(1_500).await;
+ last_lives.push(sim.next_frame - 1);
+ sim.obs_stops();
+ wait(1_800).await;
+ if round < 2 {
+ sim.obs_connects().await;
+ }
+ }
+ wait(500).await;
+
+ let mut seen = HashSet::new();
+ for (conn, last_live) in last_lives.into_iter().enumerate() {
+ let frames = sim.frames(conn);
+ assert!(!frames.is_empty(), "stream {conn} aired");
+ assert_eq!(
+ frames.last().map(|f| f.number),
+ Some(last_live),
+ "stream {conn} aired its whole delay"
+ );
+ for f in frames.iter().filter(|f| f.at > activated) {
+ assert!(seen.insert(f.number), "frame {} aired twice", f.number);
+ }
+ sim.assert_timestamps_monotonic(conn);
+ }
+}
+
+/// Asked for a longer delay than the buffer holds, then stops: what the
+/// buffer holds airs, nothing twice, then the end.
+#[tokio::test]
+async fn a_delay_longer_than_the_buffer_then_a_stop() {
+ // About 3 s of stream fits (see `Sim::with_ring_bytes`).
+ let mut sim = Sim::with_ring_bytes(false, 720).await;
+ sim.ctrl.arm_delay(6_000);
+ sim.obs_connects().await;
+ sim.destination_connects("platform").await;
+ sim.obs_sends(4_000).await;
+ let _ = sim.ctrl.activate_delay();
+ sim.obs_sends(2_000).await;
+ sim.obs_stops();
+ let stopped = Instant::now();
+ wait(5_000).await;
+
+ assert_strictly_increasing(&frames_after(&mut sim, 0, stopped), "after the stop");
+ assert!(ended(&mut sim, 0), "it ended");
+ sim.assert_timestamps_monotonic(0);
+}
diff --git a/src/crash_hold.rs b/src/crash_hold.rs
new file mode 100644
index 0000000..346b13b
--- /dev/null
+++ b/src/crash_hold.rs
@@ -0,0 +1,935 @@
+//! Crash protection at runtime: the hold window after OBS drops, and the
+//! reconnect screen played into each destination while it lasts.
+//!
+//! When OBS disconnects without saying goodbye (FCUnpublish /
+//! deleteStream), or stops sending video for `FREEZE_AFTER` while still
+//! connected, and crash protection is on, the controller opens a hold.
+//! Each destination's pump first plays out any buffered delay, then keeps
+//! every video track it receives alive until OBS publishes again (the
+//! pump's reconnect path takes over from there), the hold time runs out,
+//! or the streamer ends it early:
+//!
+//! - H.264 tracks loop the reconnect screen at the track's own resolution,
+//! framed exactly like the stream's own tags (legacy, Enhanced RTMP, or
+//! an Enhanced Broadcasting multitrack tag with its track id).
+//! - Other tracks (HEVC, AV1) hold their last keyframe, re-sent once a
+//! second, because the screen's encoder only speaks H.264.
+//! - Every AAC track gets digital silence in its own format.
+//!
+//! Timestamps continue from the last real frame, so the platform sees one
+//! unbroken stream.
+
+use crate::controller::{Controller, DestinationState};
+use crate::crash_protection::CrashProtection;
+use crate::h264::{AudioEgress, VideoEgress};
+use crate::rtmp::client::EgressSink;
+use crate::slate::{SlateLoop, StreamShape};
+use std::borrow::Cow;
+use std::collections::{HashMap, HashSet};
+use std::io;
+use std::sync::atomic::Ordering;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+/// Frame rate of the reconnect screen. It is a still-ish loop, so 30 is
+/// plenty and halves the encode work of 60.
+pub const SLATE_FPS: u32 = 30;
+
+/// No video for this long from a connected OBS counts as a freeze.
+pub const FREEZE_AFTER: Duration = Duration::from_secs(3);
+
+/// A held keyframe is re-sent once a second (every this many screen
+/// frames): a still picture needs no more, and keyframes are large.
+const HELD_FRAME_EVERY: u64 = SLATE_FPS as u64;
+
+/// OBS is back but its video hasn't rebuilt the armed delay yet: stop
+/// waiting this long past the delay itself (a buffer too small to hold
+/// it, or a delay raised meanwhile) and rejoin where the video is.
+const REBUILD_GRACE: Duration = Duration::from_secs(5);
+
+/// Used when a destination's resolution can't be read from its SPS.
+const FALLBACK_SHAPE: (usize, usize) = (1280, 720);
+
+// FLV and Enhanced RTMP tag header fields.
+const FLV_VIDEO_AVC: u8 = 7;
+const FLV_AUDIO_AAC: u8 = 10;
+const FLV_AUDIO_EX: u8 = 9;
+const AVC_NALU: u8 = 1;
+const AAC_RAW: u8 = 1;
+const EX_VIDEO: u8 = 0x80;
+const EX_SEQUENCE_START: u8 = 0;
+const EX_CODED_FRAMES: u8 = 1;
+const EX_VIDEO_MULTITRACK: u8 = 6;
+const EX_AUDIO_MULTITRACK: u8 = 5;
+const ONE_TRACK: u8 = 0;
+const FRAME_KEY: u8 = 1;
+const FRAME_INTER: u8 = 2;
+const FOURCC_AVC1: [u8; 4] = *b"avc1";
+const FOURCC_MP4A: [u8; 4] = *b"mp4a";
+
+/// Raw AAC-LC frames that decode to digital silence (verified with
+/// ffmpeg: no decoder errors, every sample 0).
+const SILENT_AAC_STEREO: &[u8] = &[0x21, 0x00, 0x49, 0x90, 0x02, 0x19, 0x00, 0x23, 0x80];
+const SILENT_AAC_MONO: &[u8] = &[0x01, 0x40, 0x20, 0x07];
+const AAC_OBJECT_LC: u8 = 2;
+const AAC_FRAME_SAMPLES: f64 = 1024.0;
+/// ISO 14496-3 sampling frequency index table.
+const AAC_SAMPLE_RATES: [u32; 13] = [
+ 96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350,
+];
+
+/// A hold on air, as the dashboard, dock and tray show it.
+#[derive(Debug, Clone, Copy)]
+pub struct HoldStatus {
+ pub reason: HoldReason,
+ pub remaining: Duration,
+ pub total: Duration,
+}
+
+/// An open hold: why, when it started, and when it gives up.
+#[derive(Debug, Clone, Copy)]
+pub struct Hold {
+ pub reason: HoldReason,
+ pub started: Instant,
+ pub deadline: Instant,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum HoldReason {
+ /// OBS's connection dropped without a goodbye.
+ Crash,
+ /// OBS is still connected but stopped sending video.
+ Freeze,
+ /// OBS stopped the stream on purpose, and crash protection holds every
+ /// disconnect.
+ Stopped,
+}
+
+/// Where the hold stands, as the pumps see it.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum HoldState {
+ /// Destinations are on the reconnect screen.
+ Holding,
+ /// The last hold closed because OBS is sending again.
+ Resumed,
+ /// The last hold closed without OBS: it ran out or was ended.
+ Ended,
+}
+
+/// Why playback of the reconnect screen stopped.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum HoldExit {
+ /// OBS is sending again and its video since then covers the armed
+ /// delay (at once without one): the pump rejoins at a keyframe at or
+ /// after `rejoin_from_seq`, never on video from before the hold.
+ Resumed { rejoin_from_seq: u64 },
+ /// The hold ended (time ran out, the streamer ended it, or the
+ /// destination can't be covered): end the platform session cleanly.
+ Ended,
+}
+
+pub struct HoldOutcome {
+ pub exit: HoldExit,
+ /// Last output timestamp sent, so the pump's timeline stays monotonic.
+ pub last_ts: u32,
+}
+
+/// How a destination already receives a track, so the hold's tags are
+/// framed exactly like the stream's own.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum Framing {
+ /// Legacy FLV: `0x17` / `0x27` AVC video, `0xAF`-style AAC audio.
+ Legacy,
+ /// Enhanced RTMP, one track: header byte, then the FourCC.
+ Enhanced,
+ /// Enhanced RTMP multitrack, OneTrack layout (Enhanced Broadcasting):
+ /// header byte, layout byte, FourCC, then this track id.
+ OneTrack(u8),
+}
+
+/// What one destination gets during a hold: a picture for every video
+/// track it receives, and silence for every AAC track.
+struct HoldPlan {
+ video: Vec,
+ audio: Vec,
+}
+
+enum VideoSource {
+ /// The reconnect screen, encoded at the track's own resolution.
+ Screen {
+ shape: StreamShape,
+ framing: Framing,
+ },
+ /// The track's last keyframe as the destination receives it.
+ HeldFrame(Arc<[u8]>),
+}
+
+/// A `VideoSource` ready to play, as the tags the destination gets.
+enum VideoTrack {
+ /// The loop wrapped once in the track's framing, not once per frame.
+ Screen {
+ shape: StreamShape,
+ /// The two IDR variants, then the P frames, in loop order.
+ keyframes: [Vec; 2],
+ deltas: Vec>,
+ },
+ HeldFrame(Arc<[u8]>),
+}
+
+impl VideoTrack {
+ fn screen(slate: &SlateLoop, framing: Framing) -> Self {
+ VideoTrack::Screen {
+ shape: slate.shape,
+ keyframes: [0, 1].map(|replay| video_tag(framing, true, &slate.keyframes[replay])),
+ deltas: slate
+ .deltas
+ .iter()
+ .map(|sample| video_tag(framing, false, sample))
+ .collect(),
+ }
+ }
+}
+
+/// Silent AAC matching one of the stream's own AudioSpecificConfigs, so
+/// the destination's decoder needs no reconfiguration.
+struct SilentAudio {
+ framing: Framing,
+ /// First byte of the stream's own legacy audio tags (format, rate,
+ /// size, channels), reused so the silent tags look like the real ones.
+ /// Unused for Enhanced RTMP framings.
+ legacy_flags: u8,
+ frame: &'static [u8],
+ frame_ms: f64,
+}
+
+/// Whether this destination can be kept live on the reconnect screen.
+/// When it can't, it ends like it did before crash protection.
+pub fn covers(ctrl: &Controller, dest: &DestinationState) -> bool {
+ plan_for(ctrl, dest).is_ok()
+}
+
+/// Keep a reconnect loop encoded for each of `dest`'s video tracks.
+pub fn prebuild_for(ctrl: &Controller, dest: &DestinationState, settings: &CrashProtection) {
+ let Some(egress) = dest.video_egress() else {
+ return;
+ };
+ for (_, header) in video_headers_for(ctrl, egress) {
+ if avc_framing(&header).is_some() {
+ ctrl.slate_cache.prebuild(settings, screen_shape(&header));
+ }
+ }
+}
+
+/// Whether the hold plays this video tag's track by holding its last
+/// keyframe: an Enhanced RTMP keyframe that isn't H.264 in a framing the
+/// reconnect screen matches (HEVC, AV1, bundled multitrack layouts). The
+/// controller keeps the latest one per track.
+pub fn is_held_keyframe(tag: &[u8]) -> bool {
+ let Some(&first) = tag.first() else {
+ return false;
+ };
+ if first & EX_VIDEO == 0 || !crate::h264::classify_video_tag(tag).is_idr {
+ return false;
+ }
+ let fourcc = match first & 0x0F {
+ EX_VIDEO_MULTITRACK if tag.get(1).map(|layout| layout >> 4) == Some(ONE_TRACK) => {
+ tag.get(2..6)
+ }
+ // A bundle of several tracks per tag: only the whole tag can be held.
+ EX_VIDEO_MULTITRACK => None,
+ _ => tag.get(1..5),
+ };
+ fourcc != Some(&FOURCC_AVC1[..])
+}
+
+/// The picture and silence for each track of `dest`, or why it can't be
+/// covered.
+fn plan_for(ctrl: &Controller, dest: &DestinationState) -> Result {
+ let Some(egress) = dest.video_egress() else {
+ return Err("its vertical canvas isn't known".into());
+ };
+ let headers = video_headers_for(ctrl, egress);
+ if headers.is_empty() {
+ return Err("no video has arrived yet".into());
+ }
+ let video = headers
+ .iter()
+ .map(|(track, header)| video_source(ctrl, egress, *track, header))
+ .collect::, _>>()?;
+ Ok(HoldPlan {
+ video,
+ audio: silent_audio_for(ctrl, dest.audio_egress()),
+ })
+}
+
+/// The video sequence headers `egress` sends, keyed by ingest track id,
+/// as the destination receives them (the same pick as the pump's
+/// `send_sequence_headers`).
+fn video_headers_for(ctrl: &Controller, egress: VideoEgress) -> Vec<(u8, Vec)> {
+ let headers = ctrl.ring.video_seq_headers.lock();
+ let target = match egress {
+ VideoEgress::Passthrough => {
+ return headers.iter().map(|(t, h)| (*t, h.clone())).collect();
+ }
+ VideoEgress::Track(target) => target,
+ };
+ let pick = headers
+ .get_key_value(&target)
+ .or_else(|| (target == 0).then(|| headers.iter().next()).flatten());
+ pick.map(|(track, header)| {
+ let sent = crate::h264::select_video_bytes(header, egress)
+ .map(Cow::into_owned)
+ .unwrap_or_else(|| header.clone());
+ (*track, sent)
+ })
+ .into_iter()
+ .collect()
+}
+
+/// The reconnect screen for an H.264 track, or its last keyframe for
+/// anything else.
+fn video_source(
+ ctrl: &Controller,
+ egress: VideoEgress,
+ track: u8,
+ header: &[u8],
+) -> Result {
+ if let Some(framing) = avc_framing(header) {
+ let shape = screen_shape(header);
+ return Ok(VideoSource::Screen { shape, framing });
+ }
+ let codec = crate::h264::seq_header_codec(header).label();
+ let Some(keyframe) = ctrl.held_keyframe(track) else {
+ return Err(format!(
+ "its {codec} video hasn't sent a keyframe to hold yet"
+ ));
+ };
+ let sent: Arc<[u8]> = match crate::h264::select_video_bytes(&keyframe, egress) {
+ Some(Cow::Borrowed(_)) => keyframe.clone(),
+ Some(Cow::Owned(flattened)) => flattened.into(),
+ None => return Err(format!("its {codec} keyframe can't be sent here")),
+ };
+ Ok(VideoSource::HeldFrame(sent))
+}
+
+/// The screen's shape for an H.264 track: its resolution, rounded down
+/// to even sides, at the screen's frame rate.
+fn screen_shape(header: &[u8]) -> StreamShape {
+ let (width, height) = crate::h264::sps_dimensions(header)
+ .map(|(w, h)| (w as usize & !1, h as usize & !1))
+ .unwrap_or(FALLBACK_SHAPE);
+ StreamShape {
+ width,
+ height,
+ fps: SLATE_FPS,
+ }
+}
+
+/// The framing of an H.264 sequence header the reconnect screen can
+/// match, or `None` for other codecs and bundled multitrack layouts.
+fn avc_framing(header: &[u8]) -> Option {
+ let &first = header.first()?;
+ if first & EX_VIDEO == 0 {
+ let is_avc_config = first & 0x0F == FLV_VIDEO_AVC && header.get(1) == Some(&0);
+ return is_avc_config.then_some(Framing::Legacy);
+ }
+ let (framing, fourcc, _) = enhanced_sequence_start(header, EX_VIDEO_MULTITRACK)?;
+ (fourcc == FOURCC_AVC1).then_some(framing)
+}
+
+/// Parse an Enhanced RTMP sequence-start tag (video or audio): its
+/// framing, FourCC and codec config. `multitrack` is the packet type that
+/// marks a multitrack tag for this media kind. Only the single-track and
+/// OneTrack layouts carry one track's config.
+fn enhanced_sequence_start(tag: &[u8], multitrack: u8) -> Option<(Framing, [u8; 4], &[u8])> {
+ let packet_type = tag.first()? & 0x0F;
+ if packet_type == EX_SEQUENCE_START {
+ let fourcc = tag.get(1..5)?.try_into().ok()?;
+ return Some((Framing::Enhanced, fourcc, tag.get(5..)?));
+ }
+ let layout = *tag.get(1)?;
+ if packet_type != multitrack || layout >> 4 != ONE_TRACK || layout & 0x0F != EX_SEQUENCE_START {
+ return None;
+ }
+ let fourcc = tag.get(2..6)?.try_into().ok()?;
+ Some((Framing::OneTrack(*tag.get(6)?), fourcc, tag.get(7..)?))
+}
+
+/// Silence for each AAC track this destination already receives. Other
+/// audio (Opus, HE-AAC, surround) gets none; video still plays.
+fn silent_audio_for(ctrl: &Controller, egress: AudioEgress) -> Vec {
+ let headers = ctrl.ring.audio_seq_headers.lock();
+ let target = match egress {
+ AudioEgress::Passthrough => {
+ return headers
+ .values()
+ .filter_map(|h| silent_audio_from_header(h))
+ .collect();
+ }
+ AudioEgress::Track(target) => target,
+ };
+ let Some(header) = headers
+ .get(&target)
+ .or_else(|| headers.get(&0))
+ .or_else(|| headers.values().next())
+ else {
+ return Vec::new();
+ };
+ crate::h264::select_audio_bytes(header, egress, ctrl.audio_target_on_wire(egress))
+ .and_then(|tag| silent_audio_from_header(&tag))
+ .into_iter()
+ .collect()
+}
+
+/// Parse an AAC sequence-header tag (legacy `[flags, 0x00, ASC...]`, or
+/// Enhanced RTMP `mp4a`) into matching silence.
+fn silent_audio_from_header(header: &[u8]) -> Option {
+ let &first = header.first()?;
+ let (framing, legacy_flags, asc) = match first >> 4 {
+ FLV_AUDIO_AAC if header.get(1) == Some(&0) => (Framing::Legacy, first, header.get(2..)?),
+ FLV_AUDIO_EX => {
+ let (framing, fourcc, asc) = enhanced_sequence_start(header, EX_AUDIO_MULTITRACK)?;
+ if fourcc != FOURCC_MP4A {
+ return None;
+ }
+ (framing, 0, asc)
+ }
+ _ => return None,
+ };
+ let asc = asc.get(..2)?;
+ let object_type = asc[0] >> 3;
+ let rate_index = ((asc[0] & 0x07) << 1) | (asc[1] >> 7);
+ let channels = (asc[1] >> 3) & 0x0F;
+ if object_type != AAC_OBJECT_LC {
+ return None;
+ }
+ let sample_rate = *AAC_SAMPLE_RATES.get(rate_index as usize)?;
+ let frame = match channels {
+ 1 => SILENT_AAC_MONO,
+ 2 => SILENT_AAC_STEREO,
+ _ => return None,
+ };
+ Some(SilentAudio {
+ framing,
+ legacy_flags,
+ frame,
+ frame_ms: AAC_FRAME_SAMPLES * 1000.0 / sample_rate as f64,
+ })
+}
+
+/// Play the reconnect screen into `dest` until the hold ends or OBS comes
+/// back. `after_ts` is the last output timestamp already sent.
+pub async fn play(
+ ctrl: &Arc,
+ dest: &Arc,
+ sink: &mut EgressSink,
+ after_ts: u32,
+) -> io::Result {
+ let ended = HoldOutcome {
+ exit: HoldExit::Ended,
+ last_ts: after_ts,
+ };
+ let plan = match plan_for(ctrl, dest) {
+ Ok(plan) => plan,
+ Err(why) => {
+ ctrl.log(format!(
+ "[{}] crash protection can't cover this destination: {why}",
+ dest.id
+ ));
+ return Ok(ended);
+ }
+ };
+ let settings = ctrl.crash_protection();
+ let mut video = Vec::with_capacity(plan.video.len());
+ for source in plan.video {
+ video.push(match source {
+ VideoSource::Screen { shape, framing } => {
+ let Some(slate) = ctrl.slate_cache.get(&settings, shape).await else {
+ ctrl.log(format!(
+ "[{}] crash protection couldn't build the reconnect screen",
+ dest.id
+ ));
+ return Ok(ended);
+ };
+ VideoTrack::screen(&slate, framing)
+ }
+ VideoSource::HeldFrame(tag) => VideoTrack::HeldFrame(tag),
+ });
+ }
+ ctrl.log(format!(
+ "[{}] crash protection: reconnect screen on air ({})",
+ dest.id,
+ describe(&video)
+ ));
+ // No sequence header: the loop's keyframes carry their own SPS/PPS
+ // in-band under ids the stream doesn't use (see slate::encoder), so the
+ // destination's decoder keeps the stream's config for the way back.
+ let base = after_ts.wrapping_add(1);
+ stream_loop(ctrl, dest, sink, &video, &plan.audio, base).await
+}
+
+/// "1920x1080, 1280x720, last keyframe held" for the log.
+fn describe(video: &[VideoTrack]) -> String {
+ video
+ .iter()
+ .map(|track| match track {
+ VideoTrack::Screen { shape, .. } => format!("{}x{}", shape.width, shape.height),
+ VideoTrack::HeldFrame(_) => "last keyframe held".to_string(),
+ })
+ .collect::>()
+ .join(", ")
+}
+
+/// Send screen frames and silent audio, interleaved by timestamp and paced
+/// in real time, until OBS returns or the hold ends.
+async fn stream_loop(
+ ctrl: &Arc,
+ dest: &Arc,
+ sink: &mut EgressSink,
+ video: &[VideoTrack],
+ audio: &[SilentAudio],
+ base: u32,
+) -> io::Result {
+ let frame_ms = 1000.0 / SLATE_FPS as f64;
+ let start = tokio::time::Instant::now();
+ let mut video_index = 0u64;
+ let mut audio_index = vec![0u64; audio.len()];
+ let mut last_ts = base;
+ let mut rebuild: Option = None;
+ loop {
+ let exit = match ctrl.hold_state() {
+ _ if dest.shutdown_requested.load(Ordering::Relaxed) => Some(HoldExit::Ended),
+ HoldState::Holding => {
+ // OBS may drop again while the delay rebuilds: start over.
+ rebuild = None;
+ None
+ }
+ // Back, then gone again without a new hold (a deliberate stop).
+ HoldState::Resumed if !ctrl.ingest_alive() => Some(HoldExit::Ended),
+ // With a delay armed the screen stays up until OBS's new video
+ // covers it, so viewers never see live.
+ HoldState::Resumed => {
+ let delay = u64::from(ctrl.target_delay_ms());
+ delay_rebuilt(ctrl, &mut rebuild, delay)
+ .map(|rejoin_from_seq| HoldExit::Resumed { rejoin_from_seq })
+ }
+ HoldState::Ended => Some(HoldExit::Ended),
+ };
+ if let Some(exit) = exit {
+ return Ok(HoldOutcome { exit, last_ts });
+ }
+ sink.drain_pings().await?;
+
+ let video_at = video_index as f64 * frame_ms;
+ // The audio track due soonest, when it is due before the next
+ // video frame.
+ let audio_due = audio
+ .iter()
+ .zip(&audio_index)
+ .map(|(track, sent)| *sent as f64 * track.frame_ms)
+ .enumerate()
+ .min_by(|a, b| a.1.total_cmp(&b.1))
+ .filter(|(_, at)| *at < video_at);
+ let at = audio_due.map_or(video_at, |(_, at)| at);
+ tokio::time::sleep_until(start + Duration::from_micros((at * 1000.0) as u64)).await;
+ let ts = base.wrapping_add(at.round() as u32);
+
+ if let Some((track, _)) = audio_due {
+ sink.send_audio(ts, &audio_tag(&audio[track])).await?;
+ audio_index[track] += 1;
+ } else {
+ for track in video {
+ let Some(tag) = video_frame(track, video_index) else {
+ continue;
+ };
+ sink.send_video(ts, tag).await?;
+ dest.tags_sent.fetch_add(1, Ordering::Relaxed);
+ dest.bytes_sent
+ .fetch_add(tag.len() as u64, Ordering::Relaxed);
+ dest.note_outbound_bytes(tag.len());
+ }
+ video_index += 1;
+ }
+ last_ts = ts;
+ }
+}
+
+/// Where OBS's video since it came back starts, tracked while the screen
+/// waits for it to rebuild the armed delay.
+pub(crate) struct Rebuild {
+ /// The newest ring seq when OBS came back (None: the ring was empty,
+ /// as it is for a new publisher). Its new video starts after it.
+ after_seq: Option,
+ since: Instant,
+}
+
+/// The ring seq OBS's new video starts at, once that video spans
+/// `delay_ms` from its first keyframe (so at that keyframe for no delay;
+/// or once the wait has run `REBUILD_GRACE` past the delay). Measured from
+/// the keyframe, not the first tag: a frozen encoder often resumes on a
+/// P-frame, and counting from it let the rejoin land short of the delay, or
+/// wait on air with nothing to send until the keyframe came.
+pub(crate) fn delay_rebuilt(
+ ctrl: &Controller,
+ rebuild: &mut Option,
+ delay_ms: u64,
+) -> Option {
+ let rebuild = rebuild.get_or_insert_with(|| Rebuild {
+ after_seq: ctrl.resumed_after_seq(),
+ since: Instant::now(),
+ });
+ let first_seq = match rebuild.after_seq {
+ Some(seq) => seq + 1,
+ None => ctrl.ring.front_seq()?,
+ };
+ if rebuild.since.elapsed() >= Duration::from_millis(delay_ms) + REBUILD_GRACE {
+ return Some(first_seq);
+ }
+ let first_keyframe = ctrl.ring.oldest_idr_at_or_after(first_seq)?;
+ let spanned = ctrl.ring.latest_ts()?.saturating_sub(first_keyframe.ts_ms);
+ (spanned >= delay_ms).then_some(first_seq)
+}
+
+/// The tag `track` sends for screen frame `index`, if any: every frame of
+/// the loop, or the held keyframe once a second.
+fn video_frame(track: &VideoTrack, index: u64) -> Option<&[u8]> {
+ match track {
+ VideoTrack::Screen {
+ keyframes, deltas, ..
+ } => {
+ let frames = 1 + deltas.len() as u64;
+ let position = (index % frames) as usize;
+ if position == 0 {
+ // Alternate the two IDR variants: back-to-back IDRs must not
+ // share an idr_pic_id.
+ let replay = (index / frames) as usize % 2;
+ Some(&keyframes[replay])
+ } else {
+ Some(&deltas[position - 1])
+ }
+ }
+ VideoTrack::HeldFrame(tag) => index.is_multiple_of(HELD_FRAME_EVERY).then_some(&tag[..]),
+ }
+}
+
+/// An H.264 coded-frame tag in `framing`, zero composition time.
+fn video_tag(framing: Framing, keyframe: bool, sample: &[u8]) -> Vec {
+ let frame_type = if keyframe { FRAME_KEY } else { FRAME_INTER };
+ let mut tag = Vec::with_capacity(sample.len() + 12);
+ match framing {
+ Framing::Legacy => tag.extend_from_slice(&[frame_type << 4 | FLV_VIDEO_AVC, AVC_NALU]),
+ Framing::Enhanced => {
+ tag.push(EX_VIDEO | frame_type << 4 | EX_CODED_FRAMES);
+ tag.extend_from_slice(&FOURCC_AVC1);
+ }
+ Framing::OneTrack(track) => {
+ tag.push(EX_VIDEO | frame_type << 4 | EX_VIDEO_MULTITRACK);
+ tag.push(ONE_TRACK << 4 | EX_CODED_FRAMES);
+ tag.extend_from_slice(&FOURCC_AVC1);
+ tag.push(track);
+ }
+ }
+ tag.extend_from_slice(&[0, 0, 0]);
+ tag.extend_from_slice(sample);
+ tag
+}
+
+/// One silent AAC frame in the track's own framing.
+fn audio_tag(audio: &SilentAudio) -> Vec {
+ let mut tag = Vec::with_capacity(audio.frame.len() + 7);
+ match audio.framing {
+ Framing::Legacy => tag.extend_from_slice(&[audio.legacy_flags, AAC_RAW]),
+ Framing::Enhanced => {
+ tag.push(FLV_AUDIO_EX << 4 | EX_CODED_FRAMES);
+ tag.extend_from_slice(&FOURCC_MP4A);
+ }
+ Framing::OneTrack(track) => {
+ tag.push(FLV_AUDIO_EX << 4 | EX_AUDIO_MULTITRACK);
+ tag.push(ONE_TRACK << 4 | EX_CODED_FRAMES);
+ tag.extend_from_slice(&FOURCC_MP4A);
+ tag.push(track);
+ }
+ }
+ tag.extend_from_slice(audio.frame);
+ tag
+}
+
+/// Encoded reconnect-screen loops, one per destination shape, rebuilt when
+/// the screen's look changes. Encoding a 1080p loop takes a few hundred
+/// ms of one core, so loops are built ahead of time while OBS is live and
+/// a hold usually starts with one ready.
+#[derive(Default)]
+pub struct SlateCache {
+ /// Per (width, height): the loop and the settings it was built with.
+ ready: crate::sync::Mutex)>>,
+ building: crate::sync::Mutex>,
+}
+
+/// A slate loop's (width, height).
+type SlateKey = (usize, usize);
+
+impl SlateCache {
+ fn cached(&self, settings: &CrashProtection, shape: StreamShape) -> Option> {
+ self.ready
+ .lock()
+ .get(&(shape.width, shape.height))
+ .filter(|(built_with, _)| built_with.same_screen(settings))
+ .map(|(_, slate)| slate.clone())
+ }
+
+ /// The loop for `shape`, building it now if it isn't ready. When a
+ /// build of it is already running (a prebuild, or another destination
+ /// at the same resolution) this waits for that one rather than encode
+ /// the same loop twice at the moment OBS drops.
+ pub async fn get(
+ self: &Arc,
+ settings: &CrashProtection,
+ shape: StreamShape,
+ ) -> Option> {
+ const BUILD_POLL: Duration = Duration::from_millis(50);
+ let key = (shape.width, shape.height);
+ loop {
+ if let Some(slate) = self.cached(settings, shape) {
+ return Some(slate);
+ }
+ if self.building.lock().insert(key) {
+ break;
+ }
+ tokio::time::sleep(BUILD_POLL).await;
+ }
+ // The build runs as its own task, so it always finishes and clears
+ // `building`: a caller cancelled mid-build (its pump aborted by the
+ // supervisor) must not leave every later caller waiting forever.
+ let cache = self.clone();
+ let settings = settings.clone();
+ let task = tokio::spawn(async move {
+ let slate = build(settings.clone(), shape).await;
+ if let Some(slate) = &slate {
+ cache.ready.lock().insert(key, (settings, slate.clone()));
+ }
+ cache.building.lock().remove(&key);
+ slate
+ });
+ task.await.ok().flatten()
+ }
+
+ /// Start building the loop for `shape` in the background, unless it is
+ /// ready or already being built.
+ pub fn prebuild(self: &Arc, settings: &CrashProtection, shape: StreamShape) {
+ let key = (shape.width, shape.height);
+ if self.cached(settings, shape).is_some() || self.building.lock().contains(&key) {
+ return;
+ }
+ let cache = self.clone();
+ let settings = settings.clone();
+ tokio::spawn(async move {
+ cache.get(&settings, shape).await;
+ });
+ }
+
+ /// Drop every loop (crash protection was switched off).
+ pub fn clear(&self) {
+ self.ready.lock().clear();
+ }
+}
+
+/// Encode a loop off the async runtime.
+async fn build(settings: CrashProtection, shape: StreamShape) -> Option> {
+ tokio::task::spawn_blocking(move || {
+ crate::slate::build_loop(&settings, shape)
+ .ok()
+ .map(Arc::new)
+ })
+ .await
+ .ok()
+ .flatten()
+}
+
+/// "1:05" for a duration, for logs and webhooks.
+pub fn minutes_seconds(duration: Duration) -> String {
+ let secs = duration.as_secs();
+ format!("{}:{:02}", secs / 60, secs % 60)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::h264::select_video_bytes;
+
+ fn aac_header(flags: u8, asc: [u8; 2]) -> Vec {
+ vec![flags, 0, asc[0], asc[1]]
+ }
+
+ /// An Enhanced Broadcasting (OneTrack) sequence-start tag.
+ fn one_track_header(first: u8, fourcc: &[u8; 4], track: u8, config: &[u8]) -> Vec {
+ let mut tag = vec![first, ONE_TRACK << 4 | EX_SEQUENCE_START];
+ tag.extend_from_slice(fourcc);
+ tag.push(track);
+ tag.extend_from_slice(config);
+ tag
+ }
+
+ #[test]
+ fn silence_matches_the_streams_own_aac_config() {
+ // AAC-LC, 48 kHz (index 3), stereo: 00010 0011 0010 000.
+ let stereo = silent_audio_from_header(&aac_header(0xAF, [0x11, 0x90])).unwrap();
+ assert_eq!(stereo.frame, SILENT_AAC_STEREO);
+ assert_eq!(stereo.framing, Framing::Legacy);
+ assert_eq!(audio_tag(&stereo)[..2], [0xAF, AAC_RAW]);
+ assert!((stereo.frame_ms - 21.333).abs() < 0.01);
+ // AAC-LC, 44.1 kHz (index 4), mono.
+ let mono = silent_audio_from_header(&aac_header(0xAE, [0x12, 0x08])).unwrap();
+ assert_eq!(mono.frame, SILENT_AAC_MONO);
+ assert!((mono.frame_ms - 23.22).abs() < 0.01);
+ }
+
+ #[test]
+ fn a_vod_audio_track_gets_silence_under_its_own_track_id() {
+ let header = one_track_header(0x95, b"mp4a", 1, &[0x11, 0x90]);
+ let silence = silent_audio_from_header(&header).unwrap();
+ assert_eq!(silence.framing, Framing::OneTrack(1));
+ let tag = audio_tag(&silence);
+ assert_eq!(tag[..7], [0x95, 0x01, b'm', b'p', b'4', b'a', 1]);
+ // It flattens back to the legacy silent frame any ingest accepts.
+ let flat = crate::h264::flatten_multitrack_audio(&tag, 1).unwrap();
+ assert_eq!(flat[..2], [0xAF, AAC_RAW]);
+ assert_eq!(&flat[2..], SILENT_AAC_STEREO);
+ }
+
+ #[test]
+ fn unsupported_audio_gets_no_silence() {
+ // HE-AAC (object type 5), 5.1 channels, a non-AAC tag, Opus.
+ assert!(silent_audio_from_header(&aac_header(0xAF, [0x29, 0x90])).is_none());
+ assert!(silent_audio_from_header(&aac_header(0xAF, [0x11, 0xB0])).is_none());
+ assert!(silent_audio_from_header(&[0x2F, 0x00, 0x11, 0x90]).is_none());
+ assert!(silent_audio_from_header(&aac_header(0xAF, [0x11, 0x90])[..2]).is_none());
+ assert!(silent_audio_from_header(&one_track_header(0x95, b"Opus", 1, &[0; 8])).is_none());
+ }
+
+ #[test]
+ fn the_screen_matches_each_h264_framing_and_nothing_else() {
+ assert_eq!(avc_framing(&[0x17, 0, 0, 0, 0, 1]), Some(Framing::Legacy));
+ let mut enhanced = vec![EX_VIDEO | FRAME_KEY << 4 | EX_SEQUENCE_START];
+ enhanced.extend_from_slice(b"avc1");
+ assert_eq!(avc_framing(&enhanced), Some(Framing::Enhanced));
+ let rung = one_track_header(0x96, b"avc1", 2, &[1]);
+ assert_eq!(avc_framing(&rung), Some(Framing::OneTrack(2)));
+ assert_eq!(avc_framing(&one_track_header(0x96, b"hvc1", 2, &[1])), None);
+ // ManyTracks bundles several tracks' configs in one tag.
+ let mut bundle = one_track_header(0x96, b"avc1", 0, &[1]);
+ bundle[1] = 1 << 4;
+ assert_eq!(avc_framing(&bundle), None);
+ }
+
+ #[test]
+ fn screen_frames_are_framed_like_the_track_they_replace() {
+ assert_eq!(
+ video_tag(Framing::Legacy, true, &[9, 9]),
+ vec![0x17, 1, 0, 0, 0, 9, 9]
+ );
+ assert_eq!(video_tag(Framing::Legacy, false, &[9])[0], 0x27);
+ let rung = video_tag(Framing::OneTrack(3), true, &[9, 9]);
+ assert_eq!(rung[..7], [0x96, 0x01, b'a', b'v', b'c', b'1', 3]);
+ // A vertical destination fed track 3 sees the plain legacy frame.
+ let flat = select_video_bytes(&rung, VideoEgress::Track(3)).unwrap();
+ assert_eq!(&flat[..], &[0x17, 1, 0, 0, 0, 9, 9]);
+ let enhanced = video_tag(Framing::Enhanced, false, &[9]);
+ assert_eq!(enhanced[..5], [0xA1, b'a', b'v', b'c', b'1']);
+ }
+
+ #[test]
+ fn only_keyframes_the_screen_cant_replace_are_held() {
+ let key = EX_VIDEO | FRAME_KEY << 4 | EX_VIDEO_MULTITRACK;
+ let coded = ONE_TRACK << 4 | EX_CODED_FRAMES;
+ let hevc = [key, coded, b'h', b'v', b'c', b'1', 1, 0, 0, 0, 7];
+ assert!(is_held_keyframe(&hevc));
+ let mut avc = hevc;
+ avc[2..6].copy_from_slice(b"avc1");
+ assert!(!is_held_keyframe(&avc));
+ let mut inter = hevc;
+ inter[0] = EX_VIDEO | FRAME_INTER << 4 | EX_VIDEO_MULTITRACK;
+ assert!(!is_held_keyframe(&inter));
+ let mut bundle = avc;
+ bundle[1] = 1 << 4 | EX_CODED_FRAMES;
+ assert!(is_held_keyframe(&bundle));
+ let av1 = [
+ EX_VIDEO | FRAME_KEY << 4 | EX_CODED_FRAMES,
+ b'a',
+ b'v',
+ b'0',
+ b'1',
+ 7,
+ ];
+ assert!(is_held_keyframe(&av1));
+ assert!(!is_held_keyframe(&[0x17, 1, 0, 0, 0, 0, 0, 0, 1, 0x65]));
+ }
+
+ #[test]
+ fn a_held_keyframe_is_resent_once_a_second() {
+ let held = VideoTrack::HeldFrame(Arc::from(&[1u8, 2, 3][..]));
+ assert!(video_frame(&held, 0).is_some());
+ assert!(video_frame(&held, 1).is_none());
+ assert!(video_frame(&held, HELD_FRAME_EVERY - 1).is_none());
+ assert_eq!(video_frame(&held, HELD_FRAME_EVERY).unwrap()[..], [1, 2, 3]);
+ }
+
+ /// A caller cancelled while its loop is being built (its pump aborted)
+ /// must not leave the resolution marked as building for everyone else.
+ #[tokio::test]
+ async fn a_cancelled_build_does_not_wedge_the_cache() {
+ let cache = Arc::new(SlateCache::default());
+ let settings = CrashProtection::default();
+ let shape = StreamShape {
+ width: 320,
+ height: 180,
+ fps: SLATE_FPS,
+ };
+ let first = {
+ let cache = cache.clone();
+ let settings = settings.clone();
+ tokio::spawn(async move { cache.get(&settings, shape).await })
+ };
+ tokio::task::yield_now().await;
+ first.abort();
+ let later = tokio::time::timeout(Duration::from_secs(10), cache.get(&settings, shape));
+ assert!(later.await.expect("not wedged").is_some());
+ }
+
+ /// OBS is back, but its new video comes in too slowly to span the
+ /// delay (a struggling encoder, or a delay raised meanwhile). The screen
+ /// can't wait forever: after the grace it rejoins where OBS's new video
+ /// starts, never on the video from before the hold.
+ #[test]
+ fn the_screen_stops_waiting_for_the_delay_after_the_grace() {
+ const DELAY_MS: u64 = 10_000;
+ let path =
+ std::env::temp_dir().join(format!("ic-rebuild-grace-{}.buf", std::process::id()));
+ let _ = std::fs::remove_file(&path);
+ let ring = Arc::new(crate::buffer::DiskRing::create(&path, 1 << 20).unwrap());
+ let ctrl = Controller::new(ring, 0);
+ let keyframe = [0x17, 1, 0, 0, 0];
+ ctrl.ring.append(9, 0, &keyframe, true, false).unwrap();
+ let first_new = ctrl.ring.append(9, 5_000, &keyframe, true, false).unwrap();
+ ctrl.ring.append(9, 6_000, &keyframe, true, false).unwrap();
+ let mut rebuild = Some(Rebuild {
+ after_seq: Some(0),
+ since: Instant::now(),
+ });
+ assert_eq!(
+ delay_rebuilt(&ctrl, &mut rebuild, DELAY_MS),
+ None,
+ "1 s of 10"
+ );
+
+ let waited = Duration::from_millis(DELAY_MS) + REBUILD_GRACE;
+ rebuild.as_mut().unwrap().since = Instant::now() - waited;
+ assert_eq!(delay_rebuilt(&ctrl, &mut rebuild, DELAY_MS), first_new);
+ let _ = std::fs::remove_file(&path);
+ }
+
+ #[test]
+ fn durations_print_as_minutes_and_seconds() {
+ assert_eq!(minutes_seconds(Duration::from_secs(65)), "1:05");
+ assert_eq!(minutes_seconds(Duration::from_secs(120)), "2:00");
+ }
+}
diff --git a/src/crash_protection.rs b/src/crash_protection.rs
new file mode 100644
index 0000000..693d368
--- /dev/null
+++ b/src/crash_protection.rs
@@ -0,0 +1,364 @@
+//! Crash protection settings.
+//!
+//! When OBS dies mid-stream, crash protection keeps every destination live
+//! with a looping reconnect screen until OBS publishes again or the hold
+//! time runs out. This file owns the persisted knobs (the
+//! `crash_protection.*` config keys); the screen itself is drawn and
+//! encoded by `crate::slate`.
+
+use crate::slate::Rgb;
+use std::io::{self, Write};
+
+pub const HOLD_SECS_DEFAULT: u32 = 120;
+pub const HOLD_SECS_MIN: u32 = 30;
+pub const HOLD_SECS_MAX: u32 = 300;
+pub const HEADLINE_MAX_CHARS: usize = 28;
+pub const SUBLINE_MAX_CHARS: usize = 48;
+const ACCENT_DEFAULT: Rgb = Rgb::new(0x5a, 0xc8, 0xfa);
+const HEADLINE_DEFAULT: &str = "Reconnecting";
+const SUBLINE_DEFAULT: &str = "The stream will be back in a moment";
+/// Config and form keys share this prefix, like `hotkey.` and `midi.`.
+pub const KEY_PREFIX: &str = "crash_protection.";
+
+/// Look of the reconnect screen.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum SlateTheme {
+ /// Minimal centred text with three pulsing dots.
+ Whisper,
+ /// 5x7 pixel font snapped to the 16 px macroblock grid.
+ Arcade,
+ /// Centred text under a dot sending out radar rings.
+ Beacon,
+ /// Centred text under a ring of eight dots, one lit and chasing round.
+ Orbit,
+ /// Broadcast lower third: text on the left with an accent bar and a
+ /// sliding progress stripe.
+ Studio,
+}
+
+impl SlateTheme {
+ /// Every theme, in the order the dashboard offers them.
+ pub const ALL: [SlateTheme; 5] = [
+ SlateTheme::Whisper,
+ SlateTheme::Beacon,
+ SlateTheme::Orbit,
+ SlateTheme::Studio,
+ SlateTheme::Arcade,
+ ];
+
+ pub fn id(self) -> &'static str {
+ match self {
+ SlateTheme::Whisper => "whisper",
+ SlateTheme::Arcade => "arcade",
+ SlateTheme::Beacon => "beacon",
+ SlateTheme::Orbit => "orbit",
+ SlateTheme::Studio => "studio",
+ }
+ }
+
+ /// Name shown in the dashboard.
+ pub fn name(self) -> &'static str {
+ match self {
+ SlateTheme::Whisper => "Whisper",
+ SlateTheme::Arcade => "Arcade",
+ SlateTheme::Beacon => "Beacon",
+ SlateTheme::Orbit => "Orbit",
+ SlateTheme::Studio => "Studio",
+ }
+ }
+
+ fn from_id(id: &str) -> Option {
+ Self::ALL.into_iter().find(|theme| theme.id() == id)
+ }
+
+ /// Background used when the streamer hasn't picked one.
+ pub fn default_background(self) -> Rgb {
+ match self {
+ SlateTheme::Whisper => Rgb::new(0x0e, 0x0f, 0x12),
+ SlateTheme::Arcade => Rgb::new(0x13, 0x0f, 0x22),
+ SlateTheme::Beacon => Rgb::new(0x0b, 0x12, 0x16),
+ SlateTheme::Orbit => Rgb::new(0x10, 0x10, 0x16),
+ SlateTheme::Studio => Rgb::new(0x0d, 0x10, 0x18),
+ }
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct CrashProtection {
+ pub enabled: bool,
+ /// How long destinations stay up on the reconnect screen before the
+ /// stream ends cleanly. Clamped to HOLD_SECS_MIN..=HOLD_SECS_MAX.
+ pub hold_secs: u32,
+ pub theme: SlateTheme,
+ pub accent: Rgb,
+ /// None means the theme's own background.
+ pub background: Option,
+ pub headline: String,
+ pub subline: String,
+ /// Treat a clean disconnect (FCUnpublish/deleteStream) as a crash too.
+ /// OBS closes cleanly on encoder errors, so this is the only way to
+ /// protect against them; the streamer ends the hold with "End now".
+ pub every_disconnect: bool,
+}
+
+impl Default for CrashProtection {
+ fn default() -> Self {
+ Self {
+ enabled: false,
+ hold_secs: HOLD_SECS_DEFAULT,
+ theme: SlateTheme::Whisper,
+ accent: ACCENT_DEFAULT,
+ background: None,
+ headline: HEADLINE_DEFAULT.into(),
+ subline: SUBLINE_DEFAULT.into(),
+ every_disconnect: false,
+ }
+ }
+}
+
+impl CrashProtection {
+ /// Apply one `field=value` pair (the part after KEY_PREFIX). Malformed
+ /// values are dropped so a hand-edited config or form can't load a
+ /// colour the rasterizer can't parse or a hold time outside the range.
+ pub fn set(&mut self, field: &str, value: &str) {
+ match field {
+ "enabled" => self.enabled = is_truthy(value),
+ "hold_secs" => {
+ if let Ok(secs) = value.trim().parse::() {
+ self.hold_secs = secs.clamp(HOLD_SECS_MIN, HOLD_SECS_MAX);
+ }
+ }
+ "theme" => {
+ if let Some(theme) = SlateTheme::from_id(value) {
+ self.theme = theme;
+ }
+ }
+ "accent" => {
+ if let Some(color) = Rgb::from_hex(value) {
+ self.accent = color;
+ }
+ }
+ "background" if value.is_empty() => self.background = None,
+ "background" => {
+ if let Some(color) = Rgb::from_hex(value) {
+ self.background = Some(color);
+ }
+ }
+ "headline" => self.headline = sanitize_text(value, HEADLINE_MAX_CHARS),
+ "subline" => self.subline = sanitize_text(value, SUBLINE_MAX_CHARS),
+ "every_disconnect" => self.every_disconnect = is_truthy(value),
+ _ => {}
+ }
+ }
+
+ /// Whether `other` draws the same reconnect screen. The hold time and
+ /// what triggers a hold don't change the picture, so editing them
+ /// never re-encodes a loop.
+ pub fn same_screen(&self, other: &CrashProtection) -> bool {
+ self.theme == other.theme
+ && self.accent == other.accent
+ && self.resolved_background() == other.resolved_background()
+ && self.headline == other.headline
+ && self.subline == other.subline
+ }
+
+ /// Background the screen actually draws with.
+ pub fn resolved_background(&self) -> Rgb {
+ self.background
+ .unwrap_or_else(|| self.theme.default_background())
+ }
+
+ /// Write every field that differs from the default, one
+ /// `crash_protection.=` line each, so an untouched
+ /// install keeps its config file lean.
+ pub fn write_lines(&self, out: &mut impl Write) -> io::Result<()> {
+ let defaults = Self::default().entries();
+ for ((field, value), (_, default_value)) in self.entries().into_iter().zip(defaults) {
+ if value != default_value {
+ writeln!(out, "{KEY_PREFIX}{field}={value}")?;
+ }
+ }
+ Ok(())
+ }
+
+ /// JSON object for `GET /config`. Nothing here is secret. `themes`
+ /// lists what the dashboard can offer, with each theme's default
+ /// background, so the page never keeps its own copy.
+ pub fn to_json(&self) -> String {
+ let themes: Vec = SlateTheme::ALL
+ .iter()
+ .map(|theme| {
+ format!(
+ r#"{{"id":{},"name":{},"background":{}}}"#,
+ crate::config::json_str(theme.id()),
+ crate::config::json_str(theme.name()),
+ crate::config::json_str(&theme.default_background().to_hex()),
+ )
+ })
+ .collect();
+ format!(
+ r#"{{"enabled":{},"hold_secs":{},"theme":{},"accent":{},"background":{},"headline":{},"subline":{},"every_disconnect":{},"defaults":{{"headline":{},"subline":{}}},"themes":[{}]}}"#,
+ self.enabled,
+ self.hold_secs,
+ crate::config::json_str(self.theme.id()),
+ crate::config::json_str(&self.accent.to_hex()),
+ crate::config::json_str(&self.background.map(Rgb::to_hex).unwrap_or_default()),
+ crate::config::json_str(&self.headline),
+ crate::config::json_str(&self.subline),
+ self.every_disconnect,
+ crate::config::json_str(HEADLINE_DEFAULT),
+ crate::config::json_str(SUBLINE_DEFAULT),
+ themes.join(","),
+ )
+ }
+
+ fn entries(&self) -> [(&'static str, String); 8] {
+ [
+ ("enabled", self.enabled.to_string()),
+ ("hold_secs", self.hold_secs.to_string()),
+ ("theme", self.theme.id().to_string()),
+ ("accent", self.accent.to_hex()),
+ (
+ "background",
+ self.background.map(Rgb::to_hex).unwrap_or_default(),
+ ),
+ ("headline", self.headline.clone()),
+ ("subline", self.subline.clone()),
+ ("every_disconnect", self.every_disconnect.to_string()),
+ ]
+ }
+}
+
+/// Checkbox values arrive as "true"/"false" or "on"/"" depending on the
+/// caller, the same convention the other boolean settings use.
+fn is_truthy(value: &str) -> bool {
+ !matches!(value, "" | "false" | "0" | "off")
+}
+
+/// Screen text: no control characters (the config format is line-based),
+/// surrounding whitespace trimmed, capped to what fits on the screen.
+fn sanitize_text(value: &str, max_chars: usize) -> String {
+ value
+ .chars()
+ .filter(|c| !c.is_control())
+ .collect::()
+ .trim()
+ .chars()
+ .take(max_chars)
+ .collect()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn written(settings: &CrashProtection) -> String {
+ let mut out = Vec::new();
+ settings.write_lines(&mut out).unwrap();
+ String::from_utf8(out).unwrap()
+ }
+
+ #[test]
+ fn defaults_write_nothing() {
+ assert_eq!(written(&CrashProtection::default()), "");
+ }
+
+ #[test]
+ fn changed_fields_round_trip_through_set() {
+ let mut settings = CrashProtection::default();
+ settings.set("enabled", "true");
+ settings.set("hold_secs", "90");
+ settings.set("theme", "arcade");
+ settings.set("accent", "#FF6B6B");
+ settings.set("background", "#101010");
+ settings.set("headline", "Be right back");
+ settings.set("subline", "Grab a drink");
+ settings.set("every_disconnect", "on");
+
+ let mut loaded = CrashProtection::default();
+ for line in written(&settings).lines() {
+ let (key, value) = line.split_once('=').unwrap();
+ loaded.set(key.strip_prefix(KEY_PREFIX).unwrap(), value);
+ }
+ assert_eq!(loaded, settings);
+ }
+
+ #[test]
+ fn hold_secs_is_clamped_and_garbage_ignored() {
+ let mut settings = CrashProtection::default();
+ settings.set("hold_secs", "5");
+ assert_eq!(settings.hold_secs, HOLD_SECS_MIN);
+ settings.set("hold_secs", "9999");
+ assert_eq!(settings.hold_secs, HOLD_SECS_MAX);
+ settings.set("hold_secs", "soon");
+ assert_eq!(settings.hold_secs, HOLD_SECS_MAX);
+ }
+
+ #[test]
+ fn bad_colours_and_themes_are_dropped() {
+ let mut settings = CrashProtection::default();
+ settings.set("accent", "red");
+ settings.set("accent", "#12345");
+ settings.set("theme", "signal");
+ assert_eq!(settings.accent, ACCENT_DEFAULT);
+ assert_eq!(settings.theme, SlateTheme::Whisper);
+ }
+
+ #[test]
+ fn empty_background_means_theme_default() {
+ let mut settings = CrashProtection::default();
+ settings.set("background", "#202020");
+ settings.set("background", "");
+ assert_eq!(settings.background, None);
+ assert_eq!(
+ settings.resolved_background(),
+ SlateTheme::Whisper.default_background()
+ );
+ }
+
+ #[test]
+ fn text_loses_control_characters_and_is_capped() {
+ let mut settings = CrashProtection::default();
+ settings.set("headline", " Back\nsoon\t ");
+ assert_eq!(settings.headline, "Backsoon");
+ settings.set("subline", &"x".repeat(100));
+ assert_eq!(settings.subline.chars().count(), SUBLINE_MAX_CHARS);
+ }
+
+ #[test]
+ fn json_carries_every_field() {
+ let json = CrashProtection::default().to_json();
+ for field in [
+ "enabled",
+ "hold_secs",
+ "theme",
+ "accent",
+ "background",
+ "headline",
+ "subline",
+ "every_disconnect",
+ "themes",
+ ] {
+ assert!(
+ json.contains(&format!("\"{field}\":")),
+ "missing {field} in {json}"
+ );
+ }
+ assert!(json.contains(r##""accent":"#5ac8fa""##));
+ }
+
+ #[test]
+ fn every_theme_round_trips_by_id() {
+ for theme in SlateTheme::ALL {
+ let mut settings = CrashProtection::default();
+ settings.set("theme", theme.id());
+ assert_eq!(settings.theme, theme);
+ // The picked theme, not the `themes` list that names every id.
+ let json = settings.to_json();
+ assert!(
+ json.contains(&format!(r#""theme":"{}""#, theme.id())),
+ "{json}"
+ );
+ }
+ }
+}
diff --git a/src/crypto.rs b/src/crypto.rs
index 8f4beee..ce1fbb0 100644
--- a/src/crypto.rs
+++ b/src/crypto.rs
@@ -206,6 +206,51 @@ mod tests {
);
}
+ /// RFC 4231 test case 6: a 131-byte key, longer than the 64-byte block, so
+ /// it must be hashed down first. Cases 1 and 2 use short keys and never
+ /// reach that branch; a password longer than 64 bytes does.
+ #[test]
+ fn hmac_rfc4231_case6_hashes_an_oversized_key() {
+ let mac = hmac_sha256(
+ &[0xaa; 131],
+ b"Test Using Larger Than Block-Size Key - Hash Key First",
+ );
+ assert_eq!(
+ sha256::hex_bytes(&mac),
+ "60e431591ee0b67f0d8a26aacbf5b77f8e0bc6213728c5140546040f0ee37f54"
+ );
+ }
+
+ /// RFC 4231 test case 7: an oversized key AND a message longer than one
+ /// block, so the inner hash spans several blocks.
+ #[test]
+ fn hmac_rfc4231_case7_oversized_key_and_message() {
+ let mac = hmac_sha256(
+ &[0xaa; 131],
+ b"This is a test using a larger than block-size key and a larger than \
+ block-size data. The key needs to be hashed before being used by the \
+ HMAC algorithm.",
+ );
+ assert_eq!(
+ sha256::hex_bytes(&mac),
+ "9b09ffa71b942fcb27635fbcd5b0e944bfdc63644f0713938a7f51535c3a35e2"
+ );
+ }
+
+ /// A key of exactly one block is used as-is, not hashed. This is the
+ /// boundary of the `>` in `hmac_sha256`: turning it into `>=` would hash a
+ /// 64-byte key and change every MAC made with one. Expected value is the
+ /// NIST HMAC-SHA256 "keylen=blocklen" example (key = bytes 0x00..0x3f).
+ #[test]
+ fn hmac_key_of_exactly_one_block_is_not_hashed() {
+ let key: Vec = (0u8..64).collect();
+ let mac = hmac_sha256(&key, b"Sample message for keylen=blocklen");
+ assert_eq!(
+ sha256::hex_bytes(&mac),
+ "8bb9a1db9806f20df7f77b82138c7914d174d59e13dc4d0169c9057b133e1d62"
+ );
+ }
+
/// Published PBKDF2-HMAC-SHA256 vectors (password "password", salt "salt").
#[test]
fn pbkdf2_known_vectors() {
@@ -233,6 +278,22 @@ mod tests {
assert_ne!(h, h2);
}
+ /// A stored hash in the exact on-disk format, built from the published
+ /// PBKDF2 vector (password "password", salt "salt" = hex 73616c74, c = 1).
+ ///
+ /// `password_round_trip` only proves `hash_password` and `verify_password`
+ /// agree with each other, so a format change made to both sides passes it
+ /// while locking every existing user out of their dashboard. This one is
+ /// what a config file written by an older build actually contains.
+ const STORED_VECTOR: &str = "pbkdf2-sha256$1$73616c74$\
+ 120fb6cffcf8b32c43e7225256c4f837a86548c92ccc35480805987cb70be17b";
+
+ #[test]
+ fn an_existing_stored_hash_still_verifies() {
+ assert!(verify_password("password", STORED_VECTOR));
+ assert!(!verify_password("Password", STORED_VECTOR));
+ }
+
#[test]
fn verify_rejects_garbage() {
assert!(!verify_password("x", ""));
@@ -241,6 +302,44 @@ mod tests {
assert!(!verify_password("x", "pbkdf2-sha256$1$zz$bb")); // non-hex salt
}
+ /// Every malformed variant of a hash that is otherwise correct for this
+ /// password. Starting from a valid value is the point: each case fails for
+ /// exactly the one reason it names, not because the password is wrong.
+ #[test]
+ fn verify_rejects_damaged_copies_of_a_valid_hash() {
+ let dk_len = STORED_VECTOR.len();
+ for (stored, why) in [
+ (
+ STORED_VECTOR.replace("pbkdf2-sha256$", "pbkdf2-sha1$"),
+ "another algorithm's prefix",
+ ),
+ (
+ STORED_VECTOR[..dk_len - 1].to_string(),
+ "an odd-length digest",
+ ),
+ (
+ STORED_VECTOR[..dk_len - 2].to_string(),
+ "a truncated digest (31 bytes)",
+ ),
+ (
+ format!("{STORED_VECTOR}00"),
+ "an overlong digest (33 bytes)",
+ ),
+ (format!("{STORED_VECTOR}$extra"), "five fields"),
+ (
+ STORED_VECTOR.replacen("$1$", "$", 1),
+ "three fields (no iteration count)",
+ ),
+ (
+ STORED_VECTOR.replacen("$1$", "$x$", 1),
+ "a non-numeric count",
+ ),
+ (STORED_VECTOR.replacen("$1$", "$-1$", 1), "a negative count"),
+ ] {
+ assert!(!verify_password("password", &stored), "{why}: {stored}");
+ }
+ }
+
#[test]
fn constant_time_eq_basics() {
assert!(constant_time_eq(b"abc", b"abc"));
diff --git a/src/h264.rs b/src/h264.rs
index 95aecbb..67fb46d 100644
--- a/src/h264.rs
+++ b/src/h264.rs
@@ -57,6 +57,18 @@ const FOURCC_AV01: [u8; 4] = *b"av01";
const FOURCC_VP09: [u8; 4] = *b"vp09";
const FOURCC_AVC1: [u8; 4] = *b"avc1";
+/// Enhanced RTMP packet types that carry decoder config: SequenceStart (0)
+/// and MPEG2TSSequenceStart (5, AV1's alternative config form).
+fn is_config_packet(packet_type: u8) -> bool {
+ matches!(packet_type, 0 | 5)
+}
+
+/// Enhanced RTMP packet types that carry coded pictures: CodedFrames (1)
+/// and CodedFramesX (3, no composition time).
+fn is_coded_frames_packet(packet_type: u8) -> bool {
+ matches!(packet_type, 1 | 3)
+}
+
pub fn classify_video_tag(payload: &[u8]) -> VideoTagInfo {
let unknown = VideoTagInfo {
is_seq_header: false,
@@ -101,14 +113,16 @@ pub fn classify_video_tag(payload: &[u8]) -> VideoTagInfo {
// 5=MPEG2TSSequenceStart, 6=Multitrack.
let frame_type = (b0 >> 4) & 0x07;
let packet_type = b0 & 0x0F;
- let is_seq_header = packet_type == 0;
+ let is_seq_header = is_config_packet(packet_type);
let is_metadata = packet_type == 4;
// Enhanced encoders always set FrameType=1 on keyframes. For AVC we
// walked NALUs because some encoders flag P-frames as key incorrectly;
// enhanced-rtmp tightens this and FrameType is the spec-blessed signal.
- // SeqStart / Metadata packets carry no slice data, so refuse to flag
- // them as IDR even if the encoder set FrameType=1.
- let is_keyframe = frame_type == 1 && !is_seq_header && !is_metadata;
+ // Only coded frames (CodedFrames / CodedFramesX) carry a picture:
+ // config, SequenceEnd and Metadata packets never count as keyframes,
+ // even with FrameType=1 - a SequenceEnd taken for one became a cut
+ // point, and crash protection's held "last keyframe".
+ let is_keyframe = frame_type == 1 && is_coded_frames_packet(packet_type);
let is_multitrack = packet_type == 6;
// For multi-track, the FourCC sits behind the multitrack header; we
@@ -126,11 +140,11 @@ pub fn classify_video_tag(payload: &[u8]) -> VideoTagInfo {
// FrameType=1 (some do).
if is_multitrack {
let nested_pt = payload.get(1).map(|b| b & 0x0F).unwrap_or(0xFF);
- let is_mt_seq_header = nested_pt == 0;
+ let is_mt_seq_header = is_config_packet(nested_pt);
let is_mt_metadata = nested_pt == 4;
return VideoTagInfo {
is_seq_header: is_mt_seq_header,
- is_idr: is_keyframe && !is_mt_seq_header && !is_mt_metadata,
+ is_idr: frame_type == 1 && is_coded_frames_packet(nested_pt),
is_multitrack: true,
is_metadata: is_mt_metadata,
codec: VideoCodec::Unknown,
@@ -163,19 +177,44 @@ pub fn classify_video_tag(payload: &[u8]) -> VideoTagInfo {
}
}
-fn contains_idr_nalu(mut data: &[u8]) -> bool {
- while data.len() >= 4 {
- let len = u32::from_be_bytes([data[0], data[1], data[2], data[3]]) as usize;
- if 4 + len > data.len() {
- return false;
- }
- let nal_unit_type = data[4] & 0x1F;
- if nal_unit_type == 5 {
- return true;
+fn contains_idr_nalu(data: &[u8]) -> bool {
+ // NAL lengths are 4 bytes in practice (OBS, FFmpeg, x264), and an IDR
+ // found that way counts even if later bytes don't parse. The AVC config
+ // allows 1- or 2-byte lengths too (lengthSizeMinusOne); for those, a
+ // 4-byte walk finds nothing and fails, and a size is trusted only when
+ // its lengths tile the frame exactly, which a wrong size almost never does.
+ let (idr, tiled) = scan_nal_units(data, 4);
+ if idr || tiled {
+ return idr;
+ }
+ [2, 1]
+ .into_iter()
+ .any(|size| scan_nal_units(data, size) == (true, true))
+}
+
+/// Walk NAL units prefixed by `size`-byte big-endian lengths: whether an
+/// IDR slice turned up, and whether the lengths tiled `data` exactly.
+fn scan_nal_units(mut data: &[u8], size: usize) -> (bool, bool) {
+ let mut idr = false;
+ while !data.is_empty() {
+ let Some(prefix) = data.get(..size) else {
+ return (idr, false);
+ };
+ let len = prefix
+ .iter()
+ .fold(0usize, |len, byte| (len << 8) | usize::from(*byte));
+ let Some(nal) = data[size..].get(..len) else {
+ return (idr, false);
+ };
+ // A zero-length NAL has no header byte to classify. Step over it:
+ // reading its header indexed past the slice when the empty NAL was
+ // the last thing in the tag, which aborts a release build.
+ if nal.first().is_some_and(|header| header & 0x1F == 5) {
+ idr = true;
}
- data = &data[4 + len..];
+ data = &data[size + len..];
}
- false
+ (idr, true)
}
/// Flatten an Enhanced RTMP multi-track video tag down to a standard
@@ -774,6 +813,12 @@ impl<'a> BitReader<'a> {
for _ in 0..size {
if next_scale != 0 {
let delta = self.read_se()?;
+ // delta_scale is -128..=127 (H.264 7.4.2.1.1.1). Anything
+ // else is a corrupt or hostile SPS, and a near-i32::MAX
+ // delta would overflow the sum below.
+ if !(-128..=127).contains(&delta) {
+ return None;
+ }
next_scale = (last_scale + delta + 256) % 256;
}
last_scale = if next_scale == 0 {
@@ -876,8 +921,9 @@ fn is_multitrack_audio(payload: &[u8]) -> bool {
/// Flatten an Enhanced-RTMP multi-track AUDIO tag down to a standard
/// single-track tag carrying only `target_track`. The audio twin of
-/// [`flatten_multitrack_video`], shifted by one byte: the multitrack header
-/// is byte 1, the FourCC is bytes 2..6, and (OneTrack) the TrackId is byte 6.
+/// [`flatten_multitrack_video`], with the same offsets: the multitrack
+/// header is byte 1, the FourCC is bytes 2..6, and (OneTrack) the TrackId is
+/// byte 6.
///
/// AAC (`mp4a`) is rewritten to a *legacy* AAC tag (`0xAF` ...), the form
/// every RTMP ingest accepts - the same reason the vertical AVC path drops to
@@ -889,15 +935,35 @@ fn is_multitrack_audio(payload: &[u8]) -> bool {
/// layout or `target_track` isn't present, so the caller drops the tag for
/// this destination.
pub fn flatten_multitrack_audio(payload: &[u8], target_track: u8) -> Option> {
+ let (fourcc, track_payload) = locate_audio_track(payload, target_track)?;
+ let nested_pt = payload[1] & 0x0F;
+ let mut out = Vec::with_capacity(track_payload.len() + 5);
+ if fourcc == FOURCC_MP4A && nested_pt <= 1 {
+ // Legacy AAC: [0xAF][AACPacketType 0=seq/1=raw][data]. The nested
+ // audio packet type maps 1:1 (SequenceStart=0, CodedFrames=1).
+ out.push(0xAF);
+ out.push(nested_pt);
+ out.extend_from_slice(track_payload);
+ } else {
+ // Enhanced-RTMP single-track audio tag (non-AAC, or an AAC packet
+ // type legacy FLV can't express such as MultichannelConfig).
+ out.push(0x90 | (nested_pt & 0x0F));
+ out.extend_from_slice(fourcc);
+ out.extend_from_slice(track_payload);
+ }
+ Some(out)
+}
+
+/// The FourCC and payload of `target_track` in an Enhanced-RTMP multi-track
+/// audio tag, across its three layouts (offsets mirror
+/// `flatten_multitrack_video`). None when the tag isn't one, or doesn't
+/// carry that track.
+fn locate_audio_track(payload: &[u8], target_track: u8) -> Option<(&[u8], &[u8])> {
if payload.len() < 7 || !is_multitrack_audio(payload) {
return None;
}
let mt_type = (payload[1] >> 4) & 0x0F;
- let nested_pt = payload[1] & 0x0F;
-
- // Locate the requested track's FourCC + payload across the three
- // multi-track layouts (offsets mirror flatten_multitrack_video).
- let (fourcc, track_payload): (&[u8], &[u8]) = match mt_type {
+ let located = match mt_type {
0 => {
// OneTrack: [FourCC(4)][TrackId(1)][payload..]
if payload[6] != target_track {
@@ -944,22 +1010,12 @@ pub fn flatten_multitrack_audio(payload: &[u8], target_track: u8) -> Option return None,
};
+ Some(located)
+}
- let mut out = Vec::with_capacity(track_payload.len() + 5);
- if fourcc == FOURCC_MP4A && nested_pt <= 1 {
- // Legacy AAC: [0xAF][AACPacketType 0=seq/1=raw][data]. The nested
- // audio packet type maps 1:1 (SequenceStart=0, CodedFrames=1).
- out.push(0xAF);
- out.push(nested_pt);
- out.extend_from_slice(track_payload);
- } else {
- // Enhanced-RTMP single-track audio tag (non-AAC, or an AAC packet
- // type legacy FLV can't express such as MultichannelConfig).
- out.push(0x90 | (nested_pt & 0x0F));
- out.extend_from_slice(fourcc);
- out.extend_from_slice(track_payload);
- }
- Some(out)
+/// Whether a multi-track audio tag carries `track`.
+pub fn multitrack_audio_has_track(payload: &[u8], track: u8) -> bool {
+ locate_audio_track(payload, track).is_some()
}
/// Decide what audio bytes to put on the wire given the destination's
@@ -969,35 +1025,39 @@ pub fn flatten_multitrack_audio(payload: &[u8], target_track: u8) -> Option Option> {
use std::borrow::Cow;
let target = match egress {
AudioEgress::Passthrough => return Some(Cow::Borrowed(payload)),
AudioEgress::Track(t) => t,
};
+ let live_fallback = target == 0 || !target_on_wire;
if !is_multitrack_audio(payload) {
- // Single track = the only audio there is. Always forward it, whatever
- // track was requested - a live track beats a silent destination when
- // the 2nd track isn't being sent.
- return Some(Cow::Borrowed(payload));
- }
- // Flatten the requested track; fall back to the primary live track 0 when
- // it isn't present, so picking "Track 2" on a single-track stream still
- // sends audio instead of silence.
+ // A single-track tag is the live track.
+ return live_fallback.then_some(Cow::Borrowed(payload));
+ }
flatten_multitrack_audio(payload, target)
- .or_else(|| flatten_multitrack_audio(payload, 0))
+ .or_else(|| {
+ live_fallback
+ .then(|| flatten_multitrack_audio(payload, 0))
+ .flatten()
+ })
.map(Cow::Owned)
}
@@ -1139,8 +1199,8 @@ fn fourcc_to_codec(fourcc: [u8; 4]) -> AudioCodec {
/// Best-effort extraction of the Enhanced-RTMP audio TrackId for the
/// OneTrack multi-track layout. Mirrors `seq_header_track_id` (the
-/// video equivalent), shifted by one byte because audio's multitrack
-/// header lives at byte 2 vs video's byte 1.
+/// video equivalent): both put the multitrack header at byte 1 and the
+/// TrackId at byte 6.
///
/// Returns 0 for:
/// * legacy AAC / MP3 (sound_format != 9)
@@ -1287,6 +1347,55 @@ mod tests {
assert_eq!(info.codec, VideoCodec::Av1);
}
+ #[test]
+ fn enhanced_rtmp_fourccs_map_to_their_codec() {
+ // hev1 is HEVC with parameter sets in-band (some encoders send it
+ // instead of hvc1); avc1 is H.264 in Enhanced framing, which is what
+ // OBS uses for the EB ladder. Each drives the dashboard codec chip
+ // and the per-destination readout.
+ for (fourcc, codec) in [
+ (b"hev1", VideoCodec::Hevc),
+ (b"vp09", VideoCodec::Vp9),
+ (b"avc1", VideoCodec::Avc),
+ (b"xxxx", VideoCodec::Unknown),
+ ] {
+ let mut frame = vec![0x80 | (2u8 << 4) | 1]; // inter, CodedFrames
+ frame.extend_from_slice(fourcc);
+ frame.extend_from_slice(&[0; 8]);
+ let info = classify_video_tag(&frame);
+ assert_eq!(info.codec, codec, "{}", String::from_utf8_lossy(fourcc));
+ assert!(!info.is_idr, "an inter frame is never a cut point");
+
+ let mut seq = vec![0x90u8]; // keyframe, SequenceStart
+ seq.extend_from_slice(fourcc);
+ assert_eq!(seq_header_codec(&seq), codec);
+ }
+ }
+
+ #[test]
+ fn seq_header_track_id_reads_only_the_onetrack_layout() {
+ // OneTrack: TrackId at byte 6.
+ assert_eq!(seq_header_track_id(&onetrack_avc_seq_header(3, 30, 40)), 3);
+ // ManyTracks / ManyTracksManyCodecs hold every track's config in
+ // one tag, so they share slot 0 even though byte 6 is a track id
+ // (ManyTracks) or a FourCC byte (ManyTracksManyCodecs).
+ let many_tracks = [0x96, 0x10, b'a', b'v', b'c', b'1', 2, 0, 0, 1, 0xAA];
+ assert_eq!(seq_header_track_id(&many_tracks), 0);
+ let many_codecs = [0x96, 0x20, 2, b'a', b'v', b'c', b'1', 0, 0, 1, 0xAA];
+ assert_eq!(seq_header_track_id(&many_codecs), 0);
+ // Legacy, Enhanced single-track and a OneTrack tag cut before its
+ // TrackId all fall back to slot 0 rather than reading past the end.
+ assert_eq!(seq_header_track_id(&legacy_avc_seq_header(40, 30)), 0);
+ assert_eq!(
+ seq_header_track_id(&[0x90, b'h', b'v', b'c', b'1', 7, 7]),
+ 0
+ );
+ assert_eq!(
+ seq_header_track_id(&[0x96, 0x00, b'a', b'v', b'c', b'1']),
+ 0
+ );
+ }
+
#[test]
fn aac_seq_header_detected() {
// sound_format 10 (AAC), packet_type 0 (AudioSpecificConfig)
@@ -1368,22 +1477,6 @@ mod tests {
assert!(!info.is_idr);
}
- #[test]
- fn enhanced_aac_seq_header_via_packet_type_0() {
- // Enhanced-RTMP single-track AAC seq header: byte 0 = 0x90
- // (SoundFormat=9, AudioPacketType=0=SequenceStart), followed
- // by FourCC. This is the exact tag OBS emits for the live
- // track when VOD audio is enabled (live = idx=0, single-track
- // because not multitrack).
- let mut payload = vec![0x90];
- payload.extend_from_slice(b"mp4a");
- let info = classify_audio_tag(&payload);
- assert!(info.is_seq_header);
- assert_eq!(info.codec, AudioCodec::Aac);
- // The convenience shim agrees.
- assert!(is_aac_seq_header(&payload));
- }
-
// ── select_video_bytes: per-destination flatten policy ───────────
//
// The pump uses this helper to decide what bytes to put on the wire
@@ -1436,7 +1529,9 @@ mod tests {
fn select_video_bytes_passes_multitrack_through_for_twitch() {
// Invariant 2: Twitch destinations get the raw multi-track
// bytes verbatim - that's what unlocks the transcoded ladder.
- // True for every TrackId, not just the primary.
+ // True for every TrackId, not just the primary. Twitch Dual Format
+ // rides on this too: the vertical canvas is just more TrackIds on
+ // the wire (`canvas_index` only exists in the config JSON).
for track in 0u8..=4 {
let tag = enhanced_rtmp_onetrack_video_bytes_track(track);
let twitch = select_video_bytes(&tag, VideoEgress::Passthrough)
@@ -1457,11 +1552,69 @@ mod tests {
let tag = enhanced_rtmp_onetrack_video_bytes_track(0);
let youtube =
select_video_bytes(&tag, VideoEgress::Track(0)).expect("track 0 must be forwarded");
- let direct_flat = flatten_multitrack_video(&tag).expect("OneTrack layout must flatten");
- assert_eq!(youtube.as_ref(), direct_flat.as_slice());
+ assert_eq!(youtube.as_ref(), flat_hvc1_keyframe().as_slice());
assert!(matches!(youtube, std::borrow::Cow::Owned(_)));
}
+ /// What `enhanced_rtmp_onetrack_video_bytes_track(_)` must flatten to,
+ /// written out byte by byte: a single-track Enhanced-RTMP keyframe
+ /// (0x80 | FrameType 1 << 4 | CodedFrames 1 = 0x91), the FourCC, then
+ /// the track body with the multitrack header and TrackId gone.
+ fn flat_hvc1_keyframe() -> Vec {
+ let mut out = vec![0x91, b'h', b'v', b'c', b'1'];
+ out.extend_from_slice(&[0xAA; 32]);
+ out
+ }
+
+ #[test]
+ fn flatten_multitrack_video_emits_known_bytes_for_every_layout() {
+ // OneTrack HEVC is the Twitch Dual Format vertical path (2K channels
+ // get HEVC, which has no legacy framing, so it is flattened rather
+ // than rewritten to 0x17/0x27). Inter frame + CodedFramesX here, so
+ // the rebuilt header byte is 0x80 | 2 << 4 | 3 = 0xA3.
+ let onetrack = [0xA6, 0x03, b'h', b'v', b'c', b'1', 1, 0x10, 0x20, 0x30];
+ assert_eq!(
+ flatten_multitrack_video(&onetrack).unwrap(),
+ [0xA3, b'h', b'v', b'c', b'1', 0x10, 0x20, 0x30]
+ );
+
+ // ManyTracks (mt_type 1): one FourCC, then [TrackId][Size(3)][body]
+ // per track. The first listed track is the primary and the rest go.
+ let many_tracks = [
+ 0x96, 0x11, b'h', b'v', b'c', b'1', 0, 0, 0, 3, 0xA1, 0xA2, 0xA3, 1, 0, 0, 2, 0xB1,
+ 0xB2,
+ ];
+ assert_eq!(
+ flatten_multitrack_video(&many_tracks).unwrap(),
+ [0x91, b'h', b'v', b'c', b'1', 0xA1, 0xA2, 0xA3]
+ );
+
+ // ManyTracksManyCodecs (mt_type 2): [TrackId][FourCC][Size(3)][body]
+ // per track, so the FourCC comes from the first track's own entry.
+ let many_codecs = [
+ 0x96, 0x21, 0, b'a', b'v', b'0', b'1', 0, 0, 2, 0xC1, 0xC2, 1, b'h', b'v', b'c', b'1',
+ 0, 0, 1, 0xD1,
+ ];
+ assert_eq!(
+ flatten_multitrack_video(&many_codecs).unwrap(),
+ [0x91, b'a', b'v', b'0', b'1', 0xC1, 0xC2]
+ );
+
+ // A first track that claims more bytes than the tag holds, and a
+ // layout type the spec reserves, both refuse rather than guess.
+ let mut overlong = many_tracks;
+ overlong[9] = 0xFF;
+ assert_eq!(flatten_multitrack_video(&overlong), None);
+ let mut reserved = onetrack;
+ reserved[1] = 0x31;
+ assert_eq!(flatten_multitrack_video(&reserved), None);
+ // Not multitrack at all.
+ assert_eq!(
+ flatten_multitrack_video(&avc_single_track_keyframe_bytes()),
+ None
+ );
+ }
+
#[test]
fn select_video_bytes_drops_nonzero_track_multitrack_for_non_twitch() {
// The core fix: OBS sends the simulcast ladder as OneTrack
@@ -1469,7 +1622,8 @@ mod tests {
// Forwarding every one to YouTube delivered N frames per PTS
// → decoder storm → reconnect cascade. These tags must be
// dropped; the primary arrives separately as a legacy /
- // Enhanced single-track tag.
+ // Enhanced single-track tag. With Dual Format, TrackId 1+ also
+ // carries the vertical canvas, which a horizontal dest never wants.
for track in 1u8..=4 {
let tag = enhanced_rtmp_onetrack_video_bytes_track(track);
assert!(
@@ -1479,48 +1633,6 @@ mod tests {
}
}
- // ── Twitch Dual Format (dual canvas) forwarding ──────────────────
- //
- // Dual Format = a horizontal (canvas 0) and a vertical (canvas 1)
- // layout sent together over the one Enhanced Broadcasting RTMP
- // connection. `canvas_index` lives only in the GetClientConfiguration
- // JSON; on the wire the canvases are just more TrackIds. So the
- // existing per-track passthrough already carries the vertical canvas:
- // these tests pin that down explicitly with dual-format framing.
-
- #[test]
- fn select_video_bytes_forwards_dual_canvas_to_twitch() {
- // Horizontal primary (TrackId 0) and a vertical-canvas rung
- // (TrackId 1) both reach Twitch byte-for-byte. Nothing about a
- // second canvas changes the Twitch passthrough contract.
- let horizontal = enhanced_rtmp_onetrack_video_bytes_track(0);
- let vertical = enhanced_rtmp_onetrack_video_bytes_track(1);
- for tag in [&horizontal, &vertical] {
- let twitch = select_video_bytes(tag, VideoEgress::Passthrough)
- .expect("twitch must forward both canvases");
- assert_eq!(twitch.as_ref(), tag.as_slice());
- assert!(matches!(twitch, std::borrow::Cow::Borrowed(_)));
- }
- }
-
- #[test]
- fn select_video_bytes_keeps_horizontal_drops_vertical_for_non_twitch() {
- // A non-Twitch destination (YouTube/Kick) running alongside a
- // Dual Format Twitch session keeps the horizontal primary and
- // drops the vertical canvas - it has no concept of a second
- // canvas, and TrackId != 0 is exactly the vertical rungs.
- let horizontal = enhanced_rtmp_onetrack_video_bytes_track(0);
- let vertical = enhanced_rtmp_onetrack_video_bytes_track(1);
- assert!(
- select_video_bytes(&horizontal, VideoEgress::Track(0)).is_some(),
- "horizontal canvas (TrackId 0) must reach non-Twitch"
- );
- assert!(
- select_video_bytes(&vertical, VideoEgress::Track(0)).is_none(),
- "vertical canvas (TrackId 1) must be dropped for non-Twitch"
- );
- }
-
// ── Vertical-canvas selection: SPS orientation + Track(n) routing ──
//
// A vertical destination forwards ONLY the portrait canvas's track,
@@ -1564,6 +1676,15 @@ mod tests {
}
self.put_bits(code, bits);
}
+ /// Signed Exp-Golomb: 0, +1, -1, +2, ... map to codes 0, 1, 2, 3, ...
+ fn put_se(&mut self, v: i32) {
+ let code = if v > 0 {
+ 2 * v.unsigned_abs() - 1
+ } else {
+ 2 * v.unsigned_abs()
+ };
+ self.put_ue(code);
+ }
fn finish(mut self) -> Vec {
self.put_bit(1); // rbsp_stop_one_bit
while self.nbits != 0 {
@@ -1655,6 +1776,220 @@ mod tests {
assert_eq!(sps_dimensions(&hevc), None);
}
+ /// The SPS fields that change how many bits sit in front of the picture
+ /// size. Everything else is fixed by `build_rich_sps_rbsp`.
+ struct SpsFields {
+ profile_idc: u32,
+ chroma_format_idc: u32,
+ /// `(list index, delta_scale values)` for each scaling list sent.
+ /// Empty means `seq_scaling_matrix_present_flag = 0`.
+ scaling_lists: Vec<(usize, Vec)>,
+ pic_order_cnt_type: u32,
+ frame_mbs_only: bool,
+ width_in_mbs: u32,
+ height_in_map_units: u32,
+ /// (left, right, top, bottom) in crop units.
+ crop: Option<(u32, u32, u32, u32)>,
+ }
+
+ impl SpsFields {
+ fn progressive(width_in_mbs: u32, height_in_map_units: u32) -> Self {
+ Self {
+ profile_idc: 100,
+ chroma_format_idc: 1,
+ scaling_lists: Vec::new(),
+ pic_order_cnt_type: 0,
+ frame_mbs_only: true,
+ width_in_mbs,
+ height_in_map_units,
+ crop: None,
+ }
+ }
+ }
+
+ /// SPS RBSP per H.264 7.3.2.1.1, written field by field. The caller
+ /// supplies scaling-list deltas that end each list the way the spec
+ /// does (the reader stops reading once nextScale hits 0).
+ fn build_rich_sps_rbsp(f: &SpsFields) -> Vec {
+ let mut w = BitWriter::new();
+ w.put_bits(f.profile_idc, 8);
+ w.put_bits(0, 8); // constraint flags + reserved
+ w.put_bits(40, 8); // level_idc
+ w.put_ue(0); // seq_parameter_set_id
+ if matches!(f.profile_idc, 100 | 110 | 122 | 244) {
+ w.put_ue(f.chroma_format_idc);
+ if f.chroma_format_idc == 3 {
+ w.put_bit(0); // separate_colour_plane_flag
+ }
+ w.put_ue(0); // bit_depth_luma_minus8
+ w.put_ue(0); // bit_depth_chroma_minus8
+ w.put_bit(0); // qpprime_y_zero_transform_bypass_flag
+ w.put_bit(u32::from(!f.scaling_lists.is_empty()));
+ if !f.scaling_lists.is_empty() {
+ let count = if f.chroma_format_idc == 3 { 12 } else { 8 };
+ for i in 0..count {
+ match f.scaling_lists.iter().find(|(idx, _)| *idx == i) {
+ Some((_, deltas)) => {
+ w.put_bit(1);
+ deltas.iter().for_each(|d| w.put_se(*d));
+ }
+ None => w.put_bit(0),
+ }
+ }
+ }
+ }
+ w.put_ue(0); // log2_max_frame_num_minus4
+ w.put_ue(f.pic_order_cnt_type);
+ match f.pic_order_cnt_type {
+ 0 => w.put_ue(2), // log2_max_pic_order_cnt_lsb_minus4
+ 1 => {
+ w.put_bit(0); // delta_pic_order_always_zero_flag
+ w.put_se(-2); // offset_for_non_ref_pic
+ w.put_se(5); // offset_for_top_to_bottom_field
+ w.put_ue(3); // num_ref_frames_in_pic_order_cnt_cycle
+ [1, -1, 7].iter().for_each(|o| w.put_se(*o));
+ }
+ _ => {}
+ }
+ w.put_ue(4); // max_num_ref_frames
+ w.put_bit(0); // gaps_in_frame_num_value_allowed_flag
+ w.put_ue(f.width_in_mbs - 1);
+ w.put_ue(f.height_in_map_units - 1);
+ w.put_bit(u32::from(f.frame_mbs_only));
+ if !f.frame_mbs_only {
+ w.put_bit(1); // mb_adaptive_frame_field_flag
+ }
+ w.put_bit(1); // direct_8x8_inference_flag
+ match f.crop {
+ Some((left, right, top, bottom)) => {
+ w.put_bit(1);
+ [left, right, top, bottom].iter().for_each(|c| w.put_ue(*c));
+ }
+ None => w.put_bit(0),
+ }
+ w.put_bit(0); // vui_parameters_present_flag
+ w.finish()
+ }
+
+ fn rich_sps_dimensions(f: &SpsFields) -> Option<(u32, u32)> {
+ sps_dimensions(&seq_header_around(&build_rich_sps_rbsp(f), false))
+ }
+
+ #[test]
+ fn sps_dimensions_decodes_real_x264_sequence_headers() {
+ // Real High-profile SPS NALs from x264, not produced by the test
+ // BitWriter, so a bug shared by the writer and the reader can't
+ // cancel out. 1280x720 has no cropping; 1920x1080 is coded as
+ // 1920x1088 with frame_crop_bottom_offset = 4 (x2 for 4:2:0).
+ let sps_720p = [
+ 0x64, 0x00, 0x1F, 0xAC, 0xD9, 0x40, 0x50, 0x05, 0xBB, 0x01, 0x10, 0x00, 0x00, 0x03,
+ 0x00, 0x10, 0x00, 0x00, 0x03, 0x03, 0xC0, 0xF1, 0x83, 0x19, 0x60,
+ ];
+ let sps_1080p = [
+ 0x64, 0x00, 0x28, 0xAC, 0xD9, 0x40, 0x78, 0x02, 0x27, 0xE5, 0xC0, 0x44, 0x00, 0x00,
+ 0x03, 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0xF0, 0x3C, 0x60, 0xC6, 0x58,
+ ];
+ assert_eq!(
+ sps_dimensions(&seq_header_around(&sps_720p, false)),
+ Some((1280, 720))
+ );
+ assert_eq!(
+ sps_dimensions(&seq_header_around(&sps_1080p, true)),
+ Some((1920, 1080))
+ );
+ }
+
+ #[test]
+ fn sps_dimensions_walks_high_profile_scaling_lists() {
+ // Each list must be skipped by exactly the bits it occupies or the
+ // size fields after it decode as garbage. The deltas cover a list
+ // that ends on its first read, one that takes two, both ends of the
+ // legal delta_scale range (-128 and 127), and an 8x8 list that never
+ // hits nextScale = 0 and so reads all 64 entries.
+ let mut f = SpsFields::progressive(120, 68);
+ f.scaling_lists = vec![
+ (0, vec![-8]),
+ (1, vec![3, -11]),
+ (2, vec![127, 121]),
+ (3, vec![-128, 120]),
+ (6, vec![0; 64]),
+ ];
+ f.crop = Some((0, 0, 0, 4));
+ assert_eq!(rich_sps_dimensions(&f), Some((1920, 1080)));
+ }
+
+ #[test]
+ fn sps_dimensions_rejects_out_of_range_scaling_deltas_without_panicking() {
+ // delta_scale is -128..=127 by spec. A hostile SPS can carry up to
+ // +/-(2^31 - 1), and `last_scale + delta + 256` overflowed i32: a
+ // panic in debug, a silently wrong parse in release. Nonsense like
+ // that means the SPS is not one we can size, so the parse gives up.
+ for bad_delta in [128, -129, i32::MAX, -i32::MAX] {
+ let mut f = SpsFields::progressive(120, 68);
+ f.scaling_lists = vec![(0, vec![bad_delta, 0])];
+ assert_eq!(rich_sps_dimensions(&f), None, "delta {bad_delta}");
+ }
+ }
+
+ #[test]
+ fn sps_dimensions_applies_cropping_per_chroma_format() {
+ // Portrait 1080x1920 is coded 1088 wide; crop_right 4 x 2 (4:2:0).
+ let mut portrait = SpsFields::progressive(68, 120);
+ portrait.crop = Some((0, 4, 0, 0));
+ assert_eq!(rich_sps_dimensions(&portrait), Some((1080, 1920)));
+
+ // 4:2:2 crops vertically in single lines: bottom 8 x 1.
+ let mut yuv422 = SpsFields::progressive(120, 68);
+ yuv422.profile_idc = 122;
+ yuv422.chroma_format_idc = 2;
+ yuv422.crop = Some((0, 0, 0, 8));
+ assert_eq!(rich_sps_dimensions(&yuv422), Some((1920, 1080)));
+
+ // 4:4:4 has 12 scaling lists (the last six 8x8) and a crop unit of
+ // one pixel in both directions.
+ let mut yuv444 = SpsFields::progressive(120, 68);
+ yuv444.profile_idc = 244;
+ yuv444.chroma_format_idc = 3;
+ yuv444.scaling_lists = vec![(8, vec![-8]), (11, vec![0; 64])];
+ yuv444.crop = Some((0, 16, 0, 8));
+ assert_eq!(rich_sps_dimensions(&yuv444), Some((1904, 1080)));
+ }
+
+ #[test]
+ fn sps_dimensions_handles_interlaced_and_every_poc_type() {
+ // frame_mbs_only = 0: height counts field pairs (2 x 34 x 16 =
+ // 1088), there is an extra mb_adaptive bit, and the vertical crop
+ // unit doubles: bottom 2 x 2 x 2 = 8 lines. Main profile, so no
+ // High-profile block, with poc type 2 (no extra fields).
+ let mut interlaced = SpsFields::progressive(120, 34);
+ interlaced.profile_idc = 77;
+ interlaced.frame_mbs_only = false;
+ interlaced.pic_order_cnt_type = 2;
+ interlaced.crop = Some((0, 0, 0, 2));
+ assert_eq!(rich_sps_dimensions(&interlaced), Some((1920, 1080)));
+
+ // poc type 1 carries signed offsets and a cycle before the size.
+ let mut poc1 = SpsFields::progressive(40, 30);
+ poc1.profile_idc = 66;
+ poc1.pic_order_cnt_type = 1;
+ assert_eq!(rich_sps_dimensions(&poc1), Some((640, 480)));
+ }
+
+ #[test]
+ fn strip_emulation_prevention_drops_only_the_escape_byte() {
+ // 00 00 03 xx is how an encoder escapes 00 00 0x inside a NAL; the
+ // 03 goes, and the zero count restarts after it.
+ assert_eq!(strip_emulation_prevention(&[0, 0, 3, 1]), [0, 0, 1]);
+ assert_eq!(strip_emulation_prevention(&[0, 0, 3, 3]), [0, 0, 3]);
+ assert_eq!(
+ strip_emulation_prevention(&[0, 0, 3, 0, 0, 3, 0]),
+ [0, 0, 0, 0, 0]
+ );
+ // One zero is not enough to start an escape.
+ assert_eq!(strip_emulation_prevention(&[0, 3, 1]), [0, 3, 1]);
+ assert_eq!(strip_emulation_prevention(&[0x42, 3]), [0x42, 3]);
+ }
+
#[test]
fn sps_dimensions_huge_crop_returns_none_not_panic() {
// A malformed SPS can carry near-u32::MAX frame-crop Exp-Golomb
@@ -1692,28 +2027,305 @@ mod tests {
}
#[test]
- fn sps_and_selection_never_panic_on_fuzz() {
- // Deterministic pseudo-random byte blobs, wrapped as AVC seq
- // headers, pushed through the parser + selector. These run on
- // hostile wire input, so the invariant is simply: never panic.
- let mut seed: u32 = 0x9e3779b9;
- for _ in 0..3000 {
- seed = seed.wrapping_mul(1664525).wrapping_add(1013904223);
- let n = (seed as usize % 48) + 1;
- let mut blob = Vec::with_capacity(n);
- let mut x = seed;
- for _ in 0..n {
- x = x.wrapping_mul(1103515245).wrapping_add(12345);
- blob.push((x >> 16) as u8);
+ fn a_zero_length_nal_at_the_end_of_a_tag_is_not_an_idr_and_does_not_panic() {
+ // A length prefix of 0 as the last 4 bytes passes the "does the NAL
+ // fit" check (4 + 0 <= 4) and the walk then read the NAL header one
+ // byte past the slice. That runs on every published video tag, and
+ // with `panic = "abort"` one malformed frame killed the process.
+ let bare = [0x17, 0x01, 0, 0, 0, 0, 0, 0, 0];
+ assert!(!classify_video_tag(&bare).is_idr);
+
+ let mut after_a_slice = avc_tag(1, &[nal(1, 10)]);
+ after_a_slice.extend_from_slice(&[0, 0, 0, 0]);
+ assert!(!classify_video_tag(&after_a_slice).is_idr);
+
+ // An empty NAL in the middle is skipped, so an IDR after it still counts.
+ let mut idr_after_empty = avc_tag(1, &[]);
+ idr_after_empty.extend_from_slice(&[0, 0, 0, 0]);
+ idr_after_empty.extend_from_slice(&nal(5, 8));
+ assert!(classify_video_tag(&idr_after_empty).is_idr);
+ }
+
+ /// Deterministic LCG for the fuzz tests: fixed seed, no dependency, and a
+ /// failure reproduces from the iteration number alone.
+ struct Lcg(u64);
+
+ impl Lcg {
+ fn next_u32(&mut self) -> u32 {
+ self.0 = self
+ .0
+ .wrapping_mul(6_364_136_223_846_793_005)
+ .wrapping_add(1_442_695_040_888_963_407);
+ (self.0 >> 33) as u32
+ }
+
+ fn below(&mut self, n: u32) -> u32 {
+ self.next_u32() % n
+ }
+
+ fn pick(&mut self, items: &[T]) -> T {
+ items[self.below(items.len() as u32) as usize]
+ }
+
+ /// Zero- and 0xFF-heavy bytes. Uniform bytes almost never form a
+ /// zero length prefix or a long Exp-Golomb run, which are exactly the
+ /// shapes that break parsers.
+ fn byte(&mut self) -> u8 {
+ match self.below(8) {
+ 0..=2 => 0x00,
+ 3 => 0xFF,
+ _ => self.next_u32() as u8,
}
- // Legacy AVC framing so the bytes reach the SPS Exp-Golomb path.
- let mut tag = vec![0x17u8, 0x00, 0, 0, 0];
- tag.extend_from_slice(&blob);
+ }
+
+ fn bytes(&mut self, max_len: u32) -> Vec {
+ let n = self.below(max_len + 1);
+ (0..n).map(|_| self.byte()).collect()
+ }
+ }
+
+ /// Length-prefixed NALs whose declared lengths are sometimes honest,
+ /// sometimes zero and sometimes past the end of the tag.
+ fn fuzz_nal_units(rng: &mut Lcg) -> Vec {
+ let mut out = Vec::new();
+ for _ in 0..rng.below(4) {
+ let body = rng.bytes(12);
+ let declared = match rng.below(4) {
+ 0 => 0,
+ 1 => rng.next_u32(),
+ _ => body.len() as u32,
+ };
+ out.extend_from_slice(&declared.to_be_bytes());
+ out.extend_from_slice(&body);
+ }
+ out
+ }
+
+ /// One tag shaped like the ones OBS sends (legacy AVC/AAC, Enhanced
+ /// single-track and multi-track video/audio) with a random body.
+ fn fuzz_tag(rng: &mut Lcg) -> Vec {
+ let b0 = match rng.below(8) {
+ 0 => 0x17,
+ 1 => 0x27,
+ 2 => 0x90 | rng.below(16) as u8,
+ 3 => 0x96,
+ 4 => 0xA6,
+ 5 => 0x95,
+ 6 => 0xAF,
+ _ => rng.next_u32() as u8,
+ };
+ let mut tag = vec![b0];
+ if rng.below(4) == 0 {
+ tag.extend(rng.bytes(24));
+ return tag;
+ }
+ let fourccs = [
+ *b"avc1", *b"hvc1", *b"hev1", *b"av01", *b"vp09", *b"mp4a", *b"zzzz",
+ ];
+ let multitrack = b0 & 0x80 != 0 && (b0 & 0x0F == 6 || b0 == 0x95);
+ if b0 & 0x80 == 0 {
+ // Legacy: [AVCPacketType][CompositionTime(3)][NALs].
+ tag.push(rng.pick(&[0u8, 1, 2]));
+ tag.extend_from_slice(&[0, 0, rng.byte()]);
+ tag.extend(fuzz_nal_units(rng));
+ } else if !multitrack {
+ tag.extend_from_slice(&rng.pick(&fourccs));
+ tag.extend(rng.bytes(16));
+ } else {
+ let mt_type = rng.below(4) as u8;
+ tag.push((mt_type << 4) | rng.pick(&[0u8, 1, 3, 4]));
+ match mt_type {
+ 0 => {
+ // OneTrack: [FourCC][TrackId][body].
+ tag.extend_from_slice(&rng.pick(&fourccs));
+ tag.push(rng.below(3) as u8);
+ tag.extend(rng.bytes(16));
+ }
+ 1 | 2 => {
+ // ManyTracks: [FourCC] then [TrackId][Size(3)][body]...
+ // ManyTracksManyCodecs: [TrackId][FourCC][Size(3)][body]...
+ if mt_type == 1 {
+ tag.extend_from_slice(&rng.pick(&fourccs));
+ }
+ for _ in 0..=rng.below(3) {
+ tag.push(rng.below(3) as u8);
+ if mt_type == 2 {
+ tag.extend_from_slice(&rng.pick(&fourccs));
+ }
+ let body = rng.bytes(8);
+ let honest = rng.below(4) != 0;
+ let size = if honest {
+ body.len() as u32
+ } else {
+ rng.below(0x100)
+ };
+ tag.extend_from_slice(&size.to_be_bytes()[1..]);
+ tag.extend(body);
+ }
+ }
+ _ => tag.extend(rng.bytes(16)),
+ }
+ }
+ if rng.below(2) == 0 {
+ let keep = rng.below(tag.len() as u32 + 1) as usize;
+ tag.truncate(keep);
+ }
+ tag
+ }
+
+ /// Exp-Golomb values a hostile SPS would use: tiny, around the limits
+ /// the parser enforces, and as large as the 32-bit code allows.
+ fn fuzz_ue(rng: &mut Lcg) -> u32 {
+ rng.pick(&[
+ 0,
+ 1,
+ 2,
+ 3,
+ 7,
+ 8,
+ 67,
+ 119,
+ 255,
+ 1 << 16,
+ 0x7FFF_FFFF,
+ 0xFFFF_FFFD,
+ ])
+ }
+
+ fn fuzz_se(rng: &mut Lcg) -> i32 {
+ rng.pick(&[
+ 0,
+ 1,
+ -1,
+ -8,
+ 127,
+ -128,
+ 128,
+ -129,
+ 1 << 20,
+ i32::MAX,
+ -i32::MAX,
+ ])
+ }
+
+ /// An SPS RBSP built by the grammar, so the fuzz reaches the scaling
+ /// lists and the crop fields instead of dying on the first bad field.
+ fn fuzz_sps_rbsp(rng: &mut Lcg) -> Vec {
+ let mut w = BitWriter::new();
+ w.put_bits(rng.pick(&[66, 77, 100, 110, 122, 244]), 8);
+ w.put_bits(0, 8);
+ w.put_bits(40, 8);
+ w.put_ue(fuzz_ue(rng) % 32);
+ w.put_ue(rng.pick(&[0, 1, 2, 3, 0x7FFF_FFFF]));
+ w.put_bit(rng.below(2));
+ w.put_ue(rng.below(3));
+ w.put_ue(rng.below(3));
+ w.put_bit(0);
+ w.put_bit(1); // seq_scaling_matrix_present_flag
+ for _ in 0..rng.below(13) {
+ w.put_bit(rng.below(2));
+ for _ in 0..rng.below(6) {
+ w.put_se(fuzz_se(rng));
+ }
+ }
+ for _ in 0..rng.below(12) {
+ w.put_ue(fuzz_ue(rng));
+ }
+ let mut rbsp = w.finish();
+ rbsp.truncate(rng.below(rbsp.len() as u32 + 1) as usize);
+ rbsp
+ }
+
+ /// An AVC sequence header with a correct spsLength around `rbsp`, in
+ /// either legacy or OneTrack framing, so the bytes reach the SPS parser.
+ fn seq_header_around(rbsp: &[u8], onetrack: bool) -> Vec {
+ let mut sps = vec![0x67u8];
+ sps.extend_from_slice(rbsp);
+ let mut tag = if onetrack {
+ let mut t = vec![0x96u8, 0x00];
+ t.extend_from_slice(b"avc1");
+ t.push(1);
+ t
+ } else {
+ vec![0x17u8, 0x00, 0, 0, 0]
+ };
+ tag.extend_from_slice(&[1, 100, 0, 40, 0xFF, 0xE1]);
+ tag.extend_from_slice(&(sps.len() as u16).to_be_bytes());
+ tag.extend_from_slice(&sps);
+ tag.push(0);
+ tag
+ }
+
+ #[test]
+ fn every_tag_parser_survives_shape_aware_fuzz() {
+ // Every function here runs on raw publisher bytes, and a release
+ // build aborts on panic, so the floor is "never panic". On top of
+ // that, the invariants the egress pumps rely on must hold for any
+ // input: passthrough is always the exact bytes, and a single-track
+ // tag always reaches a Track(0) destination untouched.
+ let mut rng = Lcg(0x1C_2026);
+ for _ in 0..20_000 {
+ let tag = fuzz_tag(&mut rng);
+ let info = classify_video_tag(&tag);
+ let _ = classify_audio_tag(&tag);
+ let _ = is_primary_video_idr(&tag);
+ let _ = seq_header_track_id(&tag);
+ let _ = audio_seq_header_track_id(&tag);
+ let _ = flatten_multitrack_video(&tag);
+ let _ = seq_header_codec(&tag);
let _ = sps_dimensions(&tag);
- let _ = select_video_bytes(&tag, VideoEgress::Track((seed & 0xFF) as u8));
- let mut m = std::collections::BTreeMap::new();
- m.insert((seed & 0x7) as u8, tag);
- let _ = detect_vertical_primary_track(&m);
+ let target = rng.below(4) as u8;
+ let _ = flatten_multitrack_audio(&tag, target);
+ for egress in [
+ VideoEgress::Track(0),
+ VideoEgress::Track(1),
+ VideoEgress::Track(target),
+ ] {
+ let _ = select_video_bytes(&tag, egress);
+ }
+ for egress in [AudioEgress::Track(0), AudioEgress::Track(1)] {
+ for on_wire in [false, true] {
+ let _ = select_audio_bytes(&tag, egress, on_wire);
+ }
+ }
+ let passthrough = select_video_bytes(&tag, VideoEgress::Passthrough).unwrap();
+ assert_eq!(passthrough.as_ref(), tag.as_slice());
+ let passthrough = select_audio_bytes(&tag, AudioEgress::Passthrough, true).unwrap();
+ assert_eq!(passthrough.as_ref(), tag.as_slice());
+ if !info.is_multitrack {
+ let primary = select_video_bytes(&tag, VideoEgress::Track(0)).unwrap();
+ assert_eq!(primary.as_ref(), tag.as_slice());
+ }
+ }
+ }
+
+ #[test]
+ fn sps_parsing_survives_random_and_grammar_built_fuzz() {
+ // Half the SPSs are raw bytes after a profile byte (High profile
+ // half the time, the one with the extra fields); half are built by
+ // the grammar with extreme field values, which is what reaches the
+ // scaling-list arithmetic. Both are wrapped with a correct spsLength
+ // so the parser, not the framing check, is what gets exercised.
+ let mut rng = Lcg(0x5B5_2026);
+ for i in 0..20_000 {
+ let rbsp = if i % 2 == 0 {
+ let mut raw = vec![if rng.below(2) == 0 {
+ 100
+ } else {
+ rng.next_u32() as u8
+ }];
+ raw.extend(rng.bytes(40));
+ raw
+ } else {
+ fuzz_sps_rbsp(&mut rng)
+ };
+ let tag = seq_header_around(&rbsp, rng.below(2) == 0);
+ if let Some((w, h)) = sps_dimensions(&tag) {
+ assert!(w > 0 && h > 0, "a decoded size is never empty");
+ }
+ let mut headers = std::collections::BTreeMap::new();
+ headers.insert(rng.below(4) as u8, tag);
+ let _ = detect_vertical_primary_track(&headers);
}
}
@@ -1761,8 +2373,7 @@ mod tests {
let kept = select_video_bytes(&track1, VideoEgress::Track(1))
.expect("vertical target track must forward");
- let flat = flatten_multitrack_video(&track1).expect("OneTrack flattens");
- assert_eq!(kept.as_ref(), flat.as_slice());
+ assert_eq!(kept.as_ref(), flat_hvc1_keyframe().as_slice());
assert!(
select_video_bytes(&track2, VideoEgress::Track(1)).is_none(),
@@ -1868,6 +2479,10 @@ mod tests {
// EB ladder rungs 1..=4 each carry their own IDR cadence but
// must NOT be picked as a cut anchor - if the cut lands here
// the legacy decoder on the destination has no reference.
+ // Twitch Dual Format's vertical canvas is one of these TrackIds,
+ // so cuts are anchored to the horizontal canvas only; a vertical
+ // cut is clean because OBS aligns both canvases' keyframes. If
+ // cuts ever become canvas-aware, this is the guard that must flip.
for track in 1u8..=4 {
let tag = enhanced_rtmp_onetrack_video_bytes_track(track);
assert!(
@@ -1877,30 +2492,6 @@ mod tests {
}
}
- #[test]
- fn is_primary_video_idr_anchors_dual_format_cuts_to_horizontal_canvas() {
- // Dual Format: the cut-point index is built only from the
- // horizontal primary (TrackId 0); the vertical canvas's primary
- // (a nonzero TrackId) is NOT a cut anchor. That means a Cut lands
- // cleanly on the vertical canvas ONLY IF OBS aligned both
- // canvases' keyframes to the same input timestamp - which it does
- // when both encoders share keyint, but that is an OBS behaviour we
- // can't prove from inside this crate (it needs one real capture).
- // This test pins our half of the contract: cuts are anchored to
- // the horizontal canvas. If a future change must make cuts
- // canvas-aware, this is the regression guard that has to flip.
- let horizontal_idr = enhanced_rtmp_onetrack_video_bytes_track(0);
- let vertical_idr = enhanced_rtmp_onetrack_video_bytes_track(1);
- assert!(
- is_primary_video_idr(&horizontal_idr),
- "horizontal canvas IDR must be a cut anchor"
- );
- assert!(
- !is_primary_video_idr(&vertical_idr),
- "vertical canvas IDR must not (yet) be an independent cut anchor"
- );
- }
-
#[test]
fn is_primary_video_idr_rejects_enhanced_p_frame() {
// Enhanced-RTMP single-track inter-frame: IsEx | FrameType=2 |
@@ -2017,6 +2608,8 @@ mod tests {
);
assert!(!info.is_multitrack);
assert_eq!(info.codec, AudioCodec::Aac);
+ // sink.rs asks through the shim, which must agree.
+ assert!(is_aac_seq_header(&tag));
}
#[test]
@@ -2084,8 +2677,8 @@ mod tests {
AudioEgress::Track(0),
AudioEgress::Track(1),
] {
- let out =
- select_audio_bytes(&legacy_aac, egress).expect("single-track audio must forward");
+ let out = select_audio_bytes(&legacy_aac, egress, false)
+ .expect("single-track audio must forward");
assert_eq!(out.as_ref(), legacy_aac.as_slice());
assert!(matches!(out, std::borrow::Cow::Borrowed(_)));
}
@@ -2097,18 +2690,37 @@ mod tests {
// track (OneTrack TrackId 0) still gets audio: the requested track is
// missing, so it falls back to the live track 0, flattened to legacy.
let live_only = enhanced_audio_onetrack(0, 1);
- let out = select_audio_bytes(&live_only, AudioEgress::Track(1))
+ let out = select_audio_bytes(&live_only, AudioEgress::Track(1), false)
.expect("must fall back to the live track, not go silent");
assert_eq!(out.as_ref(), &[0xAF, 0x01, 0x12, 0x10, 0x56, 0xe5]);
}
+ /// OBS sends its live track as plain AAC and the VOD / clean track as a
+ /// OneTrack multi-track tag. Once the clean track is on the wire, a
+ /// "Track 2" destination must get only it: forwarding the live tags too
+ /// interleaved both tracks, copyrighted live mix included.
+ #[test]
+ fn a_track_two_destination_never_gets_the_live_track_once_the_second_is_sent() {
+ let live_legacy = vec![0xaf, 0x01, 0x12, 0x10, 0x56];
+ let live_multitrack = enhanced_audio_onetrack(0, 1);
+ let clean = enhanced_audio_onetrack(1, 1);
+ let egress = AudioEgress::Track(1);
+ assert!(select_audio_bytes(&live_legacy, egress, true).is_none());
+ assert!(select_audio_bytes(&live_multitrack, egress, true).is_none());
+ let out = select_audio_bytes(&clean, egress, true).expect("the clean track goes out");
+ assert_eq!(out.as_ref(), &[0xAF, 0x01, 0x12, 0x10, 0x56, 0xe5]);
+ // The live track's own destinations are unaffected.
+ assert!(select_audio_bytes(&live_legacy, AudioEgress::Track(0), true).is_some());
+ assert!(select_audio_bytes(&clean, AudioEgress::Track(0), true).is_none());
+ }
+
#[test]
fn select_audio_bytes_passes_multitrack_through_for_twitch() {
// Twitch destination (Passthrough): every track's bytes flow
// through bit-faithfully so Twitch's VOD-audio slot keeps its feed.
for track in 0u8..=3 {
let tag = enhanced_audio_onetrack(track, 1);
- let out = select_audio_bytes(&tag, AudioEgress::Passthrough)
+ let out = select_audio_bytes(&tag, AudioEgress::Passthrough, true)
.unwrap_or_else(|| panic!("twitch must forward track {track}"));
assert_eq!(out.as_ref(), tag.as_slice());
}
@@ -2122,7 +2734,7 @@ mod tests {
for track in 1u8..=3 {
let tag = enhanced_audio_onetrack(track, 1);
assert!(
- select_audio_bytes(&tag, AudioEgress::Track(0)).is_none(),
+ select_audio_bytes(&tag, AudioEgress::Track(0), true).is_none(),
"TrackId {track} must be dropped when the dest wants Track(0)",
);
}
@@ -2133,12 +2745,12 @@ mod tests {
// The clean/second audio track routed to a single-track dest is
// flattened to legacy AAC (0xAF ...), the universally-accepted form.
let live = enhanced_audio_onetrack(0, 1);
- let out = select_audio_bytes(&live, AudioEgress::Track(0))
+ let out = select_audio_bytes(&live, AudioEgress::Track(0), true)
.expect("track 0 must reach a Track(0) destination");
assert_eq!(out.as_ref(), &[0xAF, 0x01, 0x12, 0x10, 0x56, 0xe5]);
let clean = enhanced_audio_onetrack(1, 1);
- let out = select_audio_bytes(&clean, AudioEgress::Track(1))
+ let out = select_audio_bytes(&clean, AudioEgress::Track(1), true)
.expect("track 1 must reach a Track(1) destination");
assert_eq!(out.as_ref(), &[0xAF, 0x01, 0x12, 0x10, 0x56, 0xe5]);
}
@@ -2193,4 +2805,55 @@ mod tests {
vec![0x91, b'O', b'p', b'u', b's', 0x33, 0x44],
);
}
+
+ /// Enhanced RTMP: only coded frames carry a picture. A SequenceEnd
+ /// (the encoder stopping) flagged as a keyframe was taken for one - a
+ /// cut point, and crash protection's held "last keyframe" - and AV1's
+ /// MPEG2TSSequenceStart config was never cached as a header.
+ #[test]
+ fn only_coded_frames_count_as_keyframes() {
+ let tag = |first: u8| vec![first, b'h', b'v', b'c', b'1', 0, 0, 0, 1];
+ // IsEx | FrameType 1 (key) | PacketType.
+ assert!(classify_video_tag(&tag(0x91)).is_idr, "CodedFrames");
+ assert!(classify_video_tag(&tag(0x93)).is_idr, "CodedFramesX");
+ let end = classify_video_tag(&tag(0x92));
+ assert!(!end.is_idr && !end.is_seq_header, "SequenceEnd");
+ let ts_config = classify_video_tag(&tag(0x95));
+ assert!(
+ !ts_config.is_idr && ts_config.is_seq_header,
+ "MPEG2TSSequenceStart"
+ );
+
+ // Multitrack: the nested packet type decides.
+ let multitrack = |nested: u8| vec![0x96, nested, b'a', b'v', b'c', b'1', 0, 0, 0, 1];
+ assert!(classify_video_tag(&multitrack(0x01)).is_idr);
+ assert!(
+ !classify_video_tag(&multitrack(0x02)).is_idr,
+ "nested SequenceEnd"
+ );
+ assert!(classify_video_tag(&multitrack(0x05)).is_seq_header);
+ }
+
+ /// AVC allows 1- or 2-byte NAL lengths (lengthSizeMinusOne). A stream
+ /// using them had no cut points; OBS's 4-byte lengths read as before.
+ #[test]
+ fn keyframes_are_found_with_short_nal_lengths() {
+ let legacy = |nals: &[u8]| {
+ let mut tag = vec![0x17, 1, 0, 0, 0];
+ tag.extend_from_slice(nals);
+ tag
+ };
+ // 2-byte lengths: an SEI (type 6), then an IDR slice (type 5).
+ let two = legacy(&[0, 2, 0x06, 0xAA, 0, 3, 0x65, 0xBB, 0xCC]);
+ assert!(classify_video_tag(&two).is_idr);
+ // 1-byte lengths.
+ let one = legacy(&[2, 0x06, 0xAA, 2, 0x65, 0xBB]);
+ assert!(classify_video_tag(&one).is_idr);
+ // A P-slice with short lengths is not a keyframe.
+ let p_slice = legacy(&[0, 2, 0x41, 0xAA]);
+ assert!(!classify_video_tag(&p_slice).is_idr);
+ // 4-byte lengths, as OBS sends them, unchanged.
+ let four = legacy(&[0, 0, 0, 2, 0x65, 0xBB]);
+ assert!(classify_video_tag(&four).is_idr);
+ }
}
diff --git a/src/https.rs b/src/https.rs
index 617d999..181d128 100644
--- a/src/https.rs
+++ b/src/https.rs
@@ -23,7 +23,8 @@
//!
//! Use [`https_agent`] everywhere we need to call out to an HTTPS URL.
-use ureq::tls::{TlsConfig, TlsProvider};
+use std::time::Duration;
+use ureq::tls::{RootCerts, TlsConfig, TlsProvider};
use ureq::Agent;
/// Build a ready-to-use ureq `Agent` with native-tls selected as the
@@ -31,10 +32,29 @@ use ureq::Agent;
/// access to non-2xx response bodies. Native-tls uses Windows schannel
/// under the hood - no rustls + ring dependency chain.
pub fn https_agent() -> Agent {
+ agent(None)
+}
+
+/// `https_agent` whose every call gives up after `timeout`, connect to last
+/// byte. For calls made from a loop that must keep going: a stalled server
+/// would otherwise hold it forever.
+pub fn https_agent_with_timeout(timeout: Duration) -> Agent {
+ agent(Some(timeout))
+}
+
+fn agent(timeout: Option) -> Agent {
Agent::config_builder()
+ .timeout_global(timeout)
.tls_config(
TlsConfig::builder()
.provider(TlsProvider::NativeTls)
+ // The OS certificate store (schannel on Windows). ureq's
+ // default hands native-tls a bundled root list instead, and
+ // schannel then refuses any server whose chain it builds to
+ // a root outside that list: Discord's does, so every
+ // webhook failed with "unable to find any user-specified
+ // roots in the final cert chain".
+ .root_certs(RootCerts::PlatformVerifier)
.build(),
)
// Non-2xx as Ok(resp): the Twitch proxy needs the 4xx body to
@@ -44,3 +64,20 @@ pub fn https_agent() -> Agent {
.build()
.into()
}
+
+#[cfg(test)]
+mod tests {
+ /// Network smoke test for the TLS setup; run by hand with `--ignored`.
+ #[test]
+ #[ignore]
+ fn reaches_real_https_hosts() {
+ for url in [
+ "https://discord.com/api/v10/gateway",
+ "https://id.twitch.tv/oauth2/validate",
+ "https://ntfy.sh/v1/health",
+ ] {
+ let r = super::https_agent().get(url).call();
+ assert!(r.is_ok(), "{url}: {:?}", r.err());
+ }
+ }
+}
diff --git a/src/integrations/activity.rs b/src/integrations/activity.rs
new file mode 100644
index 0000000..5411b8b
--- /dev/null
+++ b/src/integrations/activity.rs
@@ -0,0 +1,407 @@
+//! The chat activity trigger's view of chat: the last few minutes of
+//! messages, and how a trigger's rules read them right now.
+//!
+//! "Busier than normal" compares against this channel's own chat, so one
+//! setting works for twenty viewers and for twenty thousand: normal is the
+//! average speed over the last ten minutes, leaving out the window being
+//! judged, and never below `MIN_NORMAL` so three lines in a quiet chat
+//! don't count as a hype moment.
+
+use super::model::{ChatActivity, RuleKind};
+use super::twitch::irc::ChatMessage;
+use std::collections::VecDeque;
+use std::time::{Duration, Instant};
+
+/// How far back "normal" looks.
+const HISTORY: Duration = Duration::from_secs(600);
+/// "Normal" needs this much chat history before it means anything.
+const MIN_HISTORY: Duration = Duration::from_secs(60);
+/// Messages kept at most: a very busy chat keeps less than ten minutes.
+const MAX_KEPT: usize = 6000;
+/// The least "normal" can be, in messages per window.
+const MIN_NORMAL: f64 = 2.0;
+/// A share of messages needs a few messages to be a share at all.
+const MIN_FOR_SHARE: usize = 3;
+/// Characters of a message kept for word matching.
+const MAX_TEXT: usize = 200;
+
+struct Msg {
+ at: Instant,
+ login: String,
+ is_sub: bool,
+ is_vip: bool,
+ is_mod: bool,
+ is_broadcaster: bool,
+ text: String,
+}
+
+/// Recent chat, shared by every chat activity trigger.
+#[derive(Default)]
+pub struct ChatWindow {
+ msgs: VecDeque,
+ /// When chat was first seen: until `MIN_HISTORY` later, "normal" is
+ /// unknown.
+ since: Option,
+}
+
+/// How chat reads against one trigger's rules.
+#[derive(Clone, Debug, Default, PartialEq)]
+pub struct Reading {
+ pub messages: usize,
+ /// Messages per window on a normal stretch.
+ pub normal: f64,
+ /// `messages` over `normal`.
+ pub busier: f64,
+ pub chatters: usize,
+ /// Percent of messages containing one of a words rule's words: the
+ /// highest of the rules, for `{word_share}`.
+ pub word_share: f64,
+ /// Per rule, in order: its own share (0 for rules that aren't words).
+ pub shares: Vec,
+ /// The most used word of any words rule, or empty.
+ pub top_word: String,
+ /// Whether "normal" is known yet.
+ pub ready: bool,
+ /// One per rule, in order.
+ pub passing: Vec,
+ pub fires: bool,
+}
+
+impl ChatWindow {
+ pub fn push(&mut self, msg: &ChatMessage, now: Instant) {
+ self.since.get_or_insert(now);
+ if self.msgs.len() >= MAX_KEPT {
+ self.msgs.pop_front();
+ }
+ self.msgs.push_back(Msg {
+ at: now,
+ login: msg.user_login.clone(),
+ is_sub: msg.is_sub,
+ is_vip: msg.is_vip,
+ is_mod: msg.is_mod,
+ is_broadcaster: msg.is_broadcaster,
+ text: msg.text.to_lowercase().chars().take(MAX_TEXT).collect(),
+ });
+ self.trim(now);
+ }
+
+ fn trim(&mut self, now: Instant) {
+ while self
+ .msgs
+ .front()
+ .is_some_and(|m| now.duration_since(m.at) > HISTORY)
+ {
+ self.msgs.pop_front();
+ }
+ }
+
+ /// How chat reads against `rules` at `now`. The window is kept within
+ /// what can be judged: the dashboard's meter sends rules nobody
+ /// validated yet.
+ pub fn read(&self, rules: &ChatActivity, now: Instant) -> Reading {
+ let window =
+ Duration::from_millis(rules.window_ms.clamp(1_000, HISTORY.as_millis() as u64));
+ let counts = |m: &Msg| {
+ m.is_broadcaster
+ || rules.roles.everyone
+ || (rules.roles.mods && m.is_mod)
+ || (rules.roles.vips && m.is_vip)
+ || (rules.roles.subs && m.is_sub)
+ };
+ let recent: Vec<&Msg> = self
+ .msgs
+ .iter()
+ .filter(|m| now.duration_since(m.at) <= window && counts(m))
+ .collect();
+ let older = self
+ .msgs
+ .iter()
+ .filter(|m| now.duration_since(m.at) > window && counts(m))
+ .count();
+ let history = self
+ .since
+ .map_or(Duration::ZERO, |s| now.duration_since(s))
+ .min(HISTORY);
+ let ready = history >= MIN_HISTORY + window;
+ let normal = if ready {
+ let windows = (history - window).as_secs_f64() / window.as_secs_f64();
+ (older as f64 / windows).max(MIN_NORMAL)
+ } else {
+ 0.0
+ };
+ let messages = recent.len();
+ let mut chatters: Vec<&str> = recent.iter().map(|m| m.login.as_str()).collect();
+ chatters.sort_unstable();
+ chatters.dedup();
+ // Each words rule counts its own words; the top word is chat's
+ // favourite out of all of them.
+ let shares: Vec = rules
+ .rules
+ .iter()
+ .map(|r| match r.kind {
+ RuleKind::Words => word_stats(&recent, &word_list(&r.words)).0,
+ _ => 0.0,
+ })
+ .collect();
+ let all_words = word_list(
+ &rules
+ .rules
+ .iter()
+ .filter(|r| r.kind == RuleKind::Words)
+ .map(|r| r.words.as_str())
+ .collect::>()
+ .join(","),
+ );
+ let top_word = word_stats(&recent, &all_words).1;
+ let word_share = shares.iter().copied().fold(0.0, f64::max);
+ let busier = if normal > 0.0 {
+ messages as f64 / normal
+ } else {
+ 0.0
+ };
+ let passing: Vec = rules
+ .rules
+ .iter()
+ .zip(&shares)
+ .map(|(r, share)| match r.kind {
+ RuleKind::Messages => messages as f64 >= r.value,
+ RuleKind::Busier => ready && busier >= r.value,
+ RuleKind::Chatters => chatters.len() as f64 >= r.value,
+ RuleKind::Words => *share >= r.value,
+ })
+ .collect();
+ let fires = !passing.is_empty()
+ && if rules.match_all {
+ passing.iter().all(|p| *p)
+ } else {
+ passing.iter().any(|p| *p)
+ };
+ Reading {
+ messages,
+ normal,
+ busier,
+ chatters: chatters.len(),
+ word_share,
+ shares,
+ top_word,
+ ready,
+ passing,
+ fires,
+ }
+ }
+}
+
+/// The comma separated words of a words rule, lowercase, each once.
+fn word_list(words: &str) -> Vec {
+ let mut list: Vec = Vec::new();
+ for word in words.split(',').map(|w| w.trim().to_lowercase()) {
+ if !word.is_empty() && !list.contains(&word) {
+ list.push(word);
+ }
+ }
+ list
+}
+
+/// The share of messages holding any of `words`, and the most used one.
+fn word_stats(recent: &[&Msg], words: &[String]) -> (f64, String) {
+ if words.is_empty() || recent.len() < MIN_FOR_SHARE {
+ return (0.0, String::new());
+ }
+ let mut hits = vec![0usize; words.len()];
+ let mut matching = 0;
+ for m in recent {
+ let tokens: Vec<&str> = m
+ .text
+ .split(|c: char| !c.is_alphanumeric())
+ .filter(|t| !t.is_empty())
+ .collect();
+ let mut any = false;
+ for (i, w) in words.iter().enumerate() {
+ if says(&m.text, &tokens, w) {
+ hits[i] += 1;
+ any = true;
+ }
+ }
+ matching += usize::from(any);
+ }
+ // Reversed, so a tie goes to the word listed first.
+ let top = hits
+ .iter()
+ .enumerate()
+ .rev()
+ .filter(|(_, n)| **n > 0)
+ .max_by_key(|(_, n)| **n)
+ .map(|(i, _)| words[i].clone())
+ .unwrap_or_default();
+ (matching as f64 * 100.0 / recent.len() as f64, top)
+}
+
+/// Whether a message says `word`. A phrase ("let's go") is looked for as
+/// written; a single word must be a whole word, or start one when it has
+/// three letters or more ("pog" in "poggers", "clip" in "clipped"), so
+/// "w" never matches "wow".
+fn says(text: &str, tokens: &[&str], word: &str) -> bool {
+ if word.contains(char::is_whitespace) {
+ return text.contains(word);
+ }
+ let prefix_ok = word.chars().count() >= 3;
+ tokens
+ .iter()
+ .any(|t| *t == word || (prefix_ok && t.starts_with(word)))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::super::model::{ActivityRule, Roles};
+ use super::*;
+
+ fn msg(user: &str, text: &str) -> ChatMessage {
+ ChatMessage {
+ user_login: user.into(),
+ text: text.into(),
+ ..ChatMessage::default()
+ }
+ }
+
+ fn rules(kind: RuleKind, value: f64, words: &str) -> ChatActivity {
+ ChatActivity {
+ window_ms: 10_000,
+ match_all: true,
+ rules: vec![ActivityRule {
+ kind,
+ value,
+ words: words.into(),
+ }],
+ roles: Roles::EVERYONE,
+ only_live: true,
+ }
+ }
+
+ /// A steady chat for `minutes`, one message every 10 s, then a burst.
+ fn chat_then_burst(minutes: u64, burst: usize) -> (ChatWindow, Instant) {
+ let mut w = ChatWindow::default();
+ let t0 = Instant::now();
+ for s in (0..minutes * 60).step_by(10) {
+ w.push(&msg("regular", "hello"), t0 + Duration::from_secs(s));
+ }
+ let now = t0 + Duration::from_secs(minutes * 60);
+ for n in 0..burst {
+ w.push(&msg(&format!("v{n}"), "CLIP IT pog"), now);
+ }
+ (w, now)
+ }
+
+ #[test]
+ fn busier_compares_with_the_channels_own_pace() {
+ let (w, now) = chat_then_burst(5, 12);
+ let r = w.read(&rules(RuleKind::Busier, 3.0, ""), now);
+ assert!(r.ready);
+ assert_eq!(r.normal, MIN_NORMAL, "one a window is below the floor");
+ assert!(r.busier >= 6.0, "{r:?}");
+ assert!(r.fires);
+ }
+
+ #[test]
+ fn busier_waits_for_enough_history() {
+ let (w, now) = chat_then_burst(0, 30);
+ let r = w.read(&rules(RuleKind::Busier, 2.0, ""), now);
+ assert!(!r.ready);
+ assert!(!r.fires);
+ }
+
+ #[test]
+ fn words_count_a_share_of_messages_and_name_the_top_one() {
+ let (w, now) = chat_then_burst(2, 6);
+ let r = w.read(&rules(RuleKind::Words, 50.0, "clip, pog, lul"), now);
+ assert_eq!(r.messages, 7, "the burst plus one regular line");
+ assert!(r.word_share > 80.0, "{r:?}");
+ assert_eq!(r.top_word, "clip");
+ assert!(r.fires);
+ }
+
+ #[test]
+ fn short_words_match_whole_words_only() {
+ let tokens = |t: &'static str| t.split(' ').collect::>();
+ assert!(says("w", &tokens("w"), "w"));
+ assert!(!says("wow what", &tokens("wow what"), "w"));
+ assert!(says("poggers", &tokens("poggers"), "pog"));
+ assert!(says("lets go boys", &tokens("lets go boys"), "lets go"));
+ assert!(!says("clap", &tokens("clap"), "clip"));
+ }
+
+ #[test]
+ fn any_window_the_meter_sends_is_safe() {
+ let (w, now) = chat_then_burst(1, 3);
+ for window_ms in [0, u64::MAX] {
+ let mut r = rules(RuleKind::Busier, 2.0, "");
+ r.window_ms = window_ms;
+ let _ = w.read(&r, now);
+ }
+ }
+
+ #[test]
+ fn each_words_rule_counts_its_own_words() {
+ let (w, now) = chat_then_burst(2, 6);
+ let mut both = rules(RuleKind::Words, 50.0, "clip");
+ both.rules.push(ActivityRule {
+ kind: RuleKind::Words,
+ value: 50.0,
+ words: "lul".into(),
+ });
+ let r = w.read(&both, now);
+ assert_eq!(r.passing, vec![true, false], "nobody said lul");
+ assert!(!r.fires, "all rules must pass");
+ assert_eq!(r.top_word, "clip");
+ }
+
+ #[test]
+ fn one_message_is_never_a_share() {
+ let mut w = ChatWindow::default();
+ let now = Instant::now();
+ w.push(&msg("a", "clip"), now);
+ let r = w.read(&rules(RuleKind::Words, 10.0, "clip"), now);
+ assert_eq!(r.word_share, 0.0);
+ assert!(!r.fires);
+ }
+
+ #[test]
+ fn chatters_are_people_not_lines() {
+ let mut w = ChatWindow::default();
+ let now = Instant::now();
+ for _ in 0..20 {
+ w.push(&msg("spammer", "pog"), now);
+ }
+ let r = w.read(&rules(RuleKind::Chatters, 3.0, ""), now);
+ assert_eq!((r.messages, r.chatters), (20, 1));
+ assert!(!r.fires);
+ }
+
+ #[test]
+ fn any_fires_on_one_rule_all_needs_every_rule() {
+ let (w, now) = chat_then_burst(2, 5);
+ let mut both = rules(RuleKind::Messages, 3.0, "");
+ both.rules.push(ActivityRule {
+ kind: RuleKind::Chatters,
+ value: 50.0,
+ words: String::new(),
+ });
+ assert!(!w.read(&both, now).fires);
+ both.match_all = false;
+ let r = w.read(&both, now);
+ assert_eq!(r.passing, vec![true, false]);
+ assert!(r.fires);
+ }
+
+ #[test]
+ fn only_the_chosen_roles_count() {
+ let mut w = ChatWindow::default();
+ let now = Instant::now();
+ let mut m = msg("mod1", "x");
+ m.is_mod = true;
+ w.push(&m, now);
+ w.push(&msg("viewer", "x"), now);
+ let mut mods = rules(RuleKind::Messages, 2.0, "");
+ mods.roles = Roles::MODS;
+ assert_eq!(w.read(&mods, now).messages, 1);
+ }
+}
diff --git a/src/integrations/alerts.rs b/src/integrations/alerts.rs
new file mode 100644
index 0000000..49801d0
--- /dev/null
+++ b/src/integrations/alerts.rs
@@ -0,0 +1,133 @@
+//! What "Show on stream" steps put on the alerts browser source.
+//!
+//! The board keeps the last few alerts with an increasing id. The page in
+//! OBS (`/alerts`) reads them over its own event stream and plays each new
+//! id once, in order, so two alerts close together queue instead of
+//! overlapping. Only what a step chose to show is here: it is on stream
+//! anyway, which is why the page needs no login.
+
+use crate::json::{self, Value};
+use crate::sync::Mutex;
+use std::collections::VecDeque;
+use std::time::{Duration, Instant};
+
+/// Alerts kept for a page that (re)connects. Older ones are never replayed.
+const KEEP: usize = 8;
+/// How long an alert stays on the board: a page that reconnects after this
+/// no longer plays it.
+const FRESH: Duration = Duration::from_secs(30);
+pub const MIN_SECONDS: u64 = 2;
+pub const MAX_SECONDS: u64 = 30;
+const MAX_TITLE: usize = 80;
+const MAX_TEXT: usize = 300;
+
+struct Alert {
+ id: u64,
+ title: String,
+ text: String,
+ ms: u64,
+ at: Instant,
+}
+
+pub struct Board {
+ /// Changes with every start of the app, so a page that outlives a
+ /// restart knows the ids started over.
+ boot: String,
+ inner: Mutex,
+}
+
+#[derive(Default)]
+struct Inner {
+ last_id: u64,
+ alerts: VecDeque,
+}
+
+impl Board {
+ pub fn new() -> Board {
+ Board {
+ boot: crate::crypto::random_token()[..8].to_string(),
+ inner: Mutex::new(Inner::default()),
+ }
+ }
+
+ /// Put an alert up for `seconds` (clamped to what reads well).
+ pub fn show(&self, title: &str, text: &str, seconds: u64) {
+ let mut inner = self.inner.lock();
+ inner.last_id += 1;
+ let alert = Alert {
+ id: inner.last_id,
+ title: one_line(title, MAX_TITLE),
+ text: one_line(text, MAX_TEXT),
+ ms: seconds.clamp(MIN_SECONDS, MAX_SECONDS) * 1000,
+ at: Instant::now(),
+ };
+ if inner.alerts.len() >= KEEP {
+ inner.alerts.pop_front();
+ }
+ inner.alerts.push_back(alert);
+ }
+
+ /// The fresh alerts, oldest first. Unchanged between alerts, so the
+ /// page's event stream only sends when something new is up.
+ pub fn to_json(&self) -> String {
+ let mut inner = self.inner.lock();
+ inner.alerts.retain(|a| a.at.elapsed() < FRESH);
+ let alerts = Value::Arr(
+ inner
+ .alerts
+ .iter()
+ .map(|a| {
+ json::obj([
+ ("id", Value::Num(a.id as f64)),
+ ("title", json::str(&a.title)),
+ ("text", json::str(&a.text)),
+ ("ms", Value::Num(a.ms as f64)),
+ ])
+ })
+ .collect(),
+ );
+ json::obj([("boot", json::str(&self.boot)), ("alerts", alerts)]).to_json()
+ }
+}
+
+fn one_line(text: &str, max: usize) -> String {
+ let flat: String = text
+ .chars()
+ .map(|c| if c.is_control() { ' ' } else { c })
+ .take(max)
+ .collect();
+ flat.trim().to_string()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn alerts_queue_with_rising_ids_and_sane_lengths() {
+ let board = Board::new();
+ board.show("Clip!", "by Ana\r\nnice", 1);
+ board.show("", &"x".repeat(900), 99);
+ let doc = json::parse(&board.to_json()).unwrap();
+ assert_eq!(doc.str_or("boot", "").len(), 8);
+ let list = doc.get("alerts").and_then(Value::as_array).unwrap();
+ assert_eq!(list.len(), 2);
+ assert_eq!(list[0].str_or("text", ""), "by Ana nice");
+ assert_eq!(list[0].u64_or("ms", 0), MIN_SECONDS * 1000);
+ assert_eq!(list[1].u64_or("ms", 0), MAX_SECONDS * 1000);
+ assert_eq!(list[1].str_or("text", "").len(), MAX_TEXT);
+ assert!(list[1].u64_or("id", 0) > list[0].u64_or("id", 0));
+ }
+
+ #[test]
+ fn only_the_last_few_are_kept() {
+ let board = Board::new();
+ for n in 0..20 {
+ board.show("", &n.to_string(), 5);
+ }
+ let doc = json::parse(&board.to_json()).unwrap();
+ let list = doc.get("alerts").and_then(Value::as_array).unwrap();
+ assert_eq!(list.len(), KEEP);
+ assert_eq!(list[0].str_or("text", ""), "12");
+ }
+}
diff --git a/src/integrations/api.rs b/src/integrations/api.rs
new file mode 100644
index 0000000..e87eab2
--- /dev/null
+++ b/src/integrations/api.rs
@@ -0,0 +1,1381 @@
+//! The dashboard's integrations API. Every route here is admin-only (the
+//! web layer's access gate defaults to that) except `/hooks/`, which
+//! is how outside apps trigger a "web call" integration and is guarded by
+//! its secret token instead.
+//!
+//! Requests and answers are JSON. Settings writes follow the web layer's
+//! pattern: take the write lock, clone, change, validate, save, publish.
+
+use super::event;
+use super::model::{
+ valid_discord_webhook, valid_id, DiscordChannel, Integration, PhoneConnection, Roles, Trigger,
+ MAX_INTEGRATIONS,
+};
+use super::{engine, presets, recipe, runner, twitch::Which};
+use crate::config::Settings;
+use crate::controller::Controller;
+use crate::json::{self, Value};
+use std::path::Path;
+use std::sync::Arc;
+use tokio::sync::watch;
+
+pub type Reply = (&'static str, &'static str, String);
+
+fn ok(v: Value) -> Reply {
+ ("200 OK", "application/json", v.to_json())
+}
+
+fn fail(message: impl Into) -> Reply {
+ let message = capitalize(&message.into());
+ (
+ "400 Bad Request",
+ "application/json",
+ json::obj([("ok", Value::Bool(false)), ("error", json::str(message))]).to_json(),
+ )
+}
+
+fn capitalize(s: &str) -> String {
+ let mut chars = s.chars();
+ match chars.next() {
+ Some(first) => first.to_uppercase().chain(chars).collect(),
+ None => String::new(),
+ }
+}
+
+fn done() -> Reply {
+ ok(json::obj([("ok", Value::Bool(true))]))
+}
+
+/// Why a Program or File step can't be set up from this request.
+const LOCAL_STEPS_LOCKED: &str = "running programs and writing files needs a dashboard password \
+ while the dashboard is open to your network (System > Security)";
+
+pub fn new_id() -> String {
+ format!("i{}", &crate::crypto::random_token()[..12])
+}
+
+/// Handle `/integrations*`, `/connections/*`, `/twitch/*`. `None` when the
+/// path is not ours.
+///
+/// `local_steps_allowed` is false while the dashboard is open to the
+/// network with no password (see `web::may_set_up_local_steps`). Anyone on
+/// the network could then make the request, so it may not add, change or
+/// switch on a step that runs a program or writes a file: from chat, that
+/// would run anything on the PC.
+pub async fn route(
+ method: &str,
+ path: &str,
+ body: &str,
+ ctrl: &Arc,
+ settings: &Arc>,
+ cfg_path: &Path,
+ local_steps_allowed: bool,
+) -> Option {
+ let handle = ctrl.integrations()?.clone();
+ let req = || json::parse(body).unwrap_or(Value::Null);
+ let reply = match (method, path) {
+ ("GET", "/integrations") => ok(overview(&handle, &settings.borrow(), local_steps_allowed)),
+ ("GET", "/integrations/activity") => ok(json::obj([
+ ("activity", handle.activity_json()),
+ ("stats", handle.stats_json()),
+ ("twitch", handle.twitch.status_json()),
+ ])),
+ ("POST", "/integrations/save") => save(&req(), settings, cfg_path, local_steps_allowed),
+ ("POST", "/integrations/add") => add(&req(), settings, cfg_path, local_steps_allowed),
+ ("POST", "/integrations/toggle") => toggle(&req(), settings, cfg_path, local_steps_allowed),
+ ("POST", "/integrations/delete") => {
+ let id = req().str_or("id", "").to_string();
+ update(settings, cfg_path, |s| {
+ s.integrations.retain(|i| i.id != id);
+ Ok(())
+ })
+ }
+ ("POST", "/integrations/duplicate") => duplicate(&req(), settings, cfg_path),
+ ("POST", "/integrations/test") => test(&req(), &handle).await,
+ // The live meter of a chat activity trigger being tuned.
+ ("POST", "/integrations/meter") => match req().get("trigger").map(meter_rules) {
+ Some(Ok(rules)) => ok(handle.chat_meter(&rules)),
+ Some(Err(e)) => fail(e),
+ None => fail("nothing to measure"),
+ },
+ ("GET", "/integrations/obs") => ok(handle.obs_status()),
+ ("POST", "/integrations/fetch") => fetch(&req()).await,
+ ("POST", "/integrations/convert") => convert(&req()),
+ ("POST", "/integrations/alerts/test") => {
+ handle.alerts.show(
+ "InstantClone",
+ "This is where your alerts show up on stream.",
+ 6,
+ );
+ done()
+ }
+ ("POST", "/integrations/midi/learn") => midi_learn(ctrl),
+ ("POST", "/integrations/midi/poll") => {
+ let captured = ctrl.midi().take_captured_for(&[MIDI_LEARN]);
+ ok(json::obj([
+ (
+ "learning",
+ Value::Bool(ctrl.midi().learning().as_deref() == Some(MIDI_LEARN)),
+ ),
+ (
+ "captured",
+ captured.map_or(Value::Null, |(_, sig)| json::str(sig)),
+ ),
+ ]))
+ }
+ ("POST", "/integrations/midi/cancel") => {
+ if ctrl.midi().learning().as_deref() == Some(MIDI_LEARN) {
+ ctrl.midi().cancel_learn();
+ }
+ done()
+ }
+ ("POST", "/integrations/export") => export(&req(), &settings.borrow()),
+ ("POST", "/integrations/import") => import(&req(), settings, cfg_path, local_steps_allowed),
+ ("POST", "/connections/discord") => save_discord(&req(), settings, cfg_path),
+ ("POST", "/connections/discord/delete") => delete_discord(&req(), settings, cfg_path),
+ ("POST", "/connections/discord/test") => {
+ // Copy out first: a settings borrow must not live across an await.
+ let channels = settings.borrow().discord_channels.clone();
+ test_discord(&req(), channels).await
+ }
+ ("POST", "/connections/phone") => save_phone(&req(), settings, cfg_path),
+ ("POST", "/connections/phone/test") => {
+ let phone = settings.borrow().phone.clone();
+ test_phone(phone).await
+ }
+ ("POST", "/twitch/login") => {
+ handle.twitch_login(Which::from_id(req().str_or("which", "main")));
+ done()
+ }
+ ("POST", "/twitch/cancel") => {
+ handle.twitch_cancel_login();
+ done()
+ }
+ ("POST", "/twitch/shared-chat") => {
+ let on = req().bool_or("on", false);
+ update(settings, cfg_path, |s| {
+ s.twitch_shared_chat = on;
+ Ok(())
+ })
+ }
+ ("POST", "/twitch/logout") => {
+ handle.twitch_logout(Which::from_id(req().str_or("which", "main")));
+ done()
+ }
+ _ => return None,
+ };
+ Some(reply)
+}
+
+/// The rules of a chat activity trigger sent for the live meter.
+fn meter_rules(trigger: &Value) -> Result {
+ let handler = json::obj([
+ ("trigger", trigger.clone()),
+ ("steps", Value::Arr(Vec::new())),
+ ]);
+ let probe = json::obj([("handlers", Value::Arr(vec![handler]))]);
+ let integration = Integration::from_json(&probe)?;
+ match integration.handlers.into_iter().next().map(|h| h.trigger) {
+ Some(Trigger::ChatActivity(rules)) => Ok(rules),
+ _ => Err("that isn't a chat activity trigger".to_string()),
+ }
+}
+
+/// What a MIDI learn for an integration's shortcut is filed under, apart
+/// from the delay actions' own learns.
+const MIDI_LEARN: &str = crate::midi::INTEGRATION_LEARN;
+
+/// Arm a MIDI learn: the next pad pressed becomes the shortcut. The
+/// dashboard polls `/integrations/midi/poll` for it.
+fn midi_learn(ctrl: &Arc) -> Reply {
+ if !cfg!(windows) {
+ return fail("MIDI shortcuts work on Windows only");
+ }
+ // Any device will do: the learn opens them all, not only the one
+ // picked in Controls.
+ if ctrl.midi().devices().is_empty() {
+ return fail("no MIDI controller found; plug one in and try again");
+ }
+ ctrl.midi().start_learn(MIDI_LEARN);
+ done()
+}
+
+/// `/hooks/`: fire the web call integrations holding this token.
+pub fn hook(ctrl: &Arc, token: &str, body: &str, query: &str) -> Reply {
+ if let Some(handle) = ctrl.integrations() {
+ handle.hook(
+ token.to_string(),
+ body.chars().take(64 * 1024).collect(),
+ query.to_string(),
+ );
+ }
+ (
+ "202 Accepted",
+ "application/json",
+ r#"{"ok":true}"#.to_string(),
+ )
+}
+
+/// Clone the settings, change them, validate, save and publish.
+fn update(
+ settings: &Arc>,
+ cfg_path: &Path,
+ change: impl FnOnce(&mut Settings) -> Result<(), String>,
+) -> Reply {
+ let _guard = crate::web::settings_write_guard();
+ let mut next = settings.borrow().clone();
+ if let Err(e) = change(&mut next) {
+ return fail(e);
+ }
+ if let Err(e) = next.save(cfg_path) {
+ return fail(format!("couldn't save: {e}"));
+ }
+ settings.send_replace(next);
+ done()
+}
+
+fn overview(handle: &super::Handle, s: &Settings, local_steps_allowed: bool) -> Value {
+ // Previews and "Try it" use the real channel once Twitch is connected.
+ let channel = handle.twitch.channel();
+ let global = Value::Arr(
+ runner::GLOBAL_VARS
+ .iter()
+ .map(|&(name, sample)| {
+ let sample = match name {
+ "channel" if !channel.is_empty() => channel.as_str(),
+ _ => sample,
+ };
+ json::obj([("name", json::str(name)), ("sample", json::str(sample))])
+ })
+ .collect(),
+ );
+ let discord = s
+ .discord_channels
+ .iter()
+ .map(|c| {
+ json::obj([
+ ("id", json::str(&c.id)),
+ ("name", json::str(&c.name)),
+ ("hint", json::str(mask_webhook(&c.url))),
+ ])
+ })
+ .collect();
+ let vars = json::obj([
+ ("global", global),
+ ("chat", event::vars_json(runner::CHAT_VARS)),
+ (
+ "web",
+ event::vars_json(&[
+ ("body", "{\"x\":1}"),
+ ("query", "team=red"),
+ ("source", "stream_deck"),
+ ]),
+ ),
+ ("activity", event::vars_json(engine::ACTIVITY_SAMPLES)),
+ ("scene", event::vars_json(engine::SCENE_SAMPLES)),
+ ]);
+ json::obj([
+ (
+ "integrations",
+ Value::Arr(s.integrations.iter().map(Integration::to_json).collect()),
+ ),
+ (
+ "connections",
+ json::obj([
+ ("discord", Value::Arr(discord)),
+ (
+ "phone",
+ json::obj([
+ ("server", json::str(&s.phone.server)),
+ ("topic", json::str(&s.phone.topic)),
+ ]),
+ ),
+ ]),
+ ),
+ // What each integration still needs before it can run, so a card
+ // can say "Pick a Discord channel" without opening the editor.
+ (
+ "issues",
+ Value::Obj(
+ s.integrations
+ .iter()
+ .map(|i| {
+ let mut p = i.problems();
+ p.invalid.extend(p.incomplete);
+ if let Err(e) = check_connections(i, s) {
+ p.invalid.push(e);
+ }
+ (
+ i.id.clone(),
+ Value::Arr(p.invalid.into_iter().map(json::str).collect()),
+ )
+ })
+ .collect(),
+ ),
+ ),
+ ("twitch", handle.twitch.status_json()),
+ ("twitch_shared_chat", Value::Bool(s.twitch_shared_chat)),
+ // Markers, clips and chat act on the logged-in channel; say so when
+ // the Twitch destinations stream somewhere else.
+ (
+ "twitch_other_channel",
+ Value::Bool(streams_to_other_channel(
+ &handle.twitch.main_user_id(),
+ &s.destinations,
+ )),
+ ),
+ // False on another device while the dashboard has no password:
+ // Run a program and Write a file are locked there (see `route`).
+ ("local_steps", Value::Bool(local_steps_allowed)),
+ ("catalog", presets::catalog_json()),
+ // Global hotkeys and MIDI pads only exist on Windows, and a combo
+ // or pad that runs a delay action can't start an integration too.
+ (
+ "shortcuts",
+ json::obj([
+ ("available", Value::Bool(cfg!(windows))),
+ (
+ "refused",
+ Value::Arr(handle.refused_keys().into_iter().map(json::str).collect()),
+ ),
+ (
+ "taken",
+ Value::Arr(
+ s.hotkeys
+ .entries()
+ .iter()
+ .chain(s.midi.entries().iter())
+ .filter(|(_, v)| !v.is_empty())
+ .map(|(_, v)| json::str(*v))
+ .collect(),
+ ),
+ ),
+ ]),
+ ),
+ ("events", event::catalog_json()),
+ ("vars", vars),
+ ("activity", handle.activity_json()),
+ ("stats", handle.stats_json()),
+ ("dropped_events", Value::Num(handle.dropped_events() as f64)),
+ ])
+}
+
+/// `https://discord.com/api/webhooks/1234/abcd...` shown as `…/1234/••••`.
+/// The id is read right after `/webhooks/`, never by counting from the
+/// end: a link ending in `/slack` would show the token in its place.
+fn mask_webhook(url: &str) -> String {
+ let id = url
+ .split_once("/webhooks/")
+ .and_then(|(_, rest)| rest.split('/').next())
+ .filter(|id| !id.is_empty() && id.bytes().all(|b| b.is_ascii_digit()));
+ match id {
+ Some(id) => format!("…/webhooks/{id}/••••"),
+ None => "set".to_string(),
+ }
+}
+
+fn save(
+ req: &Value,
+ settings: &Arc>,
+ cfg_path: &Path,
+ local_steps_allowed: bool,
+) -> Reply {
+ let mut integration = match Integration::from_json(req) {
+ Ok(i) => i,
+ Err(e) => return fail(e),
+ };
+ if !local_steps_allowed && integration.has_local_effects() {
+ return fail(LOCAL_STEPS_LOCKED);
+ }
+ if integration.id.is_empty() {
+ integration.id = new_id();
+ }
+ let mut problems = integration.problems();
+ if !problems.invalid.is_empty() {
+ return fail(problems.invalid.join("; "));
+ }
+ if let Err(e) = check_connections(&integration, &settings.borrow()) {
+ problems.incomplete.push(e);
+ }
+ if let Some(e) = shortcut_taken(&integration, &settings.borrow()) {
+ return fail(e);
+ }
+ // Switched on, it must be ready to run; switched off, it saves as a
+ // draft and the dashboard lists what is still missing.
+ if integration.enabled && !problems.incomplete.is_empty() {
+ return fail(format!(
+ "{}. Finish it, or switch it off to save it as a draft.",
+ problems.incomplete.join("; ")
+ ));
+ }
+ let id = integration.id.clone();
+ let warnings = unknown_vars(&integration);
+ let missing = problems.incomplete;
+ // Undoing a delete puts it back where it was.
+ let restore_at = req.get("at").and_then(Value::as_f64).map(|n| n as usize);
+ let reply = update(settings, cfg_path, move |s| {
+ match s.integrations.iter().position(|i| i.id == integration.id) {
+ Some(at) => s.integrations[at] = integration,
+ None if s.integrations.len() >= MAX_INTEGRATIONS => {
+ return Err(format!("you already have {MAX_INTEGRATIONS} integrations"))
+ }
+ None => {
+ let at = restore_at.unwrap_or(usize::MAX).min(s.integrations.len());
+ s.integrations.insert(at, integration);
+ }
+ }
+ Ok(())
+ });
+ if reply.0 != "200 OK" {
+ return reply;
+ }
+ ok(json::obj([
+ ("ok", Value::Bool(true)),
+ ("id", json::str(&id)),
+ (
+ "warnings",
+ Value::Arr(warnings.into_iter().map(json::str).collect()),
+ ),
+ (
+ "missing",
+ Value::Arr(missing.into_iter().map(json::str).collect()),
+ ),
+ ]))
+}
+
+/// Variables the integration's messages use that nothing provides: most
+/// likely a typo, which would otherwise render as nothing.
+pub fn unknown_vars(integration: &Integration) -> Vec {
+ use super::model::{visit_steps, StepKind};
+ let mut unknown: Vec = Vec::new();
+ for h in &integration.handlers {
+ let mut known: Vec = runner::GLOBAL_VARS
+ .iter()
+ .map(|(n, _)| n.to_string())
+ .collect();
+ let mut prefixes = vec!["counter.".to_string(), "clip.".to_string()];
+ match &h.trigger {
+ Trigger::Event { kind, .. } => {
+ known.extend(kind.vars().iter().map(|(n, _)| n.to_string()))
+ }
+ Trigger::ChatCommand { .. } | Trigger::ChatMessage { .. } => {
+ for n in 1..=9 {
+ known.push(format!("arg{n}"));
+ }
+ known.extend(runner::CHAT_VARS.iter().map(|(n, _)| n.to_string()));
+ }
+ // A button pressed through its Stream Deck address can carry a
+ // body too.
+ Trigger::Webhook { .. } | Trigger::Shortcut { .. } => {
+ known.extend(["body", "query", "source"].map(String::from));
+ prefixes.push("body.".to_string());
+ prefixes.push("query.".to_string());
+ }
+ Trigger::ChatActivity(_) => {
+ known.extend(engine::ACTIVITY_SAMPLES.iter().map(|(n, _)| n.to_string()))
+ }
+ Trigger::Scene { .. } => {
+ known.extend(engine::SCENE_SAMPLES.iter().map(|(n, _)| n.to_string()))
+ }
+ Trigger::Timer { .. } => {}
+ }
+ visit_steps(&h.steps, &mut |s| match s.kind {
+ StepKind::Http => prefixes.push(match s.param("save_as").trim() {
+ "" => "response.".to_string(),
+ name => format!("{name}."),
+ }),
+ StepKind::SetVar => known.push(s.param("name").trim().to_string()),
+ StepKind::EditText => known.push(match s.param("save_as").trim() {
+ "" => "text".to_string(),
+ name => name.to_string(),
+ }),
+ _ => {}
+ });
+ visit_steps(&h.steps, &mut |s| {
+ for value in s.params.values() {
+ for name in super::template::names(value) {
+ let provided = known.contains(&name)
+ || prefixes.iter().any(|p| name.starts_with(p.as_str()));
+ if !provided && !unknown.contains(&name) {
+ unknown.push(name);
+ }
+ }
+ }
+ });
+ }
+ unknown
+}
+
+fn with_id(reply: Reply, id: &str) -> Reply {
+ if reply.0 != "200 OK" {
+ return reply;
+ }
+ ok(json::obj([
+ ("ok", Value::Bool(true)),
+ ("id", json::str(id)),
+ ]))
+}
+
+/// A hotkey or pad an integration wants that already runs a delay action:
+/// Windows gives a hotkey to one owner, and a pad would do both.
+fn shortcut_taken(integration: &Integration, s: &Settings) -> Option {
+ integration.handlers.iter().find_map(|h| {
+ let Trigger::Shortcut { hotkey, midi, .. } = &h.trigger else {
+ return None;
+ };
+ let key_owner = s
+ .hotkeys
+ .entries()
+ .into_iter()
+ .find(|(_, bound)| !hotkey.is_empty() && *bound == hotkey.as_str());
+ if let Some((action, _)) = key_owner {
+ return Some(format!(
+ "{hotkey} already runs the delay's {} action (Controls tab); pick another key",
+ action.replace('_', " ")
+ ));
+ }
+ let pad_owner = s
+ .midi
+ .entries()
+ .into_iter()
+ .find(|(_, bound)| super::model::pads_overlap(bound, midi));
+ pad_owner.map(|(action, _)| {
+ format!(
+ "that MIDI pad already runs the delay's {} action (Controls tab); pick another",
+ action.replace('_', " ")
+ )
+ })
+ })
+}
+
+/// Every Discord step must point at a connection that exists.
+/// Whether every enabled Twitch destination streams to a channel other
+/// than `user_id`'s. The channel comes from the id Twitch puts in a stream
+/// key (`live__…`); keys in another shape say nothing either way.
+fn streams_to_other_channel(user_id: &str, destinations: &[crate::config::Destination]) -> bool {
+ let channels: Vec<&str> = destinations
+ .iter()
+ .filter(|d| d.enabled && d.platform == "twitch")
+ .filter_map(|d| key_channel(&d.stream_key))
+ .collect();
+ !user_id.is_empty() && !channels.is_empty() && !channels.contains(&user_id)
+}
+
+/// The channel id in a Twitch stream key, `live__`.
+fn key_channel(key: &str) -> Option<&str> {
+ let (id, _) = key.trim().strip_prefix("live_")?.split_once('_')?;
+ (!id.is_empty() && id.bytes().all(|b| b.is_ascii_digit())).then_some(id)
+}
+
+fn check_connections(integration: &Integration, s: &Settings) -> Result<(), String> {
+ let mut gone = false;
+ for h in &integration.handlers {
+ super::model::visit_steps(&h.steps, &mut |step| {
+ // A blank channel is reported by validation; this catches one
+ // that was set and has since been removed.
+ let id = step.param("connection");
+ if step.kind == super::model::StepKind::Discord
+ && !id.is_empty()
+ && !s.discord_channels.iter().any(|c| c.id == id)
+ {
+ gone = true;
+ }
+ });
+ }
+ if gone {
+ Err("a Discord channel it posts to was removed; pick another".to_string())
+ } else {
+ Ok(())
+ }
+}
+
+fn add(
+ req: &Value,
+ settings: &Arc>,
+ cfg_path: &Path,
+ local_steps_allowed: bool,
+) -> Reply {
+ let presets_to_add: Vec = match (req.get("preset"), req.get("pack")) {
+ (Some(p), _) => vec![p.as_str().unwrap_or("").to_string()],
+ (_, Some(pack)) => match presets::PACKS.iter().find(|p| Some(p.id) == pack.as_str()) {
+ Some(pack) => pack.presets.iter().map(|p| p.to_string()).collect(),
+ None => return fail("unknown pack"),
+ },
+ _ => return fail("pick something to add"),
+ };
+ let mut ids = Vec::new();
+ let reply = update(settings, cfg_path, |s| {
+ let channel = s
+ .discord_channels
+ .first()
+ .map(|c| c.id.clone())
+ .unwrap_or_default();
+ for preset in &presets_to_add {
+ // A pack skips what the user already has from the catalog.
+ if presets_to_add.len() > 1 && s.integrations.iter().any(|i| &i.preset == preset) {
+ continue;
+ }
+ if s.integrations.len() >= MAX_INTEGRATIONS {
+ return Err(format!("you already have {MAX_INTEGRATIONS} integrations"));
+ }
+ let built =
+ presets::build(preset, new_id(), &channel).ok_or("unknown catalog entry")?;
+ // A catalog entry that writes a file could never be finished.
+ if !local_steps_allowed && built.has_local_effects() {
+ if presets_to_add.len() > 1 {
+ continue;
+ }
+ return Err(LOCAL_STEPS_LOCKED.to_string());
+ }
+ ids.push(built.id.clone());
+ s.integrations.push(built);
+ }
+ Ok(())
+ });
+ if reply.0 != "200 OK" {
+ return reply;
+ }
+ ok(json::obj([
+ ("ok", Value::Bool(true)),
+ ("ids", Value::Arr(ids.into_iter().map(json::str).collect())),
+ ]))
+}
+
+fn toggle(
+ req: &Value,
+ settings: &Arc>,
+ cfg_path: &Path,
+ local_steps_allowed: bool,
+) -> Reply {
+ let id = req.str_or("id", "").to_string();
+ let enabled = req.bool_or("enabled", true);
+ update(settings, cfg_path, |s| {
+ let at = s
+ .integrations
+ .iter()
+ .position(|i| i.id == id)
+ .ok_or("that integration no longer exists")?;
+ if enabled {
+ let integration = &s.integrations[at];
+ if !local_steps_allowed && integration.has_local_effects() {
+ return Err(LOCAL_STEPS_LOCKED.to_string());
+ }
+ let errors = integration.validate();
+ if !errors.is_empty() {
+ return Err(format!("finish setting it up first: {}", errors.join("; ")));
+ }
+ check_connections(integration, s)?;
+ // A delay hotkey bound since it was switched off wins otherwise,
+ // and the integration would never start from it.
+ if let Some(taken) = shortcut_taken(integration, s) {
+ return Err(taken);
+ }
+ }
+ s.integrations[at].enabled = enabled;
+ Ok(())
+ })
+}
+
+fn duplicate(req: &Value, settings: &Arc>, cfg_path: &Path) -> Reply {
+ let id = req.str_or("id", "").to_string();
+ let copy_id = new_id();
+ let returned = copy_id.clone();
+ let reply = update(settings, cfg_path, move |s| {
+ if s.integrations.len() >= MAX_INTEGRATIONS {
+ return Err(format!("you already have {MAX_INTEGRATIONS} integrations"));
+ }
+ let original = s
+ .integrations
+ .iter()
+ .find(|i| i.id == id)
+ .cloned()
+ .ok_or("that integration no longer exists")?;
+ // The copy starts off, with no hotkey or pad and its own web call
+ // link: otherwise one press, command or link would run both.
+ let mut copy = original;
+ copy.id = copy_id;
+ copy.name = format!("{} (copy)", copy.name).chars().take(80).collect();
+ copy.enabled = false;
+ for h in &mut copy.handlers {
+ match &mut h.trigger {
+ Trigger::Webhook { token } => *token = new_id() + &new_id(),
+ Trigger::Shortcut {
+ hotkey,
+ midi,
+ token,
+ ..
+ } => {
+ hotkey.clear();
+ midi.clear();
+ if !token.is_empty() {
+ *token = new_id() + &new_id();
+ }
+ }
+ _ => {}
+ }
+ }
+ s.integrations.push(copy);
+ Ok(())
+ });
+ with_id(reply, &returned)
+}
+
+async fn test(req: &Value, handle: &super::Handle) -> Reply {
+ let integration = match req.get("integration").map(Integration::from_json) {
+ Some(Ok(i)) => i,
+ Some(Err(e)) => return fail(e),
+ None => return fail("nothing to test"),
+ };
+ let handler = req.u64_or("handler", 0) as usize;
+ if handler >= integration.handlers.len() {
+ return fail("that trigger doesn't exist");
+ }
+ let values = match req.get("values") {
+ Some(Value::Obj(fields)) => fields
+ .iter()
+ .map(|(k, v)| (k.clone(), v.to_display()))
+ .collect(),
+ _ => Default::default(),
+ };
+ let run = engine::TestRun {
+ values,
+ dry: req.bool_or("dry", false),
+ };
+ match handle.test(integration, handler, run).await {
+ Some(record) => ok(json::obj([
+ ("ok", Value::Bool(true)),
+ ("status", json::str(record.status)),
+ (
+ "steps",
+ Value::Arr(
+ record
+ .steps
+ .iter()
+ .map(|s| {
+ json::obj([
+ ("label", json::str(&s.label)),
+ ("status", json::str(s.status.id())),
+ ("at_ms", Value::Num(s.at_ms as f64)),
+ ("detail", json::str(&s.detail)),
+ ])
+ })
+ .collect(),
+ ),
+ ),
+ ])),
+ None => fail("the test didn't finish within a minute"),
+ }
+}
+
+/// The editor's paste box: a bot command (or an address) to the pieces of
+/// a "command from a website". See `botcmd`.
+fn convert(req: &Value) -> Reply {
+ match super::botcmd::convert(req.str_or("text", "")) {
+ Ok(c) => ok(json::obj([
+ ("ok", Value::Bool(true)),
+ ("command", c.command.map_or(Value::Null, json::str)),
+ ("url", json::str(c.url)),
+ ("reply", c.reply.map_or(Value::Null, json::str)),
+ ("from", json::str(c.from)),
+ (
+ "unknown",
+ Value::Arr(c.unknown.into_iter().map(json::str).collect()),
+ ),
+ ])),
+ Err(e) => fail(e),
+ }
+}
+
+/// Longest answer the editor's "Send request" shows.
+const FETCH_SHOWN: usize = 200_000;
+
+/// The editor's "Send request": make the request a web request step would,
+/// with the values the user typed, and show what came back.
+async fn fetch(req: &Value) -> Reply {
+ let headers = req
+ .str_or("headers", "")
+ .lines()
+ .filter_map(|line| {
+ let (name, value) = line.split_once(':')?;
+ let name = name.trim();
+ (!name.is_empty()).then(|| (name.to_string(), value.trim().to_string()))
+ })
+ .collect();
+ let request = super::host::HttpRequest {
+ method: match req.str_or("method", "GET").trim() {
+ "" => "GET".to_string(),
+ m => m.to_ascii_uppercase(),
+ },
+ url: req.str_or("url", "").trim().to_string(),
+ headers,
+ body: req.str_or("body", "").to_string(),
+ };
+ let started = std::time::Instant::now();
+ let result = tokio::task::spawn_blocking(move || super::effects::send_http(request))
+ .await
+ .unwrap_or_else(|_| Err("the request crashed".to_string()));
+ let ms = started.elapsed().as_millis() as f64;
+ match result {
+ Ok(resp) => {
+ let truncated = resp.body.chars().count() > FETCH_SHOWN;
+ let body: String = resp.body.chars().take(FETCH_SHOWN).collect();
+ ok(json::obj([
+ ("ok", Value::Bool(true)),
+ ("status", Value::Num(f64::from(resp.status))),
+ ("ms", Value::Num(ms)),
+ ("body", json::str(body)),
+ ("truncated", Value::Bool(truncated)),
+ ]))
+ }
+ Err(e) => fail(e),
+ }
+}
+
+fn export(req: &Value, s: &Settings) -> Reply {
+ let ids: Vec<&str> = req
+ .get("ids")
+ .and_then(Value::as_array)
+ .unwrap_or(&[])
+ .iter()
+ .filter_map(Value::as_str)
+ .collect();
+ let chosen: Vec = s
+ .integrations
+ .iter()
+ .filter(|i| ids.contains(&i.id.as_str()))
+ .cloned()
+ .collect();
+ if chosen.is_empty() {
+ return fail("pick what to share");
+ }
+ let name = match req.str_or("name", "").trim() {
+ "" => chosen[0].name.clone(),
+ n => n.to_string(),
+ };
+ ok(json::obj([
+ ("ok", Value::Bool(true)),
+ ("recipe", json::str(recipe::export(&name, &chosen))),
+ ]))
+}
+
+/// Preview a recipe (`apply` false) or add it (`apply` true).
+fn import(
+ req: &Value,
+ settings: &Arc>,
+ cfg_path: &Path,
+ local_steps_allowed: bool,
+) -> Reply {
+ let mut parsed = match recipe::parse(req.str_or("recipe", ""), new_id) {
+ Ok(r) => r,
+ Err(e) => return fail(e),
+ };
+ if !req.bool_or("apply", false) {
+ let items = parsed
+ .integrations
+ .iter()
+ .map(|i| {
+ json::obj([
+ ("name", json::str(&i.name)),
+ ("summary", json::str(summary(i))),
+ ("triggers", Value::Arr(trigger_review(i))),
+ ("effects", Value::Arr(effect_review(i))),
+ // Every step, for "See every step". Parsing already
+ // dropped what a recipe never carries (addresses, paths).
+ ("integration", i.to_json()),
+ ])
+ })
+ .collect();
+ return ok(json::obj([
+ ("ok", Value::Bool(true)),
+ ("name", json::str(&parsed.name)),
+ ("items", Value::Arr(items)),
+ ("uses_discord", Value::Bool(parsed.uses_discord())),
+ ("local_effects", Value::Bool(parsed.has_local_effects())),
+ ]));
+ }
+ if parsed.has_local_effects() && !local_steps_allowed {
+ return fail(LOCAL_STEPS_LOCKED);
+ }
+ if parsed.has_local_effects() && !req.bool_or("allow_local", false) {
+ return fail("this recipe runs programs or writes files; confirm that you trust it");
+ }
+ if parsed.uses_discord() {
+ let channel = req.str_or("discord", "").to_string();
+ if !settings
+ .borrow()
+ .discord_channels
+ .iter()
+ .any(|c| c.id == channel)
+ {
+ return fail("pick which Discord channel it should post to");
+ }
+ recipe::map_discord(&mut parsed, &channel);
+ }
+ let count = parsed.integrations.len();
+ let reply = update(settings, cfg_path, move |s| {
+ if s.integrations.len() + count > MAX_INTEGRATIONS {
+ return Err(format!(
+ "that would go over {MAX_INTEGRATIONS} integrations"
+ ));
+ }
+ s.integrations.extend(parsed.integrations);
+ Ok(())
+ });
+ if reply.0 != "200 OK" {
+ return reply;
+ }
+ ok(json::obj([
+ ("ok", Value::Bool(true)),
+ ("added", Value::Num(count as f64)),
+ ]))
+}
+
+/// Who can start a chat trigger, in words. The broadcaster always can.
+fn who_can(roles: Roles) -> &'static str {
+ match (roles.everyone, roles.subs, roles.vips, roles.mods) {
+ (true, ..) => "anyone in chat",
+ (false, false, false, false) => "only you",
+ (false, false, false, true) => "only mods",
+ (false, false, true, true) => "mods and VIPs",
+ (false, true, _, _) => "subs and up",
+ _ => "chosen roles",
+ }
+}
+
+/// Every trigger of an imported integration and who can set it off, for
+/// the import preview.
+fn trigger_review(i: &Integration) -> Vec {
+ i.handlers
+ .iter()
+ .map(|h| {
+ let (what, who) = match &h.trigger {
+ Trigger::ChatCommand { command, roles, .. } => {
+ (format!("{command} in chat"), who_can(*roles))
+ }
+ Trigger::ChatMessage { roles, .. } => ("a chat message".into(), who_can(*roles)),
+ Trigger::ChatActivity(a) => ("chat gets busy".into(), who_can(a.roles)),
+ Trigger::Webhook { .. } => ("a web call".into(), "apps with its secret link"),
+ Trigger::Shortcut { .. } => ("a button".into(), "you"),
+ Trigger::Event { kind, .. } => (kind.label().to_string(), ""),
+ Trigger::Timer { every_ms, .. } => (format!("every {} min", every_ms / 60_000), ""),
+ Trigger::Scene { scene, .. } => (format!("OBS switches to {scene}"), ""),
+ };
+ json::obj([("what", json::str(what)), ("who", json::str(who))])
+ })
+ .collect()
+}
+
+/// What an imported integration reaches, most serious first: `danger` for
+/// anything that runs on this PC or lets anyone in chat change the delay,
+/// `caution` for the stream and OBS, `info` for messages it sends.
+fn effect_review(i: &Integration) -> Vec {
+ use super::model::{visit_steps, StepKind};
+ let mut found: Vec<(&'static str, &'static str)> = Vec::new();
+ let mut add = |level, text| {
+ if !found.contains(&(level, text)) {
+ found.push((level, text));
+ }
+ };
+ for h in &i.handlers {
+ let from_anyone = match &h.trigger {
+ Trigger::ChatCommand { roles, .. } | Trigger::ChatMessage { roles, .. } => {
+ roles.everyone
+ }
+ Trigger::ChatActivity(a) => a.roles.everyone,
+ _ => false,
+ };
+ visit_steps(&h.steps, &mut |s| match s.kind {
+ StepKind::Program => add("danger", "Runs a program on this PC"),
+ StepKind::File => add("danger", "Writes a file on this PC"),
+ StepKind::DelayAction if from_anyone => {
+ add("danger", "Anyone in chat can change your delay")
+ }
+ StepKind::DelayAction => add("caution", "Changes your delay"),
+ StepKind::Obs => add("caution", "Controls OBS (scenes, sources, text)"),
+ StepKind::Http => add("caution", "Calls a web address you fill in"),
+ StepKind::Clip => add("info", "Makes clips"),
+ StepKind::Marker => add("info", "Adds VOD markers"),
+ StepKind::Discord => add("info", "Posts on Discord"),
+ StepKind::Chat => add("info", "Writes in your Twitch chat"),
+ StepKind::Phone => add("info", "Sends a push to your phone"),
+ StepKind::Overlay => add("info", "Shows an alert on stream"),
+ _ => {}
+ });
+ }
+ let rank = |level: &str| match level {
+ "danger" => 0,
+ "caution" => 1,
+ _ => 2,
+ };
+ found.sort_by_key(|(level, _)| rank(level));
+ found
+ .into_iter()
+ .map(|(level, text)| json::obj([("level", json::str(level)), ("text", json::str(text))]))
+ .collect()
+}
+
+/// One line describing an integration, for the import preview.
+fn summary(i: &Integration) -> String {
+ let triggers: Vec = i
+ .handlers
+ .iter()
+ .map(|h| match &h.trigger {
+ Trigger::Event { kind, .. } => kind.label().to_string(),
+ Trigger::ChatCommand { command, .. } => format!("{command} in chat"),
+ Trigger::ChatMessage { .. } => "a chat message".to_string(),
+ Trigger::Timer { every_ms, .. } => {
+ format!("every {} min", every_ms / 60_000)
+ }
+ Trigger::Webhook { .. } => "a web call".to_string(),
+ Trigger::Shortcut { .. } => "a button (hotkey, MIDI pad or Stream Deck)".to_string(),
+ Trigger::ChatActivity(_) => "chat gets busy".to_string(),
+ Trigger::Scene { scene, .. } => format!("OBS switches to {scene}"),
+ })
+ .collect();
+ let steps: usize = i
+ .handlers
+ .iter()
+ .map(|h| super::model::count_steps(&h.steps))
+ .sum();
+ format!("When {} · {steps} steps", triggers.join(", "))
+}
+
+fn save_discord(req: &Value, settings: &Arc>, cfg_path: &Path) -> Reply {
+ let name = req
+ .str_or("name", "")
+ .trim()
+ .chars()
+ .take(40)
+ .collect::();
+ let url = req.str_or("url", "").trim().to_string();
+ let id = match req.str_or("id", "") {
+ "" => new_id(),
+ id => id.to_string(),
+ };
+ if name.is_empty() {
+ return fail("give the channel a name, like Mods");
+ }
+ if !valid_id(&id) {
+ return fail("invalid id");
+ }
+ let existing_url = settings
+ .borrow()
+ .discord_channels
+ .iter()
+ .find(|c| c.id == id)
+ .map(|c| c.url.clone());
+ // Editing keeps the saved URL unless a new one is pasted.
+ let url = match (url.is_empty(), existing_url) {
+ (true, Some(saved)) => saved,
+ _ => url,
+ };
+ if !valid_discord_webhook(&url) {
+ return fail("paste a Discord webhook link (Server settings > Integrations > Webhooks)");
+ }
+ let returned = id.clone();
+ let reply = update(settings, cfg_path, move |s| {
+ let channel = DiscordChannel { id, name, url };
+ match s.discord_channels.iter().position(|c| c.id == channel.id) {
+ Some(at) => s.discord_channels[at] = channel,
+ None if s.discord_channels.len() >= 32 => {
+ return Err("that's a lot of Discord channels; remove one first".into())
+ }
+ None => s.discord_channels.push(channel),
+ }
+ Ok(())
+ });
+ with_id(reply, &returned)
+}
+
+fn delete_discord(req: &Value, settings: &Arc>, cfg_path: &Path) -> Reply {
+ let id = req.str_or("id", "").to_string();
+ update(settings, cfg_path, |s| {
+ s.discord_channels.retain(|c| c.id != id);
+ // Integrations that posted there stop until a new channel is picked.
+ for integration in &mut s.integrations {
+ let mut uses = false;
+ for h in &integration.handlers {
+ super::model::visit_steps(&h.steps, &mut |step| {
+ uses |= step.kind == super::model::StepKind::Discord
+ && step.param("connection") == id;
+ });
+ }
+ if uses {
+ integration.enabled = false;
+ }
+ }
+ Ok(())
+ })
+}
+
+async fn test_discord(req: &Value, channels: Vec) -> Reply {
+ let id = req.str_or("id", "");
+ let Some(channel) = channels.into_iter().find(|c| c.id == id) else {
+ return fail("that channel no longer exists");
+ };
+ let sent = tokio::task::spawn_blocking(move || {
+ let body = super::effects::discord_body(&super::host::DiscordMessage {
+ content: String::new(),
+ card: Some(super::host::DiscordCard {
+ title: "🧪 Connected".into(),
+ description: "InstantClone can post here. Alerts, highlights and your stream timeline will show up like this.".into(),
+ color: runner::DEFAULT_CARD_COLOR,
+ footer: "InstantClone".into(),
+ timestamp: true,
+ ..Default::default()
+ }),
+ ping: String::new(),
+ });
+ crate::https::https_agent_with_timeout(super::effects::HTTP_TIMEOUT)
+ .post(&channel.url)
+ .header("Content-Type", "application/json")
+ .send(body)
+ .map(|r| r.status().as_u16())
+ .map_err(|e| e.to_string())
+ })
+ .await;
+ match sent {
+ Ok(Ok(status)) if (200..300).contains(&status) => done(),
+ Ok(Ok(401 | 403)) => fail("Discord refused this webhook link; copy it again from Discord"),
+ Ok(Ok(404)) => fail("Discord says this webhook no longer exists"),
+ Ok(Ok(status)) => fail(format!("Discord answered {status}")),
+ Ok(Err(e)) => fail(format!("couldn't reach Discord: {e}")),
+ Err(_) => fail("the test crashed"),
+ }
+}
+
+fn save_phone(req: &Value, settings: &Arc>, cfg_path: &Path) -> Reply {
+ let topic = req.str_or("topic", "").trim().to_string();
+ let server = req
+ .str_or("server", "")
+ .trim()
+ .trim_end_matches('/')
+ .to_string();
+ if !topic.is_empty()
+ && (topic.len() > 64
+ || !topic
+ .bytes()
+ .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-'))
+ {
+ return fail("use letters, numbers, - and _ for the topic");
+ }
+ if !server.is_empty() && !server.starts_with("https://") && !server.starts_with("http://") {
+ return fail("the server must start with https://");
+ }
+ update(settings, cfg_path, |s| {
+ s.phone = PhoneConnection { server, topic };
+ Ok(())
+ })
+}
+
+async fn test_phone(phone: PhoneConnection) -> Reply {
+ if !phone.is_set() {
+ return fail("set a topic first");
+ }
+ let body = json::obj([
+ ("topic", json::str(phone.topic.trim())),
+ ("title", json::str("InstantClone")),
+ (
+ "message",
+ json::str("🧪 Test push: your phone is connected."),
+ ),
+ ])
+ .to_json();
+ let sent = tokio::task::spawn_blocking(move || {
+ crate::https::https_agent_with_timeout(super::effects::HTTP_TIMEOUT)
+ .post(phone.server_or_default())
+ .header("Content-Type", "application/json")
+ .send(body)
+ .map(|r| r.status().as_u16())
+ .map_err(|e| e.to_string())
+ })
+ .await;
+ match sent {
+ Ok(Ok(status)) if (200..300).contains(&status) => done(),
+ Ok(Ok(status)) => fail(format!("the server answered {status}")),
+ Ok(Err(e)) => fail(format!("couldn't reach it: {e}")),
+ Err(_) => fail("the test crashed"),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn a_stream_key_names_its_channel() {
+ assert_eq!(key_channel("live_123456789_AbCdEf"), Some("123456789"));
+ assert_eq!(key_channel("live_abc_AbCdEf"), None);
+ assert_eq!(key_channel("sk_us-west_123"), None);
+ assert_eq!(key_channel("live_123"), None);
+ }
+
+ #[test]
+ fn streaming_to_another_channel_is_noticed() {
+ let twitch = |key: &str, enabled: bool| crate::config::Destination {
+ id: key.into(),
+ name: "Twitch".into(),
+ enabled,
+ platform: "twitch".into(),
+ stream_key: key.into(),
+ custom_egress_url: String::new(),
+ twitch_ingest: String::new(),
+ youtube_ingest: String::new(),
+ vod_audio: false,
+ vod_audio_inject_eb: false,
+ stream_format: "horizontal".into(),
+ audio_track: "auto".into(),
+ };
+ let mut list = vec![twitch("live_222_x", true)];
+ assert!(streams_to_other_channel("111", &list));
+ assert!(
+ !streams_to_other_channel("", &list),
+ "logged out says nothing"
+ );
+ list.push(twitch("live_111_y", true));
+ assert!(
+ !streams_to_other_channel("111", &list),
+ "one is the login's"
+ );
+ let off = vec![twitch("live_222_x", false)];
+ assert!(
+ !streams_to_other_channel("111", &off),
+ "only enabled ones count"
+ );
+ }
+
+ #[test]
+ fn webhooks_are_masked() {
+ assert_eq!(
+ mask_webhook("https://discord.com/api/webhooks/123/abcDEF"),
+ "…/webhooks/123/••••"
+ );
+ assert_eq!(
+ mask_webhook("https://discord.com/api/webhooks/123/abcDEF/slack"),
+ "…/webhooks/123/••••"
+ );
+ }
+
+ #[test]
+ fn typos_in_variables_are_reported() {
+ let mut i = presets::build("delay_command", "d".into(), "").unwrap();
+ assert!(unknown_vars(&i).is_empty());
+ i.handlers[0].steps[0]
+ .params
+ .insert("text".into(), "{delya} for {user} at {time}".into());
+ assert_eq!(unknown_vars(&i), vec!["delya".to_string()]);
+ for p in presets::PRESETS {
+ let built = presets::build(p.id, "x".into(), "c").unwrap();
+ assert!(
+ unknown_vars(&built).is_empty(),
+ "{}: {:?}",
+ p.id,
+ unknown_vars(&built)
+ );
+ }
+ }
+
+ #[test]
+ fn a_delay_hotkey_or_pad_cannot_start_an_integration_too() {
+ use super::super::model::{Handler, Step, StepKind, Trigger};
+ let mut s = Settings::defaults();
+ s.hotkeys.set("cut", "Ctrl+Alt+C");
+ s.midi.set("toggle", "note:1:36");
+ let mut i = presets::build("delay_command", "d".into(), "").unwrap();
+ let mut shortcut = |hotkey: &str, midi: &str| {
+ i.handlers = vec![Handler {
+ enabled: true,
+ trigger: Trigger::Shortcut {
+ hotkey: hotkey.into(),
+ midi: midi.into(),
+ token: String::new(),
+ only_live: false,
+ },
+ steps: vec![Step::new(StepKind::Clip, &[])],
+ }];
+ shortcut_taken(&i, &s)
+ };
+ assert!(shortcut("Ctrl+Alt+C", "").unwrap().contains("cut"));
+ assert!(shortcut("", "note:1:36").unwrap().contains("toggle"));
+ assert_eq!(shortcut("Ctrl+Alt+K", "note:1:37"), None);
+ }
+
+ #[test]
+ fn programs_and_files_are_locked_when_the_request_may_not_set_them_up() {
+ use super::super::model::{Step, StepKind};
+ let dir = std::env::temp_dir().join(format!("ic-local-steps-{}", std::process::id()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let cfg = dir.join("instantclone.config.json");
+ let settings = Arc::new(watch::channel(Settings::defaults()).0);
+ let mut i = presets::build("delay_command", "d".into(), "").unwrap();
+ i.enabled = false;
+ i.handlers[0].steps.push(Step::new(
+ StepKind::Program,
+ &[("path", r"C:\Tools\thing.exe"), ("args", "")],
+ ));
+ let locked =
+ |reply: Reply| reply.0 != "200 OK" && reply.2.contains("needs a dashboard password");
+
+ assert!(locked(save(&i.to_json(), &settings, &cfg, false)));
+ assert_eq!(save(&i.to_json(), &settings, &cfg, true).0, "200 OK");
+ let switch = |on: bool| json::obj([("id", json::str(&i.id)), ("enabled", Value::Bool(on))]);
+ assert!(locked(toggle(&switch(true), &settings, &cfg, false)));
+ assert_eq!(
+ toggle(&switch(false), &settings, &cfg, false).0,
+ "200 OK",
+ "switching it off is always allowed"
+ );
+ let shared = json::obj([
+ (
+ "recipe",
+ json::str(recipe::export("kit", std::slice::from_ref(&i))),
+ ),
+ ("apply", Value::Bool(true)),
+ ("allow_local", Value::Bool(true)),
+ ]);
+ assert!(locked(import(&shared, &settings, &cfg, false)));
+
+ let _ = std::fs::remove_dir_all(&dir);
+ }
+
+ #[test]
+ fn the_import_preview_says_who_can_start_it_and_what_it_reaches() {
+ use super::super::model::{Step, StepKind};
+ let mut i = presets::build("mod_controls", "m".into(), "").unwrap();
+ let levels = |i: &Integration| -> Vec<(String, String)> {
+ effect_review(i)
+ .iter()
+ .map(|e| (e.str_or("level", "").into(), e.str_or("text", "").into()))
+ .collect()
+ };
+ assert_eq!(
+ trigger_review(&i)[0].str_or("who", ""),
+ "only mods",
+ "mod controls stay with mods"
+ );
+ assert!(levels(&i).contains(&("caution".into(), "Changes your delay".into())));
+
+ if let Trigger::ChatCommand { roles, .. } = &mut i.handlers[0].trigger {
+ *roles = Roles::EVERYONE;
+ }
+ i.handlers[0]
+ .steps
+ .push(Step::new(StepKind::Program, &[("path", ""), ("args", "")]));
+ let effects = levels(&i);
+ assert_eq!(effects[0].0, "danger", "the worst comes first");
+ assert!(effects.contains(&(
+ "danger".into(),
+ "Anyone in chat can change your delay".into()
+ )));
+ assert!(effects.contains(&("danger".into(), "Runs a program on this PC".into())));
+ }
+
+ #[test]
+ fn summaries_name_the_triggers() {
+ let i = presets::build("mod_controls", "m".into(), "").unwrap();
+ assert_eq!(
+ summary(&i),
+ "When !cut in chat, !setdelay in chat · 8 steps"
+ );
+ }
+}
diff --git a/src/integrations/botcmd.rs b/src/integrations/botcmd.rs
new file mode 100644
index 0000000..a444b0b
--- /dev/null
+++ b/src/integrations/botcmd.rs
@@ -0,0 +1,344 @@
+//! Turn a chat bot command from Nightbot, StreamElements, Fossabot or
+//! Streamlabs into the pieces of a "command from a website": the address it
+//! fetches, the reply around the answer, and the command's name. Their
+//! variables become ours: `$(user)` is `{user}`, `${1}` is `{arg1}`.
+//!
+//! Only commands that fetch an address convert. One that runs JavaScript
+//! (`$(eval …)`) cannot, and says so; a variable with no match here is kept
+//! as written and listed, so nothing is dropped silently.
+
+/// Where the converted reply reads the fetched answer (the preset saves
+/// its web request as `api`).
+pub const ANSWER: &str = "{api.body}";
+
+#[derive(Debug)]
+pub struct Converted {
+ /// `!name` when the paste was a whole "add command" line.
+ pub command: Option,
+ pub url: String,
+ /// The reply with the answer as `{api.body}`; `None` when only an
+ /// address was pasted.
+ pub reply: Option,
+ /// Which bot the syntax came from, for the dashboard's note.
+ pub from: &'static str,
+ /// Bot variables with no InstantClone match, kept as written.
+ pub unknown: Vec,
+}
+
+/// How each bot writes "fetch this address": (opening, closing, bot).
+const FETCHES: &[(&str, u8, &str)] = &[
+ ("$(urlfetch ", b')', "Nightbot"),
+ ("$(customapi ", b')', "Fossabot"),
+ ("${customapi.", b'}', "StreamElements"),
+ ("{readapi.", b'}', "Streamlabs"),
+ ("$readapi(", b')', "Streamlabs Chatbot"),
+];
+
+/// Chat lines that add or edit a command, before the command's name.
+const ADD_COMMANDS: &[&str] = &[
+ "!addcom",
+ "!editcom",
+ "!commands",
+ "!command",
+ "!cmd",
+ "!addcmd",
+ "!editcmd",
+];
+
+pub fn convert(input: &str) -> Result {
+ let text = input.trim();
+ if text.is_empty() {
+ return Err("paste a bot command or a web address".into());
+ }
+ let lower = text.to_ascii_lowercase();
+ if lower.contains("$(eval") || lower.contains("${eval") {
+ return Err(
+ "this command runs JavaScript ($(eval)), which InstantClone doesn't. Paste just the address it calls"
+ .into(),
+ );
+ }
+ let (command, response) = split_command(text);
+ let mut unknown = Vec::new();
+ let Some(fetch) = find_fetch(response) else {
+ let url = response.trim();
+ if is_http(url) && !url.contains(char::is_whitespace) {
+ let url = translate(url, &mut unknown);
+ return Ok(Converted {
+ command,
+ url,
+ reply: None,
+ from: "a web address",
+ unknown,
+ });
+ }
+ return Err(
+ "no web address in it: paste a $(urlfetch …), ${customapi.…} or https://… line".into(),
+ );
+ };
+ // Nightbot's `$(urlfetch json URL)` asks for raw JSON, which is what we get anyway.
+ let inner = fetch.inner.trim();
+ let inner = inner.strip_prefix("json ").unwrap_or(inner).trim();
+ let url = translate(inner, &mut unknown);
+ if !is_http(&url) {
+ return Err("the address it fetches doesn't start with http:// or https://".into());
+ }
+ let before = translate(&response[..fetch.start], &mut unknown);
+ let after = translate(&response[fetch.end..], &mut unknown);
+ let reply = format!("{before}{ANSWER}{after}").trim().to_string();
+ Ok(Converted {
+ command,
+ url,
+ reply: Some(reply),
+ from: fetch.from,
+ unknown,
+ })
+}
+
+fn is_http(s: &str) -> bool {
+ let s = s.get(..8).unwrap_or(s).to_ascii_lowercase();
+ s.starts_with("http://") || s.starts_with("https://")
+}
+
+/// `!commands add !rank -cd=5 ` to (`!rank`, ``); any
+/// other text is all response.
+fn split_command(text: &str) -> (Option, &str) {
+ let first = word(text);
+ if !ADD_COMMANDS.iter().any(|w| first.eq_ignore_ascii_case(w)) {
+ return (None, text);
+ }
+ let mut rest = after(text, first);
+ let verb = word(rest);
+ if ["add", "edit", "new"]
+ .iter()
+ .any(|v| verb.eq_ignore_ascii_case(v))
+ {
+ rest = after(rest, verb);
+ }
+ rest = skip_flags(rest);
+ let name = word(rest).trim_start_matches('!').to_lowercase();
+ if name.is_empty()
+ || !name
+ .chars()
+ .all(|c| c.is_alphanumeric() || c == '_' || c == '-')
+ {
+ return (None, text);
+ }
+ let rest = skip_flags(after(rest, word(rest)));
+ (Some(format!("!{name}")), rest)
+}
+
+fn word(s: &str) -> &str {
+ s.split_whitespace().next().unwrap_or("")
+}
+
+/// `s` after its first word `w` (which `s` starts with, past whitespace).
+fn after<'a>(s: &'a str, w: &str) -> &'a str {
+ &s.trim_start()[w.len()..]
+}
+
+/// Nightbot's `-cd=5 -ul=moderator`.
+fn skip_flags(mut s: &str) -> &str {
+ loop {
+ let w = word(s);
+ if !(w.starts_with('-') && w.contains('=')) {
+ return s;
+ }
+ s = after(s, w);
+ }
+}
+
+struct Fetch<'a> {
+ start: usize,
+ end: usize,
+ inner: &'a str,
+ from: &'static str,
+}
+
+/// The first fetch expression in `s`.
+fn find_fetch(s: &str) -> Option> {
+ // ASCII lowercasing keeps byte offsets, so they index `s` too.
+ let lower = s.to_ascii_lowercase();
+ let (start, open, close, from) = FETCHES
+ .iter()
+ .filter_map(|(open, close, from)| lower.find(open).map(|at| (at, *open, *close, *from)))
+ .min_by_key(|f| f.0)?;
+ let body = start + open.len();
+ let len = closing(&s[body..], close)?;
+ Some(Fetch {
+ start,
+ end: body + len + 1,
+ inner: &s[body..body + len],
+ from,
+ })
+}
+
+/// Where the expression ends: the first `close` not inside a nested
+/// `$(…)`, `${…}` or `{…}`.
+fn closing(s: &str, close: u8) -> Option {
+ let bytes = s.as_bytes();
+ let mut depth = 0usize;
+ let mut i = 0;
+ while i < bytes.len() {
+ match bytes[i] {
+ b'$' if matches!(bytes.get(i + 1), Some(b'(' | b'{')) => {
+ depth += 1;
+ i += 2;
+ continue;
+ }
+ b'{' => depth += 1,
+ b')' | b'}' if depth > 0 => depth -= 1,
+ b if b == close => return Some(i),
+ _ => {}
+ }
+ i += 1;
+ }
+ None
+}
+
+/// Rewrite every bot variable in `s` as ours.
+fn translate(s: &str, unknown: &mut Vec) -> String {
+ let mut out = String::with_capacity(s.len());
+ let mut rest = s;
+ while let Some(at) = next_variable(rest) {
+ out.push_str(&rest[..at]);
+ let tail = &rest[at..];
+ let (open, close) = match tail.as_bytes()[0] {
+ b'$' => (
+ 2,
+ if tail.as_bytes()[1] == b'(' {
+ b')'
+ } else {
+ b'}'
+ },
+ ),
+ _ => (1, b'}'),
+ };
+ let Some(len) = closing(&tail[open..], close) else {
+ out.push_str(tail);
+ return out;
+ };
+ let whole = &tail[..open + len + 1];
+ out.push_str(&ours(&tail[open..open + len], whole, open == 1, unknown));
+ rest = &tail[whole.len()..];
+ }
+ out.push_str(rest);
+ out
+}
+
+/// Where the next `$(`, `${` or `{` starts.
+fn next_variable(s: &str) -> Option {
+ let bytes = s.as_bytes();
+ (0..bytes.len()).find(|&i| match bytes[i] {
+ b'$' => matches!(bytes.get(i + 1), Some(b'(' | b'{')),
+ b'{' => true,
+ _ => false,
+ })
+}
+
+/// One bot variable as ours. `bare`: written `{…}` (Streamlabs, or already
+/// ours), which stays as written when unknown instead of being flagged.
+fn ours(inner: &str, whole: &str, bare: bool, unknown: &mut Vec) -> String {
+ let inner = inner.trim();
+ let (name, arg) = match inner.split_once(char::is_whitespace) {
+ Some((name, arg)) => (name, Some(arg.trim())),
+ None => (inner, None),
+ };
+ let name = name.to_ascii_lowercase();
+ // Encoding helpers: InstantClone encodes values in addresses itself.
+ if let ("urlencode" | "queryescape" | "pathescape" | "urlescape", Some(arg)) =
+ (name.as_str(), arg)
+ {
+ return translate(arg, unknown);
+ }
+ let mapped = match name.as_str() {
+ "user" | "user.name" | "user.display_name" | "sender" | "source" | "username"
+ | "displayname" => "user",
+ "touser" | "target" => "target",
+ "query" | "querystring" | "args" => "args",
+ "channel" | "channel.name" | "streamer" => "channel",
+ n if n.len() == 2 && n.ends_with(':') && n.as_bytes()[0].is_ascii_digit() => "args",
+ n if n.len() == 1 && (b'1'..=b'9').contains(&n.as_bytes()[0]) => {
+ return format!("{{arg{n}}}")
+ }
+ _ if bare => return whole.to_string(),
+ _ => {
+ unknown.push(whole.to_string());
+ return whole.to_string();
+ }
+ };
+ format!("{{{mapped}}}")
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn ok(text: &str) -> Converted {
+ convert(text).unwrap_or_else(|e| panic!("{text}: {e}"))
+ }
+
+ #[test]
+ fn nightbot_add_line_gives_command_address_and_reply() {
+ let c = ok("!commands add !Rank -cd=5 $(user), your rank: $(urlfetch https://api.example/rank/$(querystring)?region=eu)");
+ assert_eq!(c.command.as_deref(), Some("!rank"));
+ assert_eq!(c.url, "https://api.example/rank/{args}?region=eu");
+ assert_eq!(c.reply.as_deref(), Some("{user}, your rank: {api.body}"));
+ assert_eq!(c.from, "Nightbot");
+ assert!(c.unknown.is_empty());
+ }
+
+ #[test]
+ fn streamelements_nested_helpers_unwrap() {
+ let c = ok("!cmd add rank ${customapi.https://api.example/v1/${pathescape ${1}}/stats}");
+ assert_eq!(c.command.as_deref(), Some("!rank"));
+ assert_eq!(c.url, "https://api.example/v1/{arg1}/stats");
+ assert_eq!(c.reply.as_deref(), Some("{api.body}"));
+ assert_eq!(c.from, "StreamElements");
+ }
+
+ #[test]
+ fn fossabot_and_streamlabs_variables_map() {
+ let c = ok("$(customapi https://x.example/followage/$(channel)/$(user))");
+ assert_eq!(c.url, "https://x.example/followage/{channel}/{user}");
+ assert_eq!(c.command, None);
+ let c = ok("{readapi.https://x.example/q?u={user}&n={1}}");
+ assert_eq!(c.url, "https://x.example/q?u={user}&n={arg1}");
+ assert_eq!(c.from, "Streamlabs");
+ }
+
+ #[test]
+ fn touser_and_json_fetch() {
+ let c = ok("$(touser) is $(urlfetch json https://x.example/$(touser))");
+ assert_eq!(c.url, "https://x.example/{target}");
+ assert_eq!(c.reply.as_deref(), Some("{target} is {api.body}"));
+ }
+
+ #[test]
+ fn a_plain_address_is_just_the_address() {
+ let c = ok(" https://x.example/a?b=1 ");
+ assert_eq!(c.url, "https://x.example/a?b=1");
+ assert_eq!(c.reply, None);
+ }
+
+ #[test]
+ fn unknown_variables_stay_and_are_listed() {
+ let c = ok("$(count) asks: $(urlfetch https://x.example)");
+ assert_eq!(c.reply.as_deref(), Some("$(count) asks: {api.body}"));
+ assert_eq!(c.unknown, vec!["$(count)".to_string()]);
+ }
+
+ #[test]
+ fn what_cannot_convert_says_why() {
+ assert!(convert("$(eval const r = 1; r)")
+ .unwrap_err()
+ .contains("JavaScript"));
+ assert!(convert("hello $(user)")
+ .unwrap_err()
+ .contains("no web address"));
+ assert!(convert("$(urlfetch ftp://x.example)")
+ .unwrap_err()
+ .contains("http"));
+ assert!(convert("$(urlfetch https://x.example").is_err());
+ assert!(convert(" ").is_err());
+ }
+}
diff --git a/src/integrations/clock.rs b/src/integrations/clock.rs
new file mode 100644
index 0000000..456f104
--- /dev/null
+++ b/src/integrations/clock.rs
@@ -0,0 +1,101 @@
+//! Local wall-clock time for the `{time}` and `{date}` variables.
+//!
+//! std has no local time zone, so this asks the OS directly: GetLocalTime on
+//! Windows, localtime_r elsewhere.
+
+pub struct LocalTime {
+ pub year: u32,
+ pub month: u32,
+ pub day: u32,
+ pub hour: u32,
+ pub minute: u32,
+}
+
+impl LocalTime {
+ /// `21:04`
+ pub fn hm(&self) -> String {
+ format!("{:02}:{:02}", self.hour, self.minute)
+ }
+
+ /// `2026-09-30`
+ pub fn date(&self) -> String {
+ format!("{:04}-{:02}-{:02}", self.year, self.month, self.day)
+ }
+}
+
+#[cfg(windows)]
+pub fn now() -> LocalTime {
+ use windows_sys::Win32::System::SystemInformation::GetLocalTime;
+ // SAFETY: GetLocalTime only writes the SYSTEMTIME we hand it.
+ let t = unsafe {
+ let mut t = std::mem::zeroed();
+ GetLocalTime(&mut t);
+ t
+ };
+ LocalTime {
+ year: t.wYear as u32,
+ month: t.wMonth as u32,
+ day: t.wDay as u32,
+ hour: t.wHour as u32,
+ minute: t.wMinute as u32,
+ }
+}
+
+#[cfg(unix)]
+pub fn now() -> LocalTime {
+ // SAFETY: time() and localtime_r() write only into the locals we pass.
+ let tm = unsafe {
+ let secs = libc::time(std::ptr::null_mut());
+ let mut tm: libc::tm = std::mem::zeroed();
+ libc::localtime_r(&secs, &mut tm);
+ tm
+ };
+ LocalTime {
+ year: (tm.tm_year + 1900) as u32,
+ month: (tm.tm_mon + 1) as u32,
+ day: tm.tm_mday as u32,
+ hour: tm.tm_hour as u32,
+ minute: tm.tm_min as u32,
+ }
+}
+
+/// `2026-10-02T18:36:46Z` for a Unix time in seconds: what a Discord card's
+/// timestamp wants (Discord shows it in each reader's own time zone).
+pub fn iso_utc(unix_secs: u64) -> String {
+ let days = (unix_secs / 86_400) as i64;
+ let secs = unix_secs % 86_400;
+ // Civil date from days since 1970-01-01 (Howard Hinnant's algorithm).
+ let z = days + 719_468;
+ let era = z.div_euclid(146_097);
+ let doe = z - era * 146_097;
+ let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365;
+ let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
+ let mp = (5 * doy + 2) / 153;
+ let day = doy - (153 * mp + 2) / 5 + 1;
+ let month = if mp < 10 { mp + 3 } else { mp - 9 };
+ let year = yoe + era * 400 + i64::from(month <= 2);
+ format!(
+ "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}Z",
+ secs / 3600,
+ secs / 60 % 60,
+ secs % 60
+ )
+}
+
+#[cfg(test)]
+mod tests {
+ #[test]
+ fn utc_timestamps_are_iso_8601() {
+ assert_eq!(super::iso_utc(0), "1970-01-01T00:00:00Z");
+ assert_eq!(super::iso_utc(1_790_966_206), "2026-10-02T18:36:46Z");
+ assert_eq!(super::iso_utc(951_782_400), "2000-02-29T00:00:00Z");
+ }
+
+ #[test]
+ fn formats_are_zero_padded_and_plausible() {
+ let t = super::now();
+ assert!(t.year >= 2024 && (1..=12).contains(&t.month) && (1..=31).contains(&t.day));
+ assert_eq!(t.hm().len(), 5);
+ assert_eq!(t.date().len(), 10);
+ }
+}
diff --git a/src/integrations/effects.rs b/src/integrations/effects.rs
new file mode 100644
index 0000000..957bd44
--- /dev/null
+++ b/src/integrations/effects.rs
@@ -0,0 +1,812 @@
+//! The real `Host`: reads the controller, and does the I/O each step asks
+//! for. Every method may block and is only called from `spawn_blocking`.
+
+use super::host::{
+ ClipInfo, DiscordCard, DiscordMessage, DiscordPosted, Host, HttpRequest, HttpResponse,
+ LiveState,
+};
+use super::twitch::Twitch;
+use crate::controller::Controller;
+use crate::json::{self, Value};
+use std::io::Write as _;
+use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
+use std::sync::Arc;
+use std::time::Duration;
+
+pub(super) const HTTP_TIMEOUT: Duration = Duration::from_secs(10);
+/// Discord's rate limit says how long to wait before trying again. A wait
+/// this long or less is waited out; a longer one fails the step.
+const DISCORD_RETRY_MAX: Duration = Duration::from_secs(10);
+/// Tries per Discord call, rate limits included.
+const DISCORD_TRIES: usize = 3;
+/// Largest response body a step keeps.
+const MAX_RESPONSE: u64 = 1024 * 1024;
+/// Largest text a file step writes.
+const MAX_FILE_TEXT: usize = 1024 * 1024;
+
+/// Programs started by integrations that may still be running. Kept so
+/// they are reaped (no zombies on Unix) without a waiting thread each, and
+/// capped so a runaway trigger can't fork the machine to its knees.
+const MAX_RUNNING_PROGRAMS: usize = 16;
+
+/// Marks every web request InstantClone makes. A step can name this PC's
+/// own dashboard as its address; arriving with this, the dashboard treats
+/// it like a request through a proxy (`web::is_proxied`), so it can never
+/// set the first password or a Run a program step.
+const VIA: (&str, &str) = ("Via", "1.1 instantclone");
+
+/// Make a web request for a step, or for the editor's "Send request".
+pub fn send_http(r: HttpRequest) -> Result {
+ if !(r.url.starts_with("http://") || r.url.starts_with("https://")) {
+ return Err("the address must start with http:// or https://".to_string());
+ }
+ let agent = crate::https::https_agent();
+ let result = match r.method.as_str() {
+ "GET" | "DELETE" | "HEAD" => {
+ let mut req = match r.method.as_str() {
+ "GET" => agent.get(&r.url),
+ "HEAD" => agent.head(&r.url),
+ _ => agent.delete(&r.url),
+ }
+ .config()
+ .timeout_global(Some(HTTP_TIMEOUT))
+ .max_redirects(0)
+ .build();
+ for (name, value) in &r.headers {
+ req = req.header(name.as_str(), value.as_str());
+ }
+ req.header(VIA.0, VIA.1).call()
+ }
+ "POST" | "PUT" | "PATCH" => {
+ let mut req = match r.method.as_str() {
+ "PUT" => agent.put(&r.url),
+ "PATCH" => agent.patch(&r.url),
+ _ => agent.post(&r.url),
+ }
+ .config()
+ .timeout_global(Some(HTTP_TIMEOUT))
+ .max_redirects(0)
+ .build();
+ let has_type = r
+ .headers
+ .iter()
+ .any(|(n, _)| n.eq_ignore_ascii_case("content-type"));
+ if !has_type && !r.body.is_empty() {
+ let kind = if json::parse(&r.body).is_ok() {
+ "application/json"
+ } else {
+ "text/plain; charset=utf-8"
+ };
+ req = req.header("Content-Type", kind);
+ }
+ for (name, value) in &r.headers {
+ req = req.header(name.as_str(), value.as_str());
+ }
+ req.header(VIA.0, VIA.1).send(r.body.as_str())
+ }
+ other => return Err(format!("{other} is not a supported method")),
+ };
+ let mut resp = result.map_err(|e| format!("request failed: {e}"))?;
+ let status = resp.status().as_u16();
+ // Redirects aren't followed: one could point a request made from this
+ // PC at the dashboard itself, and post what it answers to chat.
+ if (300..400).contains(&status) {
+ let to = resp
+ .headers()
+ .get("location")
+ .and_then(|v| v.to_str().ok())
+ .unwrap_or("another address");
+ return Err(format!("the server moved this to {to}; use that address"));
+ }
+ let body = resp
+ .body_mut()
+ .with_config()
+ .limit(MAX_RESPONSE)
+ .read_to_string()
+ .unwrap_or_default();
+ Ok(HttpResponse { status, body })
+}
+
+/// Post a Discord message, or edit message `edit` (see `Host::discord`).
+/// An edit never pings; the ping is for when it has to be posted anew.
+pub fn send_discord(
+ webhook_url: &str,
+ message: &DiscordMessage,
+ edit: Option<&str>,
+) -> Result {
+ if let Some(id) = edit {
+ let quiet = DiscordMessage {
+ ping: String::new(),
+ ..message.clone()
+ };
+ let url = webhook_message_url(webhook_url, id);
+ let (status, _) = discord_call(true, &url, &discord_body(&quiet))?;
+ // 404 here is the message, not the webhook: someone deleted it.
+ // The update still has to reach the channel, as a new message.
+ if status != 404 {
+ discord_status(status)?;
+ return Ok(DiscordPosted {
+ message_id: id.to_string(),
+ edited: true,
+ });
+ }
+ }
+ // `wait=true` makes Discord answer with the message, and its id.
+ let url = with_query(webhook_url, "wait=true");
+ let (status, answer) = discord_call(false, &url, &discord_body(message))?;
+ discord_status(status)?;
+ let message_id = json::parse(&answer)
+ .ok()
+ .and_then(|v| v.get("id").and_then(Value::as_str).map(str::to_string))
+ .filter(|id| id.bytes().all(|b| b.is_ascii_digit()))
+ .unwrap_or_default();
+ Ok(DiscordPosted {
+ message_id,
+ edited: false,
+ })
+}
+
+/// One call to Discord (an edit, or a new message), waiting out its rate
+/// limit instead of failing: a burst of timeline updates must not lose the
+/// last one. The status and the answer's body.
+fn discord_call(edit: bool, url: &str, body: &str) -> Result<(u16, String), String> {
+ let mut tries = 0;
+ loop {
+ tries += 1;
+ let agent = crate::https::https_agent();
+ let request = if edit {
+ agent.patch(url)
+ } else {
+ agent.post(url)
+ };
+ let mut resp = request
+ .config()
+ .timeout_global(Some(HTTP_TIMEOUT))
+ .build()
+ .header("Content-Type", "application/json")
+ .send(body)
+ .map_err(|e| format!("couldn't reach Discord: {e}"))?;
+ let status = resp.status().as_u16();
+ let header_wait = resp
+ .headers()
+ .get("retry-after")
+ .and_then(|v| v.to_str().ok())
+ .and_then(|v| v.trim().parse::().ok());
+ let answer = resp
+ .body_mut()
+ .with_config()
+ .limit(64 * 1024)
+ .read_to_string()
+ .unwrap_or_default();
+ if status != 429 || tries >= DISCORD_TRIES {
+ return Ok((status, answer));
+ }
+ match retry_wait(&answer, header_wait) {
+ Some(wait) => std::thread::sleep(wait),
+ None => return Ok((status, answer)),
+ }
+ }
+}
+
+/// How long Discord's rate limit asks to wait (its JSON `retry_after`, or
+/// the Retry-After header), when that is short enough to wait out.
+fn retry_wait(answer: &str, header: Option) -> Option {
+ let seconds = json::parse(answer)
+ .ok()
+ .and_then(|v| v.get("retry_after").and_then(Value::as_f64))
+ .or(header)
+ .unwrap_or(1.0);
+ if !seconds.is_finite() || seconds < 0.0 {
+ return None;
+ }
+ // A little extra, so the retry doesn't land a hair too early.
+ let wait = Duration::from_secs_f64(seconds.min(3600.0)) + Duration::from_millis(50);
+ (wait <= DISCORD_RETRY_MAX).then_some(wait)
+}
+
+pub struct RealHost {
+ pub ctrl: Arc,
+ pub twitch: Arc,
+ /// The streamer's usual delay, used when `arm` names none.
+ pub default_delay_ms: AtomicU32,
+ pub programs: crate::sync::Mutex>,
+ pub alerts: Arc,
+ /// When this stream started (Unix ms), 0 between streams. The engine
+ /// keeps it, from the stream's own events.
+ pub stream_started_ms: AtomicU64,
+ /// What the engine last worked out the on-air state to be.
+ pub onair: crate::sync::Mutex<&'static str>,
+ /// Where OBS steps send what they ask OBS to do.
+ pub obs: tokio::sync::mpsc::Sender,
+}
+
+impl Host for RealHost {
+ fn live(&self) -> LiveState {
+ let (alive, total) = self.ctrl.destination_alive_summary();
+ let hold = self.ctrl.hold_status();
+ let started = self.stream_started_ms.load(Ordering::Relaxed);
+ LiveState {
+ streaming: started != 0,
+ uptime_ms: if started == 0 {
+ 0
+ } else {
+ super::twitch::unix_ms().saturating_sub(started)
+ },
+ onair: self.onair.lock().to_string(),
+ delay_ms: self.ctrl.current_delay_ms(),
+ phase: self.ctrl.phase().to_string(),
+ hold_active: self.ctrl.hold_active(),
+ hold_left_ms: hold.map_or(0, |h| h.remaining.as_millis() as u64),
+ destinations_live: alive as usize,
+ destinations_total: total as usize,
+ obs_live: self.ctrl.ingest_alive(),
+ bitrate_kbps: self.ctrl.bitrate_kbps() as u64,
+ channel: self.twitch.channel(),
+ }
+ }
+
+ fn discord(
+ &self,
+ webhook_url: &str,
+ message: &DiscordMessage,
+ edit: Option<&str>,
+ ) -> Result {
+ send_discord(webhook_url, message, edit)
+ }
+
+ fn http(&self, r: HttpRequest) -> Result {
+ send_http(r)
+ }
+
+ fn phone(
+ &self,
+ server: &str,
+ topic: &str,
+ title: &str,
+ text: &str,
+ priority: &str,
+ ) -> Result<(), String> {
+ let priority = match priority {
+ "high" => 4.0,
+ "urgent" => 5.0,
+ _ => 3.0,
+ };
+ let body = json::obj([
+ ("topic", json::str(topic)),
+ ("title", json::str(title)),
+ ("message", json::str(text)),
+ ("priority", Value::Num(priority)),
+ ])
+ .to_json();
+ let resp = crate::https::https_agent()
+ .post(server)
+ .config()
+ .timeout_global(Some(HTTP_TIMEOUT))
+ .build()
+ .header("Content-Type", "application/json")
+ .send(body)
+ .map_err(|e| format!("couldn't reach {server}: {e}"))?;
+ match resp.status().as_u16() {
+ 200..=299 => Ok(()),
+ 429 => Err("ntfy is rate limiting; slow down".to_string()),
+ s => Err(format!("ntfy answered {s}")),
+ }
+ }
+
+ fn chat(&self, text: &str, as_bot: bool, reply_to: Option<&str>) -> Result<(), String> {
+ self.twitch.send(text, as_bot, reply_to)
+ }
+
+ fn marker(&self, description: &str) -> Result<(), String> {
+ self.twitch.marker(description)
+ }
+
+ fn clip(&self) -> Result {
+ let id = self.twitch.clip()?;
+ Ok(ClipInfo {
+ url: format!("https://clips.twitch.tv/{id}"),
+ id,
+ })
+ }
+
+ fn delay_action(&self, action: &str, ms: u32) -> Result<(), String> {
+ let default_ms = if ms > 0 {
+ ms
+ } else {
+ self.default_delay_ms.load(Ordering::Relaxed)
+ };
+ match action {
+ // "Set the delay", not the hotkey's arm/disarm toggle.
+ "arm" => {
+ self.ctrl.set_delay_to(default_ms, "integration");
+ Ok(())
+ }
+ "disarm" => {
+ self.ctrl.disarm("integration");
+ Ok(())
+ }
+ "toggle" | "activate" | "cut" | "cut_after" | "end_hold" => {
+ match self
+ .ctrl
+ .run_named_action(action, default_ms, "integration")
+ {
+ None => Ok(()),
+ Some(problem) => Err(problem),
+ }
+ }
+ other => Err(format!("unknown delay action \"{other}\"")),
+ }
+ }
+
+ fn program(&self, path: &str, args: &[String]) -> Result<(), String> {
+ if path.is_empty() {
+ return Err("pick a program to run".to_string());
+ }
+ let mut running = self.programs.lock();
+ // Reap whatever has finished since last time.
+ running.retain_mut(|c| matches!(c.try_wait(), Ok(None)));
+ if running.len() >= MAX_RUNNING_PROGRAMS {
+ return Err(format!(
+ "{MAX_RUNNING_PROGRAMS} programs started by integrations are still running; not starting another"
+ ));
+ }
+ let child = std::process::Command::new(path)
+ .args(args)
+ .stdin(std::process::Stdio::null())
+ .stdout(std::process::Stdio::null())
+ .stderr(std::process::Stdio::null())
+ .spawn()
+ .map_err(|e| format!("couldn't start it: {e}"))?;
+ running.push(child);
+ Ok(())
+ }
+
+ fn file(&self, path: &str, text: &str, append: bool) -> Result<(), String> {
+ if path.is_empty() {
+ return Err("pick a file".to_string());
+ }
+ if text.len() > MAX_FILE_TEXT {
+ return Err("that text is too big to write".to_string());
+ }
+ let path = std::path::Path::new(path);
+ if let Some(dir) = path.parent().filter(|d| !d.as_os_str().is_empty()) {
+ std::fs::create_dir_all(dir).map_err(|e| format!("couldn't make the folder: {e}"))?;
+ }
+ let result = if append {
+ std::fs::OpenOptions::new()
+ .create(true)
+ .append(true)
+ .open(path)
+ .and_then(|mut f| writeln!(f, "{text}"))
+ } else {
+ std::fs::write(path, text)
+ };
+ result.map_err(|e| format!("couldn't write it: {e}"))
+ }
+
+ fn overlay(&self, title: &str, text: &str, seconds: u64) {
+ self.alerts.show(title, text, seconds);
+ }
+
+ fn obs(&self, action: super::obsws::Action) -> Result<(), String> {
+ let (reply, outcome) = std::sync::mpsc::channel();
+ let wait = super::obsws::CALL_TIMEOUT + Duration::from_secs(1);
+ let deadline = std::time::Instant::now() + wait;
+ self.obs
+ .try_send(super::obsws::Command {
+ action,
+ reply,
+ deadline,
+ })
+ .map_err(|_| "too many OBS requests at once; slow down".to_string())?;
+ // The connection answers every request, within its own timeout.
+ outcome
+ .recv_timeout(wait)
+ .unwrap_or_else(|_| Err("OBS didn't answer in time".to_string()))
+ }
+}
+
+fn discord_status(status: u16) -> Result<(), String> {
+ match status {
+ 200..=299 => Ok(()),
+ 401 | 403 => {
+ Err("Discord refused this webhook link; copy it again from Discord".to_string())
+ }
+ 404 => Err("Discord says this webhook no longer exists".to_string()),
+ 429 => Err("Discord is rate limiting this webhook; slow down".to_string()),
+ s => Err(format!("Discord answered {s}")),
+ }
+}
+
+/// `url` with `param` added to its query (a webhook link can already carry
+/// one, like `?thread_id=` for a forum thread).
+fn with_query(url: &str, param: &str) -> String {
+ let joint = if url.contains('?') { '&' } else { '?' };
+ format!("{url}{joint}{param}")
+}
+
+/// Where one of a webhook's messages is edited: `/messages/`,
+/// keeping the webhook's query so a thread message is found in its thread.
+fn webhook_message_url(url: &str, id: &str) -> String {
+ let (base, query) = url
+ .split_once('?')
+ .map_or((url, None), |(b, q)| (b, Some(q)));
+ let base = base.trim_end_matches('/');
+ match query {
+ Some(q) => format!("{base}/messages/{id}?{q}"),
+ None => format!("{base}/messages/{id}"),
+ }
+}
+
+/// The webhook's picture, so messages carry the InstantClone mark instead
+/// of Discord's default avatar. Discord fetches it; nothing is uploaded.
+const AVATAR_URL: &str =
+ "https://raw.githubusercontent.com/Soulhackzlol/InstantClone/main/docs/icon.png";
+
+/// Discord's own limits for a card. A longer value is cut, never refused.
+const CARD_TITLE_MAX: usize = 256;
+const CARD_DESCRIPTION_MAX: usize = 4096;
+const CARD_FIELDS_MAX: usize = 25;
+const CARD_FIELD_NAME_MAX: usize = 256;
+const CARD_FIELD_VALUE_MAX: usize = 1024;
+const CARD_FOOTER_MAX: usize = 2048;
+/// What all of a card's text together may add up to.
+const CARD_TOTAL_MAX: usize = 6000;
+
+/// A Discord message that can only ping what the streamer chose. Text from
+/// chat or a web service can contain `@everyone`; `allowed_mentions` keeps
+/// that from pinging anyone unless the ping option asked for it.
+pub fn discord_body(message: &DiscordMessage) -> String {
+ let (prefix, allowed) = mention_rules(&message.ping);
+ let content: String = format!("{prefix}{}", message.content)
+ .chars()
+ .take(2000)
+ .collect();
+ let embeds = message
+ .card
+ .as_ref()
+ .and_then(card_json)
+ .map_or_else(Vec::new, |card| vec![card]);
+ json::obj([
+ ("content", json::str(content)),
+ ("embeds", Value::Arr(embeds)),
+ ("username", json::str("InstantClone")),
+ ("avatar_url", json::str(AVATAR_URL)),
+ ("allowed_mentions", allowed),
+ ])
+ .to_json()
+}
+
+/// Whether a filled-in message has anything Discord would show. An empty
+/// one is refused by Discord, so the runner skips it instead.
+pub fn discord_has_body(message: &DiscordMessage) -> bool {
+ !message.content.trim().is_empty() || message.card.as_ref().and_then(card_json).is_some()
+}
+
+/// The card as Discord's embed JSON, or None when nothing in it would show.
+fn card_json(card: &DiscordCard) -> Option {
+ let title = cut(card.title.trim(), CARD_TITLE_MAX);
+ let fields: Vec<(String, String, bool)> = card
+ .fields
+ .iter()
+ .filter(|(name, value, _)| !name.trim().is_empty() && !value.trim().is_empty())
+ .take(CARD_FIELDS_MAX)
+ .map(|(name, value, inline)| {
+ (
+ cut(name.trim(), CARD_FIELD_NAME_MAX),
+ cut(value.trim(), CARD_FIELD_VALUE_MAX),
+ *inline,
+ )
+ })
+ .collect();
+ let footer = cut(card.footer.trim(), CARD_FOOTER_MAX);
+ let used = title.chars().count()
+ + footer.chars().count()
+ + fields
+ .iter()
+ .map(|(n, v, _)| n.chars().count() + v.chars().count())
+ .sum::();
+ let room = CARD_TOTAL_MAX
+ .saturating_sub(used)
+ .min(CARD_DESCRIPTION_MAX);
+ let description = cut(card.description.trim(), room);
+ let image = web_address(&card.image);
+ if title.is_empty() && description.is_empty() && fields.is_empty() && image.is_none() {
+ return None;
+ }
+ let mut out = vec![("color".to_string(), Value::Num(f64::from(card.color)))];
+ if !title.is_empty() {
+ out.push(("title".to_string(), json::str(title)));
+ if let Some(url) = web_address(&card.url) {
+ out.push(("url".to_string(), json::str(url)));
+ }
+ }
+ if !description.is_empty() {
+ out.push(("description".to_string(), json::str(description)));
+ }
+ if !fields.is_empty() {
+ let list = fields
+ .into_iter()
+ .map(|(name, value, inline)| {
+ json::obj([
+ ("name", json::str(name)),
+ ("value", json::str(value)),
+ ("inline", Value::Bool(inline)),
+ ])
+ })
+ .collect();
+ out.push(("fields".to_string(), Value::Arr(list)));
+ }
+ if !footer.is_empty() {
+ out.push((
+ "footer".to_string(),
+ json::obj([
+ ("text", json::str(footer)),
+ ("icon_url", json::str(AVATAR_URL)),
+ ]),
+ ));
+ }
+ if card.timestamp {
+ let now = super::twitch::unix_ms() / 1000;
+ out.push((
+ "timestamp".to_string(),
+ json::str(super::clock::iso_utc(now)),
+ ));
+ }
+ if let Some(url) = image {
+ out.push(("image".to_string(), json::obj([("url", json::str(url))])));
+ }
+ if let Some(url) = web_address(&card.thumbnail) {
+ out.push((
+ "thumbnail".to_string(),
+ json::obj([("url", json::str(url))]),
+ ));
+ }
+ Some(Value::Obj(out))
+}
+
+/// `text` cut to `max` characters, with an ellipsis when it was longer.
+fn cut(text: &str, max: usize) -> String {
+ if text.chars().count() <= max {
+ return text.to_string();
+ }
+ let mut out: String = text.chars().take(max.saturating_sub(1)).collect();
+ out.push('…');
+ out
+}
+
+/// An http(s) address Discord will take, or None: Discord refuses the whole
+/// message over one malformed link, so a bad one is left out instead.
+fn web_address(text: &str) -> Option<&str> {
+ let text = text.trim();
+ let ok = (text.starts_with("https://") || text.starts_with("http://"))
+ && text.len() > 10
+ && text.len() <= 2048
+ && !text.contains(char::is_whitespace);
+ ok.then_some(text)
+}
+
+/// The text a ping puts in front of the message, and the mentions Discord
+/// may act on.
+fn mention_rules(ping: &str) -> (String, Value) {
+ match ping {
+ "here" => (
+ "@here ".to_string(),
+ json::obj([("parse", Value::Arr(vec![json::str("everyone")]))]),
+ ),
+ "everyone" => (
+ "@everyone ".to_string(),
+ json::obj([("parse", Value::Arr(vec![json::str("everyone")]))]),
+ ),
+ role if role.starts_with("role:")
+ && role[5..].bytes().all(|b| b.is_ascii_digit())
+ && role.len() > 5 =>
+ {
+ (
+ format!("<@&{}> ", &role[5..]),
+ json::obj([("roles", Value::Arr(vec![json::str(&role[5..])]))]),
+ )
+ }
+ _ => (String::new(), json::obj([("parse", Value::Arr(vec![]))])),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn text(content: &str, ping: &str) -> String {
+ discord_body(&DiscordMessage {
+ content: content.into(),
+ card: None,
+ ping: ping.into(),
+ })
+ }
+
+ fn plain(content: &str) -> DiscordMessage {
+ DiscordMessage {
+ content: content.into(),
+ ..DiscordMessage::default()
+ }
+ }
+
+ #[test]
+ fn a_card_becomes_an_embed_within_discords_limits() {
+ let card = DiscordCard {
+ title: "🔴 YouTube is down".into(),
+ url: "not a link".into(),
+ description: "x".repeat(9000),
+ color: 0xf2665a,
+ fields: vec![
+ ("Reason".into(), "timed out".into(), true),
+ ("Empty".into(), " ".into(), true),
+ ],
+ footer: "InstantClone".into(),
+ timestamp: true,
+ image: "https://example.com/a.png".into(),
+ thumbnail: String::new(),
+ };
+ let body = json::parse(&discord_body(&DiscordMessage {
+ content: "https://clips.twitch.tv/Abc".into(),
+ card: Some(card),
+ ping: String::new(),
+ }))
+ .unwrap();
+ let embed = body.path("embeds.0").unwrap();
+ assert_eq!(embed.str_or("title", ""), "🔴 YouTube is down");
+ assert!(embed.get("url").is_none(), "a broken link is left out");
+ assert_eq!(embed.path("fields").unwrap().as_array().unwrap().len(), 1);
+ assert_eq!(embed.str_or("description", "").chars().count(), 4096);
+ assert_eq!(
+ embed.path("color").and_then(Value::as_f64),
+ Some(f64::from(0xf2665a_u32))
+ );
+ assert!(embed.str_or("timestamp", "").ends_with('Z'));
+ assert_eq!(
+ embed.path("image.url").and_then(Value::as_str),
+ Some("https://example.com/a.png")
+ );
+ assert_eq!(body.str_or("content", ""), "https://clips.twitch.tv/Abc");
+ assert!(body.str_or("avatar_url", "").starts_with("https://"));
+ }
+
+ #[test]
+ fn an_empty_card_is_no_message() {
+ let empty = DiscordMessage {
+ content: " ".into(),
+ card: Some(DiscordCard::default()),
+ ping: String::new(),
+ };
+ assert!(!discord_has_body(&empty));
+ let body = json::parse(&discord_body(&empty)).unwrap();
+ assert!(body.path("embeds").unwrap().as_array().unwrap().is_empty());
+ assert!(discord_has_body(&plain("hi")));
+ }
+
+ #[test]
+ fn discord_never_pings_unless_asked() {
+ let quiet = json::parse(&text("@everyone look", "")).unwrap();
+ assert_eq!(
+ quiet
+ .path("allowed_mentions.parse")
+ .unwrap()
+ .as_array()
+ .unwrap()
+ .len(),
+ 0
+ );
+ let here = json::parse(&text("OBS dropped", "here")).unwrap();
+ assert_eq!(here.str_or("content", ""), "@here OBS dropped");
+ let role = json::parse(&text("x", "role:123")).unwrap();
+ assert_eq!(role.str_or("content", ""), "<@&123> x");
+ assert_eq!(
+ role.path("allowed_mentions.roles.0").unwrap().as_str(),
+ Some("123")
+ );
+ let bad = json::parse(&text("x", "role:1 OR 1")).unwrap();
+ assert_eq!(bad.str_or("content", ""), "x");
+ }
+
+ /// A webhook that answers like Discord does: a new message gets an id,
+ /// an edit of a known id works, an edit of a deleted one is a 404.
+ fn fake_discord(answers: usize) -> (String, std::thread::JoinHandle>) {
+ use std::io::{BufRead, BufReader, Read, Write};
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let url = format!(
+ "http://{}/api/webhooks/1/tok",
+ listener.local_addr().unwrap()
+ );
+ let seen = std::thread::spawn(move || {
+ let mut seen = Vec::new();
+ for stream in listener.incoming().take(answers) {
+ let mut stream = stream.unwrap();
+ let mut reader = BufReader::new(stream.try_clone().unwrap());
+ let mut line = String::new();
+ reader.read_line(&mut line).unwrap();
+ let request: Vec<&str> = line.split_whitespace().collect();
+ let (method, path) = (request[0].to_string(), request[1].to_string());
+ let mut length = 0;
+ loop {
+ let mut header = String::new();
+ reader.read_line(&mut header).unwrap();
+ if header.trim().is_empty() {
+ break;
+ }
+ if let Some(v) = header.to_ascii_lowercase().strip_prefix("content-length:") {
+ length = v.trim().parse().unwrap();
+ }
+ }
+ let mut body = vec![0; length];
+ reader.read_exact(&mut body).unwrap();
+ let (status, answer) = match (method.as_str(), path.as_str()) {
+ ("POST", _) => ("200 OK", r#"{"id":"111"}"#),
+ ("PATCH", p) if p.ends_with("/messages/111") => ("200 OK", r#"{"id":"111"}"#),
+ _ => ("404 Not Found", r#"{"message":"Unknown Message"}"#),
+ };
+ let pinged = String::from_utf8_lossy(&body).contains("@here");
+ seen.push(format!(
+ "{method} {path}{}",
+ if pinged { " @here" } else { "" }
+ ));
+ let reply = format!(
+ "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{answer}",
+ answer.len()
+ );
+ stream.write_all(reply.as_bytes()).unwrap();
+ }
+ seen
+ });
+ (url, seen)
+ }
+
+ #[test]
+ fn a_message_is_posted_edited_and_reposted_once_deleted() {
+ let (url, seen) = fake_discord(4);
+ let back_here = DiscordMessage {
+ ping: "here".into(),
+ ..plain("back")
+ };
+ let first = send_discord(&url, &plain("down"), None).unwrap();
+ assert_eq!((first.message_id.as_str(), first.edited), ("111", false));
+ let edit = send_discord(&url, &back_here, Some("111")).unwrap();
+ assert_eq!((edit.message_id.as_str(), edit.edited), ("111", true));
+ // Someone deleted message 999: the update still goes out, as new,
+ // and pings like a new message does.
+ let gone = send_discord(&url, &back_here, Some("999")).unwrap();
+ assert_eq!((gone.message_id.as_str(), gone.edited), ("111", false));
+ assert_eq!(
+ seen.join().unwrap(),
+ [
+ "POST /api/webhooks/1/tok?wait=true",
+ "PATCH /api/webhooks/1/tok/messages/111",
+ "PATCH /api/webhooks/1/tok/messages/999",
+ "POST /api/webhooks/1/tok?wait=true @here",
+ ]
+ );
+ }
+
+ #[test]
+ fn edits_go_to_the_message_and_keep_the_thread() {
+ let hook = "https://discord.com/api/webhooks/1/tok";
+ assert_eq!(with_query(hook, "wait=true"), format!("{hook}?wait=true"));
+ assert_eq!(
+ webhook_message_url(hook, "99"),
+ format!("{hook}/messages/99")
+ );
+ let thread = "https://discord.com/api/webhooks/1/tok?thread_id=5";
+ assert_eq!(
+ with_query(thread, "wait=true"),
+ format!("{thread}&wait=true")
+ );
+ assert_eq!(
+ webhook_message_url(thread, "99"),
+ "https://discord.com/api/webhooks/1/tok/messages/99?thread_id=5"
+ );
+ }
+
+ #[test]
+ fn discord_messages_are_capped() {
+ let long = json::parse(&text(&"a".repeat(3000), "")).unwrap();
+ assert_eq!(long.str_or("content", "").len(), 2000);
+ }
+}
diff --git a/src/integrations/engine.rs b/src/integrations/engine.rs
new file mode 100644
index 0000000..3b725a8
--- /dev/null
+++ b/src/integrations/engine.rs
@@ -0,0 +1,2199 @@
+//! The integrations engine: matches what happens to the integrations that
+//! care, and runs them.
+//!
+//! It lives on its own OS thread with its own single-threaded runtime. The
+//! stream pipeline never shares a thread with it, so a slow webhook, a
+//! chat flood or a long `wait` cannot delay a single video frame. Events
+//! come in through a bounded channel whose `try_send` works from any thread
+//! (the controller, the tray, hotkey and MIDI callbacks); if the engine ever
+//! falls behind, events are dropped and counted, never queued without end.
+//!
+//! Limits that keep a runaway integration harmless:
+//! - each integration has at most `PER_INTEGRATION` runs doing something at
+//! once (`GLOBAL_RUNS` for all of them), and at most `PER_INTEGRATION_RUNS`
+//! started and not finished (`GLOBAL_STARTED`): a run waiting for the
+//! delay gives its place back, so presses during a long delay queue up;
+//! - `cooldown_ms` spaces runs of the same trigger; chat commands also have
+//! a per-viewer cooldown;
+//! - quiet hours keep an integration silent for part of the day;
+//! - the runner caps waits, nesting and message sizes.
+
+use super::activity::ChatWindow;
+use super::alerts::Board;
+use super::effects::RealHost;
+use super::event::{Event, EventKind};
+use super::model::{
+ pad_matches, ChatActivity, DiscordChannel, Integration, MatchMode, PhoneConnection, Roles,
+ Trigger,
+};
+use super::obsws::{self, SceneNews};
+use super::runner::{self, RunContext, RunEnv, RunStatus, StepLog};
+use super::session::Session;
+use super::store::Store;
+use super::timeline::{Kind as LineKind, Timeline};
+use super::twitch::irc::ChatMessage;
+use super::twitch::{unix_ms, Twitch, Which};
+use crate::config::Settings;
+use crate::controller::Controller;
+use crate::json::{self, Value};
+use crate::sync::Mutex;
+use std::collections::{BTreeMap, HashMap, VecDeque};
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+use tokio::sync::{mpsc, oneshot, watch, Semaphore};
+
+const QUEUE: usize = 512;
+const PER_INTEGRATION: usize = 4;
+const GLOBAL_RUNS: usize = 32;
+const PER_INTEGRATION_RUNS: usize = 16;
+const GLOBAL_STARTED: usize = 64;
+const ACTIVITY_KEEP: usize = 150;
+const TICK: Duration = Duration::from_millis(250);
+/// How often per-viewer and per-trigger bookkeeping is trimmed. A busy
+/// chat adds an entry per viewer per command; without trimming, a long
+/// stream would only ever grow them.
+const PRUNE_EVERY: Duration = Duration::from_secs(60);
+/// Runs counted for `{uses}` are written at most this often: a busy chat
+/// command would otherwise rewrite the data file on every use.
+const USES_SAVE_EVERY: Duration = Duration::from_secs(5);
+/// Runs each card's history strip shows.
+const RECENT_KEEP: usize = 12;
+/// A destination stuck in a connect-then-drop loop (a key the platform
+/// accepts, then refuses) would otherwise alert on every round. Each
+/// destination gets at most one "live" and one "dropped" per window.
+const FLAP_WINDOW: Duration = Duration::from_secs(300);
+/// A chat activity trigger stays quiet this long after a stream starts:
+/// the first seconds of chat ("hi!", "hello") are a burst by nature.
+pub const ACTIVITY_WARMUP: Duration = Duration::from_secs(30);
+/// How often chat activity triggers judge chat at most.
+const ACTIVITY_READ_EVERY: Duration = Duration::from_millis(250);
+/// How long the dashboard keeps the OBS connection open after asking for
+/// the scene list, so picking a scene works before anything is saved.
+const OBS_FOR_DASHBOARD: Duration = Duration::from_secs(60);
+
+enum Input {
+ Event(Event),
+ Hook {
+ token: String,
+ body: String,
+ query: String,
+ },
+ Test {
+ integration: Box,
+ handler: usize,
+ run: TestRun,
+ reply: oneshot::Sender,
+ },
+ #[cfg_attr(not(windows), allow(dead_code))]
+ Shortcut(Shortcut, String),
+ Scene(SceneNews),
+ Login(Which),
+ CancelLogin,
+ Logout(Which),
+}
+
+/// How a test runs: values to use instead of the samples, and whether it
+/// may send anything.
+#[derive(Clone, Debug, Default)]
+pub struct TestRun {
+ pub values: BTreeMap,
+ pub dry: bool,
+}
+
+/// What a shortcut press came from. Only Windows has hotkeys and MIDI.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[cfg_attr(not(windows), allow(dead_code))]
+pub enum Shortcut {
+ Hotkey,
+ Midi,
+}
+
+/// One run, for the activity log.
+#[derive(Clone, Debug)]
+pub struct Record {
+ pub at_ms: u64,
+ pub integration_id: String,
+ pub name: String,
+ pub trigger: String,
+ /// `ok`, `stopped`, `failed` or `busy`.
+ pub status: &'static str,
+ pub test: bool,
+ pub steps: Vec,
+}
+
+impl Record {
+ fn to_json(&self) -> Value {
+ json::obj([
+ ("at_ms", Value::Num(self.at_ms as f64)),
+ ("integration_id", json::str(&self.integration_id)),
+ ("name", json::str(&self.name)),
+ ("trigger", json::str(&self.trigger)),
+ ("status", json::str(self.status)),
+ ("test", Value::Bool(self.test)),
+ (
+ "steps",
+ Value::Arr(
+ self.steps
+ .iter()
+ .map(|s| {
+ json::obj([
+ ("label", json::str(&s.label)),
+ ("status", json::str(s.status.id())),
+ ("at_ms", Value::Num(s.at_ms as f64)),
+ ("detail", json::str(&s.detail)),
+ ])
+ })
+ .collect(),
+ ),
+ ),
+ ])
+ }
+}
+
+#[derive(Clone, Default)]
+struct Stat {
+ last_ms: u64,
+ last_status: &'static str,
+ /// Failures in a row; 0 after a good run.
+ failing: u32,
+ runs: u64,
+ /// The latest runs' outcomes, oldest first.
+ recent: VecDeque<&'static str>,
+}
+
+/// The engine's public face: shared with the controller and the web layer.
+pub struct Handle {
+ tx: mpsc::Sender,
+ activity: Mutex>,
+ stats: Mutex>,
+ pub twitch: Arc,
+ /// What "Show on stream" steps put on the alerts browser source.
+ pub alerts: Arc,
+ store: Arc,
+ /// Integration hotkeys Windows wouldn't register (another app holds
+ /// them, or there are too many), for the dashboard to point out.
+ refused_keys: Mutex>,
+ dropped: AtomicU64,
+ /// The last minutes of chat, read by chat activity triggers and by the
+ /// dashboard's live meter while one is being tuned.
+ chat: Arc