From b415c033bf36235d69902060c98d6653210c523f Mon Sep 17 00:00:00 2001 From: calvin-archastro Date: Thu, 13 Aug 2026 12:49:23 -0700 Subject: [PATCH] feat: lock narration voice and add a spoken setup card Chunk long transcripts and trim Qwen trailing silence so holds follow speech. Burn-in captions are opt-in and off by default. Each job clones later sentences from one reference clip so the speaker stays put. After the intro, a setup page speaks Goal and Persona from the friction-log prompt. --- CHANGELOG.md | 14 ++ macos/Astroshots.xcodeproj/project.pbxproj | 24 ++- macos/Astroshots/App/AppState.swift | 14 +- macos/Astroshots/App/Preferences.swift | 9 + .../Features/Tray/SettingsPane.swift | 7 + macos/Astroshots/Models/FrictionLog.swift | 18 ++ .../Astroshots/Narration/NarrationBrief.swift | 80 +++++++++ .../Narration/NarrationJobQueue.swift | 102 ++++++++++- .../Narration/NarrationModelManager.swift | 123 ++++++++++++- .../Narration/NarrationModels.swift | 14 +- .../Narration/NarrationSpeech.swift | 169 ++++++++++++++++++ .../Narration/NarrationVideoComposer.swift | 140 +++++++++++++-- .../FrictionLogLoaderTests.swift | 4 + macos/AstroshotsTests/NarrationTests.swift | 139 +++++++++++++- macos/AstroshotsTests/PreferencesTests.swift | 12 ++ 15 files changed, 825 insertions(+), 44 deletions(-) create mode 100644 macos/Astroshots/Narration/NarrationBrief.swift create mode 100644 macos/Astroshots/Narration/NarrationSpeech.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a63724..c486248 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,8 +17,22 @@ The two tracks are versioned independently. the tray while keeping their files on disk, then restore them individually or all at once from Settings. +### Changed + +- **Narrated-video captions are now opt-in:** Settings → Narration has a + **Burn in captions** toggle, off by default. Word-clock captions cannot stay + locked to Qwen's actual pacing. +- **Steadier narrator across a video:** each job synthesizes one short Ryan + (or chosen speaker) clip, then clones every later sentence from that same + reference so timbre does not wander between chunks. +- **Setup card after the intro:** narrated videos open the friction-log Goal + and Persona as a spoken, typeset page before the first screenshot. + ### Fixed +- **Narrated video cutoff and crawl:** long step transcripts are now spoken in + sentence chunks, trailing TTS silence is trimmed before captions are timed, + and the voice is asked to talk at a conversational pace (temperature 0.5). - **macOS dark mode:** tray, Settings, detail views, friction logs, overlays, and review sheets now use appearance-aware backgrounds, text, borders, and accent colors instead of mixing dark system text with fixed light cards. diff --git a/macos/Astroshots.xcodeproj/project.pbxproj b/macos/Astroshots.xcodeproj/project.pbxproj index b4e63e5..4105e40 100644 --- a/macos/Astroshots.xcodeproj/project.pbxproj +++ b/macos/Astroshots.xcodeproj/project.pbxproj @@ -21,6 +21,7 @@ 395A03C70C98F97840F02AF8 /* MLXAudioCore in Frameworks */ = {isa = PBXBuildFile; productRef = 9AEB54D8607EFEF17F160806 /* MLXAudioCore */; }; 3B7C5B7B558B59E7414950DF /* ReviewFlowUITests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 182E7298047007414ABC9862 /* ReviewFlowUITests.swift */; }; 3D70DEAA463D3459B886ED95 /* NarrationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C7975C24FB04A3D1A9DA4227 /* NarrationTests.swift */; }; + 418B4A3E6A6875B8EB6437F0 /* NarrationSpeech.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5E306BB2A5486653FEA77761 /* NarrationSpeech.swift */; }; 4775F0ACFAFE194C51D69855 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2B08FD45A80670CAD9391ADB /* AppDelegate.swift */; }; 4C581895750DA89007B483FF /* NarrationPaths.swift in Sources */ = {isa = PBXBuildFile; fileRef = 10EF202600AB264B34CDB2D7 /* NarrationPaths.swift */; }; 51D0FFA42C1196281E94E2F5 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 369606653B9FD1E6FBE4B43D /* Assets.xcassets */; }; @@ -33,9 +34,11 @@ 6A8F6DA71E32DD3A01ED8141 /* Manifest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2788CCB47877372A7EF96688 /* Manifest.swift */; }; 76EB28B98DBFABBF892E3A7E /* FrictionLogListView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 33713B379610D87587D40B40 /* FrictionLogListView.swift */; }; 77469FC8404628A2CE0A1894 /* astroshots-app-icon-master.png in Resources */ = {isa = PBXBuildFile; fileRef = F217601C57C50DA926C31AF2 /* astroshots-app-icon-master.png */; }; + 842084A0B15BCB986B44ADB1 /* ThemeTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 98F5F7C7BB45AE1FD2CD699B /* ThemeTests.swift */; }; 86F48D75FA114B1E154B6595 /* FrictionLogLoaderTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = E835F1DA4729DEB7570B9FBC /* FrictionLogLoaderTests.swift */; }; 87D9D684AF70B41C69A9A1C7 /* ReviewStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = D528EB0C9FF62471B7103498 /* ReviewStore.swift */; }; 8BB7EC1365F3F71AAD8DB5D8 /* FrictionStepDetailView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2A2EE3350232CD98BC7DF3D0 /* FrictionStepDetailView.swift */; }; + 8E263B6F9A4ED674DA7F4B2D /* NarrationBrief.swift in Sources */ = {isa = PBXBuildFile; fileRef = 180276769F3E77F32E31F62F /* NarrationBrief.swift */; }; 8F2395DC60D12581FB418608 /* AppState.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9976B5034C36AABFDFD4B2C8 /* AppState.swift */; }; 967A58CBD25BFB06C924D2DD /* NarrationModels.swift in Sources */ = {isa = PBXBuildFile; fileRef = C28BEB6B07761BB8C9BE0687 /* NarrationModels.swift */; }; 96FF7A41FF2B21E652A0DB9F /* AstroshotWatcher.swift in Sources */ = {isa = PBXBuildFile; fileRef = 976E87A9FC5F5847878A8A64 /* AstroshotWatcher.swift */; }; @@ -61,7 +64,6 @@ F4ADD26B485AABC510D0CD7E /* AstroshotWatcherTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 39A1E2488E103600176B240E /* AstroshotWatcherTests.swift */; }; F57D0C06CCA71E8AED668D7D /* Theme.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84934BBEECF7545EB5C02073 /* Theme.swift */; }; F8857D65FAC69AB00ED2A558 /* StreamView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5679FA304E48ED78A5B24170 /* StreamView.swift */; }; - A11D4A2B81C643ACA8771201 /* ThemeTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A11D4A2B81C643ACA8771202 /* ThemeTests.swift */; }; FC5EB07E736FC0E1582FD805 /* ImageClipboard.swift in Sources */ = {isa = PBXBuildFile; fileRef = CEA37DEE55FC5E2992C0612C /* ImageClipboard.swift */; }; FF7CB55DF3385E5F06077544 /* ReviewStoreTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = F9EA16401CCC820E6AD2BC9A /* ReviewStoreTests.swift */; }; /* End PBXBuildFile section */ @@ -90,6 +92,7 @@ 0B03ABCA3ACC16E733DE7B00 /* TrayView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TrayView.swift; sourceTree = ""; }; 0B730D3DA7EA103359F22B14 /* FrictionLogDetailView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FrictionLogDetailView.swift; sourceTree = ""; }; 10EF202600AB264B34CDB2D7 /* NarrationPaths.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NarrationPaths.swift; sourceTree = ""; }; + 180276769F3E77F32E31F62F /* NarrationBrief.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NarrationBrief.swift; sourceTree = ""; }; 182E7298047007414ABC9862 /* ReviewFlowUITests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ReviewFlowUITests.swift; sourceTree = ""; }; 2300722B17A148F2A115D3BC /* FrictionLogTrayJourneyUITests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FrictionLogTrayJourneyUITests.swift; sourceTree = ""; }; 239B5944E0638F1D931913A7 /* ShotIndexCache.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ShotIndexCache.swift; sourceTree = ""; }; @@ -111,6 +114,7 @@ 5239CF46065A15D17D2B56CF /* AstroshotsApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AstroshotsApp.swift; sourceTree = ""; }; 5679FA304E48ED78A5B24170 /* StreamView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StreamView.swift; sourceTree = ""; }; 5C52CACF01C306AEEF350902 /* NarrationVideoComposer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NarrationVideoComposer.swift; sourceTree = ""; }; + 5E306BB2A5486653FEA77761 /* NarrationSpeech.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NarrationSpeech.swift; sourceTree = ""; }; 70B6A1CB8442F588999CF4B1 /* ShotIndexCacheTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ShotIndexCacheTests.swift; sourceTree = ""; }; 71AECECE88B0D8F41B06334C /* ReviewTakeoverView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ReviewTakeoverView.swift; sourceTree = ""; }; 75955E0D26EB206BCE2025ED /* StatusItemController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StatusItemController.swift; sourceTree = ""; }; @@ -121,6 +125,7 @@ 8C127AB1558EF7894B101A53 /* StreamGroupingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StreamGroupingTests.swift; sourceTree = ""; }; 8F2AB8506FCE8107DD28D547 /* Astroshots.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = Astroshots.entitlements; sourceTree = ""; }; 976E87A9FC5F5847878A8A64 /* AstroshotWatcher.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AstroshotWatcher.swift; sourceTree = ""; }; + 98F5F7C7BB45AE1FD2CD699B /* ThemeTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ThemeTests.swift; sourceTree = ""; }; 9976B5034C36AABFDFD4B2C8 /* AppState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppState.swift; sourceTree = ""; }; 9E64F943CCC3A0ED542021C6 /* SoftwareUpdateDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SoftwareUpdateDelegate.swift; sourceTree = ""; }; A331496EBB7CAF55CCDEF017 /* ManifestParserTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ManifestParserTests.swift; sourceTree = ""; }; @@ -142,7 +147,6 @@ E9F84D4D30E15FCC4EECC1C2 /* NarrationJobQueue.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NarrationJobQueue.swift; sourceTree = ""; }; F217601C57C50DA926C31AF2 /* astroshots-app-icon-master.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = "astroshots-app-icon-master.png"; sourceTree = ""; }; F9EA16401CCC820E6AD2BC9A /* ReviewStoreTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ReviewStoreTests.swift; sourceTree = ""; }; - A11D4A2B81C643ACA8771202 /* ThemeTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ThemeTests.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -174,7 +178,7 @@ 70B6A1CB8442F588999CF4B1 /* ShotIndexCacheTests.swift */, 5089282794B5A6E6B3636886 /* ShotPathTests.swift */, 8C127AB1558EF7894B101A53 /* StreamGroupingTests.swift */, - A11D4A2B81C643ACA8771202 /* ThemeTests.swift */, + 98F5F7C7BB45AE1FD2CD699B /* ThemeTests.swift */, ); path = AstroshotsTests; sourceTree = ""; @@ -258,10 +262,12 @@ 797776DFC44DDFA0E00939BC /* Narration */ = { isa = PBXGroup; children = ( + 180276769F3E77F32E31F62F /* NarrationBrief.swift */, E9F84D4D30E15FCC4EECC1C2 /* NarrationJobQueue.swift */, B879878D858221AB43F32C45 /* NarrationModelManager.swift */, C28BEB6B07761BB8C9BE0687 /* NarrationModels.swift */, 10EF202600AB264B34CDB2D7 /* NarrationPaths.swift */, + 5E306BB2A5486653FEA77761 /* NarrationSpeech.swift */, 5C52CACF01C306AEEF350902 /* NarrationVideoComposer.swift */, ); path = Narration; @@ -489,7 +495,7 @@ A1C2A497863C6149280EDF41 /* ShotIndexCacheTests.swift in Sources */, D862C2464A53C71CC3BD45DF /* ShotPathTests.swift in Sources */, 699399301E1BE2AFEAEF81CA /* StreamGroupingTests.swift in Sources */, - A11D4A2B81C643ACA8771201 /* ThemeTests.swift in Sources */, + 842084A0B15BCB986B44ADB1 /* ThemeTests.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -509,10 +515,12 @@ DCD8CB2F02CA756A627DDE3F /* FrictionStepTakeoverView.swift in Sources */, FC5EB07E736FC0E1582FD805 /* ImageClipboard.swift in Sources */, 6A8F6DA71E32DD3A01ED8141 /* Manifest.swift in Sources */, + 8E263B6F9A4ED674DA7F4B2D /* NarrationBrief.swift in Sources */, 1B4177ACA9522A6931715AEE /* NarrationJobQueue.swift in Sources */, 351739C9880F7D8D2B6F27EF /* NarrationModelManager.swift in Sources */, 967A58CBD25BFB06C924D2DD /* NarrationModels.swift in Sources */, 4C581895750DA89007B483FF /* NarrationPaths.swift in Sources */, + 418B4A3E6A6875B8EB6437F0 /* NarrationSpeech.swift in Sources */, AC966A901816D42D9B701C48 /* NarrationVideoComposer.swift in Sources */, BB6D913C2F3291617685815E /* OverlayController.swift in Sources */, 17BFA74EE76E1E71A5627F33 /* OverlayView.swift in Sources */, @@ -648,7 +656,7 @@ CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; CODE_SIGN_STYLE = Automatic; COPY_PHASE_STRIP = NO; - CURRENT_PROJECT_VERSION = 22; + CURRENT_PROJECT_VERSION = 23; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; ENABLE_HARDENED_RUNTIME = YES; ENABLE_NS_ASSERTIONS = NO; @@ -662,7 +670,7 @@ GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; MACOSX_DEPLOYMENT_TARGET = 14.0; - MARKETING_VERSION = 0.2.5; + MARKETING_VERSION = 0.2.6; MTL_ENABLE_DEBUG_INFO = NO; MTL_FAST_MATH = YES; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -762,7 +770,7 @@ CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; CODE_SIGN_STYLE = Automatic; COPY_PHASE_STRIP = NO; - CURRENT_PROJECT_VERSION = 22; + CURRENT_PROJECT_VERSION = 23; DEBUG_INFORMATION_FORMAT = dwarf; ENABLE_HARDENED_RUNTIME = YES; ENABLE_STRICT_OBJC_MSGSEND = YES; @@ -782,7 +790,7 @@ GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; MACOSX_DEPLOYMENT_TARGET = 14.0; - MARKETING_VERSION = 0.2.5; + MARKETING_VERSION = 0.2.6; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; ONLY_ACTIVE_ARCH = YES; diff --git a/macos/Astroshots/App/AppState.swift b/macos/Astroshots/App/AppState.swift index 7affdd2..a23d3c4 100644 --- a/macos/Astroshots/App/AppState.swift +++ b/macos/Astroshots/App/AppState.swift @@ -58,6 +58,7 @@ final class AppState { /// Opt-in narrated friction-log videos (MLX Audio + Qwen3-TTS). var narrationEnabled: Bool var narrationVoice: String + var narrationCaptionsEnabled: Bool let narration: NarrationModelManager let narrationQueue: NarrationJobQueue var watchRootPaths: [String] @@ -160,6 +161,7 @@ final class AppState { autoDismiss = preferences.autoDismiss narrationEnabled = preferences.narrationEnabled narrationVoice = preferences.narrationVoice + narrationCaptionsEnabled = preferences.narrationCaptionsEnabled hiddenFrictionLogIDs = preferences.hiddenFrictionLogIDs watchRootPaths = rootPaths needsWatchRootSetup = needsSetup @@ -601,10 +603,20 @@ final class AppState { preferences.narrationVoice = normalized } + func setNarrationCaptionsEnabled(_ enabled: Bool) { + narrationCaptionsEnabled = enabled + preferences.narrationCaptionsEnabled = enabled + } + @discardableResult func enqueueNarration(for log: FrictionLog, run: FrictionLogRun) -> Bool { do { - _ = try narrationQueue.enqueue(log: log, run: run, voice: narrationVoice) + _ = try narrationQueue.enqueue( + log: log, + run: run, + voice: narrationVoice, + showCaptions: narrationCaptionsEnabled + ) showToast("Narration queued") return true } catch { diff --git a/macos/Astroshots/App/Preferences.swift b/macos/Astroshots/App/Preferences.swift index a5265af..a597f9d 100644 --- a/macos/Astroshots/App/Preferences.swift +++ b/macos/Astroshots/App/Preferences.swift @@ -25,6 +25,9 @@ final class Preferences { static let narrationModelReady = "narrationModelReady" /// Named Qwen3 CustomVoice speaker used for previews and every render. static let narrationVoice = "narrationVoice" + /// Burned-in captions on narrated MP4s. Default off: the word clock + /// cannot stay locked to Qwen's actual pacing. + static let narrationCaptionsEnabled = "narrationCaptionsEnabled" } /// Migration schema for first-run detection. Increment when adoption rules @@ -148,6 +151,12 @@ final class Preferences { set { defaults.set(NarrationVoice.normalized(newValue), forKey: Key.narrationVoice) } } + /// Missing key is off (`UserDefaults.bool` default). + var narrationCaptionsEnabled: Bool { + get { defaults.bool(forKey: Key.narrationCaptionsEnabled) } + set { defaults.set(newValue, forKey: Key.narrationCaptionsEnabled) } + } + /// Call when the user successfully chooses initial watch folders. func markFirstRunSetupComplete() { hasCompletedFirstRunSetup = true diff --git a/macos/Astroshots/Features/Tray/SettingsPane.swift b/macos/Astroshots/Features/Tray/SettingsPane.swift index bba3f7f..c99d2fc 100644 --- a/macos/Astroshots/Features/Tray/SettingsPane.swift +++ b/macos/Astroshots/Features/Tray/SettingsPane.swift @@ -278,6 +278,13 @@ struct SettingsPane: View { .overlay(alignment: .top) { Rectangle().fill(Theme.line).frame(height: 1) } + + toggleRow( + "Burn in captions", + isOn: appState.narrationCaptionsEnabled + ) { enabled in + appState.setNarrationCaptionsEnabled(enabled) + } } VStack(alignment: .leading, spacing: 6) { diff --git a/macos/Astroshots/Models/FrictionLog.swift b/macos/Astroshots/Models/FrictionLog.swift index 0549001..a83ef00 100644 --- a/macos/Astroshots/Models/FrictionLog.swift +++ b/macos/Astroshots/Models/FrictionLog.swift @@ -201,6 +201,24 @@ enum FrictionLogLoader { var captured_at: String? } + /// Load the log + run that own a `runs//` directory. + static func loadRun(directory: URL) -> (FrictionLog, FrictionLogRun)? { + let runDir = directory.standardizedFileURL + let slugDir = runDir.deletingLastPathComponent().deletingLastPathComponent() + let frictionRoot = slugDir.deletingLastPathComponent() + let astroshot = frictionRoot.deletingLastPathComponent() + guard frictionRoot.lastPathComponent == FrictionLogPath.directoryName, + astroshot.lastPathComponent == ShotPath.astroshotDirName + else { return nil } + let logs = loadLogs(inAstroshot: astroshot) + let runID = runDir.lastPathComponent + let slug = slugDir.lastPathComponent + guard let log = logs.first(where: { $0.slug == slug }), + let run = log.runs.first(where: { $0.runID == runID }) + else { return nil } + return (log, run) + } + /// Scan `.astroshot/friction-logs/*` under a single `.astroshot` directory. static func loadLogs(inAstroshot astroshot: URL) -> [FrictionLog] { let root = FrictionLogPath.root(inAstroshot: astroshot) diff --git a/macos/Astroshots/Narration/NarrationBrief.swift b/macos/Astroshots/Narration/NarrationBrief.swift new file mode 100644 index 0000000..d66546e --- /dev/null +++ b/macos/Astroshots/Narration/NarrationBrief.swift @@ -0,0 +1,80 @@ +import Foundation + +/// Setup card after the intro: who this person is and what they came to do. +struct NarrationBrief: Equatable, Sendable { + var title: String + var persona: String + var goal: String + var transcript: String + + var hasContent: Bool { + !persona.isEmpty || !goal.isEmpty + } + + static func make(log: FrictionLog) -> NarrationBrief { + make( + title: log.title, + description: log.description, + promptMarkdown: log.promptMarkdown + ) + } + + static func make( + title: String, + description: String, + promptMarkdown: String? + ) -> NarrationBrief { + let persona = clean(section("persona", in: promptMarkdown ?? "")) + let goal = clean(section("goal", in: promptMarkdown ?? "")) + let resolvedGoal = goal.isEmpty ? clean(description) : goal + return NarrationBrief( + title: title.trimmingCharacters(in: .whitespacesAndNewlines), + persona: persona, + goal: resolvedGoal, + transcript: spokenTranscript(persona: persona, goal: resolvedGoal) + ) + } + + static func spokenTranscript(persona: String, goal: String) -> String { + var parts: [String] = [] + if !persona.isEmpty { parts.append(persona) } + if !goal.isEmpty { parts.append(goal) } + guard !parts.isEmpty else { return "" } + parts.append("We follow them through it.") + return parts.joined(separator: " ") + } + + /// Body of `## Heading` until the next ATX heading. Skips `$ENV` bullets. + static func section(_ name: String, in markdown: String) -> String { + let wanted = name.lowercased() + var capturing = false + var lines: [String] = [] + for raw in markdown.components(separatedBy: .newlines) { + let trimmed = raw.trimmingCharacters(in: .whitespaces) + if trimmed.hasPrefix("#") { + let heading = trimmed + .trimmingCharacters(in: CharacterSet(charactersIn: "#").union(.whitespaces)) + .lowercased() + if capturing { break } + capturing = heading == wanted || heading.hasPrefix(wanted + " ") + continue + } + guard capturing, !trimmed.isEmpty else { continue } + if trimmed.hasPrefix("- ") || trimmed.hasPrefix("* ") { + let item = String(trimmed.dropFirst(2)) + .trimmingCharacters(in: .whitespaces) + if item.contains("$") { continue } + lines.append(item) + } else { + lines.append(trimmed) + } + } + return lines.joined(separator: " ") + } + + private static func clean(_ text: String) -> String { + text + .replacingOccurrences(of: #"\s+"#, with: " ", options: .regularExpression) + .trimmingCharacters(in: .whitespacesAndNewlines) + } +} diff --git a/macos/Astroshots/Narration/NarrationJobQueue.swift b/macos/Astroshots/Narration/NarrationJobQueue.swift index 44af17f..f7bbb03 100644 --- a/macos/Astroshots/Narration/NarrationJobQueue.swift +++ b/macos/Astroshots/Narration/NarrationJobQueue.swift @@ -36,7 +36,8 @@ final class NarrationJobQueue { func enqueue( log: FrictionLog, run: FrictionLogRun, - voice: String = NarrationDefaults.voice + voice: String = NarrationDefaults.voice, + showCaptions: Bool = false ) throws -> NarrationJob { guard modelManager.readiness.isReady else { throw NarrationError.notReady @@ -51,7 +52,12 @@ final class NarrationJobQueue { return existing } - let job = NarrationJob.make(log: log, run: run, voice: voice) + let job = NarrationJob.make( + log: log, + run: run, + voice: voice, + showCaptions: showCaptions + ) jobs.insert(job, at: 0) if jobs.count > 40 { jobs = Array(jobs.prefix(40)) @@ -81,6 +87,47 @@ final class NarrationJobQueue { NSWorkspace.shared.activateFileViewerSelecting([URL(fileURLWithPath: path)]) } + /// Run the same pipeline as the tray queue, but await the MP4 path. + func renderImmediately( + log: FrictionLog, + run: FrictionLogRun, + voice: String = NarrationDefaults.voice, + showCaptions: Bool = false + ) async throws -> URL { + let job = NarrationJob.make( + log: log, + run: run, + voice: voice, + showCaptions: showCaptions + ) + jobs.insert(job, at: 0) + activeJobID = job.id + update(id: job.id) { + $0.phase = .synthesizing + $0.progress = 0.02 + $0.message = "Loading Qwen3-TTS…" + } + do { + let output = try await render(job: job) + update(id: job.id) { + $0.phase = .complete + $0.progress = 1 + $0.message = "Narrated video ready" + $0.outputPath = output.path + } + activeJobID = nil + return output + } catch { + update(id: job.id) { + $0.phase = .failed + $0.message = (error as? LocalizedError)?.errorDescription + ?? error.localizedDescription + } + activeJobID = nil + throw error + } + } + private func kick() { guard worker == nil else { return } worker = Task { [weak self] in @@ -140,12 +187,47 @@ final class NarrationJobQueue { try FileManager.default.createDirectory(at: workDir, withIntermediateDirectories: true) let model = try await modelManager.modelForSynthesis() - let total = max(job.stepTranscripts.count, 1) + update(id: job.id) { + $0.message = "Locking narrator voice…" + } + let anchor = try await modelManager.makeVoiceAnchor(voice: job.voice, using: model) + let hasBrief = job.brief?.hasContent == true + let total = max(job.stepTranscripts.count + (hasBrief ? 1 : 0), 1) var media: [NarrationVideoComposer.StepMedia] = [] + if let brief = job.brief, brief.hasContent { + try Task.checkCancellation() + update(id: job.id) { + $0.phase = .synthesizing + $0.progress = 0.04 + $0.message = "Speaking the setup…" + } + let wav = workDir.appendingPathComponent("context.wav") + let duration = try await synthesize( + model: model, + text: brief.transcript, + voice: job.voice, + output: wav, + anchor: anchor + ) + media.append( + NarrationVideoComposer.StepMedia( + imagePath: nil, + audioPath: wav.path, + duration: duration, + title: brief.title, + transcript: brief.transcript, + isContextCard: true, + contextPersona: brief.persona, + contextGoal: brief.goal + ) + ) + } + for (index, step) in job.stepTranscripts.enumerated() { try Task.checkCancellation() - let base = Double(index) / Double(total) + let spokenIndex = index + (hasBrief ? 1 : 0) + let base = Double(spokenIndex) / Double(total) let span = 0.75 / Double(total) update(id: job.id) { $0.phase = .synthesizing @@ -160,7 +242,8 @@ final class NarrationJobQueue { model: model, text: step.transcript, voice: job.voice, - output: wav + output: wav, + anchor: anchor ) update(id: job.id) { $0.progress = base + span @@ -193,7 +276,8 @@ final class NarrationJobQueue { try await NarrationVideoComposer.compose( steps: media, outputURL: output, - logTitle: job.logTitle + logTitle: job.logTitle, + showCaptions: job.showCaptions ) update(id: job.id) { @@ -207,7 +291,8 @@ final class NarrationJobQueue { model: any SpeechGenerationModel, text: String, voice: String, - output: URL + output: URL, + anchor: NarrationVoiceAnchor ) async throws -> Double { let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { @@ -217,7 +302,8 @@ final class NarrationJobQueue { let audio = try await modelManager.synthesizeAudio( trimmed, voice: voice, - using: model + using: model, + anchor: anchor ) let samples = audio.samples guard !samples.isEmpty else { diff --git a/macos/Astroshots/Narration/NarrationModelManager.swift b/macos/Astroshots/Narration/NarrationModelManager.swift index 42724fa..5076da0 100644 --- a/macos/Astroshots/Narration/NarrationModelManager.swift +++ b/macos/Astroshots/Narration/NarrationModelManager.swift @@ -174,17 +174,33 @@ final class NarrationModelManager { func synthesizeAudio( _ text: String, voice: String, - using model: any SpeechGenerationModel + using model: any SpeechGenerationModel, + anchor: NarrationVoiceAnchor? = nil ) async throws -> (samples: [Float], sampleRate: Int) { await acquireSynthesisSlot() defer { releaseSynthesisSlot() } try Task.checkCancellation() let box = NarrationSpeechBox(model) - let samples = try await box.speak(text, voice: NarrationVoice.normalized(voice)) + let samples = try await box.speak( + text, + voice: NarrationVoice.normalized(voice), + anchor: anchor + ) return (samples, box.sampleRate) } + /// One CustomVoice clip for the job; later chunks clone it so timbre does not wander. + func makeVoiceAnchor( + voice: String, + using model: any SpeechGenerationModel + ) async throws -> NarrationVoiceAnchor { + await acquireSynthesisSlot() + defer { releaseSynthesisSlot() } + try Task.checkCancellation() + return try await NarrationSpeechBox(model).makeAnchor(voice: voice) + } + private func acquireSynthesisSlot() async { if !synthesisBusy { synthesisBusy = true @@ -293,6 +309,20 @@ final class NarrationModelManager { } } +/// One job-scoped reference clip. Reuse the same `audio` instance so Qwen's +/// ICL cache keys on object identity instead of re-encoding every chunk. +final class NarrationVoiceAnchor: @unchecked Sendable { + let text: String + let audio: MLXArray + let canClone: Bool + + init(text: String, audio: MLXArray, canClone: Bool) { + self.text = text + self.audio = audio + self.canClone = canClone + } +} + /// Serial TTS access for non-Sendable `SpeechGenerationModel` instances. final class NarrationSpeechBox: @unchecked Sendable { private let model: any SpeechGenerationModel @@ -303,18 +333,93 @@ final class NarrationSpeechBox: @unchecked Sendable { var sampleRate: Int { model.sampleRate } - func speak(_ text: String, voice: String = NarrationDefaults.voice) async throws -> [Float] { - let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + func makeAnchor(voice: String) async throws -> NarrationVoiceAnchor { var generationParameters = model.defaultGenerationParameters - generationParameters.temperature = NarrationDefaults.temperature - let audio: MLXArray = try await model.generate( - text: trimmed, - voice: NarrationVoice.normalized(voice), + generationParameters.temperature = 0.3 + let speaker = NarrationVoice.normalized(voice) + let raw = try await model.generate( + text: NarrationSpeech.referenceText, + voice: speaker, refAudio: nil, refText: nil, language: NarrationDefaults.language, generationParameters: generationParameters ) - return audio.asArray(Float.self) + let samples = NarrationSpeech.trimSilence( + raw.asArray(Float.self), + sampleRate: sampleRate + ) + guard !samples.isEmpty else { + throw NarrationError.processFailed("Voice reference clip was empty.") + } + let audio = MLXArray(samples) + var canClone = false + if let qwen = model as? Qwen3TTSModel { + do { + _ = try qwen.prepareReferenceConditioning( + refAudio: audio, + refText: NarrationSpeech.referenceText, + language: NarrationDefaults.language + ) + canClone = true + } catch { + canClone = false + } + } + return NarrationVoiceAnchor( + text: NarrationSpeech.referenceText, + audio: audio, + canClone: canClone + ) + } + + func speak( + _ text: String, + voice: String = NarrationDefaults.voice, + anchor: NarrationVoiceAnchor? = nil + ) async throws -> [Float] { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return [] } + + var generationParameters = model.defaultGenerationParameters + generationParameters.temperature = NarrationDefaults.temperature + let speaker = NarrationVoice.normalized(voice) + let pieces = NarrationSpeech.chunks(for: trimmed) + let clone = anchor?.canClone == true + var combined: [Float] = [] + + for (index, piece) in pieces.enumerated() { + try Task.checkCancellation() + let audio: MLXArray + if clone, let anchor { + audio = try await model.generate( + text: piece, + voice: nil, + refAudio: anchor.audio, + refText: anchor.text, + language: NarrationDefaults.language, + generationParameters: generationParameters + ) + } else { + audio = try await model.generate( + text: piece, + voice: speaker, + refAudio: nil, + refText: nil, + language: NarrationDefaults.language, + generationParameters: generationParameters + ) + } + let spoken = NarrationSpeech.trimSilence( + audio.asArray(Float.self), + sampleRate: sampleRate + ) + guard !spoken.isEmpty else { continue } + if index > 0 { + combined.append(contentsOf: NarrationSpeech.interChunkSilence(sampleRate: sampleRate)) + } + combined.append(contentsOf: spoken) + } + return combined } } diff --git a/macos/Astroshots/Narration/NarrationModels.swift b/macos/Astroshots/Narration/NarrationModels.swift index aa6685e..def59be 100644 --- a/macos/Astroshots/Narration/NarrationModels.swift +++ b/macos/Astroshots/Narration/NarrationModels.swift @@ -6,8 +6,8 @@ enum NarrationDefaults { /// CustomVoice English preset (see Qwen3-TTS README). static let voice = "Ryan" static let language = "English" - /// Keep independently synthesized friction-log steps sounding like one narrator. - static let temperature: Float = 0.3 + /// Low enough to keep one speaker across steps, high enough to sound human. + static let temperature: Float = 0.5 /// Minimum step display duration when audio is shorter. static let minimumStepSeconds: Double = 1.6 static let framesPerSecond: Int32 = 12 @@ -135,6 +135,8 @@ struct NarrationJob: Identifiable, Equatable, Sendable { let runID: String let runDirectoryPath: String let voice: String + let showCaptions: Bool + let brief: NarrationBrief? let stepTranscripts: [NarrationStepInput] var phase: NarrationJobPhase var progress: Double @@ -151,7 +153,8 @@ extension NarrationJob { static func make( log: FrictionLog, run: FrictionLogRun, - voice: String = NarrationDefaults.voice + voice: String = NarrationDefaults.voice, + showCaptions: Bool = false ) -> NarrationJob { let steps = run.steps.map { step in NarrationStepInput( @@ -172,6 +175,11 @@ extension NarrationJob { runID: run.runID, runDirectoryPath: run.directoryPath, voice: NarrationVoice.normalized(voice), + showCaptions: showCaptions, + brief: { + let brief = NarrationBrief.make(log: log) + return brief.hasContent ? brief : nil + }(), stepTranscripts: steps, phase: .queued, progress: 0, diff --git a/macos/Astroshots/Narration/NarrationSpeech.swift b/macos/Astroshots/Narration/NarrationSpeech.swift new file mode 100644 index 0000000..841afca --- /dev/null +++ b/macos/Astroshots/Narration/NarrationSpeech.swift @@ -0,0 +1,169 @@ +import Foundation + +/// Pure speech-prep helpers for Qwen3-TTS: chunk long transcripts, trim the +/// silence the model leaves after it stops talking, and attach a pace prompt. +enum NarrationSpeech { + static let targetChunkWords = 24 + static let maxChunkWords = 42 + static let silenceRMSThreshold: Float = 0.01 + static let analysisWindowSeconds = 0.05 + static let leadingPadSeconds = 0.08 + static let trailingPadSeconds = 0.2 + static let interChunkSilenceSeconds = 0.12 + static let paceInstruction = + "speaking at a natural conversational pace, like a colleague walking through a product" + /// ~3s CustomVoice line, then cloned via ICL so every later chunk shares one timbre. + static let referenceText = "This is the narrator for this Astroshots friction log." + + static func voicePrompt(speaker: String) -> String { + let speaker = NarrationVoice.normalized(speaker) + return "\(speaker), \(paceInstruction)" + } + + static func chunks(for text: String) -> [String] { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return [] } + + var result: [String] = [] + var current: [String] = [] + var currentWords = 0 + + func flush() { + let joined = current.joined(separator: " ") + .trimmingCharacters(in: .whitespacesAndNewlines) + if !joined.isEmpty { result.append(joined) } + current = [] + currentWords = 0 + } + + for piece in clauses(in: trimmed) { + let words = wordCount(piece) + guard words > 0 else { continue } + if !current.isEmpty { + let combined = currentWords + words + if combined > maxChunkWords || currentWords >= targetChunkWords { + flush() + } + } + current.append(piece) + currentWords += words + } + if !current.isEmpty { flush() } + return result + } + + /// Drops leading/trailing near-silence so captions and the step hold follow + /// speech, not the leftover WAV the model fills up to its token cap. + static func trimSilence(_ samples: [Float], sampleRate: Int) -> [Float] { + guard !samples.isEmpty, sampleRate > 0 else { return samples } + let window = max(1, Int(Double(sampleRate) * analysisWindowSeconds)) + + func rms(at start: Int) -> Float { + let end = min(samples.count, start + window) + guard end > start else { return 0 } + var sum: Float = 0 + for index in start ..< end { + let sample = samples[index] + sum += sample * sample + } + return sqrt(sum / Float(end - start)) + } + + var first = 0 + var foundSpeech = false + var index = 0 + while index < samples.count { + if rms(at: index) > silenceRMSThreshold { + first = index + foundSpeech = true + break + } + index += window + } + guard foundSpeech else { return samples } + + var last = samples.count + index = ((samples.count - 1) / window) * window + while index >= 0 { + if rms(at: index) > silenceRMSThreshold { + last = min(samples.count, index + window) + break + } + index -= window + } + guard last > first else { return samples } + + let lead = Int(leadingPadSeconds * Double(sampleRate)) + let tail = Int(trailingPadSeconds * Double(sampleRate)) + let start = max(0, first - lead) + let end = min(samples.count, last + tail) + return Array(samples[start ..< end]) + } + + static func interChunkSilence(sampleRate: Int) -> [Float] { + let count = max(0, Int(interChunkSilenceSeconds * Double(max(sampleRate, 1)))) + return Array(repeating: 0, count: count) + } + + static func wordCount(_ text: String) -> Int { + text.split(whereSeparator: { $0.isWhitespace }).count + } + + /// Sentences first, then long sentences on commas / semicolons / dashes. + static func clauses(in text: String) -> [String] { + var sentences: [String] = [] + var current = "" + let scalars = Array(text) + var index = 0 + while index < scalars.count { + let character = scalars[index] + current.append(character) + let isEnd = character == "." || character == "!" || character == "?" + let nextIsSpaceOrEnd = index + 1 >= scalars.count + || scalars[index + 1].isWhitespace + if isEnd, nextIsSpaceOrEnd { + let piece = current.trimmingCharacters(in: .whitespacesAndNewlines) + if !piece.isEmpty { sentences.append(piece) } + current = "" + } + index += 1 + } + let tail = current.trimmingCharacters(in: .whitespacesAndNewlines) + if !tail.isEmpty { sentences.append(tail) } + + return sentences.flatMap { sentence -> [String] in + guard wordCount(sentence) > maxChunkWords else { return [sentence] } + return splitLongSentence(sentence) + } + } + + private static func splitLongSentence(_ sentence: String) -> [String] { + let separators = CharacterSet(charactersIn: ",;—–") + let raw = sentence.components(separatedBy: separators) + .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) } + .filter { !$0.isEmpty } + guard raw.count > 1 else { return [sentence] } + + var parts: [String] = [] + var current: [String] = [] + var currentWords = 0 + for piece in raw { + let words = wordCount(piece) + if !current.isEmpty, currentWords + words > maxChunkWords { + parts.append(current.joined(separator: ", ") + ".") + current = [] + currentWords = 0 + } + current.append(piece) + currentWords += words + } + if !current.isEmpty { + var joined = current.joined(separator: ", ") + if !joined.hasSuffix(".") && !joined.hasSuffix("!") && !joined.hasSuffix("?") { + joined += "." + } + parts.append(joined) + } + return parts + } +} diff --git a/macos/Astroshots/Narration/NarrationVideoComposer.swift b/macos/Astroshots/Narration/NarrationVideoComposer.swift index be30e1d..0c32078 100644 --- a/macos/Astroshots/Narration/NarrationVideoComposer.swift +++ b/macos/Astroshots/Narration/NarrationVideoComposer.swift @@ -17,6 +17,9 @@ enum NarrationVideoComposer { var good: [String] = [] var improve: [String] = [] var isBrandCard = false + var isContextCard = false + var contextPersona: String = "" + var contextGoal: String = "" } /// AVAssetWriter is designed for independently fed inputs but its Objective-C @@ -51,7 +54,8 @@ enum NarrationVideoComposer { steps: [StepMedia], outputURL: URL, logTitle: String = "Astroshots", - size: CGSize = CGSize(width: 1280, height: 720) + size: CGSize = CGSize(width: 1280, height: 720), + showCaptions: Bool = false ) async throws { guard !steps.isEmpty else { throw NarrationError.processFailed("No steps to encode.") @@ -81,7 +85,12 @@ enum NarrationVideoComposer { var segmentURLs: [URL] = [] for (index, step) in timeline.enumerated() { let segment = workRoot.appendingPathComponent(String(format: "seg-%02d.mp4", index)) - try await writeSegment(step: step, outputURL: segment, size: size) + try await writeSegment( + step: step, + outputURL: segment, + size: size, + showCaptions: showCaptions && !step.isBrandCard && !step.isContextCard + ) segmentURLs.append(segment) } @@ -97,7 +106,8 @@ enum NarrationVideoComposer { private static func writeSegment( step: StepMedia, outputURL: URL, - size: CGSize + size: CGSize, + showCaptions: Bool ) async throws { let duration = max(step.duration, NarrationDefaults.minimumStepSeconds) let writer = try AVAssetWriter(outputURL: outputURL, fileType: .mp4) @@ -171,7 +181,8 @@ enum NarrationVideoComposer { step: context.step, size: size, frameCount: frameCount, - frameDuration: frameDuration + frameDuration: frameDuration, + showCaptions: showCaptions ) context.videoInput.markAsFinished() } @@ -212,7 +223,8 @@ enum NarrationVideoComposer { step: StepMedia, size: CGSize, frameCount: Int, - frameDuration: CMTime + frameDuration: CMTime, + showCaptions: Bool ) async throws { var index = 0 while index < frameCount { @@ -229,7 +241,8 @@ enum NarrationVideoComposer { step: step, elapsed: elapsed, duration: Double(frameCount) / Double(frameDuration.timescale), - size: size + size: size, + showCaptions: showCaptions ) else { throw NarrationError.processFailed("Could not create video frame.") } @@ -401,7 +414,8 @@ enum NarrationVideoComposer { step: StepMedia, elapsed: Double, duration: Double, - size: CGSize + size: CGSize, + showCaptions: Bool ) -> CVPixelBuffer? { var buffer: CVPixelBuffer? let attrs: [CFString: Any] = [ @@ -432,7 +446,7 @@ enum NarrationVideoComposer { | CGBitmapInfo.byteOrder32Little.rawValue ) else { return nil } - let backgroundColor = step.isBrandCard + let backgroundColor = step.isBrandCard || step.isContextCard ? NSColor.black : NSColor(calibratedWhite: 0.035, alpha: 1) context.setFillColor(backgroundColor.cgColor) @@ -442,6 +456,10 @@ enum NarrationVideoComposer { drawBrandOverlay(title: step.title, size: size, context: context) return buffer } + if step.isContextCard { + drawContextOverlay(step: step, size: size, context: context) + return buffer + } let feedbackAlpha = feedbackOpacity( elapsed: elapsed, @@ -472,12 +490,14 @@ enum NarrationVideoComposer { context: context ) } - drawCaptions( - captionText(step.transcript, elapsed: elapsed, duration: duration), - feedbackAlpha: feedbackAlpha, - size: size, - context: context - ) + if showCaptions { + drawCaptions( + captionText(step.transcript, elapsed: elapsed, duration: duration), + feedbackAlpha: feedbackAlpha, + size: size, + context: context + ) + } return buffer } @@ -607,6 +627,98 @@ enum NarrationVideoComposer { } } + private static func drawContextOverlay( + step: StepMedia, + size: CGSize, + context: CGContext + ) { + withFlippedAppKitContext(context) { + let inset: CGFloat = 88 + let width = size.width - inset * 2 + var y: CGFloat = 56 + + ("THE SETUP" as NSString).draw( + in: CGRect(x: inset, y: y, width: width, height: 22), + withAttributes: [ + .font: NSFont.systemFont(ofSize: 13, weight: .bold), + .foregroundColor: NSColor.white.withAlphaComponent(0.55), + .kern: 1.6, + ] + ) + y += 28 + + let titleStyle = NSMutableParagraphStyle() + titleStyle.lineBreakMode = .byWordWrapping + (step.title as NSString).draw( + in: CGRect(x: inset, y: y, width: width, height: 88), + withAttributes: [ + .font: NSFont.systemFont(ofSize: 36, weight: .bold), + .foregroundColor: NSColor.white, + .paragraphStyle: titleStyle, + ] + ) + y += 100 + + if !step.contextPersona.isEmpty { + y = drawContextBlock( + label: "WHO", + body: step.contextPersona, + x: inset, + y: y, + width: width + ) + y += 28 + } + if !step.contextGoal.isEmpty { + _ = drawContextBlock( + label: "WHAT THEY WANT", + body: step.contextGoal, + x: inset, + y: y, + width: width + ) + } + } + } + + private static func drawContextBlock( + label: String, + body: String, + x: CGFloat, + y: CGFloat, + width: CGFloat + ) -> CGFloat { + (label as NSString).draw( + in: CGRect(x: x, y: y, width: width, height: 20), + withAttributes: [ + .font: NSFont.systemFont(ofSize: 12, weight: .bold), + .foregroundColor: NSColor(calibratedRed: 0.62, green: 0.48, blue: 1.0, alpha: 1), + .kern: 1.4, + ] + ) + let bodyStyle = NSMutableParagraphStyle() + bodyStyle.lineBreakMode = .byWordWrapping + bodyStyle.lineSpacing = 4 + let bodyFrame = CGRect(x: x, y: y + 26, width: width, height: 168) + (body as NSString).draw( + in: bodyFrame, + withAttributes: [ + .font: NSFont.systemFont(ofSize: 22, weight: .medium), + .foregroundColor: NSColor.white.withAlphaComponent(0.92), + .paragraphStyle: bodyStyle, + ] + ) + let bodyHeight = (body as NSString).boundingRect( + with: CGSize(width: width, height: 168), + options: [.usesLineFragmentOrigin, .usesFontLeading], + attributes: [ + .font: NSFont.systemFont(ofSize: 22, weight: .medium), + .paragraphStyle: bodyStyle, + ] + ).height + return y + 26 + min(168, ceil(bodyHeight)) + } + private static func drawFeedbackRail( step: StepMedia, alpha: CGFloat, diff --git a/macos/AstroshotsTests/FrictionLogLoaderTests.swift b/macos/AstroshotsTests/FrictionLogLoaderTests.swift index d98f941..286f545 100644 --- a/macos/AstroshotsTests/FrictionLogLoaderTests.swift +++ b/macos/AstroshotsTests/FrictionLogLoaderTests.swift @@ -98,6 +98,10 @@ struct FrictionLogLoaderTests { #expect(log.runs.count == 1) #expect(log.runs[0].runID == "20260807T120000Z") #expect(log.runs[0].steps.count == 1) + let located = try #require(FrictionLogLoader.loadRun(directory: runDir)) + #expect(located.0.slug == "checkout-as-new-user") + #expect(located.1.runID == "20260807T120000Z") + #expect(FrictionLogLoader.loadRun(directory: worktree) == nil) #expect(log.stepCount == 1) #expect(log.goodCount == 1) } diff --git a/macos/AstroshotsTests/NarrationTests.swift b/macos/AstroshotsTests/NarrationTests.swift index 3628f88..d3d3f10 100644 --- a/macos/AstroshotsTests/NarrationTests.swift +++ b/macos/AstroshotsTests/NarrationTests.swift @@ -48,7 +48,11 @@ struct NarrationTests { #expect(job.stepTranscripts.count == 1) #expect(job.stepTranscripts[0].transcript.contains("open the app")) #expect(job.voice == NarrationDefaults.voice) + #expect(job.showCaptions == false) + #expect(job.brief == nil) #expect(job.phase == .queued) + let captioned = NarrationJob.make(log: log, run: run, showCaptions: true) + #expect(captioned.showCaptions) } @Test func readinessStatusLinesAreStable() { @@ -62,7 +66,7 @@ struct NarrationTests { @Test func defaultModelIsQwen3TTS() { #expect(NarrationDefaults.modelID.contains("Qwen3-TTS")) #expect(NarrationDefaults.voice == "Ryan") - #expect(NarrationDefaults.temperature == 0.3) + #expect(NarrationDefaults.temperature == 0.5) #expect(NarrationVoice.available.map(\.id).contains("Aiden")) } @@ -177,6 +181,93 @@ struct NarrationTests { #expect(NarrationJobQueue.safeFilenameComponent("✨") == "Friction-Log") } + @Test func speechChunksPackSentencesUnderTheTokenBudget() { + let text = """ + I land on the portal as a new hire, hunting a backend for a flavor chat. \ + The hero is sharp and names every store as its own tenant. \ + The visual of Northstar and Harbor already feels like shops. \ + The hitch is the loudest button is Talk to us, with Sign in in the header. \ + I scroll for a path that sounds like embed a chat agent. + """ + let chunks = NarrationSpeech.chunks(for: text) + #expect(chunks.count >= 2) + #expect(chunks.count <= 4) + #expect(chunks.joined(separator: " ").contains("flavor chat")) + #expect(chunks.last?.contains("scroll") == true) + for chunk in chunks { + #expect(NarrationSpeech.wordCount(chunk) <= NarrationSpeech.maxChunkWords) + } + #expect(NarrationSpeech.chunks(for: " ").isEmpty) + #expect(NarrationSpeech.chunks(for: "One short line.").count == 1) + } + + @Test func speechVoicePromptKeepsTheSpeakerAndAsksForPace() { + let prompt = NarrationSpeech.voicePrompt(speaker: "Ryan") + #expect(prompt.hasPrefix("Ryan,")) + #expect(prompt.contains("conversational pace")) + #expect(NarrationSpeech.voicePrompt(speaker: "not-a-voice").hasPrefix("Ryan,")) + } + + @Test func briefReadsGoalAndPersonaFromThePrompt() { + let markdown = """ + # Ice cream + + ## Goal + Decide whether ArchAstro can back a flavor-chat storefront. + + ## Persona + Day-one contractor on the ice cream account. + + ## Environment + - Base URL: $PORTAL_URL (local) + - Fresh browser session + """ + let brief = NarrationBrief.make( + title: "Ice cream chain evaluates developer portal", + description: "meta fallback", + promptMarkdown: markdown + ) + #expect(brief.hasContent) + #expect(brief.goal.contains("flavor-chat")) + #expect(brief.persona.contains("Day-one contractor")) + #expect(!brief.transcript.contains("$PORTAL_URL")) + #expect(brief.transcript.contains("We follow them through it.")) + #expect(NarrationBrief.section("environment", in: markdown).contains("Fresh browser")) + #expect(!NarrationBrief.section("environment", in: markdown).contains("$PORTAL_URL")) + + let fallback = NarrationBrief.make( + title: "Checkout", + description: "Fresh account, empty cart.", + promptMarkdown: nil + ) + #expect(fallback.goal.contains("empty cart")) + #expect(fallback.persona.isEmpty) + #expect(NarrationBrief.make(title: "X", description: "", promptMarkdown: nil).hasContent == false) + } + + @Test func voiceReferenceLineIsShortEnoughToClone() { + let words = NarrationSpeech.wordCount(NarrationSpeech.referenceText) + #expect(words >= 6) + #expect(words <= 16) + #expect(NarrationSpeech.referenceText.hasSuffix(".")) + #expect(NarrationSpeech.chunks(for: NarrationSpeech.referenceText).count == 1) + } + + @Test func silenceTrimDropsTheTokenCapTail() { + let rate = 24_000 + var samples = [Float](repeating: 0, count: rate * 8) + for index in (rate / 2) ..< (rate * 3) { + samples[index] = 0.2 + } + let trimmed = NarrationSpeech.trimSilence(samples, sampleRate: rate) + let duration = Double(trimmed.count) / Double(rate) + #expect(duration > 2.4) + #expect(duration < 3.4) + + let silent = [Float](repeating: 0, count: rate) + #expect(NarrationSpeech.trimSilence(silent, sampleRate: rate).count == rate) + } + /// Opt-in smoke: download Qwen3-TTS, synthesize one phrase, write a WAV. /// /// xcodebuild does not always forward shell env into the test host, so we @@ -240,6 +331,52 @@ struct NarrationTests { print("narration smoke mp4: \(mp4.path) bytes=\(mp4Size?.intValue ?? 0)") } + /// Opt-in: re-render a real friction-log run with the local TTS pipeline. + /// + /// ```bash + /// export ASTROSHOTS_NARRATE_RUN=/path/to/runs/ + /// # or: touch /tmp/ASTROSHOTS_NARRATE_RUN and put the path in that file + /// ``` + @MainActor + @Test(.timeLimit(.minutes(30))) + func renderRequestedFrictionRun() async throws { + let envPath = ProcessInfo.processInfo.environment["ASTROSHOTS_NARRATE_RUN"] + let fileURL = URL(fileURLWithPath: "/tmp/ASTROSHOTS_NARRATE_RUN") + let filePath = (try? String(contentsOf: fileURL, encoding: .utf8))? + .trimmingCharacters(in: .whitespacesAndNewlines) + let raw = [envPath, filePath] + .compactMap { $0 } + .first { !$0.isEmpty && $0 != "1" } + guard let raw else { return } + + let runDir = URL(fileURLWithPath: raw, isDirectory: true) + let loaded = try #require( + FrictionLogLoader.loadRun(directory: runDir), + "Could not load friction-log run at \(runDir.path)" + ) + #expect(!loaded.1.steps.isEmpty) + + NarrationPaths.ensureDirectories() + let suite = "astroshots.narration.render.\(UUID().uuidString)" + let defaults = try #require(UserDefaults(suiteName: suite)) + defaults.removePersistentDomain(forName: suite) + defaults.set(true, forKey: "narrationEnabled") + let manager = NarrationModelManager(preferences: Preferences(defaults: defaults)) + await manager.refreshAndBootstrapIfNeeded() + #expect(manager.readiness.isReady, "bootstrap failed: \(manager.readiness.statusLine)") + + let queue = NarrationJobQueue(modelManager: manager) + let output = try await queue.renderImmediately( + log: loaded.0, + run: loaded.1, + voice: NarrationDefaults.voice + ) + #expect(FileManager.default.fileExists(atPath: output.path)) + let size = try FileManager.default.attributesOfItem(atPath: output.path)[.size] as? NSNumber + #expect((size?.intValue ?? 0) > 20_000) + print("narration render mp4: \(output.path) bytes=\(size?.intValue ?? 0)") + } + private func writeSolidPNG(to url: URL, width: Int, height: Int) throws { let image = NSImage(size: NSSize(width: width, height: height)) image.lockFocus() diff --git a/macos/AstroshotsTests/PreferencesTests.swift b/macos/AstroshotsTests/PreferencesTests.swift index 088f7da..4aae433 100644 --- a/macos/AstroshotsTests/PreferencesTests.swift +++ b/macos/AstroshotsTests/PreferencesTests.swift @@ -17,6 +17,18 @@ struct PreferencesTests { #expect(preferences.narrationVoice == "Ryan") } + @Test @MainActor + func narrationCaptionsDefaultOffAndPersist() { + let suiteName = "astroshots-preferences-captions-\(UUID().uuidString)" + let defaults = UserDefaults(suiteName: suiteName)! + defer { defaults.removePersistentDomain(forName: suiteName) } + let preferences = Preferences(defaults: defaults) + + #expect(preferences.narrationCaptionsEnabled == false) + preferences.narrationCaptionsEnabled = true + #expect(Preferences(defaults: defaults).narrationCaptionsEnabled) + } + @Test @MainActor func hiddenFrictionLogIDsAreDeduplicatedAndPersisted() { let suiteName = "astroshots-hidden-friction-\(UUID().uuidString)"