diff --git a/Apps/VoidDisplay/Resources/Localizable.xcstrings b/Apps/VoidDisplay/Resources/Localizable.xcstrings
index 127aa76..4ba4ead 100644
--- a/Apps/VoidDisplay/Resources/Localizable.xcstrings
+++ b/Apps/VoidDisplay/Resources/Localizable.xcstrings
@@ -323,13 +323,13 @@
}
}
},
- "Automatic balances H.264 compatibility, clarity, and encoder load." : {
+ "Automatic and Smooth send AV1 at source resolution on LAN. Actual frame rate follows encoder capability." : {
"extractionState" : "stale",
"localizations" : {
"zh-Hans" : {
"stringUnit" : {
"state" : "translated",
- "value" : "自动模式会平衡 H.264 兼容性、清晰度与编码负载。"
+ "value" : "自动和流畅会在局域网按源分辨率发送 AV1。实际帧率取决于编码能力。"
}
}
}
@@ -2916,16 +2916,6 @@
}
}
},
- "This Mac's WebRTC stack did not expose H.264 video encoding." : {
- "localizations" : {
- "zh-Hans" : {
- "stringUnit" : {
- "state" : "translated",
- "value" : "本机 WebRTC 链路未暴露 H.264 视频编码能力。"
- }
- }
- }
- },
"Technical Details" : {
"extractionState" : "stale",
"localizations" : {
@@ -3058,6 +3048,17 @@
}
}
},
+ "This Mac's WebRTC stack did not expose AV1 video encoding." : {
+ "extractionState" : "stale",
+ "localizations" : {
+ "zh-Hans" : {
+ "stringUnit" : {
+ "state" : "translated",
+ "value" : "本机 WebRTC 链路未暴露 AV1 视频编码能力。"
+ }
+ }
+ }
+ },
"This resolution mode already exists." : {
"extractionState" : "stale",
"localizations" : {
diff --git a/Apps/VoidDisplay/VoidDisplay.xcodeproj/project.pbxproj b/Apps/VoidDisplay/VoidDisplay.xcodeproj/project.pbxproj
index 3c9c240..7577047 100644
--- a/Apps/VoidDisplay/VoidDisplay.xcodeproj/project.pbxproj
+++ b/Apps/VoidDisplay/VoidDisplay.xcodeproj/project.pbxproj
@@ -264,6 +264,7 @@
"$(ROOT_DIR)/Tools/VoidDisplayRelay/go.mod",
"$(ROOT_DIR)/Tools/VoidDisplayRelay/go.sum",
"$(ROOT_DIR)/Tools/VoidDisplayRelay/cmd/voiddisplay-relay/main.go",
+ "$(ROOT_DIR)/Tools/VoidDisplayRelay/cmd/voiddisplay-relay/main_test.go",
"$(ROOT_DIR)/Tools/VoidDisplayRelay/internal/relay/server.go",
"$(ROOT_DIR)/Tools/VoidDisplayRelay/internal/relay/server_test.go",
);
diff --git a/Sources/VoidDisplayCapture/Services/CapturePreviewDiagnosticsSession.swift b/Sources/VoidDisplayCapture/Services/CapturePreviewDiagnosticsSession.swift
index de3edb8..c3cbcc8 100644
--- a/Sources/VoidDisplayCapture/Services/CapturePreviewDiagnosticsSession.swift
+++ b/Sources/VoidDisplayCapture/Services/CapturePreviewDiagnosticsSession.swift
@@ -64,6 +64,9 @@ package final class UITestCapturePreviewSession: @unchecked Sendable, DisplayCap
private final class DiagnosticsShareFrameConsumer: DisplayShareFrameConsumer {
nonisolated var hasDemand: Bool { false }
+ package nonisolated func updateSourceVideoSpec(_ spec: SourceVideoSpec) {
+ _ = spec
+ }
package nonisolated func updatePerformanceMode(_ mode: CapturePerformanceMode) {
_ = mode
}
diff --git a/Sources/VoidDisplayCapture/Services/DisplayCaptureRegistry.swift b/Sources/VoidDisplayCapture/Services/DisplayCaptureRegistry.swift
index 54c0d9e..b0bc007 100644
--- a/Sources/VoidDisplayCapture/Services/DisplayCaptureRegistry.swift
+++ b/Sources/VoidDisplayCapture/Services/DisplayCaptureRegistry.swift
@@ -105,6 +105,10 @@ package final class DisplayPreviewSubscription: Sendable {
private final class NoopDisplayShareFrameConsumer: DisplayShareFrameConsumer {
nonisolated var hasDemand: Bool { false }
+ package nonisolated func updateSourceVideoSpec(_ spec: SourceVideoSpec) {
+ _ = spec
+ }
+
package nonisolated func updatePerformanceMode(_ mode: CapturePerformanceMode) {
_ = mode
}
diff --git a/Sources/VoidDisplayCapture/Services/DisplayCaptureSession.swift b/Sources/VoidDisplayCapture/Services/DisplayCaptureSession.swift
index bbc0272..45aa46f 100644
--- a/Sources/VoidDisplayCapture/Services/DisplayCaptureSession.swift
+++ b/Sources/VoidDisplayCapture/Services/DisplayCaptureSession.swift
@@ -311,6 +311,7 @@ package final class DisplayCaptureSession: @unchecked Sendable, DisplayCaptureSe
nonisolated private let metrics: DisplayCaptureMetricsStore
nonisolated private let streamConfigurationCoordinator: DisplayCaptureStreamConfigurationCoordinator
nonisolated private let demandDriver: DisplayCaptureDemandDriver
+ nonisolated private let sourceVideoSpec: SourceVideoSpec
nonisolated init(
display: SCDisplay,
@@ -338,6 +339,8 @@ package final class DisplayCaptureSession: @unchecked Sendable, DisplayCaptureSe
let filter = try await Self.makeContentFilter(display: display)
self.stream = SCStream(filter: filter, configuration: config, delegate: output)
self.shareFrameConsumer = makeShareFrameConsumer()
+ self.sourceVideoSpec = captureSizeContext.sourceVideoSpec
+ self.shareFrameConsumer.updateSourceVideoSpec(captureSizeContext.sourceVideoSpec)
let metrics = DisplayCaptureMetricsStore()
self.metrics = metrics
let streamConfigurationCoordinator = DisplayCaptureStreamConfigurationCoordinator(
@@ -403,6 +406,7 @@ package final class DisplayCaptureSession: @unchecked Sendable, DisplayCaptureSe
}
package nonisolated func setDemand(_ demand: DisplayCaptureDemandSnapshot) async throws {
+ shareFrameConsumer.updateSourceVideoSpec(sourceVideoSpec)
shareFrameConsumer.updatePerformanceMode(demand.performanceMode)
try await demandDriver.setDemand(demand)
}
@@ -466,8 +470,12 @@ package extension DisplayCaptureSession {
}
nonisolated static func clampedPreviewFramesPerSecond(for refreshRate: Double) -> Int {
+ sourceFramesPerSecond(for: refreshRate)
+ }
+
+ nonisolated static func sourceFramesPerSecond(for refreshRate: Double) -> Int {
let normalizedRefreshRate = refreshRate > 0 ? refreshRate : 60.0
- return max(1, Int(min(normalizedRefreshRate, 60.0).rounded()))
+ return max(1, Int(normalizedRefreshRate.rounded()))
}
nonisolated private static func makeStreamConfigurationState(
@@ -482,10 +490,11 @@ package extension DisplayCaptureSession {
for: initialProfile,
performanceMode: initialPerformanceMode
)
- let previewFramesPerSecond = clampedPreviewFramesPerSecond(for: displayMode?.refreshRate ?? 60.0)
+ let previewFramesPerSecond = sourceFramesPerSecond(for: displayMode?.refreshRate ?? 60.0)
let initialFrameRateTier = DisplayCaptureConfigurationStateMachine.defaultFrameRateTier(
for: initialProfile,
- performanceMode: initialPerformanceMode
+ performanceMode: initialPerformanceMode,
+ sourceFramesPerSecond: captureSizeContext.sourceVideoSpec.framesPerSecond
)
let state = DisplayCaptureStreamConfigurationState(
@@ -501,7 +510,7 @@ package extension DisplayCaptureSession {
shareCursorOverrideCount: 0
)
AppLog.capture.notice(
- "Capture config display=\(display.displayID, privacy: .public) size=\(captureSize.width)x\(captureSize.height, privacy: .public) logical=\(captureSizeContext.logicalSize.width)x\(captureSizeContext.logicalSize.height, privacy: .public) physical=\(captureSizeContext.physicalSize.width)x\(captureSizeContext.physicalSize.height, privacy: .public)"
+ "Capture config display=\(display.displayID, privacy: .public) size=\(captureSize.width)x\(captureSize.height, privacy: .public) logical=\(captureSizeContext.logicalSize.width)x\(captureSizeContext.logicalSize.height, privacy: .public) physical=\(captureSizeContext.physicalSize.width)x\(captureSizeContext.physicalSize.height, privacy: .public) sourceFps=\(captureSizeContext.sourceVideoSpec.framesPerSecond, privacy: .public)"
)
return state
}
@@ -515,6 +524,7 @@ package extension DisplayCaptureSession {
let modeLogicalWidth = displayMode.map { $0.width } ?? modePixelWidth
let modeLogicalHeight = displayMode.map { $0.height } ?? modePixelHeight
let backingScale = screenBackingScaleFactor(for: display.displayID)
+ let sourceFramesPerSecond = sourceFramesPerSecond(for: displayMode?.refreshRate ?? 60.0)
let scaledLogicalWidth = max(1, Int((CGFloat(modeLogicalWidth) * backingScale).rounded()))
let scaledLogicalHeight = max(1, Int((CGFloat(modeLogicalHeight) * backingScale).rounded()))
@@ -527,7 +537,8 @@ package extension DisplayCaptureSession {
physicalSize: DisplayCaptureDimensions(
width: max(modePixelWidth, scaledLogicalWidth),
height: max(modePixelHeight, scaledLogicalHeight)
- )
+ ),
+ sourceFramesPerSecond: sourceFramesPerSecond
)
}
diff --git a/Sources/VoidDisplayCapture/Services/DisplayCaptureTypes.swift b/Sources/VoidDisplayCapture/Services/DisplayCaptureTypes.swift
index c612b9b..c53ac5e 100644
--- a/Sources/VoidDisplayCapture/Services/DisplayCaptureTypes.swift
+++ b/Sources/VoidDisplayCapture/Services/DisplayCaptureTypes.swift
@@ -5,13 +5,15 @@ import CoreGraphics
import CoreMedia
import Foundation
import ScreenCaptureKit
-package nonisolated enum DisplayCaptureFrameRateTier: Int, CaseIterable, Sendable, Equatable {
- case fps30 = 30
- case fps45 = 45
- case fps60 = 60
+package nonisolated struct DisplayCaptureFrameRateTier: Sendable, Equatable, Hashable {
+ package static let fps30 = DisplayCaptureFrameRateTier(framesPerSecond: 30)
+ package static let fps45 = DisplayCaptureFrameRateTier(framesPerSecond: 45)
+ package static let fps60 = DisplayCaptureFrameRateTier(framesPerSecond: 60)
- package var framesPerSecond: Int {
- rawValue
+ package let framesPerSecond: Int
+
+ package init(framesPerSecond: Int) {
+ self.framesPerSecond = max(1, framesPerSecond)
}
}
@@ -20,18 +22,37 @@ package typealias DisplayCaptureDimensions = CapturePixelDimensions
package nonisolated struct DisplayCaptureSizeContext: Sendable, Equatable {
package static let defaultShared = DisplayCaptureSizeContext(
logicalSize: .defaultShared,
- physicalSize: .defaultShared
+ physicalSize: .defaultShared,
+ sourceVideoSpec: .defaultShared
)
package let logicalSize: DisplayCaptureDimensions
package let physicalSize: DisplayCaptureDimensions
+ package let sourceVideoSpec: SourceVideoSpec
+
+ package init(
+ logicalSize: DisplayCaptureDimensions,
+ physicalSize: DisplayCaptureDimensions,
+ sourceFramesPerSecond: Int = 60
+ ) {
+ self.init(
+ logicalSize: logicalSize,
+ physicalSize: physicalSize,
+ sourceVideoSpec: SourceVideoSpec(
+ dimensions: physicalSize,
+ framesPerSecond: sourceFramesPerSecond
+ )
+ )
+ }
package init(
logicalSize: DisplayCaptureDimensions,
- physicalSize: DisplayCaptureDimensions
+ physicalSize: DisplayCaptureDimensions,
+ sourceVideoSpec: SourceVideoSpec
) {
self.logicalSize = logicalSize
self.physicalSize = physicalSize
+ self.sourceVideoSpec = sourceVideoSpec
}
package func captureSize(
@@ -273,19 +294,21 @@ package nonisolated enum DisplayCaptureConfigurationDecision: Sendable, Equatabl
package nonisolated enum DisplayCaptureConfigurationStateMachine {
nonisolated static func defaultFrameRateTier(
for profile: DisplayCaptureProfile,
- performanceMode: CapturePerformanceMode
+ performanceMode: CapturePerformanceMode,
+ sourceFramesPerSecond: Int = 60
) -> DisplayCaptureFrameRateTier {
- switch performanceMode {
+ let sourceFramesPerSecond = max(1, sourceFramesPerSecond)
+ return switch performanceMode {
case .automatic:
- .fps60
+ DisplayCaptureFrameRateTier(framesPerSecond: sourceFramesPerSecond)
case .smooth:
- .fps60
+ DisplayCaptureFrameRateTier(framesPerSecond: sourceFramesPerSecond)
case .powerEfficient:
switch profile {
case .previewOnly:
- .fps45
+ DisplayCaptureFrameRateTier(framesPerSecond: min(sourceFramesPerSecond, 45))
case .shareOnly, .mixed:
- .fps30
+ DisplayCaptureFrameRateTier(framesPerSecond: min(sourceFramesPerSecond, 30))
}
}
}
@@ -301,7 +324,8 @@ package nonisolated enum DisplayCaptureConfigurationStateMachine {
profile: profile,
frameRateTier: defaultFrameRateTier(
for: profile,
- performanceMode: demand.performanceMode
+ performanceMode: demand.performanceMode,
+ sourceFramesPerSecond: captureSizeContext.sourceVideoSpec.framesPerSecond
),
captureSize: captureSizeContext.captureSize(
for: profile,
diff --git a/Sources/VoidDisplayFoundation/RuntimeSupport/SharedCapturePerformanceBudget.swift b/Sources/VoidDisplayFoundation/RuntimeSupport/SharedCapturePerformanceBudget.swift
index da45bc9..c825e4c 100644
--- a/Sources/VoidDisplayFoundation/RuntimeSupport/SharedCapturePerformanceBudget.swift
+++ b/Sources/VoidDisplayFoundation/RuntimeSupport/SharedCapturePerformanceBudget.swift
@@ -58,8 +58,27 @@ package nonisolated struct CapturePixelDimensions: Sendable, Equatable {
}
}
+package nonisolated struct SourceVideoSpec: Sendable, Equatable {
+ package let dimensions: CapturePixelDimensions
+ package let framesPerSecond: Int
+
+ package init(width: Int, height: Int, framesPerSecond: Int) {
+ self.dimensions = CapturePixelDimensions(width: width, height: height)
+ self.framesPerSecond = max(1, framesPerSecond)
+ }
+
+ package init(dimensions: CapturePixelDimensions, framesPerSecond: Int) {
+ self.dimensions = dimensions
+ self.framesPerSecond = max(1, framesPerSecond)
+ }
+
+ package static let defaultShared = SourceVideoSpec(
+ dimensions: .defaultShared,
+ framesPerSecond: 60
+ )
+}
+
package nonisolated struct SharedCapturePerformanceBudget: Sendable, Equatable {
- package static let automaticPixelBudgetPerSecond: Int64 = 221_184_000
package static let powerEfficientPixelBudgetPerSecond: Int64 = 62_208_000
package let framesPerSecond: Int
@@ -78,7 +97,7 @@ package nonisolated struct SharedCapturePerformanceBudget: Sendable, Equatable {
case .automatic:
self.init(
framesPerSecond: 60,
- pixelBudgetPerSecond: Self.automaticPixelBudgetPerSecond
+ pixelBudgetPerSecond: nil
)
case .smooth:
self.init(
diff --git a/Sources/VoidDisplayFoundation/Screen/DisplaySharingBoundary.swift b/Sources/VoidDisplayFoundation/Screen/DisplaySharingBoundary.swift
index 8ac1492..15a4eda 100644
--- a/Sources/VoidDisplayFoundation/Screen/DisplaySharingBoundary.swift
+++ b/Sources/VoidDisplayFoundation/Screen/DisplaySharingBoundary.swift
@@ -5,6 +5,7 @@ import Synchronization
package protocol DisplayShareFrameConsumer: AnyObject, Sendable {
nonisolated var hasDemand: Bool { get }
+ nonisolated func updateSourceVideoSpec(_ spec: SourceVideoSpec)
nonisolated func updatePerformanceMode(_ mode: CapturePerformanceMode)
nonisolated func stopSharing()
nonisolated func submitFrame(pixelBuffer: CVPixelBuffer, ptsUs: UInt64)
diff --git a/Sources/VoidDisplaySharing/Resources/displayPage.html b/Sources/VoidDisplaySharing/Resources/displayPage.html
index 3a71ca9..bf27469 100644
--- a/Sources/VoidDisplaySharing/Resources/displayPage.html
+++ b/Sources/VoidDisplaySharing/Resources/displayPage.html
@@ -70,6 +70,10 @@
font-size: 13px;
background: rgba(255, 255, 255, 0.08);
color: var(--muted);
+ max-width: min(62vw, 560px);
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
}
.controls {
display: flex;
@@ -215,8 +219,17 @@ Connecting…
let reconnectIndex = 0;
let reconnectTimer = null;
var terminalStop = false;
- let state = "idle";
+ var state = "idle";
let originalScaleEnabled = false;
+ var negotiatedVideoCodec = null;
+ let expectedSourceVideoSpec = null;
+ const browserStatsLogInterval = 2000;
+ const browserStatsState = {
+ timer: null,
+ lastBytesReceived: null,
+ lastFramesDecoded: null,
+ lastTimestamp: null
+ };
const reconnectDelays = [250, 500, 1000, 2000, 4000];
const messages = {
en: {
@@ -233,6 +246,9 @@ Connecting…
statusError: "Error",
statusConnectionError: "Connection error",
statusReconnecting: (delay) => `Reconnecting in ${delay}ms`,
+ statusLiveWithStats: (codec, width, height, fps) => `Live ${codec} ${width}×${height} ${fps}fps`,
+ statusLiveBelowSource: (codec, width, height, fps, sourceWidth, sourceHeight, sourceFps) =>
+ `Live ${codec} ${width}×${height} ${fps}fps, source ${sourceWidth}×${sourceHeight} ${sourceFps}fps`,
scaleFit: "Fit",
scaleOriginal: "1:1",
fullscreenEnter: "Fullscreen",
@@ -251,10 +267,12 @@ Connecting…
overlayWebSocketRequiredBody: "This browser cannot open signaling transport.",
overlayWebRTCRequiredTitle: "WebRTC required",
overlayWebRTCRequiredBody: "This browser does not support RTCPeerConnection.",
- overlayH264RequiredTitle: "H.264 required",
- overlayH264RequiredBody: "This browser or device did not expose H.264 for WebRTC playback.",
- overlayH264AnswerRequiredBody: "The WebRTC answer did not negotiate H.264 video.",
- overlayH264StatsRequiredBody: "The browser did not confirm an H.264 decoder in WebRTC stats.",
+ overlayCodecRequiredTitle: "AV1 required",
+ overlayCodecRequiredBody: "This browser or device did not expose AV1 for WebRTC playback.",
+ overlayCodecAnswerRequiredBody: "The WebRTC answer did not negotiate AV1 video.",
+ overlayCodecStatsRequiredBody: "The browser did not confirm the selected AV1 decoder in WebRTC stats.",
+ overlayCodecPendingTitle: "Preparing video codec",
+ overlayCodecPendingBody: "The source stream is still preparing AV1. Retrying automatically.",
overlayNegotiationFailedTitle: "Negotiation failed",
overlayNegotiationFailedFallback: "Failed to create WebRTC offer.",
overlaySharingStoppedTitle: "Sharing stopped",
@@ -277,6 +295,9 @@ Connecting…
statusError: "发生错误",
statusConnectionError: "连接出错",
statusReconnecting: (delay) => `${delay}ms 后重连`,
+ statusLiveWithStats: (codec, width, height, fps) => `直播中 ${codec} ${width}×${height} ${fps}fps`,
+ statusLiveBelowSource: (codec, width, height, fps, sourceWidth, sourceHeight, sourceFps) =>
+ `直播中 ${codec} ${width}×${height} ${fps}fps，源 ${sourceWidth}×${sourceHeight} ${sourceFps}fps`,
scaleFit: "适应",
scaleOriginal: "1:1",
fullscreenEnter: "全屏",
@@ -295,10 +316,12 @@ Connecting…
overlayWebSocketRequiredBody: "当前浏览器无法建立信令传输通道。",
overlayWebRTCRequiredTitle: "需要 WebRTC",
overlayWebRTCRequiredBody: "当前浏览器不支持 RTCPeerConnection。",
- overlayH264RequiredTitle: "需要 H.264",
- overlayH264RequiredBody: "当前浏览器或设备未暴露 WebRTC H.264 播放能力。",
- overlayH264AnswerRequiredBody: "WebRTC answer 未协商到 H.264 视频。",
- overlayH264StatsRequiredBody: "浏览器 WebRTC stats 未确认 H.264 解码器。",
+ overlayCodecRequiredTitle: "需要 AV1",
+ overlayCodecRequiredBody: "当前浏览器或设备未暴露 WebRTC AV1 播放能力。",
+ overlayCodecAnswerRequiredBody: "WebRTC answer 未协商到 AV1 视频。",
+ overlayCodecStatsRequiredBody: "浏览器 WebRTC stats 未确认选中的 AV1 解码器。",
+ overlayCodecPendingTitle: "正在准备视频编码",
+ overlayCodecPendingBody: "源端仍在准备 AV1，正在自动重试。",
overlayNegotiationFailedTitle: "协商失败",
overlayNegotiationFailedFallback: "创建 WebRTC offer 失败。",
overlaySharingStoppedTitle: "共享已停止",
@@ -351,8 +374,8 @@ Connecting…
}
})();
const localOfferIceTimeoutMs = 2000;
- const h264StatsConfirmationTimeoutMs = 10000;
- const h264StatsPollIntervalMs = 100;
+ const codecStatsConfirmationTimeoutMs = 10000;
+ const codecStatsPollIntervalMs = 100;
function applyStaticCopy() {
document.title = t("pageTitle");
@@ -418,6 +441,7 @@ Connecting…
syncFullscreenButtonLabel();
function closePeer() {
+ stopBrowserStatsLoop();
if (peer) {
peer.ontrack = null;
peer.onicecandidate = null;
@@ -451,7 +475,7 @@ Connecting…
}
}
- function scheduleReconnect() {
+ function scheduleReconnect(overlayTitle = t("overlayReconnectTitle"), overlayBody = t("overlayReconnectBody")) {
if (terminalStop || state === "stopping" || state === "closed") {
return;
}
@@ -461,7 +485,7 @@ Connecting…
const delay = reconnectDelays[Math.min(reconnectIndex, reconnectDelays.length - 1)];
reconnectIndex += 1;
setStatus(t("statusReconnecting", delay));
- setOverlay(t("overlayReconnectTitle"), t("overlayReconnectBody"), true);
+ setOverlay(overlayTitle, overlayBody, true);
transition("handshaking");
reconnectTimer = window.setTimeout(() => {
reconnectTimer = null;
@@ -469,6 +493,47 @@ Connecting…
}, delay);
}
+ function schedulePeerRetry(overlayTitle, overlayBody) {
+ if (terminalStop || state === "stopping" || state === "closed") {
+ return;
+ }
+ if (reconnectTimer) {
+ return;
+ }
+ const delay = reconnectDelays[Math.min(reconnectIndex, reconnectDelays.length - 1)];
+ reconnectIndex += 1;
+ setStatus(t("statusReconnecting", delay));
+ setOverlay(overlayTitle, overlayBody, true);
+ transition("signalingReady");
+ reconnectTimer = window.setTimeout(async () => {
+ reconnectTimer = null;
+ if (terminalStop || state === "stopping" || state === "closed") {
+ return;
+ }
+ if (!socket || socket.readyState !== WebSocket.OPEN) {
+ scheduleReconnect(overlayTitle, overlayBody);
+ return;
+ }
+ try {
+ await startPeerConnection();
+ setOverlay(t("overlayNegotiatingTitle"), t("overlayNegotiatingBody"), true);
+ } catch (error) {
+ if (isCodecRequirementError(error)) {
+ failCodecRequirement(error);
+ return;
+ }
+ setStatus(t("statusNegotiationFailed"));
+ setOverlay(
+ t("overlayNegotiationFailedTitle"),
+ error?.message || t("overlayNegotiationFailedFallback"),
+ true
+ );
+ closeSocketAndClearReference();
+ scheduleReconnect();
+ }
+ }, delay);
+ }
+
async function sendSignal(payload) {
if (!socket || socket.readyState !== WebSocket.OPEN) return;
socket.send(JSON.stringify(payload));
@@ -502,35 +567,126 @@ Connecting…
throw new Error("Local WebRTC offer is missing ICE credentials.");
}
- function isH264Codec(codec) {
- return String(codec?.mimeType || "").toLowerCase() === "video/h264";
+ function normalizedVideoCodecName(codec) {
+ return String(codec?.mimeType || "").toLowerCase();
+ }
+
+ function isProbedReceiverVideoCodec(codec) {
+ return normalizedVideoCodecName(codec) === "video/av1";
}
- function h264ReceiverCodecPreferences() {
+ function receiverCapabilityEntry(codec) {
+ const payloadType = codecPayloadType(codec);
+ const fmtpLine = String(codec?.sdpFmtpLine || "");
+ return normalizedVideoCodecName(codec) +
+ "(pt=" + (payloadType === null ? "none" : String(payloadType)) +
+ ",fmtp=" + (fmtpLine.length === 0 ? "none" : fmtpLine) + ")";
+ }
+
+ function receiverCapabilityProbeGroup(allCodecs, mimeTypes) {
+ const entries = allCodecs
+ .filter((codec) => mimeTypes.includes(normalizedVideoCodecName(codec)))
+ .map(receiverCapabilityEntry);
+ return entries.length === 0 ? "missing" : entries.join(",");
+ }
+
+ function receiverVideoCodecCapabilitySummary() {
if (!window.RTCRtpReceiver || typeof RTCRtpReceiver.getCapabilities !== "function") {
- return null;
+ return "unavailable";
}
const capabilities = RTCRtpReceiver.getCapabilities("video");
- const codecs = Array.isArray(capabilities?.codecs)
- ? capabilities.codecs.filter(isH264Codec)
- : [];
- if (codecs.length === 0) {
- throw new Error(t("overlayH264RequiredBody"));
+ const allCodecs = Array.isArray(capabilities?.codecs) ? capabilities.codecs : [];
+ const probedCodecCount = allCodecs.filter(isProbedReceiverVideoCodec).length;
+ return [
+ "AV1=" + receiverCapabilityProbeGroup(allCodecs, ["video/av1"]),
+ "unsupportedVideoCodecCount=" + String(allCodecs.length - probedCodecCount)
+ ].join("; ");
+ }
+
+ function isSupportedPlaybackCodec(codec) {
+ return normalizedVideoCodecName(codec) === "video/av1";
+ }
+
+ function isRetransmissionCodec(codec) {
+ return normalizedVideoCodecName(codec) === "video/rtx";
+ }
+
+ function codecPayloadType(codec) {
+ const value = Number(codec?.payloadType ?? codec?.preferredPayloadType ?? NaN);
+ return Number.isFinite(value) && value >= 0 ? Math.round(value) : null;
+ }
+
+ function rtxAptPayloadType(codec) {
+ const parameterApt = Number(codec?.parameters?.apt ?? NaN);
+ if (Number.isFinite(parameterApt) && parameterApt >= 0) {
+ return Math.round(parameterApt);
+ }
+ const fmtpLine = String(codec?.sdpFmtpLine || "");
+ const match = /(?:^|;)\s*apt=(\d+)\s*(?:;|$)/u.exec(fmtpLine);
+ return match ? Number(match[1]) : null;
+ }
+
+ function matchingRtxCodecs(allCodecs, primaryCodecs) {
+ const primaryPayloadTypes = new Set(primaryCodecs
+ .map(codecPayloadType)
+ .filter((payloadType) => payloadType !== null));
+ if (primaryPayloadTypes.size === 0) {
+ return [];
}
- return codecs;
+ return allCodecs.filter((codec) => {
+ if (!isRetransmissionCodec(codec)) return false;
+ const apt = rtxAptPayloadType(codec);
+ return apt !== null && primaryPayloadTypes.has(apt);
+ });
+ }
+
+ function codecRequirementError(message) {
+ const error = new Error(message);
+ error.codecRequirement = true;
+ return error;
+ }
+
+ function isCodecRequirementError(error) {
+ return Boolean(error?.codecRequirement);
+ }
+
+ function receiverCodecPreferences() {
+ if (!window.RTCRtpReceiver || typeof RTCRtpReceiver.getCapabilities !== "function") {
+ throw codecRequirementError(t("overlayCodecRequiredBody"));
+ }
+ const capabilities = RTCRtpReceiver.getCapabilities("video");
+ const allCodecs = Array.isArray(capabilities?.codecs) ? capabilities.codecs : [];
+ const supportedCodecs = allCodecs.filter(isSupportedPlaybackCodec);
+ if (supportedCodecs.length === 0) {
+ throw codecRequirementError(t("overlayCodecRequiredBody"));
+ }
+ return supportedCodecs.concat(matchingRtxCodecs(allCodecs, supportedCodecs));
}
function videoCodecNamesFromSDP(sdp) {
const lines = String(sdp || "").split(/\r?\n/u);
- const payloadTypes = [];
- const namesByPayloadType = new Map();
+ let payloadTypes = [];
+ let namesByPayloadType = new Map();
let inVideo = false;
+ const codecNames = [];
+
+ function flushVideoMedia() {
+ if (!inVideo) return;
+ for (const payloadType of payloadTypes) {
+ const codecName = namesByPayloadType.get(payloadType);
+ if (codecName) {
+ codecNames.push(codecName);
+ }
+ }
+ }
for (const line of lines) {
if (line.startsWith("m=")) {
+ flushVideoMedia();
inVideo = line.startsWith("m=video ");
+ payloadTypes = [];
+ namesByPayloadType = new Map();
if (inVideo) {
- payloadTypes.length = 0;
const parts = line.trim().split(/\s+/u);
payloadTypes.push(...parts.slice(3));
}
@@ -542,27 +698,28 @@ Connecting…
namesByPayloadType.set(match[1], match[2].toLowerCase());
}
}
+ flushVideoMedia();
- return payloadTypes
- .map((payloadType) => namesByPayloadType.get(payloadType))
- .filter(Boolean);
+ return codecNames;
}
- function assertH264AnswerSDP(sdp) {
+ function selectedCodecFromAnswerSDP(sdp) {
const codecNames = videoCodecNamesFromSDP(sdp);
- const hasH264 = codecNames.some((name) => name === "h264");
- const hasUnexpectedVideoCodec = codecNames.some((name) => name !== "h264" && name !== "rtx");
- if (!hasH264 || hasUnexpectedVideoCodec) {
- throw new Error(t("overlayH264AnswerRequiredBody"));
+ const primaryCodecs = codecNames.filter((name) => name !== "rtx");
+ const supportedPrimaryCodecs = [...new Set(primaryCodecs.filter((name) => name === "av1"))];
+ const hasUnexpectedVideoCodec = primaryCodecs.some((name) => name !== "av1");
+ if (supportedPrimaryCodecs.length !== 1 || hasUnexpectedVideoCodec) {
+ throw new Error(t("overlayCodecAnswerRequiredBody"));
}
+ return supportedPrimaryCodecs[0];
}
- function failH264Requirement(error) {
+ function failCodecRequirement(error) {
terminalStop = true;
setStatus(t("statusNegotiationFailed"));
setOverlay(
- t("overlayH264RequiredTitle"),
- error?.message || t("overlayH264RequiredBody"),
+ t("overlayCodecRequiredTitle"),
+ error?.message || t("overlayCodecRequiredBody"),
true
);
clearReconnectTimer();
@@ -571,13 +728,18 @@ Connecting…
transition("closed");
}
- async function verifySelectedH264Codec() {
+ function isCodecErrorReason(reason) {
+ return reason === "unsupported_video_codec_offered" || reason === "supported_video_codec_missing";
+ }
+
+ async function verifySelectedCodec(expectedCodec) {
if (!peer || typeof peer.getStats !== "function") {
- throw new Error(t("overlayH264StatsRequiredBody"));
+ throw new Error(t("overlayCodecStatsRequiredBody"));
}
const startedAt = performance.now();
- while (performance.now() - startedAt < h264StatsConfirmationTimeoutMs) {
+ const expectedMimeType = `video/${expectedCodec}`;
+ while (performance.now() - startedAt < codecStatsConfirmationTimeoutMs) {
const stats = await peer.getStats();
const reports = new Map();
stats.forEach((report) => reports.set(report.id, report));
@@ -592,28 +754,182 @@ Connecting…
}
const codec = reports.get(report.codecId);
if (!codec?.mimeType) continue;
- if (String(codec.mimeType).toLowerCase() === "video/h264") {
+ const codecMimeType = String(codec.mimeType).toLowerCase();
+ if (codecMimeType === expectedMimeType) {
+ logBrowserVideoStats(report, codec, null);
return;
}
- throw new Error(t("overlayH264StatsRequiredBody"));
+ if (codecMimeType === "video/rtx") {
+ continue;
+ }
+ throw new Error(t("overlayCodecStatsRequiredBody"));
+ }
+ await new Promise((resolve) => window.setTimeout(resolve, codecStatsPollIntervalMs));
+ }
+ throw new Error(t("overlayCodecStatsRequiredBody"));
+ }
+
+ function browserStatsCodecName(codec) {
+ const mimeType = String(codec?.mimeType || "").toLowerCase();
+ if (mimeType === "video/av1") return "AV1";
+ return mimeType || "unknown";
+ }
+
+ function sourceSpecFromSignal(value) {
+ const width = Number(value?.width || 0);
+ const height = Number(value?.height || 0);
+ const framesPerSecond = Number(value?.framesPerSecond || 0);
+ if (width <= 0 || height <= 0 || framesPerSecond <= 0) {
+ return null;
+ }
+ return {
+ width: Math.round(width),
+ height: Math.round(height),
+ framesPerSecond: Math.round(framesPerSecond)
+ };
+ }
+
+ function resetBrowserStatsState() {
+ browserStatsState.lastBytesReceived = null;
+ browserStatsState.lastFramesDecoded = null;
+ browserStatsState.lastTimestamp = null;
+ }
+
+ function stopBrowserStatsLoop() {
+ if (browserStatsState.timer) {
+ window.clearInterval(browserStatsState.timer);
+ browserStatsState.timer = null;
+ }
+ resetBrowserStatsState();
+ }
+
+ function videoInboundStatsFromReport(stats) {
+ const reports = new Map();
+ stats.forEach((report) => reports.set(report.id, report));
+ for (const report of reports.values()) {
+ const reportKind = report.kind || report.mediaType;
+ if (
+ report.type !== "inbound-rtp" ||
+ reportKind !== "video" ||
+ !report.codecId
+ ) {
+ continue;
+ }
+ const codec = reports.get(report.codecId);
+ if (!codec?.mimeType) continue;
+ const mimeType = String(codec.mimeType).toLowerCase();
+ if (mimeType !== "video/av1") continue;
+ return { report, codec };
+ }
+ return null;
+ }
+
+ function logBrowserVideoStats(report, codec, derived) {
+ const codecName = browserStatsCodecName(codec);
+ const width = Number(report.frameWidth || player.videoWidth || 0);
+ const height = Number(report.frameHeight || player.videoHeight || 0);
+ const fps = Number(report.framesPerSecond || derived?.framesPerSecond || 0);
+ const bitrateBps = Number(derived?.bitrateBps || 0);
+ const droppedFrames = Number(report.framesDropped || 0);
+ const jitter = Number(report.jitter || 0);
+ const packetsLost = Number(report.packetsLost || 0);
+ const sourceSpec = expectedSourceVideoSpec;
+ const isBelowSourceSpec = Boolean(sourceSpec && (
+ (width > 0 && width < sourceSpec.width) ||
+ (height > 0 && height < sourceSpec.height) ||
+ (fps > 0 && Math.round(fps) < sourceSpec.framesPerSecond)
+ ));
+ console.info(
+ "[VoidDisplay] WebRTC browser stats",
+ {
+ selectedCodec: codecName,
+ decodedWidth: width,
+ decodedHeight: height,
+ framesPerSecond: Math.round(fps),
+ sourceWidth: sourceSpec?.width || null,
+ sourceHeight: sourceSpec?.height || null,
+ sourceFramesPerSecond: sourceSpec?.framesPerSecond || null,
+ belowSourceSpec: isBelowSourceSpec,
+ bitrateBps: Math.round(bitrateBps),
+ droppedFrames,
+ jitter,
+ packetsLost
}
- await new Promise((resolve) => window.setTimeout(resolve, h264StatsPollIntervalMs));
+ );
+ if (state === "streaming" && width > 0 && height > 0) {
+ const roundedFps = Math.max(0, Math.round(fps));
+ if (isBelowSourceSpec) {
+ setStatus(t(
+ "statusLiveBelowSource",
+ codecName,
+ width,
+ height,
+ roundedFps,
+ sourceSpec.width,
+ sourceSpec.height,
+ sourceSpec.framesPerSecond
+ ));
+ } else {
+ setStatus(t("statusLiveWithStats", codecName, width, height, roundedFps));
+ }
+ }
+ }
+
+ async function pollBrowserStats() {
+ if (!peer || typeof peer.getStats !== "function") return;
+ const stats = await peer.getStats();
+ const selected = videoInboundStatsFromReport(stats);
+ if (!selected) return;
+ const now = Number(selected.report.timestamp || performance.now());
+ const bytesReceived = Number(selected.report.bytesReceived || 0);
+ const framesDecoded = Number(selected.report.framesDecoded || 0);
+ let derived = null;
+ if (
+ browserStatsState.lastTimestamp !== null &&
+ now > browserStatsState.lastTimestamp
+ ) {
+ const elapsedSeconds = (now - browserStatsState.lastTimestamp) / 1000;
+ const byteDelta = Math.max(0, bytesReceived - browserStatsState.lastBytesReceived);
+ const frameDelta = Math.max(0, framesDecoded - browserStatsState.lastFramesDecoded);
+ derived = {
+ bitrateBps: elapsedSeconds > 0 ? (byteDelta * 8) / elapsedSeconds : 0,
+ framesPerSecond: elapsedSeconds > 0 ? frameDelta / elapsedSeconds : 0
+ };
}
- throw new Error(t("overlayH264StatsRequiredBody"));
+ browserStatsState.lastTimestamp = now;
+ browserStatsState.lastBytesReceived = bytesReceived;
+ browserStatsState.lastFramesDecoded = framesDecoded;
+ logBrowserVideoStats(selected.report, selected.codec, derived);
+ }
+
+ function startBrowserStatsLoop() {
+ stopBrowserStatsLoop();
+ browserStatsState.timer = window.setInterval(() => {
+ pollBrowserStats().catch((error) => {
+ console.warn("[VoidDisplay] WebRTC browser stats failed", error);
+ });
+ }, browserStatsLogInterval);
+ pollBrowserStats().catch(() => {});
}
async function startPeerConnection() {
closePeer();
peer = new RTCPeerConnection({ iceServers: bootstrap.iceServers ?? [] });
let usesTransceiverOffer = false;
- const h264Preferences = h264ReceiverCodecPreferences();
+ negotiatedVideoCodec = null;
+ console.info("[VoidDisplay] WebRTC receiver video capabilities " + receiverVideoCodecCapabilitySummary());
+ const codecPreferences = receiverCodecPreferences();
if (typeof peer.addTransceiver === "function") {
const transceiver = peer.addTransceiver("video", { direction: "recvonly" });
if (
- h264Preferences &&
+ codecPreferences &&
typeof transceiver.setCodecPreferences === "function"
) {
- transceiver.setCodecPreferences(h264Preferences);
+ try {
+ transceiver.setCodecPreferences(codecPreferences);
+ } catch (error) {
+ throw codecRequirementError(error?.message || t("overlayCodecRequiredBody"));
+ }
}
usesTransceiverOffer = true;
}
@@ -622,12 +938,14 @@ Connecting…
if (event.streams && event.streams[0]) {
player.srcObject = event.streams[0];
try {
- await verifySelectedH264Codec();
+ await verifySelectedCodec(negotiatedVideoCodec || "av1");
setStatus(t("statusLive"));
setOverlay(t("overlayLiveTitle"), t("overlayLiveBody"), false);
+ reconnectIndex = 0;
transition("streaming");
+ startBrowserStatsLoop();
} catch (error) {
- failH264Requirement(error);
+ failCodecRequirement(error);
}
}
};
@@ -706,6 +1024,10 @@ Connecting…
await startPeerConnection();
setOverlay(t("overlayNegotiatingTitle"), t("overlayNegotiatingBody"), true);
} catch (error) {
+ if (isCodecRequirementError(error)) {
+ failCodecRequirement(error);
+ return;
+ }
setStatus(t("statusNegotiationFailed"));
setOverlay(
t("overlayNegotiationFailedTitle"),
@@ -738,14 +1060,15 @@ Connecting…
case "answer":
if (!peer || typeof payload.sdp !== "string") return;
try {
- assertH264AnswerSDP(payload.sdp);
+ negotiatedVideoCodec = selectedCodecFromAnswerSDP(payload.sdp);
+ expectedSourceVideoSpec = sourceSpecFromSignal(payload.sourceVideoSpec);
await peer.setRemoteDescription({
type: "answer",
sdp: payload.sdp
});
setStatus(t("statusConnected"));
} catch (error) {
- failH264Requirement(error);
+ failCodecRequirement(error);
}
break;
case "ice_candidate":
@@ -766,7 +1089,17 @@ Connecting…
closeSocketAndClearReference();
transition("closed");
break;
+ case "codec_pending":
+ setStatus(t("statusReconnecting", reconnectDelays[Math.min(reconnectIndex, reconnectDelays.length - 1)]));
+ setOverlay(t("overlayCodecPendingTitle"), t("overlayCodecPendingBody"), true);
+ closePeer();
+ schedulePeerRetry(t("overlayCodecPendingTitle"), t("overlayCodecPendingBody"));
+ break;
case "error":
+ if (isCodecErrorReason(payload.reason)) {
+ failCodecRequirement(new Error(t("overlayCodecAnswerRequiredBody")));
+ break;
+ }
terminalStop = true;
setStatus(t("statusError"));
setOverlay(t("overlayStreamErrorTitle"), payload.reason || t("overlayStreamErrorFallback"), true);
diff --git a/Sources/VoidDisplaySharing/Services/WebServiceController.swift b/Sources/VoidDisplaySharing/Services/WebServiceController.swift
index 9625dc9..e6bea9e 100644
--- a/Sources/VoidDisplaySharing/Services/WebServiceController.swift
+++ b/Sources/VoidDisplaySharing/Services/WebServiceController.swift
@@ -486,6 +486,21 @@ package final class WebServiceController: WebServiceControllerProtocol {
}
defer { close(socketDescriptor) }
+ var reuseAddress = Int32(1)
+ let optionResult = setsockopt(
+ socketDescriptor,
+ SOL_SOCKET,
+ SO_REUSEADDR,
+ &reuseAddress,
+ socklen_t(MemoryLayout.size)
+ )
+ guard optionResult == 0 else {
+ return .listenerFailed(
+ port: requestedPort,
+ message: String(localized: "Failed to prepare web service socket for port \(requestedPort).")
+ )
+ }
+
var address = sockaddr_in()
address.sin_len = UInt8(MemoryLayout.size)
address.sin_family = sa_family_t(AF_INET)
diff --git a/Sources/VoidDisplaySharing/Views/SharePerformanceModePicker.swift b/Sources/VoidDisplaySharing/Views/SharePerformanceModePicker.swift
index f355235..d827963 100644
--- a/Sources/VoidDisplaySharing/Views/SharePerformanceModePicker.swift
+++ b/Sources/VoidDisplaySharing/Views/SharePerformanceModePicker.swift
@@ -25,7 +25,7 @@ package struct SharePerformanceModePicker: View {
}
.frame(maxWidth: .infinity, alignment: .center)
- Text("Automatic balances H.264 compatibility, clarity, and encoder load.")
+ Text("Automatic and Smooth send AV1 at source resolution on LAN. Actual frame rate follows encoder capability.")
.font(.caption)
.foregroundStyle(.secondary)
.multilineTextAlignment(.center)
diff --git a/Sources/VoidDisplaySharing/Web/RelayHTTPClient.swift b/Sources/VoidDisplaySharing/Web/RelayHTTPClient.swift
index 222e1a6..184ba38 100644
--- a/Sources/VoidDisplaySharing/Web/RelayHTTPClient.swift
+++ b/Sources/VoidDisplaySharing/Web/RelayHTTPClient.swift
@@ -9,7 +9,7 @@ package protocol RelayHTTPClienting: Sendable {
sdpMid: String?,
sdpMLineIndex: Int32
) async throws
- nonisolated func viewerOffer(roomID: String, clientID: String, sdp: String) async throws -> String
+ nonisolated func viewerOffer(roomID: String, clientID: String, sdp: String) async throws -> RelayViewerOfferResponse
nonisolated func viewerCandidate(
roomID: String,
clientID: String,
@@ -31,6 +31,16 @@ package struct RelayPublisherOfferResponse: Sendable, Equatable {
}
}
+package struct RelayViewerOfferResponse: Sendable, Equatable {
+ package let sdp: String
+ package let codec: WebRTCVideoCodec
+
+ package init(sdp: String, codec: WebRTCVideoCodec) {
+ self.sdp = sdp
+ self.codec = codec
+ }
+}
+
package struct RelayReadyEvent: Decodable, Sendable {
package let type: String
package let loopback: String
@@ -51,6 +61,12 @@ package final class RelayHTTPClient: RelayHTTPClienting, @unchecked Sendable {
private struct SignalResponse: Decodable {
let type: String
let sdp: String?
+ let codec: String?
+ let reason: String?
+ }
+
+ private struct ErrorResponse: Decodable {
+ let type: String?
let reason: String?
}
@@ -100,7 +116,7 @@ package final class RelayHTTPClient: RelayHTTPClienting, @unchecked Sendable {
)
}
- package nonisolated func viewerOffer(roomID: String, clientID: String, sdp: String) async throws -> String {
+ package nonisolated func viewerOffer(roomID: String, clientID: String, sdp: String) async throws -> RelayViewerOfferResponse {
try await postOffer(path: ["room", roomID, "viewer", clientID], sdp: sdp)
}
@@ -127,7 +143,7 @@ package final class RelayHTTPClient: RelayHTTPClienting, @unchecked Sendable {
try? await sendEmpty(method: "DELETE", path: ["room", roomID, "publisher", publisherID])
}
- private nonisolated func postOffer(path: [String], sdp: String) async throws -> String {
+ private nonisolated func postOffer(path: [String], sdp: String) async throws -> RelayViewerOfferResponse {
let response: SignalResponse = try await sendJSON(
method: "POST",
path: path,
@@ -136,7 +152,11 @@ package final class RelayHTTPClient: RelayHTTPClienting, @unchecked Sendable {
guard response.type == "answer", let answerSDP = response.sdp else {
throw RelayHTTPError.unexpectedResponse(response.reason ?? response.type)
}
- return answerSDP
+ guard let rawCodec = response.codec,
+ let codec = WebRTCVideoCodec(rawValue: rawCodec) else {
+ throw RelayHTTPError.unexpectedResponse(response.codec ?? "missing_codec")
+ }
+ return RelayViewerOfferResponse(sdp: answerSDP, codec: codec)
}
private nonisolated func postCandidate(
@@ -209,7 +229,9 @@ package final class RelayHTTPClient: RelayHTTPClienting, @unchecked Sendable {
throw RelayHTTPError.invalidHTTPResponse
}
guard (200..<300).contains(httpResponse.statusCode) else {
- let reason = String(data: data, encoding: .utf8) ?? "status_\(httpResponse.statusCode)"
+ let reason = (try? JSONDecoder().decode(ErrorResponse.self, from: data).reason)
+ ?? String(data: data, encoding: .utf8)
+ ?? "status_\(httpResponse.statusCode)"
throw RelayHTTPError.httpStatus(httpResponse.statusCode, reason)
}
}
@@ -220,6 +242,15 @@ package enum RelayHTTPError: Error, LocalizedError, Equatable {
case httpStatus(Int, String)
case unexpectedResponse(String)
+ package var relayReason: String? {
+ switch self {
+ case .httpStatus(_, let reason), .unexpectedResponse(let reason):
+ reason
+ case .invalidHTTPResponse:
+ nil
+ }
+ }
+
package var errorDescription: String? {
switch self {
case .invalidHTTPResponse:
diff --git a/Sources/VoidDisplaySharing/Web/RelayProcessController.swift b/Sources/VoidDisplaySharing/Web/RelayProcessController.swift
index a8c02be..fe7160c 100644
--- a/Sources/VoidDisplaySharing/Web/RelayProcessController.swift
+++ b/Sources/VoidDisplaySharing/Web/RelayProcessController.swift
@@ -156,6 +156,7 @@ package final class RelayProcessController: @unchecked Sendable {
"--control-token", controlToken,
"--listen-udp", ":0",
"--loopback-http", "127.0.0.1:0",
+ "--parent-pid", "\(ProcessInfo.processInfo.processIdentifier)",
]
process.standardOutput = stdout
process.standardError = stderr
diff --git a/Sources/VoidDisplaySharing/Web/RelaySessionHub.swift b/Sources/VoidDisplaySharing/Web/RelaySessionHub.swift
index 7dc6543..f6c893c 100644
--- a/Sources/VoidDisplaySharing/Web/RelaySessionHub.swift
+++ b/Sources/VoidDisplaySharing/Web/RelaySessionHub.swift
@@ -8,6 +8,7 @@ import Synchronization
package protocol RelayPublisherSessioning: AnyObject, Sendable {
nonisolated func start() async throws
nonisolated func updateEncodingProfile(_ profile: WebRTCStreamingProfile)
+ nonisolated func updateActiveCodecs(_ activeCodecs: Set)
nonisolated func submitFrame(pixelBuffer: CVPixelBuffer, ptsUs: UInt64)
nonisolated func close()
}
@@ -48,6 +49,7 @@ package final class RelaySessionHub: Sendable, SignalSessionHub {
let sessionEpoch: UInt64
let target: ShareTarget
let eventSink: @Sendable (SharingSessionEvent) -> Void
+ var selectedCodec: WebRTCVideoCodec?
var nextEventSequence: UInt64 = 0
var isSending = false
var pendingSignals: [QueuedSignal] = []
@@ -57,7 +59,9 @@ package final class RelaySessionHub: Sendable, SignalSessionHub {
var clients: [ObjectIdentifier: ClientState] = [:]
var nextSessionEpoch: UInt64 = 0
var onDemandChanged: @Sendable (Bool) -> Void
- var streamingProfile = WebRTCStreamingProfile(performanceMode: .automatic)
+ var performanceMode: CapturePerformanceMode = .automatic
+ var sourceVideoSpec: SourceVideoSpec = .defaultShared
+ var streamingProfile = WebRTCStreamingProfile(performanceMode: .automatic, sourceVideoSpec: .defaultShared)
var publisher: (any RelayPublisherSessioning)?
var publisherTask: Task?
var publisherGeneration: UInt64 = 0
@@ -114,12 +118,29 @@ package final class RelaySessionHub: Sendable, SignalSessionHub {
}
package nonisolated func updatePerformanceMode(_ mode: CapturePerformanceMode) {
- let profile = WebRTCStreamingProfile(performanceMode: mode)
- let publisher = state.withLock { state -> (any RelayPublisherSessioning)? in
- guard state.streamingProfile != profile else { return nil }
+ let (publisher, profile) = state.withLock { state -> ((any RelayPublisherSessioning)?, WebRTCStreamingProfile?) in
+ let profile = WebRTCStreamingProfile(performanceMode: mode, sourceVideoSpec: state.sourceVideoSpec)
+ guard state.streamingProfile != profile else { return (nil, nil) }
+ state.performanceMode = mode
state.streamingProfile = profile
- return state.publisher
+ return (state.publisher, profile)
}
+ guard let profile else { return }
+ publisher?.updateEncodingProfile(profile)
+ }
+
+ package nonisolated func updateSourceVideoSpec(_ spec: SourceVideoSpec) {
+ let (publisher, profile) = state.withLock { state -> ((any RelayPublisherSessioning)?, WebRTCStreamingProfile?) in
+ guard state.sourceVideoSpec != spec else { return (nil, nil) }
+ state.sourceVideoSpec = spec
+ let profile = WebRTCStreamingProfile(
+ performanceMode: state.performanceMode,
+ sourceVideoSpec: spec
+ )
+ state.streamingProfile = profile
+ return (state.publisher, profile)
+ }
+ guard let profile else { return }
publisher?.updateEncodingProfile(profile)
}
@@ -350,17 +371,19 @@ package final class RelaySessionHub: Sendable, SignalSessionHub {
roomID: String,
generation: UInt64
) {
- let decision = state.withLock { state -> (shouldClose: Bool, profile: WebRTCStreamingProfile?) in
+ let decision = state.withLock {
+ state -> (shouldClose: Bool, profile: WebRTCStreamingProfile?, activeCodecs: Set) in
guard state.publisherGeneration == generation else {
- return (true, nil)
+ return (true, nil, [])
}
state.publisherTask = nil
guard !state.clients.isEmpty, state.roomID == roomID else {
- return (true, nil)
+ return (true, nil, [])
}
let profile = state.streamingProfile
+ let activeCodecs = Self.activeCodecs(from: state.clients)
state.publisher = publisher
- return (false, profile)
+ return (false, profile, activeCodecs)
}
if decision.shouldClose {
publisher.close()
@@ -368,6 +391,7 @@ package final class RelaySessionHub: Sendable, SignalSessionHub {
}
if let profile = decision.profile {
publisher.updateEncodingProfile(profile)
+ publisher.updateActiveCodecs(decision.activeCodecs)
}
AppLog.web.info("Relay publisher started for room \(roomID, privacy: .public).")
}
@@ -409,15 +433,54 @@ package final class RelaySessionHub: Sendable, SignalSessionHub {
await relayClient.removeViewer(roomID: roomID, clientID: clientID)
return
}
- send(message: SignalingOutboundMessage(type: .answer, sdp: answer), to: key)
+ let update = state.withLock {
+ state -> (
+ publisher: (any RelayPublisherSessioning)?,
+ activeCodecs: Set,
+ sourceVideoSpec: SourceVideoSpec
+ ) in
+ guard var client = state.clients[key],
+ client.clientID == clientID,
+ client.sessionEpoch == sessionEpoch,
+ Self.roomID(for: client.target) == roomID else {
+ return (nil, [], state.sourceVideoSpec)
+ }
+ client.selectedCodec = answer.codec
+ state.clients[key] = client
+ return (state.publisher, Self.activeCodecs(from: state.clients), state.sourceVideoSpec)
+ }
+ update.publisher?.updateActiveCodecs(update.activeCodecs)
+ send(
+ message: SignalingOutboundMessage(
+ type: .answer,
+ sdp: answer.sdp,
+ sourceVideoSpec: SourceVideoSpecSignalPayload(spec: update.sourceVideoSpec)
+ ),
+ to: key
+ )
emitEvent(phase: .peerConnected, source: .peerConnection, for: key)
} catch {
guard isCurrentViewer(key: key, roomID: roomID, clientID: clientID, sessionEpoch: sessionEpoch) else {
return
}
AppLog.web.warning("Relay viewer offer failed: \(String(describing: error), privacy: .public)")
- send(message: SignalingOutboundMessage(type: .error, reason: "relay_viewer_offer_failed"), to: key)
- removeClient(for: key, cancelConnection: true)
+ let relayReason = (error as? RelayHTTPError)?.relayReason
+ if relayReason == "publisher_codec_pending" {
+ enqueue(
+ message: SignalingOutboundMessage(type: .codecPending, reason: "publisher_codec_pending"),
+ to: key,
+ disconnectAfterSend: false,
+ replacePending: true
+ )
+ return
+ }
+ let message: SignalingOutboundMessage
+ if relayReason == "unsupported_video_codec_offered" || relayReason == "supported_video_codec_missing" {
+ message = SignalingOutboundMessage(type: .error, reason: relayReason)
+ } else {
+ message = SignalingOutboundMessage(type: .error, reason: "relay_viewer_offer_failed")
+ }
+ enqueue(message: message, to: key, disconnectAfterSend: true, replacePending: true)
}
}
@@ -598,13 +661,28 @@ package final class RelaySessionHub: Sendable, SignalSessionHub {
}
private nonisolated func removeClient(for key: ObjectIdentifier, cancelConnection: Bool) {
- let (removed, shouldSignalDemandOff, callback, roomID) = state.withLock {
- state -> (ClientState?, Bool, @Sendable (Bool) -> Void, String?) in
+ let (removed, shouldSignalDemandOff, callback, roomID, publisher, activeCodecs) = state.withLock {
+ state -> (
+ ClientState?,
+ Bool,
+ @Sendable (Bool) -> Void,
+ String?,
+ (any RelayPublisherSessioning)?,
+ Set
+ ) in
let removed = state.clients.removeValue(forKey: key)
- return (removed, state.clients.isEmpty, state.onDemandChanged, state.roomID)
+ return (
+ removed,
+ state.clients.isEmpty,
+ state.onDemandChanged,
+ state.roomID,
+ state.publisher,
+ Self.activeCodecs(from: state.clients)
+ )
}
guard let removed else { return }
+ publisher?.updateActiveCodecs(activeCodecs)
removed.eventSink(
SharingSessionEvent(
target: removed.target,
@@ -672,6 +750,10 @@ package final class RelaySessionHub: Sendable, SignalSessionHub {
}
}
+ private nonisolated static func activeCodecs(from clients: [ObjectIdentifier: ClientState]) -> Set {
+ Set(clients.values.compactMap(\.selectedCodec))
+ }
+
private nonisolated static func roomID(for target: ShareTarget) -> String {
switch target {
case .main:
diff --git a/Sources/VoidDisplaySharing/Web/WebRTCPublisherSession.swift b/Sources/VoidDisplaySharing/Web/WebRTCPublisherSession.swift
index 41f01a1..ac43813 100644
--- a/Sources/VoidDisplaySharing/Web/WebRTCPublisherSession.swift
+++ b/Sources/VoidDisplaySharing/Web/WebRTCPublisherSession.swift
@@ -18,16 +18,23 @@ package final class WebRTCPublisherSession: NSObject, @unchecked Sendable {
case closed
}
+ private struct VideoTransceiverBinding {
+ let codec: WebRTCVideoCodec
+ let transceiver: RTCRtpTransceiver
+ }
+
nonisolated(unsafe) private let peerConnection: RTCPeerConnection
private let mediaPipeline: WebRTCMediaPipeline
private let relayClient: any RelayHTTPClienting
private let roomID: String
private let stateLock = Mutex(.idle)
private let profileState: Mutex
+ private let activeCodecsState = Mutex>([])
private let publisherIDState = Mutex(nil)
private let iceGatheringWaiters = Mutex<[CheckedContinuation]>([])
private let pendingPublisherCandidates = Mutex<[RTCIceCandidate]>([])
- nonisolated(unsafe) private var videoTransceiver: RTCRtpTransceiver?
+ nonisolated(unsafe) private var videoTransceivers: [VideoTransceiverBinding] = []
+ nonisolated(unsafe) private var diagnosticsTask: Task?
package nonisolated init?(
roomID: String,
@@ -89,12 +96,35 @@ package final class WebRTCPublisherSession: NSObject, @unchecked Sendable {
package nonisolated func updateEncodingProfile(_ profile: WebRTCStreamingProfile) {
profileState.withLock { $0 = profile }
mediaPipeline.updateEncodingProfile(profile)
- configureDesktopVideoSender(videoTransceiver?.sender, profile: profile)
- _ = peerConnection.setBweMinBitrateBps(
- NSNumber(value: profile.minBitrateBps),
- currentBitrateBps: nil,
- maxBitrateBps: NSNumber(value: profile.maxBitrateBps)
- )
+ let activeCodecs = activeCodecsState.withLock { $0 }
+ for binding in videoTransceivers {
+ configureDesktopVideoSender(
+ binding.transceiver.sender,
+ codec: binding.codec,
+ profile: profile,
+ isActive: activeCodecs.contains(binding.codec)
+ )
+ }
+ updateBandwidthEstimate(profile: profile, activeCodecs: activeCodecs)
+ }
+
+ package nonisolated func updateActiveCodecs(_ activeCodecs: Set) {
+ let configuredCodecs = Set(videoTransceivers.map(\.codec))
+ let nextActiveCodecs = activeCodecs.intersection(configuredCodecs)
+ activeCodecsState.withLock { $0 = nextActiveCodecs }
+ mediaPipeline.updateActiveCodecs(nextActiveCodecs)
+ let profile = profileState.withLock { $0 }
+ for binding in videoTransceivers {
+ configureDesktopVideoSender(
+ binding.transceiver.sender,
+ codec: binding.codec,
+ profile: profile,
+ isActive: nextActiveCodecs.contains(binding.codec)
+ )
+ }
+ updateBandwidthEstimate(profile: profile, activeCodecs: nextActiveCodecs)
+ let activeSummary = nextActiveCodecs.map(\.logName).sorted().joined(separator: ",")
+ AppLog.web.info("WebRTC publisher active codecs updated active=\(activeSummary, privacy: .public).")
}
package nonisolated func submitFrame(pixelBuffer: CVPixelBuffer, ptsUs: UInt64) {
@@ -109,6 +139,8 @@ package final class WebRTCPublisherSession: NSObject, @unchecked Sendable {
return current
}
pendingPublisherCandidates.withLock { $0.removeAll() }
+ diagnosticsTask?.cancel()
+ diagnosticsTask = nil
resumeIceGatheringWaiters()
peerConnection.close()
guard let publisherID else { return }
@@ -120,23 +152,30 @@ package final class WebRTCPublisherSession: NSObject, @unchecked Sendable {
private nonisolated func startInternal() async throws {
let capabilitySummary = mediaPipeline.senderVideoCodecCapabilitySummary()
AppLog.web.info("WebRTC sender video capabilities \(capabilitySummary, privacy: .public).")
- guard let h264Codecs = mediaPipeline.requiredH264Codecs() else {
- throw PublisherSessionError.h264Unavailable
- }
- let transceiverInit = RTCRtpTransceiverInit()
- transceiverInit.direction = .sendOnly
- transceiverInit.streamIds = ["screen"]
- guard let transceiver = peerConnection.addTransceiver(with: mediaPipeline.videoTrack, init: transceiverInit) else {
- throw PublisherSessionError.videoTransceiverUnavailable
+ guard let av1Codecs = mediaPipeline.requiredCodecs(for: .av1) else {
+ throw PublisherSessionError.av1Unavailable
}
- do {
- try transceiver.setCodecPreferences(h264Codecs, error: ())
- } catch {
- throw PublisherSessionError.codecPreferencesFailed(String(describing: error))
+ let av1Transceiver = try addVideoTransceiver(
+ codec: .av1,
+ track: mediaPipeline.av1VideoTrack,
+ codecs: av1Codecs
+ )
+ let configuredTransceivers = [VideoTransceiverBinding(codec: .av1, transceiver: av1Transceiver)]
+ let configuredCodecs: Set = [.av1]
+ videoTransceivers = configuredTransceivers
+ let initialActiveCodecs = activeCodecsState.withLock { $0.intersection(configuredCodecs) }
+ mediaPipeline.updateActiveCodecs(initialActiveCodecs)
+ let initialProfile = profileState.withLock { $0 }
+ for binding in configuredTransceivers {
+ configureDesktopVideoSender(
+ binding.transceiver.sender,
+ codec: binding.codec,
+ profile: initialProfile,
+ isActive: initialActiveCodecs.contains(binding.codec)
+ )
}
- videoTransceiver = transceiver
- configureDesktopVideoSender(transceiver.sender, profile: profileState.withLock { $0 })
- AppLog.web.info("WebRTC publisher transceiver configured H264 codecs=\(h264Codecs.count, privacy: .public).")
+ updateBandwidthEstimate(profile: initialProfile, activeCodecs: initialActiveCodecs)
+ AppLog.web.info("WebRTC publisher transceiver configured AV1 codecs=\(av1Codecs.count, privacy: .public).")
let constraints = RTCMediaConstraints(mandatoryConstraints: nil, optionalConstraints: nil)
let offer = try await createOffer(constraints: constraints)
@@ -157,9 +196,29 @@ package final class WebRTCPublisherSession: NSObject, @unchecked Sendable {
}
let answer = RTCSessionDescription(type: .answer, sdp: response.sdp)
try await setRemoteDescription(answer)
+ startDiagnosticsLoop()
AppLog.web.info("WebRTC publisher connected to relay room \(self.roomID, privacy: .public).")
}
+ private nonisolated func addVideoTransceiver(
+ codec: WebRTCVideoCodec,
+ track: RTCVideoTrack,
+ codecs: [RTCRtpCodecCapability]
+ ) throws -> RTCRtpTransceiver {
+ let transceiverInit = RTCRtpTransceiverInit()
+ transceiverInit.direction = .sendOnly
+ transceiverInit.streamIds = ["screen"]
+ guard let transceiver = peerConnection.addTransceiver(with: track, init: transceiverInit) else {
+ throw PublisherSessionError.videoTransceiverUnavailable(codec)
+ }
+ do {
+ try transceiver.setCodecPreferences(codecs, error: ())
+ } catch {
+ throw PublisherSessionError.codecPreferencesFailed("\(codec.logName): \(String(describing: error))")
+ }
+ return transceiver
+ }
+
private nonisolated func createOffer(constraints: RTCMediaConstraints) async throws -> RTCSessionDescription {
try await withCheckedThrowingContinuation { continuation in
peerConnection.offer(for: constraints) { offer, error in
@@ -253,24 +312,115 @@ package final class WebRTCPublisherSession: NSObject, @unchecked Sendable {
private nonisolated func configureDesktopVideoSender(
_ sender: RTCRtpSender?,
- profile: WebRTCStreamingProfile
+ codec: WebRTCVideoCodec,
+ profile: WebRTCStreamingProfile,
+ isActive: Bool
) {
guard let sender else { return }
let parameters = sender.parameters
if parameters.encodings.isEmpty {
parameters.encodings = [RTCRtpEncodingParameters()]
}
+ let sourceDimensions = profile.sourceVideoSpec.dimensions
+ let outputDimensions = profile.outputDimensions(
+ for: codec,
+ width: Int32(sourceDimensions.width),
+ height: Int32(sourceDimensions.height)
+ )
+ let bitrateLimits = profile.bitrateLimits(
+ for: codec,
+ outputWidth: outputDimensions.width,
+ outputHeight: outputDimensions.height
+ )
+ let framesPerSecond = profile.framesPerSecond(for: codec)
for encoding in parameters.encodings {
- encoding.maxBitrateBps = NSNumber(value: profile.maxBitrateBps)
- encoding.minBitrateBps = NSNumber(value: profile.minBitrateBps)
- encoding.maxFramerate = NSNumber(value: profile.framesPerSecond)
+ encoding.isActive = isActive
+ encoding.maxBitrateBps = NSNumber(value: bitrateLimits.maxBitrateBps)
+ encoding.minBitrateBps = NSNumber(value: bitrateLimits.minBitrateBps)
+ encoding.maxFramerate = NSNumber(value: framesPerSecond)
encoding.scaleResolutionDownBy = NSNumber(value: 1.0)
encoding.bitratePriority = 4.0
+ encoding.networkPriority = .high
}
parameters.degradationPreference = Self.maintainResolutionPreference
sender.parameters = parameters
}
+ private nonisolated func updateBandwidthEstimate(
+ profile: WebRTCStreamingProfile,
+ activeCodecs: Set
+ ) {
+ let sourceDimensions = profile.sourceVideoSpec.dimensions
+ let codecsForBudget = activeCodecs.isEmpty ? Set(WebRTCVideoCodec.allCases) : activeCodecs
+ let limits = codecsForBudget.map { codec in
+ let outputDimensions = profile.outputDimensions(
+ for: codec,
+ width: Int32(sourceDimensions.width),
+ height: Int32(sourceDimensions.height)
+ )
+ return profile.bitrateLimits(
+ for: codec,
+ outputWidth: outputDimensions.width,
+ outputHeight: outputDimensions.height
+ )
+ }
+ let minBitrateBps = limits.map(\.minBitrateBps).min() ?? profile.minBitrateBps
+ let maxBitrateBps = limits.map(\.maxBitrateBps).reduce(0, +)
+ _ = peerConnection.setBweMinBitrateBps(
+ NSNumber(value: minBitrateBps),
+ currentBitrateBps: nil,
+ maxBitrateBps: NSNumber(value: max(maxBitrateBps, profile.maxBitrateBps))
+ )
+ }
+
+ private nonisolated func startDiagnosticsLoop() {
+ diagnosticsTask?.cancel()
+ diagnosticsTask = Task { [weak self] in
+ await self?.runDiagnosticsLoop()
+ }
+ }
+
+ private nonisolated func runDiagnosticsLoop() async {
+ while !Task.isCancelled, !isClosed {
+ await logSenderDiagnostics()
+ try? await Task.sleep(nanoseconds: 5_000_000_000)
+ }
+ }
+
+ private nonisolated func logSenderDiagnostics() async {
+ let bindings = videoTransceivers
+ for binding in bindings {
+ let report = await statistics(for: binding.transceiver.sender)
+ logOutboundVideoStats(report: report, codec: binding.codec)
+ }
+ }
+
+ private nonisolated func statistics(for sender: RTCRtpSender) async -> RTCStatisticsReport {
+ await withCheckedContinuation { continuation in
+ peerConnection.statistics(for: sender) { report in
+ continuation.resume(returning: report)
+ }
+ }
+ }
+
+ private nonisolated func logOutboundVideoStats(report: RTCStatisticsReport, codec: WebRTCVideoCodec) {
+ for statistic in report.statistics.values where statistic.type == "outbound-rtp" {
+ let values = statistic.values
+ let kind = (values["kind"] ?? values["mediaType"])?.description.lowercased()
+ guard kind == nil || kind == "video" else { continue }
+ let width = values["frameWidth"]?.description ?? "unknown"
+ let height = values["frameHeight"]?.description ?? "unknown"
+ let fps = values["framesPerSecond"]?.description ?? "unknown"
+ let targetBitrate = values["targetBitrate"]?.description ?? "unknown"
+ let qualityLimitationReason = values["qualityLimitationReason"]?.description ?? "unknown"
+ let encoderImplementation = values["encoderImplementation"]?.description ?? "unknown"
+ AppLog.web.info(
+ "WebRTC publisher outbound stats codec=\(codec.logName, privacy: .public) encoded=\(width, privacy: .public)x\(height, privacy: .public) fps=\(fps, privacy: .public) targetBitrate=\(targetBitrate, privacy: .public) qualityLimitationReason=\(qualityLimitationReason, privacy: .public) encoder=\(encoderImplementation, privacy: .public)."
+ )
+ return
+ }
+ }
+
private nonisolated func sendPublisherCandidate(_ candidate: RTCIceCandidate) {
guard let publisherID = publisherIDState.withLock({ $0 }) else {
pendingPublisherCandidates.withLock { $0.append(candidate) }
@@ -342,8 +492,8 @@ extension WebRTCPublisherSession: RTCPeerConnectionDelegate {
package enum PublisherSessionError: Error, LocalizedError, Equatable {
case closed
- case h264Unavailable
- case videoTransceiverUnavailable
+ case av1Unavailable
+ case videoTransceiverUnavailable(WebRTCVideoCodec)
case codecPreferencesFailed(String)
case offerMissing
@@ -351,10 +501,10 @@ package enum PublisherSessionError: Error, LocalizedError, Equatable {
switch self {
case .closed:
"Publisher session is closed."
- case .h264Unavailable:
- String(localized: "This Mac's WebRTC stack did not expose H.264 video encoding.")
- case .videoTransceiverUnavailable:
- "WebRTC publisher video transceiver is unavailable."
+ case .av1Unavailable:
+ String(localized: "This Mac's WebRTC stack did not expose AV1 video encoding.")
+ case .videoTransceiverUnavailable(let codec):
+ "WebRTC publisher \(codec.logName) video transceiver is unavailable."
case .codecPreferencesFailed(let reason):
"Failed to set publisher codec preferences: \(reason)"
case .offerMissing:
diff --git a/Sources/VoidDisplaySharing/Web/WebRTCSessionSupport.swift b/Sources/VoidDisplaySharing/Web/WebRTCSessionSupport.swift
index 2cd0dbb..145b9b4 100644
--- a/Sources/VoidDisplaySharing/Web/WebRTCSessionSupport.swift
+++ b/Sources/VoidDisplaySharing/Web/WebRTCSessionSupport.swift
@@ -29,58 +29,147 @@ package enum SignalSessionClientAddResult: Sendable, Equatable {
case rejected(reason: String)
}
+package enum WebRTCVideoCodec: String, CaseIterable, Sendable {
+ case av1
+
+ package var logName: String {
+ switch self {
+ case .av1:
+ "AV1"
+ }
+ }
+}
+
package struct WebRTCStreamingProfile: Sendable, Equatable {
- package static let h264AutomaticPixelBudgetPerSecond: Int64 = 26_542_080
- package static let h264SmoothPixelBudgetPerSecond: Int64 = 26_542_080
- package static let h264PowerEfficientPixelBudgetPerSecond: Int64 = 15_552_000
- private static let h264Level31MaxMacroblocksPerFrame = 3_600
- private static let h264Level31MaxMacroblocksPerSecond = 108_000
+ private static let av1SourceBitsPerPixel: Double = 0.10
+ private static let av1PowerEfficientBitsPerPixel: Double = 0.05
+ package let performanceMode: CapturePerformanceMode
+ package let sourceVideoSpec: SourceVideoSpec
package let framesPerSecond: Int
package let minBitrateBps: Int
package let maxBitrateBps: Int
package let pixelBudgetPerSecond: Int64?
package init(
+ performanceMode: CapturePerformanceMode,
+ sourceVideoSpec: SourceVideoSpec,
framesPerSecond: Int,
- minBitrateBps: Int,
- maxBitrateBps: Int,
pixelBudgetPerSecond: Int64?
) {
- self.framesPerSecond = framesPerSecond
- self.minBitrateBps = minBitrateBps
- self.maxBitrateBps = maxBitrateBps
+ self.performanceMode = performanceMode
+ self.sourceVideoSpec = sourceVideoSpec
+ self.framesPerSecond = max(1, framesPerSecond)
self.pixelBudgetPerSecond = pixelBudgetPerSecond
+ let sourceDimensions = sourceVideoSpec.dimensions
+ let maxBitrateBps = Self.targetMaxBitrateBps(
+ for: .av1,
+ dimensions: sourceDimensions,
+ framesPerSecond: self.framesPerSecond,
+ performanceMode: performanceMode
+ )
+ self.maxBitrateBps = maxBitrateBps
+ self.minBitrateBps = Self.targetMinBitrateBps(maxBitrateBps: maxBitrateBps)
}
- package init(performanceMode: CapturePerformanceMode) {
+ package init(
+ performanceMode: CapturePerformanceMode,
+ sourceVideoSpec: SourceVideoSpec = .defaultShared
+ ) {
let budget = SharedCapturePerformanceBudget(performanceMode: performanceMode)
+ let sourceFramesPerSecond = sourceVideoSpec.framesPerSecond
switch performanceMode {
case .automatic:
self.init(
- framesPerSecond: 30,
- minBitrateBps: 1_500_000,
- maxBitrateBps: 8_000_000,
- pixelBudgetPerSecond: Self.h264AutomaticPixelBudgetPerSecond
+ performanceMode: performanceMode,
+ sourceVideoSpec: sourceVideoSpec,
+ framesPerSecond: sourceFramesPerSecond,
+ pixelBudgetPerSecond: nil
)
case .smooth:
self.init(
- framesPerSecond: budget.framesPerSecond,
- minBitrateBps: 1_500_000,
- maxBitrateBps: 8_000_000,
- pixelBudgetPerSecond: Self.h264SmoothPixelBudgetPerSecond
+ performanceMode: performanceMode,
+ sourceVideoSpec: sourceVideoSpec,
+ framesPerSecond: sourceFramesPerSecond,
+ pixelBudgetPerSecond: nil
)
case .powerEfficient:
self.init(
- framesPerSecond: budget.framesPerSecond,
- minBitrateBps: 800_000,
- maxBitrateBps: 5_000_000,
- pixelBudgetPerSecond: Self.h264PowerEfficientPixelBudgetPerSecond
+ performanceMode: performanceMode,
+ sourceVideoSpec: sourceVideoSpec,
+ framesPerSecond: min(sourceFramesPerSecond, budget.framesPerSecond),
+ pixelBudgetPerSecond: SharedCapturePerformanceBudget.powerEfficientPixelBudgetPerSecond
)
}
}
+ package func bitrateLimits(
+ for codec: WebRTCVideoCodec,
+ outputWidth: Int32,
+ outputHeight: Int32
+ ) -> (minBitrateBps: Int, maxBitrateBps: Int) {
+ let maxBitrateBps = Self.targetMaxBitrateBps(
+ for: codec,
+ dimensions: CapturePixelDimensions(width: Int(outputWidth), height: Int(outputHeight)),
+ framesPerSecond: framesPerSecond(for: codec),
+ performanceMode: performanceMode
+ )
+ return (
+ minBitrateBps: Self.targetMinBitrateBps(maxBitrateBps: maxBitrateBps),
+ maxBitrateBps: maxBitrateBps
+ )
+ }
+
+ package func outputDimensions(
+ for codec: WebRTCVideoCodec,
+ width: Int32,
+ height: Int32
+ ) -> (width: Int32, height: Int32) {
+ outputDimensions(
+ forWidth: width,
+ height: height,
+ framesPerSecond: framesPerSecond(for: codec),
+ pixelBudgetPerSecond: pixelBudgetPerSecond(for: codec)
+ )
+ }
+
package func outputDimensions(forWidth width: Int32, height: Int32) -> (width: Int32, height: Int32) {
+ outputDimensions(
+ forWidth: width,
+ height: height,
+ framesPerSecond: framesPerSecond,
+ pixelBudgetPerSecond: pixelBudgetPerSecond
+ )
+ }
+
+ package func framesPerSecond(for codec: WebRTCVideoCodec) -> Int {
+ framesPerSecond
+ }
+
+ package func outputVideoSpec(for codec: WebRTCVideoCodec) -> SourceVideoSpec {
+ let sourceDimensions = sourceVideoSpec.dimensions
+ let outputDimensions = outputDimensions(
+ for: codec,
+ width: Int32(sourceDimensions.width),
+ height: Int32(sourceDimensions.height)
+ )
+ return SourceVideoSpec(
+ width: Int(outputDimensions.width),
+ height: Int(outputDimensions.height),
+ framesPerSecond: framesPerSecond(for: codec)
+ )
+ }
+
+ private func pixelBudgetPerSecond(for codec: WebRTCVideoCodec) -> Int64? {
+ pixelBudgetPerSecond
+ }
+
+ private func outputDimensions(
+ forWidth width: Int32,
+ height: Int32,
+ framesPerSecond: Int,
+ pixelBudgetPerSecond: Int64?
+ ) -> (width: Int32, height: Int32) {
guard width > 0, height > 0 else {
return (width, height)
}
@@ -92,53 +181,31 @@ package struct WebRTCStreamingProfile: Sendable, Equatable {
let dimensions = budget.captureDimensions(
for: CapturePixelDimensions(width: Int(width), height: Int(height))
)
- let h264Dimensions = Self.constrainToH264Level31(
- dimensions,
- framesPerSecond: framesPerSecond
- )
return (
- width: Int32(h264Dimensions.width),
- height: Int32(h264Dimensions.height)
+ width: Int32(dimensions.width),
+ height: Int32(dimensions.height)
)
}
- private static func constrainToH264Level31(
- _ dimensions: CapturePixelDimensions,
- framesPerSecond: Int
- ) -> CapturePixelDimensions {
- let framesPerSecond = max(1, framesPerSecond)
- let frameMacroblockBudget = min(
- h264Level31MaxMacroblocksPerFrame,
- h264Level31MaxMacroblocksPerSecond / framesPerSecond
- )
- guard frameMacroblockBudget > 0,
- macroblockCount(for: dimensions) > frameMacroblockBudget else {
- return dimensions
- }
-
- var constrained = dimensions.constrained(
- toFramePixelBudget: Int64(frameMacroblockBudget * 16 * 16)
- )
- while macroblockCount(for: constrained) > frameMacroblockBudget {
- if constrained.width >= constrained.height {
- constrained = CapturePixelDimensions(
- width: constrained.width - 2,
- height: constrained.height
- )
- } else {
- constrained = CapturePixelDimensions(
- width: constrained.width,
- height: constrained.height - 2
- )
- }
+ private static func targetMaxBitrateBps(
+ for codec: WebRTCVideoCodec,
+ dimensions: CapturePixelDimensions,
+ framesPerSecond: Int,
+ performanceMode: CapturePerformanceMode
+ ) -> Int {
+ let pixelRate = Double(dimensions.pixelCount) * Double(max(1, framesPerSecond))
+ let bitsPerPixel: Double = switch (codec, performanceMode) {
+ case (.av1, .powerEfficient):
+ av1PowerEfficientBitsPerPixel
+ case (.av1, _):
+ av1SourceBitsPerPixel
}
- return constrained
+ let target = Int((pixelRate * bitsPerPixel).rounded())
+ return max(2_000_000, target)
}
- private static func macroblockCount(for dimensions: CapturePixelDimensions) -> Int {
- let macroblockWidth = (dimensions.width + 15) / 16
- let macroblockHeight = (dimensions.height + 15) / 16
- return macroblockWidth * macroblockHeight
+ private static func targetMinBitrateBps(maxBitrateBps: Int) -> Int {
+ max(1_500_000, maxBitrateBps / 4)
}
}
@@ -199,6 +266,7 @@ package enum SignalingMessageType: String, Codable {
case iceCandidate = "ice_candidate"
case iceComplete = "ice_complete"
case ready
+ case codecPending = "codec_pending"
case stopped
case error
}
@@ -211,6 +279,18 @@ package struct SignalingInboundMessage: Decodable {
package let sdpMLineIndex: Int?
}
+package struct SourceVideoSpecSignalPayload: Codable, Sendable, Equatable {
+ package let width: Int
+ package let height: Int
+ package let framesPerSecond: Int
+
+ package init(spec: SourceVideoSpec) {
+ self.width = spec.dimensions.width
+ self.height = spec.dimensions.height
+ self.framesPerSecond = spec.framesPerSecond
+ }
+}
+
package struct SignalingOutboundMessage: Encodable {
package let type: SignalingMessageType
package let reason: String?
@@ -218,6 +298,7 @@ package struct SignalingOutboundMessage: Encodable {
package let candidate: String?
package let sdpMid: String?
package let sdpMLineIndex: Int?
+ package let sourceVideoSpec: SourceVideoSpecSignalPayload?
package init(
type: SignalingMessageType,
@@ -225,7 +306,8 @@ package struct SignalingOutboundMessage: Encodable {
sdp: String? = nil,
candidate: String? = nil,
sdpMid: String? = nil,
- sdpMLineIndex: Int? = nil
+ sdpMLineIndex: Int? = nil,
+ sourceVideoSpec: SourceVideoSpecSignalPayload? = nil
) {
self.type = type
self.reason = reason
@@ -233,6 +315,7 @@ package struct SignalingOutboundMessage: Encodable {
self.candidate = candidate
self.sdpMid = sdpMid
self.sdpMLineIndex = sdpMLineIndex
+ self.sourceVideoSpec = sourceVideoSpec
}
}
@@ -319,28 +402,39 @@ package struct WebRTCCodecPreferenceDescriptor: Sendable, Equatable {
}
}
+extension WebRTCVideoCodec {
+ package var codecName: String {
+ switch self {
+ case .av1:
+ "AV1"
+ }
+ }
+}
+
package enum WebRTCCodecPreference {
- package nonisolated static func requiredH264DescriptorIndexes(
+ package nonisolated static func requiredDescriptorIndexes(
+ for codec: WebRTCVideoCodec,
from descriptors: [WebRTCCodecPreferenceDescriptor]
) -> [Int]? {
- let h264Indexes = descriptors.indices.filter {
- descriptors[$0].name.caseInsensitiveCompare(kRTCH264CodecName) == .orderedSame
+ let primaryIndexes = descriptors.indices.filter {
+ descriptors[$0].name.caseInsensitiveCompare(codec.codecName) == .orderedSame
}
- guard !h264Indexes.isEmpty else { return nil }
+ guard !primaryIndexes.isEmpty else { return nil }
- let h264PayloadTypes = Set(h264Indexes.compactMap { descriptors[$0].payloadType })
+ let primaryPayloadTypes = Set(primaryIndexes.compactMap { descriptors[$0].payloadType })
let rtxIndexes = descriptors.indices.filter { index in
let descriptor = descriptors[index]
guard descriptor.name.caseInsensitiveCompare(kRTCRtxCodecName) == .orderedSame,
let apt = descriptor.parameters["apt"].flatMap(Int.init) else {
return false
}
- return h264PayloadTypes.contains(apt)
+ return primaryPayloadTypes.contains(apt)
}
- return h264Indexes + rtxIndexes
+ return primaryIndexes + rtxIndexes
}
- package nonisolated static func requiredH264Codecs(
+ package nonisolated static func requiredCodecs(
+ for codec: WebRTCVideoCodec,
from codecs: [RTCRtpCodecCapability]
) -> [RTCRtpCodecCapability]? {
let descriptors = codecs.map {
@@ -350,7 +444,7 @@ package enum WebRTCCodecPreference {
parameters: $0.parameters
)
}
- guard let indexes = requiredH264DescriptorIndexes(from: descriptors) else {
+ guard let indexes = requiredDescriptorIndexes(for: codec, from: descriptors) else {
return nil
}
return indexes.map { codecs[$0] }
@@ -372,34 +466,63 @@ package enum WebRTCCodecPreference {
package nonisolated static func capabilitySummary(
from descriptors: [WebRTCCodecPreferenceDescriptor]
) -> String {
- let h264Descriptors = descriptors.filter {
- $0.name.caseInsensitiveCompare(kRTCH264CodecName) == .orderedSame
+ let av1Descriptors = descriptors.filter {
+ $0.name.caseInsensitiveCompare(WebRTCVideoCodec.av1.codecName) == .orderedSame
}
- let h264Summary = h264Descriptors.map { descriptor in
+ let otherCodecCount = descriptors.count - av1Descriptors.count
+ return [
+ "AV1=\(capabilityProbeSummary(from: av1Descriptors))",
+ "unsupportedVideoCodecCount=\(otherCodecCount)",
+ ].joined(separator: "; ")
+ }
+
+ private nonisolated static func capabilityProbeSummary(
+ from descriptors: [WebRTCCodecPreferenceDescriptor]
+ ) -> String {
+ guard !descriptors.isEmpty else { return "missing" }
+ return descriptors.map { descriptor in
let payloadType = descriptor.payloadType.map(String.init) ?? "none"
let parameters = descriptor.parameters
.sorted { $0.key < $1.key }
.map { "\($0.key)=\($0.value)" }
- .joined(separator: ";")
- return "\(descriptor.name)(pt=\(payloadType),params=\(parameters))"
+ .joined(separator: ",")
+ return "\(descriptor.name)(pt=\(payloadType),fmtp=\(parameters.isEmpty ? "none" : parameters))"
}
.joined(separator: ",")
- let h264Text = h264Summary.isEmpty ? "none" : h264Summary
- return "\(h264Text); nonH264CodecCount=\(descriptors.count - h264Descriptors.count)"
}
package nonisolated static func sdpVideoCodecSummary(from sdp: String) -> String {
let lines = sdp.split(whereSeparator: \.isNewline).map(String.init)
- var videoPayloadTypes: [String] = []
- var payloadNames: [String: String] = [:]
+ var currentVideoPayloadTypes: [String] = []
+ var currentPayloadNames: [String: String] = [:]
var isVideoMedia = false
+ var summary: [String] = []
+ var unexpectedCodecCount = 0
+
+ func flushVideoMedia() {
+ guard isVideoMedia else { return }
+ for payloadType in currentVideoPayloadTypes {
+ guard let name = currentPayloadNames[payloadType] else { continue }
+ let normalizedName = name.lowercased()
+ guard normalizedName == "av1" || normalizedName == "rtx" else {
+ unexpectedCodecCount += 1
+ continue
+ }
+ summary.append("\(payloadType):\(name)")
+ }
+ }
for line in lines {
if line.hasPrefix("m=") {
+ flushVideoMedia()
isVideoMedia = line.hasPrefix("m=video ")
+ currentVideoPayloadTypes = []
+ currentPayloadNames = [:]
if isVideoMedia {
let parts = line.split(separator: " ").map(String.init)
- videoPayloadTypes = parts.count > 3 ? Array(parts.dropFirst(3)) : []
+ if parts.count > 3 {
+ currentVideoPayloadTypes = Array(parts.dropFirst(3))
+ }
}
continue
}
@@ -412,14 +535,12 @@ package enum WebRTCCodecPreference {
.split(separator: "/", maxSplits: 1)
.first
.map(String.init) ?? ""
- payloadNames[payloadType] = codecName
+ currentPayloadNames[payloadType] = codecName
}
+ flushVideoMedia()
- let summary = videoPayloadTypes.compactMap { payloadType -> String? in
- guard let name = payloadNames[payloadType] else { return nil }
- return "\(payloadType):\(name)"
- }
- return summary.isEmpty ? "none" : summary.joined(separator: ",")
+ let text = summary.isEmpty ? "none" : summary.joined(separator: ",")
+ return "\(text); unexpectedVideoCodecCount=\(unexpectedCodecCount)"
}
}
@@ -436,7 +557,15 @@ package final class WebRTCMediaPipeline: @unchecked Sendable {
private struct RuntimeDiagnostics {
var submittedFrameCount = 0
- var consumedFrameCount = 0
+ var forwardedAV1FrameCount = 0
+ }
+
+ private struct CodecOutputState: Sendable, Equatable {
+ var sourceWidth: Int32
+ var sourceHeight: Int32
+ var outputWidth: Int32
+ var outputHeight: Int32
+ var framesPerSecond: Int
}
private static let sslInitialized: Void = {
@@ -444,18 +573,19 @@ package final class WebRTCMediaPipeline: @unchecked Sendable {
}()
private let factory: RTCPeerConnectionFactory
- nonisolated(unsafe) package let videoSource: RTCVideoSource
- nonisolated(unsafe) package let videoTrack: RTCVideoTrack
- nonisolated(unsafe) private let capturer: RTCVideoCapturer
+ nonisolated(unsafe) private let av1VideoSource: RTCVideoSource
+ nonisolated(unsafe) package let av1VideoTrack: RTCVideoTrack
+ nonisolated(unsafe) private let av1Capturer: RTCVideoCapturer
private let queue = DispatchQueue(
label: "com.developerchen.voiddisplay.webrtc.media",
qos: .userInitiated
)
private let profile = Mutex(WebRTCStreamingProfile(performanceMode: .automatic))
+ private let activeCodecs = Mutex>([])
private let runtimeDiagnostics = Mutex(RuntimeDiagnostics())
nonisolated(unsafe) private var frameMailbox: WebRTCFrameMailbox!
- nonisolated(unsafe) private var lastFormat: (width: Int32, height: Int32, framesPerSecond: Int)?
- nonisolated(unsafe) private var timestampSequencer = WebRTCFrameTimestampSequencer()
+ nonisolated(unsafe) private var av1OutputState: CodecOutputState?
+ nonisolated(unsafe) private var av1TimestampSequencer = WebRTCFrameTimestampSequencer()
nonisolated package init() {
_ = Self.sslInitialized
@@ -463,9 +593,9 @@ package final class WebRTCMediaPipeline: @unchecked Sendable {
encoderFactory: RTCDefaultVideoEncoderFactory(),
decoderFactory: RTCDefaultVideoDecoderFactory()
)
- self.videoSource = factory.videoSource()
- self.videoTrack = factory.videoTrack(with: videoSource, trackId: "screen-video-track")
- self.capturer = RTCVideoCapturer(delegate: videoSource)
+ self.av1VideoSource = factory.videoSource()
+ self.av1VideoTrack = factory.videoTrack(with: av1VideoSource, trackId: "screen-video-av1")
+ self.av1Capturer = RTCVideoCapturer(delegate: av1VideoSource)
self.frameMailbox = WebRTCFrameMailbox(
scheduler: { [weak self] operation in
self?.queue.async(execute: operation)
@@ -488,9 +618,9 @@ package final class WebRTCMediaPipeline: @unchecked Sendable {
return factory.peerConnection(with: configuration, constraints: constraints, delegate: nil)
}
- nonisolated package func requiredH264Codecs() -> [RTCRtpCodecCapability]? {
+ nonisolated package func requiredCodecs(for codec: WebRTCVideoCodec) -> [RTCRtpCodecCapability]? {
let capabilities = factory.rtpSenderCapabilities(forKind: kRTCMediaStreamTrackKindVideo)
- return WebRTCCodecPreference.requiredH264Codecs(from: capabilities.codecs)
+ return WebRTCCodecPreference.requiredCodecs(for: codec, from: capabilities.codecs)
}
nonisolated package func senderVideoCodecCapabilitySummary() -> String {
@@ -498,23 +628,34 @@ package final class WebRTCMediaPipeline: @unchecked Sendable {
return WebRTCCodecPreference.capabilitySummary(from: capabilities.codecs)
}
+ nonisolated package func updateActiveCodecs(_ codecs: Set) {
+ activeCodecs.withLock { $0 = codecs }
+ queue.async { [weak self] in
+ guard let self,
+ let currentState = self.av1OutputState else {
+ return
+ }
+ self.adaptOutputFormats(
+ sourceWidth: currentState.sourceWidth,
+ sourceHeight: currentState.sourceHeight,
+ profile: self.profile.withLock { $0 },
+ activeCodecs: codecs
+ )
+ }
+ }
+
nonisolated package func updateEncodingProfile(_ profile: WebRTCStreamingProfile) {
self.profile.withLock { $0 = profile }
queue.async { [weak self] in
- guard let self, let lastFormat = self.lastFormat else { return }
- let dimensions = profile.outputDimensions(
- forWidth: lastFormat.width,
- height: lastFormat.height
- )
- self.videoSource.adaptOutputFormat(
- toWidth: dimensions.width,
- height: dimensions.height,
- fps: Int32(profile.framesPerSecond)
- )
- self.lastFormat = (
- width: lastFormat.width,
- height: lastFormat.height,
- framesPerSecond: profile.framesPerSecond
+ guard let self,
+ let currentState = self.av1OutputState else {
+ return
+ }
+ self.adaptOutputFormats(
+ sourceWidth: currentState.sourceWidth,
+ sourceHeight: currentState.sourceHeight,
+ profile: profile,
+ activeCodecs: self.activeCodecs.withLock { $0 }
)
}
}
@@ -537,38 +678,133 @@ package final class WebRTCMediaPipeline: @unchecked Sendable {
let width = Int32(CVPixelBufferGetWidth(pixelBuffer))
let height = Int32(CVPixelBufferGetHeight(pixelBuffer))
let currentProfile = profile.withLock { $0 }
- let outputDimensions = currentProfile.outputDimensions(forWidth: width, height: height)
- if lastFormat?.width != width ||
- lastFormat?.height != height ||
- lastFormat?.framesPerSecond != currentProfile.framesPerSecond {
- videoSource.adaptOutputFormat(
- toWidth: outputDimensions.width,
- height: outputDimensions.height,
- fps: Int32(currentProfile.framesPerSecond)
+ let currentActiveCodecs = activeCodecs.withLock { $0 }
+ adaptOutputFormats(
+ sourceWidth: width,
+ sourceHeight: height,
+ profile: currentProfile,
+ activeCodecs: currentActiveCodecs
+ )
+
+ for codec in WebRTCVideoCodec.allCases where currentActiveCodecs.contains(codec) {
+ forwardFrame(
+ codec: codec,
+ pixelBuffer: pixelBuffer,
+ ptsUs: pendingFrame.ptsUs,
+ sourceWidth: width,
+ sourceHeight: height,
+ profile: currentProfile
)
- lastFormat = (width, height, currentProfile.framesPerSecond)
}
+ }
- let rtcBuffer = RTCCVPixelBuffer(pixelBuffer: pixelBuffer)
- let timestampNs = timestampSequencer.nextTimestampNs(
- ptsUs: pendingFrame.ptsUs,
- framesPerSecond: currentProfile.framesPerSecond
+ private func adaptOutputFormats(
+ sourceWidth: Int32,
+ sourceHeight: Int32,
+ profile: WebRTCStreamingProfile,
+ activeCodecs: Set
+ ) {
+ for codec in WebRTCVideoCodec.allCases where activeCodecs.contains(codec) {
+ adaptOutputFormat(for: codec, sourceWidth: sourceWidth, sourceHeight: sourceHeight, profile: profile)
+ }
+ }
+
+ private func adaptOutputFormat(
+ for codec: WebRTCVideoCodec,
+ sourceWidth: Int32,
+ sourceHeight: Int32,
+ profile: WebRTCStreamingProfile
+ ) {
+ let outputDimensions = profile.outputDimensions(
+ for: codec,
+ width: sourceWidth,
+ height: sourceHeight
)
- let frame = RTCVideoFrame(
- buffer: rtcBuffer,
- rotation: ._0,
- timeStampNs: timestampNs
+ let nextState = CodecOutputState(
+ sourceWidth: sourceWidth,
+ sourceHeight: sourceHeight,
+ outputWidth: outputDimensions.width,
+ outputHeight: outputDimensions.height,
+ framesPerSecond: profile.framesPerSecond(for: codec)
)
- let shouldLogFirstForwardedFrame = runtimeDiagnostics.withLock { diagnostics -> Bool in
- diagnostics.consumedFrameCount += 1
- return diagnostics.consumedFrameCount == 1
+ switch codec {
+ case .av1 where av1OutputState == nextState:
+ return
+ default:
+ break
}
- if shouldLogFirstForwardedFrame {
+ videoSource(for: codec).adaptOutputFormat(
+ toWidth: outputDimensions.width,
+ height: outputDimensions.height,
+ fps: Int32(profile.framesPerSecond(for: codec))
+ )
+ switch codec {
+ case .av1:
+ av1OutputState = nextState
+ }
+ }
+
+ private func forwardFrame(
+ codec: WebRTCVideoCodec,
+ pixelBuffer: CVPixelBuffer,
+ ptsUs: UInt64,
+ sourceWidth: Int32,
+ sourceHeight: Int32,
+ profile: WebRTCStreamingProfile
+ ) {
+ let rtcBuffer = RTCCVPixelBuffer(pixelBuffer: pixelBuffer)
+ let framesPerSecond = profile.framesPerSecond(for: codec)
+ let timestampNs = nextTimestampNs(for: codec, ptsUs: ptsUs, framesPerSecond: framesPerSecond)
+ let frame = RTCVideoFrame(buffer: rtcBuffer, rotation: ._0, timeStampNs: timestampNs)
+ let outputDimensions = profile.outputDimensions(
+ for: codec,
+ width: sourceWidth,
+ height: sourceHeight
+ )
+ if shouldLogFirstForwardedFrame(for: codec) {
AppLog.web.info(
- "WebRTC media pipeline forwarded first RTC frame input=\(width, privacy: .public)x\(height, privacy: .public) output=\(outputDimensions.width, privacy: .public)x\(outputDimensions.height, privacy: .public) fps=\(currentProfile.framesPerSecond, privacy: .public) ptsUs=\(pendingFrame.ptsUs, privacy: .public) timestampNs=\(timestampNs, privacy: .public)."
+ "WebRTC media pipeline forwarded first \(codec.logName, privacy: .public) RTC frame input=\(sourceWidth, privacy: .public)x\(sourceHeight, privacy: .public) output=\(outputDimensions.width, privacy: .public)x\(outputDimensions.height, privacy: .public) fps=\(framesPerSecond, privacy: .public) ptsUs=\(ptsUs, privacy: .public) timestampNs=\(timestampNs, privacy: .public)."
)
}
- videoSource.capturer(capturer, didCapture: frame)
+ videoSource(for: codec).capturer(capturer(for: codec), didCapture: frame)
+ }
+
+ private func nextTimestampNs(
+ for codec: WebRTCVideoCodec,
+ ptsUs: UInt64,
+ framesPerSecond: Int
+ ) -> Int64 {
+ switch codec {
+ case .av1:
+ av1TimestampSequencer.nextTimestampNs(
+ ptsUs: ptsUs,
+ framesPerSecond: framesPerSecond
+ )
+ }
+ }
+
+ private func shouldLogFirstForwardedFrame(for codec: WebRTCVideoCodec) -> Bool {
+ runtimeDiagnostics.withLock { diagnostics -> Bool in
+ switch codec {
+ case .av1:
+ diagnostics.forwardedAV1FrameCount += 1
+ return diagnostics.forwardedAV1FrameCount == 1
+ }
+ }
+ }
+
+ private func videoSource(for codec: WebRTCVideoCodec) -> RTCVideoSource {
+ switch codec {
+ case .av1:
+ av1VideoSource
+ }
+ }
+
+ private func capturer(for codec: WebRTCVideoCodec) -> RTCVideoCapturer {
+ switch codec {
+ case .av1:
+ av1Capturer
+ }
}
}
diff --git a/Tests/VoidDisplayAppTests/TestSupport/TestSignalSessionHub.swift b/Tests/VoidDisplayAppTests/TestSupport/TestSignalSessionHub.swift
index e64c8ed..5e8787f 100644
--- a/Tests/VoidDisplayAppTests/TestSupport/TestSignalSessionHub.swift
+++ b/Tests/VoidDisplayAppTests/TestSupport/TestSignalSessionHub.swift
@@ -83,6 +83,8 @@ final class TestSignalSessionHub: SignalSessionHub, @unchecked Sendable {
}
}
+ nonisolated func updateSourceVideoSpec(_: SourceVideoSpec) {}
+
nonisolated func updatePerformanceMode(_: CapturePerformanceMode) {}
nonisolated func receiveSignalText(_ text: String, from connection: any SignalSocketConnection) {
diff --git a/Tests/VoidDisplayCaptureTests/Services/DisplayCaptureProfileStateMachineTests.swift b/Tests/VoidDisplayCaptureTests/Services/DisplayCaptureProfileStateMachineTests.swift
index 7acbbc5..7ac55b9 100644
--- a/Tests/VoidDisplayCaptureTests/Services/DisplayCaptureProfileStateMachineTests.swift
+++ b/Tests/VoidDisplayCaptureTests/Services/DisplayCaptureProfileStateMachineTests.swift
@@ -199,8 +199,8 @@ struct DisplayCaptureProfileStateMachineTests {
}
}
- @Test func previewFrameRateClampCapsHighRefreshAndPreservesLowerRefresh() {
- #expect(DisplayCaptureSession.clampedPreviewFramesPerSecond(for: 144) == 60)
+ @Test func previewFrameRateKeepsHighRefreshAndPreservesFallback() {
+ #expect(DisplayCaptureSession.clampedPreviewFramesPerSecond(for: 144) == 144)
#expect(DisplayCaptureSession.clampedPreviewFramesPerSecond(for: 50) == 50)
#expect(DisplayCaptureSession.clampedPreviewFramesPerSecond(for: 0) == 60)
}
@@ -260,6 +260,27 @@ struct DisplayCaptureProfileStateMachineTests {
performanceMode: .powerEfficient
) == .fps30
)
+ #expect(
+ DisplayCaptureConfigurationStateMachine.defaultFrameRateTier(
+ for: .shareOnly,
+ performanceMode: .automatic,
+ sourceFramesPerSecond: 90
+ ).framesPerSecond == 90
+ )
+ #expect(
+ DisplayCaptureConfigurationStateMachine.defaultFrameRateTier(
+ for: .mixed,
+ performanceMode: .smooth,
+ sourceFramesPerSecond: 120
+ ).framesPerSecond == 120
+ )
+ #expect(
+ DisplayCaptureConfigurationStateMachine.defaultFrameRateTier(
+ for: .mixed,
+ performanceMode: .powerEfficient,
+ sourceFramesPerSecond: 120
+ ) == .fps30
+ )
}
@Test func captureSizeContextAppliesSharedPixelBudgetByPerformanceMode() {
@@ -269,8 +290,8 @@ struct DisplayCaptureProfileStateMachineTests {
)
#expect(context.captureSize(for: .previewOnly, performanceMode: .automatic) == DisplayCaptureDimensions(width: 3_840, height: 2_160))
- #expect(context.captureSize(for: .shareOnly, performanceMode: .automatic) == DisplayCaptureDimensions(width: 2_560, height: 1_440))
- #expect(context.captureSize(for: .mixed, performanceMode: .automatic) == DisplayCaptureDimensions(width: 2_560, height: 1_440))
+ #expect(context.captureSize(for: .shareOnly, performanceMode: .automatic) == DisplayCaptureDimensions(width: 3_840, height: 2_160))
+ #expect(context.captureSize(for: .mixed, performanceMode: .automatic) == DisplayCaptureDimensions(width: 3_840, height: 2_160))
#expect(context.captureSize(for: .shareOnly, performanceMode: .powerEfficient) == DisplayCaptureDimensions(width: 1_920, height: 1_080))
#expect(context.captureSize(for: .shareOnly, performanceMode: .smooth) == DisplayCaptureDimensions(width: 3_840, height: 2_160))
}
@@ -294,8 +315,8 @@ struct DisplayCaptureProfileStateMachineTests {
physicalSize: DisplayCaptureDimensions(width: 2_161, height: 3_841)
)
- #expect(wideContext.captureSize(for: .shareOnly, performanceMode: .automatic) == DisplayCaptureDimensions(width: 2_968, height: 1_242))
- #expect(portraitContext.captureSize(for: .shareOnly, performanceMode: .automatic) == DisplayCaptureDimensions(width: 1_440, height: 2_560))
+ #expect(wideContext.captureSize(for: .shareOnly, performanceMode: .automatic) == DisplayCaptureDimensions(width: 3_440, height: 1_440))
+ #expect(portraitContext.captureSize(for: .shareOnly, performanceMode: .automatic) == DisplayCaptureDimensions(width: 2_161, height: 3_841))
}
@Test func automaticMixedModeKeeps60AcrossPreviewPressureWindows() {
diff --git a/Tests/VoidDisplayCaptureTests/TestSupport/CaptureTestSupport.swift b/Tests/VoidDisplayCaptureTests/TestSupport/CaptureTestSupport.swift
index b7ce79a..fba8165 100644
--- a/Tests/VoidDisplayCaptureTests/TestSupport/CaptureTestSupport.swift
+++ b/Tests/VoidDisplayCaptureTests/TestSupport/CaptureTestSupport.swift
@@ -35,6 +35,8 @@ final class TestDisplayShareFrameConsumer: DisplayShareFrameConsumer {
nonisolated var hasDemand: Bool { false }
+ nonisolated func updateSourceVideoSpec(_: SourceVideoSpec) {}
+
nonisolated func updatePerformanceMode(_ mode: CapturePerformanceMode) {
performanceModes.withLock { $0.append(mode) }
}
diff --git a/Tests/VoidDisplayFoundationTests/SharedCapturePerformanceBudgetTests.swift b/Tests/VoidDisplayFoundationTests/SharedCapturePerformanceBudgetTests.swift
index 1dc280a..1055bcc 100644
--- a/Tests/VoidDisplayFoundationTests/SharedCapturePerformanceBudgetTests.swift
+++ b/Tests/VoidDisplayFoundationTests/SharedCapturePerformanceBudgetTests.swift
@@ -5,7 +5,7 @@ struct SharedCapturePerformanceBudgetTests {
@Test func performanceModesMapToExpectedPixelBudgets() {
#expect(SharedCapturePerformanceBudget(performanceMode: .automatic) == SharedCapturePerformanceBudget(
framesPerSecond: 60,
- pixelBudgetPerSecond: 221_184_000
+ pixelBudgetPerSecond: nil
))
#expect(SharedCapturePerformanceBudget(performanceMode: .smooth) == SharedCapturePerformanceBudget(
framesPerSecond: 60,
@@ -18,27 +18,27 @@ struct SharedCapturePerformanceBudgetTests {
}
@Test func pixelBudgetComputesEvenDimensionsWithoutUpscaling() {
- let automaticBudget = SharedCapturePerformanceBudget(performanceMode: .automatic)
+ let powerEfficientBudget = SharedCapturePerformanceBudget(performanceMode: .powerEfficient)
#expect(
- automaticBudget.captureDimensions(
+ powerEfficientBudget.captureDimensions(
for: CapturePixelDimensions(width: 3_840, height: 2_160)
- ) == CapturePixelDimensions(width: 2_560, height: 1_440)
+ ) == CapturePixelDimensions(width: 1_920, height: 1_080)
)
#expect(
- automaticBudget.captureDimensions(
+ powerEfficientBudget.captureDimensions(
for: CapturePixelDimensions(width: 1_920, height: 1_080)
) == CapturePixelDimensions(width: 1_920, height: 1_080)
)
#expect(
- automaticBudget.captureDimensions(
+ powerEfficientBudget.captureDimensions(
for: CapturePixelDimensions(width: 3_440, height: 1_440)
- ) == CapturePixelDimensions(width: 2_968, height: 1_242)
+ ) == CapturePixelDimensions(width: 2_224, height: 932)
)
#expect(
- automaticBudget.captureDimensions(
+ powerEfficientBudget.captureDimensions(
for: CapturePixelDimensions(width: 2_161, height: 3_841)
- ) == CapturePixelDimensions(width: 1_440, height: 2_560)
+ ) == CapturePixelDimensions(width: 1_080, height: 1_920)
)
}
diff --git a/Tests/VoidDisplaySharingTests/Integration/WebServerSocketIntegrationTests.swift b/Tests/VoidDisplaySharingTests/Integration/WebServerSocketIntegrationTests.swift
index 42c555d..b733782 100644
--- a/Tests/VoidDisplaySharingTests/Integration/WebServerSocketIntegrationTests.swift
+++ b/Tests/VoidDisplaySharingTests/Integration/WebServerSocketIntegrationTests.swift
@@ -121,23 +121,46 @@ struct WebServerSocketIntegrationTests {
#expect(responseText.contains(#"function localOfferSDPWithIceCredentials() {"#))
#expect(responseText.contains(#"async function waitForLocalOfferSDP() {"#))
#expect(responseText.contains(#"RTCRtpReceiver.getCapabilities("video")"#))
- #expect(responseText.contains(#"transceiver.setCodecPreferences(h264Preferences);"#))
- #expect(responseText.contains(#"assertH264AnswerSDP(payload.sdp);"#))
- #expect(responseText.contains(#"await verifySelectedH264Codec();"#))
+ #expect(responseText.contains(#"function receiverCodecPreferences() {"#))
+ #expect(responseText.contains(#"function receiverVideoCodecCapabilitySummary() {"#))
+ #expect(responseText.contains(#""unsupportedVideoCodecCount=" + String(allCodecs.length - probedCodecCount)"#))
+ #expect(responseText.contains(#"WebRTC receiver video capabilities "#))
+ #expect(!responseText.contains(#"function initialForceH264Only() {"#))
+ #expect(!responseText.contains(#"forceH264Only"#))
+ #expect(responseText.contains(#"return normalizedVideoCodecName(codec) === "video/av1";"#))
+ #expect(responseText.contains(#"function isRetransmissionCodec(codec) {"#))
+ #expect(!responseText.contains(#"function receiverSupportsCodec(mimeType) {"#))
+ #expect(responseText.contains(#"function matchingRtxCodecs(allCodecs, primaryCodecs) {"#))
+ #expect(responseText.contains(#"return supportedCodecs.concat(matchingRtxCodecs(allCodecs, supportedCodecs));"#))
+ #expect(responseText.contains(#"transceiver.setCodecPreferences(codecPreferences);"#))
+ #expect(responseText.contains(#"const supportedPrimaryCodecs = [...new Set("#))
+ #expect(responseText.contains(#"negotiatedVideoCodec = selectedCodecFromAnswerSDP(payload.sdp);"#))
+ #expect(responseText.contains(#"await verifySelectedCodec(negotiatedVideoCodec || "av1");"#))
+ #expect(responseText.contains(#"function isCodecErrorReason(reason) {"#))
+ #expect(!responseText.contains(#"function shouldRetryWithH264Fallback() {"#))
+ #expect(!responseText.contains(#"function retryWithH264Fallback() {"#))
+ #expect(responseText.contains(#"reason === "unsupported_video_codec_offered" || reason === "supported_video_codec_missing""#))
+ #expect(responseText.contains(#"if (codecMimeType === "video/rtx") {"#))
#expect(responseText.contains(#"peer.addTransceiver("video", { direction: "recvonly" });"#))
#expect(responseText.contains(#"sdp: await waitForLocalOfferSDP()"#))
#expect(!responseText.contains(#"sdp: offer.sdp"#))
#expect(responseText.contains(#"const reconnectDelays = [250, 500, 1000, 2000, 4000];"#))
- #expect(responseText.contains(#"function scheduleReconnect() {"#))
+ #expect(responseText.contains(#"function scheduleReconnect(overlayTitle = t("overlayReconnectTitle"), overlayBody = t("overlayReconnectBody")) {"#))
+ #expect(responseText.contains(#"function schedulePeerRetry(overlayTitle, overlayBody) {"#))
#expect(responseText.contains(#"peer = new RTCPeerConnection({ iceServers: bootstrap.iceServers ?? [] });"#))
- #expect(responseText.contains(#"setOverlay(t("overlayReconnectTitle"), t("overlayReconnectBody"), true);"#))
+ #expect(responseText.contains(#"setOverlay(overlayTitle, overlayBody, true);"#))
#expect(responseText.contains(#"setOverlay(t("overlaySharingStoppedTitle"), t("overlaySharingStoppedBody"), true);"#))
#expect(responseText.contains(#"case "stopped":"#))
+ #expect(responseText.contains(#"case "codec_pending":"#))
+ #expect(responseText.contains(#"schedulePeerRetry(t("overlayCodecPendingTitle"), t("overlayCodecPendingBody"));"#))
#expect(responseText.contains(#"case "error":"#))
#expect(responseText.contains(#"connect();"#))
#expect(responseText.contains(#"heroEyebrow: "VOIDDISPLAY 实时画面""#))
- #expect(responseText.contains(#"overlayH264RequiredTitle: "H.264 required""#))
- #expect(responseText.contains(#"overlayH264RequiredTitle: "需要 H.264""#))
+ #expect(responseText.contains(#"overlayCodecRequiredTitle: "AV1 required""#))
+ #expect(responseText.contains(#"overlayCodecRequiredTitle: "需要 AV1""#))
+ #expect(responseText.contains(#"overlayCodecPendingTitle: "Preparing video codec""#))
+ #expect(responseText.contains(#"overlayCodecPendingTitle: "正在准备视频编码""#))
+ #expect(!responseText.contains(#"overlayCodecFallbackTitle"#))
#expect(responseText.contains(#"fullscreenEnter: "全屏""#))
#expect(responseText.contains(#"pageTitle: "Screen Share""#))
#expect(responseText.contains("hero-eyebrow"))
@@ -202,6 +225,15 @@ struct WebServerSocketIntegrationTests {
#expect(smokeResult.documentTitle == "Screen Share")
#expect(smokeResult.scaleButtonText == "Fit")
#expect(smokeResult.toggleCallCount >= 2)
+ #expect(smokeResult.codecPreferenceMimeTypes == ["video/AV1", "video/rtx"])
+ #expect(smokeResult.codecPreferenceFmtpLines == ["", "apt=98"])
+ #expect(smokeResult.receiverCapabilitySummary.contains("AV1=video/av1(pt=98,fmtp=none)"))
+ #expect(smokeResult.receiverCapabilitySummary.contains("unsupportedVideoCodecCount=7"))
+ #expect(smokeResult.selectedCodec == "av1")
+ #expect(smokeResult.rejectsUnexpectedCodec)
+ #expect(smokeResult.marksMissingCodecAsRequirementError)
+ #expect(smokeResult.marksMissingCapabilitiesAsRequirementError)
+ #expect(smokeResult.belowSourceStatus.contains("source 2560×1440 60fps"))
}
@Test func oversizedIncompleteSignalFrameClosesConnection() async throws {
@@ -832,6 +864,14 @@ private struct DisplayPageScriptSmokeResult: Equatable {
let documentTitle: String
let scaleButtonText: String
let toggleCallCount: Int
+ let codecPreferenceMimeTypes: [String]
+ let codecPreferenceFmtpLines: [String]
+ let receiverCapabilitySummary: String
+ let selectedCodec: String
+ let rejectsUnexpectedCodec: Bool
+ let marksMissingCodecAsRequirementError: Bool
+ let marksMissingCapabilitiesAsRequirementError: Bool
+ let belowSourceStatus: String
}
@MainActor
@@ -946,7 +986,8 @@ private func evaluateDisplayPageRuntimeScript(
RTCPeerConnection: RTCPeerConnection,
location: {
protocol: "http:",
- host: "127.0.0.1"
+ host: "127.0.0.1",
+ search: ""
},
setTimeout: function() { return 1; },
clearTimeout: function() {},
@@ -955,6 +996,7 @@ private func evaluateDisplayPageRuntimeScript(
var console = {
log: function() {},
+ info: function() {},
warn: function() {},
error: function() {}
};
@@ -973,6 +1015,81 @@ private func evaluateDisplayPageRuntimeScript(
"""
transition("streaming");
__elements["scale-mode-btn"].__handlers["click"]();
+ function RTCRtpReceiver() {}
+ RTCRtpReceiver.getCapabilities = function() {
+ return {
+ codecs: [
+ { mimeType: "video/VP8", payloadType: 96 },
+ { mimeType: "video/AV1", payloadType: 98 },
+ { mimeType: "video/H264", payloadType: 102 },
+ { mimeType: "video/H265", payloadType: 116, sdpFmtpLine: "profile-id=1;tx-mode=SRST" },
+ { mimeType: "video/rtx", sdpFmtpLine: "apt=96" },
+ { mimeType: "video/rtx", sdpFmtpLine: "apt=98" },
+ { mimeType: "video/rtx", sdpFmtpLine: "apt=102" },
+ { mimeType: "video/rtx", sdpFmtpLine: "apt=116" }
+ ]
+ };
+ };
+ window.RTCRtpReceiver = RTCRtpReceiver;
+ var __codecPreferenceMimeTypes = receiverCodecPreferences().map(function(codec) {
+ return codec.mimeType;
+ });
+ var __codecPreferenceFmtpLines = receiverCodecPreferences().map(function(codec) {
+ return codec.sdpFmtpLine || "";
+ });
+ var __receiverCapabilitySummary = receiverVideoCodecCapabilitySummary();
+ var __selectedCodec = selectedCodecFromAnswerSDP(
+ "v=0\\r\\n" +
+ "m=video 9 UDP/TLS/RTP/SAVPF 98 99\\r\\n" +
+ "a=rtpmap:98 AV1/90000\\r\\n" +
+ "a=rtpmap:99 rtx/90000\\r\\n"
+ );
+ var __rejectsUnexpectedCodec = false;
+ try {
+ selectedCodecFromAnswerSDP(
+ "v=0\\r\\n" +
+ "m=video 9 UDP/TLS/RTP/SAVPF 96\\r\\n" +
+ "a=rtpmap:96 VP8/90000\\r\\n" +
+ "m=video 9 UDP/TLS/RTP/SAVPF 102\\r\\n" +
+ "a=rtpmap:102 H264/90000\\r\\n"
+ );
+ } catch (error) {
+ __rejectsUnexpectedCodec = true;
+ }
+ RTCRtpReceiver.getCapabilities = function() {
+ return {
+ codecs: [
+ { mimeType: "video/VP8" }
+ ]
+ };
+ };
+ var __marksMissingCodecAsRequirementError = false;
+ try {
+ receiverCodecPreferences();
+ } catch (error) {
+ __marksMissingCodecAsRequirementError = isCodecRequirementError(error);
+ }
+ delete window.RTCRtpReceiver;
+ var __marksMissingCapabilitiesAsRequirementError = false;
+ try {
+ receiverCodecPreferences();
+ } catch (error) {
+ __marksMissingCapabilitiesAsRequirementError = isCodecRequirementError(error);
+ }
+ expectedSourceVideoSpec = sourceSpecFromSignal({ width: 2560, height: 1440, framesPerSecond: 60 });
+ logBrowserVideoStats(
+ {
+ frameWidth: 1280,
+ frameHeight: 720,
+ framesPerSecond: 30,
+ framesDropped: 0,
+ jitter: 0,
+ packetsLost: 0
+ },
+ { mimeType: "video/AV1" },
+ { bitrateBps: 4000000 }
+ );
+ var __belowSourceStatus = __elements["status"].textContent;
"""
)
if let exceptionMessage {
@@ -982,10 +1099,32 @@ private func evaluateDisplayPageRuntimeScript(
let documentTitle = context.evaluateScript("document.title")?.toString() ?? ""
let scaleButtonText = context.evaluateScript("__elements['scale-mode-btn'].textContent")?.toString() ?? ""
let toggleCallCount = Int(context.evaluateScript("__toggleCount")?.toInt32() ?? 0)
+ let codecPreferenceMimeTypesJSON = context.evaluateScript("JSON.stringify(__codecPreferenceMimeTypes)")?.toString() ?? "[]"
+ let codecPreferenceMimeTypesData = Data(codecPreferenceMimeTypesJSON.utf8)
+ let codecPreferenceMimeTypes = (try? JSONDecoder().decode([String].self, from: codecPreferenceMimeTypesData)) ?? []
+ let codecPreferenceFmtpLinesJSON = context.evaluateScript("JSON.stringify(__codecPreferenceFmtpLines)")?.toString() ?? "[]"
+ let codecPreferenceFmtpLinesData = Data(codecPreferenceFmtpLinesJSON.utf8)
+ let codecPreferenceFmtpLines = (try? JSONDecoder().decode([String].self, from: codecPreferenceFmtpLinesData)) ?? []
+ let receiverCapabilitySummary = context.evaluateScript("__receiverCapabilitySummary")?.toString() ?? ""
+ let selectedCodec = context.evaluateScript("__selectedCodec")?.toString() ?? ""
+ let rejectsUnexpectedCodec = context.evaluateScript("__rejectsUnexpectedCodec")?.toBool() ?? false
+ let marksMissingCodecAsRequirementError =
+ context.evaluateScript("__marksMissingCodecAsRequirementError")?.toBool() ?? false
+ let marksMissingCapabilitiesAsRequirementError =
+ context.evaluateScript("__marksMissingCapabilitiesAsRequirementError")?.toBool() ?? false
+ let belowSourceStatus = context.evaluateScript("__belowSourceStatus")?.toString() ?? ""
return DisplayPageScriptSmokeResult(
documentTitle: documentTitle,
scaleButtonText: scaleButtonText,
- toggleCallCount: toggleCallCount
+ toggleCallCount: toggleCallCount,
+ codecPreferenceMimeTypes: codecPreferenceMimeTypes,
+ codecPreferenceFmtpLines: codecPreferenceFmtpLines,
+ receiverCapabilitySummary: receiverCapabilitySummary,
+ selectedCodec: selectedCodec,
+ rejectsUnexpectedCodec: rejectsUnexpectedCodec,
+ marksMissingCodecAsRequirementError: marksMissingCodecAsRequirementError,
+ marksMissingCapabilitiesAsRequirementError: marksMissingCapabilitiesAsRequirementError,
+ belowSourceStatus: belowSourceStatus
)
}
diff --git a/Tests/VoidDisplaySharingTests/TestSupport/TestSignalSessionHub.swift b/Tests/VoidDisplaySharingTests/TestSupport/TestSignalSessionHub.swift
index e64c8ed..5e8787f 100644
--- a/Tests/VoidDisplaySharingTests/TestSupport/TestSignalSessionHub.swift
+++ b/Tests/VoidDisplaySharingTests/TestSupport/TestSignalSessionHub.swift
@@ -83,6 +83,8 @@ final class TestSignalSessionHub: SignalSessionHub, @unchecked Sendable {
}
}
+ nonisolated func updateSourceVideoSpec(_: SourceVideoSpec) {}
+
nonisolated func updatePerformanceMode(_: CapturePerformanceMode) {}
nonisolated func receiveSignalText(_ text: String, from connection: any SignalSocketConnection) {
diff --git a/Tests/VoidDisplaySharingTests/Web/RelayHTTPClientTests.swift b/Tests/VoidDisplaySharingTests/Web/RelayHTTPClientTests.swift
index 9c8e335..2f9fd76 100644
--- a/Tests/VoidDisplaySharingTests/Web/RelayHTTPClientTests.swift
+++ b/Tests/VoidDisplaySharingTests/Web/RelayHTTPClientTests.swift
@@ -60,6 +60,53 @@ struct RelayHTTPClientTests {
#expect(requests.map(\.method) == ["POST", "DELETE"])
#expect(requests.first?.body.contains(#""candidate":"candidate:1""#) == true)
}
+
+ @Test func viewerOfferExposesRelayErrorReason() async throws {
+ let transport = RelayHTTPClientMockTransport(
+ responses: [
+ MockHTTPResponse(
+ statusCode: 400,
+ body: #"{"type":"error","reason":"publisher_codec_pending"}"#
+ )
+ ]
+ )
+ let client = RelayHTTPClient(
+ baseURL: URL(string: "http://relay.test")!,
+ controlToken: "token",
+ session: transport.session
+ )
+
+ do {
+ _ = try await client.viewerOffer(roomID: "2", clientID: "viewer-1", sdp: "viewer-offer")
+ Issue.record("viewerOffer succeeded for relay error response")
+ } catch let error as RelayHTTPError {
+ #expect(error == .httpStatus(400, "publisher_codec_pending"))
+ #expect(error.relayReason == "publisher_codec_pending")
+ }
+ }
+
+ @Test func viewerOfferDecodesSelectedCodec() async throws {
+ let transport = RelayHTTPClientMockTransport(
+ responses: [
+ MockHTTPResponse(
+ statusCode: 200,
+ body: #"{"type":"answer","sdp":"viewer-answer","codec":"av1"}"#
+ )
+ ]
+ )
+ let client = RelayHTTPClient(
+ baseURL: URL(string: "http://relay.test")!,
+ controlToken: "token",
+ session: transport.session
+ )
+
+ let response = try await client.viewerOffer(roomID: "2", clientID: "viewer-1", sdp: "viewer-offer")
+
+ #expect(response == RelayViewerOfferResponse(sdp: "viewer-answer", codec: .av1))
+ let request = try #require(transport.requests().first)
+ #expect(request.path == "/room/2/viewer/viewer-1")
+ #expect(request.body.contains(#""sdp":"viewer-offer""#))
+ }
}
private struct MockHTTPResponse: Sendable {
diff --git a/Tests/VoidDisplaySharingTests/Web/RelaySessionHubTests.swift b/Tests/VoidDisplaySharingTests/Web/RelaySessionHubTests.swift
index 1cf1841..d9cae82 100644
--- a/Tests/VoidDisplaySharingTests/Web/RelaySessionHubTests.swift
+++ b/Tests/VoidDisplaySharingTests/Web/RelaySessionHubTests.swift
@@ -53,11 +53,11 @@ private final class FakeRelayClient: RelayHTTPClienting, @unchecked Sendable {
}
private let state = Mutex(State())
- private let onViewerOffer: (@Sendable () async -> Void)?
+ private let onViewerOffer: (@Sendable () async throws -> Void)?
private let onViewerCandidate: (@Sendable () async -> Void)?
init(
- onViewerOffer: (@Sendable () async -> Void)? = nil,
+ onViewerOffer: (@Sendable () async throws -> Void)? = nil,
onViewerCandidate: (@Sendable () async -> Void)? = nil
) {
self.onViewerOffer = onViewerOffer
@@ -79,10 +79,10 @@ private final class FakeRelayClient: RelayHTTPClienting, @unchecked Sendable {
state.withLock { $0.publisherCandidates.append((roomID, publisherID, candidate)) }
}
- nonisolated func viewerOffer(roomID: String, clientID: String, sdp: String) async throws -> String {
+ nonisolated func viewerOffer(roomID: String, clientID: String, sdp: String) async throws -> RelayViewerOfferResponse {
state.withLock { $0.viewerOffers.append((roomID, clientID, sdp)) }
- await onViewerOffer?()
- return "relay-viewer-answer-\(clientID)"
+ try await onViewerOffer?()
+ return RelayViewerOfferResponse(sdp: "relay-viewer-answer-\(clientID)", codec: .av1)
}
nonisolated func viewerCandidate(
@@ -126,6 +126,7 @@ private final class FakePublisherSession: RelayPublisherSessioning, @unchecked S
var startCallCount = 0
var closeCallCount = 0
var profiles: [WebRTCStreamingProfile] = []
+ var activeCodecs: [Set] = []
}
private let state = Mutex(State())
@@ -150,6 +151,10 @@ private final class FakePublisherSession: RelayPublisherSessioning, @unchecked S
state.withLock { $0.profiles.append(profile) }
}
+ nonisolated func updateActiveCodecs(_ activeCodecs: Set) {
+ state.withLock { $0.activeCodecs.append(activeCodecs) }
+ }
+
nonisolated func submitFrame(pixelBuffer _: CVPixelBuffer, ptsUs _: UInt64) {}
nonisolated func close() {
@@ -167,6 +172,10 @@ private final class FakePublisherSession: RelayPublisherSessioning, @unchecked S
func profiles() -> [WebRTCStreamingProfile] {
state.withLock { $0.profiles }
}
+
+ func activeCodecs() -> [Set] {
+ state.withLock { $0.activeCodecs }
+ }
}
private final class PublisherFactoryRecorder: @unchecked Sendable {
@@ -201,6 +210,10 @@ private final class RelayHubBox: @unchecked Sendable {
func updatePerformanceMode(_ mode: CapturePerformanceMode) {
state.withLock { $0 }?.updatePerformanceMode(mode)
}
+
+ func updateSourceVideoSpec(_ spec: SourceVideoSpec) {
+ state.withLock { $0 }?.updateSourceVideoSpec(spec)
+ }
}
private final class AsyncGate: @unchecked Sendable {
@@ -285,6 +298,11 @@ struct RelaySessionHubTests {
#expect(client.viewerOffers().first?.sdp == "viewer-offer")
let answer = try #require(socket.decodedTextPayloads().first(where: { $0.contains(#""type":"answer""#) }))
#expect(answer.contains(#""sdp":"relay-viewer-answer-viewer-1""#))
+ #expect(answer.contains(#""sourceVideoSpec""#))
+ #expect(answer.contains(#""width":1920"#))
+ #expect(answer.contains(#""height":1080"#))
+ #expect(answer.contains(#""framesPerSecond":60"#))
+ #expect(await waitUntil { factory.records().first?.publisher.activeCodecs().contains(Set([.av1])) == true })
}
@MainActor @Test func staleViewerOfferCleansRelayViewerAndDoesNotSendAnswer() async {
@@ -309,6 +327,54 @@ struct RelaySessionHubTests {
#expect(socket.decodedTextPayloads().contains(where: { $0.contains(#""type":"answer""#) }) == false)
}
+ @MainActor @Test func publisherCodecPendingSendsRetryMessageAndKeepsViewerConnected() async throws {
+ let client = FakeRelayClient(onViewerOffer: {
+ throw RelayHTTPError.httpStatus(400, "publisher_codec_pending")
+ })
+ let factory = PublisherFactoryRecorder()
+ let demandEvents = Mutex<[Bool]>([])
+ let hub = RelaySessionHub(
+ onDemandChanged: { value in demandEvents.withLock { $0.append(value) } },
+ relayClientProvider: { client },
+ publisherFactory: factory.make
+ )
+ let socket = RelayTestSocketConnection()
+ #expect(isRelayAccepted(hub.addClient(socket, target: .id(2), makeClientID: { "viewer-1" }, eventSink: { _ in })))
+
+ hub.receiveSignalText(#"{"type":"offer","sdp":"viewer-offer"}"#, from: socket)
+
+ #expect(await waitUntil { client.viewerOffers().count == 1 })
+ #expect(await waitUntil { socket.decodedTextPayloads().contains(where: { $0.contains(#""type":"codec_pending""#) }) })
+ #expect(await waitUntil { factory.records().count == 1 })
+ #expect(socket.cancelCallCount == 0)
+ #expect(hub.activeClientCount == 1)
+ #expect(factory.records().first?.publisher.closeCallCount() == 0)
+ #expect(demandEvents.withLock { $0 } == [true])
+ let codecPending = try #require(socket.decodedTextPayloads().first(where: { $0.contains(#""type":"codec_pending""#) }))
+ #expect(codecPending.contains(#""reason":"publisher_codec_pending""#))
+ #expect(socket.decodedTextPayloads().contains(where: { $0.contains(#""type":"error""#) }) == false)
+ }
+
+ @MainActor @Test func viewerCodecRelayErrorForwardsSpecificReason() async throws {
+ let client = FakeRelayClient(onViewerOffer: {
+ throw RelayHTTPError.httpStatus(400, "unsupported_video_codec_offered")
+ })
+ let factory = PublisherFactoryRecorder()
+ let hub = RelaySessionHub(
+ relayClientProvider: { client },
+ publisherFactory: factory.make
+ )
+ let socket = RelayTestSocketConnection()
+ #expect(isRelayAccepted(hub.addClient(socket, target: .id(2), makeClientID: { "viewer-1" }, eventSink: { _ in })))
+
+ hub.receiveSignalText(#"{"type":"offer","sdp":"viewer-offer"}"#, from: socket)
+
+ #expect(await waitUntil { client.viewerOffers().count == 1 })
+ #expect(await waitUntil { socket.cancelCallCount == 1 })
+ let error = try #require(socket.decodedTextPayloads().first(where: { $0.contains(#""type":"error""#) }))
+ #expect(error.contains(#""reason":"unsupported_video_codec_offered""#))
+ }
+
@MainActor @Test func tenViewersCreateOnePublisherSession() async {
let client = FakeRelayClient()
let factory = PublisherFactoryRecorder()
@@ -349,6 +415,28 @@ struct RelaySessionHubTests {
#expect(factory.records().count == 1)
}
+ @MainActor @Test func sourceSpecChangeUpdatesPublisherProfile() async {
+ let client = FakeRelayClient()
+ let factory = PublisherFactoryRecorder()
+ let hub = RelaySessionHub(
+ relayClientProvider: { client },
+ publisherFactory: factory.make
+ )
+ let socket = RelayTestSocketConnection()
+ #expect(isRelayAccepted(hub.addClient(socket, target: .id(2), makeClientID: { "viewer-1" }, eventSink: { _ in })))
+ #expect(await waitUntil { factory.records().count == 1 })
+
+ let sourceSpec = SourceVideoSpec(width: 2_560, height: 1_440, framesPerSecond: 120)
+ hub.updateSourceVideoSpec(sourceSpec)
+
+ let expectedProfile = WebRTCStreamingProfile(
+ performanceMode: .automatic,
+ sourceVideoSpec: sourceSpec
+ )
+ let publisher = factory.records().first?.publisher
+ #expect(publisher?.profiles().contains(expectedProfile) == true)
+ }
+
@MainActor @Test func publisherStartupRefreshesProfileOutsideStateLock() async {
let client = FakeRelayClient()
let hubBox = RelayHubBox()
diff --git a/Tests/VoidDisplaySharingTests/Web/WebRTCSessionSupportTests.swift b/Tests/VoidDisplaySharingTests/Web/WebRTCSessionSupportTests.swift
index 2e5d22a..96c8ba3 100644
--- a/Tests/VoidDisplaySharingTests/Web/WebRTCSessionSupportTests.swift
+++ b/Tests/VoidDisplaySharingTests/Web/WebRTCSessionSupportTests.swift
@@ -34,68 +34,47 @@ private final class MailboxReference: @unchecked Sendable {
}
struct WebRTCSessionSupportTests {
- @Test func streamingProfilesMatchPerformanceModes() {
- #expect(WebRTCStreamingProfile(performanceMode: .smooth) == WebRTCStreamingProfile(
- framesPerSecond: 60,
- minBitrateBps: 1_500_000,
- maxBitrateBps: 8_000_000,
- pixelBudgetPerSecond: WebRTCStreamingProfile.h264SmoothPixelBudgetPerSecond
- ))
- #expect(WebRTCStreamingProfile(performanceMode: .automatic) == WebRTCStreamingProfile(
- framesPerSecond: 30,
- minBitrateBps: 1_500_000,
- maxBitrateBps: 8_000_000,
- pixelBudgetPerSecond: WebRTCStreamingProfile.h264AutomaticPixelBudgetPerSecond
- ))
- #expect(WebRTCStreamingProfile(performanceMode: .powerEfficient) == WebRTCStreamingProfile(
- framesPerSecond: 30,
- minBitrateBps: 800_000,
- maxBitrateBps: 5_000_000,
- pixelBudgetPerSecond: WebRTCStreamingProfile.h264PowerEfficientPixelBudgetPerSecond
- ))
+ @Test func automaticAndSmoothProfilesUseSourceSpecForAV1() {
+ let sourceSpec = SourceVideoSpec(width: 2_560, height: 1_440, framesPerSecond: 60)
+ for mode in [CapturePerformanceMode.automatic, .smooth] {
+ let profile = WebRTCStreamingProfile(performanceMode: mode, sourceVideoSpec: sourceSpec)
+ let av1Dimensions = profile.outputDimensions(for: .av1, width: 2_560, height: 1_440)
+
+ #expect(profile.framesPerSecond == 60)
+ #expect(profile.framesPerSecond(for: .av1) == 60)
+ #expect(profile.pixelBudgetPerSecond == nil)
+ #expect(av1Dimensions.width == 2_560)
+ #expect(av1Dimensions.height == 1_440)
+ #expect(profile.outputVideoSpec(for: .av1) == sourceSpec)
+ }
}
- @Test func streamingProfilesApplyPixelBudgetByMode() {
- let smoothDimensions = WebRTCStreamingProfile(performanceMode: .smooth)
- .outputDimensions(forWidth: 3_840, height: 2_160)
- let automaticDimensions = WebRTCStreamingProfile(performanceMode: .automatic)
- .outputDimensions(forWidth: 3_840, height: 2_160)
- let powerEfficientDimensions = WebRTCStreamingProfile(performanceMode: .powerEfficient)
- .outputDimensions(forWidth: 3_840, height: 2_160)
- let wideDimensions = WebRTCStreamingProfile(performanceMode: .automatic)
- .outputDimensions(forWidth: 3_440, height: 1_440)
- let portraitDimensions = WebRTCStreamingProfile(performanceMode: .automatic)
- .outputDimensions(forWidth: 2_160, height: 3_840)
-
- #expect(smoothDimensions.width == 886)
- #expect(smoothDimensions.height == 498)
- #expect(automaticDimensions.width == 1_254)
- #expect(automaticDimensions.height == 704)
- #expect(powerEfficientDimensions.width == 960)
- #expect(powerEfficientDimensions.height == 540)
- #expect(wideDimensions.width == 1_454)
- #expect(wideDimensions.height == 608)
- #expect(portraitDimensions.width == 704)
- #expect(portraitDimensions.height == 1_254)
+ @Test func powerEfficientProfileKeepsOnlyActiveDowngradeBudget() {
+ let sourceSpec = SourceVideoSpec(width: 2_560, height: 1_440, framesPerSecond: 60)
+ let profile = WebRTCStreamingProfile(performanceMode: .powerEfficient, sourceVideoSpec: sourceSpec)
+ let dimensions = profile.outputDimensions(for: .av1, width: 2_560, height: 1_440)
+
+ #expect(profile.framesPerSecond == 30)
+ #expect(profile.pixelBudgetPerSecond == SharedCapturePerformanceBudget.powerEfficientPixelBudgetPerSecond)
+ #expect(dimensions.width == 1_920)
+ #expect(dimensions.height == 1_080)
}
- @Test func streamingProfileOutputFitsH264Level31MacroblockBudget() {
- let samples: [(WebRTCStreamingProfile, Int32, Int32)] = [
- (WebRTCStreamingProfile(performanceMode: .smooth), 3_840, 2_160),
- (WebRTCStreamingProfile(performanceMode: .automatic), 3_840, 2_160),
- (WebRTCStreamingProfile(performanceMode: .powerEfficient), 3_840, 2_160),
- (WebRTCStreamingProfile(performanceMode: .automatic), 2_428, 1_518),
- (WebRTCStreamingProfile(performanceMode: .smooth), 2_428, 1_518),
- (WebRTCStreamingProfile(performanceMode: .automatic), 2_160, 3_840),
- ]
+ @Test func bitrateProfilesScaleWithAV1PixelRate() {
+ let profile1440p60 = WebRTCStreamingProfile(
+ performanceMode: .automatic,
+ sourceVideoSpec: SourceVideoSpec(width: 2_560, height: 1_440, framesPerSecond: 60)
+ )
+ let profile4K60 = WebRTCStreamingProfile(
+ performanceMode: .automatic,
+ sourceVideoSpec: SourceVideoSpec(width: 3_840, height: 2_160, framesPerSecond: 60)
+ )
- for (profile, width, height) in samples {
- let dimensions = profile.outputDimensions(forWidth: width, height: height)
- let macroblocks = macroblockCount(width: dimensions.width, height: dimensions.height)
+ let av1Limits = profile1440p60.bitrateLimits(for: .av1, outputWidth: 2_560, outputHeight: 1_440)
+ let av14KLimits = profile4K60.bitrateLimits(for: .av1, outputWidth: 3_840, outputHeight: 2_160)
- #expect(macroblocks <= 3_600)
- #expect(macroblocks * profile.framesPerSecond <= 108_000)
- }
+ #expect(av1Limits.maxBitrateBps == 22_118_400)
+ #expect(av14KLimits.maxBitrateBps == 49_766_400)
}
#if canImport(WebRTC)
@@ -114,7 +93,7 @@ struct WebRTCSessionSupportTests {
#expect(sequencer.nextTimestampNs(ptsUs: 0, framesPerSecond: 30) == 49_999_999)
}
- @Test func h264CodecPreferenceKeepsOnlyH264AndMatchingRtx() {
+ @Test func codecPreferenceKeepsOnlyRequestedCodecAndMatchingRtx() {
let descriptors = [
WebRTCCodecPreferenceDescriptor(
name: kRTCVp8CodecName,
@@ -127,19 +106,19 @@ struct WebRTCSessionSupportTests {
parameters: ["apt": "96"]
),
WebRTCCodecPreferenceDescriptor(
- name: kRTCH264CodecName,
- payloadType: 102,
+ name: "AV1",
+ payloadType: 104,
parameters: [:]
),
WebRTCCodecPreferenceDescriptor(
name: kRTCRtxCodecName,
- payloadType: 103,
- parameters: ["apt": "102"]
+ payloadType: 105,
+ parameters: ["apt": "104"]
),
]
- #expect(WebRTCCodecPreference.requiredH264DescriptorIndexes(from: descriptors) == [2, 3])
- #expect(WebRTCCodecPreference.requiredH264DescriptorIndexes(from: Array(descriptors.prefix(2))) == nil)
+ #expect(WebRTCCodecPreference.requiredDescriptorIndexes(for: .av1, from: descriptors) == [2, 3])
+ #expect(WebRTCCodecPreference.requiredDescriptorIndexes(for: .av1, from: Array(descriptors.prefix(2))) == nil)
}
@Test func sdpVideoCodecSummaryListsVideoPayloadNames() {
@@ -147,20 +126,33 @@ struct WebRTCSessionSupportTests {
v=0
m=audio 9 UDP/TLS/RTP/SAVPF 111
a=rtpmap:111 opus/48000/2
+ m=video 9 UDP/TLS/RTP/SAVPF 104 105
+ a=rtpmap:104 AV1/90000
+ a=rtpmap:105 rtx/90000
m=video 9 UDP/TLS/RTP/SAVPF 102 103
- a=rtpmap:102 H264/90000
+ a=rtpmap:102 VP8/90000
a=rtpmap:103 rtx/90000
"""
- #expect(WebRTCCodecPreference.sdpVideoCodecSummary(from: sdp) == "102:H264,103:rtx")
+ #expect(WebRTCCodecPreference.sdpVideoCodecSummary(from: sdp) == "104:AV1,105:rtx,103:rtx; unexpectedVideoCodecCount=1")
+
+ let reusedPayloadTypeSDP = """
+ v=0
+ m=video 9 UDP/TLS/RTP/SAVPF 96
+ a=rtpmap:96 AV1/90000
+ m=video 9 UDP/TLS/RTP/SAVPF 96
+ a=rtpmap:96 VP8/90000
+ """
+
+ #expect(WebRTCCodecPreference.sdpVideoCodecSummary(from: reusedPayloadTypeSDP) == "96:AV1; unexpectedVideoCodecCount=1")
}
- @Test func capabilitySummaryDoesNotPrintNonH264CodecNames() {
+ @Test func capabilitySummaryPrintsAV1ProbeWithoutUnsupportedCodecNames() {
let descriptors = [
WebRTCCodecPreferenceDescriptor(
- name: kRTCH264CodecName,
- payloadType: 102,
- parameters: ["profile-level-id": "42e01f"]
+ name: "AV1",
+ payloadType: 35,
+ parameters: [:]
),
WebRTCCodecPreferenceDescriptor(
name: kRTCVp8CodecName,
@@ -171,10 +163,19 @@ struct WebRTCSessionSupportTests {
let summary = WebRTCCodecPreference.capabilitySummary(from: descriptors)
- #expect(summary.contains("H264"))
- #expect(summary.contains("nonH264CodecCount=1"))
+ #expect(summary.contains("AV1=AV1(pt=35,fmtp=none)"))
+ #expect(summary.contains("unsupportedVideoCodecCount=1"))
#expect(!summary.contains("VP8"))
}
+
+ @Test func runtimeSenderCapabilityProbePrintsCurrentWebRTCBinaryCodecs() {
+ let pipeline = WebRTCMediaPipeline()
+ let summary = pipeline.senderVideoCodecCapabilitySummary()
+
+ print("VoidDisplay WebRTC sender codec capability probe: \(summary)")
+ #expect(summary.contains("AV1="))
+ #expect(summary.contains("unsupportedVideoCodecCount="))
+ }
#endif
@Test func frameMailboxKeepsOnlyLatestPendingFrameBeforeDrainStarts() {
@@ -218,9 +219,3 @@ struct WebRTCSessionSupportTests {
#expect(scheduler.count() == 0)
}
}
-
-private func macroblockCount(width: Int32, height: Int32) -> Int {
- let macroblockWidth = (Int(width) + 15) / 16
- let macroblockHeight = (Int(height) + 15) / 16
- return macroblockWidth * macroblockHeight
-}
diff --git a/Tests/VoidDisplayTestingSupport/TestDisplayShareFrameConsumer.swift b/Tests/VoidDisplayTestingSupport/TestDisplayShareFrameConsumer.swift
index 0d93ad5..8399295 100644
--- a/Tests/VoidDisplayTestingSupport/TestDisplayShareFrameConsumer.swift
+++ b/Tests/VoidDisplayTestingSupport/TestDisplayShareFrameConsumer.swift
@@ -6,6 +6,10 @@ package final class TestDisplayShareFrameConsumer: DisplayShareFrameConsumer {
package nonisolated var hasDemand: Bool { false }
+ package nonisolated func updateSourceVideoSpec(_ spec: SourceVideoSpec) {
+ _ = spec
+ }
+
package nonisolated func updatePerformanceMode(_ mode: CapturePerformanceMode) {
_ = mode
}
diff --git a/Tools/VoidDisplayRelay/cmd/voiddisplay-relay/main.go b/Tools/VoidDisplayRelay/cmd/voiddisplay-relay/main.go
index af40f15..acdc6f0 100644
--- a/Tools/VoidDisplayRelay/cmd/voiddisplay-relay/main.go
+++ b/Tools/VoidDisplayRelay/cmd/voiddisplay-relay/main.go
@@ -8,6 +8,7 @@ import (
"os"
"os/signal"
"syscall"
+ "time"
"voiddisplay-relay/internal/relay"
)
@@ -16,9 +17,11 @@ func main() {
var controlToken string
var loopbackHTTP string
var listenUDP string
+ var parentPID int
flag.StringVar(&controlToken, "control-token", "", "shared control token for loopback requests")
flag.StringVar(&loopbackHTTP, "loopback-http", "127.0.0.1:0", "loopback HTTP listen address")
flag.StringVar(&listenUDP, "listen-udp", ":0", "UDP4 listen address for WebRTC traffic")
+ flag.IntVar(&parentPID, "parent-pid", 0, "parent process identifier to monitor")
flag.Parse()
logger := slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo}))
@@ -30,6 +33,9 @@ func main() {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
+ if parentPID > 1 {
+ go monitorParentProcess(ctx, stop, parentPID, logger)
+ }
err := server.ListenAndServe(ctx, loopbackHTTP, func(loopback string) {
fmt.Fprintln(os.Stdout, relay.ReadyJSON(loopback))
@@ -39,3 +45,30 @@ func main() {
os.Exit(1)
}
}
+
+func monitorParentProcess(ctx context.Context, cancel context.CancelFunc, parentPID int, logger *slog.Logger) {
+ ticker := time.NewTicker(time.Second)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ if os.Getppid() == 1 {
+ logger.Info("relay parent process exited", "parent_pid", parentPID)
+ cancel()
+ return
+ }
+ err := syscall.Kill(parentPID, 0)
+ if err == nil || err == syscall.EPERM {
+ continue
+ }
+ if err == syscall.ESRCH {
+ logger.Info("relay parent process is no longer alive", "parent_pid", parentPID)
+ cancel()
+ return
+ }
+ logger.Warn("relay parent process check failed", "parent_pid", parentPID, "error", err)
+ }
+ }
+}
diff --git a/Tools/VoidDisplayRelay/cmd/voiddisplay-relay/main_test.go b/Tools/VoidDisplayRelay/cmd/voiddisplay-relay/main_test.go
new file mode 100644
index 0000000..8de2080
--- /dev/null
+++ b/Tools/VoidDisplayRelay/cmd/voiddisplay-relay/main_test.go
@@ -0,0 +1,23 @@
+package main
+
+import (
+ "context"
+ "io"
+ "log/slog"
+ "testing"
+ "time"
+)
+
+func TestMonitorParentProcessCancelsWhenParentMissing(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ logger := slog.New(slog.NewTextHandler(io.Discard, nil))
+
+ go monitorParentProcess(ctx, cancel, 1<<30, logger)
+
+ select {
+ case <-ctx.Done():
+ case <-time.After(2500 * time.Millisecond):
+ t.Fatal("expected missing parent process to cancel relay context")
+ }
+}
diff --git a/Tools/VoidDisplayRelay/internal/relay/server.go b/Tools/VoidDisplayRelay/internal/relay/server.go
index 5b092b7..d8b889f 100644
--- a/Tools/VoidDisplayRelay/internal/relay/server.go
+++ b/Tools/VoidDisplayRelay/internal/relay/server.go
@@ -9,13 +9,13 @@ import (
"log/slog"
"net"
"net/http"
- "strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pion/ice/v4"
+ "github.com/pion/interceptor"
"github.com/pion/rtcp"
"github.com/pion/rtp"
pionsdp "github.com/pion/sdp/v3"
@@ -56,6 +56,7 @@ type candidateRequest struct {
type signalResponse struct {
Type string `json:"type"`
SDP string `json:"sdp,omitempty"`
+ Codec string `json:"codec,omitempty"`
Reason string `json:"reason,omitempty"`
}
@@ -77,152 +78,61 @@ type Snapshot struct {
}
type RoomSnapshot struct {
- ID string `json:"id"`
- HasPublisher bool `json:"hasPublisher"`
- PublisherID string `json:"publisherID,omitempty"`
- SubscriberCount int `json:"subscriberCount"`
- PublisherPacketCount uint64 `json:"publisherPacketCount"`
- ForwardedPacketCount uint64 `json:"forwardedPacketCount"`
- WrittenPacketCount uint64 `json:"writtenPacketCount"`
- DroppedPacketCount uint64 `json:"droppedPacketCount"`
- SlowSubscriberCount int `json:"slowSubscriberCount"`
- PLIForwardCount uint64 `json:"pliForwardCount"`
- FIRForwardCount uint64 `json:"firForwardCount"`
-}
-
-var errH264VideoCodecMissing = errors.New("h264_video_codec_missing")
-
-var h264RTCPFeedback = []webrtc.RTCPFeedback{
+ ID string `json:"id"`
+ HasPublisher bool `json:"hasPublisher"`
+ PublisherID string `json:"publisherID,omitempty"`
+ PublisherCodecs []string `json:"publisherCodecs,omitempty"`
+ SubscriberCodecCounts map[string]int `json:"subscriberCodecCounts,omitempty"`
+ SubscriberCount int `json:"subscriberCount"`
+ PublisherPacketCount uint64 `json:"publisherPacketCount"`
+ ForwardedPacketCount uint64 `json:"forwardedPacketCount"`
+ WrittenPacketCount uint64 `json:"writtenPacketCount"`
+ DroppedPacketCount uint64 `json:"droppedPacketCount"`
+ SlowSubscriberCount int `json:"slowSubscriberCount"`
+ PLIForwardCount uint64 `json:"pliForwardCount"`
+ FIRForwardCount uint64 `json:"firForwardCount"`
+ NACKForwardCount uint64 `json:"nackForwardCount"`
+}
+
+type videoCodec string
+
+const (
+ videoCodecAV1 videoCodec = "av1"
+)
+
+var errSupportedVideoCodecMissing = errors.New("supported_video_codec_missing")
+var errUnsupportedVideoCodecOffered = errors.New("unsupported_video_codec_offered")
+var errPublisherCodecPending = errors.New("publisher_codec_pending")
+var errPublisherCodecDuplicate = errors.New("publisher_video_codec_duplicate")
+
+var av1RTCPFeedback = []webrtc.RTCPFeedback{
{Type: "goog-remb"},
{Type: "ccm", Parameter: "fir"},
{Type: "nack"},
{Type: "nack", Parameter: "pli"},
}
-var h264CodecParameters = []webrtc.RTPCodecParameters{
- {
- RTPCodecCapability: webrtc.RTPCodecCapability{
- MimeType: webrtc.MimeTypeH264,
- ClockRate: 90000,
- SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f",
- RTCPFeedback: h264RTCPFeedback,
- },
- PayloadType: 102,
- },
- {
- RTPCodecCapability: webrtc.RTPCodecCapability{
- MimeType: webrtc.MimeTypeRTX,
- ClockRate: 90000,
- SDPFmtpLine: "apt=102",
- },
- PayloadType: 103,
- },
- {
- RTPCodecCapability: webrtc.RTPCodecCapability{
- MimeType: webrtc.MimeTypeH264,
- ClockRate: 90000,
- SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f",
- RTCPFeedback: h264RTCPFeedback,
- },
- PayloadType: 104,
- },
- {
- RTPCodecCapability: webrtc.RTPCodecCapability{
- MimeType: webrtc.MimeTypeRTX,
- ClockRate: 90000,
- SDPFmtpLine: "apt=104",
- },
- PayloadType: 105,
- },
- {
- RTPCodecCapability: webrtc.RTPCodecCapability{
- MimeType: webrtc.MimeTypeH264,
- ClockRate: 90000,
- SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",
- RTCPFeedback: h264RTCPFeedback,
- },
- PayloadType: 106,
- },
- {
- RTPCodecCapability: webrtc.RTPCodecCapability{
- MimeType: webrtc.MimeTypeRTX,
- ClockRate: 90000,
- SDPFmtpLine: "apt=106",
- },
- PayloadType: 107,
- },
- {
- RTPCodecCapability: webrtc.RTPCodecCapability{
- MimeType: webrtc.MimeTypeH264,
- ClockRate: 90000,
- SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f",
- RTCPFeedback: h264RTCPFeedback,
- },
- PayloadType: 108,
- },
- {
- RTPCodecCapability: webrtc.RTPCodecCapability{
- MimeType: webrtc.MimeTypeRTX,
- ClockRate: 90000,
- SDPFmtpLine: "apt=108",
- },
- PayloadType: 109,
- },
- {
- RTPCodecCapability: webrtc.RTPCodecCapability{
- MimeType: webrtc.MimeTypeH264,
- ClockRate: 90000,
- SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f",
- RTCPFeedback: h264RTCPFeedback,
- },
- PayloadType: 127,
- },
- {
- RTPCodecCapability: webrtc.RTPCodecCapability{
- MimeType: webrtc.MimeTypeRTX,
- ClockRate: 90000,
- SDPFmtpLine: "apt=127",
- },
- PayloadType: 125,
- },
- {
- RTPCodecCapability: webrtc.RTPCodecCapability{
- MimeType: webrtc.MimeTypeH264,
- ClockRate: 90000,
- SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f",
- RTCPFeedback: h264RTCPFeedback,
- },
- PayloadType: 39,
- },
+var av1CodecParameters = []webrtc.RTPCodecParameters{
{
RTPCodecCapability: webrtc.RTPCodecCapability{
- MimeType: webrtc.MimeTypeRTX,
- ClockRate: 90000,
- SDPFmtpLine: "apt=39",
- },
- PayloadType: 40,
- },
- {
- RTPCodecCapability: webrtc.RTPCodecCapability{
- MimeType: webrtc.MimeTypeH264,
+ MimeType: webrtc.MimeTypeAV1,
ClockRate: 90000,
- SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c1f",
- RTCPFeedback: h264RTCPFeedback,
+ RTCPFeedback: av1RTCPFeedback,
},
- PayloadType: 112,
+ PayloadType: 45,
},
{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: webrtc.MimeTypeRTX,
ClockRate: 90000,
- SDPFmtpLine: "apt=112",
+ SDPFmtpLine: "apt=45",
},
- PayloadType: 113,
+ PayloadType: 46,
},
}
-func registerH264Codecs(mediaEngine *webrtc.MediaEngine) error {
- for _, codec := range h264CodecParameters {
+func registerVideoCodecs(mediaEngine *webrtc.MediaEngine) error {
+ for _, codec := range av1CodecParameters {
if err := mediaEngine.RegisterCodec(codec, webrtc.RTPCodecTypeVideo); err != nil {
return err
}
@@ -230,42 +140,220 @@ func registerH264Codecs(mediaEngine *webrtc.MediaEngine) error {
return nil
}
-func h264TrackCapability() webrtc.RTPCodecCapability {
+func trackCapability(codec videoCodec) (webrtc.RTPCodecCapability, error) {
+ if codec != videoCodecAV1 {
+ return webrtc.RTPCodecCapability{}, errUnsupportedVideoCodecOffered
+ }
return webrtc.RTPCodecCapability{
- MimeType: webrtc.MimeTypeH264,
+ MimeType: webrtc.MimeTypeAV1,
ClockRate: 90000,
- RTCPFeedback: h264RTCPFeedback,
+ RTCPFeedback: av1RTCPFeedback,
+ }, nil
+}
+
+func codecParametersForVideoCodec(codec videoCodec) ([]webrtc.RTPCodecParameters, error) {
+ if codec != videoCodecAV1 {
+ return nil, errUnsupportedVideoCodecOffered
+ }
+ return append([]webrtc.RTPCodecParameters(nil), av1CodecParameters...), nil
+}
+
+func codecFromName(name string) (videoCodec, bool) {
+ switch {
+ case strings.EqualFold(name, "AV1"), strings.EqualFold(name, webrtc.MimeTypeAV1):
+ return videoCodecAV1, true
+ default:
+ return "", false
}
}
-func requireH264VideoCodec(sdp string) error {
+type videoMediaCodecSet struct {
+ codecs map[videoCodec]struct{}
+ unsupportedPrimaryCount int
+}
+
+func videoMediaCodecSets(sdp string) ([]videoMediaCodecSet, error) {
var description pionsdp.SessionDescription
if err := description.UnmarshalString(sdp); err != nil {
- return err
- }
- for _, media := range description.MediaDescriptions {
- if !strings.EqualFold(media.MediaName.Media, "video") {
- continue
+ return nil, err
+ }
+ mediaCodecSets := make([]videoMediaCodecSet, 0)
+ payloadTypes := make(map[string]struct{})
+ payloadNames := make(map[string]string)
+ inVideo := false
+ flush := func() {
+ if !inVideo {
+ return
}
- payloadTypes := make([]uint8, 0, len(media.MediaName.Formats))
- for _, format := range media.MediaName.Formats {
- payloadType, err := strconv.ParseUint(format, 10, 8)
- if err != nil {
+ codecSet := make(map[videoCodec]struct{})
+ unsupportedPrimaryCount := 0
+ for payloadType := range payloadTypes {
+ name := payloadNames[payloadType]
+ if videoCodec, ok := codecFromName(name); ok {
+ codecSet[videoCodec] = struct{}{}
continue
}
- payloadTypes = append(payloadTypes, uint8(payloadType))
+ if !strings.EqualFold(name, "rtx") {
+ unsupportedPrimaryCount++
+ }
}
- codecs, err := description.GetCodecsForPayloadTypes(payloadTypes)
- if err != nil {
- return err
+ mediaCodecSets = append(mediaCodecSets, videoMediaCodecSet{
+ codecs: codecSet,
+ unsupportedPrimaryCount: unsupportedPrimaryCount,
+ })
+ }
+ for _, line := range strings.Split(sdp, "\n") {
+ line = strings.TrimSpace(line)
+ if strings.HasPrefix(line, "m=") {
+ flush()
+ inVideo = strings.HasPrefix(line, "m=video ")
+ payloadTypes = make(map[string]struct{})
+ payloadNames = make(map[string]string)
+ if inVideo {
+ parts := strings.Fields(line)
+ for _, payloadType := range parts[3:] {
+ payloadTypes[payloadType] = struct{}{}
+ }
+ }
+ continue
}
- for _, codec := range codecs {
- if strings.EqualFold(codec.Name, "H264") {
- return nil
+ if !inVideo || !strings.HasPrefix(line, "a=rtpmap:") {
+ continue
+ }
+ parts := strings.Fields(strings.TrimPrefix(line, "a=rtpmap:"))
+ if len(parts) < 2 {
+ continue
+ }
+ payloadType := parts[0]
+ payloadNames[payloadType] = strings.SplitN(parts[1], "/", 2)[0]
+ }
+ flush()
+ return mediaCodecSets, nil
+}
+
+func publisherVideoCodecs(sdp string) ([]videoCodec, error) {
+ mediaCodecSets, err := videoMediaCodecSets(sdp)
+ if err != nil {
+ return nil, err
+ }
+ codecs := make([]videoCodec, 0, len(mediaCodecSets))
+ seen := make(map[videoCodec]struct{})
+ for _, mediaCodecSet := range mediaCodecSets {
+ if mediaCodecSet.unsupportedPrimaryCount > 0 {
+ return nil, errUnsupportedVideoCodecOffered
+ }
+ if len(mediaCodecSet.codecs) == 0 {
+ return nil, errSupportedVideoCodecMissing
+ }
+ if _, ok := mediaCodecSet.codecs[videoCodecAV1]; ok && len(mediaCodecSet.codecs) == 1 {
+ if _, duplicate := seen[videoCodecAV1]; duplicate {
+ return nil, errPublisherCodecDuplicate
}
+ seen[videoCodecAV1] = struct{}{}
+ codecs = append(codecs, videoCodecAV1)
+ continue
}
+ return nil, errUnsupportedVideoCodecOffered
+ }
+ if len(codecs) == 0 {
+ return nil, errSupportedVideoCodecMissing
+ }
+ return codecs, nil
+}
+
+func codecSetFromList(codecs []videoCodec) map[videoCodec]struct{} {
+ result := make(map[videoCodec]struct{}, len(codecs))
+ for _, codec := range codecs {
+ result[codec] = struct{}{}
}
- return errH264VideoCodecMissing
+ return result
+}
+
+func codecListFromSet(codecSet map[videoCodec]struct{}) []videoCodec {
+ codecs := make([]videoCodec, 0, len(codecSet))
+ if _, ok := codecSet[videoCodecAV1]; ok {
+ codecs = append(codecs, videoCodecAV1)
+ }
+ return codecs
+}
+
+func codecStrings(codecs []videoCodec) []string {
+ result := make([]string, 0, len(codecs))
+ for _, codec := range codecs {
+ result = append(result, string(codec))
+ }
+ return result
+}
+
+func headerExtensionIDs(parameters []webrtc.RTPHeaderExtensionParameter) map[string]uint8 {
+ result := make(map[string]uint8, len(parameters))
+ for _, parameter := range parameters {
+ if parameter.ID <= 0 || parameter.ID > 255 || parameter.URI == "" {
+ continue
+ }
+ result[parameter.URI] = uint8(parameter.ID)
+ }
+ return result
+}
+
+func headerExtensionRewrites(
+ publisherExtensions map[string]uint8,
+ viewerExtensions map[string]uint8,
+) map[uint8]uint8 {
+ rewrites := make(map[uint8]uint8)
+ for uri, publisherID := range publisherExtensions {
+ viewerID, ok := viewerExtensions[uri]
+ if !ok {
+ rewrites[publisherID] = 0
+ continue
+ }
+ rewrites[publisherID] = viewerID
+ }
+ return rewrites
+}
+
+func copyHeaderExtensionMap(input map[string]uint8) map[string]uint8 {
+ output := make(map[string]uint8, len(input))
+ for key, value := range input {
+ output[key] = value
+ }
+ return output
+}
+
+func copyExtensionRewriteMap(input map[uint8]uint8) map[uint8]uint8 {
+ output := make(map[uint8]uint8, len(input))
+ for key, value := range input {
+ output[key] = value
+ }
+ return output
+}
+
+func selectViewerCodec(sdp string, available map[videoCodec]struct{}) (videoCodec, error) {
+ mediaCodecSets, err := videoMediaCodecSets(sdp)
+ if err != nil {
+ return "", err
+ }
+ if len(available) == 0 {
+ return "", errPublisherCodecPending
+ }
+ offered := make(map[videoCodec]struct{})
+ for _, mediaCodecSet := range mediaCodecSets {
+ if mediaCodecSet.unsupportedPrimaryCount > 0 {
+ return "", errUnsupportedVideoCodecOffered
+ }
+ if len(mediaCodecSet.codecs) == 0 {
+ return "", errSupportedVideoCodecMissing
+ }
+ for codec := range mediaCodecSet.codecs {
+ offered[codec] = struct{}{}
+ }
+ }
+ if _, publisherHasAV1 := available[videoCodecAV1]; publisherHasAV1 {
+ if _, viewerHasAV1 := offered[videoCodecAV1]; viewerHasAV1 {
+ return videoCodecAV1, nil
+ }
+ }
+ return "", errSupportedVideoCodecMissing
}
func NewServer(config Config) *Server {
@@ -408,7 +496,23 @@ func (s *Server) startWebRTC() error {
}
udpMux := ice.NewUDPMuxDefault(ice.UDPMuxParams{UDPConn: udpConn})
mediaEngine := &webrtc.MediaEngine{}
- if err := registerH264Codecs(mediaEngine); err != nil {
+ if err := registerVideoCodecs(mediaEngine); err != nil {
+ _ = udpMux.Close()
+ _ = udpConn.Close()
+ return err
+ }
+ interceptorRegistry := &interceptor.Registry{}
+ if err := webrtc.ConfigureNack(mediaEngine, interceptorRegistry); err != nil {
+ _ = udpMux.Close()
+ _ = udpConn.Close()
+ return err
+ }
+ if err := webrtc.ConfigureRTCPReports(interceptorRegistry); err != nil {
+ _ = udpMux.Close()
+ _ = udpConn.Close()
+ return err
+ }
+ if err := webrtc.ConfigureStatsInterceptor(interceptorRegistry); err != nil {
_ = udpMux.Close()
_ = udpConn.Close()
return err
@@ -419,6 +523,7 @@ func (s *Server) startWebRTC() error {
api := webrtc.NewAPI(
webrtc.WithMediaEngine(mediaEngine),
webrtc.WithSettingEngine(settingEngine),
+ webrtc.WithInterceptorRegistry(interceptorRegistry),
)
listenAddresses := make([]string, 0)
@@ -537,7 +642,7 @@ func (s *Server) handleViewerOffer(w http.ResponseWriter, r *http.Request, roomI
writeJSON(w, http.StatusBadRequest, signalResponse{Type: "error", Reason: err.Error()})
return
}
- writeJSON(w, http.StatusOK, signalResponse{Type: "answer", SDP: answer})
+ writeJSON(w, http.StatusOK, signalResponse{Type: "answer", SDP: answer.SDP, Codec: string(answer.Codec)})
}
func (s *Server) handleViewerCandidate(w http.ResponseWriter, r *http.Request, roomID string, clientID string) {
@@ -632,11 +737,14 @@ type Room struct {
viewers map[string]*viewerSession
subscribers map[string]*viewerRTPWriter
pendingViewerICE map[string][]webrtc.ICECandidateInit
- publisherSSRC uint32
+ publisherCodecs map[videoCodec]struct{}
+ publisherSSRCs map[videoCodec]uint32
+ publisherExtensions map[videoCodec]map[string]uint8
publisherPacketCount atomic.Uint64
forwardedPacketCount atomic.Uint64
pliForwardCount atomic.Uint64
firForwardCount atomic.Uint64
+ nackForwardCount atomic.Uint64
}
type rtpSink interface {
@@ -654,6 +762,11 @@ type publisherOfferResult struct {
PublisherID string
}
+type viewerOfferResult struct {
+ SDP string
+ Codec videoCodec
+}
+
type publisherSession struct {
id string
pc peerConnection
@@ -663,6 +776,7 @@ type viewerSession struct {
pc peerConnection
sender *webrtc.RTPSender
writer *viewerRTPWriter
+ codec videoCodec
}
type peerConnection interface {
@@ -674,29 +788,35 @@ type peerConnection interface {
const subscriberRTPQueueSize = 512
type viewerRTPWriter struct {
- roomID string
- clientID string
- logger *slog.Logger
- sink rtpSink
- queue chan *rtp.Packet
- done chan struct{}
- mu sync.Mutex
- closed bool
- written atomic.Uint64
- dropped atomic.Uint64
-}
-
-func newViewerRTPWriter(roomID string, clientID string, sink rtpSink, logger *slog.Logger) *viewerRTPWriter {
+ roomID string
+ clientID string
+ codec videoCodec
+ logger *slog.Logger
+ sink rtpSink
+ queue chan *rtp.Packet
+ done chan struct{}
+ mu sync.Mutex
+ closed bool
+ viewerExtensions map[string]uint8
+ extensionRewrites map[uint8]uint8
+ written atomic.Uint64
+ dropped atomic.Uint64
+}
+
+func newViewerRTPWriter(roomID string, clientID string, codec videoCodec, sink rtpSink, logger *slog.Logger) *viewerRTPWriter {
if logger == nil {
logger = slog.Default()
}
writer := &viewerRTPWriter{
- roomID: roomID,
- clientID: clientID,
- logger: logger,
- sink: sink,
- queue: make(chan *rtp.Packet, subscriberRTPQueueSize),
- done: make(chan struct{}),
+ roomID: roomID,
+ clientID: clientID,
+ codec: codec,
+ logger: logger,
+ sink: sink,
+ queue: make(chan *rtp.Packet, subscriberRTPQueueSize),
+ done: make(chan struct{}),
+ viewerExtensions: make(map[string]uint8),
+ extensionRewrites: make(map[uint8]uint8),
}
go writer.run()
return writer
@@ -709,6 +829,7 @@ func (w *viewerRTPWriter) enqueue(packet *rtp.Packet) bool {
return false
}
packetCopy := packet.Clone()
+ w.rewriteHeaderExtensions(packetCopy)
select {
case w.queue <- packetCopy:
return true
@@ -719,6 +840,49 @@ func (w *viewerRTPWriter) enqueue(packet *rtp.Packet) bool {
}
}
+func (w *viewerRTPWriter) setViewerExtensions(extensions map[string]uint8) {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ w.viewerExtensions = copyHeaderExtensionMap(extensions)
+}
+
+func (w *viewerRTPWriter) setExtensionRewrites(rewrites map[uint8]uint8) {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ w.extensionRewrites = copyExtensionRewriteMap(rewrites)
+}
+
+func (w *viewerRTPWriter) setPublisherExtensions(extensions map[string]uint8) {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ w.extensionRewrites = headerExtensionRewrites(extensions, w.viewerExtensions)
+}
+
+func (w *viewerRTPWriter) rewriteHeaderExtensions(packet *rtp.Packet) {
+ type pendingRewrite struct {
+ viewerID uint8
+ payload []byte
+ }
+ pendingRewrites := make([]pendingRewrite, 0, len(w.extensionRewrites))
+ for publisherID, viewerID := range w.extensionRewrites {
+ payload := packet.GetExtension(publisherID)
+ if payload == nil {
+ continue
+ }
+ payloadCopy := append([]byte(nil), payload...)
+ _ = packet.DelExtension(publisherID)
+ if viewerID != 0 {
+ pendingRewrites = append(pendingRewrites, pendingRewrite{
+ viewerID: viewerID,
+ payload: payloadCopy,
+ })
+ }
+ }
+ for _, rewrite := range pendingRewrites {
+ _ = packet.SetExtension(rewrite.viewerID, rewrite.payload)
+ }
+}
+
func (w *viewerRTPWriter) run() {
for {
select {
@@ -753,12 +917,15 @@ func NewRoom(id string, logger *slog.Logger, newPeerConnection peerConnectionFac
logger = slog.Default()
}
return &Room{
- id: id,
- logger: logger,
- newPeerConnection: newPeerConnection,
- viewers: make(map[string]*viewerSession),
- subscribers: make(map[string]*viewerRTPWriter),
- pendingViewerICE: make(map[string][]webrtc.ICECandidateInit),
+ id: id,
+ logger: logger,
+ newPeerConnection: newPeerConnection,
+ viewers: make(map[string]*viewerSession),
+ subscribers: make(map[string]*viewerRTPWriter),
+ pendingViewerICE: make(map[string][]webrtc.ICECandidateInit),
+ publisherCodecs: make(map[videoCodec]struct{}),
+ publisherSSRCs: make(map[videoCodec]uint32),
+ publisherExtensions: make(map[videoCodec]map[string]uint8),
}
}
@@ -767,11 +934,14 @@ func newRoomForTest(id string, logger *slog.Logger) *Room {
logger = slog.Default()
}
return &Room{
- id: id,
- logger: logger,
- viewers: make(map[string]*viewerSession),
- subscribers: make(map[string]*viewerRTPWriter),
- pendingViewerICE: make(map[string][]webrtc.ICECandidateInit),
+ id: id,
+ logger: logger,
+ viewers: make(map[string]*viewerSession),
+ subscribers: make(map[string]*viewerRTPWriter),
+ pendingViewerICE: make(map[string][]webrtc.ICECandidateInit),
+ publisherCodecs: make(map[videoCodec]struct{}),
+ publisherSSRCs: make(map[videoCodec]uint32),
+ publisherExtensions: make(map[videoCodec]map[string]uint8),
}
}
@@ -781,9 +951,12 @@ func (r *Room) Snapshot() RoomSnapshot {
if r.publisher != nil {
publisherID = r.publisher.id
}
+ publisherCodecs := codecStrings(codecListFromSet(r.publisherCodecs))
writers := make([]*viewerRTPWriter, 0, len(r.subscribers))
+ subscriberCodecCounts := make(map[string]int)
for _, writer := range r.subscribers {
writers = append(writers, writer)
+ subscriberCodecCounts[string(writer.codec)]++
}
r.mu.Unlock()
var writtenPacketCount uint64
@@ -798,17 +971,20 @@ func (r *Room) Snapshot() RoomSnapshot {
}
}
return RoomSnapshot{
- ID: r.id,
- HasPublisher: publisherID != "",
- PublisherID: publisherID,
- SubscriberCount: len(writers),
- PublisherPacketCount: r.publisherPacketCount.Load(),
- ForwardedPacketCount: r.forwardedPacketCount.Load(),
- WrittenPacketCount: writtenPacketCount,
- DroppedPacketCount: droppedPacketCount,
- SlowSubscriberCount: slowSubscriberCount,
- PLIForwardCount: r.pliForwardCount.Load(),
- FIRForwardCount: r.firForwardCount.Load(),
+ ID: r.id,
+ HasPublisher: publisherID != "",
+ PublisherID: publisherID,
+ PublisherCodecs: publisherCodecs,
+ SubscriberCodecCounts: subscriberCodecCounts,
+ SubscriberCount: len(writers),
+ PublisherPacketCount: r.publisherPacketCount.Load(),
+ ForwardedPacketCount: r.forwardedPacketCount.Load(),
+ WrittenPacketCount: writtenPacketCount,
+ DroppedPacketCount: droppedPacketCount,
+ SlowSubscriberCount: slowSubscriberCount,
+ PLIForwardCount: r.pliForwardCount.Load(),
+ FIRForwardCount: r.firForwardCount.Load(),
+ NACKForwardCount: r.nackForwardCount.Load(),
}
}
@@ -816,38 +992,37 @@ func (r *Room) SetPublisherOffer(sdp string) (publisherOfferResult, error) {
if r.newPeerConnection == nil {
return publisherOfferResult{}, errors.New("room_peer_connection_factory_missing")
}
- if err := requireH264VideoCodec(sdp); err != nil {
+ if _, err := publisherVideoCodecs(sdp); err != nil {
return publisherOfferResult{}, err
}
pc, err := r.newPeerConnection()
if err != nil {
return publisherOfferResult{}, err
}
- _, err = pc.AddTransceiverFromKind(webrtc.RTPCodecTypeVideo, webrtc.RTPTransceiverInit{
- Direction: webrtc.RTPTransceiverDirectionRecvonly,
- })
- if err != nil {
- _ = pc.Close()
- return publisherOfferResult{}, err
- }
publisherID := r.nextPublisherIdentifier()
r.mu.Lock()
previous := r.publisher
- previousSSRC := r.publisherSSRC
+ previousCodecs := r.publisherCodecs
+ previousSSRCs := r.publisherSSRCs
+ previousExtensions := r.publisherExtensions
r.publisher = &publisherSession{id: publisherID, pc: pc}
- r.publisherSSRC = 0
+ r.publisherCodecs = make(map[videoCodec]struct{})
+ r.publisherSSRCs = make(map[videoCodec]uint32)
+ r.publisherExtensions = make(map[videoCodec]map[string]uint8)
r.mu.Unlock()
rollbackPublisher := func() {
r.mu.Lock()
if r.publisher != nil && r.publisher.id == publisherID {
r.publisher = previous
- r.publisherSSRC = previousSSRC
+ r.publisherCodecs = previousCodecs
+ r.publisherSSRCs = previousSSRCs
+ r.publisherExtensions = previousExtensions
}
r.mu.Unlock()
_ = pc.Close()
}
pc.OnTrack(func(remote *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) {
- r.publisherTrackStarted(publisherID, remote)
+ r.publisherTrackStarted(publisherID, remote, receiver)
})
pc.OnConnectionStateChange(func(state webrtc.PeerConnectionState) {
if state == webrtc.PeerConnectionStateFailed ||
@@ -877,11 +1052,22 @@ func (r *Room) SetPublisherOffer(sdp string) (publisherOfferResult, error) {
rollbackPublisher()
return publisherOfferResult{}, errors.New("publisher_local_description_missing")
}
+ negotiatedCodecs, err := publisherVideoCodecs(localDescription.SDP)
+ if err != nil {
+ rollbackPublisher()
+ return publisherOfferResult{}, err
+ }
+ negotiatedCodecSet := codecSetFromList(negotiatedCodecs)
+ r.mu.Lock()
+ if r.publisher != nil && r.publisher.id == publisherID {
+ r.publisherCodecs = negotiatedCodecSet
+ }
+ r.mu.Unlock()
if previous != nil {
_ = previous.pc.Close()
}
- r.logger.Info("publisher ready", "room", r.id, "publisherID", publisherID)
+ r.logger.Info("publisher ready", "room", r.id, "publisherID", publisherID, "codecs", codecStrings(negotiatedCodecs))
return publisherOfferResult{SDP: localDescription.SDP, PublisherID: publisherID}, nil
}
@@ -904,30 +1090,58 @@ func (r *Room) AddPublisherCandidate(publisherID string, candidate webrtc.ICECan
return publisher.pc.AddICECandidate(candidate)
}
-func (r *Room) SetViewerOffer(clientID string, sdp string) (string, error) {
+func (r *Room) SetViewerOffer(clientID string, sdp string) (viewerOfferResult, error) {
if r.newPeerConnection == nil {
- return "", errors.New("room_peer_connection_factory_missing")
+ return viewerOfferResult{}, errors.New("room_peer_connection_factory_missing")
}
- if err := requireH264VideoCodec(sdp); err != nil {
- return "", err
+ r.mu.Lock()
+ publisherCodecs := make(map[videoCodec]struct{}, len(r.publisherCodecs))
+ for codec := range r.publisherCodecs {
+ publisherCodecs[codec] = struct{}{}
+ }
+ r.mu.Unlock()
+ selectedCodec, err := selectViewerCodec(sdp, publisherCodecs)
+ if err != nil {
+ return viewerOfferResult{}, err
}
pc, err := r.newPeerConnection()
if err != nil {
- return "", err
+ return viewerOfferResult{}, err
+ }
+ capability, err := trackCapability(selectedCodec)
+ if err != nil {
+ _ = pc.Close()
+ return viewerOfferResult{}, err
}
track, err := webrtc.NewTrackLocalStaticRTP(
- h264TrackCapability(),
- "screen",
+ capability,
+ "screen-"+string(selectedCodec),
"voiddisplay",
)
if err != nil {
_ = pc.Close()
- return "", err
+ return viewerOfferResult{}, err
+ }
+ transceiver, err := pc.AddTransceiverFromTrack(track, webrtc.RTPTransceiverInit{
+ Direction: webrtc.RTPTransceiverDirectionSendonly,
+ })
+ if err != nil {
+ _ = pc.Close()
+ return viewerOfferResult{}, err
}
- sender, err := pc.AddTrack(track)
+ codecParameters, err := codecParametersForVideoCodec(selectedCodec)
if err != nil {
_ = pc.Close()
- return "", err
+ return viewerOfferResult{}, err
+ }
+ if err := transceiver.SetCodecPreferences(codecParameters); err != nil {
+ _ = pc.Close()
+ return viewerOfferResult{}, err
+ }
+ sender := transceiver.Sender()
+ if sender == nil {
+ _ = pc.Close()
+ return viewerOfferResult{}, errors.New("viewer_sender_missing")
}
pc.OnConnectionStateChange(func(state webrtc.PeerConnectionState) {
if state == webrtc.PeerConnectionStateFailed ||
@@ -939,29 +1153,34 @@ func (r *Room) SetViewerOffer(clientID string, sdp string) (string, error) {
offer := webrtc.SessionDescription{Type: webrtc.SDPTypeOffer, SDP: sdp}
if err := pc.SetRemoteDescription(offer); err != nil {
_ = pc.Close()
- return "", err
+ return viewerOfferResult{}, err
}
answer, err := pc.CreateAnswer(nil)
if err != nil {
_ = pc.Close()
- return "", err
+ return viewerOfferResult{}, err
}
gatherComplete := webrtc.GatheringCompletePromise(pc)
if err := pc.SetLocalDescription(answer); err != nil {
_ = pc.Close()
- return "", err
+ return viewerOfferResult{}, err
}
<-gatherComplete
localDescription := pc.LocalDescription()
if localDescription == nil {
_ = pc.Close()
- return "", errors.New("viewer_local_description_missing")
+ return viewerOfferResult{}, errors.New("viewer_local_description_missing")
}
- writer := newViewerRTPWriter(r.id, clientID, track, r.logger)
+ writer := newViewerRTPWriter(r.id, clientID, selectedCodec, track, r.logger)
+ viewerExtensions := headerExtensionIDs(sender.GetParameters().HeaderExtensions)
+ writer.setViewerExtensions(viewerExtensions)
r.mu.Lock()
previous := r.viewers[clientID]
- r.viewers[clientID] = &viewerSession{pc: pc, sender: sender, writer: writer}
+ if publisherExtensions := r.publisherExtensions[selectedCodec]; len(publisherExtensions) > 0 {
+ writer.setExtensionRewrites(headerExtensionRewrites(publisherExtensions, viewerExtensions))
+ }
+ r.viewers[clientID] = &viewerSession{pc: pc, sender: sender, writer: writer, codec: selectedCodec}
r.subscribers[clientID] = writer
pending := append([]webrtc.ICECandidateInit(nil), r.pendingViewerICE[clientID]...)
delete(r.pendingViewerICE, clientID)
@@ -972,9 +1191,9 @@ func (r *Room) SetViewerOffer(clientID string, sdp string) (string, error) {
}
r.applyICECandidates(clientID, pc, pending)
- r.logger.Info("viewer ready", "room", r.id, "clientID", clientID, "subscribers", subscriberCount)
- go r.readViewerRTCP(clientID, sender)
- return localDescription.SDP, nil
+ r.logger.Info("viewer ready", "room", r.id, "clientID", clientID, "codec", selectedCodec, "subscribers", subscriberCount)
+ go r.readViewerRTCP(clientID, selectedCodec, sender)
+ return viewerOfferResult{SDP: localDescription.SDP, Codec: selectedCodec}, nil
}
func (r *Room) AddViewerCandidate(clientID string, candidate webrtc.ICECandidateInit) error {
@@ -1015,7 +1234,9 @@ func (r *Room) Close() {
r.viewers = make(map[string]*viewerSession)
r.subscribers = make(map[string]*viewerRTPWriter)
r.pendingViewerICE = make(map[string][]webrtc.ICECandidateInit)
- r.publisherSSRC = 0
+ r.publisherCodecs = make(map[videoCodec]struct{})
+ r.publisherSSRCs = make(map[videoCodec]uint32)
+ r.publisherExtensions = make(map[videoCodec]map[string]uint8)
r.mu.Unlock()
if publisher != nil {
_ = publisher.pc.Close()
@@ -1031,7 +1252,9 @@ func (r *Room) StopPublisher(publisherID string) {
if r.publisher != nil && r.publisher.id == publisherID {
publisher = r.publisher
r.publisher = nil
- r.publisherSSRC = 0
+ r.publisherCodecs = make(map[videoCodec]struct{})
+ r.publisherSSRCs = make(map[videoCodec]uint32)
+ r.publisherExtensions = make(map[videoCodec]map[string]uint8)
}
r.mu.Unlock()
if publisher == nil {
@@ -1079,15 +1302,32 @@ func (r *Room) handlePublisherDisconnected(publisherID string, state webrtc.Peer
r.Close()
}
-func (r *Room) publisherTrackStarted(publisherID string, remote *webrtc.TrackRemote) {
+func (r *Room) publisherTrackStarted(publisherID string, remote *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) {
+ codec, ok := codecFromName(remote.Codec().MimeType)
+ if !ok {
+ r.logger.Warn("publisher track ignored unsupported codec", "room", r.id, "publisherID", publisherID, "codec", remote.Codec().MimeType)
+ return
+ }
r.mu.Lock()
if r.publisher == nil || r.publisher.id != publisherID {
r.mu.Unlock()
r.logger.Debug("ignored stale publisher track", "room", r.id, "publisherID", publisherID)
return
}
- r.publisherSSRC = uint32(remote.SSRC())
+ ssrc := uint32(remote.SSRC())
+ r.publisherSSRCs[codec] = ssrc
+ publisherExtensions := headerExtensionIDs(receiver.GetParameters().HeaderExtensions)
+ r.publisherExtensions[codec] = publisherExtensions
+ writers := make([]*viewerRTPWriter, 0, len(r.subscribers))
+ for _, writer := range r.subscribers {
+ if writer.codec == codec {
+ writers = append(writers, writer)
+ }
+ }
r.mu.Unlock()
+ for _, writer := range writers {
+ writer.setPublisherExtensions(publisherExtensions)
+ }
r.logger.Info(
"publisher track started",
"room",
@@ -1097,8 +1337,9 @@ func (r *Room) publisherTrackStarted(publisherID string, remote *webrtc.TrackRem
"codec",
remote.Codec().MimeType,
"ssrc",
- uint32(remote.SSRC()),
+ ssrc,
)
+ defer r.publisherTrackStopped(publisherID, codec, ssrc)
for {
packet, _, err := remote.ReadRTP()
if err != nil {
@@ -1107,18 +1348,35 @@ func (r *Room) publisherTrackStarted(publisherID string, remote *webrtc.TrackRem
}
return
}
- if !r.ForwardRTPFromPublisher(publisherID, packet) {
+ if !r.ForwardRTPFromPublisher(publisherID, codec, packet) {
return
}
r.publisherPacketCount.Add(1)
}
}
+func (r *Room) publisherTrackStopped(publisherID string, codec videoCodec, ssrc uint32) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ if r.publisher == nil || r.publisher.id != publisherID {
+ return
+ }
+ if r.publisherSSRCs[codec] != ssrc {
+ return
+ }
+ delete(r.publisherSSRCs, codec)
+ delete(r.publisherExtensions, codec)
+}
+
func (r *Room) ForwardRTP(packet *rtp.Packet) {
- r.forwardRTP(packet)
+ r.forwardRTPToSubscribers("", packet)
}
-func (r *Room) ForwardRTPFromPublisher(publisherID string, packet *rtp.Packet) bool {
+func (r *Room) ForwardRTPForCodec(codec videoCodec, packet *rtp.Packet) {
+ r.forwardRTPToSubscribers(codec, packet)
+}
+
+func (r *Room) ForwardRTPFromPublisher(publisherID string, codec videoCodec, packet *rtp.Packet) bool {
r.mu.Lock()
isCurrent := r.publisher != nil && r.publisher.id == publisherID
r.mu.Unlock()
@@ -1126,14 +1384,17 @@ func (r *Room) ForwardRTPFromPublisher(publisherID string, packet *rtp.Packet) b
r.logger.Debug("ignored stale publisher RTP", "room", r.id, "publisherID", publisherID)
return false
}
- r.forwardRTP(packet)
+ r.forwardRTPToSubscribers(codec, packet)
return true
}
-func (r *Room) forwardRTP(packet *rtp.Packet) {
+func (r *Room) forwardRTPToSubscribers(codec videoCodec, packet *rtp.Packet) {
r.mu.Lock()
subscribers := make([]*viewerRTPWriter, 0, len(r.subscribers))
for _, subscriber := range r.subscribers {
+ if codec != "" && subscriber.codec != codec {
+ continue
+ }
subscribers = append(subscribers, subscriber)
}
r.mu.Unlock()
@@ -1144,7 +1405,7 @@ func (r *Room) forwardRTP(packet *rtp.Packet) {
}
}
-func (r *Room) readViewerRTCP(clientID string, sender *webrtc.RTPSender) {
+func (r *Room) readViewerRTCP(clientID string, codec videoCodec, sender *webrtc.RTPSender) {
for {
packets, _, err := sender.ReadRTCP()
if err != nil {
@@ -1153,14 +1414,14 @@ func (r *Room) readViewerRTCP(clientID string, sender *webrtc.RTPSender) {
}
return
}
- r.forwardFeedback(packets)
+ r.forwardFeedback(codec, packets)
}
}
-func (r *Room) forwardFeedback(packets []rtcp.Packet) {
+func (r *Room) forwardFeedback(codec videoCodec, packets []rtcp.Packet) {
r.mu.Lock()
publisher := r.publisher
- publisherSSRC := r.publisherSSRC
+ publisherSSRC := r.publisherSSRCs[codec]
r.mu.Unlock()
if publisher == nil || publisherSSRC == 0 {
return
@@ -1187,6 +1448,15 @@ func (r *Room) forwardFeedback(packets []rtcp.Packet) {
MediaSSRC: publisherSSRC,
FIR: entries,
})
+ case *rtcp.TransportLayerNack:
+ r.nackForwardCount.Add(1)
+ nacks := make([]rtcp.NackPair, len(value.Nacks))
+ copy(nacks, value.Nacks)
+ forwarded = append(forwarded, &rtcp.TransportLayerNack{
+ SenderSSRC: value.SenderSSRC,
+ MediaSSRC: publisherSSRC,
+ Nacks: nacks,
+ })
}
}
if len(forwarded) == 0 {
diff --git a/Tools/VoidDisplayRelay/internal/relay/server_test.go b/Tools/VoidDisplayRelay/internal/relay/server_test.go
index b334157..816dab3 100644
--- a/Tools/VoidDisplayRelay/internal/relay/server_test.go
+++ b/Tools/VoidDisplayRelay/internal/relay/server_test.go
@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"encoding/json"
+ "errors"
"net/http"
"strings"
"sync"
@@ -20,8 +21,8 @@ func TestRoomForwardRTPFansOutOnePublisherPacketToViewers(t *testing.T) {
room := newRoomForTest("2", nil)
first := &recordingSink{}
second := &recordingSink{}
- room.subscribers["first"] = newViewerRTPWriter("2", "first", first, nil)
- room.subscribers["second"] = newViewerRTPWriter("2", "second", second, nil)
+ room.subscribers["first"] = newViewerRTPWriter("2", "first", videoCodecAV1, first, nil)
+ room.subscribers["second"] = newViewerRTPWriter("2", "second", videoCodecAV1, second, nil)
defer room.Close()
packet := &rtp.Packet{
@@ -49,8 +50,8 @@ func TestRoomRemoveViewerStopsFutureRTPForThatViewer(t *testing.T) {
room := newRoomForTest("2", nil)
removed := &recordingSink{}
active := &recordingSink{}
- room.subscribers["removed"] = newViewerRTPWriter("2", "removed", removed, nil)
- room.subscribers["active"] = newViewerRTPWriter("2", "active", active, nil)
+ room.subscribers["removed"] = newViewerRTPWriter("2", "removed", videoCodecAV1, removed, nil)
+ room.subscribers["active"] = newViewerRTPWriter("2", "active", videoCodecAV1, active, nil)
room.viewers["removed"] = &viewerSession{pc: &fakePeerConnection{}}
room.pendingViewerICE["removed"] = []webrtc.ICECandidateInit{{Candidate: "candidate:removed"}}
defer room.Close()
@@ -70,6 +71,20 @@ func TestRoomRemoveViewerStopsFutureRTPForThatViewer(t *testing.T) {
}
}
+func TestRoomSnapshotCountsSubscriberCodecs(t *testing.T) {
+ room := newRoomForTest("2", nil)
+ room.subscribers["av1-a"] = newViewerRTPWriter("2", "av1-a", videoCodecAV1, &recordingSink{}, nil)
+ room.subscribers["av1-b"] = newViewerRTPWriter("2", "av1-b", videoCodecAV1, &recordingSink{}, nil)
+ room.subscribers["av1-c"] = newViewerRTPWriter("2", "av1-c", videoCodecAV1, &recordingSink{}, nil)
+ defer room.Close()
+
+ snapshot := room.Snapshot()
+
+ if snapshot.SubscriberCodecCounts["av1"] != 3 {
+ t.Fatalf("AV1 subscriber count = %d, want 3", snapshot.SubscriberCodecCounts["av1"])
+ }
+}
+
func TestRoomIgnoresStalePublisherCandidate(t *testing.T) {
room := newRoomForTest("2", nil)
pc := &fakePeerConnection{}
@@ -84,6 +99,26 @@ func TestRoomIgnoresStalePublisherCandidate(t *testing.T) {
}
}
+func TestRoomPublisherTrackStoppedClearsOnlyMatchingActiveCodec(t *testing.T) {
+ room := newRoomForTest("2", nil)
+ room.publisher = &publisherSession{id: "publisher-1", pc: &fakePeerConnection{}}
+ room.publisherSSRCs[videoCodecAV1] = 1234
+ room.publisherExtensions[videoCodecAV1] = map[string]uint8{"av1-extension": 3}
+
+ room.publisherTrackStopped("publisher-1", videoCodecAV1, 9999)
+ if room.publisherSSRCs[videoCodecAV1] != 1234 {
+ t.Fatalf("AV1 SSRC cleared for nonmatching SSRC")
+ }
+
+ room.publisherTrackStopped("publisher-1", videoCodecAV1, 1234)
+ if _, ok := room.publisherSSRCs[videoCodecAV1]; ok {
+ t.Fatal("AV1 SSRC kept after matching track stopped")
+ }
+ if _, ok := room.publisherExtensions[videoCodecAV1]; ok {
+ t.Fatal("AV1 extensions kept after matching track stopped")
+ }
+}
+
func TestRoomBuffersViewerCandidateBeforeViewerExists(t *testing.T) {
room := newRoomForTest("2", nil)
@@ -115,24 +150,29 @@ func TestRoomApplyPendingICECandidates(t *testing.T) {
}
}
-func TestRoomForwardFeedbackRetargetsPLIAndFIRToPublisherSSRC(t *testing.T) {
+func TestRoomForwardFeedbackRetargetsPLIAndFIRAndNACKToPublisherSSRC(t *testing.T) {
pc := &fakePeerConnection{}
room := newRoomForTest("2", nil)
room.publisher = &publisherSession{id: "publisher-1", pc: pc}
- room.publisherSSRC = 1234
+ room.publisherSSRCs[videoCodecAV1] = 1234
- room.forwardFeedback([]rtcp.Packet{
+ room.forwardFeedback(videoCodecAV1, []rtcp.Packet{
&rtcp.PictureLossIndication{SenderSSRC: 1, MediaSSRC: 55},
&rtcp.FullIntraRequest{
SenderSSRC: 2,
MediaSSRC: 56,
FIR: []rtcp.FIREntry{{SSRC: 56, SequenceNumber: 9}},
},
+ &rtcp.TransportLayerNack{
+ SenderSSRC: 3,
+ MediaSSRC: 57,
+ Nacks: []rtcp.NackPair{{PacketID: 100}},
+ },
})
packets := pc.writtenRTCP()
- if len(packets) != 2 {
- t.Fatalf("written RTCP count = %d, want 2", len(packets))
+ if len(packets) != 3 {
+ t.Fatalf("written RTCP count = %d, want 3", len(packets))
}
pli, ok := packets[0].(*rtcp.PictureLossIndication)
if !ok || pli.MediaSSRC != 1234 {
@@ -142,12 +182,19 @@ func TestRoomForwardFeedbackRetargetsPLIAndFIRToPublisherSSRC(t *testing.T) {
if !ok || fir.MediaSSRC != 1234 || fir.FIR[0].SSRC != 1234 {
t.Fatalf("FIR retarget mismatch: %#v", packets[1])
}
+ nack, ok := packets[2].(*rtcp.TransportLayerNack)
+ if !ok || nack.MediaSSRC != 1234 || nack.Nacks[0].PacketID != 100 {
+ t.Fatalf("NACK retarget mismatch: %#v", packets[2])
+ }
if got := room.pliForwardCount.Load(); got != 1 {
t.Fatalf("PLI count = %d, want 1", got)
}
if got := room.firForwardCount.Load(); got != 1 {
t.Fatalf("FIR count = %d, want 1", got)
}
+ if got := room.nackForwardCount.Load(); got != 1 {
+ t.Fatalf("NACK count = %d, want 1", got)
+ }
}
func TestRoomStalePublisherStopAndRTPDoNotAffectCurrentPublisher(t *testing.T) {
@@ -155,11 +202,11 @@ func TestRoomStalePublisherStopAndRTPDoNotAffectCurrentPublisher(t *testing.T) {
pc := &fakePeerConnection{}
sink := &recordingSink{}
room.publisher = &publisherSession{id: "current", pc: pc}
- room.subscribers["viewer"] = newViewerRTPWriter("2", "viewer", sink, nil)
+ room.subscribers["viewer"] = newViewerRTPWriter("2", "viewer", videoCodecAV1, sink, nil)
defer room.Close()
room.StopPublisher("stale")
- forwarded := room.ForwardRTPFromPublisher("stale", &rtp.Packet{
+ forwarded := room.ForwardRTPFromPublisher("stale", videoCodecAV1, &rtp.Packet{
Header: rtp.Header{Timestamp: 77},
Payload: []byte{1},
})
@@ -180,8 +227,8 @@ func TestRoomSlowViewerDropsOnlyThatViewer(t *testing.T) {
slow := newBlockingSink()
defer slow.release()
active := &recordingSink{}
- room.subscribers["slow"] = newViewerRTPWriter("2", "slow", slow, nil)
- room.subscribers["active"] = newViewerRTPWriter("2", "active", active, nil)
+ room.subscribers["slow"] = newViewerRTPWriter("2", "slow", videoCodecAV1, slow, nil)
+ room.subscribers["active"] = newViewerRTPWriter("2", "active", videoCodecAV1, active, nil)
defer room.Close()
for index := 0; index < subscriberRTPQueueSize+20; index++ {
@@ -207,7 +254,7 @@ func TestRoomSlowViewerDropsOnlyThatViewer(t *testing.T) {
func TestViewerWriterRejectsEnqueueAfterClose(t *testing.T) {
sink := &recordingSink{}
- writer := newViewerRTPWriter("2", "viewer", sink, nil)
+ writer := newViewerRTPWriter("2", "viewer", videoCodecAV1, sink, nil)
writer.close()
if writer.enqueue(&rtp.Packet{Header: rtp.Header{Timestamp: 10}, Payload: []byte{1}}) {
@@ -218,10 +265,40 @@ func TestViewerWriterRejectsEnqueueAfterClose(t *testing.T) {
}
}
+func TestViewerWriterRewritesHeaderExtensionIDs(t *testing.T) {
+ sink := &recordingSink{}
+ writer := newViewerRTPWriter("2", "viewer", videoCodecAV1, sink, nil)
+ defer writer.close()
+ writer.setExtensionRewrites(map[uint8]uint8{3: 4, 4: 0})
+ packet := &rtp.Packet{
+ Header: rtp.Header{Timestamp: 10},
+ Payload: []byte{1},
+ }
+ if err := packet.SetExtension(3, []byte{9, 8}); err != nil {
+ t.Fatal(err)
+ }
+ if err := packet.SetExtension(4, []byte{7, 6}); err != nil {
+ t.Fatal(err)
+ }
+
+ if !writer.enqueue(packet) {
+ t.Fatal("writer rejected RTP")
+ }
+
+ waitFor(t, func() bool { return sink.count() == 1 })
+ forwarded := sink.onlyPacket(t)
+ if got := forwarded.GetExtension(4); !bytes.Equal(got, []byte{9, 8}) {
+ t.Fatalf("viewer extension 4 = %v, want [9 8]", got)
+ }
+ if got := forwarded.GetExtension(3); got != nil {
+ t.Fatalf("publisher extension 3 was not removed: %v", got)
+ }
+}
+
func TestRoomConcurrentRemoveAndForwardDoesNotRace(t *testing.T) {
room := newRoomForTest("2", nil)
sink := &recordingSink{}
- room.subscribers["viewer"] = newViewerRTPWriter("2", "viewer", sink, nil)
+ room.subscribers["viewer"] = newViewerRTPWriter("2", "viewer", videoCodecAV1, sink, nil)
room.viewers["viewer"] = &viewerSession{pc: &fakePeerConnection{}, writer: room.subscribers["viewer"]}
var wg sync.WaitGroup
@@ -266,7 +343,8 @@ func TestRoomInvalidPublisherOfferPreservesCurrentPublisher(t *testing.T) {
room := NewRoom("2", nil, server.newPeerConnection)
current := &fakePeerConnection{}
room.publisher = &publisherSession{id: "current", pc: current}
- room.publisherSSRC = 1234
+ room.publisherCodecs[videoCodecAV1] = struct{}{}
+ room.publisherSSRCs[videoCodecAV1] = 1234
if _, err := room.SetPublisherOffer("invalid-sdp"); err == nil {
t.Fatal("SetPublisherOffer succeeded for invalid SDP")
@@ -275,8 +353,8 @@ func TestRoomInvalidPublisherOfferPreservesCurrentPublisher(t *testing.T) {
if room.publisher == nil || room.publisher.id != "current" {
t.Fatalf("current publisher was not preserved: %#v", room.publisher)
}
- if room.publisherSSRC != 1234 {
- t.Fatalf("publisher SSRC = %d, want 1234", room.publisherSSRC)
+ if room.publisherSSRCs[videoCodecAV1] != 1234 {
+ t.Fatalf("publisher AV1 SSRC = %d, want 1234", room.publisherSSRCs[videoCodecAV1])
}
if current.isClosed() {
t.Fatal("current publisher was closed")
@@ -296,7 +374,7 @@ func TestRoomPublisherRejectsVP8OnlyOffer(t *testing.T) {
}
}
-func TestRoomPublisherAcceptsH264Offer(t *testing.T) {
+func TestRoomPublisherRejectsH265OnlyOffer(t *testing.T) {
server := NewServer(Config{ListenUDP: "127.0.0.1:0"})
if err := server.startWebRTC(); err != nil {
t.Fatal(err)
@@ -304,14 +382,51 @@ func TestRoomPublisherAcceptsH264Offer(t *testing.T) {
defer server.Close()
room := NewRoom("2", nil, server.newPeerConnection)
- result, err := room.SetPublisherOffer(createPublisherOfferWithCodec(t, webrtc.MimeTypeH264))
+ if _, err := room.SetPublisherOffer(createPublisherOfferWithCodec(t, webrtc.MimeTypeH265)); err == nil {
+ t.Fatal("SetPublisherOffer accepted H265-only SDP")
+ }
+}
+
+func TestRoomPublisherRejectsMixedSupportedAndUnsupportedVideoOffer(t *testing.T) {
+ server := NewServer(Config{ListenUDP: "127.0.0.1:0"})
+ if err := server.startWebRTC(); err != nil {
+ t.Fatal(err)
+ }
+ defer server.Close()
+ room := NewRoom("2", nil, server.newPeerConnection)
+ offer := appendUnsupportedVideoCodecForTest(
+ createPublisherOfferWithCodec(t, webrtc.MimeTypeAV1),
+ "96",
+ "VP8",
+ )
+
+ _, err := room.SetPublisherOffer(offer)
+
+ if !errors.Is(err, errUnsupportedVideoCodecOffered) {
+ t.Fatalf("SetPublisherOffer error = %v, want unsupported codec", err)
+ }
+}
+
+func TestRoomPublisherAcceptsAV1Offer(t *testing.T) {
+ server := NewServer(Config{ListenUDP: "127.0.0.1:0"})
+ if err := server.startWebRTC(); err != nil {
+ t.Fatal(err)
+ }
+ defer server.Close()
+ room := NewRoom("2", nil, server.newPeerConnection)
+
+ result, err := room.SetPublisherOffer(createPublisherOfferWithCodec(t, webrtc.MimeTypeAV1))
if err != nil {
t.Fatalf("SetPublisherOffer returned error: %v", err)
}
- assertVideoSDPOnlyH264(t, result.SDP)
+ assertVideoSDPOnlyCodec(t, result.SDP, videoCodecAV1)
+ snapshot := room.Snapshot()
+ if strings.Join(snapshot.PublisherCodecs, ",") != "av1" {
+ t.Fatalf("publisher codecs = %v, want [av1]", snapshot.PublisherCodecs)
+ }
}
-func TestRoomPublisherAcceptsSwiftH264HighProfileOffer(t *testing.T) {
+func TestRoomPublisherRejectsDuplicateCodecVideoMLine(t *testing.T) {
server := NewServer(Config{ListenUDP: "127.0.0.1:0"})
if err := server.startWebRTC(); err != nil {
t.Fatal(err)
@@ -319,44 +434,163 @@ func TestRoomPublisherAcceptsSwiftH264HighProfileOffer(t *testing.T) {
defer server.Close()
room := NewRoom("2", nil, server.newPeerConnection)
- result, err := room.SetPublisherOffer(createPublisherOfferWithH264Fmtp(
- t,
- "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c1f",
- ))
+ _, err := room.SetPublisherOffer(createPublisherOfferWithCodecs(t, []videoCodec{videoCodecAV1, videoCodecAV1}))
+
+ if !errors.Is(err, errPublisherCodecDuplicate) {
+ t.Fatalf("SetPublisherOffer error = %v, want duplicate codec", err)
+ }
+}
+
+func TestRoomViewerAnswerUsesOnlyAV1(t *testing.T) {
+ server := NewServer(Config{ListenUDP: "127.0.0.1:0"})
+ if err := server.startWebRTC(); err != nil {
+ t.Fatal(err)
+ }
+ defer server.Close()
+ room := NewRoom("2", nil, server.newPeerConnection)
+ room.publisherCodecs[videoCodecAV1] = struct{}{}
+ room.publisherSSRCs[videoCodecAV1] = 1234
+
+ answer, err := room.SetViewerOffer("viewer", createViewerOfferWithCodec(t, videoCodecAV1))
if err != nil {
- t.Fatalf("SetPublisherOffer returned error: %v", err)
+ t.Fatalf("SetViewerOffer returned error: %v", err)
+ }
+ assertVideoSDPOnlyCodec(t, answer.SDP, videoCodecAV1)
+ if answer.Codec != videoCodecAV1 {
+ t.Fatalf("viewer answer codec = %s, want av1", answer.Codec)
+ }
+ if room.viewers["viewer"].codec != videoCodecAV1 {
+ t.Fatalf("viewer codec = %s, want av1", room.viewers["viewer"].codec)
+ }
+}
+
+func TestRoomViewerRejectsOfferWhenViewerLacksAV1(t *testing.T) {
+ server := NewServer(Config{ListenUDP: "127.0.0.1:0"})
+ if err := server.startWebRTC(); err != nil {
+ t.Fatal(err)
+ }
+ defer server.Close()
+ room := NewRoom("2", nil, server.newPeerConnection)
+ room.publisherCodecs[videoCodecAV1] = struct{}{}
+ room.publisherSSRCs[videoCodecAV1] = 1234
+
+ _, err := room.SetViewerOffer("viewer", createViewerOfferWithMimeType(t, webrtc.MimeTypeH264))
+
+ if !errors.Is(err, errUnsupportedVideoCodecOffered) && !errors.Is(err, errSupportedVideoCodecMissing) {
+ t.Fatalf("SetViewerOffer error = %v, want unsupported or missing codec", err)
+ }
+}
+
+func TestRoomViewerUsesNegotiatedAV1BeforeAV1RTPStarts(t *testing.T) {
+ server := NewServer(Config{ListenUDP: "127.0.0.1:0"})
+ if err := server.startWebRTC(); err != nil {
+ t.Fatal(err)
}
- assertVideoSDPOnlyH264(t, result.SDP)
- if !strings.Contains(result.SDP, "profile-level-id=640c1f") {
- t.Fatalf("publisher answer did not retain Swift H264 profile 640c1f:\n%s", result.SDP)
+ defer server.Close()
+ room := NewRoom("2", nil, server.newPeerConnection)
+ room.publisherCodecs[videoCodecAV1] = struct{}{}
+ room.publisherSSRCs[videoCodecAV1] = 5678
+
+ answer, err := room.SetViewerOffer("viewer", createViewerOfferWithCodec(t, videoCodecAV1))
+ if err != nil {
+ t.Fatalf("SetViewerOffer returned error: %v", err)
+ }
+ assertVideoSDPOnlyCodec(t, answer.SDP, videoCodecAV1)
+ if answer.Codec != videoCodecAV1 {
+ t.Fatalf("viewer answer codec = %s, want av1", answer.Codec)
+ }
+ if room.viewers["viewer"].codec != videoCodecAV1 {
+ t.Fatalf("viewer codec = %s, want av1", room.viewers["viewer"].codec)
}
}
-func TestRoomViewerAnswerUsesOnlyH264(t *testing.T) {
+func TestRoomViewerUsesNegotiatedCodecsBeforePublisherRTPStarts(t *testing.T) {
server := NewServer(Config{ListenUDP: "127.0.0.1:0"})
if err := server.startWebRTC(); err != nil {
t.Fatal(err)
}
defer server.Close()
room := NewRoom("2", nil, server.newPeerConnection)
+ room.publisherCodecs[videoCodecAV1] = struct{}{}
- answer, err := room.SetViewerOffer("viewer", createViewerOffer(t))
+ answer, err := room.SetViewerOffer("viewer", createViewerOfferWithCodec(t, videoCodecAV1))
if err != nil {
t.Fatalf("SetViewerOffer returned error: %v", err)
}
- assertVideoSDPOnlyH264(t, answer)
+ assertVideoSDPOnlyCodec(t, answer.SDP, videoCodecAV1)
+ if answer.Codec != videoCodecAV1 {
+ t.Fatalf("viewer answer codec = %s, want av1", answer.Codec)
+ }
+ if room.viewers["viewer"].codec != videoCodecAV1 {
+ t.Fatalf("viewer codec = %s, want av1", room.viewers["viewer"].codec)
+ }
+}
+
+func TestRoomViewerReturnsCodecPendingUntilPublisherCodecsAreNegotiated(t *testing.T) {
+ server := NewServer(Config{ListenUDP: "127.0.0.1:0"})
+ if err := server.startWebRTC(); err != nil {
+ t.Fatal(err)
+ }
+ defer server.Close()
+ room := NewRoom("2", nil, server.newPeerConnection)
+
+ _, err := room.SetViewerOffer("viewer", createViewerOfferWithCodecs(t, []videoCodec{videoCodecAV1, videoCodecAV1}))
+
+ if !errors.Is(err, errPublisherCodecPending) {
+ t.Fatalf("SetViewerOffer error = %v, want codec pending", err)
+ }
+}
+
+func TestRoomViewerRejectsUnsupportedVideoCodecOffer(t *testing.T) {
+ server := NewServer(Config{ListenUDP: "127.0.0.1:0"})
+ if err := server.startWebRTC(); err != nil {
+ t.Fatal(err)
+ }
+ defer server.Close()
+ room := NewRoom("2", nil, server.newPeerConnection)
+ room.publisherCodecs[videoCodecAV1] = struct{}{}
+ room.publisherSSRCs[videoCodecAV1] = 1234
+
+ if _, err := room.SetViewerOffer("viewer-vp8", createViewerOfferWithMimeType(t, webrtc.MimeTypeVP8)); err == nil {
+ t.Fatal("SetViewerOffer accepted VP8-only SDP")
+ }
+ if _, err := room.SetViewerOffer("viewer-h265", createViewerOfferWithMimeType(t, webrtc.MimeTypeH265)); err == nil {
+ t.Fatal("SetViewerOffer accepted H265-only SDP")
+ }
}
-func TestRoomForwardRTPRewritesViewerPayloadTypeFromNegotiatedH264Binding(t *testing.T) {
+func TestRoomViewerRejectsMixedSupportedAndUnsupportedVideoOffer(t *testing.T) {
+ server := NewServer(Config{ListenUDP: "127.0.0.1:0"})
+ if err := server.startWebRTC(); err != nil {
+ t.Fatal(err)
+ }
+ defer server.Close()
+ room := NewRoom("2", nil, server.newPeerConnection)
+ room.publisherCodecs[videoCodecAV1] = struct{}{}
+ room.publisherSSRCs[videoCodecAV1] = 1234
+ offer := appendUnsupportedVideoCodecForTest(
+ createViewerOfferWithCodec(t, videoCodecAV1),
+ "116",
+ "H265",
+ )
+
+ _, err := room.SetViewerOffer("viewer", offer)
+
+ if !errors.Is(err, errUnsupportedVideoCodecOffered) {
+ t.Fatalf("SetViewerOffer error = %v, want unsupported codec", err)
+ }
+}
+
+func TestRoomForwardRTPRewritesViewerPayloadTypeFromNegotiatedAV1Binding(t *testing.T) {
room := newRoomForTest("2", nil)
- track, err := webrtc.NewTrackLocalStaticRTP(h264TrackCapability(), "screen", "voiddisplay")
+ track, err := webrtc.NewTrackLocalStaticRTP(mustTrackCapability(t, videoCodecAV1), "screen", "voiddisplay")
if err != nil {
t.Fatal(err)
}
stream := &capturingTrackLocalWriter{}
_, err = track.Bind(fakeTrackLocalContext{
codecs: []webrtc.RTPCodecParameters{{
- RTPCodecCapability: h264TrackCapability(),
+ RTPCodecCapability: mustTrackCapability(t, videoCodecAV1),
PayloadType: 124,
}},
ssrc: 5678,
@@ -365,10 +599,10 @@ func TestRoomForwardRTPRewritesViewerPayloadTypeFromNegotiatedH264Binding(t *tes
if err != nil {
t.Fatal(err)
}
- room.subscribers["viewer"] = newViewerRTPWriter("2", "viewer", track, nil)
+ room.subscribers["viewer"] = newViewerRTPWriter("2", "viewer", videoCodecAV1, track, nil)
defer room.Close()
- room.ForwardRTP(&rtp.Packet{
+ room.ForwardRTPForCodec(videoCodecAV1, &rtp.Packet{
Header: rtp.Header{
PayloadType: 102,
SSRC: 1234,
@@ -388,6 +622,28 @@ func TestRoomForwardRTPRewritesViewerPayloadTypeFromNegotiatedH264Binding(t *tes
}
}
+func TestRoomForwardRTPForCodecSendsAV1PacketsToAV1Viewers(t *testing.T) {
+ room := newRoomForTest("2", nil)
+ first := &recordingSink{}
+ second := &recordingSink{}
+ room.subscribers["first"] = newViewerRTPWriter("2", "first", videoCodecAV1, first, nil)
+ room.subscribers["second"] = newViewerRTPWriter("2", "second", videoCodecAV1, second, nil)
+ defer room.Close()
+
+ room.ForwardRTPForCodec(videoCodecAV1, &rtp.Packet{
+ Header: rtp.Header{Timestamp: 11},
+ Payload: []byte{1},
+ })
+
+ waitFor(t, func() bool { return first.count() == 1 && second.count() == 1 })
+ if got := first.onlyPacket(t).Timestamp; got != 11 {
+ t.Fatalf("first viewer timestamp = %d, want 11", got)
+ }
+ if got := second.onlyPacket(t).Timestamp; got != 11 {
+ t.Fatalf("second viewer timestamp = %d, want 11", got)
+ }
+}
+
func TestServerListenUDPBindsSocketAndEventsExposeAddress(t *testing.T) {
loopback, stopServer := startTestServer(t)
defer stopServer()
@@ -607,7 +863,7 @@ func startTestServer(t *testing.T) (string, func()) {
func createPublisherOffer(t *testing.T) string {
t.Helper()
- pc, _ := createPublisherPeerWithCodec(t, webrtc.MimeTypeH264)
+ pc, _ := createPublisherPeerWithCodec(t, webrtc.MimeTypeAV1)
defer pc.Close()
return pc.LocalDescription().SDP
}
@@ -619,24 +875,32 @@ func createPublisherOfferWithCodec(t *testing.T, mimeType string) string {
return pc.LocalDescription().SDP
}
-func createPublisherOfferWithH264Fmtp(t *testing.T, fmtp string) string {
+func appendUnsupportedVideoCodecForTest(sdp string, payloadType string, codecName string) string {
+ lines := strings.Split(sdp, "\n")
+ output := make([]string, 0, len(lines)+1)
+ insertedRTPMap := false
+ inFirstVideo := false
+ for _, line := range lines {
+ trimmed := strings.TrimSpace(line)
+ if strings.HasPrefix(trimmed, "m=video ") && !insertedRTPMap {
+ line = strings.TrimRight(line, "\r") + " " + payloadType
+ inFirstVideo = true
+ } else if strings.HasPrefix(trimmed, "m=") {
+ inFirstVideo = false
+ }
+ output = append(output, line)
+ if inFirstVideo && !insertedRTPMap && strings.HasPrefix(trimmed, "m=video ") {
+ output = append(output, "a=rtpmap:"+payloadType+" "+codecName+"/90000")
+ insertedRTPMap = true
+ }
+ }
+ return strings.Join(output, "\n")
+}
+
+func createPublisherOfferWithCodecs(t *testing.T, codecs []videoCodec) string {
t.Helper()
mediaEngine := &webrtc.MediaEngine{}
- videoRTCPFeedback := []webrtc.RTCPFeedback{
- {Type: "goog-remb"},
- {Type: "ccm", Parameter: "fir"},
- {Type: "nack"},
- {Type: "nack", Parameter: "pli"},
- }
- if err := mediaEngine.RegisterCodec(webrtc.RTPCodecParameters{
- RTPCodecCapability: webrtc.RTPCodecCapability{
- MimeType: webrtc.MimeTypeH264,
- ClockRate: 90000,
- SDPFmtpLine: fmtp,
- RTCPFeedback: videoRTCPFeedback,
- },
- PayloadType: 96,
- }, webrtc.RTPCodecTypeVideo); err != nil {
+ if err := registerVideoCodecs(mediaEngine); err != nil {
t.Fatal(err)
}
api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine))
@@ -645,20 +909,24 @@ func createPublisherOfferWithH264Fmtp(t *testing.T, fmtp string) string {
t.Fatal(err)
}
defer pc.Close()
- track, err := webrtc.NewTrackLocalStaticRTP(
- webrtc.RTPCodecCapability{
- MimeType: webrtc.MimeTypeH264,
- ClockRate: 90000,
- SDPFmtpLine: fmtp,
- },
- "screen",
- "voiddisplay",
- )
- if err != nil {
- t.Fatal(err)
- }
- if _, err := pc.AddTrack(track); err != nil {
- t.Fatal(err)
+ for _, codec := range codecs {
+ track, err := webrtc.NewTrackLocalStaticRTP(
+ mustTrackCapability(t, codec),
+ "screen-"+string(codec),
+ "voiddisplay",
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+ transceiver, err := pc.AddTransceiverFromTrack(track, webrtc.RTPTransceiverInit{
+ Direction: webrtc.RTPTransceiverDirectionSendonly,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := transceiver.SetCodecPreferences(mustCodecParameters(t, codec)); err != nil {
+ t.Fatal(err)
+ }
}
offer, err := pc.CreateOffer(nil)
if err != nil {
@@ -673,7 +941,7 @@ func createPublisherOfferWithH264Fmtp(t *testing.T, fmtp string) string {
if localDescription == nil {
t.Fatal("publisher local description missing")
}
- return localDescription.SDP
+ return filterVideoMediaCodecsForTest(t, localDescription.SDP, codecs)
}
func createPublisherPeerWithCodec(t *testing.T, mimeType string) (*webrtc.PeerConnection, *webrtc.TrackLocalStaticRTP) {
@@ -683,11 +951,17 @@ func createPublisherPeerWithCodec(t *testing.T, mimeType string) (*webrtc.PeerCo
var err error
switch mimeType {
case webrtc.MimeTypeH264:
- err = registerH264Codecs(mediaEngine)
- codec = h264TrackCapability()
+ err = registerH264CodecsForRejectedOfferTest(mediaEngine)
+ codec = webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264, ClockRate: 90000}
+ case webrtc.MimeTypeAV1:
+ err = registerCodecParametersForTest(mediaEngine, av1CodecParameters)
+ codec = mustTrackCapability(t, videoCodecAV1)
case webrtc.MimeTypeVP8:
err = registerVP8CodecsForTest(mediaEngine)
codec = webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8, ClockRate: 90000}
+ case webrtc.MimeTypeH265:
+ err = registerH265CodecsForTest(mediaEngine)
+ codec = webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH265, ClockRate: 90000}
default:
t.Fatalf("unsupported test codec: %s", mimeType)
}
@@ -726,21 +1000,91 @@ func createPublisherPeerWithCodec(t *testing.T, mimeType string) (*webrtc.PeerCo
return pc, track
}
-func createViewerOffer(t *testing.T) string {
+func createViewerOfferWithCodec(t *testing.T, codec videoCodec) string {
+ t.Helper()
+ return createViewerOfferWithCodecs(t, []videoCodec{codec})
+}
+
+func createViewerOfferWithCodecs(t *testing.T, codecs []videoCodec) string {
t.Helper()
mediaEngine := &webrtc.MediaEngine{}
- if err := mediaEngine.RegisterDefaultCodecs(); err != nil {
- t.Fatal(err)
+ codecParams := make([]webrtc.RTPCodecParameters, 0)
+ for _, codec := range codecs {
+ codecParams = append(codecParams, mustCodecParameters(t, codec)...)
+ }
+ for _, codec := range codecParams {
+ if err := mediaEngine.RegisterCodec(codec, webrtc.RTPCodecTypeVideo); err != nil {
+ t.Fatal(err)
+ }
+ }
+ return createViewerOfferWithCodecParameters(t, mediaEngine, codecParams)
+}
+
+func createViewerOfferWithMimeType(t *testing.T, mimeType string) string {
+ t.Helper()
+ mediaEngine := &webrtc.MediaEngine{}
+ var codecParams []webrtc.RTPCodecParameters
+ switch mimeType {
+ case webrtc.MimeTypeAV1:
+ codecParams = mustCodecParameters(t, videoCodecAV1)
+ case webrtc.MimeTypeH264:
+ if err := registerH264CodecsForRejectedOfferTest(mediaEngine); err != nil {
+ t.Fatal(err)
+ }
+ codecParams = []webrtc.RTPCodecParameters{{
+ RTPCodecCapability: webrtc.RTPCodecCapability{
+ MimeType: webrtc.MimeTypeH264,
+ ClockRate: 90000,
+ },
+ PayloadType: 96,
+ }}
+ case webrtc.MimeTypeVP8:
+ if err := registerVP8CodecsForTest(mediaEngine); err != nil {
+ t.Fatal(err)
+ }
+ codecParams = []webrtc.RTPCodecParameters{{
+ RTPCodecCapability: webrtc.RTPCodecCapability{
+ MimeType: webrtc.MimeTypeVP8,
+ ClockRate: 90000,
+ },
+ PayloadType: 96,
+ }}
+ case webrtc.MimeTypeH265:
+ if err := registerH265CodecsForTest(mediaEngine); err != nil {
+ t.Fatal(err)
+ }
+ codecParams = []webrtc.RTPCodecParameters{{
+ RTPCodecCapability: webrtc.RTPCodecCapability{
+ MimeType: webrtc.MimeTypeH265,
+ ClockRate: 90000,
+ },
+ PayloadType: 116,
+ }}
+ default:
+ t.Fatalf("unsupported viewer mime type: %s", mimeType)
}
+ return createViewerOfferWithCodecParameters(t, mediaEngine, codecParams)
+}
+
+func createViewerOfferWithCodecParameters(
+ t *testing.T,
+ mediaEngine *webrtc.MediaEngine,
+ codecParams []webrtc.RTPCodecParameters,
+) string {
+ t.Helper()
api := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine))
pc, err := api.NewPeerConnection(webrtc.Configuration{})
if err != nil {
t.Fatal(err)
}
defer pc.Close()
- if _, err = pc.AddTransceiverFromKind(webrtc.RTPCodecTypeVideo, webrtc.RTPTransceiverInit{
+ transceiver, err := pc.AddTransceiverFromKind(webrtc.RTPCodecTypeVideo, webrtc.RTPTransceiverInit{
Direction: webrtc.RTPTransceiverDirectionRecvonly,
- }); err != nil {
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := transceiver.SetCodecPreferences(codecParams); err != nil {
t.Fatal(err)
}
offer, err := pc.CreateOffer(nil)
@@ -759,6 +1103,110 @@ func createViewerOffer(t *testing.T) string {
return localDescription.SDP
}
+func filterVideoMediaCodecsForTest(t *testing.T, sdp string, targetCodecs []videoCodec) string {
+ t.Helper()
+ lines := strings.Split(sdp, "\n")
+ sections := make([][]string, 0)
+ current := make([]string, 0)
+ for _, line := range lines {
+ trimmed := strings.TrimSpace(line)
+ if strings.HasPrefix(trimmed, "m=") && len(current) > 0 {
+ sections = append(sections, current)
+ current = nil
+ }
+ current = append(current, line)
+ }
+ if len(current) > 0 {
+ sections = append(sections, current)
+ }
+
+ videoIndex := 0
+ filteredSections := make([]string, 0, len(sections))
+ for _, section := range sections {
+ if len(section) == 0 || !strings.HasPrefix(strings.TrimSpace(section[0]), "m=video ") {
+ filteredSections = append(filteredSections, strings.Join(section, "\n"))
+ continue
+ }
+ if videoIndex >= len(targetCodecs) {
+ t.Fatalf("SDP has more video media sections than target codecs:\n%s", sdp)
+ }
+ filteredSections = append(filteredSections, strings.Join(
+ filterVideoMediaSectionForTest(section, targetCodecs[videoIndex]),
+ "\n",
+ ))
+ videoIndex++
+ }
+ if videoIndex != len(targetCodecs) {
+ t.Fatalf("SDP video media sections = %d, want %d:\n%s", videoIndex, len(targetCodecs), sdp)
+ }
+ return strings.Join(filteredSections, "\n")
+}
+
+func filterVideoMediaSectionForTest(lines []string, targetCodec videoCodec) []string {
+ formats := strings.Fields(strings.TrimSpace(lines[0]))
+ payloadNames := make(map[string]string)
+ rtxApt := make(map[string]string)
+ for _, line := range lines[1:] {
+ trimmed := strings.TrimSpace(line)
+ if strings.HasPrefix(trimmed, "a=rtpmap:") {
+ parts := strings.Fields(strings.TrimPrefix(trimmed, "a=rtpmap:"))
+ if len(parts) >= 2 {
+ payloadNames[parts[0]] = strings.ToLower(strings.SplitN(parts[1], "/", 2)[0])
+ }
+ continue
+ }
+ if strings.HasPrefix(trimmed, "a=fmtp:") {
+ parts := strings.Fields(strings.TrimPrefix(trimmed, "a=fmtp:"))
+ if len(parts) >= 2 && strings.Contains(parts[1], "apt=") {
+ payloadType := parts[0]
+ for _, parameter := range strings.Split(parts[1], ";") {
+ parameter = strings.TrimSpace(parameter)
+ if strings.HasPrefix(parameter, "apt=") {
+ rtxApt[payloadType] = strings.TrimPrefix(parameter, "apt=")
+ }
+ }
+ }
+ }
+ }
+ keptPrimary := make(map[string]struct{})
+ keptPayloads := make(map[string]struct{})
+ for _, payloadType := range formats[3:] {
+ if payloadNames[payloadType] == string(targetCodec) {
+ keptPrimary[payloadType] = struct{}{}
+ keptPayloads[payloadType] = struct{}{}
+ }
+ }
+ for payloadType, apt := range rtxApt {
+ if _, ok := keptPrimary[apt]; ok {
+ keptPayloads[payloadType] = struct{}{}
+ }
+ }
+
+ filtered := make([]string, 0, len(lines))
+ mline := append([]string(nil), formats[:3]...)
+ for _, payloadType := range formats[3:] {
+ if _, ok := keptPayloads[payloadType]; ok {
+ mline = append(mline, payloadType)
+ }
+ }
+ filtered = append(filtered, strings.Join(mline, " "))
+ for _, line := range lines[1:] {
+ trimmed := strings.TrimSpace(line)
+ if strings.HasPrefix(trimmed, "a=rtpmap:") ||
+ strings.HasPrefix(trimmed, "a=fmtp:") ||
+ strings.HasPrefix(trimmed, "a=rtcp-fb:") {
+ payloadType := strings.Fields(strings.TrimPrefix(strings.TrimPrefix(strings.TrimPrefix(trimmed, "a=rtpmap:"), "a=fmtp:"), "a=rtcp-fb:"))
+ if len(payloadType) > 0 {
+ if _, ok := keptPayloads[payloadType[0]]; !ok {
+ continue
+ }
+ }
+ }
+ filtered = append(filtered, line)
+ }
+ return filtered
+}
+
func registerVP8CodecsForTest(mediaEngine *webrtc.MediaEngine) error {
videoRTCPFeedback := []webrtc.RTCPFeedback{
{Type: "goog-remb"},
@@ -791,40 +1239,179 @@ func registerVP8CodecsForTest(mediaEngine *webrtc.MediaEngine) error {
return nil
}
-func assertVideoSDPOnlyH264(t *testing.T, sdp string) {
+func registerCodecParametersForTest(mediaEngine *webrtc.MediaEngine, codecs []webrtc.RTPCodecParameters) error {
+ for _, codec := range codecs {
+ if err := mediaEngine.RegisterCodec(codec, webrtc.RTPCodecTypeVideo); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func registerH264CodecsForRejectedOfferTest(mediaEngine *webrtc.MediaEngine) error {
+ videoRTCPFeedback := []webrtc.RTCPFeedback{
+ {Type: "goog-remb"},
+ {Type: "ccm", Parameter: "fir"},
+ {Type: "nack"},
+ {Type: "nack", Parameter: "pli"},
+ }
+ for _, codec := range []webrtc.RTPCodecParameters{
+ {
+ RTPCodecCapability: webrtc.RTPCodecCapability{
+ MimeType: webrtc.MimeTypeH264,
+ ClockRate: 90000,
+ RTCPFeedback: videoRTCPFeedback,
+ },
+ PayloadType: 96,
+ },
+ {
+ RTPCodecCapability: webrtc.RTPCodecCapability{
+ MimeType: webrtc.MimeTypeRTX,
+ ClockRate: 90000,
+ SDPFmtpLine: "apt=96",
+ },
+ PayloadType: 97,
+ },
+ } {
+ if err := mediaEngine.RegisterCodec(codec, webrtc.RTPCodecTypeVideo); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func registerH265CodecsForTest(mediaEngine *webrtc.MediaEngine) error {
+ videoRTCPFeedback := []webrtc.RTCPFeedback{
+ {Type: "goog-remb"},
+ {Type: "ccm", Parameter: "fir"},
+ {Type: "nack"},
+ {Type: "nack", Parameter: "pli"},
+ }
+ for _, codec := range []webrtc.RTPCodecParameters{
+ {
+ RTPCodecCapability: webrtc.RTPCodecCapability{
+ MimeType: webrtc.MimeTypeH265,
+ ClockRate: 90000,
+ RTCPFeedback: videoRTCPFeedback,
+ },
+ PayloadType: 116,
+ },
+ {
+ RTPCodecCapability: webrtc.RTPCodecCapability{
+ MimeType: webrtc.MimeTypeRTX,
+ ClockRate: 90000,
+ SDPFmtpLine: "apt=116",
+ },
+ PayloadType: 117,
+ },
+ } {
+ if err := mediaEngine.RegisterCodec(codec, webrtc.RTPCodecTypeVideo); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func mustTrackCapability(t *testing.T, codec videoCodec) webrtc.RTPCodecCapability {
+ t.Helper()
+ capability, err := trackCapability(codec)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return capability
+}
+
+func mustCodecParameters(t *testing.T, codec videoCodec) []webrtc.RTPCodecParameters {
+ t.Helper()
+ codecs, err := codecParametersForVideoCodec(codec)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return codecs
+}
+
+func assertVideoSDPOnlyCodec(t *testing.T, sdp string, expected videoCodec) {
t.Helper()
+ assertNoVideoExtmap(t, sdp)
codecs := videoCodecNamesFromSDP(sdp)
if len(codecs) == 0 {
t.Fatalf("SDP video codecs empty:\n%s", sdp)
}
- hasH264 := false
+ hasExpected := false
for _, codec := range codecs {
switch strings.ToLower(codec) {
- case "h264":
- hasH264 = true
+ case string(expected):
+ hasExpected = true
case "rtx":
default:
- t.Fatalf("SDP contains non-H264 video codec %q in codecs %v:\n%s", codec, codecs, sdp)
+ t.Fatalf("SDP contains unexpected video codec %q in codecs %v:\n%s", codec, codecs, sdp)
}
}
- if !hasH264 {
- t.Fatalf("SDP did not contain H264 in video codecs %v:\n%s", codecs, sdp)
+ if !hasExpected {
+ t.Fatalf("SDP did not contain %s in video codecs %v:\n%s", expected, codecs, sdp)
+ }
+}
+
+func assertNoVideoExtmap(t *testing.T, sdp string) {
+ t.Helper()
+ inVideo := false
+ for _, line := range strings.Split(sdp, "\n") {
+ line = strings.TrimSpace(line)
+ if strings.HasPrefix(line, "m=") {
+ inVideo = strings.HasPrefix(line, "m=video ")
+ continue
+ }
+ if inVideo && strings.HasPrefix(line, "a=extmap:") {
+ t.Fatalf("SDP contains video extmap line %q:\n%s", line, sdp)
+ }
}
}
func videoCodecNamesFromSDP(sdp string) []string {
+ byMedia := videoCodecNamesByVideoMediaFromSDP(sdp)
+ if len(byMedia) == 0 {
+ return nil
+ }
+ return byMedia[len(byMedia)-1]
+}
+
+func videoCodecNamesFromAllVideoSDP(sdp string) []string {
+ byMedia := videoCodecNamesByVideoMediaFromSDP(sdp)
+ codecs := make([]string, 0)
+ for _, mediaCodecs := range byMedia {
+ codecs = append(codecs, mediaCodecs...)
+ }
+ return codecs
+}
+
+func videoCodecNamesByVideoMediaFromSDP(sdp string) [][]string {
+ var byMedia [][]string
var payloadTypes []string
payloadNames := make(map[string]string)
inVideo := false
+ flush := func() {
+ if !inVideo {
+ return
+ }
+ codecs := make([]string, 0, len(payloadTypes))
+ for _, payloadType := range payloadTypes {
+ if codecName, ok := payloadNames[payloadType]; ok {
+ codecs = append(codecs, codecName)
+ }
+ }
+ byMedia = append(byMedia, codecs)
+ }
for _, line := range strings.Split(sdp, "\n") {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "m=") {
+ flush()
inVideo = strings.HasPrefix(line, "m=video ")
if inVideo {
parts := strings.Fields(line)
if len(parts) > 3 {
- payloadTypes = append(payloadTypes[:0], parts[3:]...)
+ payloadTypes = append([]string(nil), parts[3:]...)
}
+ payloadNames = make(map[string]string)
}
continue
}
@@ -840,13 +1427,8 @@ func videoCodecNamesFromSDP(sdp string) []string {
codecName := strings.SplitN(parts[1], "/", 2)[0]
payloadNames[payloadType] = codecName
}
- codecs := make([]string, 0, len(payloadTypes))
- for _, payloadType := range payloadTypes {
- if codecName, ok := payloadNames[payloadType]; ok {
- codecs = append(codecs, codecName)
- }
- }
- return codecs
+ flush()
+ return byMedia
}
type capturingTrackLocalWriter struct {