Fix missing WebM recording duration metadata

packages/conversation-client/src/recording.ts

@@ -0,0 +1,11 @@
+/** Finalizes browser-recorded audio containers before they are uploaded. */
+import { fixWebmDuration } from "./vendor/fix-webm-duration/fix/lib/fixWebmDuration";
+
+/** Adds the duration metadata omitted from WebM files produced by MediaRecorder. */
+export async function addRecordingDurationMetadata(
+  recording: Blob,
+  durationMs: number,
+): Promise<Blob> {
+  if (!recording.type.toLowerCase().startsWith("audio/webm")) return recording;
+  return fixWebmDuration(recording, durationMs, { logger: false });
+}

packages/conversation-client/src/runtime.ts

@@ -15,6 +15,7 @@ import {
 import { Result } from "better-result";
 
 import { ConversationClientError, conversationClientError } from "./errors";
+import { addRecordingDurationMetadata } from "./recording";
 import type { ConversationApi, ConversationRuntime, RuntimeEvents, RuntimeFactory } from "./types";
 
 /** Maximum time to wait for the server to enter a shutdown state. */
@@ -36,6 +37,7 @@ class RealtimeConversationRuntime implements ConversationRuntime {
   private microphone: MediaStream | null = null;
   private audioContext: AudioContext | null = null;
   private recorder: MediaRecorder | null = null;
+  private recordingStartedAt: number | null = null;
   private readonly recordingChunks: Blob[] = [];
   private recordingUpload: RecordingUpload | null = null;
   private recordingFinalized = false;
@@ -108,6 +110,7 @@ class RealtimeConversationRuntime implements ConversationRuntime {
           if (event.data.size > 0) this.recordingChunks.push(event.data);
         });
         recorder.start(1_000);
+        this.recordingStartedAt = performance.now();
 
         const upload = await this.api.beginRecording(
           this.conversationId,
@@ -181,8 +184,9 @@ class RealtimeConversationRuntime implements ConversationRuntime {
 
     this.finalization = Result.tryPromise({
       try: async () => {
-        const recording = await this.stopRecorder();
+        const stopped = await this.stopRecorder();
         this.closeMedia();
+        const recording = await addRecordingDurationMetadata(stopped.blob, stopped.durationMs);
         const started = await this.api.beginRecordingUpload(this.conversationId, upload);
         if (!started.isOk()) throw started.error;
 
@@ -285,16 +289,27 @@ class RealtimeConversationRuntime implements ConversationRuntime {
     return ending;
   }
 
-  /** Stops the recorder and returns all accumulated mixed-audio data. */
-  private stopRecorder(): Promise<Blob> {
+  /** Stops the recorder and returns its data with the measured recording duration. */
+  private stopRecorder(): Promise<{ readonly blob: Blob; readonly durationMs: number }> {
     const recorder = this.recorder;
+    const durationMs =
+      this.recordingStartedAt === null
+        ? 0
+        : Math.max(0, performance.now() - this.recordingStartedAt);
     if (recorder === null || recorder.state === "inactive") {
-      return Promise.resolve(new Blob(this.recordingChunks, { type: recorder?.mimeType ?? "" }));
+      return Promise.resolve({
+        blob: new Blob(this.recordingChunks, { type: recorder?.mimeType ?? "" }),
+        durationMs,
+      });
     }
     return new Promise((resolve) => {
       recorder.addEventListener(
         "stop",
-        () => resolve(new Blob(this.recordingChunks, { type: recorder.mimeType })),
+        () =>
+          resolve({
+            blob: new Blob(this.recordingChunks, { type: recorder.mimeType }),
+            durationMs,
+          }),
         { once: true },
       );
       recorder.stop();

packages/conversation-client/src/vendor/fix-webm-duration/LICENSE

@@ -0,0 +1,21 @@
+The MIT license
+
+Copyright (c) 2018 Yury Sitnikov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

packages/conversation-client/src/vendor/fix-webm-duration/UPSTREAM.md

@@ -0,0 +1,20 @@
+# Vendored fix-webm-duration
+
+This directory contains the parser and duration-fixing source from
+[fix-webm-duration](https://github.com/yusitnikov/fix-webm-duration), vendored from upstream commit
+`7bbd85d3d42e29f3c0f7225588cf1b553309550b` under the MIT license in `LICENSE`.
+
+Applied upstream change:
+
+- PR [#28](https://github.com/yusitnikov/fix-webm-duration/pull/28), commit
+  `9124d811ef9eff237903227e2cbf425075d8a5b9`: parse section data with `Uint8Array.subarray()`
+  instead of copying it with `slice()`.
+
+Local adaptations:
+
+- Package imports use relative paths, and unused package entry barrels are omitted.
+- Float parsing uses `DataView` rather than reversing section bytes in place. This is required after
+  PR #28 because parsed sections are views into the original recording buffer.
+- Parser failures return the original Blob and are reported when a logger is provided.
+- Source formatting follows this repository's formatter.
+- The demo and React viewer packages are not included.

packages/conversation-client/src/vendor/fix-webm-duration/fix/lib/fixParsedWebmDuration.ts

@@ -0,0 +1,68 @@
+import type { Options } from "../../parser/lib/Options";
+import { WebmFile } from "../../parser/lib/WebmFile";
+import { WebmFloat } from "../../parser/lib/WebmFloat";
+
+export const fixParsedWebmDuration = (
+  file: WebmFile,
+  duration: number,
+  options: Options = {},
+): boolean => {
+  let logger = options.logger;
+  if (logger === undefined) {
+    logger = (message) => console.debug(message);
+  } else if (!logger) {
+    logger = () => {
+      // NOOP
+    };
+  }
+
+  const segmentSection = file.getSectionById(0x8538067);
+  if (!segmentSection) {
+    logger("[fix-webm-duration] Segment section is missing");
+    return false;
+  }
+
+  const infoSection = segmentSection.getSectionById(0x549a966);
+  if (!infoSection) {
+    logger("[fix-webm-duration] Info section is missing");
+    return false;
+  }
+
+  const timeScaleSection = infoSection.getSectionById(0xad7b1);
+  if (!timeScaleSection) {
+    logger("[fix-webm-duration] TimecodeScale section is missing");
+    return false;
+  }
+
+  let durationSection = infoSection.getSectionById(0x489);
+  if (durationSection) {
+    if (durationSection.getValue() <= 0) {
+      logger(
+        `[fix-webm-duration] Duration section is present, but the value is ${durationSection.getValue()}`,
+      );
+      durationSection.setValue(duration);
+    } else {
+      logger(
+        `[fix-webm-duration] Duration section is present, and the value is ${durationSection.getValue()}`,
+      );
+      return false;
+    }
+  } else {
+    logger("[fix-webm-duration] Duration section is missing");
+    // append Duration section
+    durationSection = new WebmFloat("Duration");
+    durationSection.setValue(duration);
+    infoSection.data!.push({
+      id: 0x489,
+      data: durationSection,
+    });
+  }
+
+  // set default time scale to 1 millisecond (1000000 nanoseconds)
+  timeScaleSection.setValue(1000000);
+  infoSection.updateByData();
+  segmentSection.updateByData();
+  file.updateByData();
+
+  return true;
+};

packages/conversation-client/src/vendor/fix-webm-duration/fix/lib/fixWebmDuration.ts

@@ -0,0 +1,24 @@
+/* oxlint-disable eslint-js/no-restricted-syntax -- Invalid media returns the original Blob. */
+import type { Options } from "../../parser/lib/Options";
+import { WebmFile } from "../../parser/lib/WebmFile";
+import { fixParsedWebmDuration } from "./fixParsedWebmDuration";
+
+export const fixWebmDuration = async (
+  blob: Blob,
+  duration: number,
+  options?: Options,
+): Promise<Blob> => {
+  try {
+    const file = await WebmFile.fromBlob(blob);
+    if (fixParsedWebmDuration(file, duration, options)) {
+      return file.toBlob(blob.type);
+    }
+  } catch (cause) {
+    if (options?.logger) {
+      const message = cause instanceof Error ? cause.message : "Unknown parser failure";
+      options.logger(`[fix-webm-duration] ${message}`);
+    }
+  }
+
+  return blob;
+};

packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/Options.ts

@@ -0,0 +1,5 @@
+export type LoggerCallback = (message: string) => void;
+
+export interface Options {
+  logger?: LoggerCallback | false;
+}

packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/SectionType.ts

@@ -0,0 +1,9 @@
+export enum SectionType {
+  Container = "Container",
+  Uint = "Uint",
+  Int = "Int",
+  Float = "Float",
+  String = "String",
+  Date = "Date",
+  Binary = "Binary",
+}

packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/WebmBase.ts

@@ -0,0 +1,39 @@
+export class WebmBase<DataT, ValueT> {
+  public source: Uint8Array | undefined;
+  public data: DataT | undefined;
+
+  protected constructor(
+    public name = "Unknown",
+    public start = 0,
+  ) {}
+
+  getType() {
+    return "Unknown";
+  }
+
+  updateBySource() {
+    // NOOP
+  }
+
+  setSource(source: Uint8Array) {
+    this.source = source;
+    this.updateBySource();
+  }
+
+  updateByData() {
+    // NOOP
+  }
+
+  setData(data: DataT) {
+    this.data = data;
+    this.updateByData();
+  }
+
+  getValue(): ValueT {
+    return this.data as unknown as ValueT;
+  }
+
+  setValue(value: ValueT) {
+    this.setData(value as unknown as DataT);
+  }
+}