diff --git a/packages/conversation-client/src/recording.ts b/packages/conversation-client/src/recording.ts new file mode 100644 index 0000000..4356993 --- /dev/null +++ b/packages/conversation-client/src/recording.ts @@ -0,0 +1,11 @@ +/** Finalizes browser-recorded audio containers before they are uploaded. */ +import { fixWebmDuration } from "./vendor/fix-webm-duration/fix/lib/fixWebmDuration"; + +/** Adds the duration metadata omitted from WebM files produced by MediaRecorder. */ +export async function addRecordingDurationMetadata( + recording: Blob, + durationMs: number, +): Promise { + if (!recording.type.toLowerCase().startsWith("audio/webm")) return recording; + return fixWebmDuration(recording, durationMs, { logger: false }); +} diff --git a/packages/conversation-client/src/runtime.ts b/packages/conversation-client/src/runtime.ts index 007b843..3d21028 100644 --- a/packages/conversation-client/src/runtime.ts +++ b/packages/conversation-client/src/runtime.ts @@ -15,6 +15,7 @@ import { import { Result } from "better-result"; import { ConversationClientError, conversationClientError } from "./errors"; +import { addRecordingDurationMetadata } from "./recording"; import type { ConversationApi, ConversationRuntime, RuntimeEvents, RuntimeFactory } from "./types"; /** Maximum time to wait for the server to enter a shutdown state. */ @@ -36,6 +37,7 @@ class RealtimeConversationRuntime implements ConversationRuntime { private microphone: MediaStream | null = null; private audioContext: AudioContext | null = null; private recorder: MediaRecorder | null = null; + private recordingStartedAt: number | null = null; private readonly recordingChunks: Blob[] = []; private recordingUpload: RecordingUpload | null = null; private recordingFinalized = false; @@ -108,6 +110,7 @@ class RealtimeConversationRuntime implements ConversationRuntime { if (event.data.size > 0) this.recordingChunks.push(event.data); }); recorder.start(1_000); + this.recordingStartedAt = performance.now(); const upload = await this.api.beginRecording( this.conversationId, @@ -181,8 +184,9 @@ class RealtimeConversationRuntime implements ConversationRuntime { this.finalization = Result.tryPromise({ try: async () => { - const recording = await this.stopRecorder(); + const stopped = await this.stopRecorder(); this.closeMedia(); + const recording = await addRecordingDurationMetadata(stopped.blob, stopped.durationMs); const started = await this.api.beginRecordingUpload(this.conversationId, upload); if (!started.isOk()) throw started.error; @@ -285,16 +289,27 @@ class RealtimeConversationRuntime implements ConversationRuntime { return ending; } - /** Stops the recorder and returns all accumulated mixed-audio data. */ - private stopRecorder(): Promise { + /** Stops the recorder and returns its data with the measured recording duration. */ + private stopRecorder(): Promise<{ readonly blob: Blob; readonly durationMs: number }> { const recorder = this.recorder; + const durationMs = + this.recordingStartedAt === null + ? 0 + : Math.max(0, performance.now() - this.recordingStartedAt); if (recorder === null || recorder.state === "inactive") { - return Promise.resolve(new Blob(this.recordingChunks, { type: recorder?.mimeType ?? "" })); + return Promise.resolve({ + blob: new Blob(this.recordingChunks, { type: recorder?.mimeType ?? "" }), + durationMs, + }); } return new Promise((resolve) => { recorder.addEventListener( "stop", - () => resolve(new Blob(this.recordingChunks, { type: recorder.mimeType })), + () => + resolve({ + blob: new Blob(this.recordingChunks, { type: recorder.mimeType }), + durationMs, + }), { once: true }, ); recorder.stop(); diff --git a/packages/conversation-client/src/vendor/fix-webm-duration/LICENSE b/packages/conversation-client/src/vendor/fix-webm-duration/LICENSE new file mode 100644 index 0000000..755aa25 --- /dev/null +++ b/packages/conversation-client/src/vendor/fix-webm-duration/LICENSE @@ -0,0 +1,21 @@ +The MIT license + +Copyright (c) 2018 Yury Sitnikov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/conversation-client/src/vendor/fix-webm-duration/UPSTREAM.md b/packages/conversation-client/src/vendor/fix-webm-duration/UPSTREAM.md new file mode 100644 index 0000000..a29a351 --- /dev/null +++ b/packages/conversation-client/src/vendor/fix-webm-duration/UPSTREAM.md @@ -0,0 +1,20 @@ +# Vendored fix-webm-duration + +This directory contains the parser and duration-fixing source from +[fix-webm-duration](https://github.com/yusitnikov/fix-webm-duration), vendored from upstream commit +`7bbd85d3d42e29f3c0f7225588cf1b553309550b` under the MIT license in `LICENSE`. + +Applied upstream change: + +- PR [#28](https://github.com/yusitnikov/fix-webm-duration/pull/28), commit + `9124d811ef9eff237903227e2cbf425075d8a5b9`: parse section data with `Uint8Array.subarray()` + instead of copying it with `slice()`. + +Local adaptations: + +- Package imports use relative paths, and unused package entry barrels are omitted. +- Float parsing uses `DataView` rather than reversing section bytes in place. This is required after + PR #28 because parsed sections are views into the original recording buffer. +- Parser failures return the original Blob and are reported when a logger is provided. +- Source formatting follows this repository's formatter. +- The demo and React viewer packages are not included. diff --git a/packages/conversation-client/src/vendor/fix-webm-duration/fix/lib/fixParsedWebmDuration.ts b/packages/conversation-client/src/vendor/fix-webm-duration/fix/lib/fixParsedWebmDuration.ts new file mode 100644 index 0000000..55f445a --- /dev/null +++ b/packages/conversation-client/src/vendor/fix-webm-duration/fix/lib/fixParsedWebmDuration.ts @@ -0,0 +1,68 @@ +import type { Options } from "../../parser/lib/Options"; +import { WebmFile } from "../../parser/lib/WebmFile"; +import { WebmFloat } from "../../parser/lib/WebmFloat"; + +export const fixParsedWebmDuration = ( + file: WebmFile, + duration: number, + options: Options = {}, +): boolean => { + let logger = options.logger; + if (logger === undefined) { + logger = (message) => console.debug(message); + } else if (!logger) { + logger = () => { + // NOOP + }; + } + + const segmentSection = file.getSectionById(0x8538067); + if (!segmentSection) { + logger("[fix-webm-duration] Segment section is missing"); + return false; + } + + const infoSection = segmentSection.getSectionById(0x549a966); + if (!infoSection) { + logger("[fix-webm-duration] Info section is missing"); + return false; + } + + const timeScaleSection = infoSection.getSectionById(0xad7b1); + if (!timeScaleSection) { + logger("[fix-webm-duration] TimecodeScale section is missing"); + return false; + } + + let durationSection = infoSection.getSectionById(0x489); + if (durationSection) { + if (durationSection.getValue() <= 0) { + logger( + `[fix-webm-duration] Duration section is present, but the value is ${durationSection.getValue()}`, + ); + durationSection.setValue(duration); + } else { + logger( + `[fix-webm-duration] Duration section is present, and the value is ${durationSection.getValue()}`, + ); + return false; + } + } else { + logger("[fix-webm-duration] Duration section is missing"); + // append Duration section + durationSection = new WebmFloat("Duration"); + durationSection.setValue(duration); + infoSection.data!.push({ + id: 0x489, + data: durationSection, + }); + } + + // set default time scale to 1 millisecond (1000000 nanoseconds) + timeScaleSection.setValue(1000000); + infoSection.updateByData(); + segmentSection.updateByData(); + file.updateByData(); + + return true; +}; diff --git a/packages/conversation-client/src/vendor/fix-webm-duration/fix/lib/fixWebmDuration.ts b/packages/conversation-client/src/vendor/fix-webm-duration/fix/lib/fixWebmDuration.ts new file mode 100644 index 0000000..7a2b8d4 --- /dev/null +++ b/packages/conversation-client/src/vendor/fix-webm-duration/fix/lib/fixWebmDuration.ts @@ -0,0 +1,24 @@ +/* oxlint-disable eslint-js/no-restricted-syntax -- Invalid media returns the original Blob. */ +import type { Options } from "../../parser/lib/Options"; +import { WebmFile } from "../../parser/lib/WebmFile"; +import { fixParsedWebmDuration } from "./fixParsedWebmDuration"; + +export const fixWebmDuration = async ( + blob: Blob, + duration: number, + options?: Options, +): Promise => { + try { + const file = await WebmFile.fromBlob(blob); + if (fixParsedWebmDuration(file, duration, options)) { + return file.toBlob(blob.type); + } + } catch (cause) { + if (options?.logger) { + const message = cause instanceof Error ? cause.message : "Unknown parser failure"; + options.logger(`[fix-webm-duration] ${message}`); + } + } + + return blob; +}; diff --git a/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/Options.ts b/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/Options.ts new file mode 100644 index 0000000..fdf8435 --- /dev/null +++ b/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/Options.ts @@ -0,0 +1,5 @@ +export type LoggerCallback = (message: string) => void; + +export interface Options { + logger?: LoggerCallback | false; +} diff --git a/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/SectionType.ts b/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/SectionType.ts new file mode 100644 index 0000000..af9d75c --- /dev/null +++ b/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/SectionType.ts @@ -0,0 +1,9 @@ +export enum SectionType { + Container = "Container", + Uint = "Uint", + Int = "Int", + Float = "Float", + String = "String", + Date = "Date", + Binary = "Binary", +} diff --git a/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/WebmBase.ts b/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/WebmBase.ts new file mode 100644 index 0000000..4995c03 --- /dev/null +++ b/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/WebmBase.ts @@ -0,0 +1,39 @@ +export class WebmBase { + public source: Uint8Array | undefined; + public data: DataT | undefined; + + protected constructor( + public name = "Unknown", + public start = 0, + ) {} + + getType() { + return "Unknown"; + } + + updateBySource() { + // NOOP + } + + setSource(source: Uint8Array) { + this.source = source; + this.updateBySource(); + } + + updateByData() { + // NOOP + } + + setData(data: DataT) { + this.data = data; + this.updateByData(); + } + + getValue(): ValueT { + return this.data as unknown as ValueT; + } + + setValue(value: ValueT) { + this.setData(value as unknown as DataT); + } +} diff --git a/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/WebmContainer.ts b/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/WebmContainer.ts new file mode 100644 index 0000000..8522175 --- /dev/null +++ b/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/WebmContainer.ts @@ -0,0 +1,150 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { WebmBase } from "./WebmBase"; +import { sections, SectionKey, SectionsMap } from "./sections"; +import { SectionType } from "./SectionType"; +import { WebmUint } from "./WebmUint"; +import { WebmFloat } from "./WebmFloat"; +import { WebmString } from "./WebmString"; + +type SectionTypeMap = { + [SectionType.Container]: WebmContainer; + [SectionType.Uint]: WebmUint; + [SectionType.Float]: WebmFloat; + [SectionType.String]: WebmString; +}; +type TypeBySectionKey = + SectionsMap[IdType]["type"] extends keyof SectionTypeMap + ? SectionTypeMap[SectionsMap[IdType]["type"]] + : WebmBase; + +export interface WebmContainerItem { + id: IdType; + idHex?: string; + data: TypeBySectionKey; +} + +export class WebmContainer extends WebmBase { + public offset = 0; + + constructor( + name?: string, + public isInfinite = false, + start = 0, + ) { + super(name, start); + } + + override getType() { + return "Container"; + } + + readByte() { + return this.source![this.offset++]!; + } + + readUint() { + const firstByte = this.readByte(); + const bytes = 8 - firstByte.toString(2).length; + let value = firstByte - (1 << (7 - bytes)); + for (let i = 0; i < bytes; i++) { + value <<= 8; + value |= this.readByte(); + } + return value; + } + + override updateBySource() { + this.data = []; + let end: number; + for (this.offset = 0; this.offset < this.source!.length; this.offset = end) { + const start = this.offset; + + const id = this.readUint() as SectionKey; + const { name, type } = sections[id] ?? {}; + + const len = this.readUint(); + + end = this.source!.length; + if (len >= 0) end = Math.min(this.offset + len, end); + + const data = this.source!.subarray(this.offset, end); + + let section: WebmBase; + switch (type) { + case SectionType.Container: + section = new WebmContainer(name, len < 0, start); + break; + case SectionType.Uint: + section = new WebmUint(name, start); + break; + case SectionType.Float: + section = new WebmFloat(name, start); + break; + case SectionType.String: + section = new WebmString(name, start); + break; + default: + section = new WebmBase(name, start); + break; + } + section.setSource(data); + this.data.push({ + id, + idHex: id.toString(16), + data: section, + }); + } + } + + writeUint(x: number, draft = false) { + let bytes = 1; + // oxlint-disable-next-line eslint/no-unmodified-loop-condition -- bytes changes the bound + while ((x < 0 || x >= 2 ** (7 * bytes)) && bytes < 8) bytes++; + + if (!draft) { + for (let i = 0; i < bytes; i++) { + this.source![this.offset + i] = (x >> (8 * (bytes - 1 - i))) & 0xff; + } + + const firstByte = this.source![this.offset]!; + this.source![this.offset] = (firstByte & ((1 << (8 - bytes)) - 1)) | (1 << (8 - bytes)); + } + + this.offset += bytes; + } + + writeSections(draft = false) { + this.offset = 0; + for (const section of this.data!) { + const content = section.data.source!; + const contentLength = content.length; + this.writeUint(section.id, draft); + this.writeUint( + section.data instanceof WebmContainer && section.data.isInfinite ? -1 : contentLength, + draft, + ); + if (!draft) { + this.source!.set(content, this.offset); + } + this.offset += contentLength; + } + return this.offset; + } + + override updateByData() { + // run without accessing this.source to determine total length - need to know it to create Uint8Array + const length = this.writeSections(true); + this.source = new Uint8Array(length); + // now really write data + this.writeSections(); + } + + getSectionById(id: IdType): TypeBySectionKey | null { + for (const section of this.data!) { + if (section.id === id) { + return section.data as unknown as TypeBySectionKey; + } + } + return null; + } +} diff --git a/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/WebmFile.ts b/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/WebmFile.ts new file mode 100644 index 0000000..a491307 --- /dev/null +++ b/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/WebmFile.ts @@ -0,0 +1,20 @@ +import { WebmContainer } from "./WebmContainer"; + +export class WebmFile extends WebmContainer { + constructor(source: Uint8Array) { + super("File"); + this.setSource(source); + } + + override getType() { + return "File"; + } + + toBlob(mimeType = "video/webm") { + return new Blob([this.source!.buffer as ArrayBuffer], { type: mimeType }); + } + + static async fromBlob(blob: Blob) { + return new WebmFile(new Uint8Array(await blob.arrayBuffer())); + } +} diff --git a/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/WebmFloat.ts b/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/WebmFloat.ts new file mode 100644 index 0000000..47bff39 --- /dev/null +++ b/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/WebmFloat.ts @@ -0,0 +1,25 @@ +import { WebmBase } from "./WebmBase"; + +export class WebmFloat extends WebmBase { + constructor(name?: string, start = 0) { + super(name, start); + } + + override getType() { + return "Float"; + } + + override updateBySource() { + const source = this.source!; + const view = new DataView(source.buffer, source.byteOffset, source.byteLength); + this.data = source.byteLength === 4 ? view.getFloat32(0) : view.getFloat64(0); + } + + override updateByData() { + const byteLength = this.source?.byteLength === 4 ? 4 : 8; + this.source = new Uint8Array(byteLength); + const view = new DataView(this.source.buffer); + if (byteLength === 4) view.setFloat32(0, this.data!); + else view.setFloat64(0, this.data!); + } +} diff --git a/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/WebmString.ts b/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/WebmString.ts new file mode 100644 index 0000000..2e2dfa4 --- /dev/null +++ b/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/WebmString.ts @@ -0,0 +1,27 @@ +import { WebmBase } from "./WebmBase"; + +export class WebmString extends WebmBase { + constructor(name?: string, start = 0) { + super(name, start); + } + + override getType() { + return "String"; + } + + override updateBySource() { + this.data = this.source; + } + + override updateByData() { + this.source = this.data; + } + + override getValue(): string { + let result = ""; + this.source!.forEach((code) => { + result += String.fromCharCode(code); + }); + return result; + } +} diff --git a/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/WebmUint.ts b/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/WebmUint.ts new file mode 100644 index 0000000..2b135bb --- /dev/null +++ b/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/WebmUint.ts @@ -0,0 +1,41 @@ +import { WebmBase } from "./WebmBase"; + +function padHex(hex: string) { + return hex.length % 2 === 1 ? "0" + hex : hex; +} + +export class WebmUint extends WebmBase { + constructor(name?: string, start = 0) { + super(name, start); + } + + override getType() { + return "Uint"; + } + + override updateBySource() { + // use hex representation of a number instead of number value + this.data = ""; + for (let i = 0; i < this.source!.length; i++) { + const hex = this.source![i]!.toString(16); + this.data += padHex(hex); + } + } + + override updateByData() { + const length = this.data!.length / 2; + this.source = new Uint8Array(length); + for (let i = 0; i < length; i++) { + const hex = this.data!.substring(i * 2, i * 2 + 2); + this.source[i] = parseInt(hex, 16); + } + } + + override getValue() { + return parseInt(this.data!, 16); + } + + override setValue(value: number) { + this.setData(padHex(value.toString(16))); + } +} diff --git a/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/sections.ts b/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/sections.ts new file mode 100644 index 0000000..53a9016 --- /dev/null +++ b/packages/conversation-client/src/vendor/fix-webm-duration/parser/lib/sections.ts @@ -0,0 +1,254 @@ +import { SectionType } from "./SectionType"; + +/* + * This is the list of possible WEBM file sections by their IDs. + * + * See: + * - https://github.com/MediaArea/MediaInfoLib/blob/master/Source/MediaInfo/Multiple/File_Mk.cpp + * - https://datatracker.ietf.org/doc/draft-ietf-cellar-matroska/ + */ +export const sections = { + 0xa45dfa3: { name: "EBML", type: SectionType.Container }, + 0x286: { name: "EBMLVersion", type: SectionType.Uint }, + 0x2f7: { name: "EBMLReadVersion", type: SectionType.Uint }, + 0x2f2: { name: "EBMLMaxIDLength", type: SectionType.Uint }, + 0x2f3: { name: "EBMLMaxSizeLength", type: SectionType.Uint }, + 0x282: { name: "DocType", type: SectionType.String }, + 0x287: { name: "DocTypeVersion", type: SectionType.Uint }, + 0x285: { name: "DocTypeReadVersion", type: SectionType.Uint }, + 0x6c: { name: "Void", type: SectionType.Binary }, + 0x3f: { name: "CRC-32", type: SectionType.Binary }, + 0xb538667: { name: "SignatureSlot", type: SectionType.Container }, + 0x3e8a: { name: "SignatureAlgo", type: SectionType.Uint }, + 0x3e9a: { name: "SignatureHash", type: SectionType.Uint }, + 0x3ea5: { name: "SignaturePublicKey", type: SectionType.Binary }, + 0x3eb5: { name: "Signature", type: SectionType.Binary }, + 0x3e5b: { name: "SignatureElements", type: SectionType.Container }, + 0x3e7b: { name: "SignatureElementList", type: SectionType.Container }, + 0x2532: { name: "SignedElement", type: SectionType.Binary }, + 0x8538067: { name: "Segment", type: SectionType.Container }, + 0x14d9b74: { name: "SeekHead", type: SectionType.Container }, + 0xdbb: { name: "Seek", type: SectionType.Container }, + 0x13ab: { name: "SeekID", type: SectionType.Binary }, + 0x13ac: { name: "SeekPosition", type: SectionType.Uint }, + 0x549a966: { name: "Info", type: SectionType.Container }, + 0x33a4: { name: "SegmentUID", type: SectionType.Binary }, + 0x3384: { name: "SegmentFilename", type: SectionType.String }, + 0x1cb923: { name: "PrevUID", type: SectionType.Binary }, + 0x1c83ab: { name: "PrevFilename", type: SectionType.String }, + 0x1eb923: { name: "NextUID", type: SectionType.Binary }, + 0x1e83bb: { name: "NextFilename", type: SectionType.String }, + 0x444: { name: "SegmentFamily", type: SectionType.Binary }, + 0x2924: { name: "ChapterTranslate", type: SectionType.Container }, + 0x29fc: { name: "ChapterTranslateEditionUID", type: SectionType.Uint }, + 0x29bf: { name: "ChapterTranslateCodec", type: SectionType.Uint }, + 0x29a5: { name: "ChapterTranslateID", type: SectionType.Binary }, + 0xad7b1: { name: "TimecodeScale", type: SectionType.Uint }, + 0x489: { name: "Duration", type: SectionType.Float }, + 0x461: { name: "DateUTC", type: SectionType.Date }, + 0x3ba9: { name: "Title", type: SectionType.String }, + 0xd80: { name: "MuxingApp", type: SectionType.String }, + 0x1741: { name: "WritingApp", type: SectionType.String }, + // Intentionally don't recognize the Cluster section because it's large + // 0xf43b675: { name: "Cluster", type: SectionType.Container }, + 0x67: { name: "Timecode", type: SectionType.Uint }, + 0x1854: { name: "SilentTracks", type: SectionType.Container }, + 0x18d7: { name: "SilentTrackNumber", type: SectionType.Uint }, + 0x27: { name: "Position", type: SectionType.Uint }, + 0x2b: { name: "PrevSize", type: SectionType.Uint }, + 0x23: { name: "SimpleBlock", type: SectionType.Binary }, + 0x20: { name: "BlockGroup", type: SectionType.Container }, + 0x21: { name: "Block", type: SectionType.Binary }, + 0x22: { name: "BlockVirtual", type: SectionType.Binary }, + 0x35a1: { name: "BlockAdditions", type: SectionType.Container }, + 0x26: { name: "BlockMore", type: SectionType.Container }, + 0x6e: { name: "BlockAddID", type: SectionType.Uint }, + 0x25: { name: "BlockAdditional", type: SectionType.Binary }, + 0x1b: { name: "BlockDuration", type: SectionType.Uint }, + 0x7a: { name: "ReferencePriority", type: SectionType.Uint }, + 0x7b: { name: "ReferenceBlock", type: SectionType.Int }, + 0x7d: { name: "ReferenceVirtual", type: SectionType.Int }, + 0x24: { name: "CodecState", type: SectionType.Binary }, + 0x35a2: { name: "DiscardPadding", type: SectionType.Int }, + 0xe: { name: "Slices", type: SectionType.Container }, + 0x68: { name: "TimeSlice", type: SectionType.Container }, + 0x4c: { name: "LaceNumber", type: SectionType.Uint }, + 0x4d: { name: "FrameNumber", type: SectionType.Uint }, + 0x4b: { name: "BlockAdditionID", type: SectionType.Uint }, + 0x4e: { name: "Delay", type: SectionType.Uint }, + 0x4f: { name: "SliceDuration", type: SectionType.Uint }, + 0x48: { name: "ReferenceFrame", type: SectionType.Container }, + 0x49: { name: "ReferenceOffset", type: SectionType.Uint }, + 0x4a: { name: "ReferenceTimeCode", type: SectionType.Uint }, + 0x2f: { name: "EncryptedBlock", type: SectionType.Binary }, + 0x654ae6b: { name: "Tracks", type: SectionType.Container }, + 0x2e: { name: "TrackEntry", type: SectionType.Container }, + 0x57: { name: "TrackNumber", type: SectionType.Uint }, + 0x33c5: { name: "TrackUID", type: SectionType.Uint }, + 0x3: { name: "TrackType", type: SectionType.Uint }, + 0x39: { name: "FlagEnabled", type: SectionType.Uint }, + 0x8: { name: "FlagDefault", type: SectionType.Uint }, + 0x15aa: { name: "FlagForced", type: SectionType.Uint }, + 0x1c: { name: "FlagLacing", type: SectionType.Uint }, + 0x2de7: { name: "MinCache", type: SectionType.Uint }, + 0x2df8: { name: "MaxCache", type: SectionType.Uint }, + 0x3e383: { name: "DefaultDuration", type: SectionType.Uint }, + 0x34e7a: { name: "DefaultDecodedFieldDuration", type: SectionType.Uint }, + 0x3314f: { name: "TrackTimecodeScale", type: SectionType.Float }, + 0x137f: { name: "TrackOffset", type: SectionType.Int }, + 0x15ee: { name: "MaxBlockAdditionID", type: SectionType.Uint }, + 0x136e: { name: "Name", type: SectionType.String }, + 0x2b59c: { name: "Language", type: SectionType.String }, + 0x6: { name: "CodecID", type: SectionType.String }, + 0x23a2: { name: "CodecPrivate", type: SectionType.Binary }, + 0x58688: { name: "CodecName", type: SectionType.String }, + 0x3446: { name: "AttachmentLink", type: SectionType.Uint }, + 0x1a9697: { name: "CodecSettings", type: SectionType.String }, + 0x1b4040: { name: "CodecInfoURL", type: SectionType.String }, + 0x6b240: { name: "CodecDownloadURL", type: SectionType.String }, + 0x2a: { name: "CodecDecodeAll", type: SectionType.Uint }, + 0x2fab: { name: "TrackOverlay", type: SectionType.Uint }, + 0x16aa: { name: "CodecDelay", type: SectionType.Uint }, + 0x16bb: { name: "SeekPreRoll", type: SectionType.Uint }, + 0x2624: { name: "TrackTranslate", type: SectionType.Container }, + 0x26fc: { name: "TrackTranslateEditionUID", type: SectionType.Uint }, + 0x26bf: { name: "TrackTranslateCodec", type: SectionType.Uint }, + 0x26a5: { name: "TrackTranslateTrackID", type: SectionType.Binary }, + 0x60: { name: "Video", type: SectionType.Container }, + 0x1a: { name: "FlagInterlaced", type: SectionType.Uint }, + 0x13b8: { name: "StereoMode", type: SectionType.Uint }, + 0x13c0: { name: "AlphaMode", type: SectionType.Uint }, + 0x13b9: { name: "OldStereoMode", type: SectionType.Uint }, + 0x30: { name: "PixelWidth", type: SectionType.Uint }, + 0x3a: { name: "PixelHeight", type: SectionType.Uint }, + 0x14aa: { name: "PixelCropBottom", type: SectionType.Uint }, + 0x14bb: { name: "PixelCropTop", type: SectionType.Uint }, + 0x14cc: { name: "PixelCropLeft", type: SectionType.Uint }, + 0x14dd: { name: "PixelCropRight", type: SectionType.Uint }, + 0x14b0: { name: "DisplayWidth", type: SectionType.Uint }, + 0x14ba: { name: "DisplayHeight", type: SectionType.Uint }, + 0x14b2: { name: "DisplayUnit", type: SectionType.Uint }, + 0x14b3: { name: "AspectRatioType", type: SectionType.Uint }, + 0xeb524: { name: "ColourSpace", type: SectionType.Binary }, + 0xfb523: { name: "GammaValue", type: SectionType.Float }, + 0x383e3: { name: "FrameRate", type: SectionType.Float }, + 0x61: { name: "Audio", type: SectionType.Container }, + 0x35: { name: "SamplingFrequency", type: SectionType.Float }, + 0x38b5: { name: "OutputSamplingFrequency", type: SectionType.Float }, + 0x1f: { name: "Channels", type: SectionType.Uint }, + 0x3d7b: { name: "ChannelPositions", type: SectionType.Binary }, + 0x2264: { name: "BitDepth", type: SectionType.Uint }, + 0x62: { name: "TrackOperation", type: SectionType.Container }, + 0x63: { name: "TrackCombinePlanes", type: SectionType.Container }, + 0x64: { name: "TrackPlane", type: SectionType.Container }, + 0x65: { name: "TrackPlaneUID", type: SectionType.Uint }, + 0x66: { name: "TrackPlaneType", type: SectionType.Uint }, + 0x69: { name: "TrackJoinBlocks", type: SectionType.Container }, + 0x6d: { name: "TrackJoinUID", type: SectionType.Uint }, + 0x40: { name: "TrickTrackUID", type: SectionType.Uint }, + 0x41: { name: "TrickTrackSegmentUID", type: SectionType.Binary }, + 0x46: { name: "TrickTrackFlag", type: SectionType.Uint }, + 0x47: { name: "TrickMasterTrackUID", type: SectionType.Uint }, + 0x44: { name: "TrickMasterTrackSegmentUID", type: SectionType.Binary }, + 0x2d80: { name: "ContentEncodings", type: SectionType.Container }, + 0x2240: { name: "ContentEncoding", type: SectionType.Container }, + 0x1031: { name: "ContentEncodingOrder", type: SectionType.Uint }, + 0x1032: { name: "ContentEncodingScope", type: SectionType.Uint }, + 0x1033: { name: "ContentEncodingType", type: SectionType.Uint }, + 0x1034: { name: "ContentCompression", type: SectionType.Container }, + 0x254: { name: "ContentCompAlgo", type: SectionType.Uint }, + 0x255: { name: "ContentCompSettings", type: SectionType.Binary }, + 0x1035: { name: "ContentEncryption", type: SectionType.Container }, + 0x7e1: { name: "ContentEncAlgo", type: SectionType.Uint }, + 0x7e2: { name: "ContentEncKeyID", type: SectionType.Binary }, + 0x7e3: { name: "ContentSignature", type: SectionType.Binary }, + 0x7e4: { name: "ContentSigKeyID", type: SectionType.Binary }, + 0x7e5: { name: "ContentSigAlgo", type: SectionType.Uint }, + 0x7e6: { name: "ContentSigHashAlgo", type: SectionType.Uint }, + 0xc53bb6b: { name: "Cues", type: SectionType.Container }, + 0x3b: { name: "CuePoint", type: SectionType.Container }, + 0x33: { name: "CueTime", type: SectionType.Uint }, + 0x37: { name: "CueTrackPositions", type: SectionType.Container }, + 0x77: { name: "CueTrack", type: SectionType.Uint }, + 0x71: { name: "CueClusterPosition", type: SectionType.Uint }, + 0x70: { name: "CueRelativePosition", type: SectionType.Uint }, + 0x32: { name: "CueDuration", type: SectionType.Uint }, + 0x1378: { name: "CueBlockNumber", type: SectionType.Uint }, + 0x6a: { name: "CueCodecState", type: SectionType.Uint }, + 0x5b: { name: "CueReference", type: SectionType.Container }, + 0x16: { name: "CueRefTime", type: SectionType.Uint }, + 0x17: { name: "CueRefCluster", type: SectionType.Uint }, + 0x135f: { name: "CueRefNumber", type: SectionType.Uint }, + 0x6b: { name: "CueRefCodecState", type: SectionType.Uint }, + 0x941a469: { name: "Attachments", type: SectionType.Container }, + 0x21a7: { name: "AttachedFile", type: SectionType.Container }, + 0x67e: { name: "FileDescription", type: SectionType.String }, + 0x66e: { name: "FileName", type: SectionType.String }, + 0x660: { name: "FileMimeType", type: SectionType.String }, + 0x65c: { name: "FileData", type: SectionType.Binary }, + 0x6ae: { name: "FileUID", type: SectionType.Uint }, + 0x675: { name: "FileReferral", type: SectionType.Binary }, + 0x661: { name: "FileUsedStartTime", type: SectionType.Uint }, + 0x662: { name: "FileUsedEndTime", type: SectionType.Uint }, + 0x43a770: { name: "Chapters", type: SectionType.Container }, + 0x5b9: { name: "EditionEntry", type: SectionType.Container }, + 0x5bc: { name: "EditionUID", type: SectionType.Uint }, + 0x5bd: { name: "EditionFlagHidden", type: SectionType.Uint }, + 0x5db: { name: "EditionFlagDefault", type: SectionType.Uint }, + 0x5dd: { name: "EditionFlagOrdered", type: SectionType.Uint }, + 0x36: { name: "ChapterAtom", type: SectionType.Container }, + 0x33c4: { name: "ChapterUID", type: SectionType.Uint }, + 0x1654: { name: "ChapterStringUID", type: SectionType.String }, + 0x11: { name: "ChapterTimeStart", type: SectionType.Uint }, + 0x12: { name: "ChapterTimeEnd", type: SectionType.Uint }, + 0x18: { name: "ChapterFlagHidden", type: SectionType.Uint }, + 0x598: { name: "ChapterFlagEnabled", type: SectionType.Uint }, + 0x2e67: { name: "ChapterSegmentUID", type: SectionType.Binary }, + 0x2ebc: { name: "ChapterSegmentEditionUID", type: SectionType.Uint }, + 0x23c3: { name: "ChapterPhysicalEquiv", type: SectionType.Uint }, + 0xf: { name: "ChapterTrack", type: SectionType.Container }, + 0x9: { name: "ChapterTrackNumber", type: SectionType.Uint }, + 0x0: { name: "ChapterDisplay", type: SectionType.Container }, + 0x5: { name: "ChapString", type: SectionType.String }, + 0x37c: { name: "ChapLanguage", type: SectionType.String }, + 0x37e: { name: "ChapCountry", type: SectionType.String }, + 0x2944: { name: "ChapProcess", type: SectionType.Container }, + 0x2955: { name: "ChapProcessCodecID", type: SectionType.Uint }, + 0x50d: { name: "ChapProcessPrivate", type: SectionType.Binary }, + 0x2911: { name: "ChapProcessCommand", type: SectionType.Container }, + 0x2922: { name: "ChapProcessTime", type: SectionType.Uint }, + 0x2933: { name: "ChapProcessData", type: SectionType.Binary }, + 0x254c367: { name: "Tags", type: SectionType.Container }, + 0x3373: { name: "Tag", type: SectionType.Container }, + 0x23c0: { name: "Targets", type: SectionType.Container }, + 0x28ca: { name: "TargetTypeValue", type: SectionType.Uint }, + 0x23ca: { name: "TargetType", type: SectionType.String }, + 0x23c5: { name: "TagTrackUID", type: SectionType.Uint }, + 0x23c9: { name: "TagEditionUID", type: SectionType.Uint }, + 0x23c4: { name: "TagChapterUID", type: SectionType.Uint }, + 0x23c6: { name: "TagAttachmentUID", type: SectionType.Uint }, + 0x27c8: { name: "SimpleTag", type: SectionType.Container }, + 0x5a3: { name: "TagName", type: SectionType.String }, + 0x47a: { name: "TagLanguage", type: SectionType.String }, + 0x484: { name: "TagDefault", type: SectionType.Uint }, + 0x487: { name: "TagString", type: SectionType.String }, + 0x485: { name: "TagBinary", type: SectionType.Binary }, + 0x15b0: { name: "Colour", type: SectionType.Container }, + 0x15b1: { name: "MatrixCoefficients", type: SectionType.Uint }, + 0x15b2: { name: "BitsPerChannel", type: SectionType.Uint }, + 0x15b3: { name: "ChromaSubsamplingHorz", type: SectionType.Uint }, + 0x15b4: { name: "ChromaSubsamplingVert", type: SectionType.Uint }, + 0x15b5: { name: "CbSubsamplingHorz", type: SectionType.Uint }, + 0x15b6: { name: "CbSubsamplingVert", type: SectionType.Uint }, + 0x15b7: { name: "ChromaSitingHorz", type: SectionType.Uint }, + 0x15b8: { name: "ChromaSitingVert", type: SectionType.Uint }, + 0x15b9: { name: "Range", type: SectionType.Uint }, + 0x15ba: { name: "TransferCharacteristics", type: SectionType.Uint }, + 0x15bb: { name: "Primaries", type: SectionType.Uint }, + 0x15bc: { name: "MaxCLL", type: SectionType.Uint }, + 0x15bd: { name: "MaxFALL", type: SectionType.Uint }, +} as const; + +export type SectionsMap = typeof sections; + +export type SectionKey = keyof SectionsMap; diff --git a/packages/conversation-client/test/recording.test.ts b/packages/conversation-client/test/recording.test.ts new file mode 100644 index 0000000..8fbb3c8 --- /dev/null +++ b/packages/conversation-client/test/recording.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; + +import { addRecordingDurationMetadata } from "../src/recording"; +import { WebmContainer } from "../src/vendor/fix-webm-duration/parser/lib/WebmContainer"; + +const SEGMENT_ID = 0x8538067; +const INFO_ID = 0x549a966; +const TIMECODE_SCALE_ID = 0xad7b1; +const DURATION_ID = 0x489; + +function recordingWithoutDuration() { + return new Blob( + [ + new Uint8Array([ + 0x18, 0x53, 0x80, 0x67, 0x8c, 0x15, 0x49, 0xa9, 0x66, 0x87, 0x2a, 0xd7, 0xb1, 0x83, 0x0f, + 0x42, 0x40, + ]), + ], + { type: "audio/webm;codecs=opus" }, + ); +} + +describe("recording finalization", () => { + it("adds measured duration metadata to WebM recordings", async () => { + const fixed = await addRecordingDurationMetadata(recordingWithoutDuration(), 12_345); + const file = new WebmContainer("File"); + file.setSource(new Uint8Array(await fixed.arrayBuffer())); + + const duration = file + .getSectionById(SEGMENT_ID) + ?.getSectionById(INFO_ID) + ?.getSectionById(DURATION_ID) + ?.getValue(); + const timecodeScale = file + .getSectionById(SEGMENT_ID) + ?.getSectionById(INFO_ID) + ?.getSectionById(TIMECODE_SCALE_ID) + ?.getValue(); + + expect(duration).toBe(12_345); + expect(timecodeScale).toBe(1_000_000); + }); + + it("does not modify recording formats that carry their own duration", async () => { + const recording = new Blob(["recording"], { type: "audio/mp4" }); + + await expect(addRecordingDurationMetadata(recording, 12_345)).resolves.toBe(recording); + }); + + it("parses float views without changing the recording buffer", () => { + const source = new Uint8Array([0x44, 0x89, 0x88, 0x40, 0xc8, 0x1c, 0x80, 0, 0, 0, 0]); + const original = source.slice(); + const container = new WebmContainer("Info"); + + container.setSource(source); + + expect(container.getSectionById(DURATION_ID)?.getValue()).toBe(12_345); + expect(source).toEqual(original); + }); +});