From 4c221228af7acac163ff5a345bf943e06bfa6e13 Mon Sep 17 00:00:00 2001 From: Filip Nowakowski Date: Fri, 20 Mar 2026 12:21:27 +0100 Subject: [PATCH 01/28] refactor(meetings): multistream --- .../plugin-meetings/src/metrics/constants.ts | 1 + .../codec/mediaCodecHelper.h264.ts | 25 ++-- .../src/multistream/codec/types.ts | 8 +- .../src/multistream/mediaRequestManager.ts | 140 ++++++++---------- .../src/multistream/receiveSlot.ts | 21 ++- .../src/multistream/remoteMedia.ts | 98 ++++-------- .../src/multistream/remoteMediaGroup.ts | 49 ++++-- .../plugin-meetings/src/multistream/types.ts | 24 +-- 8 files changed, 185 insertions(+), 181 deletions(-) diff --git a/packages/@webex/plugin-meetings/src/metrics/constants.ts b/packages/@webex/plugin-meetings/src/metrics/constants.ts index 84f4c39b02b..5f1653f7922 100644 --- a/packages/@webex/plugin-meetings/src/metrics/constants.ts +++ b/packages/@webex/plugin-meetings/src/metrics/constants.ts @@ -91,6 +91,7 @@ const BEHAVIORAL_METRICS = { LOCUS_CLASSIC_VS_HASH_TREE_MISMATCH: 'js_sdk_locus_classic_vs_hash_tree_mismatch', LOCUS_HASH_TREE_UNSUPPORTED_OPERATION: 'js_sdk_locus_hash_tree_unsupported_operation', MEDIA_STILL_NOT_CONNECTED: 'js_sdk_media_still_not_connected', + DEPRECATED_SET_MAX_FS_USED: 'js_sdk_deprecated_set_max_fs_used', }; export {BEHAVIORAL_METRICS as default}; diff --git a/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts b/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts index b4922e7d671..32dacb2edf1 100644 --- a/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts +++ b/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts @@ -6,35 +6,34 @@ import { CodecInfo as WcmeCodecInfo, } from '@webex/internal-media-core'; import {CODEC_DEFAULTS, H264_CODEC_PARAMETERS, PANE_SIZE_TO_RESOLUTION} from './constants'; -import {MediaCodecHelper, H264CodecInfo} from './types'; +import {MediaCodecHelper, H264CodecInfo, GetCodecInfoOptions} from './types'; import {MediaRequest, RemoteVideoResolution} from '../types'; import LoggerProxy from '../../common/logs/logger-proxy'; -type H264CodecOptions = { - getMaxFs?: () => number; -}; - /** * Class for H264 media codec info */ -export default class MediaCodecHelperH264 - implements MediaCodecHelper -{ +export default class MediaCodecHelperH264 implements MediaCodecHelper { /** * Gets the H264 codec info * - * @param {Object} options - The options for the H264 codec info - * @param {number} options.maxFs - The maximum frame size + * @param {GetCodecInfoOptions} options - The options for the H264 codec info * @returns {H264CodecInfo} The H264 codec info */ - getCodecInfo(options: H264CodecOptions): H264CodecInfo | undefined { - if (!options.getMaxFs) { + getCodecInfo({sizeHint}: GetCodecInfoOptions): H264CodecInfo | undefined { + let maxFs: number; + + if (sizeHint?.width > 0 && sizeHint?.height > 0) { + maxFs = this.getSizeHintMaxFs(sizeHint.width, sizeHint.height); + } else if (sizeHint?.resolution) { + maxFs = this.getMaxFs(sizeHint.resolution); + } else { return undefined; } return { codec: 'h264', - maxFs: options.getMaxFs(), + maxFs, }; } diff --git a/packages/@webex/plugin-meetings/src/multistream/codec/types.ts b/packages/@webex/plugin-meetings/src/multistream/codec/types.ts index 350f97eb9cf..8bb89acff34 100644 --- a/packages/@webex/plugin-meetings/src/multistream/codec/types.ts +++ b/packages/@webex/plugin-meetings/src/multistream/codec/types.ts @@ -3,7 +3,7 @@ import { SupportedResolution, CodecInfo as WcmeCodecInfo, } from '@webex/internal-media-core'; -import {MediaRequest} from '../types'; +import {MediaRequest, SizeHint} from '../types'; export type H264CodecInfo = H264EncodingParams & { codec: 'h264'; @@ -11,8 +11,10 @@ export type H264CodecInfo = H264EncodingParams & { export type CodecInfo = H264CodecInfo; -export interface MediaCodecHelper { - getCodecInfo(options: TCodecOptions): TCodecInfo | undefined; +export type GetCodecInfoOptions = {sizeHint?: SizeHint}; + +export interface MediaCodecHelper { + getCodecInfo(options: GetCodecInfoOptions): TCodecInfo | undefined; getWCMECodecInfos(mediaRequest: MediaRequest): WcmeCodecInfo[]; degradeMediaRequest(mediaRequest: MediaRequest, resolution: SupportedResolution): number; getMaxPayloadBitsPerSecond(mediaRequest: MediaRequest): number; diff --git a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts index d4d3ba61a6e..0754fc45688 100644 --- a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts +++ b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts @@ -4,10 +4,8 @@ import { Policy, ActiveSpeakerInfo, ReceiverSelectedInfo, - CodecInfo as WcmeCodecInfo, - H264Codec, - getRecommendedMaxBitrateForFrameSize, RecommendedOpusBitrates, + SupportedResolution, } from '@webex/internal-media-core'; import {cloneDeepWith, debounce, isEmpty} from 'lodash'; @@ -15,7 +13,8 @@ import LoggerProxy from '../common/logs/logger-proxy'; import {ReceiveSlotEvents} from './receiveSlot'; import {MediaRequest, MediaRequestId} from './types'; -import {CODEC_DEFAULTS, H264_CODEC_PARAMETERS} from './codec/constants'; +import {CODEC_DEFAULTS} from './codec/constants'; +import MediaCodecHelper from './codec/mediaCodecHelper'; const DEBOUNCED_SOURCE_UPDATE_TIME = 1000; @@ -77,40 +76,26 @@ export default class MediaRequestManager { } private getDegradedClientRequests(clientRequests: ClientRequestsMap) { - const maxFsLimits = [ - H264_CODEC_PARAMETERS['1080p'].maxFs, - H264_CODEC_PARAMETERS['720p'].maxFs, - H264_CODEC_PARAMETERS['540p'].maxFs, - H264_CODEC_PARAMETERS['360p'].maxFs, - H264_CODEC_PARAMETERS['180p'].maxFs, - H264_CODEC_PARAMETERS['90p'].maxFs, - ]; - - // reduce max-fs until total macroblocks is below limit - for (let i = 0; i < maxFsLimits.length; i += 1) { + const resolutions: SupportedResolution[] = ['1080p', '720p', '540p', '360p', '180p', '90p']; + + for (const resolution of resolutions) { let totalMacroblocksRequested = 0; + Object.values(clientRequests).forEach((mr) => { - if (mr.codecInfo) { - mr.codecInfo.maxFs = Math.min( - mr.preferredMaxFs || CODEC_DEFAULTS.h264.maxFs, - mr.codecInfo.maxFs || CODEC_DEFAULTS.h264.maxFs, - maxFsLimits[i] - ); - // we only consider sources with "live" state - const slotsWithLiveSource = mr.receiveSlots.filter((rs) => rs.sourceState === 'live'); - totalMacroblocksRequested += mr.codecInfo.maxFs * slotsWithLiveSource.length; - } + const mediaCodecHelper = MediaCodecHelper.get(mr.codecInfo?.codec); + totalMacroblocksRequested += mediaCodecHelper.degradeMediaRequest(mr, resolution); }); + if (totalMacroblocksRequested <= this.degradationPreferences.maxMacroblocksLimit) { - if (i !== 0) { + if (resolution !== '1080p') { LoggerProxy.logger.warn( - `multistream:mediaRequestManager --> too many streams with high max-fs, frame size will be limited to ${maxFsLimits[i]}` + `multistream:mediaRequestManager --> too many streams with high macroblocks requested, resolution will be limited to ${resolution}` ); } break; - } else if (i === maxFsLimits.length - 1) { + } else if (resolution === '90p') { LoggerProxy.logger.warn( - `multistream:mediaRequestManager --> even with frame size limited to ${maxFsLimits[i]} you are still requesting too many streams, consider reducing the number of requests` + `multistream:mediaRequestManager --> even with resolution limited to ${resolution} you are still requesting too many streams, consider reducing the number of requests` ); } } @@ -151,20 +136,28 @@ export default class MediaRequestManager { * * If MediaRequestManager kind is "audio", a constant bitrate will be returned. * If MediaRequestManager kind is "video", the bitrate will be calculated based - * on maxFs (default h264 maxFs as fallback if maxFs is not defined) + * on maxFs (default maxFs as fallback if maxFs is not defined) * * @param {MediaRequest} mediaRequest - mediaRequest to take data from * @returns {number} maxPayloadBitsPerSecond */ private getMaxPayloadBitsPerSecond(mediaRequest: MediaRequest): number { if (this.kind === 'audio') { - // return mono_music bitrate default if the kind of mediarequest manager is audio: + // return mono_music bitrate default if the kind of media request manager is audio: return RecommendedOpusBitrates.FB_MONO_MUSIC; } - return getRecommendedMaxBitrateForFrameSize( - mediaRequest.codecInfo.maxFs || CODEC_DEFAULTS.h264.maxFs + if (mediaRequest.codecInfo?.codec) { + const mediaCodecHelper = MediaCodecHelper.get(mediaRequest.codecInfo.codec); + + return mediaCodecHelper.getMaxPayloadBitsPerSecond(mediaRequest); + } + + LoggerProxy.logger.warn( + 'multistream:mediaRequestManager --> no codec info found for media request' ); + + return 0; } /** @@ -293,51 +286,38 @@ export default class MediaRequestManager { // map all the client media requests to wcme stream requests Object.values(clientRequests).forEach((mr) => { - if (mr.receiveSlots.length > 0) { - streamRequests.push( - new StreamRequest( - mr.policyInfo.policy === 'active-speaker' - ? Policy.ActiveSpeaker - : Policy.ReceiverSelected, - mr.policyInfo.policy === 'active-speaker' - ? new ActiveSpeakerInfo( - mr.policyInfo.priority, - mr.policyInfo.crossPriorityDuplication, - mr.policyInfo.crossPolicyDuplication, - mr.policyInfo.preferLiveVideo, - mr.policyInfo.namedMediaGroups - ) - : new ReceiverSelectedInfo(mr.policyInfo.csi), - mr.receiveSlots.map((receiveSlot) => receiveSlot.wcmeReceiveSlot), - this.getMaxPayloadBitsPerSecond(mr), - mr.codecInfo && [ - WcmeCodecInfo.fromH264( - 0x80, - new H264Codec( - mr.codecInfo.maxFs, - mr.codecInfo.maxFps || CODEC_DEFAULTS.h264.maxFps, - this.getH264MaxMbps(mr), - mr.codecInfo.maxWidth, - mr.codecInfo.maxHeight - ) - ), - ] - ) - ); + if (mr.receiveSlots.length <= 0) { + return; } - }); - //! IMPORTANT: this is only a temporary fix. This will soon be done in the jmp layer (@webex/json-multistream) - // https://jira-eng-gpk2.cisco.com/jira/browse/WEBEX-326713 - if (!this.checkIsNewRequestsEqualToPrev(streamRequests)) { - this.sendMediaRequestsCallback(streamRequests); - this.previousStreamRequests = streamRequests; - LoggerProxy.logger.info(`multistream:sendRequests --> media requests sent. `); - } else { - LoggerProxy.logger.info( - `multistream:sendRequests --> detected duplicate WCME requests, skipping them... ` + const policy = + mr.policyInfo.policy === 'active-speaker' ? Policy.ActiveSpeaker : Policy.ReceiverSelected; + const policySpecificInfo = + mr.policyInfo.policy === 'active-speaker' + ? new ActiveSpeakerInfo( + mr.policyInfo.priority, + mr.policyInfo.crossPriorityDuplication, + mr.policyInfo.crossPolicyDuplication, + mr.policyInfo.preferLiveVideo, + mr.policyInfo.namedMediaGroups + ) + : new ReceiverSelectedInfo(mr.policyInfo.csi); + + const receiveSlots = mr.receiveSlots.map((receiveSlot) => receiveSlot.wcmeReceiveSlot); + const maxPayloadBitsPerSecond = this.getMaxPayloadBitsPerSecond(mr); + const codecInfos = [...MediaCodecHelper.H264.getWCMECodecInfos(mr)]; + + const streamRequest = new StreamRequest( + policy, + policySpecificInfo, + receiveSlots, + maxPayloadBitsPerSecond, + codecInfos ); - } + streamRequests.push(streamRequest); + }); + + this.sendMediaRequestsCallback(streamRequests); } public addRequest(mediaRequest: MediaRequest, commit = true): MediaRequestId { @@ -346,15 +326,20 @@ export default class MediaRequestManager { this.clientRequests[newId] = mediaRequest; - const eventHandler = ({maxFs}) => { + mediaRequest.handleMaxFs = ({maxFs}) => { mediaRequest.preferredMaxFs = maxFs; this.debouncedSourceUpdateListener(); }; - mediaRequest.handleMaxFs = eventHandler; + + mediaRequest.handleSizeHint = (sizeHint) => { + mediaRequest.sizeHint = sizeHint; + this.debouncedSourceUpdateListener(); + }; mediaRequest.receiveSlots.forEach((rs) => { rs.on(ReceiveSlotEvents.SourceUpdate, this.sourceUpdateListener); rs.on(ReceiveSlotEvents.MaxFsUpdate, mediaRequest.handleMaxFs); + rs.on(ReceiveSlotEvents.SizeHintUpdate, mediaRequest.handleSizeHint); }); if (commit) { @@ -370,6 +355,7 @@ export default class MediaRequestManager { mediaRequest?.receiveSlots.forEach((rs) => { rs.off(ReceiveSlotEvents.SourceUpdate, this.sourceUpdateListener); rs.off(ReceiveSlotEvents.MaxFsUpdate, mediaRequest.handleMaxFs); + rs.off(ReceiveSlotEvents.SizeHintUpdate, mediaRequest.handleSizeHint); }); delete this.clientRequests[requestId]; diff --git a/packages/@webex/plugin-meetings/src/multistream/receiveSlot.ts b/packages/@webex/plugin-meetings/src/multistream/receiveSlot.ts index df349f76ddc..c3c1ee88e56 100644 --- a/packages/@webex/plugin-meetings/src/multistream/receiveSlot.ts +++ b/packages/@webex/plugin-meetings/src/multistream/receiveSlot.ts @@ -8,10 +8,12 @@ import { import LoggerProxy from '../common/logs/logger-proxy'; import EventsScope from '../common/events/events-scope'; +import {SizeHint} from './types'; export const ReceiveSlotEvents = { SourceUpdate: 'sourceUpdate', MaxFsUpdate: 'maxFsUpdate', + SizeHintUpdate: 'sizeHintUpdate', }; export type {StreamState} from '@webex/internal-media-core'; @@ -82,6 +84,23 @@ export class ReceiveSlot extends EventsScope { return this.#csi; } + /** + * Supply the width and height of the video element + * to restrict the requested resolution to this size + * @param width width of the video element + * @param height height of the video element + */ + public setSizeHint(sizeHint: SizeHint) { + this.emit( + { + file: 'meeting/receiveSlot', + function: 'setSizeHint', + }, + ReceiveSlotEvents.SizeHintUpdate, + sizeHint + ); + } + /** * Set the max frame size for this slot * @param newFs frame size @@ -92,7 +111,7 @@ export class ReceiveSlot extends EventsScope { this.emit( { file: 'meeting/receiveSlot', - function: 'findMemberId', + function: 'setMaxFs', }, ReceiveSlotEvents.MaxFsUpdate, { diff --git a/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts b/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts index 13b2e31cdd9..5a605e54450 100644 --- a/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts +++ b/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts @@ -1,12 +1,13 @@ /* eslint-disable valid-jsdoc */ import {MediaType, StreamState} from '@webex/internal-media-core'; -import LoggerProxy from '../common/logs/logger-proxy'; +import Metrics from '@webex/internal-plugin-metrics'; import EventsScope from '../common/events/events-scope'; import MediaRequestManager from './mediaRequestManager'; import {CSI, ReceiveSlot, ReceiveSlotEvents} from './receiveSlot'; -import type {MediaRequestId, RemoteVideoResolution} from './types'; -import {H264_CODEC_PARAMETERS} from './codec/constants'; +import type {MediaRequestId, RemoteVideoResolution, SizeHint} from './types'; +import BEHAVIORAL_METRICS from '../metrics/constants'; +import MediaCodecHelper from './codec/mediaCodecHelper'; export const RemoteMediaEvents = { SourceUpdate: ReceiveSlotEvents.SourceUpdate, @@ -17,37 +18,15 @@ export const RemoteMediaEvents = { * Converts pane size into h264 maxFs * @param {RemoteVideoResolution} paneSize * @returns {number} + * @deprecated use MediaCodecHelper from plugin-meetings/src/codec/mediaCodecHelper instead */ export function getMaxFs(paneSize: RemoteVideoResolution): number { - let maxFs; - - switch (paneSize) { - case 'thumbnail': - maxFs = H264_CODEC_PARAMETERS['90p'].maxFs; - break; - case 'very small': - maxFs = H264_CODEC_PARAMETERS['180p'].maxFs; - break; - case 'small': - maxFs = H264_CODEC_PARAMETERS['360p'].maxFs; - break; - case 'medium': - maxFs = H264_CODEC_PARAMETERS['720p'].maxFs; - break; - case 'large': - maxFs = H264_CODEC_PARAMETERS['1080p'].maxFs; - break; - case 'best': - maxFs = H264_CODEC_PARAMETERS['1080p'].maxFs; // for now 'best' is 1080p, so same as 'large' - break; - default: - LoggerProxy.logger.warn( - `RemoteMedia#getMaxFs --> unsupported paneSize: ${paneSize}, using "medium" instead` - ); - maxFs = H264_CODEC_PARAMETERS['720p'].maxFs; - } + this.LoggerProxy.logger.warn( + 'RemoteMedia->getMaxFs --> [DEPRECATION WARNING]: getMaxFs has been deprecated, use MediaCodecHelper instead' + ); + Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.DEPRECATED_SET_MAX_FS_USED, {paneSize}); - return maxFs; + return MediaCodecHelper.H264.getMaxFs(paneSize); } type Options = { @@ -76,11 +55,11 @@ export class RemoteMedia extends EventsScope { public readonly id: RemoteMediaId; /** - * The max frame size of the media request, used for logging and media requests. + * The size hint of the media request, used for logging and media requests. * Set by setSizeHint() based on video element dimensions. - * When > 0, this value takes precedence over options.resolution in sendMediaRequest(). + * @todo remove this once deprecation of getEffectiveMaxFs() is complete */ - private maxFrameSize = 0; + private sizeHint?: SizeHint; /** * Constructs RemoteMedia instance @@ -110,48 +89,35 @@ export class RemoteMedia extends EventsScope { * @param height height of the video element * @note width/height of 0 will be ignored */ - public setSizeHint(width, height) { - // only base on height for now - let fs: number; - + public setSizeHint(width: number, height: number) { if (width === 0 || height === 0) { return; } - // we switch to the next resolution level when the height is 10% more than the current resolution height - // except for 1080p - we switch to it immediately when the height is more than 720p - const threshold = 1.1; - const getThresholdHeight = (h: number) => Math.round(h * threshold); - - if (height < getThresholdHeight(90)) { - fs = H264_CODEC_PARAMETERS['90p'].maxFs; - } else if (height < getThresholdHeight(180)) { - fs = H264_CODEC_PARAMETERS['180p'].maxFs; - } else if (height < getThresholdHeight(360)) { - fs = H264_CODEC_PARAMETERS['360p'].maxFs; - } else if (height < getThresholdHeight(540)) { - fs = H264_CODEC_PARAMETERS['540p'].maxFs; - } else if (height <= 720) { - fs = H264_CODEC_PARAMETERS['720p'].maxFs; - } else { - fs = H264_CODEC_PARAMETERS['1080p'].maxFs; - } + this.sizeHint = {width, height, resolution: this.options.resolution}; + this.receiveSlot?.setSizeHint(this.sizeHint); + } - this.maxFrameSize = fs; - this.receiveSlot?.setMaxFs(fs); + /** + * Get the current size hint that would be used in media requests + * @returns {SizeHint | undefined} The size hint, or undefined if no size hint has been set + */ + public getSizeHint(): SizeHint | undefined { + return this.sizeHint; } /** * Get the current effective maxFs value that would be used in media requests * @returns {number | undefined} The maxFs value, or undefined if no constraints + * @deprecated use getSizeHint() instead */ public getEffectiveMaxFs(): number | undefined { - if (this.maxFrameSize > 0) { - return this.maxFrameSize; + if (this.sizeHint) { + return MediaCodecHelper.H264.getSizeHintMaxFs(this.sizeHint.width, this.sizeHint.height); } if (this.options.resolution) { - return getMaxFs(this.options.resolution); + return MediaCodecHelper.H264.getMaxFs(this.options.resolution); } return undefined; @@ -197,9 +163,6 @@ export class RemoteMedia extends EventsScope { throw new Error('sendMediaRequest() called on an invalidated RemoteMedia instance'); } - // Use maxFrameSize from setSizeHint if available, otherwise fallback to options.resolution - const maxFs = this.getEffectiveMaxFs(); - this.mediaRequestId = this.mediaRequestManager.addRequest( { policyInfo: { @@ -207,10 +170,9 @@ export class RemoteMedia extends EventsScope { csi, }, receiveSlots: [this.receiveSlot], - codecInfo: maxFs && { - codec: 'h264', - maxFs, - }, + codecInfo: MediaCodecHelper.H264.getCodecInfo({ + sizeHint: this.sizeHint, + }), }, commit ); diff --git a/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts b/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts index 3d0f170ee70..b5d16521bac 100644 --- a/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts +++ b/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts @@ -5,10 +5,11 @@ import {forEach} from 'lodash'; import {NamedMediaGroup} from '@webex/internal-media-core'; import LoggerProxy from '../common/logs/logger-proxy'; -import {getMaxFs, RemoteMedia} from './remoteMedia'; +import {RemoteMedia} from './remoteMedia'; import MediaRequestManager from './mediaRequestManager'; -import type {MediaRequestId, RemoteVideoResolution} from './types'; +import type {MediaRequestId, RemoteVideoResolution, SizeHint} from './types'; import {CSI, ReceiveSlot} from './receiveSlot'; +import MediaCodecHelper from './codec/mediaCodecHelper'; type Options = { resolution?: RemoteVideoResolution; // applies only to groups of type MediaType.VideoMain and MediaType.VideoSlides @@ -216,8 +217,7 @@ export class RemoteMediaGroup { private sendActiveSpeakerMediaRequest(commit: boolean) { this.cancelActiveSpeakerMediaRequest(false); - // Calculate the effective maxFs based on all unpinned RemoteMedia instances - const effectiveMaxFs = this.getEffectiveMaxFsForActiveSpeaker(); + const sizeHint = this.getSizeHintForActiveSpeaker(); this.mediaRequestId = this.mediaRequestManager.addRequest( { @@ -234,10 +234,11 @@ export class RemoteMediaGroup { receiveSlots: this.unpinnedRemoteMedia.map((remoteMedia) => remoteMedia.getUnderlyingReceiveSlot() ) as ReceiveSlot[], - codecInfo: effectiveMaxFs && { - codec: 'h264', - maxFs: effectiveMaxFs, - }, + codecInfo: sizeHint + ? MediaCodecHelper.H264.getCodecInfo({ + sizeHint, + }) + : undefined, }, commit ); @@ -305,10 +306,39 @@ export class RemoteMediaGroup { ); } + private getSizeHintForActiveSpeaker(): SizeHint | undefined { + // Get all size hint values from unpinned RemoteMedia instances + const sizeHints = this.unpinnedRemoteMedia + .map((remoteMedia) => remoteMedia.getSizeHint()) + .filter((sizeHint) => !!sizeHint); + + // Use the highest sizeHint based on width*height value to ensure we don't under-request resolution for any instance + if (sizeHints.length > 0) { + return sizeHints.reduce((maxSizeHint, currentSizeHint) => { + const currentSize = currentSizeHint.width * currentSizeHint.height; + const maxSize = maxSizeHint.width * maxSizeHint.height; + + return currentSize > maxSize ? currentSizeHint : maxSizeHint; + }, sizeHints[0]); + } + + // Fall back to group's resolution option + if (this.options.resolution) { + return { + width: 0, + height: 0, + resolution: this.options.resolution, + }; + } + + return undefined; + } + /** * Calculate the effective maxFs for the active speaker media request based on unpinned RemoteMedia instances * @returns {number | undefined} The calculated maxFs value, or undefined if no constraints * @private + * @deprecated */ private getEffectiveMaxFsForActiveSpeaker(): number | undefined { // Get all effective maxFs values from unpinned RemoteMedia instances @@ -323,7 +353,7 @@ export class RemoteMediaGroup { // Fall back to group's resolution option if (this.options.resolution) { - return getMaxFs(this.options.resolution); + return MediaCodecHelper.H264.getMaxFs(this.options.resolution); } return undefined; @@ -332,6 +362,7 @@ export class RemoteMediaGroup { /** * Get the current effective maxFs that would be used for the active speaker media request * @returns {number | undefined} The effective maxFs value + * @deprecated */ public getEffectiveMaxFs(): number | undefined { return this.getEffectiveMaxFsForActiveSpeaker(); diff --git a/packages/@webex/plugin-meetings/src/multistream/types.ts b/packages/@webex/plugin-meetings/src/multistream/types.ts index 8c992cb64ea..08b57afe152 100644 --- a/packages/@webex/plugin-meetings/src/multistream/types.ts +++ b/packages/@webex/plugin-meetings/src/multistream/types.ts @@ -18,16 +18,6 @@ export interface ReceiverSelectedPolicyInfo { export type PolicyInfo = ActiveSpeakerPolicyInfo | ReceiverSelectedPolicyInfo; -export interface MediaRequest { - policyInfo: PolicyInfo; - receiveSlots: Array; - codecInfo?: CodecInfo; - preferredMaxFs?: number; - handleMaxFs?: ({maxFs}: {maxFs: number}) => void; -} - -export type MediaRequestId = string; - export type RemoteVideoResolution = /** the smallest possible resolution, 90p or less */ | 'thumbnail' @@ -41,3 +31,17 @@ export type RemoteVideoResolution = | 'large' /** highest possible resolution */ | 'best'; + +export type SizeHint = {width: number; height: number; resolution?: RemoteVideoResolution}; + +export interface MediaRequest { + policyInfo: PolicyInfo; + receiveSlots: Array; + codecInfo?: CodecInfo; + preferredMaxFs?: number; + sizeHint?: SizeHint; + handleMaxFs?: ({maxFs}: {maxFs: number}) => void; + handleSizeHint?: (sizeHint: SizeHint) => void; +} + +export type MediaRequestId = string; From b359ca498a22fc69424eeb8db9eca95fddd860bc Mon Sep 17 00:00:00 2001 From: Jordan Rowan <86778628+jor-row@users.noreply.github.com> Date: Fri, 20 Mar 2026 17:27:25 +0000 Subject: [PATCH 02/28] fix(ca): send stayLobbyTime with lobby exit event (#4744) Co-authored-by: Jordan Rowan Co-authored-by: Gabriel Lee --- .../call-diagnostic-metrics-latencies.ts | 8 +- .../call-diagnostic-metrics.util.ts | 6 +- .../call-diagnostic-metrics-batcher.ts | 42 +++- .../call-diagnostic-metrics-latencies.ts | 237 +++++++++--------- .../call-diagnostic-metrics.util.ts | 13 +- 5 files changed, 171 insertions(+), 135 deletions(-) diff --git a/packages/@webex/internal-plugin-metrics/src/call-diagnostic/call-diagnostic-metrics-latencies.ts b/packages/@webex/internal-plugin-metrics/src/call-diagnostic/call-diagnostic-metrics-latencies.ts index 84f577ef2bd..3e8b42966a5 100644 --- a/packages/@webex/internal-plugin-metrics/src/call-diagnostic/call-diagnostic-metrics-latencies.ts +++ b/packages/@webex/internal-plugin-metrics/src/call-diagnostic/call-diagnostic-metrics-latencies.ts @@ -303,10 +303,7 @@ export default class CallDiagnosticLatencies extends WebexPlugin { * @returns - latency */ public getStayLobbyTime() { - return this.getDiffBetweenTimestamps( - 'client.locus.join.response', - 'internal.host.meeting.participant.admitted' - ); + return this.getDiffBetweenTimestamps('client.locus.join.response', 'client.lobby.exited'); } /** @@ -480,7 +477,8 @@ export default class CallDiagnosticLatencies extends WebexPlugin { const clickToInterstitial = this.getClickToInterstitial(); const interstitialToJoinOk = this.getInterstitialToJoinOK(); const joinConfJMT = this.getJoinConfJMT(); - const lobbyTime = this.getStayLobbyTime(); + const lobbyTimeLatency = this.getStayLobbyTime(); + const lobbyTime = typeof lobbyTimeLatency === 'number' ? lobbyTimeLatency : 0; if (clickToInterstitial && interstitialToJoinOk && joinConfJMT) { const totalMediaJMT = clamp( diff --git a/packages/@webex/internal-plugin-metrics/src/call-diagnostic/call-diagnostic-metrics.util.ts b/packages/@webex/internal-plugin-metrics/src/call-diagnostic/call-diagnostic-metrics.util.ts index 064008516d7..e5003a6dae9 100644 --- a/packages/@webex/internal-plugin-metrics/src/call-diagnostic/call-diagnostic-metrics.util.ts +++ b/packages/@webex/internal-plugin-metrics/src/call-diagnostic/call-diagnostic-metrics.util.ts @@ -361,7 +361,6 @@ export const prepareDiagnosticMetricItem = (webex: any, item: any) => { joinTimes.totalMediaJMT = cdl.getTotalMediaJMT(); joinTimes.interstitialToMediaOKJMT = cdl.getInterstitialToMediaOKJMT(); joinTimes.callInitMediaEngineReady = cdl.getCallInitMediaEngineReady(); - joinTimes.stayLobbyTime = cdl.getStayLobbyTime(); joinTimes.totalMediaJMTWithUserDelay = cdl.getTotalMediaJMTWithUserDelay(); joinTimes.totalJMTWithUserDelay = cdl.getTotalJMTWithUserDelay(); break; @@ -369,6 +368,11 @@ export const prepareDiagnosticMetricItem = (webex: any, item: any) => { case 'client.media.tx.start': audioSetupDelay.joinRespTxStart = cdl.getAudioJoinRespTxStart(); videoSetupDelay.joinRespTxStart = cdl.getVideoJoinRespTxStart(); + break; + + case 'client.lobby.exited': + joinTimes.stayLobbyTime = cdl.getStayLobbyTime(); + break; } if (!isEmpty(joinTimes)) { diff --git a/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics-batcher.ts b/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics-batcher.ts index df43c1263dd..2b628cffb6d 100644 --- a/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics-batcher.ts +++ b/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics-batcher.ts @@ -142,9 +142,7 @@ describe('plugin-metrics', () => { webex.internal.newMetrics.callDiagnosticLatencies.getDiffBetweenTimestamps = sinon .stub() .returns(10); - webex.internal.newMetrics.callDiagnosticLatencies.getU2CTime = sinon - .stub() - .returns(20); + webex.internal.newMetrics.callDiagnosticLatencies.getU2CTime = sinon.stub().returns(20); webex.internal.newMetrics.callDiagnosticLatencies.getReachabilityClustersReqResp = sinon .stub() .returns(10); @@ -165,7 +163,7 @@ describe('plugin-metrics', () => { registerWDMDeviceJMT: 10, showInterstitialTime: 10, getU2CTime: 20, - getReachabilityClustersReqResp: 10 + getReachabilityClustersReqResp: 10, }, }); assert.lengthOf( @@ -189,9 +187,8 @@ describe('plugin-metrics', () => { webex.internal.newMetrics.callDiagnosticLatencies.getDownloadTimeJMT = sinon .stub() .returns(100); - webex.internal.newMetrics.callDiagnosticLatencies.getClickToInterstitialWithUserDelay = sinon - .stub() - .returns(43); + webex.internal.newMetrics.callDiagnosticLatencies.getClickToInterstitialWithUserDelay = + sinon.stub().returns(43); webex.internal.newMetrics.callDiagnosticLatencies.getTotalJMTWithUserDelay = sinon .stub() .returns(64); @@ -346,7 +343,7 @@ describe('plugin-metrics', () => { webex.internal.newMetrics.callDiagnosticLatencies.getInterstitialToJoinOK = sinon .stub() .returns(7); - webex.internal.newMetrics.callDiagnosticLatencies.getStayLobbyTime = sinon + webex.internal.newMetrics.callDiagnosticLatencies.getStayLobbyTime = sinon .stub() .returns(1); webex.internal.newMetrics.callDiagnosticLatencies.getTotalMediaJMTWithUserDelay = sinon @@ -372,7 +369,6 @@ describe('plugin-metrics', () => { totalMediaJMT: 61, interstitialToMediaOKJMT: 22, callInitMediaEngineReady: 10, - stayLobbyTime: 1, totalMediaJMTWithUserDelay: 43, totalJMTWithUserDelay: 64, }, @@ -382,6 +378,34 @@ describe('plugin-metrics', () => { 0 ); }); + + it('appends the correct join times to the request for client.lobby.exited', async () => { + webex.internal.newMetrics.callDiagnosticLatencies.getStayLobbyTime = sinon + .stub() + .returns(10); + + const promise = webex.internal.newMetrics.callDiagnosticMetrics.submitToCallDiagnostics( + //@ts-ignore + {event: {name: 'client.lobby.exited'}} + ); + await flushPromises(); + clock.tick(config.metrics.batcherWait); + + await promise; + + //@ts-ignore + assert.calledOnce(webex.request); + assert.deepEqual(webex.request.getCalls()[0].args[0].body.metrics[0].eventPayload.event, { + name: 'client.lobby.exited', + joinTimes: { + stayLobbyTime: 10, + }, + }); + assert.lengthOf( + webex.internal.newMetrics.callDiagnosticMetrics.callDiagnosticEventsBatcher.queue, + 0 + ); + }); }); describe('when the request fails', () => { diff --git a/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics-latencies.ts b/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics-latencies.ts index 9a1e786a8e0..17499eee22e 100644 --- a/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics-latencies.ts +++ b/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics-latencies.ts @@ -143,7 +143,7 @@ describe('internal-plugin-metrics', () => { cdl.saveTimestamp({key: 'client.alert.removed', value: 50}); const res = cdl.getDiffBetweenTimestamps('client.alert.displayed', 'client.alert.removed', { minimum: 0, - maximum: 100 + maximum: 100, }); assert.deepEqual(res, 40); }); @@ -153,7 +153,7 @@ describe('internal-plugin-metrics', () => { cdl.saveTimestamp({key: 'client.alert.removed', value: 45}); const res = cdl.getDiffBetweenTimestamps('client.alert.displayed', 'client.alert.removed', { minimum: 10, - maximum: 100 + maximum: 100, }); assert.deepEqual(res, 10); }); @@ -163,7 +163,7 @@ describe('internal-plugin-metrics', () => { cdl.saveTimestamp({key: 'client.alert.removed', value: 210}); const res = cdl.getDiffBetweenTimestamps('client.alert.displayed', 'client.alert.removed', { minimum: 0, - maximum: 100 + maximum: 100, }); assert.deepEqual(res, 100); }); @@ -172,7 +172,7 @@ describe('internal-plugin-metrics', () => { cdl.saveTimestamp({key: 'client.alert.displayed', value: 50}); cdl.saveTimestamp({key: 'client.alert.removed', value: 45}); const res = cdl.getDiffBetweenTimestamps('client.alert.displayed', 'client.alert.removed', { - maximum: 100 + maximum: 100, }); assert.deepEqual(res, 0); }); @@ -181,7 +181,7 @@ describe('internal-plugin-metrics', () => { cdl.saveTimestamp({key: 'client.alert.displayed', value: 10}); cdl.saveTimestamp({key: 'client.alert.removed', value: 2000}); const res = cdl.getDiffBetweenTimestamps('client.alert.displayed', 'client.alert.removed', { - minimum: 5 + minimum: 5, }); assert.deepEqual(res, 1990); }); @@ -191,7 +191,7 @@ describe('internal-plugin-metrics', () => { cdl.saveTimestamp({key: 'client.alert.removed', value: 50}); const res = cdl.getDiffBetweenTimestamps('client.alert.displayed', 'client.alert.removed', { minimum: 10, - maximum: 1000 + maximum: 1000, }); assert.deepEqual(res, 10); }); @@ -200,7 +200,7 @@ describe('internal-plugin-metrics', () => { cdl.saveTimestamp({key: 'client.alert.displayed', value: 10}); const res = cdl.getDiffBetweenTimestamps('client.alert.displayed', 'client.alert.removed', { minimum: 0, - maximum: 100 + maximum: 100, }); assert.deepEqual(res, undefined); }); @@ -513,7 +513,7 @@ describe('internal-plugin-metrics', () => { value: 10, }); cdl.saveTimestamp({ - key: 'internal.host.meeting.participant.admitted', + key: 'client.lobby.exited', value: 20, }); assert.deepEqual(cdl.getStayLobbyTime(), 10); @@ -656,56 +656,56 @@ describe('internal-plugin-metrics', () => { }); it('calculates getTotalJMT correctly when clickToInterstitial is 0', () => { - cdl.saveLatency('internal.click.to.interstitial', 0); - cdl.saveTimestamp({ - key: 'internal.client.interstitial-window.click.joinbutton', - value: 20, - }); - cdl.saveTimestamp({ - key: 'client.locus.join.response', - value: 40, - }); - assert.deepEqual(cdl.getTotalJMT(), 20); + cdl.saveLatency('internal.click.to.interstitial', 0); + cdl.saveTimestamp({ + key: 'internal.client.interstitial-window.click.joinbutton', + value: 20, }); + cdl.saveTimestamp({ + key: 'client.locus.join.response', + value: 40, + }); + assert.deepEqual(cdl.getTotalJMT(), 20); + }); - it('calculates getTotalJMT correctly when interstitialToJoinOk is 0', () => { - cdl.saveTimestamp({ - key: 'internal.client.interstitial-window.click.joinbutton', - value: 40, - }); - cdl.saveLatency('internal.click.to.interstitial', 12); - cdl.saveTimestamp({ - key: 'client.locus.join.response', - value: 40, - }); - assert.deepEqual(cdl.getTotalJMT(), 12); + it('calculates getTotalJMT correctly when interstitialToJoinOk is 0', () => { + cdl.saveTimestamp({ + key: 'internal.client.interstitial-window.click.joinbutton', + value: 40, + }); + cdl.saveLatency('internal.click.to.interstitial', 12); + cdl.saveTimestamp({ + key: 'client.locus.join.response', + value: 40, }); + assert.deepEqual(cdl.getTotalJMT(), 12); + }); - it('calculates getTotalJMT correctly when both clickToInterstitial and interstitialToJoinOk are 0', () => { - cdl.saveTimestamp({ - key: 'internal.client.interstitial-window.click.joinbutton', - value: 40, - }); - cdl.saveLatency('internal.click.to.interstitial', 0); - cdl.saveTimestamp({ - key: 'client.locus.join.response', - value: 40, - }); - assert.deepEqual(cdl.getTotalJMT(), 0); + it('calculates getTotalJMT correctly when both clickToInterstitial and interstitialToJoinOk are 0', () => { + cdl.saveTimestamp({ + key: 'internal.client.interstitial-window.click.joinbutton', + value: 40, }); + cdl.saveLatency('internal.click.to.interstitial', 0); + cdl.saveTimestamp({ + key: 'client.locus.join.response', + value: 40, + }); + assert.deepEqual(cdl.getTotalJMT(), 0); + }); - it('calculates getTotalJMT correctly when both clickToInterstitial is not a number', () => { - cdl.saveTimestamp({ - key: 'internal.client.interstitial-window.click.joinbutton', - value: 40, - }); - cdl.saveLatency('internal.click.to.interstitial', 'eleven' as unknown as number); - cdl.saveTimestamp({ - key: 'client.locus.join.response', - value: 40, - }); - assert.deepEqual(cdl.getTotalJMT(), undefined); + it('calculates getTotalJMT correctly when both clickToInterstitial is not a number', () => { + cdl.saveTimestamp({ + key: 'internal.client.interstitial-window.click.joinbutton', + value: 40, }); + cdl.saveLatency('internal.click.to.interstitial', 'eleven' as unknown as number); + cdl.saveTimestamp({ + key: 'client.locus.join.response', + value: 40, + }); + assert.deepEqual(cdl.getTotalJMT(), undefined); + }); it('calculates getTotalJMT correctly when it is greater than MAX_INTEGER', () => { cdl.saveTimestamp({ @@ -740,70 +740,73 @@ describe('internal-plugin-metrics', () => { assert.deepEqual(cdl.getTotalJMTWithUserDelay(), 45); }); - it('calculates getTotalJMTWithUserDelay correctly when clickToInterstitialWithUserDelay is 0', () => { - cdl.saveLatency('internal.click.to.interstitial.with.user.delay', 0); - cdl.saveTimestamp({ - key: 'internal.client.interstitial-window.click.joinbutton', - value: 20, - }); - cdl.saveTimestamp({ - key: 'client.locus.join.response', - value: 40, - }); - assert.deepEqual(cdl.getTotalJMTWithUserDelay(), 20); + it('calculates getTotalJMTWithUserDelay correctly when clickToInterstitialWithUserDelay is 0', () => { + cdl.saveLatency('internal.click.to.interstitial.with.user.delay', 0); + cdl.saveTimestamp({ + key: 'internal.client.interstitial-window.click.joinbutton', + value: 20, + }); + cdl.saveTimestamp({ + key: 'client.locus.join.response', + value: 40, }); + assert.deepEqual(cdl.getTotalJMTWithUserDelay(), 20); + }); - it('calculates getTotalJMTWithUserDelay correctly when interstitialToJoinOk is 0', () => { - cdl.saveTimestamp({ - key: 'internal.client.interstitial-window.click.joinbutton', - value: 40, - }); - cdl.saveLatency('internal.click.to.interstitial.with.user.delay', 12); - cdl.saveTimestamp({ - key: 'client.locus.join.response', - value: 40, - }); - assert.deepEqual(cdl.getTotalJMTWithUserDelay(), 12); + it('calculates getTotalJMTWithUserDelay correctly when interstitialToJoinOk is 0', () => { + cdl.saveTimestamp({ + key: 'internal.client.interstitial-window.click.joinbutton', + value: 40, + }); + cdl.saveLatency('internal.click.to.interstitial.with.user.delay', 12); + cdl.saveTimestamp({ + key: 'client.locus.join.response', + value: 40, }); + assert.deepEqual(cdl.getTotalJMTWithUserDelay(), 12); + }); - it('calculates getTotalJMTWithUserDelay correctly when both clickToInterstitialWithUserDelay and interstitialToJoinOk are 0', () => { - cdl.saveTimestamp({ - key: 'internal.client.interstitial-window.click.joinbutton', - value: 40, - }); - cdl.saveLatency('internal.click.to.interstitial.with.user.delay', 0); - cdl.saveTimestamp({ - key: 'client.locus.join.response', - value: 40, - }); - assert.deepEqual(cdl.getTotalJMTWithUserDelay(), 0); + it('calculates getTotalJMTWithUserDelay correctly when both clickToInterstitialWithUserDelay and interstitialToJoinOk are 0', () => { + cdl.saveTimestamp({ + key: 'internal.client.interstitial-window.click.joinbutton', + value: 40, }); + cdl.saveLatency('internal.click.to.interstitial.with.user.delay', 0); + cdl.saveTimestamp({ + key: 'client.locus.join.response', + value: 40, + }); + assert.deepEqual(cdl.getTotalJMTWithUserDelay(), 0); + }); - it('calculates getTotalJMTWithUserDelay correctly when both clickToInterstitialWithUserDelay is not a number', () => { - cdl.saveTimestamp({ - key: 'internal.client.interstitial-window.click.joinbutton', - value: 40, - }); - cdl.saveLatency('internal.click.to.interstitial.with.user.delay', 'eleven' as unknown as number); - cdl.saveTimestamp({ - key: 'client.locus.join.response', - value: 40, - }); - assert.deepEqual(cdl.getTotalJMTWithUserDelay(), undefined); + it('calculates getTotalJMTWithUserDelay correctly when both clickToInterstitialWithUserDelay is not a number', () => { + cdl.saveTimestamp({ + key: 'internal.client.interstitial-window.click.joinbutton', + value: 40, + }); + cdl.saveLatency( + 'internal.click.to.interstitial.with.user.delay', + 'eleven' as unknown as number + ); + cdl.saveTimestamp({ + key: 'client.locus.join.response', + value: 40, }); + assert.deepEqual(cdl.getTotalJMTWithUserDelay(), undefined); + }); - it('calculates getTotalJMTWithUserDelay correctly when it is greater than MAX_INTEGER', () => { - cdl.saveLatency('internal.click.to.interstitial.with.user.delay', 2147483648); - cdl.saveTimestamp({ - key: 'internal.client.interstitial-window.click.joinbutton', - value: 20, - }); - cdl.saveTimestamp({ - key: 'client.locus.join.response', - value: 40, - }); - assert.deepEqual(cdl.getTotalJMTWithUserDelay(), 2147483647); + it('calculates getTotalJMTWithUserDelay correctly when it is greater than MAX_INTEGER', () => { + cdl.saveLatency('internal.click.to.interstitial.with.user.delay', 2147483648); + cdl.saveTimestamp({ + key: 'internal.client.interstitial-window.click.joinbutton', + value: 20, }); + cdl.saveTimestamp({ + key: 'client.locus.join.response', + value: 40, + }); + assert.deepEqual(cdl.getTotalJMTWithUserDelay(), 2147483647); + }); it('calculates getTotalMediaJMT correctly', () => { cdl.saveTimestamp({ @@ -827,7 +830,7 @@ describe('internal-plugin-metrics', () => { value: 20, }); cdl.saveTimestamp({ - key: 'internal.host.meeting.participant.admitted', + key: 'client.lobby.exited', value: 24, }); cdl.saveTimestamp({ @@ -863,7 +866,7 @@ describe('internal-plugin-metrics', () => { value: 2147483700, }); cdl.saveTimestamp({ - key: 'internal.host.meeting.participant.admitted', + key: 'client.lobby.exited', value: 2147483800, }); cdl.saveTimestamp({ @@ -900,7 +903,7 @@ describe('internal-plugin-metrics', () => { value: 20, }); cdl.saveTimestamp({ - key: 'internal.host.meeting.participant.admitted', + key: 'client.lobby.exited', value: 24, }); cdl.saveTimestamp({ @@ -937,7 +940,7 @@ describe('internal-plugin-metrics', () => { value: 2147483700, }); cdl.saveTimestamp({ - key: 'internal.host.meeting.participant.admitted', + key: 'client.lobby.exited', value: 2147483800, }); cdl.saveTimestamp({ @@ -1041,20 +1044,20 @@ describe('internal-plugin-metrics', () => { // the maximum possible sum is 2400000, which is less than MAX_INTEGER (2147483647). // This test should verify that the final clamping works by mocking the intermediate methods // to return values that would sum to more than MAX_INTEGER. - + const originalGetJoinReqResp = cdl.getJoinReqResp; const originalGetICESetupTime = cdl.getICESetupTime; - + // Mock the methods to return large values that would exceed MAX_INTEGER when summed cdl.getJoinReqResp = () => 1500000000; cdl.getICESetupTime = () => 1000000000; - + const result = cdl.getJoinConfJMT(); - + // Restore original methods cdl.getJoinReqResp = originalGetJoinReqResp; cdl.getICESetupTime = originalGetICESetupTime; - + assert.deepEqual(result, 2147483647); }); @@ -1140,7 +1143,7 @@ describe('internal-plugin-metrics', () => { value: 10, }); cdl.saveTimestamp({ - key: 'internal.host.meeting.participant.admitted', + key: 'client.lobby.exited', value: 12, }); cdl.saveTimestamp({ @@ -1160,7 +1163,7 @@ describe('internal-plugin-metrics', () => { value: 10, }); cdl.saveTimestamp({ - key: 'internal.host.meeting.participant.admitted', + key: 'client.lobby.exited', value: 12, }); cdl.saveTimestamp({ diff --git a/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics.util.ts b/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics.util.ts index 406ea91019b..53e5009d15c 100644 --- a/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics.util.ts +++ b/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics.util.ts @@ -311,7 +311,7 @@ describe('internal-plugin-metrics', () => { origin: { buildType: 'prod', networkType: 'unknown', - upgradeChannel: expectedUpgradeChannel + upgradeChannel: expectedUpgradeChannel, }, event: {name: eventName, ...expectedEvent}, }, @@ -393,7 +393,7 @@ describe('internal-plugin-metrics', () => { totalJmt: undefined, clientJmt: undefined, downloadTime: undefined, - clickToInterstitialWithUserDelay: undefined, + clickToInterstitialWithUserDelay: undefined, totalJMTWithUserDelay: undefined, }, }, @@ -430,7 +430,6 @@ describe('internal-plugin-metrics', () => { totalMediaJMT: undefined, interstitialToMediaOKJMT: undefined, callInitMediaEngineReady: undefined, - stayLobbyTime: undefined, totalMediaJMTWithUserDelay: undefined, totalJMTWithUserDelay: undefined, }, @@ -447,6 +446,14 @@ describe('internal-plugin-metrics', () => { }, }, ], + [ + 'client.lobby.exited', + { + joinTimes: { + stayLobbyTime: undefined, + }, + }, + ], ].forEach(([eventName, expectedEvent]) => { it(`returns expected result for ${eventName}`, () => { check(eventName as string, expectedEvent, 'gold'); From 3db31e87c725ccfea3f6fa086028bc573ad9684d Mon Sep 17 00:00:00 2001 From: Hem Dutt Date: Mon, 23 Mar 2026 15:33:54 +0530 Subject: [PATCH 03/28] feat(internal-call-ai-assistant): ai generated summaries in call history (#4753) Co-authored-by: Hem Dutt --- .../.eslintrc.js | 6 + .../internal-plugin-call-ai-summary/README.md | 257 ++++ .../ai-docs/AGENTS.md | 300 +++++ .../ai-docs/ARCHITECTURE.md | 1189 +++++++++++++++++ .../babel.config.js | 3 + .../jest.config.js | 3 + .../package.json | 46 + .../src/ai-summary.ts | 318 +++++ .../src/config.ts | 7 + .../src/constants.ts | 19 + .../src/index.ts | 13 + .../src/manual-integration-test.js | 129 ++ .../src/manual-pragya-api-test.js | 166 +++ .../src/types.ts | 154 +++ yarn.lock | 19 + 15 files changed, 2629 insertions(+) create mode 100644 packages/@webex/internal-plugin-call-ai-summary/.eslintrc.js create mode 100644 packages/@webex/internal-plugin-call-ai-summary/README.md create mode 100644 packages/@webex/internal-plugin-call-ai-summary/ai-docs/AGENTS.md create mode 100644 packages/@webex/internal-plugin-call-ai-summary/ai-docs/ARCHITECTURE.md create mode 100644 packages/@webex/internal-plugin-call-ai-summary/babel.config.js create mode 100644 packages/@webex/internal-plugin-call-ai-summary/jest.config.js create mode 100644 packages/@webex/internal-plugin-call-ai-summary/package.json create mode 100644 packages/@webex/internal-plugin-call-ai-summary/src/ai-summary.ts create mode 100644 packages/@webex/internal-plugin-call-ai-summary/src/config.ts create mode 100644 packages/@webex/internal-plugin-call-ai-summary/src/constants.ts create mode 100644 packages/@webex/internal-plugin-call-ai-summary/src/index.ts create mode 100644 packages/@webex/internal-plugin-call-ai-summary/src/manual-integration-test.js create mode 100644 packages/@webex/internal-plugin-call-ai-summary/src/manual-pragya-api-test.js create mode 100644 packages/@webex/internal-plugin-call-ai-summary/src/types.ts diff --git a/packages/@webex/internal-plugin-call-ai-summary/.eslintrc.js b/packages/@webex/internal-plugin-call-ai-summary/.eslintrc.js new file mode 100644 index 00000000000..a4fc83c6f44 --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/.eslintrc.js @@ -0,0 +1,6 @@ +const config = { + root: true, + extends: ['@webex/eslint-config-legacy'], +}; + +module.exports = config; diff --git a/packages/@webex/internal-plugin-call-ai-summary/README.md b/packages/@webex/internal-plugin-call-ai-summary/README.md new file mode 100644 index 00000000000..ef72be791a7 --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/README.md @@ -0,0 +1,257 @@ +# @webex/internal-plugin-call-ai-summary + +Internal Webex JS SDK plugin for retrieving AI-generated call summaries, notes, action items, and transcripts from completed calls. + +## Overview + +This plugin resolves AI summary containers via the **Pragya** service and fetches encrypted summary content from URLs returned by Pragya. All AI-generated content is decrypted using KMS keys provided in the container response. + +**Discovery flow:** + +1. **Janus** (call history) returns `extensionPayload.callingContainerIds` per call session +2. **Pragya** resolves a container ID into metadata including content URLs and encryption key +3. **Plugin** fetches content from those URLs and decrypts using `@webex/internal-plugin-encryption` + +## Install + +This plugin is part of the Webex JS SDK monorepo. It self-registers when imported — no changes to `packages/webex` are needed. + +```bash +# From the SDK monorepo root +yarn +``` + +To use in a consuming application: + +```javascript +// Importing the plugin auto-registers it on webex.internal.aisummary +import '@webex/internal-plugin-call-ai-summary'; +``` + +## Prerequisites + +- An authenticated Webex SDK instance with a registered device +- `@webex/internal-plugin-encryption` (pulled in automatically as a dependency) +- A valid Pragya container ID (obtained from Janus call history `extensionPayload.callingContainerIds`) + +## API + +All methods are accessible via `webex.internal.aisummary`. + +### `getContainer({ containerId })` + +Resolves a Pragya container by ID. Returns container metadata with summary content URLs and the KMS encryption key. + +The raw Pragya response nests URLs under `summaryData.data` — this method flattens it so you can access `summaryData.summaryUrl` directly. + +```typescript +const container = await webex.internal.aisummary.getContainer({ + containerId: '34125120-13b5-11f1-9b36-adb685725098', +}); + +// container.summaryData.summaryUrl — full summary URL +// container.summaryData.transcriptUrl — transcript URL +// container.summaryData.status — "Active" when ready +// container.encryptionKeyUrl — KMS key for decryption +``` + +**Returns:** `Promise` + +### `getSummary({ containerInfo })` + +Fetches and decrypts all summary content (note, short note, and action items) in a single request via `summaryUrl?fields=note,shortnote,actionitems`. + +This is the **recommended** method for retrieving summary content. + +```typescript +const summary = await webex.internal.aisummary.getSummary({ + containerInfo: container, +}); + +console.log(summary.note); // Decrypted full note +console.log(summary.shortNote); // Decrypted short note +summary.actionItems.forEach((item) => { + console.log(item.aiGeneratedContent); // Decrypted action item +}); +``` + +**Returns:** `Promise` — `{ id, note, shortNote, actionItems, feedbackUrl? }` + +### `getNotes({ containerInfo })` + +Fetches and decrypts notes from the standalone `notesUrl` endpoint. Only available if `notesUrl` is present in the Pragya response. + +```typescript +const notes = await webex.internal.aisummary.getNotes({ + containerInfo: container, +}); + +console.log(notes.content); // Decrypted notes +``` + +**Returns:** `Promise` — `{ id, content, feedbackUrl? }` + +### `getActionItems({ containerInfo })` + +Fetches and decrypts action items from the standalone `actionItemsUrl` endpoint. Only available if `actionItemsUrl` is present in the Pragya response. + +```typescript +const actionItems = await webex.internal.aisummary.getActionItems({ + containerInfo: container, +}); + +actionItems.snippets.forEach((snippet) => { + console.log(snippet.aiGeneratedContent); // Decrypted + console.log(snippet.editedContent); // User-edited version (if any) +}); +``` + +**Returns:** `Promise` — `{ id?, snippets[], feedbackUrl? }` + +### `getTranscriptUrl({ containerInfo })` + +Returns the transcript URL string without fetching or decrypting. Use this when you need the URL for downstream processing. + +```typescript +const url = webex.internal.aisummary.getTranscriptUrl({ + containerInfo: container, +}); +``` + +**Returns:** `string` + +### `getTranscript({ containerInfo })` + +Fetches and decrypts the full call transcript. + +```typescript +const transcript = await webex.internal.aisummary.getTranscript({ + containerInfo: container, +}); + +transcript.snippets.forEach((snippet) => { + console.log(`[${snippet.startTime}] ${snippet.speaker?.speakerName}: ${snippet.content}`); +}); +``` + +**Returns:** `Promise` — `{ id, totalCount, snippets[] }` + +## Full Usage Example + +```typescript +import '@webex/internal-plugin-call-ai-summary'; + +// 1. Get call history (existing SDK API) +const callHistory = await callHistoryInstance.getCallHistoryData(10, 50); +const sessions = callHistory.data.userSessions; + +// 2. Find a session with AI summary +const session = sessions.find( + (s) => s.extensionPayload?.callingContainerIds?.length > 0 +); +if (!session) return; + +// 3. Resolve the container +const containerId = session.extensionPayload.callingContainerIds[0]; +const container = await webex.internal.aisummary.getContainer({ containerId }); + +if (container.summaryData.status !== 'Active') { + console.log('Summary not ready yet'); + return; +} + +// 4. Fetch all summary content in one call +const summary = await webex.internal.aisummary.getSummary({ containerInfo: container }); +console.log('Note:', summary.note); +console.log('Short Note:', summary.shortNote); +summary.actionItems.forEach((item, i) => { + console.log(`Action ${i + 1}: ${item.aiGeneratedContent}`); +}); + +// 5. Fetch transcript +const transcript = await webex.internal.aisummary.getTranscript({ containerInfo: container }); +transcript.snippets.forEach((s) => { + console.log(`[${s.startTime}] ${s.speaker?.speakerName}: ${s.content}`); +}); +``` + +## Manual Testing + +A manual integration test is provided for verifying against live APIs: + +```bash +cd packages/@webex/internal-plugin-call-ai-summary + +# Provide a fresh token and container ID +WEBEX_TOKEN='' CONTAINER_ID='' node src/manual-integration-test.js +``` + +This script registers a device (WDM), resolves the Pragya service via the SDK service catalog, fetches the container, decrypts summary content via KMS, and prints the results. + +## Error Handling + +| Error | Cause | Recovery | +|-------|-------|----------| +| `containerId is required and must be a non-empty string` | Empty or missing containerId | Validate input before calling | +| `containerInfo with valid summaryData and encryptionKeyUrl is required` | Missing container info or URL field | Call `getContainer()` first | +| `Container not found` | 404 from Pragya | Verify containerId from Janus | +| `Summary content not available or expired` | 404 from content endpoint | Content may have been deleted | +| `Access denied: User not authorized to view this summary` | 403 | Check user permissions / org AI settings | +| `Authentication failed: Invalid or expired token` | 401 | Re-authenticate the user | + +## Encryption + +All AI-generated content (`aiGeneratedContent` fields) is encrypted with KMS. The plugin decrypts automatically using: + +- **Key source:** `encryptionKeyUrl` from the Pragya container response, with fallback to `keyUrl` from the content response body +- **Decryption method:** `webex.internal.encryption.decryptText(keyUrl, ciphertext)` + +This is the same pattern used by `@webex/internal-plugin-ai-assistant` and `@webex/internal-plugin-task`. + +## Development + +```bash +cd packages/@webex/internal-plugin-call-ai-summary + +# Build +yarn build + +# Lint +yarn test:style + +# Unit tests +yarn test:unit + +# All checks +yarn test +``` + +## Package Structure + +``` +src/ + index.ts # Self-registration via registerInternalPlugin('aisummary', ...) + ai-summary.ts # Plugin implementation (WebexPlugin.extend) + config.ts # Plugin config + constants.ts # Service name, error messages + types.ts # TypeScript interfaces +test/ + unit/ + spec/ + ai-summary.ts # Unit tests (26 tests) + data/ + responses.ts # Mock API response fixtures +ai-docs/ + ARCHITECTURE.md # Detailed architecture document +``` + +## Dependencies + +| Package | Purpose | +|---------|---------| +| `@webex/webex-core` | Plugin infrastructure (`WebexPlugin`, `registerInternalPlugin`) | +| `@webex/internal-plugin-encryption` | KMS content decryption via `decryptText()` | + +## Architecture + +See [ai-docs/ARCHITECTURE.md](ai-docs/ARCHITECTURE.md) for the full architecture document covering data flows, API request/response details, DTOs, security considerations, and testing strategy. diff --git a/packages/@webex/internal-plugin-call-ai-summary/ai-docs/AGENTS.md b/packages/@webex/internal-plugin-call-ai-summary/ai-docs/AGENTS.md new file mode 100644 index 00000000000..263c4265e40 --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/ai-docs/AGENTS.md @@ -0,0 +1,300 @@ +# @webex/internal-plugin-call-ai-summary + +This is an internal Cisco Webex plugin. As such, it does not strictly adhere to semantic versioning. Use at your own risk. If you're not working on one of our first party clients, please look at our developer api and stick to our public plugins. +Internal Webex JS SDK plugin for retrieving AI-generated call summaries, notes, action items, and transcript URLs from the Pragya and AI Bridge services. + +## Overview + +This plugin provides methods to: + +1. Resolve a **Pragya container** by ID (returns metadata, summary URLs, and encryption key) +2. Fetch and decrypt **AI-generated summaries** (note, short note, action items) in a single call +3. Fetch and decrypt **AI-generated notes** via a dedicated notes endpoint +4. Fetch and decrypt **AI-generated action items** via a dedicated action items endpoint +5. Retrieve the **transcript URL** for a call + +All AI-generated content is **JWE-encrypted** and decrypted via the KMS (Key Management Service) using `@webex/internal-plugin-encryption`. + +## Architecture + +``` +Pragya Service AI Bridge Service +(container metadata) (summary content) + | | + getContainer() getSummary() / getNotes() / getActionItems() + | | + v v + PragyaContainerResponse Encrypted JWE content + (summaryData, encryptionKeyUrl) | + v + KMS Decryption + (internal-plugin-encryption) + | + v + Decrypted plaintext (HTML) +``` + +**Note:** The Pragya API returns summary URLs nested under `summaryData.data`. The `getContainer()` method normalizes this automatically, flattening `summaryData.data` into `summaryData` so consumers can access `summaryData.summaryUrl` directly. + +## Registration + +The plugin registers itself as `aisummary` on the internal namespace: + +```typescript +import '@webex/internal-plugin-call-ai-summary'; + +// Accessed via: +webex.internal.aisummary.getContainer({ containerId: '...' }); +``` + +## Source Files + +| File | Description | +|------|-------------| +| `src/index.ts` | Entry point. Registers the plugin via `registerInternalPlugin('aisummary', ...)`. | +| `src/ai-summary.ts` | Main plugin class extending `WebexPlugin`. Contains all public and private methods. | +| `src/types.ts` | TypeScript interfaces for request/response DTOs. | +| `src/constants.ts` | Service name, resource path, and error message constants. | +| `src/config.ts` | Plugin configuration (currently empty). | + +## API Reference + +### `getContainer(options: GetContainerOptions): Promise` + +Resolves a Pragya container by ID. Returns container metadata including summary URLs and the KMS encryption key URL. Normalizes the response by flattening `summaryData.data` into `summaryData`. + +```typescript +const container = await webex.internal.aisummary.getContainer({ + containerId: '34125120-13b5-11f1-9b36-adb685725098', +}); + +// After normalization, URLs are directly on summaryData: +console.log(container.summaryData.summaryUrl); // https://aibridge-.../summaries/... +console.log(container.summaryData.transcriptUrl); // https://aibridge-.../transcripts/... +``` + +**Request**: `GET {pragya-service}/containers/{containerId}` + +**Response fields**: +- `summaryData` — Contains summary URLs (`summaryUrl`, `transcriptUrl`, `status`, `summarizeAfterCall`) +- `encryptionKeyUrl` — KMS key URL for decrypting content (e.g., `kms://kms-aore.wbx2.com/keys/...`) +- `kmsResourceObjectUrl`, `aclUrl`, `forkSessionId`, `callSessionId`, `ownerUserId`, `orgId`, `start`, `end` + +### `getSummary(options: GetSummaryContentOptions): Promise` + +Fetches all AI-generated summary content (note, short note, and action items) from a single request to the summary URL, and decrypts each field via KMS. This is the primary method for retrieving summary content. + +```typescript +const summary = await webex.internal.aisummary.getSummary({ + containerInfo: container, +}); + +console.log(summary.note); // Decrypted full note (HTML) +console.log(summary.shortNote); // Decrypted short note (HTML) +console.log(summary.actionItems); // Array of decrypted action item snippets +console.log(summary.feedbackUrl); // Feedback URL from links (if available) +``` + +**Request**: `GET {summaryUrl}?fields=note,shortnote,actionitems` + +**Response structure** (from AI Bridge, before decryption): +```json +{ + "id": "...", + "keyUrl": "kms://...", + "note": { "aiGeneratedContent": "" }, + "shortnote": { "aiGeneratedContent": "" }, + "actionitems": { + "snippets": [ + { "id": "...", "aiGeneratedContent": "" } + ] + }, + "links": [ + { "rel": "feedback", "href": "https://..." } + ] +} +``` + +**Return type** (`SummaryContent`): +- `id` — Summary identifier +- `note` — Decrypted full note (HTML string) +- `shortNote` — Decrypted short note (HTML string) +- `actionItems` — Array of `ActionItemSnippet` objects +- `feedbackUrl` — Extracted from `links` array (`rel: "feedback"`), if available + +### `getNotes(options: GetSummaryContentOptions): Promise` + +Fetches AI-generated notes from the dedicated notes endpoint and decrypts via KMS. Requires `notesUrl` to be present in the container's `summaryData`. + +```typescript +const notes = await webex.internal.aisummary.getNotes({ + containerInfo: container, +}); + +console.log(notes.content); // Decrypted notes content +``` + +**Request**: `GET {notesUrl}` + +> **Note:** The `notesUrl` may not be present in all API versions. Prefer `getSummary()` which returns notes, short notes, and action items in a single call. + +### `getActionItems(options: GetSummaryContentOptions): Promise` + +Fetches AI-generated action items from the dedicated action items endpoint and decrypts each snippet via KMS. Requires `actionItemsUrl` to be present in the container's `summaryData`. + +```typescript +const actionItems = await webex.internal.aisummary.getActionItems({ + containerInfo: container, +}); + +actionItems.snippets.forEach((item) => { + console.log(item.aiGeneratedContent); // Decrypted action item +}); +``` + +**Request**: `GET {actionItemsUrl}` + +> **Note:** The `actionItemsUrl` may not be present in all API versions. Prefer `getSummary()` which returns notes, short notes, and action items in a single call. + +### `getTranscriptUrl(options: GetSummaryContentOptions): string` + +Returns the transcript URL from the container info. Does not fetch or decrypt content. + +```typescript +const transcriptUrl = webex.internal.aisummary.getTranscriptUrl({ + containerInfo: container, +}); +``` + +## Types + +### Request Types + +```typescript +interface GetContainerOptions { + containerId: string; // Pragya container ID +} + +interface GetSummaryContentOptions { + containerInfo: PragyaContainerResponse; // Resolved container from getContainer() +} +``` + +### Response Types + +```typescript +interface PragyaContainerResponse { + summaryData: PragyaSummaryData; + encryptionKeyUrl: string; + kmsResourceObjectUrl: string; + aclUrl: string; + forkSessionId: string; + callSessionId: string; + ownerUserId: string; + orgId: string; + start: string; + end: string; +} + +interface PragyaSummaryData { + status: string; + summaryUrl: string; + transcriptUrl: string; + summarizeAfterCall: boolean; + notesUrl?: string; // May not be present in all API versions + actionItemsUrl?: string; // May not be present in all API versions +} + +interface SummaryContent { + id: string; + note: string; // Decrypted full note (HTML) + shortNote: string; // Decrypted short note (HTML) + actionItems: ActionItemSnippet[]; + feedbackUrl?: string; // From links array (rel="feedback") +} + +interface SummaryNotes { + id: string; + content: string; // Decrypted notes content + feedbackUrl?: string; +} + +interface SummaryActionItems { + id: string; + snippets: ActionItemSnippet[]; + feedbackUrl?: string; +} + +interface ActionItemSnippet { + id: string; + editedContent?: string; // User-edited version (if available) + aiGeneratedContent: string; // Decrypted AI-generated content +} +``` + +## Error Handling + +The plugin normalizes HTTP errors into descriptive messages: + +| Status Code | Error Message | +|-------------|---------------| +| 401 | `Authentication failed: Invalid or expired token` | +| 403 | `Access denied: User not authorized to view this summary` | +| 404 | `Container not found` | +| Other | `{methodName} failed: {error.message}` | + +Validation errors are thrown synchronously: +- Missing or empty `containerId` throws `containerId is required and must be a non-empty string` +- Missing `containerInfo`, `summaryData` URL, or `encryptionKeyUrl` throws `containerInfo with valid summaryData and encryptionKeyUrl is required` + +## Encryption / Decryption + +All AI-generated content from the AI Bridge service is JWE-encrypted. Decryption uses: + +``` +webex.internal.encryption.decryptText(encryptionKeyUrl, encryptedContent) +``` + +This requires: +1. A registered device (`webex.internal.device.register()`) +2. Mercury WebSocket connection (initiated automatically during KMS key fetch) +3. ECDHE key exchange with KMS +4. Key retrieval from KMS using the `encryptionKeyUrl` + +The SDK handles steps 1-4 automatically when `decryptText` is called. + +## Dependencies + +- `@webex/webex-core` — Base plugin class, request handling, auth interceptor +- `@webex/internal-plugin-encryption` — KMS decryption + +## Token Requirements + +The Pragya and AI Bridge APIs require a valid Webex access token. The SDK's auth interceptor automatically attaches the token for URLs in the service catalog or on allowed domains (e.g., `wbx2.com`, `webex.com`). + +## Manual Testing + +Two manual test scripts are provided in `src/`: + +### `manual-pragya-api-test.js` +Validates the Pragya container response structure (34 checks). + +```bash +cd packages/@webex/internal-plugin-call-ai-summary +WEBEX_TOKEN='' node src/manual-pragya-api-test.js +``` + +### `manual-integration-test.js` +Tests the full end-to-end flow using the SDK service catalog: +1. Device registration (WDM) to populate the service catalog +2. `getContainer` via plugin (resolves `service: 'pragya'` from the catalog) +3. `getSummary` via plugin (fetches + decrypts note, short note, and action items via KMS) +4. `getTranscriptUrl` via plugin +5. Transcript content fetch + +```bash +cd packages/@webex/internal-plugin-call-ai-summary +WEBEX_TOKEN='' CONTAINER_ID='' node src/manual-integration-test.js +``` + +Both scripts require a valid Webex access token. Set `WEBEX_TOKEN` and optionally `CONTAINER_ID` as environment variables, or update the placeholder values in the scripts. diff --git a/packages/@webex/internal-plugin-call-ai-summary/ai-docs/ARCHITECTURE.md b/packages/@webex/internal-plugin-call-ai-summary/ai-docs/ARCHITECTURE.md new file mode 100644 index 00000000000..0200ee89f2b --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/ai-docs/ARCHITECTURE.md @@ -0,0 +1,1189 @@ +# AI Call Summary Architecture + +## 1. Overview + +The Webex JS SDK will provide AI-generated call summary retrieval capabilities through a new **`internal-plugin-call-ai-summary`** internal plugin. This document describes the architecture for retrieving AI-generated notes, action items, and transcripts from completed calls. + +### 1.1 Summary Discovery Flow + +AI summary content is discovered through a two-step lookup: **Janus** (call history) provides container IDs, and **Pragya** (AI container service) resolves those IDs into direct URLs for summary content. + +**Step 1: Get container IDs from Janus call history** + +The Janus `UserSession` response includes an `extensionPayload` field containing container IDs for AI artifacts related to a call: + +```typescript +export type UserSession = { + id: string; + sessionId: string; + disposition: Disposition; + startTime: string; + endTime: string; + url: string; + durationSeconds: number; + joinedDurationSeconds: number; + participantCount: number; + isDeleted: boolean; + isPMR: boolean; + correlationIds: string[]; + links: CallRecordLink; + self: CallRecordSelf; + other: CallRecordListOther; + sessionType: SessionType; + direction: string; + callingSpecifics?: { redirectionDetails: RedirectionDetails }; + extensionPayload?: { + callingContainerIds?: string[]; + }; +}; +``` + +> **Note:** The `extensionPayload.callingContainerIds` field is already present in the Janus API response but is not yet in the SDK's `UserSession` type definition at `packages/calling/src/Events/types.ts`. However, this plugin does **not** modify that type. It accepts a plain `containerId` string as input, keeping the plugin self-contained. The `UserSession` type update can be handled separately by the calling package team when convenient. + +**Step 2: Resolve container IDs via Pragya** + +For each `containerId`, call the Pragya container API: + +``` +GET https://{pragya-host}/pragya/api/v1/containers/{containerId} +``` + +**Pragya response:** + +> **Note:** The raw Pragya response nests summary URLs under `summaryData.data`. The plugin's `getContainer()` method flattens this so consumers can access `summaryData.summaryUrl` directly. + +```json +{ + "id": "34125120-13b5-11f1-9b36-adb685725098", + "objectType": "callingAIContainer", + "memberships": { + "items": [ + { "id": "...", "roles": ["OWNER"], "objectType": "containerMembership" } + ] + }, + "summaryData": { + "extensionId": "...", + "objectType": "extension", + "extensionType": "callingAISummary", + "data": { + "id": "...", + "objectType": "callingAISummary", + "status": "Active", + "summaryUrl": "https://aibridge-url/summaries/c635e870-7b3b-4b3b-8b3b-7b3b7b3b7b3c", + "transcriptUrl": "https://aibridge-url/summaries/c635e870-7b3b-4b3b-8b3b-7b3b7b3b7b3c/transcripts", + "summarizeAfterCall": true, + "aclUrl": "https://acl-a.wbx2.com/...", + "kmsResourceObjectUrl": "kms://kms-cisco.wbx2.com/resources/...", + "contentRetention": { ... } + } + }, + "encryptionKeyUrl": "kms://kms-cisco.wbx2.com/keys/897e4d2d-6219-433d-be77-7ec73fe1c0db", + "kmsResourceObjectUrl": "kms://kms-cisco.wbx2.com/resources/f7316435-2147-4d23-bf4a-762d831cb58c", + "aclUrl": "https://acl-a.wbx2.com/acl/api/v1/acls/78c4cd90-f880-11ee-96e9-3932dce37910", + "forkSessionId": "123e4567-e89b-12d3-a456-426614174000", + "callSessionId": "123e4567-e89b-12d3-a456-426614174000", + "ownerUserId": "123e4567-e89b-12d3-a456-426614174000", + "orgId": "123e4567-e89b-12d3-a456-426614174000", + "start": "2023-10-01T12:00:00Z", + "end": "2023-10-01T12:00:00Z" +} +``` + +**Step 3: Fetch summary content from the URLs** + +The `summaryData` object provides direct, region-correct URLs to each content type. The plugin fetches content from these URLs and decrypts it using the `encryptionKeyUrl` from the same Pragya response. + +### 1.2 Key Design Decisions + +- **Self-contained plugin with zero changes to existing packages.** The plugin owns all of its types, constants, and logic. It does not modify `UserSession`, `CallHistory`, or any other existing code. Consumers pass a `containerId` string; how they obtain it (Janus, Mercury event, hard-coded for testing) is their concern. +- **No separate service discovery for summary endpoints.** Pragya returns fully-qualified URLs that already include the correct regional host. The SDK fetches from these URLs directly using `uri:` rather than `service:` + `resource:`. +- **Pragya is the source of truth** for both the content URLs and the encryption key. +- **Pragya is discoverable via U2C** as `serviceName: "pragya"` (validated: e.g., load-us resolves to `https://pragya-loada.ciscospark.com/pragya/api/v1`). + +### 1.3 Goals + +- Resolve AI summary container IDs from Janus call history via Pragya +- Retrieve AI-generated notes (full notes) for a call +- Retrieve AI-generated action items for a call +- Retrieve transcript download URLs for a call +- Handle encrypted content decryption via KMS +- Maintain consistency with existing Webex JS SDK internal plugin patterns +- Provide type-safe interfaces for all operations +- Support both browser and Node.js environments + +### 1.4 Non-Goals + +- Start/stop AI assistant during active calls (handled by Pragya start/stop APIs, out of scope) +- Generate or regenerate summaries (backend-managed during/after calls) +- Provide real-time in-call AI responses +- Handle recording storage or deletion +- Implement feedback UI components + +### 1.5 Prerequisites + +1. Janus API already returns `extensionPayload.callingContainerIds` in the response +2. Testing environment with AI-enabled calls that generate summaries + +## 2. High-Level Design + +### 2.1 Component Architecture + +``` ++---------------------------------------------------------------+ +| Client Application | ++-------------------------------+-------------------------------+ + | + | webex.internal.aisummary.* + | ++-------------------------------v-------------------------------+ +| internal-plugin-call-ai-summary | +| (Internal Plugin) | +| +----------------------------------------------------------+ | +| | Public API Methods | | +| | - getContainer(containerId) | | +| | - getSummary(containerInfo) | | +| | - getNotes(containerInfo) | | +| | - getActionItems(containerInfo) | | +| | - getTranscriptUrl(containerInfo) | | +| +----------------------------+-----------------------------+ | +| | | +| +----------------------------v-----------------------------+ | +| | Internal Logic | | +| | - Input validation | | +| | - Content decryption (KMS via encryptionKeyUrl) | | +| | - Response normalization | | +| | - Error handling & mapping | | +| +----------------------------+-----------------------------+ | ++-------------------------------+-------------------------------+ + | + +-----------------+-----------------+ + | | ++-------------v--------------+ +-----------------v--------------+ +| Pragya Service | | Summary Content URLs | +| (U2C: serviceName:pragya) | | (Direct URLs from Pragya) | +| GET /containers/{id} | | GET {summaryUrl} | ++----------------------------+ | GET {notesUrl} | + | GET {actionItemsUrl} | + +--------------------------------+ + | + +-----------v--------------------+ + | internal-plugin-encryption | + | decryptText(keyUrl, cipher) | + +--------------------------------+ +``` + +### 2.2 Key Components + +| Component | Responsibility | +|-----------|----------------| +| `internal-plugin-call-ai-summary` | Internal plugin; resolves Pragya containers, fetches and decrypts summary content | +| `internal-plugin-encryption` | KMS integration for decrypting AI-generated content using `encryptionKeyUrl` | +| `http-core` | HTTP transport; adds authorization headers, handles retries | +| Pragya Service | Container metadata; provides content URLs and encryption key | +| Summary Content Endpoints | Serve encrypted AI-generated content (notes, action items, transcripts) | + +## 3. Data Flow + +### 3.1 End-to-End Summary Retrieval Flow + +``` +Client + | + | 1. Get call history + +-> callHistory.getCallHistoryData() + | +-> Janus API: GET /history/userSessions + | +-> Response includes extensionPayload.callingContainerIds + | + | 2. Resolve container + +-> webex.internal.aisummary.getContainer(containerId) + | +-> Pragya API: GET /pragya/api/v1/containers/{containerId} + | +-> Response: { summaryData: { summaryUrl, notesUrl, ... }, encryptionKeyUrl } + | + | 3. Fetch all summary content in one call + +-> webex.internal.aisummary.getSummary({ containerInfo: container }) + +-> HTTP GET {summaryUrl}?fields=note,shortnote,actionitems + +-> Response: { note: {...}, shortnote: {...}, actionitems: {...} } + +-> Decrypt note, shortNote, and all action item snippets + +-> Return { id, note, shortNote, actionItems, feedbackUrl } +``` + +### 3.2 Get Container Info Flow + +``` +Client + +-> webex.internal.aisummary.getContainer({ containerId }) + +-> Validate containerId (non-empty string) + +-> webex.request({ + method: 'GET', + service: 'pragya', + resource: `containers/${containerId}`, + }) + +-> Flatten: if body.summaryData.data exists, set body.summaryData = body.summaryData.data + +-> Return PragyaContainerResponse (with flat summaryData) +``` + +### 3.3 Get Notes Flow + +``` +Client + +-> webex.internal.aisummary.getNotes(containerInfo) + +-> Validate containerInfo has summaryData.notesUrl and encryptionKeyUrl + +-> webex.request({ + method: 'GET', + uri: containerInfo.summaryData.notesUrl, + }) + +-> Response: { id, aiGeneratedContent: "", feedbackUrl?, keyUrl } + +-> Decrypt aiGeneratedContent using containerInfo.encryptionKeyUrl + +-> Return decrypted SummaryNotes +``` + +### 3.4 Get Action Items Flow + +``` +Client + +-> webex.internal.aisummary.getActionItems(containerInfo) + +-> Validate containerInfo has summaryData.actionItemsUrl and encryptionKeyUrl + +-> webex.request({ + method: 'GET', + uri: containerInfo.summaryData.actionItemsUrl, + }) + +-> Response: [{ id, keyUrl, snippets: [{ id, content, aiGeneratedContent }] }] + +-> Decrypt all aiGeneratedContent fields using containerInfo.encryptionKeyUrl + +-> Return decrypted SummaryActionItems +``` + +## 4. SDK Method Interfaces + +### 4.1 Internal API Methods + +```typescript +/** + * AISummary namespace accessible via webex.internal.aisummary + */ +interface AISummary { + /** + * Resolve a Pragya container by ID to get summary URLs and encryption key. + */ + getContainer(options: GetContainerOptions): Promise; + + /** + * Get AI-generated full summary for a call. + * Fetches from summaryUrl with ?fields=note,shortnote,actionitems and decrypts all content. + * Returns note, shortNote, and actionItems in a single response. + */ + getSummary(options: GetSummaryContentOptions): Promise; + + /** + * Get AI-generated notes for a call. + * Fetches from containerInfo.summaryData.notesUrl and decrypts content. + * Only available if notesUrl is present in the Pragya response. + */ + getNotes(options: GetSummaryContentOptions): Promise; + + /** + * Get AI-generated action items for a call. + * Fetches from containerInfo.summaryData.actionItemsUrl and decrypts content. + * Only available if actionItemsUrl is present in the Pragya response. + */ + getActionItems(options: GetSummaryContentOptions): Promise; + + /** + * Get the transcript URL for a call. + * Returns the URL from containerInfo.summaryData.transcriptUrl. + * Does not fetch or decrypt - the consumer uses this URL directly. + */ + getTranscriptUrl(options: GetSummaryContentOptions): string; + + /** + * Get decrypted transcript for a call. + * Fetches from containerInfo.summaryData.transcriptUrl and decrypts each snippet. + */ + getTranscript(options: GetSummaryContentOptions): Promise; +} +``` + +## 5. Data Transfer Objects (DTOs) + +### 5.1 Request DTOs + +```typescript +/** + * Options for resolving a Pragya container + */ +export interface GetContainerOptions { + /** Pragya container ID from Janus extensionPayload.callingContainerIds */ + containerId: string; +} + +/** + * Options for fetching summary content. + * Requires the resolved Pragya container info. + */ +export interface GetSummaryContentOptions { + /** The resolved Pragya container response */ + containerInfo: PragyaContainerResponse; +} +``` + +### 5.2 Pragya Response DTOs + +```typescript +/** + * Summary data URLs from a Pragya container + */ +export interface PragyaSummaryData { + /** Status of the summary (e.g., "Active") */ + status: string; + /** Full summary URL (AI Bridge) */ + summaryUrl: string; + /** Transcript URL (AI Bridge) */ + transcriptUrl: string; + /** Whether summarization runs after call ends */ + summarizeAfterCall: boolean; + /** Notes-specific URL (may not be present in all API versions) */ + notesUrl?: string; + /** Action items URL (may not be present in all API versions) */ + actionItemsUrl?: string; +} + +/** + * Complete Pragya container response + */ +export interface PragyaContainerResponse { + /** Summary data with content URLs */ + summaryData: PragyaSummaryData; + /** KMS encryption key URL for decrypting content */ + encryptionKeyUrl: string; + /** KMS resource object URL */ + kmsResourceObjectUrl: string; + /** ACL URL for access control */ + aclUrl: string; + /** Fork session ID */ + forkSessionId: string; + /** Call session ID */ + callSessionId: string; + /** Owner user ID */ + ownerUserId: string; + /** Organization ID */ + orgId: string; + /** Call start time */ + start: string; + /** Call end time */ + end: string; +} +``` + +### 5.3 Summary Response DTOs + +```typescript +/** + * Decrypted AI-generated summary content. + * Contains all three content types returned by the summary API. + */ +export interface SummaryContent { + /** Unique identifier */ + id: string; + /** Decrypted full note content */ + note: string; + /** Decrypted short note content */ + shortNote: string; + /** Decrypted action item snippets */ + actionItems: ActionItemSnippet[]; + /** Feedback URL (if available) */ + feedbackUrl?: string; +} + +/** + * Decrypted AI-generated notes + */ +export interface SummaryNotes { + /** Unique identifier */ + id: string; + /** Decrypted notes content */ + content: string; + /** Feedback URL (if available) */ + feedbackUrl?: string; +} + +/** + * Single action item snippet + */ +export interface ActionItemSnippet { + /** Unique identifier */ + id: string; + /** User-edited version (if available) */ + editedContent?: string; + /** Decrypted AI-generated content */ + aiGeneratedContent: string; +} + +/** + * Decrypted AI-generated action items + */ +export interface SummaryActionItems { + /** Unique identifier (absent when no action items exist) */ + id?: string; + /** Array of action item snippets */ + snippets: ActionItemSnippet[]; + /** Feedback URL (if available) */ + feedbackUrl?: string; +} + +/** + * Single decrypted transcript snippet + */ +export interface TranscriptSnippet { + /** Start time in milliseconds */ + startTime: string; + /** End time in milliseconds */ + endTime: string; + /** Decrypted transcript content */ + content: string; + /** Audio CSI identifier */ + audioCSI?: string; + /** Speaker information */ + speaker?: { + speakerName: string; + speakerId: string; + }; +} + +/** + * Decrypted transcript response + */ +export interface TranscriptContent { + /** Unique identifier */ + id: string; + /** Total number of snippets */ + totalCount: number; + /** Decrypted transcript snippets */ + snippets: TranscriptSnippet[]; +} +``` + +## 6. Low-Level Design & Pseudo Code + +### 6.1 Plugin Registration + +```typescript +// packages/@webex/internal-plugin-call-ai-summary/src/index.ts + +import '@webex/internal-plugin-encryption'; +import {registerInternalPlugin} from '@webex/webex-core'; + +import AISummary from './ai-summary'; +import config from './config'; + +registerInternalPlugin('aisummary', AISummary, {config}); + +export {default} from './ai-summary'; +``` + +### 6.2 Config + +```typescript +// packages/@webex/internal-plugin-call-ai-summary/src/config.ts + +export default { + aisummary: {}, +}; +``` + +### 6.3 Constants + +```typescript +// packages/@webex/internal-plugin-call-ai-summary/src/constants.ts + +export const AI_SUMMARY_SERVICE = 'pragya'; +export const AI_SUMMARY_CONTAINERS_RESOURCE = 'containers'; + +export const SUMMARY_STATUSES = { + ACTIVE: 'Active', +} as const; + +export const ERROR_MESSAGES = { + INVALID_CONTAINER_ID: 'containerId is required and must be a non-empty string', + INVALID_CONTAINER_INFO: 'containerInfo with valid summaryData and encryptionKeyUrl is required', + CONTAINER_NOT_FOUND: 'Container not found', + CONTENT_NOT_FOUND: 'Summary content not available or expired', + ACCESS_DENIED: 'Access denied: User not authorized to view this summary', + AUTHENTICATION_FAILED: 'Authentication failed: Invalid or expired token', +} as const; +``` + +### 6.4 Plugin Implementation + +```typescript +// packages/@webex/internal-plugin-call-ai-summary/src/ai-summary.ts + +import {WebexPlugin} from '@webex/webex-core'; + +import {AI_SUMMARY_SERVICE, AI_SUMMARY_CONTAINERS_RESOURCE, ERROR_MESSAGES} from './constants'; +import type { + GetContainerOptions, + GetSummaryContentOptions, + PragyaContainerResponse, + SummaryContent, + SummaryNotes, + SummaryActionItems, + TranscriptContent, +} from './types'; + +const AISummary = WebexPlugin.extend({ + namespace: 'AISummary', + + /** + * Resolve a Pragya container by ID. + * Flattens the nested summaryData.data structure for consumer convenience. + */ + getContainer(options: GetContainerOptions): Promise { + const {containerId} = options; + this._validateContainerId(containerId); + + return this.webex + .request({ + method: 'GET', + service: AI_SUMMARY_SERVICE, + resource: `${AI_SUMMARY_CONTAINERS_RESOURCE}/${containerId}`, + }) + .then(({body}) => { + // Pragya API nests summary URLs under summaryData.data — flatten + if (body.summaryData?.data) { + body.summaryData = body.summaryData.data; + } + return body; + }) + .catch((error) => { + this.logger.error('AISummary->getContainer failed', {error, containerId}); + throw this._handleError(error, 'getContainer'); + }); + }, + + /** + * Get AI-generated full summary for a call. + * Fetches note, shortNote, and actionItems in a single request via + * summaryUrl?fields=note,shortnote,actionitems, then decrypts all content. + */ + async getSummary(options: GetSummaryContentOptions): Promise { + const {containerInfo} = options; + this._validateContainerInfo(containerInfo, 'summaryUrl'); + + try { + const {body} = await this.webex.request({ + method: 'GET', + uri: `${containerInfo.summaryData.summaryUrl}?fields=note,shortnote,actionitems`, + }); + + const keyUrl = body.keyUrl || containerInfo.encryptionKeyUrl; + const decryptedNote = await this._decryptContent(body.note.aiGeneratedContent, keyUrl); + const decryptedShortNote = await this._decryptContent( + body.shortnote.aiGeneratedContent, keyUrl + ); + + const decryptedSnippets = await Promise.all( + (body.actionitems?.snippets || []).map(async (snippet: any) => { + const decryptedAiContent = await this._decryptContent( + snippet.aiGeneratedContent, keyUrl + ); + return { + id: snippet.id, + editedContent: snippet.content || undefined, + aiGeneratedContent: decryptedAiContent, + }; + }) + ); + + const feedbackLink = (body.links || []).find((link: any) => link.rel === 'feedback'); + + return { + id: body.id, + note: decryptedNote, + shortNote: decryptedShortNote, + actionItems: decryptedSnippets, + feedbackUrl: feedbackLink?.href, + }; + } catch (error) { + this.logger.error('AISummary->getSummary failed', {error}); + throw this._handleError(error, 'getSummary'); + } + }, + + /** + * Get AI-generated notes for a call (standalone endpoint). + * Uses body.keyUrl as decryption key with fallback to containerInfo.encryptionKeyUrl. + */ + async getNotes(options: GetSummaryContentOptions): Promise { + const {containerInfo} = options; + this._validateContainerInfo(containerInfo, 'notesUrl'); + + try { + const {body} = await this.webex.request({ + method: 'GET', + uri: containerInfo.summaryData.notesUrl, + }); + + const keyUrl = body.keyUrl || containerInfo.encryptionKeyUrl; + const decryptedContent = await this._decryptContent(body.aiGeneratedContent, keyUrl); + + return { id: body.id, content: decryptedContent, feedbackUrl: body.feedbackUrl }; + } catch (error) { + this.logger.error('AISummary->getNotes failed', {error}); + throw this._handleError(error, 'getNotes'); + } + }, + + /** + * Get AI-generated action items for a call (standalone endpoint). + * Response is an array; takes the first element and decrypts all snippets. + */ + async getActionItems(options: GetSummaryContentOptions): Promise { + const {containerInfo} = options; + this._validateContainerInfo(containerInfo, 'actionItemsUrl'); + + try { + const {body} = await this.webex.request({ + method: 'GET', + uri: containerInfo.summaryData.actionItemsUrl, + }); + + const actionItemsData = Array.isArray(body) ? body[0] : body; + if (!actionItemsData) return {id: undefined, snippets: []}; + + const keyUrl = actionItemsData.keyUrl || containerInfo.encryptionKeyUrl; + const decryptedSnippets = await Promise.all( + (actionItemsData.snippets || []).map(async (snippet: any) => { + const decryptedAiContent = await this._decryptContent( + snippet.aiGeneratedContent, keyUrl + ); + return { + id: snippet.id, + editedContent: snippet.content || undefined, + aiGeneratedContent: decryptedAiContent, + }; + }) + ); + + return { + id: actionItemsData.id, + snippets: decryptedSnippets, + feedbackUrl: actionItemsData.feedbackUrl, + }; + } catch (error) { + this.logger.error('AISummary->getActionItems failed', {error}); + throw this._handleError(error, 'getActionItems'); + } + }, + + /** Returns the transcript URL string from the container info. */ + getTranscriptUrl(options: GetSummaryContentOptions): string { + const {containerInfo} = options; + this._validateContainerInfo(containerInfo, 'transcriptUrl'); + return containerInfo.summaryData.transcriptUrl; + }, + + /** Fetches and decrypts the full transcript, returning all snippets. */ + async getTranscript(options: GetSummaryContentOptions): Promise { + const {containerInfo} = options; + this._validateContainerInfo(containerInfo, 'transcriptUrl'); + + try { + const {body} = await this.webex.request({ + method: 'GET', + uri: containerInfo.summaryData.transcriptUrl, + }); + + const keyUrl = body.keyUrl || containerInfo.encryptionKeyUrl; + const decryptedSnippets = await Promise.all( + (body.transcriptSnippetList || []).map(async (snippet: any) => { + const decryptedContent = await this._decryptContent(snippet.content, keyUrl); + return { + startTime: snippet.startTime, + endTime: snippet.endTime, + content: decryptedContent, + audioCSI: snippet.audioCSI, + speaker: snippet.speaker, + }; + }) + ); + + return { id: body.id, totalCount: body.totalCount, snippets: decryptedSnippets }; + } catch (error) { + this.logger.error('AISummary->getTranscript failed', {error}); + throw this._handleError(error, 'getTranscript'); + } + }, + + // --- Private helpers --- + + _validateContainerId(containerId: string): void { /* ... */ }, + _validateContainerInfo(containerInfo: PragyaContainerResponse, urlField: string): void { /* ... */ }, + _decryptContent(encryptedContent: string, encryptionKeyUrl: string): Promise { + return this.webex.internal.encryption.decryptText(encryptionKeyUrl, encryptedContent); + }, + _handleError(error: any, methodName: string): Error { + if (error.statusCode === 404) { + const msg = methodName === 'getContainer' + ? ERROR_MESSAGES.CONTAINER_NOT_FOUND + : ERROR_MESSAGES.CONTENT_NOT_FOUND; + return new Error(msg); + } + if (error.statusCode === 403) return new Error(ERROR_MESSAGES.ACCESS_DENIED); + if (error.statusCode === 401) return new Error(ERROR_MESSAGES.AUTHENTICATION_FAILED); + return new Error(`${methodName} failed: ${error.message || 'Unknown error'}`); + }, +}); + +export default AISummary; +``` + +### 6.5 Usage Examples + +```typescript +// Step 1: Get call history (existing SDK API) +const callHistory = await callHistoryInstance.getCallHistoryData(10, 50); +const sessions = callHistory.data.userSessions; + +// Step 2: Find sessions with AI summaries +const sessionWithSummary = sessions.find( + (session) => session.extensionPayload?.callingContainerIds?.length > 0 +); + +if (!sessionWithSummary) { + console.log('No AI summaries available'); + return; +} + +// Step 3: Resolve the container (plugin flattens summaryData.data automatically) +const containerId = sessionWithSummary.extensionPayload.callingContainerIds[0]; +const container = await webex.internal.aisummary.getContainer({ containerId }); + +// Check if summary is available +if (container.summaryData.status !== 'Active') { + console.log('Summary is not yet ready'); + return; +} + +// Step 4: Fetch all summary content (note + shortNote + actionItems) in one call +const summary = await webex.internal.aisummary.getSummary({ containerInfo: container }); +console.log('Note:', summary.note); +console.log('Short Note:', summary.shortNote); +summary.actionItems.forEach((item, i) => { + console.log(`Action Item ${i + 1}: ${item.aiGeneratedContent}`); +}); + +// Step 5: Get transcript URL (or fetch full transcript) +const transcriptUrl = webex.internal.aisummary.getTranscriptUrl({ containerInfo: container }); +console.log('Transcript URL:', transcriptUrl); + +// Step 6: Fetch and decrypt full transcript +const transcript = await webex.internal.aisummary.getTranscript({ containerInfo: container }); +transcript.snippets.forEach((snippet) => { + console.log(`[${snippet.startTime}] ${snippet.speaker?.speakerName}: ${snippet.content}`); +}); +``` + +## 7. API Request/Response Details + +### 7.1 Pragya Container Lookup + +**Request:** +```http +GET /pragya/api/v1/containers/{containerId} HTTP/1.1 +Authorization: Bearer {user_access_token} +Accept: application/json +``` + +**Success Response (200 OK):** + +> The raw response nests URLs under `summaryData.data`. The plugin's `getContainer()` flattens this automatically. + +```json +{ + "id": "34125120-13b5-11f1-9b36-adb685725098", + "objectType": "callingAIContainer", + "memberships": { + "items": [{ "id": "...", "roles": ["OWNER"], "objectType": "containerMembership" }] + }, + "summaryData": { + "extensionId": "...", + "objectType": "extension", + "extensionType": "callingAISummary", + "data": { + "id": "...", + "objectType": "callingAISummary", + "status": "Active", + "summaryUrl": "https://aibridge-url/summaries/c635e870-...", + "transcriptUrl": "https://aibridge-url/summaries/c635e870-.../transcripts", + "summarizeAfterCall": true, + "aclUrl": "https://acl-a.wbx2.com/...", + "kmsResourceObjectUrl": "kms://kms-cisco.wbx2.com/resources/..." + } + }, + "encryptionKeyUrl": "kms://kms-cisco.wbx2.com/keys/897e4d2d-...", + "kmsResourceObjectUrl": "kms://kms-cisco.wbx2.com/resources/f7316435-...", + "aclUrl": "https://acl-a.wbx2.com/acl/api/v1/acls/78c4cd90-...", + "forkSessionId": "123e4567-...", + "callSessionId": "123e4567-...", + "ownerUserId": "123e4567-...", + "orgId": "123e4567-...", + "start": "2023-10-01T12:00:00Z", + "end": "2023-10-01T12:00:00Z" +} +``` + +**Error Responses:** +- `401 Unauthorized` - Invalid or expired token +- `403 Forbidden` - User not authorized to access this container +- `404 Not Found` - Container not found + +### 7.2 Summary Content (fetched via summaryUrl with fields query) + +The primary way to fetch all summary content is via `getSummary()`, which appends `?fields=note,shortnote,actionitems` to the `summaryUrl`. + +**Request:** +```http +GET {summaryData.summaryUrl}?fields=note,shortnote,actionitems HTTP/1.1 +Authorization: Bearer {user_access_token} +Accept: application/json +``` + +**Success Response (200 OK):** +```json +{ + "id": "10293-dk93-ddie-odir-did932j3kdde", + "keyUrl": "kms://kms-us-int.wbx2.com/keys/f19d4d28-...", + "note": { + "aiGeneratedContent": "" + }, + "shortnote": { + "aiGeneratedContent": "" + }, + "actionitems": { + "snippets": [ + { + "id": "394r0087-...", + "content": "edited version", + "aiGeneratedContent": "" + } + ] + }, + "links": [ + { "rel": "feedback", "href": "https://summarizer-r.wbx2.com/summarizer/api/v1/feedback/..." } + ] +} +``` + +### 7.3 Notes (standalone, fetched via notesUrl) + +**Request:** +```http +GET {summaryData.notesUrl} HTTP/1.1 +Authorization: Bearer {user_access_token} +Accept: application/json +``` + +**Success Response (200 OK):** +```json +{ + "id": "10293-dk93-ddie-odir-did932j3kdde", + "aiGeneratedContent": "", + "feedbackUrl": "https://summarizer-r.wbx2.com/summarizer/api/v1/feedback/report/...", + "keyUrl": "kms://kms-us-int.wbx2.com/keys/f19d4d28-..." +} +``` + +### 7.4 Action Items (standalone, fetched via actionItemsUrl) + +**Request:** +```http +GET {summaryData.actionItemsUrl} HTTP/1.1 +Authorization: Bearer {user_access_token} +Accept: application/json +``` + +**Success Response (200 OK):** +```json +[ + { + "id": "1234-dk93-ddie-odir-dk93dj33", + "keyUrl": "kms://kms-us-int.wbx2.com/keys/f19d4d28-...", + "snippets": [ + { + "id": "394r0087-...", + "content": "edited version", + "aiGeneratedContent": "" + } + ] + } +] +``` + +## 8. Encryption & Decryption + +### 8.1 Content Encryption + +All AI-generated content is encrypted using KMS (Key Management Service): + +- **Encryption Key**: The `encryptionKeyUrl` from the Pragya container response (format: `kms://kms-{region}.wbx2.com/keys/{key-id}`) +- **Encrypted Fields**: `aiGeneratedContent` in notes and action item snippets +- **Decryption**: Uses `@webex/internal-plugin-encryption` via `decryptText()` + +### 8.2 Decryption Pattern + +The SDK uses the existing `@webex/internal-plugin-encryption` plugin: + +```typescript +// Decrypt using the encryptionKeyUrl from the Pragya container response +const decryptedContent = await this.webex.internal.encryption.decryptText( + containerInfo.encryptionKeyUrl, + body.aiGeneratedContent +); +``` + +This is the same pattern used by existing plugins: + +**AI Assistant Plugin** (`internal-plugin-ai-assistant/src/utils.ts`): +```typescript +const decryptedValue = await webex.internal.encryption.decryptText( + encryptionKeyUrl, + encryptedValue +); +``` + +**Task Plugin** (`internal-plugin-task/src/helpers/decrypt.helper.js`): +```javascript +ctx.webex.internal.encryption.decryptText(key.uri || key, object[name]) +``` + +## 9. Error Handling + +### 9.1 Error Scenarios + +| Error Type | HTTP Status | SDK Error Message | Recovery Action | +|------------|-------------|-------------------|-----------------| +| Invalid Container ID | N/A (client) | "containerId is required and must be a non-empty string" | Validate input | +| Invalid Container Info | N/A (client) | "containerInfo with valid summaryData and encryptionKeyUrl is required" | Ensure getContainer was called first | +| Authentication Failed | 401 | "Authentication failed: Invalid or expired token" | Re-authenticate user | +| Access Denied | 403 | "Access denied: User not authorized to view this summary" | Check user permissions | +| Container Not Found | 404 | "Container not found" | Verify containerId from Janus | +| Content Not Found | 404 (non-getContainer) | "Summary content not available or expired" | Content may have been deleted or expired | +| Summary Not Ready | N/A | summaryData.status !== "Active" | Retry after delay | + +## 10. Security Considerations + +### 10.1 Authentication +- All API calls (Pragya and content URLs) require a valid user bearer token +- Token is automatically attached by the SDK's HTTP layer + +### 10.2 Authorization +- Only call participants or authorized users can access containers and summaries +- Org-level AI features must be enabled +- Per-call consent: AI assistant must have been enabled during the call + +### 10.3 Content Protection +- All AI-generated content is encrypted at rest with KMS +- `encryptionKeyUrl` from Pragya container is the decryption key +- HTTPS required for all API calls + +## 11. Testing Strategy + +### 11.1 Unit Tests + +```typescript +import {assert, expect} from '@webex/test-helper-chai'; +import MockWebex from '@webex/test-helper-mock-webex'; +import sinon from 'sinon'; +import AISummary from '@webex/internal-plugin-call-ai-summary'; +import config from '@webex/internal-plugin-call-ai-summary/src/config'; + +describe('internal-plugin-call-ai-summary', () => { + let webex; + + beforeEach(() => { + webex = MockWebex({ + children: { + aisummary: AISummary, + }, + }); + webex.config.aisummary = config.aisummary; + webex.internal.encryption = { + decryptText: sinon.stub().resolves('decrypted content'), + }; + }); + + describe('#getContainer', () => { + it('should resolve a Pragya container by ID', async () => { + const mockContainer = { + summaryData: { + status: 'Active', + summaryUrl: 'https://aibridge-url/summaries/abc123', + notesUrl: 'https://aibridge-url/summaries/abc123/notes', + actionItemsUrl: 'https://aibridge-url/summaries/abc123/action-items', + transcriptUrl: 'https://aibridge-url/summaries/abc123/transcripts', + summarizeAfterCall: true, + }, + encryptionKeyUrl: 'kms://kms.url/keys/key-id', + }; + + webex.request = sinon.stub().resolves({body: mockContainer}); + + const result = await webex.internal.aisummary.getContainer({ + containerId: 'container-123', + }); + + expect(result.summaryData.status).to.equal('Active'); + assert.calledWith(webex.request, sinon.match({ + method: 'GET', + service: 'pragya', + resource: 'containers/container-123', + })); + }); + + it('should throw for empty containerId', async () => { + await expect( + webex.internal.aisummary.getContainer({containerId: ''}) + ).to.be.rejectedWith('containerId is required'); + }); + }); + + describe('#getNotes', () => { + const mockContainerInfo = { + summaryData: { + notesUrl: 'https://aibridge-url/summaries/abc123/notes', + }, + encryptionKeyUrl: 'kms://kms.url/keys/key-id', + }; + + it('should fetch and decrypt notes', async () => { + webex.request = sinon.stub().resolves({ + body: { + id: 'note-id', + aiGeneratedContent: 'encrypted-notes', + feedbackUrl: 'https://feedback.url', + }, + }); + + const result = await webex.internal.aisummary.getNotes({ + containerInfo: mockContainerInfo, + }); + + expect(result.id).to.equal('note-id'); + expect(result.content).to.equal('decrypted content'); + expect(result.feedbackUrl).to.equal('https://feedback.url'); + assert.calledWith( + webex.internal.encryption.decryptText, + 'kms://kms.url/keys/key-id', + 'encrypted-notes' + ); + }); + + it('should throw when containerInfo is missing notesUrl', async () => { + await expect( + webex.internal.aisummary.getNotes({containerInfo: {summaryData: {}}}) + ).to.be.rejectedWith('containerInfo with valid summaryData'); + }); + }); + + describe('#getActionItems', () => { + const mockContainerInfo = { + summaryData: { + actionItemsUrl: 'https://aibridge-url/summaries/abc123/action-items', + }, + encryptionKeyUrl: 'kms://kms.url/keys/key-id', + }; + + it('should fetch and decrypt all action item snippets', async () => { + webex.request = sinon.stub().resolves({ + body: [{ + id: 'action-items-id', + keyUrl: 'kms://kms.url/keys/key-id', + snippets: [ + {id: 's1', aiGeneratedContent: 'encrypted-1'}, + {id: 's2', content: 'edited', aiGeneratedContent: 'encrypted-2'}, + ], + }], + }); + + webex.internal.encryption.decryptText + .onFirstCall().resolves('Decrypted item 1') + .onSecondCall().resolves('Decrypted item 2'); + + const result = await webex.internal.aisummary.getActionItems({ + containerInfo: mockContainerInfo, + }); + + expect(result.snippets).to.have.lengthOf(2); + expect(result.snippets[0].aiGeneratedContent).to.equal('Decrypted item 1'); + expect(result.snippets[1].aiGeneratedContent).to.equal('Decrypted item 2'); + expect(result.snippets[1].editedContent).to.equal('edited'); + }); + }); + + describe('#getTranscriptUrl', () => { + it('should return the transcript URL', () => { + const containerInfo = { + summaryData: { + transcriptUrl: 'https://aibridge-url/summaries/abc123/transcripts', + }, + encryptionKeyUrl: 'kms://kms.url/keys/key-id', + }; + + const url = webex.internal.aisummary.getTranscriptUrl({containerInfo}); + + expect(url).to.equal('https://aibridge-url/summaries/abc123/transcripts'); + }); + }); +}); +``` + +## 12. Modularity & Existing Code Impact + +### 12.1 Zero Changes to Existing Packages + +This plugin is fully self-contained. It does **not** require modifications to any existing package: + +| Concern | Approach | +|---------|----------| +| `UserSession` type in `@webex/calling` | **Not modified.** The plugin accepts a plain `containerId: string`. Consumers extract it from the Janus response at the application layer. The `UserSession` type update is a separate, optional task for the calling package team. | +| `packages/webex` bundle | **Not modified.** Consumers import `@webex/internal-plugin-call-ai-summary` directly, which self-registers via `registerInternalPlugin()`. No changes to the webex package index are needed. | +| `@webex/internal-plugin-encryption` | **Not modified.** Used as a runtime dependency via `this.webex.internal.encryption.decryptText()`. | + +### 12.2 Plugin Package Structure + +``` +packages/@webex/internal-plugin-call-ai-summary/ + src/ + index.ts # registerInternalPlugin('aisummary', ...) + ai-summary.ts # WebexPlugin.extend({...}) + config.ts # { aisummary: {} } + constants.ts # Service name, error messages + types.ts # All TypeScript interfaces + test/ + unit/ + spec/ + ai-summary.ts + data/ + responses.ts # Mock Pragya and content responses + package.json + jest.config.js + babel.config.js + .eslintrc.js +``` + +## 13. Dependencies + +### 13.1 Internal Dependencies + +| Package | Purpose | +|---------|---------| +| `@webex/webex-core` | Plugin infrastructure (`WebexPlugin`, `registerInternalPlugin`) | +| `@webex/internal-plugin-encryption` | Content decryption via `decryptText()` | + +### 13.2 External Service Dependencies + +| Service | Purpose | Discovery | +|---------|---------|-----------| +| **Janus** | Call history; provides `extensionPayload.callingContainerIds` | U2C: `serviceName: "janus"` | +| **Pragya** | Container metadata; provides content URLs and encryption key | U2C: `serviceName: "pragya"` | +| **Summary Content Endpoints** | Serve encrypted AI-generated content | Direct URLs from Pragya response | +| **KMS** | Encryption key management | Via `encryptionKeyUrl` from Pragya | diff --git a/packages/@webex/internal-plugin-call-ai-summary/babel.config.js b/packages/@webex/internal-plugin-call-ai-summary/babel.config.js new file mode 100644 index 00000000000..71a8b034b1f --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/babel.config.js @@ -0,0 +1,3 @@ +const babelConfigLegacy = require('@webex/babel-config-legacy'); + +module.exports = babelConfigLegacy; diff --git a/packages/@webex/internal-plugin-call-ai-summary/jest.config.js b/packages/@webex/internal-plugin-call-ai-summary/jest.config.js new file mode 100644 index 00000000000..0e9d38b401c --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/jest.config.js @@ -0,0 +1,3 @@ +const config = require('@webex/jest-config-legacy'); + +module.exports = config; diff --git a/packages/@webex/internal-plugin-call-ai-summary/package.json b/packages/@webex/internal-plugin-call-ai-summary/package.json new file mode 100644 index 00000000000..eebb7f2bc71 --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/package.json @@ -0,0 +1,46 @@ +{ + "name": "@webex/internal-plugin-call-ai-summary", + "description": "A Webex internal plugin for AI-generated call summary retrieval", + "license": "MIT", + "author": "Webex JS SDK Team", + "main": "dist/index.js", + "devMain": "src/index.ts", + "repository": { + "type": "git", + "url": "https://github.com/webex/webex-js-sdk.git", + "directory": "packages/@webex/internal-plugin-call-ai-summary" + }, + "engines": { + "node": ">=16" + }, + "browserify": { + "transform": [ + "babelify", + "envify" + ] + }, + "dependencies": { + "@webex/internal-plugin-encryption": "workspace:*", + "@webex/webex-core": "workspace:*" + }, + "devDependencies": { + "@babel/core": "^7.17.10", + "@webex/babel-config-legacy": "workspace:*", + "@webex/eslint-config-legacy": "workspace:*", + "@webex/jest-config-legacy": "workspace:*", + "@webex/legacy-tools": "workspace:*", + "@webex/test-helper-chai": "workspace:*", + "@webex/test-helper-mock-webex": "workspace:*", + "eslint": "^8.24.0", + "prettier": "^2.7.1", + "sinon": "^9.2.4" + }, + "scripts": { + "build": "yarn build:src", + "build:src": "webex-legacy-tools build -dest \"./dist\" -src \"./src\" -js -ts -maps", + "deploy:npm": "yarn npm publish", + "test": "yarn test:style && yarn test:unit", + "test:style": "eslint ./src/**/*.*", + "test:unit": "webex-legacy-tools test --unit --runner jest" + } +} diff --git a/packages/@webex/internal-plugin-call-ai-summary/src/ai-summary.ts b/packages/@webex/internal-plugin-call-ai-summary/src/ai-summary.ts new file mode 100644 index 00000000000..c9e6a2a23cd --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/src/ai-summary.ts @@ -0,0 +1,318 @@ +/*! + * Copyright (c) 2015-2025 Cisco Systems, Inc. See LICENSE file. + */ + +import {WebexPlugin} from '@webex/webex-core'; + +import {AI_SUMMARY_SERVICE, AI_SUMMARY_CONTAINERS_RESOURCE, ERROR_MESSAGES} from './constants'; +import type { + GetContainerOptions, + GetSummaryContentOptions, + PragyaContainerResponse, + SummaryContent, + SummaryNotes, + SummaryActionItems, + TranscriptContent, +} from './types'; + +const AISummary = WebexPlugin.extend({ + namespace: 'AISummary', + + /** + * Resolve a Pragya container by ID. + * Returns container metadata including summary URLs and encryption key. + * + * @param {GetContainerOptions} options + * @returns {Promise} + */ + getContainer(options: GetContainerOptions): Promise { + const {containerId} = options; + + this._validateContainerId(containerId); + + return this.webex + .request({ + method: 'GET', + service: AI_SUMMARY_SERVICE, + resource: `${AI_SUMMARY_CONTAINERS_RESOURCE}/${containerId}`, + }) + .then(({body}) => { + // Pragya API nests summary URLs under summaryData.data — flatten + // so consumers can access summaryData.summaryUrl directly. + if (body.summaryData?.data) { + body.summaryData = body.summaryData.data; + } + + return body; + }) + .catch((error) => { + this.logger.error('AISummary->getContainer failed', {error, containerId}); + throw this._handleError(error, 'getContainer'); + }); + }, + + /** + * Get AI-generated full summary for a call. + * Fetches from containerInfo.summaryData.summaryUrl and decrypts content. + * + * @param {GetSummaryContentOptions} options + * @returns {Promise} + */ + async getSummary(options: GetSummaryContentOptions): Promise { + const {containerInfo} = options; + + this._validateContainerInfo(containerInfo, 'summaryUrl'); + + try { + const {body} = await this.webex.request({ + method: 'GET', + uri: `${containerInfo.summaryData.summaryUrl}?fields=note,shortnote,actionitems`, + }); + + const keyUrl = body.keyUrl || containerInfo.encryptionKeyUrl; + + const decryptedNote = await this._decryptContent(body.note.aiGeneratedContent, keyUrl); + + const decryptedShortNote = await this._decryptContent( + body.shortnote.aiGeneratedContent, + keyUrl + ); + + const decryptedSnippets = await Promise.all( + (body.actionitems?.snippets || []).map(async (snippet: any) => { + const decryptedAiContent = await this._decryptContent(snippet.aiGeneratedContent, keyUrl); + + return { + id: snippet.id, + editedContent: snippet.content || undefined, + aiGeneratedContent: decryptedAiContent, + }; + }) + ); + + // feedbackUrl may be in the links array as rel="feedback" + const feedbackLink = (body.links || []).find((link: any) => link.rel === 'feedback'); + + return { + id: body.id, + note: decryptedNote, + shortNote: decryptedShortNote, + actionItems: decryptedSnippets, + feedbackUrl: feedbackLink?.href, + }; + } catch (error) { + this.logger.error('AISummary->getSummary failed', {error}); + throw this._handleError(error, 'getSummary'); + } + }, + + /** + * Get AI-generated notes for a call. + * Fetches from containerInfo.summaryData.notesUrl and decrypts content. + * + * @param {GetSummaryContentOptions} options + * @returns {Promise} + */ + async getNotes(options: GetSummaryContentOptions): Promise { + const {containerInfo} = options; + + this._validateContainerInfo(containerInfo, 'notesUrl'); + + try { + const {body} = await this.webex.request({ + method: 'GET', + uri: containerInfo.summaryData.notesUrl, + }); + + const keyUrl = body.keyUrl || containerInfo.encryptionKeyUrl; + + const decryptedContent = await this._decryptContent(body.aiGeneratedContent, keyUrl); + + return { + id: body.id, + content: decryptedContent, + feedbackUrl: body.feedbackUrl, + }; + } catch (error) { + this.logger.error('AISummary->getNotes failed', {error}); + throw this._handleError(error, 'getNotes'); + } + }, + + /** + * Get AI-generated action items for a call. + * Fetches from containerInfo.summaryData.actionItemsUrl and decrypts content. + * + * @param {GetSummaryContentOptions} options + * @returns {Promise} + */ + async getActionItems(options: GetSummaryContentOptions): Promise { + const {containerInfo} = options; + + this._validateContainerInfo(containerInfo, 'actionItemsUrl'); + + try { + const {body} = await this.webex.request({ + method: 'GET', + uri: containerInfo.summaryData.actionItemsUrl, + }); + + // Action items response is an array; take the first element + const actionItemsData = Array.isArray(body) ? body[0] : body; + + if (!actionItemsData) { + return {id: undefined, snippets: []}; + } + + const keyUrl = actionItemsData.keyUrl || containerInfo.encryptionKeyUrl; + + const decryptedSnippets = await Promise.all( + (actionItemsData.snippets || []).map(async (snippet: any) => { + const decryptedAiContent = await this._decryptContent(snippet.aiGeneratedContent, keyUrl); + + return { + id: snippet.id, + editedContent: snippet.content || undefined, + aiGeneratedContent: decryptedAiContent, + }; + }) + ); + + return { + id: actionItemsData.id, + snippets: decryptedSnippets, + feedbackUrl: actionItemsData.feedbackUrl, + }; + } catch (error) { + this.logger.error('AISummary->getActionItems failed', {error}); + throw this._handleError(error, 'getActionItems'); + } + }, + + /** + * Get the transcript URL for a call. + * Returns the URL string from the container info. + * + * @param {GetSummaryContentOptions} options + * @returns {string} + */ + getTranscriptUrl(options: GetSummaryContentOptions): string { + const {containerInfo} = options; + + this._validateContainerInfo(containerInfo, 'transcriptUrl'); + + return containerInfo.summaryData.transcriptUrl; + }, + + /** + * Get decrypted transcript for a call. + * Fetches from containerInfo.summaryData.transcriptUrl and decrypts each snippet. + * + * @param {GetSummaryContentOptions} options + * @returns {Promise} + */ + async getTranscript(options: GetSummaryContentOptions): Promise { + const {containerInfo} = options; + + this._validateContainerInfo(containerInfo, 'transcriptUrl'); + + try { + const {body} = await this.webex.request({ + method: 'GET', + uri: containerInfo.summaryData.transcriptUrl, + }); + + const keyUrl = body.keyUrl || containerInfo.encryptionKeyUrl; + + const decryptedSnippets = await Promise.all( + (body.transcriptSnippetList || []).map(async (snippet: any) => { + const decryptedContent = await this._decryptContent(snippet.content, keyUrl); + + return { + startTime: snippet.startTime, + endTime: snippet.endTime, + content: decryptedContent, + audioCSI: snippet.audioCSI, + speaker: snippet.speaker, + }; + }) + ); + + return { + id: body.id, + totalCount: body.totalCount, + snippets: decryptedSnippets, + }; + } catch (error) { + this.logger.error('AISummary->getTranscript failed', {error}); + throw this._handleError(error, 'getTranscript'); + } + }, + + /** + * Validate containerId parameter. + * @param {string} containerId - The container ID to validate. + * @returns {void} + * @private + */ + _validateContainerId(containerId: string): void { + if (!containerId || typeof containerId !== 'string' || containerId.trim().length === 0) { + throw new Error(ERROR_MESSAGES.INVALID_CONTAINER_ID); + } + }, + + /** + * Validate containerInfo has the required URL field and encryption key. + * @param {PragyaContainerResponse} containerInfo - The container info to validate. + * @param {string} urlField - The summaryData field name to check. + * @returns {void} + * @private + */ + _validateContainerInfo(containerInfo: PragyaContainerResponse, urlField: string): void { + if (!containerInfo?.summaryData?.[urlField] || !containerInfo?.encryptionKeyUrl) { + throw new Error(ERROR_MESSAGES.INVALID_CONTAINER_INFO); + } + }, + + /** + * Decrypt encrypted content using KMS. + * Delegates to the internal encryption plugin. + * @param {string} encryptedContent - The encrypted text (JWE format). + * @param {string} encryptionKeyUrl - KMS key URL. + * @returns {Promise} Decrypted plaintext. + * @private + */ + _decryptContent(encryptedContent: string, encryptionKeyUrl: string): Promise { + return this.webex.internal.encryption.decryptText(encryptionKeyUrl, encryptedContent); + }, + + /** + * Handle and normalize errors. + * @param {object} error - The error object from the request. + * @param {string} methodName - The name of the calling method. + * @returns {Error} A normalized Error instance. + * @private + */ + _handleError(error: any, methodName: string): Error { + if (error.statusCode === 404) { + const message = + methodName === 'getContainer' + ? ERROR_MESSAGES.CONTAINER_NOT_FOUND + : ERROR_MESSAGES.CONTENT_NOT_FOUND; + + return new Error(message); + } + + if (error.statusCode === 403) { + return new Error(ERROR_MESSAGES.ACCESS_DENIED); + } + + if (error.statusCode === 401) { + return new Error(ERROR_MESSAGES.AUTHENTICATION_FAILED); + } + + return new Error(`${methodName} failed: ${error.message || 'Unknown error'}`); + }, +}); + +export default AISummary; diff --git a/packages/@webex/internal-plugin-call-ai-summary/src/config.ts b/packages/@webex/internal-plugin-call-ai-summary/src/config.ts new file mode 100644 index 00000000000..3285944abd9 --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/src/config.ts @@ -0,0 +1,7 @@ +/*! + * Copyright (c) 2015-2025 Cisco Systems, Inc. See LICENSE file. + */ + +export default { + aisummary: {}, +}; diff --git a/packages/@webex/internal-plugin-call-ai-summary/src/constants.ts b/packages/@webex/internal-plugin-call-ai-summary/src/constants.ts new file mode 100644 index 00000000000..cedeb65f39e --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/src/constants.ts @@ -0,0 +1,19 @@ +/*! + * Copyright (c) 2015-2025 Cisco Systems, Inc. See LICENSE file. + */ + +export const AI_SUMMARY_SERVICE = 'pragya'; +export const AI_SUMMARY_CONTAINERS_RESOURCE = 'containers'; + +export const SUMMARY_STATUSES = { + ACTIVE: 'Active', +} as const; + +export const ERROR_MESSAGES = { + INVALID_CONTAINER_ID: 'containerId is required and must be a non-empty string', + INVALID_CONTAINER_INFO: 'containerInfo with valid summaryData and encryptionKeyUrl is required', + CONTAINER_NOT_FOUND: 'Container not found', + CONTENT_NOT_FOUND: 'Summary content not available or expired', + ACCESS_DENIED: 'Access denied: User not authorized to view this summary', + AUTHENTICATION_FAILED: 'Authentication failed: Invalid or expired token', +} as const; diff --git a/packages/@webex/internal-plugin-call-ai-summary/src/index.ts b/packages/@webex/internal-plugin-call-ai-summary/src/index.ts new file mode 100644 index 00000000000..16c8f7347fb --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/src/index.ts @@ -0,0 +1,13 @@ +/*! + * Copyright (c) 2015-2025 Cisco Systems, Inc. See LICENSE file. + */ + +import '@webex/internal-plugin-encryption'; +import {registerInternalPlugin} from '@webex/webex-core'; + +import AISummary from './ai-summary'; +import config from './config'; + +registerInternalPlugin('aisummary', AISummary, {config}); + +export {default} from './ai-summary'; diff --git a/packages/@webex/internal-plugin-call-ai-summary/src/manual-integration-test.js b/packages/@webex/internal-plugin-call-ai-summary/src/manual-integration-test.js new file mode 100644 index 00000000000..c1cadc30ec9 --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/src/manual-integration-test.js @@ -0,0 +1,129 @@ +/*! + * Copyright (c) 2015-2025 Cisco Systems, Inc. See LICENSE file. + * + * Manual integration test for internal-plugin-call-ai-summary + * Tests the full flow using the SDK service catalog (WDM): + * device.register() -> getContainer -> getSummary (with KMS decryption) + * + * The SDK resolves `service: 'pragya'` to the correct base URL via the + * service catalog populated during device registration. + * + * Usage: + * WEBEX_TOKEN='' node src/manual-integration-test.js + */ + +/* eslint-disable no-console, require-jsdoc */ + +require('@webex/internal-plugin-call-ai-summary'); + +const WebexCore = require('@webex/webex-core').default; + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- +const WEBEX_TOKEN = process.env.WEBEX_TOKEN || ''; +const CONTAINER_ID = process.env.CONTAINER_ID || ''; + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- +async function main() { + console.log('=== Step 1: Create WebexCore ===\n'); + const webex = new WebexCore({ + credentials: { + access_token: WEBEX_TOKEN, + }, + }); + + // Step 2: Register device to populate service catalog + console.log('=== Step 2: Register device (WDM) ===\n'); + await webex.internal.device.register(); + console.log('Device registered successfully.'); + console.log('Device URL:', webex.internal.device.url); + + // Log the pragya service URL from the service catalog + try { + const pragyaUrl = webex.internal.services.get('pragya'); + console.log('Pragya service URL (from catalog):', pragyaUrl); + } catch (e) { + console.log('Could not resolve pragya from service catalog:', e.message); + } + + // Step 3: Get container via plugin (uses service: 'pragya' + resource) + console.log('\n=== Step 3: getContainer via plugin ===\n'); + const container = await webex.internal.aisummary.getContainer({ + containerId: CONTAINER_ID, + }); + console.log( + 'Container Info:', + JSON.stringify( + { + id: container.id, + objectType: container.objectType, + encryptionKeyUrl: container.encryptionKeyUrl, + summaryUrl: container.summaryData?.summaryUrl, + transcriptUrl: container.summaryData?.transcriptUrl, + }, + null, + 2 + ) + ); + + // Step 4: Call getSummary via plugin (fetches + decrypts all content) + console.log('\n=== Step 4: getSummary via plugin ===\n'); + const summaryResult = await webex.internal.aisummary.getSummary({ + containerInfo: container, + }); + + console.log('=== getSummary return structure ==='); + const noteStr = summaryResult.note || ''; + const shortNoteStr = summaryResult.shortNote || ''; + const truncNote = noteStr.length > 200 ? `${noteStr.substring(0, 200)}...` : noteStr; + const truncShort = + shortNoteStr.length > 200 ? `${shortNoteStr.substring(0, 200)}...` : shortNoteStr; + const truncated = { + id: summaryResult.id, + note: truncNote, + shortNote: truncShort, + actionItems: (summaryResult.actionItems || []).map((item) => { + const content = item.aiGeneratedContent || ''; + const truncContent = content.length > 100 ? `${content.substring(0, 100)}...` : content; + + return { + id: item.id, + aiGeneratedContent: truncContent, + editedContent: item.editedContent, + }; + }), + feedbackUrl: summaryResult.feedbackUrl, + }; + console.log(JSON.stringify(truncated, null, 2)); + + // Step 5: Get transcript URL via plugin + console.log('\n=== Step 5: getTranscriptUrl via plugin ===\n'); + const transcriptUrl = webex.internal.aisummary.getTranscriptUrl({ + containerInfo: container, + }); + console.log('Transcript URL:', transcriptUrl); + + // Step 6: Fetch transcript content + console.log('\n=== Step 6: Fetch transcript content ===\n'); + try { + const {body: transcriptBody} = await webex.request({ + method: 'GET', + uri: `${transcriptUrl}?fields=id,content`, + }); + console.log('Transcript response keys:', Object.keys(transcriptBody)); + console.log(JSON.stringify(transcriptBody, null, 2).substring(0, 500)); + } catch (err) { + console.error('Transcript fetch failed:', err.message); + } + + console.log('\nDone.'); + process.exit(0); +} + +main().catch((err) => { + console.error('FATAL:', err.message || err); + process.exit(1); +}); diff --git a/packages/@webex/internal-plugin-call-ai-summary/src/manual-pragya-api-test.js b/packages/@webex/internal-plugin-call-ai-summary/src/manual-pragya-api-test.js new file mode 100644 index 00000000000..08878f1772d --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/src/manual-pragya-api-test.js @@ -0,0 +1,166 @@ +/*! + * Copyright (c) 2015-2025 Cisco Systems, Inc. See LICENSE file. + * + * Manual test for internal-plugin-call-ai-summary + * + * Usage: + * WEBEX_TOKEN='' node manual-test.js + * + * Or paste your token directly into WEBEX_TOKEN below. + */ + +/* eslint-disable no-console, require-jsdoc */ + +require('@webex/internal-plugin-call-ai-summary'); + +const WebexCore = require('@webex/webex-core').default; + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- +const WEBEX_TOKEN = process.env.WEBEX_TOKEN || ''; +const CONTAINER_ID = ''; +const PRAGYA_BASE_URL = ''; + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- +async function main() { + if (WEBEX_TOKEN === '') { + console.error('ERROR: Set WEBEX_TOKEN env var or paste your token in the script.'); + process.exit(1); + } + + const webex = new WebexCore({ + credentials: { + access_token: WEBEX_TOKEN, + }, + }); + + console.log('--- Fetching container', CONTAINER_ID, '(SDK auth) ---\n'); + + const response = await webex.request({ + method: 'GET', + uri: `${PRAGYA_BASE_URL}/containers/${CONTAINER_ID}`, + headers: { + 'content-type': 'application/json', + }, + }); + + const container = response.body; + let passed = 0; + let failed = 0; + + function check(label, actual, expected) { + if (actual === expected) { + console.log(` PASS: ${label}`); + passed += 1; + } else { + console.log( + ` FAIL: ${label} — expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}` + ); + failed += 1; + } + } + + function checkExists(label, value) { + if (value !== undefined && value !== null) { + console.log(` PASS: ${label} exists`); + passed += 1; + } else { + console.log(` FAIL: ${label} is missing`); + failed += 1; + } + } + + function checkType(label, value, type) { + const actual = Array.isArray(value) ? 'array' : typeof value; + const expected = type; + if (type === 'array' ? Array.isArray(value) : actual === expected) { + console.log(` PASS: ${label} is ${type}`); + passed += 1; + } else { + console.log(` FAIL: ${label} — expected ${type}, got ${actual}`); + failed += 1; + } + } + + function checkMatch(label, value, regex) { + if (regex.test(value)) { + console.log(` PASS: ${label} matches ${regex}`); + passed += 1; + } else { + console.log(` FAIL: ${label} — ${JSON.stringify(value)} does not match ${regex}`); + failed += 1; + } + } + + // --- Top-level container --- + console.log('\n--- Top-level container ---'); + checkType('container', container, 'object'); + check('container.id', container.id, CONTAINER_ID); + check('container.objectType', container.objectType, 'callingAIContainer'); + checkExists('container.summaryData', container.summaryData); + checkExists('container.memberships', container.memberships); + checkMatch('container.encryptionKeyUrl', container.encryptionKeyUrl, /^kms:\/\//); + checkMatch('container.kmsResourceObjectUrl', container.kmsResourceObjectUrl, /^kms:\/\//); + checkMatch('container.aclUrl', container.aclUrl, /^https?:\/\//); + checkExists('container.start', container.start); + checkExists('container.end', container.end); + checkExists('container.forkSessionId', container.forkSessionId); + checkExists('container.callSessionId', container.callSessionId); + checkExists('container.ownerUserId', container.ownerUserId); + checkExists('container.orgId', container.orgId); + + // --- summaryData --- + console.log('\n--- summaryData ---'); + const {summaryData} = container; + checkType('summaryData', summaryData, 'object'); + checkExists('summaryData.extensionId', summaryData.extensionId); + check('summaryData.objectType', summaryData.objectType, 'extension'); + check('summaryData.extensionType', summaryData.extensionType, 'callingAISummary'); + + // --- summaryData.data --- + console.log('\n--- summaryData.data ---'); + const summaryDataData = summaryData.data; + checkType('summaryData.data', summaryDataData, 'object'); + checkExists('summaryData.data.id', summaryDataData.id); + check('summaryData.data.objectType', summaryDataData.objectType, 'callingAISummary'); + checkExists('summaryData.data.status', summaryDataData.status); + checkMatch('summaryData.data.summaryUrl', summaryDataData.summaryUrl, /^https?:\/\//); + checkMatch('summaryData.data.transcriptUrl', summaryDataData.transcriptUrl, /^https?:\/\//); + checkMatch('summaryData.data.aclUrl', summaryDataData.aclUrl, /^https?:\/\//); + checkMatch( + 'summaryData.data.kmsResourceObjectUrl', + summaryDataData.kmsResourceObjectUrl, + /^kms:\/\// + ); + checkType('summaryData.data.summarizeAfterCall', summaryDataData.summarizeAfterCall, 'boolean'); + checkType('summaryData.data.contentRetention', summaryDataData.contentRetention, 'object'); + + // --- memberships --- + console.log('\n--- memberships ---'); + const {memberships} = container; + checkType('memberships', memberships, 'object'); + checkType('memberships.items', memberships.items, 'array'); + if (memberships.items.length > 0) { + console.log(` PASS: memberships.items has ${memberships.items.length} member(s)`); + passed += 1; + const first = memberships.items[0]; + checkExists('memberships.items[0].id', first.id); + checkType('memberships.items[0].roles', first.roles, 'array'); + check('memberships.items[0].objectType', first.objectType, 'containerMembership'); + } else { + console.log(' FAIL: memberships.items is empty'); + failed += 1; + } + + // --- Summary --- + console.log(`\n=== ${passed} passed, ${failed} failed ===`); + process.exit(failed > 0 ? 1 : 0); +} + +main().catch((err) => { + console.error('FAILED:', err.message || err); + process.exit(1); +}); diff --git a/packages/@webex/internal-plugin-call-ai-summary/src/types.ts b/packages/@webex/internal-plugin-call-ai-summary/src/types.ts new file mode 100644 index 00000000000..ce4fdfc4c14 --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/src/types.ts @@ -0,0 +1,154 @@ +/*! + * Copyright (c) 2015-2025 Cisco Systems, Inc. See LICENSE file. + */ + +// --- Pragya Response DTOs --- + +/** + * Summary data URLs from a Pragya container. + */ +export interface PragyaSummaryData { + /** Status of the summary (e.g., "Active") */ + status: string; + /** Full summary URL (AI Bridge) */ + summaryUrl: string; + /** Transcript URL (AI Bridge) */ + transcriptUrl: string; + /** Whether summarization runs after call ends */ + summarizeAfterCall: boolean; + /** Notes-specific URL (may not be present in all API versions) */ + notesUrl?: string; + /** Action items URL (may not be present in all API versions) */ + actionItemsUrl?: string; +} + +/** + * Complete Pragya container response. + */ +export interface PragyaContainerResponse { + /** Summary data with content URLs */ + summaryData: PragyaSummaryData; + /** KMS encryption key URL for decrypting content */ + encryptionKeyUrl: string; + /** KMS resource object URL */ + kmsResourceObjectUrl: string; + /** ACL URL for access control */ + aclUrl: string; + /** Fork session ID */ + forkSessionId: string; + /** Call session ID */ + callSessionId: string; + /** Owner user ID */ + ownerUserId: string; + /** Organization ID */ + orgId: string; + /** Call start time */ + start: string; + /** Call end time */ + end: string; +} + +// --- Request DTOs --- + +/** + * Options for resolving a Pragya container. + */ +export interface GetContainerOptions { + /** Pragya container ID (from Janus extensionPayload.callingContainerIds) */ + containerId: string; +} + +/** + * Options for fetching summary content. + * Requires the resolved Pragya container info. + */ +export interface GetSummaryContentOptions { + /** The resolved Pragya container response */ + containerInfo: PragyaContainerResponse; +} + +// --- Summary Response DTOs --- + +/** + * Decrypted AI-generated summary content. + * Contains all three content types returned by the summary API. + */ +export interface SummaryContent { + /** Unique identifier */ + id: string; + /** Decrypted full note content */ + note: string; + /** Decrypted short note content */ + shortNote: string; + /** Decrypted action item snippets */ + actionItems: ActionItemSnippet[]; + /** Feedback URL (if available) */ + feedbackUrl?: string; +} + +/** + * Decrypted AI-generated notes. + */ +export interface SummaryNotes { + /** Unique identifier */ + id: string; + /** Decrypted notes content */ + content: string; + /** Feedback URL (if available) */ + feedbackUrl?: string; +} + +/** + * Single action item snippet. + */ +export interface ActionItemSnippet { + /** Unique identifier */ + id: string; + /** User-edited version (if available) */ + editedContent?: string; + /** Decrypted AI-generated content */ + aiGeneratedContent: string; +} + +/** + * Decrypted AI-generated action items. + */ +export interface SummaryActionItems { + /** Unique identifier (absent when no action items exist) */ + id?: string; + /** Array of action item snippets */ + snippets: ActionItemSnippet[]; + /** Feedback URL (if available) */ + feedbackUrl?: string; +} + +/** + * Single decrypted transcript snippet. + */ +export interface TranscriptSnippet { + /** Start time in milliseconds */ + startTime: string; + /** End time in milliseconds */ + endTime: string; + /** Decrypted transcript content */ + content: string; + /** Audio CSI identifier */ + audioCSI?: string; + /** Speaker information */ + speaker?: { + speakerName: string; + speakerId: string; + }; +} + +/** + * Decrypted transcript response. + */ +export interface TranscriptContent { + /** Unique identifier */ + id: string; + /** Total number of snippets */ + totalCount: number; + /** Decrypted transcript snippets */ + snippets: TranscriptSnippet[]; +} diff --git a/yarn.lock b/yarn.lock index c00ce400761..a624aa069c0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7853,6 +7853,25 @@ __metadata: languageName: unknown linkType: soft +"@webex/internal-plugin-call-ai-summary@workspace:packages/@webex/internal-plugin-call-ai-summary": + version: 0.0.0-use.local + resolution: "@webex/internal-plugin-call-ai-summary@workspace:packages/@webex/internal-plugin-call-ai-summary" + dependencies: + "@babel/core": ^7.17.10 + "@webex/babel-config-legacy": "workspace:*" + "@webex/eslint-config-legacy": "workspace:*" + "@webex/internal-plugin-encryption": "workspace:*" + "@webex/jest-config-legacy": "workspace:*" + "@webex/legacy-tools": "workspace:*" + "@webex/test-helper-chai": "workspace:*" + "@webex/test-helper-mock-webex": "workspace:*" + "@webex/webex-core": "workspace:*" + eslint: ^8.24.0 + prettier: ^2.7.1 + sinon: ^9.2.4 + languageName: unknown + linkType: soft + "@webex/internal-plugin-conversation@workspace:*, @webex/internal-plugin-conversation@workspace:packages/@webex/internal-plugin-conversation": version: 0.0.0-use.local resolution: "@webex/internal-plugin-conversation@workspace:packages/@webex/internal-plugin-conversation" From d24b4b83453487420fa3682132c528e8e968bf99 Mon Sep 17 00:00:00 2001 From: Filip Nowakowski Date: Tue, 24 Mar 2026 07:41:16 +0100 Subject: [PATCH 04/28] refactor(meetings): enhance size hint handling and simplify codec info retrieval --- .../codec/mediaCodecHelper.h264.ts | 80 +++++++++---------- .../src/multistream/remoteMedia.ts | 24 +++--- .../src/multistream/remoteMediaGroup.ts | 12 +-- .../plugin-meetings/src/multistream/types.ts | 2 +- 4 files changed, 52 insertions(+), 66 deletions(-) diff --git a/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts b/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts index 32dacb2edf1..e7d22a1df5e 100644 --- a/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts +++ b/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts @@ -7,7 +7,7 @@ import { } from '@webex/internal-media-core'; import {CODEC_DEFAULTS, H264_CODEC_PARAMETERS, PANE_SIZE_TO_RESOLUTION} from './constants'; import {MediaCodecHelper, H264CodecInfo, GetCodecInfoOptions} from './types'; -import {MediaRequest, RemoteVideoResolution} from '../types'; +import {MediaRequest, RemoteVideoResolution, SizeHint} from '../types'; import LoggerProxy from '../../common/logs/logger-proxy'; /** @@ -21,13 +21,9 @@ export default class MediaCodecHelperH264 implements MediaCodecHelper 0 && sizeHint?.height > 0) { - maxFs = this.getSizeHintMaxFs(sizeHint.width, sizeHint.height); - } else if (sizeHint?.resolution) { - maxFs = this.getMaxFs(sizeHint.resolution); - } else { + if (!maxFs) { return undefined; } @@ -82,19 +78,15 @@ export default class MediaCodecHelperH264 implements MediaCodecHelper 0 && height > 0) { + // we switch to the next resolution level when the height is 10% more than the current resolution height + // except for 1080p - we switch to it immediately when the height is more than 720p + const threshold = 1.1; + const getThresholdHeight = (h: number) => Math.round(h * threshold); + + if (height < getThresholdHeight(90)) { + return H264_CODEC_PARAMETERS['90p'].maxFs; + } + if (height < getThresholdHeight(180)) { + return H264_CODEC_PARAMETERS['180p'].maxFs; + } + if (height < getThresholdHeight(360)) { + return H264_CODEC_PARAMETERS['360p'].maxFs; + } + if (height < getThresholdHeight(540)) { + return H264_CODEC_PARAMETERS['540p'].maxFs; + } + if (height <= 720) { + return H264_CODEC_PARAMETERS['720p'].maxFs; + } + + return H264_CODEC_PARAMETERS['1080p'].maxFs; } - // we switch to the next resolution level when the height is 10% more than the current resolution height - // except for 1080p - we switch to it immediately when the height is more than 720p - const threshold = 1.1; - const getThresholdHeight = (h: number) => Math.round(h * threshold); - - if (height < getThresholdHeight(90)) { - return H264_CODEC_PARAMETERS['90p'].maxFs; - } - if (height < getThresholdHeight(180)) { - return H264_CODEC_PARAMETERS['180p'].maxFs; - } - if (height < getThresholdHeight(360)) { - return H264_CODEC_PARAMETERS['360p'].maxFs; - } - if (height < getThresholdHeight(540)) { - return H264_CODEC_PARAMETERS['540p'].maxFs; - } - if (height <= 720) { - return H264_CODEC_PARAMETERS['720p'].maxFs; + // Fall back to resolution option + if (resolution) { + return this.getMaxFs(resolution); } - return H264_CODEC_PARAMETERS['1080p'].maxFs; + return undefined; } } diff --git a/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts b/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts index 5a605e54450..fe679eef4a9 100644 --- a/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts +++ b/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts @@ -59,7 +59,7 @@ export class RemoteMedia extends EventsScope { * Set by setSizeHint() based on video element dimensions. * @todo remove this once deprecation of getEffectiveMaxFs() is complete */ - private sizeHint?: SizeHint; + private sizeHint: SizeHint = {}; /** * Constructs RemoteMedia instance @@ -78,6 +78,7 @@ export class RemoteMedia extends EventsScope { this.receiveSlot = receiveSlot; this.mediaRequestManager = mediaRequestManager; this.options = options || {}; + this.sizeHint = {resolution: this.options.resolution}; this.setupEventListeners(); this.id = `RM${remoteMediaCounter}-${this.receiveSlot.id}`; } @@ -94,7 +95,8 @@ export class RemoteMedia extends EventsScope { return; } - this.sizeHint = {width, height, resolution: this.options.resolution}; + this.sizeHint.width = width; + this.sizeHint.height = height; this.receiveSlot?.setSizeHint(this.sizeHint); } @@ -112,15 +114,11 @@ export class RemoteMedia extends EventsScope { * @deprecated use getSizeHint() instead */ public getEffectiveMaxFs(): number | undefined { - if (this.sizeHint) { - return MediaCodecHelper.H264.getSizeHintMaxFs(this.sizeHint.width, this.sizeHint.height); - } - - if (this.options.resolution) { - return MediaCodecHelper.H264.getMaxFs(this.options.resolution); - } - - return undefined; + return MediaCodecHelper.H264.getSizeHintMaxFs({ + width: this.sizeHint?.width, + height: this.sizeHint?.height, + resolution: this.options.resolution, + }); } /** @@ -170,9 +168,7 @@ export class RemoteMedia extends EventsScope { csi, }, receiveSlots: [this.receiveSlot], - codecInfo: MediaCodecHelper.H264.getCodecInfo({ - sizeHint: this.sizeHint, - }), + sizeHint: this.sizeHint, }, commit ); diff --git a/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts b/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts index b5d16521bac..29453464b11 100644 --- a/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts +++ b/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts @@ -217,8 +217,6 @@ export class RemoteMediaGroup { private sendActiveSpeakerMediaRequest(commit: boolean) { this.cancelActiveSpeakerMediaRequest(false); - const sizeHint = this.getSizeHintForActiveSpeaker(); - this.mediaRequestId = this.mediaRequestManager.addRequest( { policyInfo: { @@ -234,11 +232,7 @@ export class RemoteMediaGroup { receiveSlots: this.unpinnedRemoteMedia.map((remoteMedia) => remoteMedia.getUnderlyingReceiveSlot() ) as ReceiveSlot[], - codecInfo: sizeHint - ? MediaCodecHelper.H264.getCodecInfo({ - sizeHint, - }) - : undefined, + sizeHint: this.getSizeHintForActiveSpeaker(), }, commit ); @@ -325,8 +319,6 @@ export class RemoteMediaGroup { // Fall back to group's resolution option if (this.options.resolution) { return { - width: 0, - height: 0, resolution: this.options.resolution, }; } @@ -335,6 +327,8 @@ export class RemoteMediaGroup { } /** + * @todo: Why do we calculate maxFs based on all unpinned RemoteMedia instances? + * * Calculate the effective maxFs for the active speaker media request based on unpinned RemoteMedia instances * @returns {number | undefined} The calculated maxFs value, or undefined if no constraints * @private diff --git a/packages/@webex/plugin-meetings/src/multistream/types.ts b/packages/@webex/plugin-meetings/src/multistream/types.ts index 08b57afe152..0401e9d6814 100644 --- a/packages/@webex/plugin-meetings/src/multistream/types.ts +++ b/packages/@webex/plugin-meetings/src/multistream/types.ts @@ -32,7 +32,7 @@ export type RemoteVideoResolution = /** highest possible resolution */ | 'best'; -export type SizeHint = {width: number; height: number; resolution?: RemoteVideoResolution}; +export type SizeHint = {width?: number; height?: number; resolution?: RemoteVideoResolution}; export interface MediaRequest { policyInfo: PolicyInfo; From 9cc40fdf78f6bc8a7a303b4f869c69e93fb72440 Mon Sep 17 00:00:00 2001 From: Filip Nowakowski Date: Tue, 24 Mar 2026 09:07:06 +0100 Subject: [PATCH 05/28] refactor(meetings): add deprecation warnings and enhance codec handling in media components --- .../plugin-meetings/src/metrics/constants.ts | 2 + .../plugin-meetings/src/metrics/index.ts | 4 + .../src/multistream/mediaRequestManager.ts | 82 ++++++++++++++----- .../src/multistream/receiveSlot.ts | 8 +- .../src/multistream/remoteMedia.ts | 23 ++++-- .../src/multistream/remoteMediaGroup.ts | 74 ++++++++++++----- .../plugin-meetings/src/multistream/types.ts | 5 ++ 7 files changed, 149 insertions(+), 49 deletions(-) diff --git a/packages/@webex/plugin-meetings/src/metrics/constants.ts b/packages/@webex/plugin-meetings/src/metrics/constants.ts index 5f1653f7922..0f313b43df2 100644 --- a/packages/@webex/plugin-meetings/src/metrics/constants.ts +++ b/packages/@webex/plugin-meetings/src/metrics/constants.ts @@ -92,6 +92,8 @@ const BEHAVIORAL_METRICS = { LOCUS_HASH_TREE_UNSUPPORTED_OPERATION: 'js_sdk_locus_hash_tree_unsupported_operation', MEDIA_STILL_NOT_CONNECTED: 'js_sdk_media_still_not_connected', DEPRECATED_SET_MAX_FS_USED: 'js_sdk_deprecated_set_max_fs_used', + DEPRECATED_GET_EFFECTIVE_MAX_FS_USED: 'js_sdk_deprecated_get_effective_max_fs_used', + DEPRECATED_RECEIVE_SLOT_SET_MAX_FS_USED: 'js_sdk_deprecated_receive_slot_set_max_fs_used', }; export {BEHAVIORAL_METRICS as default}; diff --git a/packages/@webex/plugin-meetings/src/metrics/index.ts b/packages/@webex/plugin-meetings/src/metrics/index.ts index 90f4bbe7f26..c4120686ac7 100644 --- a/packages/@webex/plugin-meetings/src/metrics/index.ts +++ b/packages/@webex/plugin-meetings/src/metrics/index.ts @@ -59,6 +59,10 @@ class Metrics { * @returns {void} */ sendBehavioralMetric(metricName: string, metricFields: object = {}, metricTags: object = {}) { + if (!this.webex?.internal?.metrics) { + return; + } + this.webex.internal.metrics.submitClientMetrics(metricName, { type: this.webex.config.metrics.type, fields: metricFields, diff --git a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts index 0754fc45688..0cb03311e2d 100644 --- a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts +++ b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts @@ -12,7 +12,7 @@ import {cloneDeepWith, debounce, isEmpty} from 'lodash'; import LoggerProxy from '../common/logs/logger-proxy'; import {ReceiveSlotEvents} from './receiveSlot'; -import {MediaRequest, MediaRequestId} from './types'; +import {MediaRequest, MediaRequestId, RemoteVideoResolution, SizeHint} from './types'; import {CODEC_DEFAULTS} from './codec/constants'; import MediaCodecHelper from './codec/mediaCodecHelper'; @@ -34,6 +34,15 @@ type Options = { type ClientRequestsMap = {[key: MediaRequestId]: MediaRequest}; export default class MediaRequestManager { + /** + * @deprecated Legacy pane-size to H264 maxFs mapping; prefer sizeHint / resolution on RemoteMedia. + * @param {RemoteVideoResolution} paneSize - Layout pane size label. + * @returns {number} H264 maxFs for that pane size. + */ + static getLegacyMaxFsForPaneSize(paneSize: RemoteVideoResolution): number { + return MediaCodecHelper.H264.getMaxFs(paneSize); + } + private sendMediaRequestsCallback: SendMediaRequestsCallback; private kind: Kind; @@ -75,6 +84,54 @@ export default class MediaRequestManager { this.sendRequests(); // re-send requests after preferences are set } + /** + * Fills or refreshes H264 `codecInfo` from `sizeHint` (and defaults). Video managers only; + * callers of `addRequest` should not need to pass `codecInfo`. + * @param {MediaRequest} mr - Request to mutate. + * @returns {void} + */ + private ensureH264CodecInfo(mr: MediaRequest): void { + if (this.kind !== 'video') { + return; + } + + const helper = MediaCodecHelper.H264; + const fromSizeHint = helper.getCodecInfo({sizeHint: mr.sizeHint || {}}); + + if (mr.codecInfo?.codec === 'h264') { + mr.codecInfo = { + ...mr.codecInfo, + maxFs: mr.codecInfo.maxFs ?? fromSizeHint?.maxFs ?? CODEC_DEFAULTS.h264.maxFs, + }; + + return; + } + + mr.codecInfo = fromSizeHint ?? {codec: 'h264', maxFs: CODEC_DEFAULTS.h264.maxFs}; + } + + /** + * Re-applies {@link ensureH264CodecInfo} for every request (e.g. on a clone before degradation). + * @param {ClientRequestsMap} clientRequests - Request map to mutate. + * @returns {void} + */ + private hydrateVideoCodecInfos(clientRequests: ClientRequestsMap): void { + Object.values(clientRequests).forEach((mr) => this.ensureH264CodecInfo(mr)); + } + + /** + * @internal Legacy H264 maxFs preview for deprecated getEffectiveMaxFs APIs. + * @param {SizeHint | undefined} sizeHint - Width/height and/or resolution hint. + * @returns {number | undefined} Effective maxFs, if derivable. + */ + public getLegacyEffectiveMaxFsFromSizeHint(sizeHint: SizeHint | undefined): number | undefined { + if (this.kind !== 'video') { + return undefined; + } + + return MediaCodecHelper.H264.getSizeHintMaxFs(sizeHint || {}); + } + private getDegradedClientRequests(clientRequests: ClientRequestsMap) { const resolutions: SupportedResolution[] = ['1080p', '720p', '540p', '360p', '180p', '90p']; @@ -160,24 +217,6 @@ export default class MediaRequestManager { return 0; } - /** - * Returns the max Macro Blocks per second (maxMbps) per H264 Stream - * - * The maxMbps will be calculated based on maxFs and maxFps - * (default h264 maxFps as fallback if maxFps is not defined) - * - * @param {MediaRequest} mediaRequest - mediaRequest to take data from - * @returns {number} maxMbps - */ - // eslint-disable-next-line class-methods-use-this - private getH264MaxMbps(mediaRequest: MediaRequest): number { - // fallback for maxFps (not needed for maxFs, since there is a fallback already in getDegradedClientRequests) - const maxFps = mediaRequest.codecInfo.maxFps || CODEC_DEFAULTS.h264.maxFps; - - // divided by 100 since maxFps is 3000 (for 30 frames per seconds) - return (mediaRequest.codecInfo.maxFs * maxFps) / 100; - } - /** * Clears the previous stream requests. * @@ -282,6 +321,7 @@ export default class MediaRequestManager { const clientRequests = this.cloneClientRequests(); this.trimRequests(clientRequests); + this.hydrateVideoCodecInfos(clientRequests); this.getDegradedClientRequests(clientRequests); // map all the client media requests to wcme stream requests @@ -326,13 +366,17 @@ export default class MediaRequestManager { this.clientRequests[newId] = mediaRequest; + this.ensureH264CodecInfo(mediaRequest); + mediaRequest.handleMaxFs = ({maxFs}) => { mediaRequest.preferredMaxFs = maxFs; + this.ensureH264CodecInfo(mediaRequest); this.debouncedSourceUpdateListener(); }; mediaRequest.handleSizeHint = (sizeHint) => { mediaRequest.sizeHint = sizeHint; + this.ensureH264CodecInfo(mediaRequest); this.debouncedSourceUpdateListener(); }; diff --git a/packages/@webex/plugin-meetings/src/multistream/receiveSlot.ts b/packages/@webex/plugin-meetings/src/multistream/receiveSlot.ts index c3c1ee88e56..0a0591f6bf5 100644 --- a/packages/@webex/plugin-meetings/src/multistream/receiveSlot.ts +++ b/packages/@webex/plugin-meetings/src/multistream/receiveSlot.ts @@ -7,7 +7,9 @@ import { } from '@webex/internal-media-core'; import LoggerProxy from '../common/logs/logger-proxy'; +import Metrics from '../metrics'; import EventsScope from '../common/events/events-scope'; +import BEHAVIORAL_METRICS from '../metrics/constants'; import {SizeHint} from './types'; export const ReceiveSlotEvents = { @@ -104,9 +106,13 @@ export class ReceiveSlot extends EventsScope { /** * Set the max frame size for this slot * @param newFs frame size + * @deprecated Prefer {@link ReceiveSlot.setSizeHint} or layout resolution; H264 maxFs is handled inside MediaRequestManager. */ public setMaxFs(newFs) { - // emit event for media request manager to listen to + LoggerProxy.logger.warn( + 'ReceiveSlot->setMaxFs --> [DEPRECATION WARNING]: use setSizeHint() / layout resolution instead' + ); + Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.DEPRECATED_RECEIVE_SLOT_SET_MAX_FS_USED, {}); this.emit( { diff --git a/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts b/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts index fe679eef4a9..f9c1473e2e0 100644 --- a/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts +++ b/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts @@ -1,13 +1,13 @@ /* eslint-disable valid-jsdoc */ import {MediaType, StreamState} from '@webex/internal-media-core'; -import Metrics from '@webex/internal-plugin-metrics'; import EventsScope from '../common/events/events-scope'; +import Metrics from '../metrics'; +import LoggerProxy from '../common/logs/logger-proxy'; import MediaRequestManager from './mediaRequestManager'; import {CSI, ReceiveSlot, ReceiveSlotEvents} from './receiveSlot'; import type {MediaRequestId, RemoteVideoResolution, SizeHint} from './types'; import BEHAVIORAL_METRICS from '../metrics/constants'; -import MediaCodecHelper from './codec/mediaCodecHelper'; export const RemoteMediaEvents = { SourceUpdate: ReceiveSlotEvents.SourceUpdate, @@ -18,15 +18,15 @@ export const RemoteMediaEvents = { * Converts pane size into h264 maxFs * @param {RemoteVideoResolution} paneSize * @returns {number} - * @deprecated use MediaCodecHelper from plugin-meetings/src/codec/mediaCodecHelper instead + * @deprecated Prefer `RemoteMedia` resolution options and `setSizeHint()`; see `multistream/codec/mediaCodecHelper` for codec details. */ export function getMaxFs(paneSize: RemoteVideoResolution): number { - this.LoggerProxy.logger.warn( - 'RemoteMedia->getMaxFs --> [DEPRECATION WARNING]: getMaxFs has been deprecated, use MediaCodecHelper instead' + LoggerProxy.logger.warn( + 'RemoteMedia->getMaxFs --> [DEPRECATION WARNING]: getMaxFs has been deprecated; use size hints / resolution on RemoteMedia instead' ); Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.DEPRECATED_SET_MAX_FS_USED, {paneSize}); - return MediaCodecHelper.H264.getMaxFs(paneSize); + return MediaRequestManager.getLegacyMaxFsForPaneSize(paneSize); } type Options = { @@ -111,10 +111,17 @@ export class RemoteMedia extends EventsScope { /** * Get the current effective maxFs value that would be used in media requests * @returns {number | undefined} The maxFs value, or undefined if no constraints - * @deprecated use getSizeHint() instead + * @deprecated Use {@link RemoteMedia.getSizeHint} and layout resolution instead. */ public getEffectiveMaxFs(): number | undefined { - return MediaCodecHelper.H264.getSizeHintMaxFs({ + LoggerProxy.logger.warn( + 'RemoteMedia->getEffectiveMaxFs --> [DEPRECATION WARNING]: use getSizeHint() and resolution options instead' + ); + Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.DEPRECATED_GET_EFFECTIVE_MAX_FS_USED, { + surface: 'RemoteMedia', + }); + + return this.mediaRequestManager.getLegacyEffectiveMaxFsFromSizeHint({ width: this.sizeHint?.width, height: this.sizeHint?.height, resolution: this.options.resolution, diff --git a/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts b/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts index 29453464b11..9007349a16f 100644 --- a/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts +++ b/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts @@ -4,12 +4,23 @@ import {forEach} from 'lodash'; import {NamedMediaGroup} from '@webex/internal-media-core'; import LoggerProxy from '../common/logs/logger-proxy'; +import Metrics from '../metrics'; import {RemoteMedia} from './remoteMedia'; import MediaRequestManager from './mediaRequestManager'; import type {MediaRequestId, RemoteVideoResolution, SizeHint} from './types'; import {CSI, ReceiveSlot} from './receiveSlot'; -import MediaCodecHelper from './codec/mediaCodecHelper'; +import BEHAVIORAL_METRICS from '../metrics/constants'; + +/** Higher rank = larger nominal pane / resolution for active-speaker sizeHint merge. */ +const REMOTE_VIDEO_RESOLUTION_RANK: Record = { + thumbnail: 1, + 'very small': 2, + small: 3, + medium: 4, + large: 5, + best: 6, +}; type Options = { resolution?: RemoteVideoResolution; // applies only to groups of type MediaType.VideoMain and MediaType.VideoSlides @@ -301,29 +312,42 @@ export class RemoteMediaGroup { } private getSizeHintForActiveSpeaker(): SizeHint | undefined { - // Get all size hint values from unpinned RemoteMedia instances const sizeHints = this.unpinnedRemoteMedia .map((remoteMedia) => remoteMedia.getSizeHint()) - .filter((sizeHint) => !!sizeHint); + .filter((sizeHint): sizeHint is SizeHint => !!sizeHint); - // Use the highest sizeHint based on width*height value to ensure we don't under-request resolution for any instance - if (sizeHints.length > 0) { - return sizeHints.reduce((maxSizeHint, currentSizeHint) => { - const currentSize = currentSizeHint.width * currentSizeHint.height; - const maxSize = maxSizeHint.width * maxSizeHint.height; + if (sizeHints.length === 0) { + if (this.options.resolution) { + return {resolution: this.options.resolution}; + } - return currentSize > maxSize ? currentSizeHint : maxSizeHint; - }, sizeHints[0]); + return undefined; + } + + const withPixels = sizeHints.filter((sh) => (sh.width ?? 0) > 0 && (sh.height ?? 0) > 0); + + if (withPixels.length > 0) { + return withPixels.reduce((best, cur) => + cur.width! * cur.height! > best.width! * best.height! ? cur : best + ); + } + + const withResolution = sizeHints.filter((sh) => sh.resolution); + + if (withResolution.length > 0) { + return withResolution.reduce((best, cur) => + REMOTE_VIDEO_RESOLUTION_RANK[cur.resolution!] > + REMOTE_VIDEO_RESOLUTION_RANK[best.resolution!] + ? cur + : best + ); } - // Fall back to group's resolution option if (this.options.resolution) { - return { - resolution: this.options.resolution, - }; + return {resolution: this.options.resolution}; } - return undefined; + return sizeHints[0]; } /** @@ -335,19 +359,20 @@ export class RemoteMediaGroup { * @deprecated */ private getEffectiveMaxFsForActiveSpeaker(): number | undefined { - // Get all effective maxFs values from unpinned RemoteMedia instances const maxFsValues = this.unpinnedRemoteMedia - .map((remoteMedia) => remoteMedia.getEffectiveMaxFs()) + .map((remoteMedia) => + this.mediaRequestManager.getLegacyEffectiveMaxFsFromSizeHint(remoteMedia.getSizeHint()) + ) .filter((maxFs) => maxFs !== undefined); - // Use the highest maxFs value to ensure we don't under-request resolution for any instance if (maxFsValues.length > 0) { return Math.max(...maxFsValues); } - // Fall back to group's resolution option if (this.options.resolution) { - return MediaCodecHelper.H264.getMaxFs(this.options.resolution); + return this.mediaRequestManager.getLegacyEffectiveMaxFsFromSizeHint({ + resolution: this.options.resolution, + }); } return undefined; @@ -356,9 +381,16 @@ export class RemoteMediaGroup { /** * Get the current effective maxFs that would be used for the active speaker media request * @returns {number | undefined} The effective maxFs value - * @deprecated + * @deprecated Use unpinned {@link RemoteMedia.getSizeHint} values and group resolution options instead. */ public getEffectiveMaxFs(): number | undefined { + LoggerProxy.logger.warn( + 'RemoteMediaGroup->getEffectiveMaxFs --> [DEPRECATION WARNING]: use getSizeHint() on remote media instances instead' + ); + Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.DEPRECATED_GET_EFFECTIVE_MAX_FS_USED, { + surface: 'RemoteMediaGroup', + }); + return this.getEffectiveMaxFsForActiveSpeaker(); } } diff --git a/packages/@webex/plugin-meetings/src/multistream/types.ts b/packages/@webex/plugin-meetings/src/multistream/types.ts index 0401e9d6814..1c4165d60fa 100644 --- a/packages/@webex/plugin-meetings/src/multistream/types.ts +++ b/packages/@webex/plugin-meetings/src/multistream/types.ts @@ -37,6 +37,11 @@ export type SizeHint = {width?: number; height?: number; resolution?: RemoteVide export interface MediaRequest { policyInfo: PolicyInfo; receiveSlots: Array; + /** + * For {@link MediaRequestManager} with `kind: 'video'`, H264 `codecInfo` is always filled from + * `sizeHint` (and defaults) inside the manager. Callers should pass `sizeHint` / layout resolution + * only and must not rely on setting this field. + */ codecInfo?: CodecInfo; preferredMaxFs?: number; sizeHint?: SizeHint; From c8d4a145e2269826e69fd6b9ec0fdec5f5ed2bd0 Mon Sep 17 00:00:00 2001 From: Filip Nowakowski Date: Tue, 24 Mar 2026 09:26:58 +0100 Subject: [PATCH 06/28] refactor(meetings): streamline codec handling and improve size hint resolution logic --- .../src/multistream/codec/constants.ts | 10 +++++ .../src/multistream/mediaRequestManager.ts | 43 +------------------ .../src/multistream/remoteMediaGroup.ts | 23 +++------- .../plugin-meetings/src/multistream/types.ts | 2 +- 4 files changed, 18 insertions(+), 60 deletions(-) diff --git a/packages/@webex/plugin-meetings/src/multistream/codec/constants.ts b/packages/@webex/plugin-meetings/src/multistream/codec/constants.ts index d57f6ebd699..9b84e82e66a 100644 --- a/packages/@webex/plugin-meetings/src/multistream/codec/constants.ts +++ b/packages/@webex/plugin-meetings/src/multistream/codec/constants.ts @@ -38,3 +38,13 @@ export const PANE_SIZE_TO_RESOLUTION = { large: '1080p', best: '1080p', } satisfies Record; + +/** Higher rank = larger nominal pane / resolution */ +export const PANE_SIZE_RANK = { + thumbnail: 1, + 'very small': 2, + small: 3, + medium: 4, + large: 5, + best: 6, +} satisfies Record; diff --git a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts index 0cb03311e2d..da95d69ba15 100644 --- a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts +++ b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts @@ -13,7 +13,6 @@ import LoggerProxy from '../common/logs/logger-proxy'; import {ReceiveSlotEvents} from './receiveSlot'; import {MediaRequest, MediaRequestId, RemoteVideoResolution, SizeHint} from './types'; -import {CODEC_DEFAULTS} from './codec/constants'; import MediaCodecHelper from './codec/mediaCodecHelper'; const DEBOUNCED_SOURCE_UPDATE_TIME = 1000; @@ -84,41 +83,6 @@ export default class MediaRequestManager { this.sendRequests(); // re-send requests after preferences are set } - /** - * Fills or refreshes H264 `codecInfo` from `sizeHint` (and defaults). Video managers only; - * callers of `addRequest` should not need to pass `codecInfo`. - * @param {MediaRequest} mr - Request to mutate. - * @returns {void} - */ - private ensureH264CodecInfo(mr: MediaRequest): void { - if (this.kind !== 'video') { - return; - } - - const helper = MediaCodecHelper.H264; - const fromSizeHint = helper.getCodecInfo({sizeHint: mr.sizeHint || {}}); - - if (mr.codecInfo?.codec === 'h264') { - mr.codecInfo = { - ...mr.codecInfo, - maxFs: mr.codecInfo.maxFs ?? fromSizeHint?.maxFs ?? CODEC_DEFAULTS.h264.maxFs, - }; - - return; - } - - mr.codecInfo = fromSizeHint ?? {codec: 'h264', maxFs: CODEC_DEFAULTS.h264.maxFs}; - } - - /** - * Re-applies {@link ensureH264CodecInfo} for every request (e.g. on a clone before degradation). - * @param {ClientRequestsMap} clientRequests - Request map to mutate. - * @returns {void} - */ - private hydrateVideoCodecInfos(clientRequests: ClientRequestsMap): void { - Object.values(clientRequests).forEach((mr) => this.ensureH264CodecInfo(mr)); - } - /** * @internal Legacy H264 maxFs preview for deprecated getEffectiveMaxFs APIs. * @param {SizeHint | undefined} sizeHint - Width/height and/or resolution hint. @@ -321,7 +285,6 @@ export default class MediaRequestManager { const clientRequests = this.cloneClientRequests(); this.trimRequests(clientRequests); - this.hydrateVideoCodecInfos(clientRequests); this.getDegradedClientRequests(clientRequests); // map all the client media requests to wcme stream requests @@ -360,23 +323,19 @@ export default class MediaRequestManager { this.sendMediaRequestsCallback(streamRequests); } - public addRequest(mediaRequest: MediaRequest, commit = true): MediaRequestId { + public addRequest(mediaRequest: Omit, commit = true): MediaRequestId { // eslint-disable-next-line no-plusplus const newId = `${this.counter++}`; this.clientRequests[newId] = mediaRequest; - this.ensureH264CodecInfo(mediaRequest); - mediaRequest.handleMaxFs = ({maxFs}) => { mediaRequest.preferredMaxFs = maxFs; - this.ensureH264CodecInfo(mediaRequest); this.debouncedSourceUpdateListener(); }; mediaRequest.handleSizeHint = (sizeHint) => { mediaRequest.sizeHint = sizeHint; - this.ensureH264CodecInfo(mediaRequest); this.debouncedSourceUpdateListener(); }; diff --git a/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts b/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts index 9007349a16f..a5bd5c225ec 100644 --- a/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts +++ b/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts @@ -11,16 +11,7 @@ import MediaRequestManager from './mediaRequestManager'; import type {MediaRequestId, RemoteVideoResolution, SizeHint} from './types'; import {CSI, ReceiveSlot} from './receiveSlot'; import BEHAVIORAL_METRICS from '../metrics/constants'; - -/** Higher rank = larger nominal pane / resolution for active-speaker sizeHint merge. */ -const REMOTE_VIDEO_RESOLUTION_RANK: Record = { - thumbnail: 1, - 'very small': 2, - small: 3, - medium: 4, - large: 5, - best: 6, -}; +import {PANE_SIZE_RANK} from './codec/constants'; type Options = { resolution?: RemoteVideoResolution; // applies only to groups of type MediaType.VideoMain and MediaType.VideoSlides @@ -325,21 +316,19 @@ export class RemoteMediaGroup { } const withPixels = sizeHints.filter((sh) => (sh.width ?? 0) > 0 && (sh.height ?? 0) > 0); - if (withPixels.length > 0) { + // return the size hint with the largest area return withPixels.reduce((best, cur) => - cur.width! * cur.height! > best.width! * best.height! ? cur : best + cur.width * cur.height > best.width * best.height ? cur : best ); } const withResolution = sizeHints.filter((sh) => sh.resolution); if (withResolution.length > 0) { + // return the size hint with the highest resolution rank return withResolution.reduce((best, cur) => - REMOTE_VIDEO_RESOLUTION_RANK[cur.resolution!] > - REMOTE_VIDEO_RESOLUTION_RANK[best.resolution!] - ? cur - : best + PANE_SIZE_RANK[cur.resolution] > PANE_SIZE_RANK[best.resolution] ? cur : best ); } @@ -347,7 +336,7 @@ export class RemoteMediaGroup { return {resolution: this.options.resolution}; } - return sizeHints[0]; + return undefined; } /** diff --git a/packages/@webex/plugin-meetings/src/multistream/types.ts b/packages/@webex/plugin-meetings/src/multistream/types.ts index 1c4165d60fa..fa2d54ab5f2 100644 --- a/packages/@webex/plugin-meetings/src/multistream/types.ts +++ b/packages/@webex/plugin-meetings/src/multistream/types.ts @@ -1,6 +1,6 @@ import {NamedMediaGroup} from '@webex/internal-media-core'; import type {CodecInfo} from './codec/types'; -import {ReceiveSlot} from './receiveSlot'; +import type {ReceiveSlot} from './receiveSlot'; export interface ActiveSpeakerPolicyInfo { policy: 'active-speaker'; From 379171a7f1db0724d81d22f98eba252a97f396ab Mon Sep 17 00:00:00 2001 From: Filip Nowakowski Date: Tue, 24 Mar 2026 09:27:44 +0100 Subject: [PATCH 07/28] refactor(meetings): remove unnecessary metrics check in behavioral metric submission --- packages/@webex/plugin-meetings/src/metrics/index.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/@webex/plugin-meetings/src/metrics/index.ts b/packages/@webex/plugin-meetings/src/metrics/index.ts index c4120686ac7..90f4bbe7f26 100644 --- a/packages/@webex/plugin-meetings/src/metrics/index.ts +++ b/packages/@webex/plugin-meetings/src/metrics/index.ts @@ -59,10 +59,6 @@ class Metrics { * @returns {void} */ sendBehavioralMetric(metricName: string, metricFields: object = {}, metricTags: object = {}) { - if (!this.webex?.internal?.metrics) { - return; - } - this.webex.internal.metrics.submitClientMetrics(metricName, { type: this.webex.config.metrics.type, fields: metricFields, From 6e8fbd5707054e6f3d71271534881cf783d09ed9 Mon Sep 17 00:00:00 2001 From: Filip Nowakowski Date: Tue, 24 Mar 2026 09:33:17 +0100 Subject: [PATCH 08/28] refactor(meetings): remove deprecated methods and streamline codec resolution in media components --- .../src/multistream/mediaRequestManager.ts | 22 ------------------- .../src/multistream/remoteMedia.ts | 5 +++-- .../src/multistream/remoteMediaGroup.ts | 7 +++--- 3 files changed, 6 insertions(+), 28 deletions(-) diff --git a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts index da95d69ba15..bb8c14d111e 100644 --- a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts +++ b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts @@ -33,15 +33,6 @@ type Options = { type ClientRequestsMap = {[key: MediaRequestId]: MediaRequest}; export default class MediaRequestManager { - /** - * @deprecated Legacy pane-size to H264 maxFs mapping; prefer sizeHint / resolution on RemoteMedia. - * @param {RemoteVideoResolution} paneSize - Layout pane size label. - * @returns {number} H264 maxFs for that pane size. - */ - static getLegacyMaxFsForPaneSize(paneSize: RemoteVideoResolution): number { - return MediaCodecHelper.H264.getMaxFs(paneSize); - } - private sendMediaRequestsCallback: SendMediaRequestsCallback; private kind: Kind; @@ -83,19 +74,6 @@ export default class MediaRequestManager { this.sendRequests(); // re-send requests after preferences are set } - /** - * @internal Legacy H264 maxFs preview for deprecated getEffectiveMaxFs APIs. - * @param {SizeHint | undefined} sizeHint - Width/height and/or resolution hint. - * @returns {number | undefined} Effective maxFs, if derivable. - */ - public getLegacyEffectiveMaxFsFromSizeHint(sizeHint: SizeHint | undefined): number | undefined { - if (this.kind !== 'video') { - return undefined; - } - - return MediaCodecHelper.H264.getSizeHintMaxFs(sizeHint || {}); - } - private getDegradedClientRequests(clientRequests: ClientRequestsMap) { const resolutions: SupportedResolution[] = ['1080p', '720p', '540p', '360p', '180p', '90p']; diff --git a/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts b/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts index f9c1473e2e0..5cc365efc8b 100644 --- a/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts +++ b/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts @@ -8,6 +8,7 @@ import MediaRequestManager from './mediaRequestManager'; import {CSI, ReceiveSlot, ReceiveSlotEvents} from './receiveSlot'; import type {MediaRequestId, RemoteVideoResolution, SizeHint} from './types'; import BEHAVIORAL_METRICS from '../metrics/constants'; +import MediaCodecHelper from './codec/mediaCodecHelper'; export const RemoteMediaEvents = { SourceUpdate: ReceiveSlotEvents.SourceUpdate, @@ -26,7 +27,7 @@ export function getMaxFs(paneSize: RemoteVideoResolution): number { ); Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.DEPRECATED_SET_MAX_FS_USED, {paneSize}); - return MediaRequestManager.getLegacyMaxFsForPaneSize(paneSize); + return MediaCodecHelper.H264.getMaxFs(paneSize); } type Options = { @@ -121,7 +122,7 @@ export class RemoteMedia extends EventsScope { surface: 'RemoteMedia', }); - return this.mediaRequestManager.getLegacyEffectiveMaxFsFromSizeHint({ + return MediaCodecHelper.H264.getSizeHintMaxFs({ width: this.sizeHint?.width, height: this.sizeHint?.height, resolution: this.options.resolution, diff --git a/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts b/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts index a5bd5c225ec..420a8c350d7 100644 --- a/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts +++ b/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts @@ -12,6 +12,7 @@ import type {MediaRequestId, RemoteVideoResolution, SizeHint} from './types'; import {CSI, ReceiveSlot} from './receiveSlot'; import BEHAVIORAL_METRICS from '../metrics/constants'; import {PANE_SIZE_RANK} from './codec/constants'; +import MediaCodecHelper from './codec/mediaCodecHelper'; type Options = { resolution?: RemoteVideoResolution; // applies only to groups of type MediaType.VideoMain and MediaType.VideoSlides @@ -349,9 +350,7 @@ export class RemoteMediaGroup { */ private getEffectiveMaxFsForActiveSpeaker(): number | undefined { const maxFsValues = this.unpinnedRemoteMedia - .map((remoteMedia) => - this.mediaRequestManager.getLegacyEffectiveMaxFsFromSizeHint(remoteMedia.getSizeHint()) - ) + .map((remoteMedia) => MediaCodecHelper.H264.getSizeHintMaxFs(remoteMedia.getSizeHint())) .filter((maxFs) => maxFs !== undefined); if (maxFsValues.length > 0) { @@ -359,7 +358,7 @@ export class RemoteMediaGroup { } if (this.options.resolution) { - return this.mediaRequestManager.getLegacyEffectiveMaxFsFromSizeHint({ + return MediaCodecHelper.H264.getSizeHintMaxFs({ resolution: this.options.resolution, }); } From 9f97523ac10d64fa9b700757e4e7cdaab181e672 Mon Sep 17 00:00:00 2001 From: Filip Nowakowski Date: Tue, 24 Mar 2026 10:05:10 +0100 Subject: [PATCH 09/28] refactor(meetings): enhance codec info handling in MediaRequest and improve degradation logic --- .../codec/mediaCodecHelper.h264.ts | 38 ++++++++++++------- .../src/multistream/mediaRequestManager.ts | 32 +++++++++++----- .../plugin-meetings/src/multistream/types.ts | 7 +--- 3 files changed, 48 insertions(+), 29 deletions(-) diff --git a/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts b/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts index e7d22a1df5e..9af59581678 100644 --- a/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts +++ b/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts @@ -41,20 +41,25 @@ export default class MediaCodecHelperH264 implements MediaCodecHelper codecInfo.codec === 'h264'); + if (codecInfos.length === 0) { return 0; } - mr.codecInfo.maxFs = Math.min( + const maxFs = Math.min( mr.preferredMaxFs || CODEC_DEFAULTS.h264.maxFs, - mr.codecInfo.maxFs || CODEC_DEFAULTS.h264.maxFs, + Math.min(...codecInfos.map((codecInfo) => codecInfo.maxFs)), H264_CODEC_PARAMETERS[resolution].maxFs ); + codecInfos.forEach((codecInfo) => { + codecInfo.maxFs = maxFs; + }); + // we only consider sources with "live" state const slotsWithLiveSource = mr.receiveSlots.filter((rs) => rs.sourceState === 'live'); - return mr.codecInfo.maxFs * slotsWithLiveSource.length; + return maxFs * slotsWithLiveSource.length; } /** @@ -64,11 +69,14 @@ export default class MediaCodecHelperH264 implements MediaCodecHelper codecInfo.codec === 'h264'); + if (codecInfos.length === 0) { return 0; } - return getRecommendedMaxBitrateForFrameSize(mediaRequest.codecInfo.maxFs); + return getRecommendedMaxBitrateForFrameSize( + Math.min(...codecInfos.map((codecInfo) => codecInfo.maxFs)) + ); } /** @@ -79,14 +87,16 @@ export default class MediaCodecHelperH264 implements MediaCodecHelper + WcmeCodecInfo.fromH264( + 0x80, // TODO: Fix this constant + new H264Codec( + codecInfo.maxFs, + codecInfo.maxFps || CODEC_DEFAULTS.h264.maxFps, + codecInfo.maxMbps || CODEC_DEFAULTS.h264.maxMbps, + codecInfo.maxWidth, + codecInfo.maxHeight + ) ) ), ]; diff --git a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts index bb8c14d111e..94313d80efc 100644 --- a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts +++ b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts @@ -12,7 +12,7 @@ import {cloneDeepWith, debounce, isEmpty} from 'lodash'; import LoggerProxy from '../common/logs/logger-proxy'; import {ReceiveSlotEvents} from './receiveSlot'; -import {MediaRequest, MediaRequestId, RemoteVideoResolution, SizeHint} from './types'; +import {MediaRequest, MediaRequestId} from './types'; import MediaCodecHelper from './codec/mediaCodecHelper'; const DEBOUNCED_SOURCE_UPDATE_TIME = 1000; @@ -81,8 +81,11 @@ export default class MediaRequestManager { let totalMacroblocksRequested = 0; Object.values(clientRequests).forEach((mr) => { - const mediaCodecHelper = MediaCodecHelper.get(mr.codecInfo?.codec); - totalMacroblocksRequested += mediaCodecHelper.degradeMediaRequest(mr, resolution); + totalMacroblocksRequested += Math.max( + ...mr.codecInfos.map((codecInfo) => + MediaCodecHelper.get(codecInfo.codec).degradeMediaRequest(mr, resolution) + ) + ); }); if (totalMacroblocksRequested <= this.degradationPreferences.maxMacroblocksLimit) { @@ -146,10 +149,12 @@ export default class MediaRequestManager { return RecommendedOpusBitrates.FB_MONO_MUSIC; } - if (mediaRequest.codecInfo?.codec) { - const mediaCodecHelper = MediaCodecHelper.get(mediaRequest.codecInfo.codec); + if (mediaRequest.codecInfos) { + const maxPbps = mediaRequest.codecInfos.map((codecInfo) => + MediaCodecHelper.get(codecInfo.codec).getMaxPayloadBitsPerSecond(mediaRequest) + ); - return mediaCodecHelper.getMaxPayloadBitsPerSecond(mediaRequest); + return Math.max(...maxPbps); } LoggerProxy.logger.warn( @@ -286,7 +291,9 @@ export default class MediaRequestManager { const receiveSlots = mr.receiveSlots.map((receiveSlot) => receiveSlot.wcmeReceiveSlot); const maxPayloadBitsPerSecond = this.getMaxPayloadBitsPerSecond(mr); - const codecInfos = [...MediaCodecHelper.H264.getWCMECodecInfos(mr)]; + const codecInfos = mr.codecInfos.flatMap((codecInfo) => + MediaCodecHelper.get(codecInfo.codec).getWCMECodecInfos(mr) + ); const streamRequest = new StreamRequest( policy, @@ -301,11 +308,18 @@ export default class MediaRequestManager { this.sendMediaRequestsCallback(streamRequests); } - public addRequest(mediaRequest: Omit, commit = true): MediaRequestId { + public addRequest(mediaRequest: Omit, commit = true): MediaRequestId { // eslint-disable-next-line no-plusplus const newId = `${this.counter++}`; - this.clientRequests[newId] = mediaRequest; + this.clientRequests[newId] = { + ...mediaRequest, + codecInfos: [ + MediaCodecHelper.H264.getCodecInfo({ + sizeHint: mediaRequest.sizeHint, + }), + ], + }; mediaRequest.handleMaxFs = ({maxFs}) => { mediaRequest.preferredMaxFs = maxFs; diff --git a/packages/@webex/plugin-meetings/src/multistream/types.ts b/packages/@webex/plugin-meetings/src/multistream/types.ts index fa2d54ab5f2..0355e8090a9 100644 --- a/packages/@webex/plugin-meetings/src/multistream/types.ts +++ b/packages/@webex/plugin-meetings/src/multistream/types.ts @@ -37,12 +37,7 @@ export type SizeHint = {width?: number; height?: number; resolution?: RemoteVide export interface MediaRequest { policyInfo: PolicyInfo; receiveSlots: Array; - /** - * For {@link MediaRequestManager} with `kind: 'video'`, H264 `codecInfo` is always filled from - * `sizeHint` (and defaults) inside the manager. Callers should pass `sizeHint` / layout resolution - * only and must not rely on setting this field. - */ - codecInfo?: CodecInfo; + codecInfos?: CodecInfo[]; preferredMaxFs?: number; sizeHint?: SizeHint; handleMaxFs?: ({maxFs}: {maxFs: number}) => void; From 27daeee60335599c1ac146ef4df5cd3cf2ecfeb0 Mon Sep 17 00:00:00 2001 From: Filip Nowakowski Date: Wed, 25 Mar 2026 10:05:05 +0100 Subject: [PATCH 10/28] refactor(meetings): improve media request handling and enhance size hint logic --- .../codec/mediaCodecHelper.h264.ts | 5 +- .../src/multistream/mediaRequestManager.ts | 48 +++++-- .../spec/multistream/mediaRequestManager.ts | 130 ++++++++---------- .../test/unit/spec/multistream/receiveSlot.ts | 33 ++++- .../test/unit/spec/multistream/remoteMedia.ts | 36 +++-- .../unit/spec/multistream/remoteMediaGroup.ts | 57 +++----- .../spec/multistream/remoteMediaManager.ts | 53 +++---- 7 files changed, 193 insertions(+), 169 deletions(-) diff --git a/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts b/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts index 9af59581678..8adeeab9479 100644 --- a/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts +++ b/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts @@ -20,7 +20,7 @@ export default class MediaCodecHelperH264 implements MediaCodecHelper 0 && height > 0) { // we switch to the next resolution level when the height is 10% more than the current resolution height // except for 1080p - we switch to it immediately when the height is more than 720p diff --git a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts index 94313d80efc..9ebb7a9fae4 100644 --- a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts +++ b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts @@ -14,6 +14,8 @@ import LoggerProxy from '../common/logs/logger-proxy'; import {ReceiveSlotEvents} from './receiveSlot'; import {MediaRequest, MediaRequestId} from './types'; import MediaCodecHelper from './codec/mediaCodecHelper'; +import {CODEC_DEFAULTS} from './codec/constants'; +import type {CodecInfo} from './codec/types'; const DEBOUNCED_SOURCE_UPDATE_TIME = 1000; @@ -305,6 +307,11 @@ export default class MediaRequestManager { streamRequests.push(streamRequest); }); + if (this.checkIsNewRequestsEqualToPrev(streamRequests)) { + return; + } + + this.previousStreamRequests = [...streamRequests]; this.sendMediaRequestsCallback(streamRequests); } @@ -312,29 +319,43 @@ export default class MediaRequestManager { // eslint-disable-next-line no-plusplus const newId = `${this.counter++}`; - this.clientRequests[newId] = { + const codecInfos: CodecInfo[] = + this.kind === 'audio' + ? [] + : (() => { + const info = MediaCodecHelper.H264.getCodecInfo({ + sizeHint: mediaRequest.sizeHint, + }); + + return info ? [info] : [{codec: 'h264', maxFs: CODEC_DEFAULTS.h264.maxFs}]; + })(); + + const storedRequest: MediaRequest = { ...mediaRequest, - codecInfos: [ - MediaCodecHelper.H264.getCodecInfo({ - sizeHint: mediaRequest.sizeHint, - }), - ], + codecInfos, }; - mediaRequest.handleMaxFs = ({maxFs}) => { - mediaRequest.preferredMaxFs = maxFs; + this.clientRequests[newId] = storedRequest; + + const handleMaxFs = ({maxFs}: {maxFs: number}) => { + storedRequest.preferredMaxFs = maxFs; this.debouncedSourceUpdateListener(); }; - mediaRequest.handleSizeHint = (sizeHint) => { - mediaRequest.sizeHint = sizeHint; + const handleSizeHint = (sizeHint: MediaRequest['sizeHint']) => { + storedRequest.sizeHint = sizeHint; this.debouncedSourceUpdateListener(); }; - mediaRequest.receiveSlots.forEach((rs) => { + storedRequest.handleMaxFs = handleMaxFs; + storedRequest.handleSizeHint = handleSizeHint; + mediaRequest.handleMaxFs = handleMaxFs; + mediaRequest.handleSizeHint = handleSizeHint; + + storedRequest.receiveSlots.forEach((rs) => { rs.on(ReceiveSlotEvents.SourceUpdate, this.sourceUpdateListener); - rs.on(ReceiveSlotEvents.MaxFsUpdate, mediaRequest.handleMaxFs); - rs.on(ReceiveSlotEvents.SizeHintUpdate, mediaRequest.handleSizeHint); + rs.on(ReceiveSlotEvents.MaxFsUpdate, handleMaxFs); + rs.on(ReceiveSlotEvents.SizeHintUpdate, handleSizeHint); }); if (commit) { @@ -368,6 +389,7 @@ export default class MediaRequestManager { this.clientRequests = {}; this.numTotalSources = 0; this.numLiveSources = 0; + this.previousStreamRequests = []; } public setNumCurrentSources(numTotalSources: number, numLiveSources: number) { diff --git a/packages/@webex/plugin-meetings/test/unit/spec/multistream/mediaRequestManager.ts b/packages/@webex/plugin-meetings/test/unit/spec/multistream/mediaRequestManager.ts index 17d59677a65..40523f105d3 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/multistream/mediaRequestManager.ts +++ b/packages/@webex/plugin-meetings/test/unit/spec/multistream/mediaRequestManager.ts @@ -1,13 +1,35 @@ import 'jsdom-global/register'; import MediaRequestManager from '@webex/plugin-meetings/src/multistream/mediaRequestManager'; import {ReceiveSlot} from '@webex/plugin-meetings/src/multistream/receiveSlot'; +import type {SizeHint} from '@webex/plugin-meetings/src/multistream/types'; import sinon from 'sinon'; import {assert} from '@webex/test-helper-chai'; -import {getMaxFs} from '@webex/plugin-meetings/src/multistream/remoteMedia'; +import MediaCodecHelper from '@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper'; import FakeTimers from '@sinonjs/fake-timers'; import * as InternalMediaCoreModule from '@webex/internal-media-core'; import { expect } from 'chai'; +/** Maps a target H264 maxFs to a size hint that yields that maxFs via MediaCodecHelper.H264. */ +const sizeHintForMaxFs = (maxFs: number): SizeHint => { + if (maxFs <= 60) { + return {resolution: 'thumbnail'}; + } + if (maxFs <= 240) { + return {resolution: 'very small'}; + } + if (maxFs <= 920) { + return {resolution: 'small'}; + } + if (maxFs <= 2040) { + return {height: 500}; + } + if (maxFs <= 3600) { + return {resolution: 'medium'}; + } + + return {resolution: 'large'}; +}; + type ExpectedActiveSpeaker = { policy: 'active-speaker'; maxPayloadBitsPerSecond?: number; @@ -78,6 +100,7 @@ describe('MediaRequestManager', () => { id: `fake receive slot ${index}`, on: sinon.stub(), off: sinon.stub(), + setSizeHint: sinon.stub(), sourceState: 'live', wcmeReceiveSlot: fakeWcmeSlots[index], } as unknown as ReceiveSlot) @@ -104,10 +127,7 @@ describe('MediaRequestManager', () => { namedMediaGroups, }, receiveSlots, - codecInfo: { - codec: 'h264', - maxFs: maxFs, - }, + sizeHint: sizeHintForMaxFs(maxFs), }, commit ); @@ -121,10 +141,7 @@ describe('MediaRequestManager', () => { csi, }, receiveSlots: [receiveSlot], - codecInfo: { - codec: 'h264', - maxFs: maxFs, - }, + sizeHint: sizeHintForMaxFs(maxFs), }, commit ); @@ -160,7 +177,7 @@ describe('MediaRequestManager', () => { sinon.match({ payloadType: 0x80, h264: sinon.match({ - maxMbps: expectedRequest.maxMbps, + maxMbps: MAX_MBPS_1080p, maxFs: expectedRequest.maxFs, }), }), @@ -181,7 +198,7 @@ describe('MediaRequestManager', () => { sinon.match({ payloadType: 0x80, h264: sinon.match({ - maxMbps: expectedRequest.maxMbps, + maxMbps: MAX_MBPS_1080p, maxFs: expectedRequest.maxFs, }), }), @@ -218,17 +235,11 @@ describe('MediaRequestManager', () => { preferLiveVideo: false, }, receiveSlots: [fakeReceiveSlots[0], fakeReceiveSlots[1], fakeReceiveSlots[2]], - codecInfo: { - codec: 'h264', - maxFs: MAX_FS_360p, - maxFps: MAX_FPS, - }, + sizeHint: sizeHintForMaxFs(MAX_FS_360p), }, false ); - - mediaRequestManager.addRequest( { policyInfo: { @@ -236,12 +247,7 @@ describe('MediaRequestManager', () => { csi: 123, }, receiveSlots: [fakeReceiveSlots[3]], - codecInfo: { - codec: 'h264', - maxFs: MAX_FS_720p, - maxFps: MAX_FPS, - maxMbps: MAX_MBPS_720p, - }, + sizeHint: sizeHintForMaxFs(MAX_FS_720p), }, false ); @@ -254,12 +260,7 @@ describe('MediaRequestManager', () => { csi: 123, }, receiveSlots: [fakeReceiveSlots[4]], - codecInfo: { - codec: 'h264', - maxFs: MAX_FS_1080p, - maxFps: MAX_FPS, - maxMbps: MAX_MBPS_1080p, - }, + sizeHint: sizeHintForMaxFs(MAX_FS_1080p), }, true ); @@ -283,7 +284,7 @@ describe('MediaRequestManager', () => { h264: sinon.match({ maxFs: MAX_FS_360p, maxFps: MAX_FPS, - maxMbps: MAX_MBPS_360p, + maxMbps: MAX_MBPS_1080p, }), }), ], @@ -301,7 +302,7 @@ describe('MediaRequestManager', () => { h264: sinon.match({ maxFs: MAX_FS_720p, maxFps: MAX_FPS, - maxMbps: MAX_MBPS_720p, + maxMbps: MAX_MBPS_1080p, }), }), ], @@ -400,24 +401,20 @@ describe('MediaRequestManager', () => { ]); }); - it('removes the events maxFsUpdate and sourceUpdate when cancelRequest() is called', async () => { - + it('removes sourceUpdate, maxFsUpdate, and sizeHintUpdate when cancelRequest() is called', () => { const requestId = addActiveSpeakerRequest(255, [fakeReceiveSlots[2], fakeReceiveSlots[3]], MAX_FS_720p); mediaRequestManager.cancelRequest(requestId, true); - const sourceUpdateHandler = fakeReceiveSlots[2].off.getCall(0); - - const maxFsHandlerCall = fakeReceiveSlots[2].off.getCall(1); + const offCalls = fakeReceiveSlots[2].off.getCalls(); + const offFor = (event: string) => offCalls.find((c) => c.args[0] === event); - const maxFsEventName = maxFsHandlerCall.args[0]; - const sourceUpdateEventName = sourceUpdateHandler.args[0]; + ['sourceUpdate', 'maxFsUpdate', 'sizeHintUpdate'].forEach((event) => { + const call = offFor(event); - expect(sourceUpdateHandler.args[1]).to.be.a('function'); - expect(maxFsHandlerCall.args[1]).to.be.a('function'); - - assert.equal(maxFsEventName, 'maxFsUpdate') - assert.equal(sourceUpdateEventName, 'sourceUpdate') + assert.isDefined(call, `expected off() for ${event}`); + expect(call.args[1]).to.be.a('function'); + }); }); it('cancels the requests correctly when cancelRequest() is called with commit=true', () => { @@ -830,7 +827,7 @@ describe('MediaRequestManager', () => { sendMediaRequestsCallback.resetHistory(); // request 4 "large" 1080p streams, which should degrade to 720p if live - addActiveSpeakerRequest(255, fakeReceiveSlots.slice(0, 4), getMaxFs('large'), true); + addActiveSpeakerRequest(255, fakeReceiveSlots.slice(0, 4), MediaCodecHelper.H264.getMaxFs('large'), true); // check that resulting requests are 4 "large" 1080p streams checkMediaRequestsSent([ @@ -839,7 +836,7 @@ describe('MediaRequestManager', () => { priority: 255, receiveSlots: fakeWcmeSlots.slice(0, 4), maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: getMaxFs('large'), + maxFs: MediaCodecHelper.H264.getMaxFs('large'), maxMbps: MAX_MBPS_1080p, }, ]); @@ -851,13 +848,13 @@ describe('MediaRequestManager', () => { sendMediaRequestsCallback.resetHistory(); // request 3 "large" 1080p streams - addActiveSpeakerRequest(255, fakeReceiveSlots.slice(0, 3), getMaxFs('large'), false); + addActiveSpeakerRequest(255, fakeReceiveSlots.slice(0, 3), MediaCodecHelper.H264.getMaxFs('large'), false); // request additional "large" 1080p stream to exceed max macroblocks limit const additionalRequestId = addReceiverSelectedRequest( 123, fakeReceiveSlots[3], - getMaxFs('large'), + MediaCodecHelper.H264.getMaxFs('large'), true ); @@ -868,7 +865,7 @@ describe('MediaRequestManager', () => { priority: 255, receiveSlots: fakeWcmeSlots.slice(0, 3), maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_720p, - maxFs: getMaxFs('medium'), + maxFs: MediaCodecHelper.H264.getMaxFs('medium'), maxMbps: MAX_MBPS_720p, }, { @@ -876,7 +873,7 @@ describe('MediaRequestManager', () => { csi: 123, receiveSlot: fakeWcmeSlots[3], maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_720p, - maxFs: getMaxFs('medium'), + maxFs: MediaCodecHelper.H264.getMaxFs('medium'), maxMbps: MAX_MBPS_720p, }, ]); @@ -891,7 +888,7 @@ describe('MediaRequestManager', () => { priority: 255, receiveSlots: fakeWcmeSlots.slice(0, 3), maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: getMaxFs('large'), + maxFs: MediaCodecHelper.H264.getMaxFs('large'), maxMbps: MAX_MBPS_1080p, }, ]); @@ -903,7 +900,7 @@ describe('MediaRequestManager', () => { sendMediaRequestsCallback.resetHistory(); // request 10 "large" 1080p streams - addActiveSpeakerRequest(255, fakeReceiveSlots.slice(0, 10), getMaxFs('large'), true); + addActiveSpeakerRequest(255, fakeReceiveSlots.slice(0, 10), MediaCodecHelper.H264.getMaxFs('large'), true); // check that resulting requests are 10 540p streams checkMediaRequestsSent([ @@ -924,8 +921,8 @@ describe('MediaRequestManager', () => { sendMediaRequestsCallback.resetHistory(); // request 5 "large" 1080p streams and 5 "small" 360p streams - addActiveSpeakerRequest(255, fakeReceiveSlots.slice(0, 5), getMaxFs('large'), false); - addActiveSpeakerRequest(254, fakeReceiveSlots.slice(5, 10), getMaxFs('small'), true); + addActiveSpeakerRequest(255, fakeReceiveSlots.slice(0, 5), MediaCodecHelper.H264.getMaxFs('large'), false); + addActiveSpeakerRequest(254, fakeReceiveSlots.slice(5, 10), MediaCodecHelper.H264.getMaxFs('small'), true); // check that resulting requests are 5 "medium" 720p streams and 5 "small" 360p streams checkMediaRequestsSent([ @@ -934,7 +931,7 @@ describe('MediaRequestManager', () => { priority: 255, receiveSlots: fakeWcmeSlots.slice(0, 5), maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_720p, - maxFs: getMaxFs('medium'), + maxFs: MediaCodecHelper.H264.getMaxFs('medium'), maxMbps: MAX_MBPS_720p, }, { @@ -942,7 +939,7 @@ describe('MediaRequestManager', () => { priority: 254, receiveSlots: fakeWcmeSlots.slice(5, 10), maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_360p, - maxFs: getMaxFs('small'), + maxFs: MediaCodecHelper.H264.getMaxFs('small'), maxMbps: MAX_MBPS_360p, }, ]); @@ -952,7 +949,7 @@ describe('MediaRequestManager', () => { sendMediaRequestsCallback.resetHistory(); const clock = FakeTimers.install({now: Date.now()}); - addActiveSpeakerRequest(255, fakeReceiveSlots.slice(0, 10), getMaxFs('large'), true); + addActiveSpeakerRequest(255, fakeReceiveSlots.slice(0, 10), MediaCodecHelper.H264.getMaxFs('large'), true); sendMediaRequestsCallback.resetHistory(); @@ -980,7 +977,7 @@ describe('MediaRequestManager', () => { receiveSlots: fakeWcmeSlots.slice(0, 10), maxFs: preferredFrameSize, maxPayloadBitsPerSecond: 99000, - maxMbps: 3000, + maxMbps: MAX_MBPS_1080p, }, ]); clock.uninstall() @@ -1017,7 +1014,6 @@ describe('MediaRequestManager', () => { csi: 123, }, receiveSlots: [fakeReceiveSlots[0]], - codecInfo: undefined, }, false ); @@ -1049,12 +1045,7 @@ describe('MediaRequestManager', () => { csi: 123, }, receiveSlots: [fakeReceiveSlots[0]], - codecInfo: { - codec: 'h264', - maxFs: MAX_FS_1080p, - maxFps: MAX_FPS, - maxMbps: MAX_MBPS_1080p, - }, + sizeHint: sizeHintForMaxFs(MAX_FS_1080p), }, false ); @@ -1091,14 +1082,7 @@ describe('MediaRequestManager', () => { csi: 123, }, receiveSlots: [fakeReceiveSlots[0]], - codecInfo: { - codec: 'h264', - maxFs: MAX_FS_1080p, - maxFps: MAX_FPS, - // random value to pass in, to show that the output (below) is calculated - // from the maxFs and maxFps values only: - maxMbps: 123, - }, + sizeHint: sizeHintForMaxFs(MAX_FS_1080p), }, false ); diff --git a/packages/@webex/plugin-meetings/test/unit/spec/multistream/receiveSlot.ts b/packages/@webex/plugin-meetings/test/unit/spec/multistream/receiveSlot.ts index 860173568d6..d84bc141a42 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/multistream/receiveSlot.ts +++ b/packages/@webex/plugin-meetings/test/unit/spec/multistream/receiveSlot.ts @@ -4,6 +4,7 @@ import EventEmitter from 'events'; import {MediaType, ReceiveSlotEvents as WcmeReceiveSlotEvents} from '@webex/internal-media-core'; import {ReceiveSlot, ReceiveSlotEvents} from '@webex/plugin-meetings/src/multistream/receiveSlot'; +import Metrics from '@webex/plugin-meetings/src/metrics'; import sinon from 'sinon'; import {assert} from '@webex/test-helper-chai'; @@ -144,7 +145,12 @@ describe('ReceiveSlot', () => { }); describe('setMaxFs()', () => { + afterEach(() => { + sinon.restore(); + }); + it('emits the correct event', () => { + sinon.stub(Metrics, 'sendBehavioralMetric'); sinon.stub(receiveSlot, 'emit'); receiveSlot.setMaxFs(100); @@ -152,13 +158,36 @@ describe('ReceiveSlot', () => { receiveSlot.emit, { file: 'meeting/receiveSlot', - function: 'findMemberId', + function: 'setMaxFs', }, ReceiveSlotEvents.MaxFsUpdate, { maxFs: 100, } ); - }) + }); + }); + + describe('setSizeHint()', () => { + afterEach(() => { + sinon.restore(); + }); + + it('emits SizeHintUpdate with the given hint', () => { + sinon.stub(receiveSlot, 'emit'); + const hint = {width: 640, height: 360}; + + receiveSlot.setSizeHint(hint); + + assert.calledOnceWithExactly( + receiveSlot.emit, + { + file: 'meeting/receiveSlot', + function: 'setSizeHint', + }, + ReceiveSlotEvents.SizeHintUpdate, + hint + ); + }); }); }); diff --git a/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMedia.ts b/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMedia.ts index 84af9235087..d23672157ef 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMedia.ts +++ b/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMedia.ts @@ -6,6 +6,7 @@ import {MediaType} from '@webex/internal-media-core'; import {RemoteMedia, RemoteMediaEvents} from '@webex/plugin-meetings/src/multistream/remoteMedia'; import {RemoteVideoResolution} from '@webex/plugin-meetings/src/multistream/types'; import {ReceiveSlotEvents} from '@webex/plugin-meetings/src/multistream/receiveSlot'; +import Metrics from '@webex/plugin-meetings/src/metrics'; import sinon from 'sinon'; import {assert} from '@webex/test-helper-chai'; import {forEach} from 'lodash'; @@ -25,6 +26,7 @@ describe('RemoteMedia', () => { fakeReceiveSlot.sourceState = 'avatar'; fakeReceiveSlot.stream = fakeStream; fakeReceiveSlot.setMaxFs = sinon.stub(); + fakeReceiveSlot.setSizeHint = sinon.stub(); fakeMediaRequestManager = { addRequest: sinon.stub(), @@ -83,9 +85,8 @@ describe('RemoteMedia', () => { csi, }), receiveSlots: [fakeReceiveSlot], - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 3600, + sizeHint: sinon.match({ + resolution: 'medium', }), }), true @@ -105,9 +106,8 @@ describe('RemoteMedia', () => { csi: csi2, }), receiveSlots: [fakeReceiveSlot], - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 3600, + sizeHint: sinon.match({ + resolution: 'medium', }), }), false @@ -139,9 +139,8 @@ describe('RemoteMedia', () => { csi: 5678, }), receiveSlots: [fakeReceiveSlot], - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 3600, + sizeHint: sinon.match({ + resolution: 'medium', }), }), false @@ -243,7 +242,7 @@ describe('RemoteMedia', () => { it(`skip updating the max fs when applied ${width}:${height}`, () => { remoteMedia.setSizeHint(width, height); - assert.notCalled(fakeReceiveSlot.setMaxFs); + assert.notCalled(fakeReceiveSlot.setSizeHint); }); } ); @@ -269,13 +268,28 @@ describe('RemoteMedia', () => { it(`sets the max fs to ${fs} correctly when height is ${height}`, () => { remoteMedia.setSizeHint(100, height); - assert.calledOnceWithExactly(fakeReceiveSlot.setMaxFs, fs); + assert.calledOnceWithExactly( + fakeReceiveSlot.setSizeHint, + sinon.match({ + resolution: 'medium', + width: 100, + height, + }) + ); }); } ); }); describe('getEffectiveMaxFs()', () => { + beforeEach(() => { + sinon.stub(Metrics, 'sendBehavioralMetric'); + }); + + afterEach(() => { + Metrics.sendBehavioralMetric.restore(); + }); + it('returns maxFrameSize when it is greater than 0', () => { remoteMedia.setSizeHint(960, 540); diff --git a/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMediaGroup.ts b/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMediaGroup.ts index 4eb3bc71ed5..d0d964a3bf8 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMediaGroup.ts +++ b/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMediaGroup.ts @@ -93,9 +93,8 @@ describe('RemoteMediaGroup', () => { priority: 211, }), receiveSlots: fakeReceiveSlots, - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 3600, + sizeHint: sinon.match({ + resolution: 'medium', }), }), true @@ -126,9 +125,8 @@ describe('RemoteMediaGroup', () => { preferLiveVideo: true }), receiveSlots: fakeReceiveSlots, - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 3600, + sizeHint: sinon.match({ + resolution: 'medium', }), }), false, @@ -174,7 +172,6 @@ describe('RemoteMediaGroup', () => { namedMediaGroups: sinon.match([{type: 1, value: 24}]), }), receiveSlots: fakeNamedMediaSlots, - codecInfo: undefined, }), false, ); @@ -215,7 +212,6 @@ describe('RemoteMediaGroup', () => { nameMediaGroups: undefined, }), receiveSlots: fakeNamedMediaSlots, - codecInfo: undefined, }), true, ); @@ -271,9 +267,8 @@ describe('RemoteMediaGroup', () => { priority: 255, }), receiveSlots: expectedActiveSpeakerReceiveSlots, - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 3600, + sizeHint: sinon.match({ + resolution: 'medium', }), }) ); @@ -285,9 +280,8 @@ describe('RemoteMediaGroup', () => { csi: CSI, }), receiveSlots: expectedReceiverSelectedSlots, - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 3600, + sizeHint: sinon.match({ + resolution: 'medium', }), }) ); @@ -342,9 +336,8 @@ describe('RemoteMediaGroup', () => { csi: 1234, }), receiveSlots: expectedReceiverSelectedSlots, - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 3600, + sizeHint: sinon.match({ + resolution: 'medium', }), }) ); @@ -495,9 +488,8 @@ describe('RemoteMediaGroup', () => { priority: 255, }), receiveSlots: expectedActiveSpeakerReceiveSlots, - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 3600, + sizeHint: sinon.match({ + resolution: 'medium', }), }) ); @@ -509,9 +501,8 @@ describe('RemoteMediaGroup', () => { csi: CSI, }), receiveSlots: expectedReceiverSelectedSlots, - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 3600, + sizeHint: sinon.match({ + resolution: 'medium', }), }) ); @@ -549,9 +540,8 @@ describe('RemoteMediaGroup', () => { priority: 255, }), receiveSlots: expectedActiveSpeakerReceiveSlots2, - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 3600, + sizeHint: sinon.match({ + resolution: 'medium', }), }) ); @@ -563,9 +553,8 @@ describe('RemoteMediaGroup', () => { csi: CSI2, }), receiveSlots: expectedReceiverSelectedSlots2, - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 3600, + sizeHint: sinon.match({ + resolution: 'medium', }), }) ); @@ -597,9 +586,8 @@ describe('RemoteMediaGroup', () => { policy: 'active-speaker', priority: 255, }), - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 3600, + sizeHint: sinon.match({ + resolution: 'medium', }), }) ); @@ -658,9 +646,8 @@ describe('RemoteMediaGroup', () => { csi: 2345, }), receiveSlots: [fakeReceiveSlots[PINNED_INDEX]], - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 3600, + sizeHint: sinon.match({ + resolution: 'medium', }), }) ); diff --git a/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMediaManager.ts b/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMediaManager.ts index a6526e96c8b..1e1d2b01421 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMediaManager.ts +++ b/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMediaManager.ts @@ -290,7 +290,6 @@ describe('RemoteMediaManager', () => { priority: 255, }), receiveSlots: Array(5).fill(fakeAudioSlot), - codecInfo: undefined, }) ); }); @@ -349,7 +348,6 @@ describe('RemoteMediaManager', () => { namedMediaGroups: sinon.match([{type: 1, value: 20}]), }), receiveSlots: Array(1).fill(fakeAudioSlot), - codecInfo: undefined, }), false ); @@ -601,7 +599,6 @@ describe('RemoteMediaManager', () => { priority: 255, }), receiveSlots: Array(NUM_STREAMS).fill(fakeScreenShareAudioSlot), - codecInfo: undefined, }) ); }); @@ -1523,9 +1520,8 @@ describe('RemoteMediaManager', () => { priority: 255, }), receiveSlots: Array(6).fill(fakeVideoSlot), - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 60, + sizeHint: sinon.match({ + resolution: 'thumbnail', }), }) ); @@ -1537,9 +1533,8 @@ describe('RemoteMediaManager', () => { csi: 11111, }), receiveSlots: Array(1).fill(fakeVideoSlot), - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 3600, + sizeHint: sinon.match({ + resolution: 'medium', }), }) ); @@ -1551,9 +1546,8 @@ describe('RemoteMediaManager', () => { csi: 22222, }), receiveSlots: Array(1).fill(fakeVideoSlot), - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 3600, + sizeHint: sinon.match({ + resolution: 'medium', }), }) ); @@ -1596,9 +1590,8 @@ describe('RemoteMediaManager', () => { priority: 255, }), receiveSlots: Array(1).fill(fakeVideoSlot), - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 8192, + sizeHint: sinon.match({ + resolution: 'large', }), }) ); @@ -1610,9 +1603,8 @@ describe('RemoteMediaManager', () => { priority: 254, }), receiveSlots: Array(5).fill(fakeVideoSlot), - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 240, + sizeHint: sinon.match({ + resolution: 'very small', }), }) ); @@ -1733,9 +1725,8 @@ describe('RemoteMediaManager', () => { priority: 255, }), receiveSlots: [fakeScreenShareVideoSlot], - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 3600, + sizeHint: sinon.match({ + resolution: 'medium', }), }) ); @@ -2049,9 +2040,8 @@ describe('RemoteMediaManager', () => { csi: 1001, }), receiveSlots: Array(1).fill(fakeVideoSlot), - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 3600, + sizeHint: sinon.match({ + resolution: 'medium', }), }) ); @@ -2072,9 +2062,8 @@ describe('RemoteMediaManager', () => { csi: 1002, }), receiveSlots: Array(1).fill(fakeVideoSlot), - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 3600, + sizeHint: sinon.match({ + resolution: 'medium', }), }) ); @@ -2099,9 +2088,8 @@ describe('RemoteMediaManager', () => { csi: 2001, }), receiveSlots: Array(1).fill(fakeVideoSlot), - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 3600, + sizeHint: sinon.match({ + resolution: 'medium', }), }) ); @@ -2164,9 +2152,8 @@ describe('RemoteMediaManager', () => { csi: 54321, }), receiveSlots: Array(1).fill(fakeVideoSlot), - codecInfo: sinon.match({ - codec: 'h264', - maxFs: 8192, + sizeHint: sinon.match({ + resolution: 'best', }), }) ); From 32d4f9cd6a9bf42e90c5188fc1d1ee353cc09dbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edmond=20Vuji=C4=87i?= <67634227+edvujic@users.noreply.github.com> Date: Wed, 25 Mar 2026 10:13:39 +0100 Subject: [PATCH 11/28] fix(plugin-meetings): remove JMP deduplication logic (#4773) Co-authored-by: evujici --- .../src/multistream/mediaRequestManager.ts | 56 +----------- .../src/reconnection-manager/index.ts | 1 - .../spec/multistream/mediaRequestManager.ts | 87 +------------------ .../unit/spec/reconnection-manager/index.js | 12 +-- 4 files changed, 9 insertions(+), 147 deletions(-) diff --git a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts index 7004def8749..fb16c7643bd 100644 --- a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts +++ b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts @@ -10,7 +10,7 @@ import { RecommendedOpusBitrates, NamedMediaGroup, } from '@webex/internal-media-core'; -import {cloneDeepWith, debounce, isEmpty} from 'lodash'; +import {cloneDeepWith, debounce} from 'lodash'; import LoggerProxy from '../common/logs/logger-proxy'; @@ -94,8 +94,6 @@ export class MediaRequestManager { private debouncedSourceUpdateListener: () => void; - private previousStreamRequests: Array = []; - private trimRequestsToNumOfSources: boolean; private numTotalSources: number; private numLiveSources: number; @@ -161,36 +159,6 @@ export class MediaRequestManager { } } - /** - * Returns true if two stream requests are the same, false otherwise. - * - * @param {StreamRequest} streamRequestA - Stream request A for comparison. - * @param {StreamRequest} streamRequestB - Stream request B for comparison. - * @returns {boolean} - Whether they are equal. - */ - // eslint-disable-next-line class-methods-use-this - public isEqual(streamRequestA: StreamRequest, streamRequestB: StreamRequest) { - return ( - JSON.stringify(streamRequestA._toJmpStreamRequest()) === - JSON.stringify(streamRequestB._toJmpStreamRequest()) - ); - } - - /** - * Compares new stream requests to previous ones and determines - * if they are the same. - * - * @param {StreamRequest[]} newRequests - Array with new requests. - * @returns {boolean} - True if they are equal, false otherwise. - */ - private checkIsNewRequestsEqualToPrev(newRequests: StreamRequest[]) { - return ( - !isEmpty(this.previousStreamRequests) && - this.previousStreamRequests.length === newRequests.length && - this.previousStreamRequests.every((req, idx) => this.isEqual(req, newRequests[idx])) - ); - } - /** * Returns the maxPayloadBitsPerSecond per Stream * @@ -230,15 +198,6 @@ export class MediaRequestManager { return (mediaRequest.codecInfo.maxFs * maxFps) / 100; } - /** - * Clears the previous stream requests. - * - * @returns {void} - */ - public clearPreviousRequests(): void { - this.previousStreamRequests = []; - } - /** Modifies the passed in clientRequests and makes sure that in total they don't ask * for more streams than there are available. * @@ -372,17 +331,8 @@ export class MediaRequestManager { } }); - //! IMPORTANT: this is only a temporary fix. This will soon be done in the jmp layer (@webex/json-multistream) - // https://jira-eng-gpk2.cisco.com/jira/browse/WEBEX-326713 - if (!this.checkIsNewRequestsEqualToPrev(streamRequests)) { - this.sendMediaRequestsCallback(streamRequests); - this.previousStreamRequests = streamRequests; - LoggerProxy.logger.info(`multistream:sendRequests --> media requests sent. `); - } else { - LoggerProxy.logger.info( - `multistream:sendRequests --> detected duplicate WCME requests, skipping them... ` - ); - } + this.sendMediaRequestsCallback(streamRequests); + LoggerProxy.logger.info(`multistream:sendRequests --> media requests sent. `); } public addRequest(mediaRequest: MediaRequest, commit = true): MediaRequestId { diff --git a/packages/@webex/plugin-meetings/src/reconnection-manager/index.ts b/packages/@webex/plugin-meetings/src/reconnection-manager/index.ts index fd8407336a0..eabafe0991f 100644 --- a/packages/@webex/plugin-meetings/src/reconnection-manager/index.ts +++ b/packages/@webex/plugin-meetings/src/reconnection-manager/index.ts @@ -609,7 +609,6 @@ export default class ReconnectionManager { if (this.meeting.isMultistream) { Object.values(this.meeting.mediaRequestManagers).forEach( (mediaRequestManager: MediaRequestManager) => { - mediaRequestManager.clearPreviousRequests(); mediaRequestManager.commit(); } ); diff --git a/packages/@webex/plugin-meetings/test/unit/spec/multistream/mediaRequestManager.ts b/packages/@webex/plugin-meetings/test/unit/spec/multistream/mediaRequestManager.ts index ede46c14585..637df1cc34a 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/multistream/mediaRequestManager.ts +++ b/packages/@webex/plugin-meetings/test/unit/spec/multistream/mediaRequestManager.ts @@ -666,8 +666,8 @@ describe('MediaRequestManager', () => { ]); }); - it('avoids sending duplicate requests and clears all the requests on reset()', () => { - // send some requests and commit them one by one + it('clears all the requests on reset()', () => { + // send some requests and commit them addReceiverSelectedRequest(1500, fakeReceiveSlots[0], MAX_FS_1080p, false); addReceiverSelectedRequest(1501, fakeReceiveSlots[1], MAX_FS_1080p, false); addActiveSpeakerRequest( @@ -722,95 +722,12 @@ describe('MediaRequestManager', () => { }, ]); - // check that when calling commit() - // all requests are not re-sent again (avoid duplicate requests) - mediaRequestManager.commit(); - - assert.notCalled(sendMediaRequestsCallback); - - // now reset everything - mediaRequestManager.reset(); - - // calling commit now should not cause any requests to be sent out - mediaRequestManager.commit(); - checkMediaRequestsSent([]); - }); - - it('makes sure to call requests correctly after reset was called and another request was added', () => { - addReceiverSelectedRequest(1500, fakeReceiveSlots[0], MAX_FS_1080p, false); - - assert.notCalled(sendMediaRequestsCallback); - - mediaRequestManager.commit(); - checkMediaRequestsSent([ - { - policy: 'receiver-selected', - csi: 1500, - receiveSlot: fakeWcmeSlots[0], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, - }, - ]); - // now reset everything mediaRequestManager.reset(); // calling commit now should not cause any requests to be sent out mediaRequestManager.commit(); checkMediaRequestsSent([]); - - //add new request - addReceiverSelectedRequest(1501, fakeReceiveSlots[1], MAX_FS_1080p, false); - - // commit - mediaRequestManager.commit(); - - // check the new request was sent - checkMediaRequestsSent([ - { - policy: 'receiver-selected', - csi: 1501, - receiveSlot: fakeWcmeSlots[1], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, - }, - ]); - }); - - it('can send same media request after previous requests have been cleared', () => { - // add a request and commit - addReceiverSelectedRequest(1500, fakeReceiveSlots[0], MAX_FS_1080p, false); - mediaRequestManager.commit(); - checkMediaRequestsSent([ - { - policy: 'receiver-selected', - csi: 1500, - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - receiveSlot: fakeWcmeSlots[0], - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, - }, - ]); - - // clear previous requests - mediaRequestManager.clearPreviousRequests(); - - // commit same request - mediaRequestManager.commit(); - - // check the request was sent - checkMediaRequestsSent([ - { - policy: 'receiver-selected', - csi: 1500, - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - receiveSlot: fakeWcmeSlots[0], - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, - }, - ]); }); it('re-sends media requests after degradation preferences are set', () => { diff --git a/packages/@webex/plugin-meetings/test/unit/spec/reconnection-manager/index.js b/packages/@webex/plugin-meetings/test/unit/spec/reconnection-manager/index.js index a831fb8b71c..e72b16b4464 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/reconnection-manager/index.js +++ b/packages/@webex/plugin-meetings/test/unit/spec/reconnection-manager/index.js @@ -54,8 +54,8 @@ describe('plugin-meetings', () => { webrtcMediaConnection: fakeMediaConnection, }, mediaRequestManagers: { - audio: {commit: sinon.stub(), clearPreviousRequests: sinon.stub()}, - video: {commit: sinon.stub(), clearPreviousRequests: sinon.stub()}, + audio: {commit: sinon.stub()}, + video: {commit: sinon.stub()}, }, roap: { doTurnDiscovery: sinon.stub().resolves({ @@ -179,26 +179,22 @@ describe('plugin-meetings', () => { }); }); - it('does not clear previous requests and re-request media for non-multistream meetings', async () => { + it('does not re-request media for non-multistream meetings', async () => { fakeMeeting.isMultistream = false; const rm = new ReconnectionManager(fakeMeeting); await rm.reconnect(); - assert.notCalled(fakeMeeting.mediaRequestManagers.audio.clearPreviousRequests); - assert.notCalled(fakeMeeting.mediaRequestManagers.video.clearPreviousRequests); assert.notCalled(fakeMeeting.mediaRequestManagers.audio.commit); assert.notCalled(fakeMeeting.mediaRequestManagers.video.commit); }); - it('does clear previous requests and re-request media for multistream meetings', async () => { + it('does re-request media for multistream meetings', async () => { fakeMeeting.isMultistream = true; const rm = new ReconnectionManager(fakeMeeting); await rm.reconnect(); - assert.calledOnce(fakeMeeting.mediaRequestManagers.audio.clearPreviousRequests); - assert.calledOnce(fakeMeeting.mediaRequestManagers.video.clearPreviousRequests); assert.calledOnce(fakeMeeting.mediaRequestManagers.audio.commit); assert.calledOnce(fakeMeeting.mediaRequestManagers.video.commit); }); From 20103a6e13e055d19e095a32e11d8fb4cafb9e6d Mon Sep 17 00:00:00 2001 From: Tianhui-Han <133636998+Tianhui-Han@users.noreply.github.com> Date: Wed, 25 Mar 2026 18:40:56 +0800 Subject: [PATCH 12/28] feat(datachannel): add update subchannel subscriptions function (#4749) Co-authored-by: mickelr <121160648+mickelr@users.noreply.github.com> --- packages/@webex/internal-plugin-llm/README.md | 6 +- .../internal-plugin-llm/src/constants.ts | 8 + .../@webex/internal-plugin-llm/src/llm.ts | 70 ++++- .../internal-plugin-llm/src/llm.types.ts | 4 +- .../internal-plugin-llm/test/unit/spec/llm.js | 285 ++++++++++++------ .../internal-plugin-voicea/src/voicea.ts | 71 +++++ .../test/unit/spec/voicea.js | 208 ++++++++++++- packages/@webex/plugin-meetings/package.json | 1 + .../src/interceptors/dataChannelAuthToken.ts | 28 ++ .../plugin-meetings/src/interceptors/utils.ts | 16 + .../plugin-meetings/src/meeting/index.ts | 10 +- .../plugin-meetings/src/meeting/request.ts | 8 +- .../plugin-meetings/src/webinar/index.ts | 5 + .../spec/interceptors/dataChannelAuthToken.ts | 69 +++++ .../test/unit/spec/interceptors/utils.ts | 75 +++++ .../test/unit/spec/meeting/index.js | 12 +- .../test/unit/spec/meeting/request.js | 30 +- .../test/unit/spec/webinar/index.ts | 18 ++ yarn.lock | 8 + 19 files changed, 793 insertions(+), 139 deletions(-) create mode 100644 packages/@webex/plugin-meetings/src/interceptors/utils.ts create mode 100644 packages/@webex/plugin-meetings/test/unit/spec/interceptors/utils.ts diff --git a/packages/@webex/internal-plugin-llm/README.md b/packages/@webex/internal-plugin-llm/README.md index de0a161c117..6ab304f4fb7 100644 --- a/packages/@webex/internal-plugin-llm/README.md +++ b/packages/@webex/internal-plugin-llm/README.md @@ -79,8 +79,8 @@ llm.on(`event:${sessionB}`, (envelope) => { }); // Optional: store/retrieve token by token type -webex.internal.llm.setDatachannelToken(datachannelToken, 'DEFAULT'); -webex.internal.llm.getDatachannelToken('DEFAULT'); +webex.internal.llm.setDatachannelToken(datachannelToken, 'llm-default-session'); +webex.internal.llm.getDatachannelToken('llm-default-session'); // Optional: inject token refresh handler webex.internal.llm.setRefreshHandler(async () => { @@ -88,7 +88,7 @@ webex.internal.llm.setRefreshHandler(async () => { return { body: { datachannelToken: '', - datachannelTokenType: 'DEFAULT', + datachannelTokenType: 'llm-default-session', }, }; }); diff --git a/packages/@webex/internal-plugin-llm/src/constants.ts b/packages/@webex/internal-plugin-llm/src/constants.ts index 45b0c5b43d7..7d267df4392 100644 --- a/packages/@webex/internal-plugin-llm/src/constants.ts +++ b/packages/@webex/internal-plugin-llm/src/constants.ts @@ -4,3 +4,11 @@ export const LLM = 'llm'; export const LLM_DEFAULT_SESSION = 'llm-default-session'; export const DATA_CHANNEL_WITH_JWT_TOKEN = 'data-channel-with-jwt-token'; + +export const SUBSCRIPTION_AWARE_SUBCHANNELS_PARAM = 'subscriptionAwareSubchannels'; + +export const DATA_CHNANEL_TYPE = { + TRANSCRIPTION: 'transcription', +}; + +export const AWARE_DATA_CHANNEL = [DATA_CHNANEL_TYPE.TRANSCRIPTION]; diff --git a/packages/@webex/internal-plugin-llm/src/llm.ts b/packages/@webex/internal-plugin-llm/src/llm.ts index 57ee462324e..0c1ac265e4e 100644 --- a/packages/@webex/internal-plugin-llm/src/llm.ts +++ b/packages/@webex/internal-plugin-llm/src/llm.ts @@ -2,8 +2,14 @@ import Mercury from '@webex/internal-plugin-mercury'; -import {LLM, DATA_CHANNEL_WITH_JWT_TOKEN, LLM_DEFAULT_SESSION} from './constants'; // eslint-disable-next-line no-unused-vars +import { + LLM, + DATA_CHANNEL_WITH_JWT_TOKEN, + AWARE_DATA_CHANNEL, + SUBSCRIPTION_AWARE_SUBCHANNELS_PARAM, + LLM_DEFAULT_SESSION, +} from './constants'; import {ILLMChannel, DataChannelTokenType} from './llm.types'; export const config = { @@ -118,7 +124,7 @@ export default class LLMChannel extends (Mercury as any) implements ILLMChannel datachannelToken?: string, sessionId: string = LLM_DEFAULT_SESSION ): Promise => - this.register(datachannelUrl, datachannelToken, sessionId).then(() => { + this.register(datachannelUrl, datachannelToken, sessionId).then(async () => { if (!locusUrl || !datachannelUrl) return undefined; // Get or create connection data @@ -128,7 +134,13 @@ export default class LLMChannel extends (Mercury as any) implements ILLMChannel sessionData.datachannelToken = datachannelToken; this.connections.set(sessionId, sessionData); - return this.connect(sessionData.webSocketUrl, sessionId); + const isDataChannelTokenEnabled = await this.isDataChannelTokenEnabled(); + + const connectUrl = isDataChannelTokenEnabled + ? LLMChannel.buildUrlWithAwareSubchannels(sessionData.webSocketUrl, AWARE_DATA_CHANNEL) + : sessionData.webSocketUrl; + + return this.connect(connectUrl, sessionId); }); /** @@ -180,7 +192,9 @@ export default class LLMChannel extends (Mercury as any) implements ILLMChannel * @param {DataChannelTokenType} dataChannelTokenType * @returns {string} data channel token */ - public getDatachannelToken = (dataChannelTokenType: DataChannelTokenType): string => { + public getDatachannelToken = ( + dataChannelTokenType: DataChannelTokenType = DataChannelTokenType.Default + ): string => { return this.datachannelTokens[dataChannelTokenType]; }; @@ -192,11 +206,23 @@ export default class LLMChannel extends (Mercury as any) implements ILLMChannel */ public setDatachannelToken = ( datachannelToken: string, - dataChannelTokenType: DataChannelTokenType + dataChannelTokenType: DataChannelTokenType = DataChannelTokenType.Default ): void => { this.datachannelTokens[dataChannelTokenType] = datachannelToken; }; + /** + * Resets all data‑channel tokens to their initial undefined values. + * Used when leaving or disconnecting from a meeting. + * @returns {void} + */ + private resetDatachannelTokens() { + this.datachannelTokens = { + [DataChannelTokenType.Default]: undefined, + [DataChannelTokenType.PracticeSession]: undefined, + }; + } + /** * Set the handler used to refresh the DataChannel token * @@ -219,9 +245,11 @@ export default class LLMChannel extends (Mercury as any) implements ILLMChannel */ public async refreshDataChannelToken() { if (!this.refreshHandler) { - const error = new Error('LLM refreshHandler is not set'); - this.logger.error(`Error refreshing DataChannel token: ${error.message}`); - throw error; + this.logger.warn( + 'llm#refreshDataChannelToken --> LLM refreshHandler is not set, skipping token refresh' + ); + + return null; } try { @@ -229,8 +257,13 @@ export default class LLMChannel extends (Mercury as any) implements ILLMChannel return res; } catch (error: any) { - this.logger.error(`Error refreshing DataChannel token: ${error}`); - throw error; + this.logger.warn( + `llm#refreshDataChannelToken --> DataChannel token refresh failed (likely locus changed or participant left): ${ + error?.message || error + }` + ); + + return null; } } @@ -247,6 +280,7 @@ export default class LLMChannel extends (Mercury as any) implements ILLMChannel this.disconnect(options, sessionId).then(() => { // Clean up sessions data this.connections.delete(sessionId); + this.datachannelTokens[sessionId] = undefined; }); /** @@ -258,6 +292,7 @@ export default class LLMChannel extends (Mercury as any) implements ILLMChannel this.disconnectAll(options).then(() => { // Clean up all connection data this.connections.clear(); + this.resetDatachannelTokens(); }); /** @@ -283,4 +318,19 @@ export default class LLMChannel extends (Mercury as any) implements ILLMChannel // @ts-ignore return this.webex.internal.feature.getFeature('developer', DATA_CHANNEL_WITH_JWT_TOKEN); } + + /** + * Builds a WebSocket URL with the `subscriptionAwareSubchannels` query parameter. + * + * @param {string} baseUrl - The original WebSocket URL. + * @param {string[]} subchannels - List of subchannels to declare as subscription-aware. + * @returns {string} The final URL with updated query parameters. + */ + + public static buildUrlWithAwareSubchannels = (baseUrl: string, subchannels: string[]) => { + const urlObj = new URL(baseUrl); + urlObj.searchParams.set(SUBSCRIPTION_AWARE_SUBCHANNELS_PARAM, subchannels.join(',')); + + return urlObj.toString(); + }; } diff --git a/packages/@webex/internal-plugin-llm/src/llm.types.ts b/packages/@webex/internal-plugin-llm/src/llm.types.ts index a708c2c5b76..0bdb261164c 100644 --- a/packages/@webex/internal-plugin-llm/src/llm.types.ts +++ b/packages/@webex/internal-plugin-llm/src/llm.types.ts @@ -24,8 +24,8 @@ interface ILLMChannel { } export enum DataChannelTokenType { - Default = 'default', - PracticeSession = 'practiceSession', + Default = 'llm-default-session', + PracticeSession = 'llm-practice-session', } // eslint-disable-next-line import/prefer-default-export diff --git a/packages/@webex/internal-plugin-llm/test/unit/spec/llm.js b/packages/@webex/internal-plugin-llm/test/unit/spec/llm.js index 055a1cb19a3..c1948325fb8 100644 --- a/packages/@webex/internal-plugin-llm/test/unit/spec/llm.js +++ b/packages/@webex/internal-plugin-llm/test/unit/spec/llm.js @@ -25,55 +25,77 @@ describe('plugin-llm', () => { }; llmService = webex.internal.llm; - llmService.connect = sinon.stub().callsFake(() => { - // Simulate a successful connection by stubbing getSocket to return connected: true - llmService.getSocket = sinon.stub().returns({connected: true}); - }); + llmService.webSocketUrl = 'wss://example.com/socket'; llmService.disconnect = sinon.stub().resolves(true); llmService.request = sinon.stub().resolves({ headers: {}, body: { binding: 'binding', - webSocketUrl: 'url', + webSocketUrl: 'wss://example.com/socket', }, }); + const sockets = new Map(); + + llmService.connect = sinon.stub().callsFake((url, sessionId) => { + sockets.set(sessionId, {connected: true}); + llmService.getSocket = sinon.stub().callsFake((sid) => sockets.get(sid)); + }); + llmService.connections.set('llm-default-session',{ + webSocketUrl: 'wss://example.com/socket', + }) }); + afterEach(() => sinon.restore()); + describe('#registerAndConnect', () => { it('registers connection', async () => { - llmService.register = sinon.stub().resolves({ - body: { - binding: 'binding', - webSocketUrl: 'url', - }, + llmService.register = sinon.stub().callsFake(async () => { + llmService.binding = 'binding'; + llmService.webSocketUrl = 'wss://example.com/socket'; + return { + body: { + binding: 'binding', + webSocketUrl: 'wss://example.com/socket', + }, + }; }); - assert.equal(llmService.isConnected(), false); - await llmService.registerAndConnect(locusUrl, datachannelUrl); - assert.equal(llmService.isConnected(), true); + + assert.equal(llmService.isConnected('llm-default-session'), false); + await llmService.registerAndConnect(locusUrl, datachannelUrl,undefined); + assert.equal(llmService.isConnected('llm-default-session'), true); }); - it("doesn't registers connection for invalid input", async () => { - llmService.register = sinon.stub().resolves({ - body: { - binding: 'binding', - webSocketUrl: 'url', - }, + it("doesn't register connection for invalid input", async () => { + llmService.register = sinon.stub().callsFake(async () => { + llmService.binding = 'binding'; + llmService.webSocketUrl = 'wss://example.com/socket'; + return { + body: { + binding: 'binding', + webSocketUrl: 'wss://example.com/socket', + }, + }; }); + await llmService.registerAndConnect(); assert.equal(llmService.isConnected(), false); }); it('registers connection with token', async () => { - llmService.register = sinon.stub().resolves({ - body: { - binding: 'binding', - webSocketUrl: 'url', - }, + llmService.register = sinon.stub().callsFake(async () => { + llmService.binding = 'binding'; + llmService.webSocketUrl = 'wss://example.com/socket'; + return { + body: { + binding: 'binding', + webSocketUrl: 'wss://example.com/socket', + }, + }; }); assert.equal(llmService.isConnected(), false); - await llmService.registerAndConnect(locusUrl, datachannelUrl, 'abc123'); + await llmService.registerAndConnect(locusUrl, datachannelUrl,'abc123'); sinon.assert.calledOnceWithExactly( llmService.register, @@ -84,6 +106,72 @@ describe('plugin-llm', () => { assert.equal(llmService.isConnected(), true); }); + + it('connects with subscriptionAwareSubchannels when token enabled', async () => { + llmService.isDataChannelTokenEnabled = sinon.stub().returns(true); + + llmService.register = sinon.stub().callsFake(async () => { + llmService.binding = 'binding'; + llmService.webSocketUrl = 'wss://example.com/socket'; + return { + body: { + binding: 'binding', + webSocketUrl: 'wss://example.com/socket', + }, + }; + }); + + const buildSpy = sinon.spy(LLMService, 'buildUrlWithAwareSubchannels'); + + await llmService.registerAndConnect(locusUrl, datachannelUrl,'abc123'); + + sinon.assert.calledOnce(buildSpy); + sinon.assert.calledOnce(llmService.connect); + + const calledUrl = llmService.connect.getCall(0).args[0]; + assert.include(calledUrl, 'subscriptionAwareSubchannels='); + }); + + it('connects without subscriptionAwareSubchannels when token disabled', async () => { + llmService.isDataChannelTokenEnabled = sinon.stub().returns(false); + + llmService.register = sinon.stub().callsFake(async () => { + llmService.binding = 'binding'; + llmService.webSocketUrl = 'wss://example.com/socket'; + return { + body: { + binding: 'binding', + webSocketUrl: 'wss://example.com/socket', + }, + }; + }); + + const buildSpy = sinon.spy(LLMService, 'buildUrlWithAwareSubchannels'); + + await llmService.registerAndConnect(locusUrl, datachannelUrl); + + sinon.assert.notCalled(buildSpy); + sinon.assert.calledOnce(llmService.connect); + + const calledUrl = llmService.connect.getCall(0).args[0]; + assert.equal(calledUrl, llmService.webSocketUrl); + }); + + it('connects without subscriptionAwareSubchannels when token enabled BUT token missing', async () => { + llmService.isDataChannelTokenEnabled = sinon.stub().resolves(true); + + const buildSpy = sinon.spy(LLMService, 'buildUrlWithAwareSubchannels'); + + await llmService.registerAndConnect(locusUrl, datachannelUrl, undefined); + + sinon.assert.calledOnce(buildSpy); + sinon.assert.calledOnce(llmService.connect); + + const calledUrl = llmService.connect.getCall(0).args[0]; + assert.include(calledUrl, 'subscriptionAwareSubchannels='); + + buildSpy.restore(); + }); }); describe('#register', () => { @@ -136,15 +224,19 @@ describe('plugin-llm', () => { }); }); - describe('#getLocusUrl', () => { it('gets LocusUrl', async () => { - llmService.register = sinon.stub().resolves({ - body: { - binding: 'binding', - webSocketUrl: 'url', - }, + llmService.register = sinon.stub().callsFake(async () => { + llmService.binding = 'binding'; + llmService.webSocketUrl = 'wss://example.com/socket'; + return { + body: { + binding: 'binding', + webSocketUrl: 'wss://example.com/socket', + }, + }; }); + await llmService.registerAndConnect(locusUrl, datachannelUrl); assert.equal(llmService.getLocusUrl(), locusUrl); }); @@ -152,11 +244,15 @@ describe('plugin-llm', () => { describe('#getDatachannelUrl', () => { it('gets dataChannel Url', async () => { - llmService.register = sinon.stub().resolves({ - body: { - binding: 'binding', - webSocketUrl: 'url', - }, + llmService.register = sinon.stub().callsFake(async () => { + llmService.binding = 'binding'; + llmService.webSocketUrl = 'wss://example.com/socket'; + return { + body: { + binding: 'binding', + webSocketUrl: 'wss://example.com/socket', + }, + }; }); await llmService.registerAndConnect(locusUrl, datachannelUrl); assert.equal(llmService.getDatachannelUrl(), datachannelUrl); @@ -174,41 +270,47 @@ describe('plugin-llm', () => { }); }); - describe('disconnectLLM', () => { + describe('#disconnectLLM', () => { let instance; beforeEach(() => { instance = { disconnect: jest.fn(() => Promise.resolve()), - locusUrl: 'someUrl', - datachannelUrl: 'someUrl', - binding: {}, - webSocketUrl: 'someUrl', - disconnectLLM: function (options) { - return this.disconnect(options).then(() => { - this.locusUrl = undefined; - this.datachannelUrl = undefined; - this.binding = undefined; - this.webSocketUrl = undefined; + connections: new Map([ + ['llm-default-session', { foo: 'bar' }], + ]), + datachannelTokens: { + 'llm-default-session': 'session-token', + }, + + disconnectLLM: function (options, sessionId = 'llm-default-session') { + return this.disconnect(options, sessionId).then(() => { + this.connections.delete(sessionId); + this.datachannelTokens[sessionId] = undefined; }); - } + }, }; }); - it('should call disconnect and clear relevant properties', async () => { - await instance.disconnectLLM({}); + it('calls disconnect and clears session connection + token', async () => { + await instance.disconnectLLM({ code: 3000, reason: 'bye' }); + + expect(instance.disconnect).toHaveBeenCalledWith( + { code: 3000, reason: 'bye' }, + 'llm-default-session' + ); + + expect(instance.connections.has('llm-default-session')).toBe(false); - expect(instance.disconnect).toHaveBeenCalledWith({}); - expect(instance.locusUrl).toBeUndefined(); - expect(instance.datachannelUrl).toBeUndefined(); - expect(instance.binding).toBeUndefined(); - expect(instance.webSocketUrl).toBeUndefined(); + expect(instance.datachannelTokens['llm-default-session']).toBeUndefined(); }); - it('should handle errors from disconnect gracefully', async () => { - instance.disconnect.mockRejectedValue(new Error('Disconnect failed')); + it('propagates disconnect errors', async () => { + instance.disconnect.mockRejectedValue(new Error('disconnect failed')); - await expect(instance.disconnectLLM({})).rejects.toThrow('Disconnect failed'); + await expect( + instance.disconnectLLM({ code: 3000, reason: 'bye' }) + ).rejects.toThrow('disconnect failed'); }); }); @@ -239,53 +341,56 @@ describe('plugin-llm', () => { }); describe('#refreshDataChannelToken', () => { - it('throws if no handler is set', async () => { - try { - await llmService.refreshDataChannelToken(); - assert.fail('Should have thrown'); - } catch (err) { - assert.match(err.message, 'LLM refreshHandler is not set'); - } + it('returns null and logs warn if no handler is set', async () => { + const warnSpy = llmService.logger.warn + + const result = await llmService.refreshDataChannelToken(); + + assert.equal(result, null); + + sinon.assert.calledOnce(warnSpy); + sinon.assert.calledWithMatch( + warnSpy, + sinon.match('LLM refreshHandler is not set') + ); }); it('returns token when handler resolves', async () => { - const mockToken = { body: { datachannelToken: 'newToken' ,isPracticeSession: false} } + const mockToken = { body: { datachannelToken: 'newToken', isPracticeSession: false } }; const handler = sinon.stub().resolves(mockToken); + llmService.setRefreshHandler(handler); const token = await llmService.refreshDataChannelToken(); + assert.equal(token, mockToken); sinon.assert.calledOnce(handler); }); - it('logs and rethrows when handler rejects', async () => { + it('logs warn and returns null when handler rejects', async () => { const handler = sinon.stub().rejects(new Error('throw error')); + llmService.setRefreshHandler(handler); - const loggerSpy = llmService.logger.error; + const warnSpy = llmService.logger.warn - llmService.setRefreshHandler(handler); + const result = await llmService.refreshDataChannelToken(); - try { - await llmService.refreshDataChannelToken(); - assert.fail('Should have thrown'); - } catch (err) { - assert.match(err.message, /throw error/); - } + assert.equal(result, null); - sinon.assert.calledOnce(loggerSpy); + sinon.assert.calledOnce(warnSpy); sinon.assert.calledWithMatch( - loggerSpy, - sinon.match("Error refreshing DataChannel token: Error: throw error") + warnSpy, + sinon.match('DataChannel token refresh failed'), ); }); }); describe('#getDatachannelToken / #setDatachannelToken', () => { it('sets and gets datachannel token', () => { - llmService.setDatachannelToken('abc123','default'); - assert.equal(llmService.getDatachannelToken('default'), 'abc123'); - llmService.setDatachannelToken('123abc','practiceSession'); - assert.equal(llmService.getDatachannelToken('practiceSession'), '123abc'); + llmService.setDatachannelToken('abc123','llm-default-session'); + assert.equal(llmService.getDatachannelToken('llm-default-session'), 'abc123'); + llmService.setDatachannelToken('123abc','llm-practice-session'); + assert.equal(llmService.getDatachannelToken('llm-practice-session'), '123abc'); }); }); @@ -293,15 +398,6 @@ describe('plugin-llm', () => { const locusUrl2 = 'locusUrl2'; const datachannelUrl2 = 'datachannelUrl2'; - beforeEach(() => { - const sockets = new Map(); - - llmService.connect = sinon.stub().callsFake((url, sessionId) => { - sockets.set(sessionId, {connected: true}); - llmService.getSocket = sinon.stub().callsFake((sid) => sockets.get(sid)); - }); - }); - it('tracks multiple sessions independently', async () => { await llmService.registerAndConnect(locusUrl, datachannelUrl, undefined, 's1'); await llmService.registerAndConnect(locusUrl2, datachannelUrl2, undefined, 's2'); @@ -314,7 +410,6 @@ describe('plugin-llm', () => { assert.equal(llmService.getDatachannelUrl('s2'), datachannelUrl2); const all = llmService.getAllConnections(); - assert.equal(all.size, 2); assert.equal(all.has('s1'), true); assert.equal(all.has('s2'), true); }); @@ -333,10 +428,13 @@ describe('plugin-llm', () => { const all = llmService.getAllConnections(); assert.equal(all.has('s1'), false); assert.equal(all.has('s2'), true); + + assert.equal(llmService.datachannelTokens['s1'], undefined); }); it('disconnectAllLLM clears all sessions', async () => { llmService.disconnectAll = sinon.stub().resolves(true); + sinon.spy(llmService, 'resetDatachannelTokens'); await llmService.registerAndConnect(locusUrl, datachannelUrl, undefined, 's1'); await llmService.registerAndConnect(locusUrl2, datachannelUrl2, undefined, 's2'); @@ -345,7 +443,10 @@ describe('plugin-llm', () => { sinon.assert.calledOnce(llmService.disconnectAll); assert.equal(llmService.getAllConnections().size, 0); + + sinon.assert.calledOnce(llmService.resetDatachannelTokens); }); }); + }); }); diff --git a/packages/@webex/internal-plugin-voicea/src/voicea.ts b/packages/@webex/internal-plugin-voicea/src/voicea.ts index c300eebf422..2e1424dbfc5 100644 --- a/packages/@webex/internal-plugin-voicea/src/voicea.ts +++ b/packages/@webex/internal-plugin-voicea/src/voicea.ts @@ -42,6 +42,8 @@ export class VoiceaChannel extends WebexPlugin implements IVoiceaChannel { private captionStatus: string; + private isCaptionBoxOn: boolean; + private toggleManualCaptionStatus: string; private currentSpokenLanguage?: string; @@ -102,6 +104,7 @@ export class VoiceaChannel extends WebexPlugin implements IVoiceaChannel { */ public deregisterEvents() { this.areCaptionsEnabled = false; + this.isCaptionBoxOn = false; this.captionServiceId = undefined; // @ts-ignore this.webex.internal.llm.off('event:relay.event', this.eventProcessor); @@ -272,6 +275,8 @@ export class VoiceaChannel extends WebexPlugin implements IVoiceaChannel { // @ts-ignore this.webex.internal.llm.isConnected(LLM_PRACTICE_SESSION); + public getIsCaptionBoxOn = (): boolean => this.isCaptionBoxOn; + /** * Resolves the active LLM publish transport, preferring the practice-session * connection only when that session is fully connected. @@ -286,6 +291,9 @@ export class VoiceaChannel extends WebexPlugin implements IVoiceaChannel { socket: (isPracticeSessionConnected && llm.getSocket(LLM_PRACTICE_SESSION)) || llm.socket, binding: (isPracticeSessionConnected && llm.getBinding(LLM_PRACTICE_SESSION)) || llm.getBinding(), + datachannelUrl: + (isPracticeSessionConnected && llm.getDatachannelUrl(LLM_PRACTICE_SESSION)) || + llm.getDatachannelUrl(), }; }; @@ -461,6 +469,7 @@ export class VoiceaChannel extends WebexPlugin implements IVoiceaChannel { this.areCaptionsEnabled = true; this.captionStatus = TURN_ON_CAPTION_STATUS.ENABLED; this.announce(); + this.updateSubchannelSubscriptionsAndSyncCaptionState({subscribe: ['transcription']}, true); }) .catch(() => { this.captionStatus = TURN_ON_CAPTION_STATUS.IDLE; @@ -620,6 +629,68 @@ export class VoiceaChannel extends WebexPlugin implements IVoiceaChannel { * @returns {string} */ public getAnnounceStatus = () => this.announceStatus; + /** + * update LLM sub‑channel subscriptions. + * + * sends a single `subchannelSubscriptionRequest` to LLM, + * allowing subscribe and unsubscribe subchannel. + * + * @param {string[]} options.subscribe Sub‑channels to subscribe to. + * @param {string[]} options.unsubscribe Sub‑channels to unsubscribe from. + * @returns {Promise} + */ + public updateSubchannelSubscriptions = async ({ + subscribe = [], + unsubscribe = [], + }: { + subscribe?: string[]; + unsubscribe?: string[]; + } = {}): Promise => { + // @ts-ignore + const isDataChannelTokenEnabled = await this.webex.internal.llm.isDataChannelTokenEnabled(); + // @ts-ignore + if (!this.isLLMConnected() || !isDataChannelTokenEnabled) return; + + const {socket, datachannelUrl} = this.getPublishTransport(); + + // @ts-ignore + socket.send({ + id: `${this.seqNum}`, + type: 'subchannelSubscriptionRequest', + data: { + // @ts-ignore + datachannelUri: datachannelUrl, + subscribe, + unsubscribe, + }, + trackingId: `${config.trackingIdPrefix}_${uuid.v4().toString()}`, + }); + + this.seqNum += 1; + }; + + /** + * Syncs the UI caption intent and updates transcription subchannel + * subscriptions accordingly. + * + * @param {Object} [options] - Subscription options. + * @param {string[]} [options.subscribe] - Subchannels to subscribe to. + * @param {string[]} [options.unsubscribe] - Subchannels to unsubscribe from. + * @param {boolean} [isCaptionBoxOn=false] - Whether captions are intended to be enabled. + * + * @returns {Promise} + */ + public updateSubchannelSubscriptionsAndSyncCaptionState = ( + options: { + subscribe?: string[]; + unsubscribe?: string[]; + } = {}, + isCaptionBoxOn = false + ): Promise => { + this.isCaptionBoxOn = isCaptionBoxOn; + + return this.updateSubchannelSubscriptions(options); + }; } export default VoiceaChannel; diff --git a/packages/@webex/internal-plugin-voicea/test/unit/spec/voicea.js b/packages/@webex/internal-plugin-voicea/test/unit/spec/voicea.js index 4ea914f3e72..70dbc483ad0 100644 --- a/packages/@webex/internal-plugin-voicea/test/unit/spec/voicea.js +++ b/packages/@webex/internal-plugin-voicea/test/unit/spec/voicea.js @@ -221,19 +221,17 @@ describe('plugin-voicea', () => { assert.notCalled(voiceaService.webex.internal.llm.socket.send); }); }); - describe('#deregisterEvents', () => { beforeEach(async () => { const mockWebSocket = new MockWebSocket(); - voiceaService.webex.internal.llm.socket = mockWebSocket; + voiceaService.isCaptionBoxOn = true; }); - it('deregisters voicea service', async () => { + it('deregisters voicea service and resets caption state', async () => { voiceaService.listenToEvents(); await voiceaService.toggleTranscribing(true); - // eslint-disable-next-line no-underscore-dangle voiceaService.webex.internal.llm._emit('event:relay.event', { headers: {from: 'ws'}, data: {relayType: 'voicea.annc', voiceaPayload: {}}, @@ -241,12 +239,14 @@ describe('plugin-voicea', () => { assert.equal(voiceaService.areCaptionsEnabled, true); assert.equal(voiceaService.captionServiceId, 'ws'); + assert.equal(voiceaService.isCaptionBoxOn, true); voiceaService.deregisterEvents(); assert.equal(voiceaService.areCaptionsEnabled, false); assert.equal(voiceaService.captionServiceId, undefined); assert.equal(voiceaService.announceStatus, 'idle'); assert.equal(voiceaService.captionStatus, 'idle'); + assert.equal(voiceaService.isCaptionBoxOn, false); }); }); describe('#processAnnouncementMessage', () => { @@ -408,6 +408,7 @@ describe('plugin-voicea', () => { it('turns on captions', async () => { const announcementSpy = sinon.spy(voiceaService, 'announce'); + const updateSubchannelSubscriptionsAndSyncCaptionStateSpy = sinon.spy(voiceaService, 'updateSubchannelSubscriptionsAndSyncCaptionState'); const triggerSpy = sinon.spy(); @@ -428,6 +429,11 @@ describe('plugin-voicea', () => { assert.calledOnceWithExactly(triggerSpy); assert.calledOnce(announcementSpy); + assert.calledOnceWithExactly( + updateSubchannelSubscriptionsAndSyncCaptionStateSpy, + { subscribe: ['transcription'] }, + true + ); }); it("should handle request fail", async () => { @@ -486,6 +492,28 @@ describe('plugin-voicea', () => { }); }); + describe('#getIsCaptionBoxOn', () => { + beforeEach(() => { + voiceaService.isCaptionBoxOn = false; + }); + + it('returns false when captions are disabled', () => { + voiceaService.isCaptionBoxOn = false; + + const result = voiceaService.getIsCaptionBoxOn(); + + assert.equal(result, false); + }); + + it('returns true when captions are enabled', () => { + voiceaService.isCaptionBoxOn = true; + + const result = voiceaService.getIsCaptionBoxOn(); + + assert.equal(result, true); + }); + }); + describe("#announce", () => { let isAnnounceProcessed, sendAnnouncement; beforeEach(() => { @@ -1256,6 +1284,177 @@ describe('plugin-voicea', () => { }); }); + describe('#updateSubchannelSubscriptions', () => { + beforeEach(() => { + const mockWebSocket = new MockWebSocket(); + + sinon.stub(voiceaService, 'getPublishTransport').returns({ + socket: mockWebSocket, + datachannelUrl: 'mock-datachannel-uri', + }); + + voiceaService.seqNum = 1; + + voiceaService.isLLMConnected = sinon.stub().returns(true); + voiceaService.webex.internal.llm.isDataChannelTokenEnabled = sinon.stub().resolves(true); + }); + + it('sends subchannelSubscriptionRequest with subscribe and unsubscribe lists', async () => { + await voiceaService.updateSubchannelSubscriptions({ + subscribe: ['transcription'], + unsubscribe: ['polls'], + }); + + const socket = voiceaService.getPublishTransport().socket; + + sinon.assert.calledOnceWithExactly( + socket.send, + { + id: '1', + type: 'subchannelSubscriptionRequest', + data: { + datachannelUri: 'mock-datachannel-uri', + subscribe: ['transcription'], + unsubscribe: ['polls'], + }, + trackingId: sinon.match.string, + } + ); + + sinon.assert.match(voiceaService.seqNum, 2); + }); + + it('sends empty arrays when no subscribe/unsubscribe provided', async () => { + await voiceaService.updateSubchannelSubscriptions({}); + + const socket = voiceaService.getPublishTransport().socket; + + sinon.assert.calledOnceWithExactly( + socket.send, + { + id: '1', + type: 'subchannelSubscriptionRequest', + data: { + datachannelUri: 'mock-datachannel-uri', + subscribe: [], + unsubscribe: [], + }, + trackingId: sinon.match.string, + } + ); + + sinon.assert.match(voiceaService.seqNum, 2); + }); + + it('does nothing when LLM is not connected', async () => { + voiceaService.isLLMConnected = sinon.stub().returns(false); + + await voiceaService.updateSubchannelSubscriptions({ + subscribe: ['transcription'], + }); + + const socket = voiceaService.getPublishTransport().socket; + + sinon.assert.notCalled(socket.send); + sinon.assert.match(voiceaService.seqNum, 1); + }); + + it('does nothing when dataChannelToken is not enabled', async () => { + voiceaService.webex.internal.llm.isDataChannelTokenEnabled = sinon.stub().resolves(false); + + await voiceaService.updateSubchannelSubscriptions({ + subscribe: ['transcription'], + }); + + const socket = voiceaService.getPublishTransport().socket; + + sinon.assert.notCalled(socket.send); + sinon.assert.match(voiceaService.seqNum, 1); + }); + }); + + + describe('#updateSubchannelSubscriptionsAndSyncCaptionState', () => { + beforeEach(() => { + const mockWebSocket = new MockWebSocket(); + voiceaService.webex.internal.llm.socket = mockWebSocket; + + voiceaService.webex.internal.llm.getDatachannelUrl = sinon.stub().returns('mock-datachannel-uri'); + + voiceaService.seqNum = 1; + + voiceaService.isLLMConnected = sinon.stub().returns(true); + voiceaService.webex.internal.llm.isDataChannelTokenEnabled = sinon.stub().resolves(true); + + sinon.spy(voiceaService, 'updateSubchannelSubscriptions'); + }); + + afterEach(() => { + sinon.restore(); + }); + + it('updates caption intent and forwards subscribe/unsubscribe to updateSubchannelSubscriptions', async () => { + await voiceaService.updateSubchannelSubscriptionsAndSyncCaptionState( + { + subscribe: ['transcription'], + unsubscribe: ['polls'], + }, + true + ); + + assert.equal(voiceaService.isCaptionBoxOn, true); + + assert.calledOnceWithExactly( + voiceaService.updateSubchannelSubscriptions, + { + subscribe: ['transcription'], + unsubscribe: ['polls'], + } + ); + }); + + it('sets caption intent to false when isCCBoxOpen is false', async () => { + await voiceaService.updateSubchannelSubscriptionsAndSyncCaptionState( + { subscribe: ['transcription'] }, + false + ); + + assert.equal(voiceaService.isCaptionBoxOn, false); + + assert.calledOnceWithExactly( + voiceaService.updateSubchannelSubscriptions, + { subscribe: ['transcription'] } + ); + }); + + it('defaults subscribe/unsubscribe to empty arrays when options is empty', async () => { + await voiceaService.updateSubchannelSubscriptionsAndSyncCaptionState({}, true); + + assert.equal(voiceaService.isCaptionBoxOn, true); + + assert.calledOnceWithExactly( + voiceaService.updateSubchannelSubscriptions, + {} + ); + }); + + it('still updates caption intent even if updateSubchannelSubscriptions does nothing (e.g., LLM not connected)', async () => { + voiceaService.isLLMConnected = sinon.stub().returns(false); + + await voiceaService.updateSubchannelSubscriptionsAndSyncCaptionState( + { subscribe: ['transcription'] }, + true + ); + + assert.equal(voiceaService.isCaptionBoxOn, true); + + assert.calledOnceWithExactly( + voiceaService.updateSubchannelSubscriptions, + { subscribe: ['transcription'] } + ); + }); + }); + describe('#multiple llm connections', () => { let defaultSocket; let practiceSocket; @@ -1380,6 +1579,5 @@ describe('plugin-voicea', () => { assert.equal(voiceaService.captionServiceId, 'svc-practice'); }); }); - }); }); diff --git a/packages/@webex/plugin-meetings/package.json b/packages/@webex/plugin-meetings/package.json index 3203b4f49c7..c4b17a17ea0 100644 --- a/packages/@webex/plugin-meetings/package.json +++ b/packages/@webex/plugin-meetings/package.json @@ -84,6 +84,7 @@ "global": "^4.4.0", "ip-anonymize": "^0.1.0", "javascript-state-machine": "^3.1.0", + "jose": "^5.8.0", "jwt-decode": "3.1.2", "lodash": "^4.17.21", "uuid": "^3.3.2", diff --git a/packages/@webex/plugin-meetings/src/interceptors/dataChannelAuthToken.ts b/packages/@webex/plugin-meetings/src/interceptors/dataChannelAuthToken.ts index 77a79273357..54659385956 100644 --- a/packages/@webex/plugin-meetings/src/interceptors/dataChannelAuthToken.ts +++ b/packages/@webex/plugin-meetings/src/interceptors/dataChannelAuthToken.ts @@ -5,6 +5,7 @@ import {Interceptor} from '@webex/http-core'; import LoggerProxy from '../common/logs/logger-proxy'; import {DATA_CHANNEL_AUTH_HEADER, MAX_RETRY, RETRY_INTERVAL, RETRY_KEY} from './constant'; +import {isJwtTokenExpired} from './utils'; /*! * Copyright (c) 2015-2026 Cisco Systems, Inc. See LICENSE file. @@ -69,6 +70,33 @@ export default class DataChannelAuthTokenInterceptor extends Interceptor { return key ? headers[key] : undefined; } + /** + * Intercepts outgoing requests and refreshes the Data-Channel-Auth-Token + * if the current JWT token is expired before the request is sent. + * + * @param {Object} options - The original request options. + * @returns {Promise} Updated request options with refreshed token if needed. + */ + async onRequest(options) { + const token = this.getHeader(options.headers, DATA_CHANNEL_AUTH_HEADER); + const enabled = await this._isDataChannelTokenEnabled(); + + if (!token || !enabled) { + return options; + } + + if (isJwtTokenExpired(token)) { + try { + const newToken = await this._refreshDataChannelToken(); + options.headers[DATA_CHANNEL_AUTH_HEADER] = newToken; + } catch (e) { + LoggerProxy.logger.warn(`DataChannelAuthTokenInterceptor: refresh failed: ${e.message}`); + } + } + + return options; + } + /** * Intercept responses and, on 401/403 with `Data-Channel-Auth-Token` header, * attempt to refresh the data channel token and retry the original request once. diff --git a/packages/@webex/plugin-meetings/src/interceptors/utils.ts b/packages/@webex/plugin-meetings/src/interceptors/utils.ts new file mode 100644 index 00000000000..396e291e823 --- /dev/null +++ b/packages/@webex/plugin-meetings/src/interceptors/utils.ts @@ -0,0 +1,16 @@ +import * as jose from 'jose'; + +const EXPIRY_BUFFER = 30 * 1000; + +// eslint-disable-next-line import/prefer-default-export +export const isJwtTokenExpired = (token: string): boolean => { + try { + const payload = jose.decodeJwt(token); + + if (!payload?.exp) return false; + + return payload.exp * 1000 < Date.now() + EXPIRY_BUFFER; + } catch { + return true; + } +}; diff --git a/packages/@webex/plugin-meetings/src/meeting/index.ts b/packages/@webex/plugin-meetings/src/meeting/index.ts index e49098de3cc..bf9a045f42f 100644 --- a/packages/@webex/plugin-meetings/src/meeting/index.ts +++ b/packages/@webex/plugin-meetings/src/meeting/index.ts @@ -10317,12 +10317,12 @@ export default class Meeting extends StatelessWebexPlugin { } catch (e) { const msg = e?.message || String(e); - const err = Object.assign(new Error(`Failed to refresh data channel token: ${msg}`), { - statusCode: e?.statusCode, - original: e, - }); + LoggerProxy.logger.warn( + `Meeting:index#refreshDataChannelToken --> DataChannel token refresh failed (likely locus changed or participant left): ${msg}`, + {statusCode: e?.statusCode} + ); - throw err; + return null; } } diff --git a/packages/@webex/plugin-meetings/src/meeting/request.ts b/packages/@webex/plugin-meetings/src/meeting/request.ts index 97e70fd3c78..ca684ca1a65 100644 --- a/packages/@webex/plugin-meetings/src/meeting/request.ts +++ b/packages/@webex/plugin-meetings/src/meeting/request.ts @@ -1159,13 +1159,13 @@ export default class MeetingRequest extends StatelessWebexPlugin { method: HTTP_VERBS.GET, uri, }).catch((err) => { - LoggerProxy.logger.error( - `Meeting:request#fetchDatachannelToken --> Error retrieving ${ + LoggerProxy.logger.warn( + `Meeting:request#fetchDatachannelToken --> Failed to retrieve ${ isPracticeSession ? 'practice session ' : '' - }datachannel token, error ${err}` + }datachannel token: ${err?.message || err}` ); - throw err; + return null; }); } } diff --git a/packages/@webex/plugin-meetings/src/webinar/index.ts b/packages/@webex/plugin-meetings/src/webinar/index.ts index 6aedec1010b..4c918083c0d 100644 --- a/packages/@webex/plugin-meetings/src/webinar/index.ts +++ b/packages/@webex/plugin-meetings/src/webinar/index.ts @@ -177,6 +177,8 @@ const Webinar = WebexPlugin.extend({ const finalToken = currentToken ?? practiceSessionDatachannelToken; + const isCaptionBoxOn = this.webex.internal.voicea.getIsCaptionBoxOn(); + if (!currentToken && practiceSessionDatachannelToken) { // @ts-ignore this.webex.internal.llm.setDatachannelToken( @@ -219,6 +221,9 @@ const Webinar = WebexPlugin.extend({ ); // @ts-ignore - Fix type this.webex.internal.voicea?.announce?.(); + if (isCaptionBoxOn) { + this.webex.internal.voicea.updateSubchannelSubscriptions({subscribe: ['transcription']}); + } LoggerProxy.logger.info( `Webinar:index#updatePSDataChannel --> enabled to receive relay events for default session for ${LLM_PRACTICE_SESSION}!` ); diff --git a/packages/@webex/plugin-meetings/test/unit/spec/interceptors/dataChannelAuthToken.ts b/packages/@webex/plugin-meetings/test/unit/spec/interceptors/dataChannelAuthToken.ts index c09b247a057..d62e5285ddb 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/interceptors/dataChannelAuthToken.ts +++ b/packages/@webex/plugin-meetings/test/unit/spec/interceptors/dataChannelAuthToken.ts @@ -5,6 +5,7 @@ import MockWebex from '@webex/test-helper-mock-webex'; import {WebexHttpError} from '@webex/webex-core'; import DataChannelAuthTokenInterceptor from '@webex/plugin-meetings/src/interceptors/dataChannelAuthToken'; import LoggerProxy from '@webex/plugin-meetings/src/common/logs/logger-proxy'; +import * as utils from '@webex/plugin-meetings/src/interceptors/utils'; import {DATA_CHANNEL_AUTH_HEADER, MAX_RETRY} from '@webex/plugin-meetings/src/interceptors/constant'; describe('plugin-meetings', () => { @@ -14,6 +15,10 @@ describe('plugin-meetings', () => { beforeEach(() => { clock = sinon.useFakeTimers(); + sinon.stub(LoggerProxy, 'logger').value({ + error: sinon.stub(), + warn: sinon.stub(), + }); webex = new MockWebex({children: {}}); webex.request = sinon.stub().resolves({}); @@ -25,6 +30,7 @@ describe('plugin-meetings', () => { }); afterEach(() => { + sinon.restore(); clock.restore(); }); @@ -86,6 +92,69 @@ describe('plugin-meetings', () => { }); }); + describe('#onRequest', () => { + let isJwtTokenExpiredStub; + + beforeEach(() => { + isJwtTokenExpiredStub = sinon.stub(utils, 'isJwtTokenExpired').returns(false); + }); + + it('does nothing when token is missing', async () => { + const options = {headers: {}}; + + const res = await interceptor.onRequest(options); + + expect(res).to.equal(options); + sinon.assert.notCalled(isJwtTokenExpiredStub); + }); + + it('does nothing when feature is disabled', async () => { + interceptor._isDataChannelTokenEnabled.resolves(false); + + const options = {headers: {[DATA_CHANNEL_AUTH_HEADER]: 'old-token'}}; + const res = await interceptor.onRequest(options); + + expect(res).to.equal(options); + sinon.assert.notCalled(isJwtTokenExpiredStub); + }); + + it('does not refresh when token is not expired', async () => { + interceptor._isDataChannelTokenEnabled.resolves(true); + isJwtTokenExpiredStub.returns(false); + + const options = {headers: {[DATA_CHANNEL_AUTH_HEADER]: 'old-token'}}; + const res = await interceptor.onRequest(options); + + sinon.assert.notCalled(interceptor._refreshDataChannelToken); + expect(res.headers[DATA_CHANNEL_AUTH_HEADER]).to.equal('old-token'); + }); + + it('refreshes token when expired', async () => { + interceptor._isDataChannelTokenEnabled.resolves(true); + isJwtTokenExpiredStub.returns(true); + + interceptor._refreshDataChannelToken.resolves('new-token'); + + const options = {headers: {[DATA_CHANNEL_AUTH_HEADER]: 'old-token'}}; + const res = await interceptor.onRequest(options); + + sinon.assert.calledOnce(interceptor._refreshDataChannelToken); + expect(res.headers[DATA_CHANNEL_AUTH_HEADER]).to.equal('new-token'); + }); + + it('continues request when refresh fails', async () => { + interceptor._isDataChannelTokenEnabled.resolves(true); + isJwtTokenExpiredStub.returns(true); + + interceptor._refreshDataChannelToken.rejects(new Error('refresh failed')); + + const options = {headers: {[DATA_CHANNEL_AUTH_HEADER]: 'old-token'}}; + const res = await interceptor.onRequest(options); + + expect(res.headers[DATA_CHANNEL_AUTH_HEADER]).to.equal('old-token'); + }); + }); + describe('#refreshTokenAndRetryWithDelay', () => { const options = { headers: {[DATA_CHANNEL_AUTH_HEADER]: 'old-token'}, diff --git a/packages/@webex/plugin-meetings/test/unit/spec/interceptors/utils.ts b/packages/@webex/plugin-meetings/test/unit/spec/interceptors/utils.ts new file mode 100644 index 00000000000..1ad08a71fbf --- /dev/null +++ b/packages/@webex/plugin-meetings/test/unit/spec/interceptors/utils.ts @@ -0,0 +1,75 @@ +import 'jsdom-global/register'; +import {expect} from '@webex/test-helper-chai'; +import sinon from 'sinon'; +import {isJwtTokenExpired} from '@webex/plugin-meetings/src/interceptors/utils'; + +const makeJwt = (payload) => + [ + Buffer.from(JSON.stringify({alg: 'none', typ: 'JWT'})).toString('base64url'), + Buffer.from(JSON.stringify(payload)).toString('base64url'), + '' + ].join('.'); + +describe('plugin-meetings', () => { + describe('Interceptors', () => { + describe('utils - isJwtTokenExpired', () => { + let clock; + + beforeEach(() => { + clock = sinon.useFakeTimers(); + }); + + afterEach(() => { + sinon.restore(); + clock.restore(); + }); + + it('returns false when token has no exp', () => { + const token = makeJwt({}); // no exp + + const result = isJwtTokenExpired(token); + + expect(result).to.equal(false); + }); + + it('returns false when token is not expired', () => { + const now = Date.now(); + const futureExp = Math.floor((now + 60 * 1000) / 1000); + + const token = makeJwt({exp: futureExp}); + + const result = isJwtTokenExpired(token); + + expect(result).to.equal(false); + }); + + it('returns true when token is expired', () => { + const now = Date.now(); + const pastExp = Math.floor((now - 60 * 1000) / 1000); + + const token = makeJwt({exp: pastExp}); + + const result = isJwtTokenExpired(token); + + expect(result).to.equal(true); + }); + + it('returns true when token expires within EXPIRY_BUFFER', () => { + const now = Date.now(); + const expSoon = Math.floor((now + 10 * 1000) / 1000); + + const token = makeJwt({exp: expSoon}); + + const result = isJwtTokenExpired(token); + + expect(result).to.equal(true); + }); + + it('returns true when token is invalid', () => { + const result = isJwtTokenExpired('not-a-jwt'); + + expect(result).to.equal(true); + }); + }); + }); +}); diff --git a/packages/@webex/plugin-meetings/test/unit/spec/meeting/index.js b/packages/@webex/plugin-meetings/test/unit/spec/meeting/index.js index a527971f936..0e50ef81dd6 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/meeting/index.js +++ b/packages/@webex/plugin-meetings/test/unit/spec/meeting/index.js @@ -13029,7 +13029,7 @@ describe('plugin-meetings', () => { 'a datachannel url', 'token-123' ); - assert.calledWithExactly(webex.internal.llm.setDatachannelToken, 'token-123', 'default'); + assert.calledWithExactly(webex.internal.llm.setDatachannelToken, 'token-123', 'llm-default-session'); }); it('prefers refreshed token over locus self token', async () => { meeting.joinedWith = {state: 'JOINED'}; @@ -13039,7 +13039,7 @@ describe('plugin-meetings', () => { self: {datachannelToken: 'locus-token'}, }; - webex.internal.llm.getDatachannelToken.withArgs('default').returns('refreshed-token'); + webex.internal.llm.getDatachannelToken.withArgs('llm-default-session').returns('refreshed-token'); await meeting.updateLLMConnection(); @@ -13072,7 +13072,7 @@ describe('plugin-meetings', () => { 'a datachannel url', 'token-123' ); - assert.calledWithExactly(webex.internal.llm.setDatachannelToken, 'token-123', 'default'); + assert.calledWithExactly(webex.internal.llm.setDatachannelToken, 'token-123', 'llm-default-session'); }); describe('#clearMeetingData', () => { @@ -14735,7 +14735,7 @@ describe('plugin-meetings', () => { expect(result).to.deep.equal({ body: { datachannelToken: 'mock-token', - dataChannelTokenType: 'practiceSession', + dataChannelTokenType: 'llm-practice-session', }, }); }); @@ -14748,7 +14748,7 @@ describe('plugin-meetings', () => { const result = meeting.getDataChannelTokenType(); - expect(result).to.equal('practiceSession'); + expect(result).to.equal('llm-practice-session'); }); it('returns Default when not in practice session mode', () => { @@ -14758,7 +14758,7 @@ describe('plugin-meetings', () => { const result = meeting.getDataChannelTokenType(); - expect(result).to.equal('default'); + expect(result).to.equal('llm-default-session'); }); }); describe('#stopKeepAlive', () => { diff --git a/packages/@webex/plugin-meetings/test/unit/spec/meeting/request.js b/packages/@webex/plugin-meetings/test/unit/spec/meeting/request.js index 9cd9d9579e6..46094edaea4 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/meeting/request.js +++ b/packages/@webex/plugin-meetings/test/unit/spec/meeting/request.js @@ -924,7 +924,14 @@ describe('plugin-meetings', () => { const locusUrl = 'https://locus.example.com/locus/api/v1/loci/123'; const participantId = 'participant-123'; + beforeEach(() => { + sinon.restore(); + locusDeltaRequestSpy = sinon.stub(meetingsRequest, 'locusDeltaRequest'); + }); + it('sends GET request to regular datachannel token endpoint', async () => { + locusDeltaRequestSpy.resolves({body: {}}); + await meetingsRequest.fetchDatachannelToken({ locusUrl, requestingParticipantId: participantId, @@ -938,6 +945,8 @@ describe('plugin-meetings', () => { }); it('sends GET request to practice session datachannel token endpoint', async () => { + locusDeltaRequestSpy.resolves({body: {}}); + await meetingsRequest.fetchDatachannelToken({ locusUrl, requestingParticipantId: participantId, @@ -950,7 +959,7 @@ describe('plugin-meetings', () => { }); }); - it('throws if locusUrl or participantId is missing', async () => { + it('rejects when locusUrl or participantId is missing', async () => { await assert.isRejected( meetingsRequest.fetchDatachannelToken({ locusUrl: null, @@ -968,18 +977,15 @@ describe('plugin-meetings', () => { ); }); - it('logs and rethrows error when locusDeltaRequest fails', async () => { - const error = new Error('network error'); - locusDeltaRequestSpy.restore(); - sinon.stub(meetingsRequest, 'locusDeltaRequest').rejects(error); + it('returns null when locusDeltaRequest fails', async () => { + locusDeltaRequestSpy.rejects(new Error('network error')); - await assert.isRejected( - meetingsRequest.fetchDatachannelToken({ - locusUrl, - requestingParticipantId: participantId, - }), - /network error/ - ); + const result = await meetingsRequest.fetchDatachannelToken({ + locusUrl, + requestingParticipantId: participantId, + }); + + assert.equal(result, null); }); }); }); diff --git a/packages/@webex/plugin-meetings/test/unit/spec/webinar/index.ts b/packages/@webex/plugin-meetings/test/unit/spec/webinar/index.ts index 35c54b62478..4678f440a70 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/webinar/index.ts +++ b/packages/@webex/plugin-meetings/test/unit/spec/webinar/index.ts @@ -233,6 +233,8 @@ describe('plugin-meetings', () => { // Ensure connect path is eligible webinar.selfIsPanelist = true; webinar.practiceSessionEnabled = true; + webex.internal.voicea.getIsCaptionBoxOn = sinon.stub().returns(false); + webex.internal.voicea.updateSubchannelSubscriptions = sinon.stub(); }); it('no-ops when practice session join eligibility is false', async () => { @@ -342,6 +344,22 @@ describe('plugin-meetings', () => { processRelayEvent ); }); + + it('subscribes to transcription when caption intent is enabled', async () => { + webex.internal.voicea.getIsCaptionBoxOn = sinon.stub().returns(true); + + await webinar.updatePSDataChannel(); + + assert.calledOnceWithExactly(webex.internal.voicea.updateSubchannelSubscriptions, { subscribe: ['transcription'] }); + }); + + it('does not subscribe to transcription when caption intent is disabled', async () => { + webex.internal.voicea.getIsCaptionBoxOn = sinon.stub().returns(false); + + await webinar.updatePSDataChannel(); + + assert.notCalled(webex.internal.voicea.updateSubchannelSubscriptions); + }); }); describe('#updateStatusByRole', () => { diff --git a/yarn.lock b/yarn.lock index a624aa069c0..4d7e383a897 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8872,6 +8872,7 @@ __metadata: global: ^4.4.0 ip-anonymize: ^0.1.0 javascript-state-machine: ^3.1.0 + jose: ^5.8.0 jsdom: 19.0.0 jsdom-global: 3.0.2 jwt-decode: 3.1.2 @@ -22458,6 +22459,13 @@ __metadata: languageName: node linkType: hard +"jose@npm:^5.8.0": + version: 5.10.0 + resolution: "jose@npm:5.10.0" + checksum: e80965ef3ab47baafac3517f53fa9c74b948b57690de524f51320c314cd545ef51ec7b18761605d58fb5965b7c5e12b2bb6ddae87a6ccf55e3f4ad077347d8d7 + languageName: node + linkType: hard + "js-logger@npm:^1.6.1": version: 1.6.1 resolution: "js-logger@npm:1.6.1" From d736170aa62904780cf0cd01245b5502c0566170 Mon Sep 17 00:00:00 2001 From: Filip Nowakowski Date: Wed, 25 Mar 2026 11:57:36 +0100 Subject: [PATCH 13/28] refactor(meetings): update size hint logic and deprecate getEffectiveMaxFs method --- .../src/multistream/remoteMedia.ts | 6 + .../spec/multistream/mediaRequestManager.ts | 549 +++++++----------- .../test/unit/spec/multistream/receiveSlot.ts | 9 +- .../test/unit/spec/multistream/remoteMedia.ts | 167 ++++-- 4 files changed, 343 insertions(+), 388 deletions(-) diff --git a/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts b/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts index 5cc365efc8b..3af714983df 100644 --- a/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts +++ b/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts @@ -99,6 +99,12 @@ export class RemoteMedia extends EventsScope { this.sizeHint.width = width; this.sizeHint.height = height; this.receiveSlot?.setSizeHint(this.sizeHint); + + // TODO: remove this once deprecation of getEffectiveMaxFs() is complete + const maxFs = MediaCodecHelper.H264.getSizeHintMaxFs(this.sizeHint); + if (maxFs !== undefined) { + this.receiveSlot?.setMaxFs(maxFs); + } } /** diff --git a/packages/@webex/plugin-meetings/test/unit/spec/multistream/mediaRequestManager.ts b/packages/@webex/plugin-meetings/test/unit/spec/multistream/mediaRequestManager.ts index 40523f105d3..1a4a3409bae 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/multistream/mediaRequestManager.ts +++ b/packages/@webex/plugin-meetings/test/unit/spec/multistream/mediaRequestManager.ts @@ -5,47 +5,27 @@ import type {SizeHint} from '@webex/plugin-meetings/src/multistream/types'; import sinon from 'sinon'; import {assert} from '@webex/test-helper-chai'; import MediaCodecHelper from '@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper'; +import {getRecommendedMaxBitrateForFrameSize} from '@webex/internal-media-core'; import FakeTimers from '@sinonjs/fake-timers'; import * as InternalMediaCoreModule from '@webex/internal-media-core'; import { expect } from 'chai'; -/** Maps a target H264 maxFs to a size hint that yields that maxFs via MediaCodecHelper.H264. */ -const sizeHintForMaxFs = (maxFs: number): SizeHint => { - if (maxFs <= 60) { - return {resolution: 'thumbnail'}; - } - if (maxFs <= 240) { - return {resolution: 'very small'}; - } - if (maxFs <= 920) { - return {resolution: 'small'}; - } - if (maxFs <= 2040) { - return {height: 500}; - } - if (maxFs <= 3600) { - return {resolution: 'medium'}; - } - - return {resolution: 'large'}; -}; - type ExpectedActiveSpeaker = { policy: 'active-speaker'; - maxPayloadBitsPerSecond?: number; priority: number; receiveSlots: Array; + sizeHint?: SizeHint; maxFs?: number; - maxMbps?: number; + maxPayloadBitsPerSecond?: number; namedMediaGroups?:[{type: number, value: number}]; }; type ExpectedReceiverSelected = { policy: 'receiver-selected'; - maxPayloadBitsPerSecond?: number; csi: number; receiveSlot: ReceiveSlot; + sizeHint?: SizeHint; maxFs?: number; - maxMbps?: number; + maxPayloadBitsPerSecond?: number; }; type ExpectedRequest = ExpectedActiveSpeaker | ExpectedReceiverSelected; @@ -53,22 +33,26 @@ const degradationPreferences = { maxMacroblocksLimit: Infinity, // no limit }; +const resolveExpectedMaxFs = (req: ExpectedRequest): number | undefined => { + if (req.maxFs !== undefined) return req.maxFs; + if (req.sizeHint) return MediaCodecHelper.H264.getSizeHintMaxFs(req.sizeHint); + return undefined; +}; + +const resolveExpectedBitrate = (req: ExpectedRequest): number | undefined => { + if (req.maxPayloadBitsPerSecond !== undefined) return req.maxPayloadBitsPerSecond; + const maxFs = resolveExpectedMaxFs(req); + return maxFs ? getRecommendedMaxBitrateForFrameSize(maxFs) : undefined; +}; + describe('MediaRequestManager', () => { const CROSS_PRIORITY_DUPLICATION = true; const CROSS_POLICY_DUPLICATION = true; - const MAX_FPS = 3000; - const MAX_FS_360p = 920; - const MAX_FS_540p = 2040; - const MAX_FS_720p = 3600; - const MAX_FS_1080p = 8192; - const MAX_MBPS_360p = 27600; - const MAX_MBPS_540p = 61200; - const MAX_MBPS_720p = 108000; - const MAX_MBPS_1080p = 245760; - const MAX_PAYLOADBITSPS_360p = 640000; - const MAX_PAYLOADBITSPS_540p = 880000; - const MAX_PAYLOADBITSPS_720p = 2500000; - const MAX_PAYLOADBITSPS_1080p = 4000000; + + const SIZE_HINT_SMALL: SizeHint = {resolution: 'small'}; + const SIZE_HINT_MEDIUM: SizeHint = {resolution: 'medium'}; + const SIZE_HINT_LARGE: SizeHint = {resolution: 'large'}; + const SIZE_HINT_540p: SizeHint = {width: 960, height: 540}; const NUM_SLOTS = 15; @@ -111,7 +95,7 @@ describe('MediaRequestManager', () => { const addActiveSpeakerRequest = ( priority, receiveSlots, - maxFs, + sizeHint: SizeHint, commit = false, preferLiveVideo = true, namedMediaGroups = undefined @@ -127,13 +111,13 @@ describe('MediaRequestManager', () => { namedMediaGroups, }, receiveSlots, - sizeHint: sizeHintForMaxFs(maxFs), + sizeHint, }, commit ); // helper function for adding a receiver selected request - const addReceiverSelectedRequest = (csi, receiveSlot, maxFs, commit = false) => + const addReceiverSelectedRequest = (csi, receiveSlot, sizeHint: SizeHint, commit = false) => mediaRequestManager.addRequest( { policyInfo: { @@ -141,7 +125,7 @@ describe('MediaRequestManager', () => { csi, }, receiveSlots: [receiveSlot], - sizeHint: sizeHintForMaxFs(maxFs), + sizeHint, }, commit ); @@ -161,50 +145,58 @@ describe('MediaRequestManager', () => { assert.calledWith( sendMediaRequestsCallback, expectedRequests.map((expectedRequest) => { + const maxFs = resolveExpectedMaxFs(expectedRequest); + const maxPayloadBitsPerSecond = resolveExpectedBitrate(expectedRequest); + + const codecInfosMatcher = isCodecInfoDefined && maxFs !== undefined + ? [sinon.match({ + payloadType: 0x80, + h264: sinon.match({maxFs}), + })] + : []; + if (expectedRequest.policy === 'active-speaker') { - return sinon.match({ + const policyMatch: Record = { + priority: expectedRequest.priority, + crossPriorityDuplication: CROSS_PRIORITY_DUPLICATION, + crossPolicyDuplication: CROSS_POLICY_DUPLICATION, + preferLiveVideo, + }; + + if (expectedRequest.namedMediaGroups) { + policyMatch.namedMediaGroups = sinon.match( + expectedRequest.namedMediaGroups.map((nmg) => sinon.match(nmg)) + ); + } + + const match: Record = { policy: 'active-speaker', - policySpecificInfo: sinon.match({ - priority: expectedRequest.priority, - crossPriorityDuplication: CROSS_PRIORITY_DUPLICATION, - crossPolicyDuplication: CROSS_POLICY_DUPLICATION, - preferLiveVideo, - }), + policySpecificInfo: sinon.match(policyMatch), receiveSlots: expectedRequest.receiveSlots, - maxPayloadBitsPerSecond: expectedRequest.maxPayloadBitsPerSecond, - codecInfos: isCodecInfoDefined - ? [ - sinon.match({ - payloadType: 0x80, - h264: sinon.match({ - maxMbps: MAX_MBPS_1080p, - maxFs: expectedRequest.maxFs, - }), - }), - ] - : [], - }); + codecInfos: codecInfosMatcher, + }; + + if (maxPayloadBitsPerSecond !== undefined) { + match.maxPayloadBitsPerSecond = maxPayloadBitsPerSecond; + } + + return sinon.match(match); } if (expectedRequest.policy === 'receiver-selected') { - return sinon.match({ + const match: Record = { policy: 'receiver-selected', policySpecificInfo: sinon.match({ csi: expectedRequest.csi, }), receiveSlots: [expectedRequest.receiveSlot], - maxPayloadBitsPerSecond: expectedRequest.maxPayloadBitsPerSecond, - codecInfos: isCodecInfoDefined - ? [ - sinon.match({ - payloadType: 0x80, - h264: sinon.match({ - maxMbps: MAX_MBPS_1080p, - maxFs: expectedRequest.maxFs, - }), - }), - ] - : [], - }); + codecInfos: codecInfosMatcher, + }; + + if (maxPayloadBitsPerSecond !== undefined) { + match.maxPayloadBitsPerSecond = maxPayloadBitsPerSecond; + } + + return sinon.match(match); } return undefined; @@ -235,7 +227,7 @@ describe('MediaRequestManager', () => { preferLiveVideo: false, }, receiveSlots: [fakeReceiveSlots[0], fakeReceiveSlots[1], fakeReceiveSlots[2]], - sizeHint: sizeHintForMaxFs(MAX_FS_360p), + sizeHint: SIZE_HINT_SMALL, }, false ); @@ -247,7 +239,7 @@ describe('MediaRequestManager', () => { csi: 123, }, receiveSlots: [fakeReceiveSlots[3]], - sizeHint: sizeHintForMaxFs(MAX_FS_720p), + sizeHint: SIZE_HINT_MEDIUM, }, false ); @@ -260,11 +252,15 @@ describe('MediaRequestManager', () => { csi: 123, }, receiveSlots: [fakeReceiveSlots[4]], - sizeHint: sizeHintForMaxFs(MAX_FS_1080p), + sizeHint: SIZE_HINT_LARGE, }, true ); + const expectedSmallMaxFs = MediaCodecHelper.H264.getSizeHintMaxFs(SIZE_HINT_SMALL); + const expectedMediumMaxFs = MediaCodecHelper.H264.getSizeHintMaxFs(SIZE_HINT_MEDIUM); + const expectedLargeMaxFs = MediaCodecHelper.H264.getSizeHintMaxFs(SIZE_HINT_LARGE); + // all 3 requests should be sent out together assert.calledOnce(sendMediaRequestsCallback); assert.calledWith(sendMediaRequestsCallback, [ @@ -277,14 +273,12 @@ describe('MediaRequestManager', () => { preferLiveVideo: false, }), receiveSlots: [fakeWcmeSlots[0], fakeWcmeSlots[1], fakeWcmeSlots[2]], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_360p, + maxPayloadBitsPerSecond: getRecommendedMaxBitrateForFrameSize(expectedSmallMaxFs), codecInfos: [ sinon.match({ payloadType: 0x80, h264: sinon.match({ - maxFs: MAX_FS_360p, - maxFps: MAX_FPS, - maxMbps: MAX_MBPS_1080p, + maxFs: expectedSmallMaxFs, }), }), ], @@ -295,14 +289,12 @@ describe('MediaRequestManager', () => { csi: 123, }), receiveSlots: [fakeWcmeSlots[3]], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_720p, + maxPayloadBitsPerSecond: getRecommendedMaxBitrateForFrameSize(expectedMediumMaxFs), codecInfos: [ sinon.match({ payloadType: 0x80, h264: sinon.match({ - maxFs: MAX_FS_720p, - maxFps: MAX_FPS, - maxMbps: MAX_MBPS_1080p, + maxFs: expectedMediumMaxFs, }), }), ], @@ -313,14 +305,12 @@ describe('MediaRequestManager', () => { csi: 123, }), receiveSlots: [fakeWcmeSlots[4]], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, + maxPayloadBitsPerSecond: getRecommendedMaxBitrateForFrameSize(expectedLargeMaxFs), codecInfos: [ sinon.match({ payloadType: 0x80, h264: sinon.match({ - maxFs: MAX_FS_1080p, - maxFps: MAX_FPS, - maxMbps: MAX_MBPS_1080p, + maxFs: expectedLargeMaxFs, }), }), ], @@ -330,38 +320,32 @@ describe('MediaRequestManager', () => { it('keeps adding requests with every call to addRequest()', () => { // start with 1 request - addReceiverSelectedRequest(100, fakeReceiveSlots[0], MAX_FS_1080p, true); + addReceiverSelectedRequest(100, fakeReceiveSlots[0], SIZE_HINT_LARGE, true); checkMediaRequestsSent([ { policy: 'receiver-selected', csi: 100, receiveSlot: fakeWcmeSlots[0], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, ]); // now add another one - addReceiverSelectedRequest(101, fakeReceiveSlots[1], MAX_FS_1080p, true); + addReceiverSelectedRequest(101, fakeReceiveSlots[1], SIZE_HINT_LARGE, true); checkMediaRequestsSent([ { policy: 'receiver-selected', csi: 100, receiveSlot: fakeWcmeSlots[0], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, { policy: 'receiver-selected', csi: 101, receiveSlot: fakeWcmeSlots[1], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, ]); @@ -369,7 +353,7 @@ describe('MediaRequestManager', () => { addActiveSpeakerRequest( 1, [fakeReceiveSlots[2], fakeReceiveSlots[3], fakeReceiveSlots[4]], - MAX_FS_720p, + SIZE_HINT_MEDIUM, true ); @@ -378,31 +362,25 @@ describe('MediaRequestManager', () => { policy: 'receiver-selected', csi: 100, receiveSlot: fakeWcmeSlots[0], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, { policy: 'receiver-selected', csi: 101, receiveSlot: fakeWcmeSlots[1], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, { policy: 'active-speaker', priority: 1, receiveSlots: [fakeWcmeSlots[2], fakeWcmeSlots[3], fakeWcmeSlots[4]], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_720p, - maxFs: MAX_FS_720p, - maxMbps: MAX_MBPS_720p, + sizeHint: SIZE_HINT_MEDIUM, }, ]); }); it('removes sourceUpdate, maxFsUpdate, and sizeHintUpdate when cancelRequest() is called', () => { - const requestId = addActiveSpeakerRequest(255, [fakeReceiveSlots[2], fakeReceiveSlots[3]], MAX_FS_720p); + const requestId = addActiveSpeakerRequest(255, [fakeReceiveSlots[2], fakeReceiveSlots[3]], SIZE_HINT_MEDIUM); mediaRequestManager.cancelRequest(requestId, true); @@ -419,10 +397,10 @@ describe('MediaRequestManager', () => { it('cancels the requests correctly when cancelRequest() is called with commit=true', () => { const requestIds = [ - addActiveSpeakerRequest(255, [fakeReceiveSlots[0], fakeReceiveSlots[1]], MAX_FS_720p), - addActiveSpeakerRequest(255, [fakeReceiveSlots[2], fakeReceiveSlots[3]], MAX_FS_720p), - addReceiverSelectedRequest(100, fakeReceiveSlots[4], MAX_FS_1080p), - addReceiverSelectedRequest(200, fakeReceiveSlots[5], MAX_FS_1080p), + addActiveSpeakerRequest(255, [fakeReceiveSlots[0], fakeReceiveSlots[1]], SIZE_HINT_MEDIUM), + addActiveSpeakerRequest(255, [fakeReceiveSlots[2], fakeReceiveSlots[3]], SIZE_HINT_MEDIUM), + addReceiverSelectedRequest(100, fakeReceiveSlots[4], SIZE_HINT_LARGE), + addReceiverSelectedRequest(200, fakeReceiveSlots[5], SIZE_HINT_LARGE), ]; // cancel one of the active speaker requests @@ -434,25 +412,19 @@ describe('MediaRequestManager', () => { policy: 'active-speaker', priority: 255, receiveSlots: [fakeWcmeSlots[0], fakeWcmeSlots[1]], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_720p, - maxFs: MAX_FS_720p, - maxMbps: MAX_MBPS_720p, + sizeHint: SIZE_HINT_MEDIUM, }, { policy: 'receiver-selected', csi: 100, receiveSlot: fakeWcmeSlots[4], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, { policy: 'receiver-selected', csi: 200, receiveSlot: fakeWcmeSlots[5], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, ]); @@ -465,17 +437,13 @@ describe('MediaRequestManager', () => { policy: 'active-speaker', priority: 255, receiveSlots: [fakeWcmeSlots[0], fakeWcmeSlots[1]], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_720p, - maxFs: MAX_FS_720p, - maxMbps: MAX_MBPS_720p, + sizeHint: SIZE_HINT_MEDIUM, }, { policy: 'receiver-selected', csi: 100, receiveSlot: fakeWcmeSlots[4], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, ]); }); @@ -484,10 +452,10 @@ describe('MediaRequestManager', () => { addActiveSpeakerRequest( 10, [fakeReceiveSlots[0], fakeReceiveSlots[1], fakeReceiveSlots[2]], - MAX_FS_720p, + SIZE_HINT_MEDIUM, false ); - addReceiverSelectedRequest(123, fakeReceiveSlots[3], MAX_FS_1080p, false); + addReceiverSelectedRequest(123, fakeReceiveSlots[3], SIZE_HINT_LARGE, false); // nothing should be sent out as we didn't commit the requests assert.notCalled(sendMediaRequestsCallback); @@ -501,17 +469,13 @@ describe('MediaRequestManager', () => { policy: 'active-speaker', priority: 10, receiveSlots: [fakeWcmeSlots[0], fakeWcmeSlots[1], fakeWcmeSlots[2]], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_720p, - maxFs: MAX_FS_720p, - maxMbps: MAX_MBPS_720p, + sizeHint: SIZE_HINT_MEDIUM, }, { policy: 'receiver-selected', csi: 123, receiveSlot: fakeWcmeSlots[3], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, ]); }); @@ -522,12 +486,12 @@ describe('MediaRequestManager', () => { addActiveSpeakerRequest( 250, [fakeReceiveSlots[0], fakeReceiveSlots[1], fakeReceiveSlots[2]], - MAX_FS_720p, + SIZE_HINT_MEDIUM, false ), - addReceiverSelectedRequest(98765, fakeReceiveSlots[3], MAX_FS_1080p, false), - addReceiverSelectedRequest(99999, fakeReceiveSlots[4], MAX_FS_1080p, false), - addReceiverSelectedRequest(88888, fakeReceiveSlots[5], MAX_FS_1080p, true), + addReceiverSelectedRequest(98765, fakeReceiveSlots[3], SIZE_HINT_LARGE, false), + addReceiverSelectedRequest(99999, fakeReceiveSlots[4], SIZE_HINT_LARGE, false), + addReceiverSelectedRequest(88888, fakeReceiveSlots[5], SIZE_HINT_LARGE, true), ]; checkMediaRequestsSent([ @@ -535,33 +499,25 @@ describe('MediaRequestManager', () => { policy: 'active-speaker', priority: 250, receiveSlots: [fakeWcmeSlots[0], fakeWcmeSlots[1], fakeWcmeSlots[2]], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_720p, - maxFs: MAX_FS_720p, - maxMbps: MAX_MBPS_720p, + sizeHint: SIZE_HINT_MEDIUM, }, { policy: 'receiver-selected', csi: 98765, receiveSlot: fakeWcmeSlots[3], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, { policy: 'receiver-selected', csi: 99999, receiveSlot: fakeWcmeSlots[4], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, { policy: 'receiver-selected', csi: 88888, receiveSlot: fakeWcmeSlots[5], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, ]); @@ -580,33 +536,31 @@ describe('MediaRequestManager', () => { policy: 'receiver-selected', csi: 98765, receiveSlot: fakeWcmeSlots[3], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, ]); }); it('sends the wcme media requests when commit() is called', () => { // send some requests, all of them with commit=false - addReceiverSelectedRequest(123000, fakeReceiveSlots[0], MAX_FS_1080p, false); - addReceiverSelectedRequest(456000, fakeReceiveSlots[1], MAX_FS_1080p, false); + addReceiverSelectedRequest(123000, fakeReceiveSlots[0], SIZE_HINT_LARGE, false); + addReceiverSelectedRequest(456000, fakeReceiveSlots[1], SIZE_HINT_LARGE, false); addActiveSpeakerRequest( 255, [fakeReceiveSlots[2], fakeReceiveSlots[3], fakeReceiveSlots[4]], - MAX_FS_720p, + SIZE_HINT_MEDIUM, false ); addActiveSpeakerRequest( 254, [fakeReceiveSlots[5], fakeReceiveSlots[6], fakeReceiveSlots[7]], - MAX_FS_720p, + SIZE_HINT_MEDIUM, false ); addActiveSpeakerRequest( 254, [fakeReceiveSlots[8], fakeReceiveSlots[9], fakeReceiveSlots[10]], - MAX_FS_720p, + SIZE_HINT_MEDIUM, false, true, [{type: 1, value: 20}], @@ -623,41 +577,31 @@ describe('MediaRequestManager', () => { policy: 'receiver-selected', csi: 123000, receiveSlot: fakeWcmeSlots[0], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, { policy: 'receiver-selected', csi: 456000, receiveSlot: fakeWcmeSlots[1], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, { policy: 'active-speaker', priority: 255, receiveSlots: [fakeWcmeSlots[2], fakeWcmeSlots[3], fakeWcmeSlots[4]], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_720p, - maxFs: MAX_FS_720p, - maxMbps: MAX_MBPS_720p, + sizeHint: SIZE_HINT_MEDIUM, }, { policy: 'active-speaker', priority: 254, receiveSlots: [fakeWcmeSlots[5], fakeWcmeSlots[6], fakeWcmeSlots[7]], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_720p, - maxFs: MAX_FS_720p, - maxMbps: MAX_MBPS_720p, + sizeHint: SIZE_HINT_MEDIUM, }, { policy: 'active-speaker', priority: 254, receiveSlots: [fakeWcmeSlots[8], fakeWcmeSlots[9], fakeWcmeSlots[10]], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_720p, - maxFs: MAX_FS_720p, - maxMbps: MAX_MBPS_720p, + sizeHint: SIZE_HINT_MEDIUM, namedMediaGroups: [{type: 1, value: 20}], }, ]); @@ -665,18 +609,18 @@ describe('MediaRequestManager', () => { it('avoids sending duplicate requests and clears all the requests on reset()', () => { // send some requests and commit them one by one - addReceiverSelectedRequest(1500, fakeReceiveSlots[0], MAX_FS_1080p, false); - addReceiverSelectedRequest(1501, fakeReceiveSlots[1], MAX_FS_1080p, false); + addReceiverSelectedRequest(1500, fakeReceiveSlots[0], SIZE_HINT_LARGE, false); + addReceiverSelectedRequest(1501, fakeReceiveSlots[1], SIZE_HINT_LARGE, false); addActiveSpeakerRequest( 255, [fakeReceiveSlots[2], fakeReceiveSlots[3], fakeReceiveSlots[4]], - MAX_FS_720p, + SIZE_HINT_MEDIUM, false ); addActiveSpeakerRequest( 254, [fakeReceiveSlots[5], fakeReceiveSlots[6], fakeReceiveSlots[7]], - MAX_FS_720p, + SIZE_HINT_MEDIUM, false ); @@ -689,33 +633,25 @@ describe('MediaRequestManager', () => { policy: 'receiver-selected', csi: 1500, receiveSlot: fakeWcmeSlots[0], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, { policy: 'receiver-selected', csi: 1501, receiveSlot: fakeWcmeSlots[1], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, { policy: 'active-speaker', priority: 255, receiveSlots: [fakeWcmeSlots[2], fakeWcmeSlots[3], fakeWcmeSlots[4]], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_720p, - maxFs: MAX_FS_720p, - maxMbps: MAX_MBPS_720p, + sizeHint: SIZE_HINT_MEDIUM, }, { policy: 'active-speaker', priority: 254, receiveSlots: [fakeWcmeSlots[5], fakeWcmeSlots[6], fakeWcmeSlots[7]], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_720p, - maxFs: MAX_FS_720p, - maxMbps: MAX_MBPS_720p, + sizeHint: SIZE_HINT_MEDIUM, }, ]); @@ -734,7 +670,7 @@ describe('MediaRequestManager', () => { }); it('makes sure to call requests correctly after reset was called and another request was added', () => { - addReceiverSelectedRequest(1500, fakeReceiveSlots[0], MAX_FS_1080p, false); + addReceiverSelectedRequest(1500, fakeReceiveSlots[0], SIZE_HINT_LARGE, false); assert.notCalled(sendMediaRequestsCallback); @@ -744,9 +680,7 @@ describe('MediaRequestManager', () => { policy: 'receiver-selected', csi: 1500, receiveSlot: fakeWcmeSlots[0], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, ]); @@ -758,7 +692,7 @@ describe('MediaRequestManager', () => { checkMediaRequestsSent([]); //add new request - addReceiverSelectedRequest(1501, fakeReceiveSlots[1], MAX_FS_1080p, false); + addReceiverSelectedRequest(1501, fakeReceiveSlots[1], SIZE_HINT_LARGE, false); // commit mediaRequestManager.commit(); @@ -769,25 +703,21 @@ describe('MediaRequestManager', () => { policy: 'receiver-selected', csi: 1501, receiveSlot: fakeWcmeSlots[1], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, ]); }); it('can send same media request after previous requests have been cleared', () => { // add a request and commit - addReceiverSelectedRequest(1500, fakeReceiveSlots[0], MAX_FS_1080p, false); + addReceiverSelectedRequest(1500, fakeReceiveSlots[0], SIZE_HINT_LARGE, false); mediaRequestManager.commit(); checkMediaRequestsSent([ { policy: 'receiver-selected', csi: 1500, - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, receiveSlot: fakeWcmeSlots[0], - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, ]); @@ -802,10 +732,8 @@ describe('MediaRequestManager', () => { { policy: 'receiver-selected', csi: 1500, - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, receiveSlot: fakeWcmeSlots[0], - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, ]); }); @@ -826,18 +754,16 @@ describe('MediaRequestManager', () => { mediaRequestManager.setDegradationPreferences({maxMacroblocksLimit: 32400}); sendMediaRequestsCallback.resetHistory(); - // request 4 "large" 1080p streams, which should degrade to 720p if live - addActiveSpeakerRequest(255, fakeReceiveSlots.slice(0, 4), MediaCodecHelper.H264.getMaxFs('large'), true); + // request 4 "large" streams, which should degrade to "medium" if live + addActiveSpeakerRequest(255, fakeReceiveSlots.slice(0, 4), SIZE_HINT_LARGE, true); - // check that resulting requests are 4 "large" 1080p streams + // check that resulting requests remain "large" (no degradation because sources are not live) checkMediaRequestsSent([ { policy: 'active-speaker', priority: 255, receiveSlots: fakeWcmeSlots.slice(0, 4), - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MediaCodecHelper.H264.getMaxFs('large'), - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, ]); }); @@ -847,49 +773,43 @@ describe('MediaRequestManager', () => { mediaRequestManager.setDegradationPreferences({maxMacroblocksLimit: 32400}); sendMediaRequestsCallback.resetHistory(); - // request 3 "large" 1080p streams - addActiveSpeakerRequest(255, fakeReceiveSlots.slice(0, 3), MediaCodecHelper.H264.getMaxFs('large'), false); + // request 3 "large" streams + addActiveSpeakerRequest(255, fakeReceiveSlots.slice(0, 3), SIZE_HINT_LARGE, false); - // request additional "large" 1080p stream to exceed max macroblocks limit + // request additional "large" stream to exceed max macroblocks limit const additionalRequestId = addReceiverSelectedRequest( 123, fakeReceiveSlots[3], - MediaCodecHelper.H264.getMaxFs('large'), + SIZE_HINT_LARGE, true ); - // check that resulting requests are 4 "medium" 720p streams + // check that resulting requests are degraded to "medium" checkMediaRequestsSent([ { policy: 'active-speaker', priority: 255, receiveSlots: fakeWcmeSlots.slice(0, 3), - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_720p, - maxFs: MediaCodecHelper.H264.getMaxFs('medium'), - maxMbps: MAX_MBPS_720p, + sizeHint: SIZE_HINT_MEDIUM, }, { policy: 'receiver-selected', csi: 123, receiveSlot: fakeWcmeSlots[3], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_720p, - maxFs: MediaCodecHelper.H264.getMaxFs('medium'), - maxMbps: MAX_MBPS_720p, + sizeHint: SIZE_HINT_MEDIUM, }, ]); // cancel additional request mediaRequestManager.cancelRequest(additionalRequestId); - // check that resulting requests are 3 "large" 1080p streams + // check that resulting requests bounce back to "large" checkMediaRequestsSent([ { policy: 'active-speaker', priority: 255, receiveSlots: fakeWcmeSlots.slice(0, 3), - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MediaCodecHelper.H264.getMaxFs('large'), - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, ]); }); @@ -899,18 +819,16 @@ describe('MediaRequestManager', () => { mediaRequestManager.setDegradationPreferences({maxMacroblocksLimit: 32400}); sendMediaRequestsCallback.resetHistory(); - // request 10 "large" 1080p streams - addActiveSpeakerRequest(255, fakeReceiveSlots.slice(0, 10), MediaCodecHelper.H264.getMaxFs('large'), true); + // request 10 "large" streams + addActiveSpeakerRequest(255, fakeReceiveSlots.slice(0, 10), SIZE_HINT_LARGE, true); - // check that resulting requests are 10 540p streams + // check that resulting requests are degraded to 540p checkMediaRequestsSent([ { policy: 'active-speaker', priority: 255, receiveSlots: fakeWcmeSlots.slice(0, 10), - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_540p, - maxFs: MAX_FS_540p, - maxMbps: MAX_MBPS_540p, + sizeHint: SIZE_HINT_540p, }, ]); }); @@ -920,27 +838,23 @@ describe('MediaRequestManager', () => { mediaRequestManager.setDegradationPreferences({maxMacroblocksLimit: 32400}); sendMediaRequestsCallback.resetHistory(); - // request 5 "large" 1080p streams and 5 "small" 360p streams - addActiveSpeakerRequest(255, fakeReceiveSlots.slice(0, 5), MediaCodecHelper.H264.getMaxFs('large'), false); - addActiveSpeakerRequest(254, fakeReceiveSlots.slice(5, 10), MediaCodecHelper.H264.getMaxFs('small'), true); + // request 5 "large" streams and 5 "small" streams + addActiveSpeakerRequest(255, fakeReceiveSlots.slice(0, 5), SIZE_HINT_LARGE, false); + addActiveSpeakerRequest(254, fakeReceiveSlots.slice(5, 10), SIZE_HINT_SMALL, true); - // check that resulting requests are 5 "medium" 720p streams and 5 "small" 360p streams + // check that only "large" streams are degraded to "medium", "small" stays unchanged checkMediaRequestsSent([ { policy: 'active-speaker', priority: 255, receiveSlots: fakeWcmeSlots.slice(0, 5), - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_720p, - maxFs: MediaCodecHelper.H264.getMaxFs('medium'), - maxMbps: MAX_MBPS_720p, + sizeHint: SIZE_HINT_MEDIUM, }, { policy: 'active-speaker', priority: 254, receiveSlots: fakeWcmeSlots.slice(5, 10), - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_360p, - maxFs: MediaCodecHelper.H264.getMaxFs('small'), - maxMbps: MAX_MBPS_360p, + sizeHint: SIZE_HINT_SMALL, }, ]); }); @@ -949,7 +863,7 @@ describe('MediaRequestManager', () => { sendMediaRequestsCallback.resetHistory(); const clock = FakeTimers.install({now: Date.now()}); - addActiveSpeakerRequest(255, fakeReceiveSlots.slice(0, 10), MediaCodecHelper.H264.getMaxFs('large'), true); + addActiveSpeakerRequest(255, fakeReceiveSlots.slice(0, 10), SIZE_HINT_LARGE, true); sendMediaRequestsCallback.resetHistory(); @@ -976,8 +890,6 @@ describe('MediaRequestManager', () => { priority: 255, receiveSlots: fakeWcmeSlots.slice(0, 10), maxFs: preferredFrameSize, - maxPayloadBitsPerSecond: 99000, - maxMbps: MAX_MBPS_1080p, }, ]); clock.uninstall() @@ -1029,7 +941,6 @@ describe('MediaRequestManager', () => { // returns RecommendedOpusBitrates.FB_MONO_MUSIC as expected: maxPayloadBitsPerSecond: 64000, }, - // set isCodecInfoDefined to false, since we don't pass in a codec info when audio: ], {isCodecInfoDefined: false} ); @@ -1045,7 +956,7 @@ describe('MediaRequestManager', () => { csi: 123, }, receiveSlots: [fakeReceiveSlots[0]], - sizeHint: sizeHintForMaxFs(MAX_FS_1080p), + sizeHint: SIZE_HINT_LARGE, }, false ); @@ -1057,24 +968,23 @@ describe('MediaRequestManager', () => { policy: 'receiver-selected', csi: 123, receiveSlot: fakeWcmeSlots[0], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, ]); // calls the utility function as expected with maxFs passed in (no need to do // further tests here, since the util function itself should be tested for different inputs) - assert.calledWith(getRecommendedMaxBitrateForFrameSizeSpy, MAX_FS_1080p); + const expectedMaxFs = MediaCodecHelper.H264.getSizeHintMaxFs(SIZE_HINT_LARGE); + assert.calledWith(getRecommendedMaxBitrateForFrameSizeSpy, expectedMaxFs); }); }); - describe('maxMbps', () => { + describe('codec info', () => { beforeEach(() => { sendMediaRequestsCallback.resetHistory(); }); - it('returns the correct maxMbps value', () => { + it('includes codec info matching the requested size hint', () => { mediaRequestManager.addRequest( { policyInfo: { @@ -1082,7 +992,7 @@ describe('MediaRequestManager', () => { csi: 123, }, receiveSlots: [fakeReceiveSlots[0]], - sizeHint: sizeHintForMaxFs(MAX_FS_1080p), + sizeHint: SIZE_HINT_LARGE, }, false ); @@ -1094,9 +1004,7 @@ describe('MediaRequestManager', () => { policy: 'receiver-selected', csi: 123, receiveSlot: fakeWcmeSlots[0], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_1080p, - maxFs: MAX_FS_1080p, - maxMbps: MAX_MBPS_1080p, + sizeHint: SIZE_HINT_LARGE, }, ]); }); @@ -1123,29 +1031,29 @@ describe('MediaRequestManager', () => { describe(`preferLiveVideo=${preferLiveVideo}`, () => { it(`trims the active speaker request with lowest priority first and maintains slot order`, () => { // add some receiver-selected and active-speaker requests, in a mixed up order - addReceiverSelectedRequest(100, fakeReceiveSlots[0], MAX_FS_360p, false); + addReceiverSelectedRequest(100, fakeReceiveSlots[0], SIZE_HINT_SMALL, false); addActiveSpeakerRequest( // AS request 1 - it will get 1 slot trimmed 254, [fakeReceiveSlots[1], fakeReceiveSlots[2], fakeReceiveSlots[3]], - MAX_FS_360p, + SIZE_HINT_SMALL, false, preferLiveVideo ); addActiveSpeakerRequest( // AS request 2 - lowest priority, it will have all slots trimmed 253, [fakeReceiveSlots[7], fakeReceiveSlots[8], fakeReceiveSlots[9]], - MAX_FS_360p, + SIZE_HINT_SMALL, false, preferLiveVideo ); addActiveSpeakerRequest( // AS request 3 - highest priority, nothing will be trimmed 255, [fakeReceiveSlots[4], fakeReceiveSlots[5], fakeReceiveSlots[6]], - MAX_FS_360p, + SIZE_HINT_SMALL, false, preferLiveVideo ); - addReceiverSelectedRequest(101, fakeReceiveSlots[10], MAX_FS_360p, false); + addReceiverSelectedRequest(101, fakeReceiveSlots[10], SIZE_HINT_SMALL, false); /* Set number of available streams to 7 so that there will be enough sources only for the 2 RS requests and 2 of the 3 AS requests. The lowest priority AS request will @@ -1158,34 +1066,26 @@ describe('MediaRequestManager', () => { policy: 'receiver-selected', csi: 100, receiveSlot: fakeWcmeSlots[0], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_360p, - maxFs: MAX_FS_360p, - maxMbps: MAX_MBPS_360p, + sizeHint: SIZE_HINT_SMALL, }, { policy: 'active-speaker', priority: 254, receiveSlots: [fakeWcmeSlots[1], fakeWcmeSlots[2]], // fakeWcmeSlots[3] got trimmed - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_360p, - maxFs: MAX_FS_360p, - maxMbps: MAX_MBPS_360p, + sizeHint: SIZE_HINT_SMALL, }, // AS request with priority 253 is missing, because all of its slots got trimmed { policy: 'active-speaker', priority: 255, receiveSlots: [fakeWcmeSlots[4], fakeWcmeSlots[5], fakeWcmeSlots[6]], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_360p, - maxFs: MAX_FS_360p, - maxMbps: MAX_MBPS_360p, + sizeHint: SIZE_HINT_SMALL, }, { policy: 'receiver-selected', csi: 101, receiveSlot: fakeWcmeSlots[10], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_360p, - maxFs: MAX_FS_360p, - maxMbps: MAX_MBPS_360p, + sizeHint: SIZE_HINT_SMALL, }, ], {preferLiveVideo}); @@ -1197,60 +1097,50 @@ describe('MediaRequestManager', () => { policy: 'receiver-selected', csi: 100, receiveSlot: fakeWcmeSlots[0], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_360p, - maxFs: MAX_FS_360p, - maxMbps: MAX_MBPS_360p, + sizeHint: SIZE_HINT_SMALL, }, { policy: 'active-speaker', priority: 254, receiveSlots: [fakeWcmeSlots[1], fakeWcmeSlots[2], fakeWcmeSlots[3]], // all slots are used, nothing trimmed - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_360p, - maxFs: MAX_FS_360p, - maxMbps: MAX_MBPS_360p, + sizeHint: SIZE_HINT_SMALL, }, { policy: 'active-speaker', priority: 253, receiveSlots: [fakeWcmeSlots[7], fakeWcmeSlots[8]], // only 1 slot is trimmed - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_360p, - maxFs: MAX_FS_360p, - maxMbps: MAX_MBPS_360p, + sizeHint: SIZE_HINT_SMALL, }, { policy: 'active-speaker', priority: 255, receiveSlots: [fakeWcmeSlots[4], fakeWcmeSlots[5], fakeWcmeSlots[6]], // all slots are used, nothing trimmed - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_360p, - maxFs: MAX_FS_360p, - maxMbps: MAX_MBPS_360p, + sizeHint: SIZE_HINT_SMALL, }, { policy: 'receiver-selected', csi: 101, receiveSlot: fakeWcmeSlots[10], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_360p, - maxFs: MAX_FS_360p, - maxMbps: MAX_MBPS_360p, + sizeHint: SIZE_HINT_SMALL, }, ], {preferLiveVideo}); }) it('does not trim the receiver selected requests', async () => { // add some receiver-selected and active-speaker requests, in a mixed up order - addReceiverSelectedRequest(200, fakeReceiveSlots[0], MAX_FS_360p, false); + addReceiverSelectedRequest(200, fakeReceiveSlots[0], SIZE_HINT_SMALL, false); addActiveSpeakerRequest( 255, [fakeReceiveSlots[1], fakeReceiveSlots[2], fakeReceiveSlots[3]], - MAX_FS_360p, + SIZE_HINT_SMALL, false, preferLiveVideo ); - addReceiverSelectedRequest(201, fakeReceiveSlots[4], MAX_FS_720p, false); + addReceiverSelectedRequest(201, fakeReceiveSlots[4], SIZE_HINT_MEDIUM, false); addActiveSpeakerRequest( 254, [fakeReceiveSlots[5], fakeReceiveSlots[6], fakeReceiveSlots[7]], - MAX_FS_720p, + SIZE_HINT_MEDIUM, false, preferLiveVideo ); @@ -1265,45 +1155,44 @@ describe('MediaRequestManager', () => { policy: 'receiver-selected', csi: 200, receiveSlot: fakeWcmeSlots[0], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_360p, - maxFs: MAX_FS_360p, - maxMbps: MAX_MBPS_360p, + sizeHint: SIZE_HINT_SMALL, }, { policy: 'receiver-selected', csi: 201, receiveSlot: fakeWcmeSlots[4], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_720p, - maxFs: MAX_FS_720p, - maxMbps: MAX_MBPS_720p, + sizeHint: SIZE_HINT_MEDIUM, }, ], {preferLiveVideo}); }); it('does trimming first and applies degradationPreferences after that', async () => { // add some receiver-selected and active-speaker requests - addReceiverSelectedRequest(200, fakeReceiveSlots[0], MAX_FS_360p, false); + addReceiverSelectedRequest(200, fakeReceiveSlots[0], SIZE_HINT_SMALL, false); addActiveSpeakerRequest( 255, [fakeReceiveSlots[1], fakeReceiveSlots[2], fakeReceiveSlots[3]], - MAX_FS_360p, + SIZE_HINT_SMALL, false, preferLiveVideo ); - addReceiverSelectedRequest(201, fakeReceiveSlots[4], MAX_FS_720p, false); + addReceiverSelectedRequest(201, fakeReceiveSlots[4], SIZE_HINT_MEDIUM, false); addActiveSpeakerRequest( 254, [fakeReceiveSlots[5], fakeReceiveSlots[6], fakeReceiveSlots[7]], - MAX_FS_720p, + SIZE_HINT_MEDIUM, false, preferLiveVideo ); - // Set maxMacroblocksLimit to a value that's big enough just for the 2 RS requests and 1 AS with 1 slot of 360p. + const smallMaxFs = MediaCodecHelper.H264.getSizeHintMaxFs(SIZE_HINT_SMALL); + const mediumMaxFs = MediaCodecHelper.H264.getSizeHintMaxFs(SIZE_HINT_MEDIUM); + + // Set maxMacroblocksLimit to a value that's big enough just for the 2 RS requests and 1 AS with 1 slot of "small". // but not big enough for all of the RS and AS requests. If maxMacroblocksLimit // was applied first, the resolution of all requests (including RS ones) would be degraded // This test verifies that it's not happening and the resolutions are not affected. - mediaRequestManager.setDegradationPreferences({maxMacroblocksLimit: MAX_FS_360p + MAX_FS_720p + MAX_FS_360p}); + mediaRequestManager.setDegradationPreferences({maxMacroblocksLimit: smallMaxFs + mediumMaxFs + smallMaxFs}); sendMediaRequestsCallback.resetHistory(); /* Limit the num of streams so that only 2 RS requests and 1 AS with 1 slot can be sent out */ @@ -1315,43 +1204,37 @@ describe('MediaRequestManager', () => { policy: 'receiver-selected', csi: 200, receiveSlot: fakeWcmeSlots[0], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_360p, - maxFs: MAX_FS_360p, - maxMbps: MAX_MBPS_360p, + sizeHint: SIZE_HINT_SMALL, }, { policy: 'active-speaker', priority: 255, receiveSlots: [fakeWcmeSlots[1]], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_360p, - maxFs: MAX_FS_360p, - maxMbps: MAX_MBPS_360p, + sizeHint: SIZE_HINT_SMALL, }, { policy: 'receiver-selected', csi: 201, receiveSlot: fakeWcmeSlots[4], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_720p, - maxFs: MAX_FS_720p, - maxMbps: MAX_MBPS_720p, + sizeHint: SIZE_HINT_MEDIUM, }, ], {preferLiveVideo}); }); it('trims all AS requests completely until setNumCurrentSources() is called with non-zero values', async () => { // add some receiver-selected and active-speaker requests - addReceiverSelectedRequest(200, fakeReceiveSlots[0], MAX_FS_360p, false); + addReceiverSelectedRequest(200, fakeReceiveSlots[0], SIZE_HINT_SMALL, false); addActiveSpeakerRequest( 255, [fakeReceiveSlots[1], fakeReceiveSlots[2], fakeReceiveSlots[3]], - MAX_FS_360p, + SIZE_HINT_SMALL, false, preferLiveVideo ); addActiveSpeakerRequest( 254, [fakeReceiveSlots[5]], - MAX_FS_360p, + SIZE_HINT_SMALL, false, preferLiveVideo ); @@ -1365,9 +1248,7 @@ describe('MediaRequestManager', () => { policy: 'receiver-selected', csi: 200, receiveSlot: fakeWcmeSlots[0], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_360p, - maxFs: MAX_FS_360p, - maxMbps: MAX_MBPS_360p, + sizeHint: SIZE_HINT_SMALL, }, ], {preferLiveVideo}); }); @@ -1381,11 +1262,11 @@ describe('MediaRequestManager', () => { mediaRequestManager.reset(); // add some receiver-selected and active-speaker requests - addReceiverSelectedRequest(200, fakeReceiveSlots[0], MAX_FS_360p, false); + addReceiverSelectedRequest(200, fakeReceiveSlots[0], SIZE_HINT_SMALL, false); addActiveSpeakerRequest( 255, [fakeReceiveSlots[1], fakeReceiveSlots[2], fakeReceiveSlots[3]], - MAX_FS_360p, + SIZE_HINT_SMALL, false, preferLiveVideo ); @@ -1398,9 +1279,7 @@ describe('MediaRequestManager', () => { policy: 'receiver-selected', csi: 200, receiveSlot: fakeWcmeSlots[0], - maxPayloadBitsPerSecond: MAX_PAYLOADBITSPS_360p, - maxFs: MAX_FS_360p, - maxMbps: MAX_MBPS_360p, + sizeHint: SIZE_HINT_SMALL, }, ], {preferLiveVideo}); }); @@ -1412,15 +1291,15 @@ describe('MediaRequestManager', () => { addActiveSpeakerRequest( 255, [fakeReceiveSlots[0]], - MAX_FS_360p, + SIZE_HINT_SMALL, false, true ); - addReceiverSelectedRequest(201, fakeReceiveSlots[4], MAX_FS_720p, false); + addReceiverSelectedRequest(201, fakeReceiveSlots[4], SIZE_HINT_MEDIUM, false); addActiveSpeakerRequest( 254, [fakeReceiveSlots[2]], - MAX_FS_360p, + SIZE_HINT_SMALL, false, false ); @@ -1429,7 +1308,3 @@ describe('MediaRequestManager', () => { }) }) }); -function assertEqual(arg0: any, arg1: string) { - throw new Error('Function not implemented.'); -} - diff --git a/packages/@webex/plugin-meetings/test/unit/spec/multistream/receiveSlot.ts b/packages/@webex/plugin-meetings/test/unit/spec/multistream/receiveSlot.ts index d84bc141a42..7f9b2d7d206 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/multistream/receiveSlot.ts +++ b/packages/@webex/plugin-meetings/test/unit/spec/multistream/receiveSlot.ts @@ -144,7 +144,7 @@ describe('ReceiveSlot', () => { }); }); - describe('setMaxFs()', () => { + describe('setMaxFs() [deprecated]', () => { afterEach(() => { sinon.restore(); }); @@ -166,6 +166,13 @@ describe('ReceiveSlot', () => { } ); }); + + it('sends deprecation metric', () => { + sinon.stub(Metrics, 'sendBehavioralMetric'); + receiveSlot.setMaxFs(100); + + assert.calledOnce(Metrics.sendBehavioralMetric); + }); }); describe('setSizeHint()', () => { diff --git a/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMedia.ts b/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMedia.ts index d23672157ef..487c93a5ff0 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMedia.ts +++ b/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMedia.ts @@ -6,7 +6,9 @@ import {MediaType} from '@webex/internal-media-core'; import {RemoteMedia, RemoteMediaEvents} from '@webex/plugin-meetings/src/multistream/remoteMedia'; import {RemoteVideoResolution} from '@webex/plugin-meetings/src/multistream/types'; import {ReceiveSlotEvents} from '@webex/plugin-meetings/src/multistream/receiveSlot'; +import MediaCodecHelper from '@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper'; import Metrics from '@webex/plugin-meetings/src/metrics'; +import BEHAVIORAL_METRICS from '@webex/plugin-meetings/src/metrics/constants'; import sinon from 'sinon'; import {assert} from '@webex/test-helper-chai'; import {forEach} from 'lodash'; @@ -147,6 +149,26 @@ describe('RemoteMedia', () => { ); }); + it('includes updated size hint after setSizeHint is called', () => { + remoteMedia.setSizeHint(640, 360); + + fakeMediaRequestManager.addRequest.resetHistory(); + + remoteMedia.sendMediaRequest(1234, true); + + assert.calledWith( + fakeMediaRequestManager.addRequest, + sinon.match({ + sizeHint: sinon.match({ + resolution: 'medium', + width: 640, + height: 360, + }), + }), + true + ); + }); + it('throws when called on a stopped RemoteMedia instance', () => { remoteMedia.stop(); assert.throws( @@ -239,33 +261,19 @@ describe('RemoteMedia', () => { {width: 0, height: 240}, ], ({width, height}) => { - it(`skip updating the max fs when applied ${width}:${height}`, () => { + it(`skips update when applied ${width}x${height}`, () => { remoteMedia.setSizeHint(width, height); assert.notCalled(fakeReceiveSlot.setSizeHint); + assert.notCalled(fakeReceiveSlot.setMaxFs); }); } ); forEach( - [ - {height: 90, fs: 60}, // 90p - {height: 98, fs: 60}, - {height: 99, fs: 240}, // 180p - {height: 180, fs: 240}, - {height: 197, fs: 240}, - {height: 198, fs: 920}, // 360p - {height: 360, fs: 920}, - {height: 395, fs: 920}, - {height: 396, fs: 2040}, // 540p - {height: 540, fs: 2040}, - {height: 610, fs: 3600}, // 720p - {height: 720, fs: 3600}, - {height: 721, fs: 8192}, // 1080p - {height: 1080, fs: 8192}, - ], - ({height, fs}) => { - it(`sets the max fs to ${fs} correctly when height is ${height}`, () => { + [90, 98, 99, 180, 197, 198, 360, 395, 396, 540, 610, 720, 721, 1080], + (height) => { + it(`forwards size hint to receive slot when height is ${height}`, () => { remoteMedia.setSizeHint(100, height); assert.calledOnceWithExactly( @@ -279,9 +287,54 @@ describe('RemoteMedia', () => { }); } ); + + it('also calls setMaxFs on the receive slot for backward compatibility', () => { + remoteMedia.setSizeHint(960, 540); + + assert.calledOnce(fakeReceiveSlot.setMaxFs); + + const expectedMaxFs = MediaCodecHelper.H264.getSizeHintMaxFs({ + resolution: 'medium', + width: 960, + height: 540, + }); + + assert.calledWith(fakeReceiveSlot.setMaxFs, expectedMaxFs); + }); }); - describe('getEffectiveMaxFs()', () => { + describe('getSizeHint()', () => { + it('returns initial size hint based on resolution option', () => { + const hint = remoteMedia.getSizeHint(); + + assert.deepEqual(hint, {resolution: 'medium'}); + }); + + it('returns undefined resolution when no resolution option was provided', () => { + const rmWithoutResolution = new RemoteMedia(fakeReceiveSlot, fakeMediaRequestManager); + const hint = rmWithoutResolution.getSizeHint(); + + assert.deepEqual(hint, {resolution: undefined}); + }); + + it('includes width and height after setSizeHint is called', () => { + remoteMedia.setSizeHint(640, 360); + + const hint = remoteMedia.getSizeHint(); + + assert.deepEqual(hint, {resolution: 'medium', width: 640, height: 360}); + }); + + it('is not affected by zero-dimension calls to setSizeHint', () => { + remoteMedia.setSizeHint(0, 0); + + const hint = remoteMedia.getSizeHint(); + + assert.deepEqual(hint, {resolution: 'medium'}); + }); + }); + + describe('getEffectiveMaxFs() [deprecated]', () => { beforeEach(() => { sinon.stub(Metrics, 'sendBehavioralMetric'); }); @@ -290,63 +343,77 @@ describe('RemoteMedia', () => { Metrics.sendBehavioralMetric.restore(); }); - it('returns maxFrameSize when it is greater than 0', () => { + it('sends deprecation metric when called', () => { + remoteMedia.getEffectiveMaxFs(); + + assert.calledWith( + Metrics.sendBehavioralMetric, + BEHAVIORAL_METRICS.DEPRECATED_GET_EFFECTIVE_MAX_FS_USED, + {surface: 'RemoteMedia'} + ); + }); + + it('returns correct maxFs after setSizeHint is called', () => { remoteMedia.setSizeHint(960, 540); const result = remoteMedia.getEffectiveMaxFs(); - assert.strictEqual(result, 2040); + const expected = MediaCodecHelper.H264.getSizeHintMaxFs({ + width: 960, + height: 540, + resolution: 'medium', + }); + + assert.strictEqual(result, expected); }); - it('returns getMaxFs result when maxFrameSize is 0 and resolution is provided', () => { + it('falls back to resolution option when no pixel dimensions are set', () => { remoteMedia.setSizeHint(0, 0); - // remoteMedia was created with {resolution: 'medium'} in beforeEach - const result = remoteMedia.getEffectiveMaxFs(); - // 'medium' resolution should map to 720p which is 3600 - assert.strictEqual(result, 3600); + assert.strictEqual(result, MediaCodecHelper.H264.getMaxFs('medium')); }); - it('returns undefined when maxFrameSize is 0 and no resolution is provided', () => { - remoteMedia.setSizeHint(0, 0); - - // Create a new RemoteMedia without resolution option - const remoteMediaWithoutResolution = new RemoteMedia(fakeReceiveSlot, fakeMediaRequestManager); + it('returns undefined when no resolution and no pixel dimensions', () => { + const rmWithoutResolution = new RemoteMedia(fakeReceiveSlot, fakeMediaRequestManager); + rmWithoutResolution.setSizeHint(0, 0); - const result = remoteMediaWithoutResolution.getEffectiveMaxFs(); + const result = rmWithoutResolution.getEffectiveMaxFs(); assert.strictEqual(result, undefined); }); - it('prioritizes maxFrameSize over resolution option', () => { + it('uses pixel dimensions over resolution option when both are set', () => { remoteMedia.setSizeHint(640, 360); - // remoteMedia was created with {resolution: 'medium'} in beforeEach const result = remoteMedia.getEffectiveMaxFs(); - // Should return maxFrameSize (500) instead of resolution-based value (3600) - assert.strictEqual(result, 920); + const expected = MediaCodecHelper.H264.getSizeHintMaxFs({ + width: 640, + height: 360, + resolution: 'medium', + }); + + assert.strictEqual(result, expected); }); - it('works correctly with different resolution options', () => { - const testCases: Array<{ resolution: RemoteVideoResolution; expected: number }> = [ - { resolution: 'thumbnail', expected: 60 }, - { resolution: 'very small', expected: 240 }, - { resolution: 'small', expected: 920 }, - { resolution: 'medium', expected: 3600 }, - { resolution: 'large', expected: 8192 }, - { resolution: 'best', expected: 8192 }, + it('returns correct values for all resolution options', () => { + const resolutions: RemoteVideoResolution[] = [ + 'thumbnail', 'very small', 'small', 'medium', 'large', 'best', ]; - testCases.forEach(({ resolution, expected }) => { - const testRemoteMedia = new RemoteMedia(fakeReceiveSlot, fakeMediaRequestManager, { resolution }); - testRemoteMedia.setSizeHint(0, 0); // Ensure maxFrameSize doesn't interfere + resolutions.forEach((resolution) => { + const testRM = new RemoteMedia(fakeReceiveSlot, fakeMediaRequestManager, {resolution}); + testRM.setSizeHint(0, 0); - const result = testRemoteMedia.getEffectiveMaxFs(); + const result = testRM.getEffectiveMaxFs(); - assert.strictEqual(result, expected, `Failed for resolution: ${resolution}`); + assert.strictEqual( + result, + MediaCodecHelper.H264.getMaxFs(resolution), + `Failed for resolution: ${resolution}` + ); }); }); }); From 994fc03bf5c49a1bd20baaf8682b3c810eac7342 Mon Sep 17 00:00:00 2001 From: Jordan Rowan <86778628+jor-row@users.noreply.github.com> Date: Fri, 20 Mar 2026 17:27:25 +0000 Subject: [PATCH 14/28] fix(ca): send stayLobbyTime with lobby exit event (#4744) Co-authored-by: Jordan Rowan Co-authored-by: Gabriel Lee --- .../call-diagnostic-metrics-latencies.ts | 8 +- .../call-diagnostic-metrics.util.ts | 6 +- .../call-diagnostic-metrics-batcher.ts | 42 +++- .../call-diagnostic-metrics-latencies.ts | 237 +++++++++--------- .../call-diagnostic-metrics.util.ts | 13 +- 5 files changed, 171 insertions(+), 135 deletions(-) diff --git a/packages/@webex/internal-plugin-metrics/src/call-diagnostic/call-diagnostic-metrics-latencies.ts b/packages/@webex/internal-plugin-metrics/src/call-diagnostic/call-diagnostic-metrics-latencies.ts index 84f577ef2bd..3e8b42966a5 100644 --- a/packages/@webex/internal-plugin-metrics/src/call-diagnostic/call-diagnostic-metrics-latencies.ts +++ b/packages/@webex/internal-plugin-metrics/src/call-diagnostic/call-diagnostic-metrics-latencies.ts @@ -303,10 +303,7 @@ export default class CallDiagnosticLatencies extends WebexPlugin { * @returns - latency */ public getStayLobbyTime() { - return this.getDiffBetweenTimestamps( - 'client.locus.join.response', - 'internal.host.meeting.participant.admitted' - ); + return this.getDiffBetweenTimestamps('client.locus.join.response', 'client.lobby.exited'); } /** @@ -480,7 +477,8 @@ export default class CallDiagnosticLatencies extends WebexPlugin { const clickToInterstitial = this.getClickToInterstitial(); const interstitialToJoinOk = this.getInterstitialToJoinOK(); const joinConfJMT = this.getJoinConfJMT(); - const lobbyTime = this.getStayLobbyTime(); + const lobbyTimeLatency = this.getStayLobbyTime(); + const lobbyTime = typeof lobbyTimeLatency === 'number' ? lobbyTimeLatency : 0; if (clickToInterstitial && interstitialToJoinOk && joinConfJMT) { const totalMediaJMT = clamp( diff --git a/packages/@webex/internal-plugin-metrics/src/call-diagnostic/call-diagnostic-metrics.util.ts b/packages/@webex/internal-plugin-metrics/src/call-diagnostic/call-diagnostic-metrics.util.ts index 064008516d7..e5003a6dae9 100644 --- a/packages/@webex/internal-plugin-metrics/src/call-diagnostic/call-diagnostic-metrics.util.ts +++ b/packages/@webex/internal-plugin-metrics/src/call-diagnostic/call-diagnostic-metrics.util.ts @@ -361,7 +361,6 @@ export const prepareDiagnosticMetricItem = (webex: any, item: any) => { joinTimes.totalMediaJMT = cdl.getTotalMediaJMT(); joinTimes.interstitialToMediaOKJMT = cdl.getInterstitialToMediaOKJMT(); joinTimes.callInitMediaEngineReady = cdl.getCallInitMediaEngineReady(); - joinTimes.stayLobbyTime = cdl.getStayLobbyTime(); joinTimes.totalMediaJMTWithUserDelay = cdl.getTotalMediaJMTWithUserDelay(); joinTimes.totalJMTWithUserDelay = cdl.getTotalJMTWithUserDelay(); break; @@ -369,6 +368,11 @@ export const prepareDiagnosticMetricItem = (webex: any, item: any) => { case 'client.media.tx.start': audioSetupDelay.joinRespTxStart = cdl.getAudioJoinRespTxStart(); videoSetupDelay.joinRespTxStart = cdl.getVideoJoinRespTxStart(); + break; + + case 'client.lobby.exited': + joinTimes.stayLobbyTime = cdl.getStayLobbyTime(); + break; } if (!isEmpty(joinTimes)) { diff --git a/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics-batcher.ts b/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics-batcher.ts index df43c1263dd..2b628cffb6d 100644 --- a/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics-batcher.ts +++ b/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics-batcher.ts @@ -142,9 +142,7 @@ describe('plugin-metrics', () => { webex.internal.newMetrics.callDiagnosticLatencies.getDiffBetweenTimestamps = sinon .stub() .returns(10); - webex.internal.newMetrics.callDiagnosticLatencies.getU2CTime = sinon - .stub() - .returns(20); + webex.internal.newMetrics.callDiagnosticLatencies.getU2CTime = sinon.stub().returns(20); webex.internal.newMetrics.callDiagnosticLatencies.getReachabilityClustersReqResp = sinon .stub() .returns(10); @@ -165,7 +163,7 @@ describe('plugin-metrics', () => { registerWDMDeviceJMT: 10, showInterstitialTime: 10, getU2CTime: 20, - getReachabilityClustersReqResp: 10 + getReachabilityClustersReqResp: 10, }, }); assert.lengthOf( @@ -189,9 +187,8 @@ describe('plugin-metrics', () => { webex.internal.newMetrics.callDiagnosticLatencies.getDownloadTimeJMT = sinon .stub() .returns(100); - webex.internal.newMetrics.callDiagnosticLatencies.getClickToInterstitialWithUserDelay = sinon - .stub() - .returns(43); + webex.internal.newMetrics.callDiagnosticLatencies.getClickToInterstitialWithUserDelay = + sinon.stub().returns(43); webex.internal.newMetrics.callDiagnosticLatencies.getTotalJMTWithUserDelay = sinon .stub() .returns(64); @@ -346,7 +343,7 @@ describe('plugin-metrics', () => { webex.internal.newMetrics.callDiagnosticLatencies.getInterstitialToJoinOK = sinon .stub() .returns(7); - webex.internal.newMetrics.callDiagnosticLatencies.getStayLobbyTime = sinon + webex.internal.newMetrics.callDiagnosticLatencies.getStayLobbyTime = sinon .stub() .returns(1); webex.internal.newMetrics.callDiagnosticLatencies.getTotalMediaJMTWithUserDelay = sinon @@ -372,7 +369,6 @@ describe('plugin-metrics', () => { totalMediaJMT: 61, interstitialToMediaOKJMT: 22, callInitMediaEngineReady: 10, - stayLobbyTime: 1, totalMediaJMTWithUserDelay: 43, totalJMTWithUserDelay: 64, }, @@ -382,6 +378,34 @@ describe('plugin-metrics', () => { 0 ); }); + + it('appends the correct join times to the request for client.lobby.exited', async () => { + webex.internal.newMetrics.callDiagnosticLatencies.getStayLobbyTime = sinon + .stub() + .returns(10); + + const promise = webex.internal.newMetrics.callDiagnosticMetrics.submitToCallDiagnostics( + //@ts-ignore + {event: {name: 'client.lobby.exited'}} + ); + await flushPromises(); + clock.tick(config.metrics.batcherWait); + + await promise; + + //@ts-ignore + assert.calledOnce(webex.request); + assert.deepEqual(webex.request.getCalls()[0].args[0].body.metrics[0].eventPayload.event, { + name: 'client.lobby.exited', + joinTimes: { + stayLobbyTime: 10, + }, + }); + assert.lengthOf( + webex.internal.newMetrics.callDiagnosticMetrics.callDiagnosticEventsBatcher.queue, + 0 + ); + }); }); describe('when the request fails', () => { diff --git a/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics-latencies.ts b/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics-latencies.ts index 9a1e786a8e0..17499eee22e 100644 --- a/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics-latencies.ts +++ b/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics-latencies.ts @@ -143,7 +143,7 @@ describe('internal-plugin-metrics', () => { cdl.saveTimestamp({key: 'client.alert.removed', value: 50}); const res = cdl.getDiffBetweenTimestamps('client.alert.displayed', 'client.alert.removed', { minimum: 0, - maximum: 100 + maximum: 100, }); assert.deepEqual(res, 40); }); @@ -153,7 +153,7 @@ describe('internal-plugin-metrics', () => { cdl.saveTimestamp({key: 'client.alert.removed', value: 45}); const res = cdl.getDiffBetweenTimestamps('client.alert.displayed', 'client.alert.removed', { minimum: 10, - maximum: 100 + maximum: 100, }); assert.deepEqual(res, 10); }); @@ -163,7 +163,7 @@ describe('internal-plugin-metrics', () => { cdl.saveTimestamp({key: 'client.alert.removed', value: 210}); const res = cdl.getDiffBetweenTimestamps('client.alert.displayed', 'client.alert.removed', { minimum: 0, - maximum: 100 + maximum: 100, }); assert.deepEqual(res, 100); }); @@ -172,7 +172,7 @@ describe('internal-plugin-metrics', () => { cdl.saveTimestamp({key: 'client.alert.displayed', value: 50}); cdl.saveTimestamp({key: 'client.alert.removed', value: 45}); const res = cdl.getDiffBetweenTimestamps('client.alert.displayed', 'client.alert.removed', { - maximum: 100 + maximum: 100, }); assert.deepEqual(res, 0); }); @@ -181,7 +181,7 @@ describe('internal-plugin-metrics', () => { cdl.saveTimestamp({key: 'client.alert.displayed', value: 10}); cdl.saveTimestamp({key: 'client.alert.removed', value: 2000}); const res = cdl.getDiffBetweenTimestamps('client.alert.displayed', 'client.alert.removed', { - minimum: 5 + minimum: 5, }); assert.deepEqual(res, 1990); }); @@ -191,7 +191,7 @@ describe('internal-plugin-metrics', () => { cdl.saveTimestamp({key: 'client.alert.removed', value: 50}); const res = cdl.getDiffBetweenTimestamps('client.alert.displayed', 'client.alert.removed', { minimum: 10, - maximum: 1000 + maximum: 1000, }); assert.deepEqual(res, 10); }); @@ -200,7 +200,7 @@ describe('internal-plugin-metrics', () => { cdl.saveTimestamp({key: 'client.alert.displayed', value: 10}); const res = cdl.getDiffBetweenTimestamps('client.alert.displayed', 'client.alert.removed', { minimum: 0, - maximum: 100 + maximum: 100, }); assert.deepEqual(res, undefined); }); @@ -513,7 +513,7 @@ describe('internal-plugin-metrics', () => { value: 10, }); cdl.saveTimestamp({ - key: 'internal.host.meeting.participant.admitted', + key: 'client.lobby.exited', value: 20, }); assert.deepEqual(cdl.getStayLobbyTime(), 10); @@ -656,56 +656,56 @@ describe('internal-plugin-metrics', () => { }); it('calculates getTotalJMT correctly when clickToInterstitial is 0', () => { - cdl.saveLatency('internal.click.to.interstitial', 0); - cdl.saveTimestamp({ - key: 'internal.client.interstitial-window.click.joinbutton', - value: 20, - }); - cdl.saveTimestamp({ - key: 'client.locus.join.response', - value: 40, - }); - assert.deepEqual(cdl.getTotalJMT(), 20); + cdl.saveLatency('internal.click.to.interstitial', 0); + cdl.saveTimestamp({ + key: 'internal.client.interstitial-window.click.joinbutton', + value: 20, }); + cdl.saveTimestamp({ + key: 'client.locus.join.response', + value: 40, + }); + assert.deepEqual(cdl.getTotalJMT(), 20); + }); - it('calculates getTotalJMT correctly when interstitialToJoinOk is 0', () => { - cdl.saveTimestamp({ - key: 'internal.client.interstitial-window.click.joinbutton', - value: 40, - }); - cdl.saveLatency('internal.click.to.interstitial', 12); - cdl.saveTimestamp({ - key: 'client.locus.join.response', - value: 40, - }); - assert.deepEqual(cdl.getTotalJMT(), 12); + it('calculates getTotalJMT correctly when interstitialToJoinOk is 0', () => { + cdl.saveTimestamp({ + key: 'internal.client.interstitial-window.click.joinbutton', + value: 40, + }); + cdl.saveLatency('internal.click.to.interstitial', 12); + cdl.saveTimestamp({ + key: 'client.locus.join.response', + value: 40, }); + assert.deepEqual(cdl.getTotalJMT(), 12); + }); - it('calculates getTotalJMT correctly when both clickToInterstitial and interstitialToJoinOk are 0', () => { - cdl.saveTimestamp({ - key: 'internal.client.interstitial-window.click.joinbutton', - value: 40, - }); - cdl.saveLatency('internal.click.to.interstitial', 0); - cdl.saveTimestamp({ - key: 'client.locus.join.response', - value: 40, - }); - assert.deepEqual(cdl.getTotalJMT(), 0); + it('calculates getTotalJMT correctly when both clickToInterstitial and interstitialToJoinOk are 0', () => { + cdl.saveTimestamp({ + key: 'internal.client.interstitial-window.click.joinbutton', + value: 40, }); + cdl.saveLatency('internal.click.to.interstitial', 0); + cdl.saveTimestamp({ + key: 'client.locus.join.response', + value: 40, + }); + assert.deepEqual(cdl.getTotalJMT(), 0); + }); - it('calculates getTotalJMT correctly when both clickToInterstitial is not a number', () => { - cdl.saveTimestamp({ - key: 'internal.client.interstitial-window.click.joinbutton', - value: 40, - }); - cdl.saveLatency('internal.click.to.interstitial', 'eleven' as unknown as number); - cdl.saveTimestamp({ - key: 'client.locus.join.response', - value: 40, - }); - assert.deepEqual(cdl.getTotalJMT(), undefined); + it('calculates getTotalJMT correctly when both clickToInterstitial is not a number', () => { + cdl.saveTimestamp({ + key: 'internal.client.interstitial-window.click.joinbutton', + value: 40, }); + cdl.saveLatency('internal.click.to.interstitial', 'eleven' as unknown as number); + cdl.saveTimestamp({ + key: 'client.locus.join.response', + value: 40, + }); + assert.deepEqual(cdl.getTotalJMT(), undefined); + }); it('calculates getTotalJMT correctly when it is greater than MAX_INTEGER', () => { cdl.saveTimestamp({ @@ -740,70 +740,73 @@ describe('internal-plugin-metrics', () => { assert.deepEqual(cdl.getTotalJMTWithUserDelay(), 45); }); - it('calculates getTotalJMTWithUserDelay correctly when clickToInterstitialWithUserDelay is 0', () => { - cdl.saveLatency('internal.click.to.interstitial.with.user.delay', 0); - cdl.saveTimestamp({ - key: 'internal.client.interstitial-window.click.joinbutton', - value: 20, - }); - cdl.saveTimestamp({ - key: 'client.locus.join.response', - value: 40, - }); - assert.deepEqual(cdl.getTotalJMTWithUserDelay(), 20); + it('calculates getTotalJMTWithUserDelay correctly when clickToInterstitialWithUserDelay is 0', () => { + cdl.saveLatency('internal.click.to.interstitial.with.user.delay', 0); + cdl.saveTimestamp({ + key: 'internal.client.interstitial-window.click.joinbutton', + value: 20, + }); + cdl.saveTimestamp({ + key: 'client.locus.join.response', + value: 40, }); + assert.deepEqual(cdl.getTotalJMTWithUserDelay(), 20); + }); - it('calculates getTotalJMTWithUserDelay correctly when interstitialToJoinOk is 0', () => { - cdl.saveTimestamp({ - key: 'internal.client.interstitial-window.click.joinbutton', - value: 40, - }); - cdl.saveLatency('internal.click.to.interstitial.with.user.delay', 12); - cdl.saveTimestamp({ - key: 'client.locus.join.response', - value: 40, - }); - assert.deepEqual(cdl.getTotalJMTWithUserDelay(), 12); + it('calculates getTotalJMTWithUserDelay correctly when interstitialToJoinOk is 0', () => { + cdl.saveTimestamp({ + key: 'internal.client.interstitial-window.click.joinbutton', + value: 40, + }); + cdl.saveLatency('internal.click.to.interstitial.with.user.delay', 12); + cdl.saveTimestamp({ + key: 'client.locus.join.response', + value: 40, }); + assert.deepEqual(cdl.getTotalJMTWithUserDelay(), 12); + }); - it('calculates getTotalJMTWithUserDelay correctly when both clickToInterstitialWithUserDelay and interstitialToJoinOk are 0', () => { - cdl.saveTimestamp({ - key: 'internal.client.interstitial-window.click.joinbutton', - value: 40, - }); - cdl.saveLatency('internal.click.to.interstitial.with.user.delay', 0); - cdl.saveTimestamp({ - key: 'client.locus.join.response', - value: 40, - }); - assert.deepEqual(cdl.getTotalJMTWithUserDelay(), 0); + it('calculates getTotalJMTWithUserDelay correctly when both clickToInterstitialWithUserDelay and interstitialToJoinOk are 0', () => { + cdl.saveTimestamp({ + key: 'internal.client.interstitial-window.click.joinbutton', + value: 40, }); + cdl.saveLatency('internal.click.to.interstitial.with.user.delay', 0); + cdl.saveTimestamp({ + key: 'client.locus.join.response', + value: 40, + }); + assert.deepEqual(cdl.getTotalJMTWithUserDelay(), 0); + }); - it('calculates getTotalJMTWithUserDelay correctly when both clickToInterstitialWithUserDelay is not a number', () => { - cdl.saveTimestamp({ - key: 'internal.client.interstitial-window.click.joinbutton', - value: 40, - }); - cdl.saveLatency('internal.click.to.interstitial.with.user.delay', 'eleven' as unknown as number); - cdl.saveTimestamp({ - key: 'client.locus.join.response', - value: 40, - }); - assert.deepEqual(cdl.getTotalJMTWithUserDelay(), undefined); + it('calculates getTotalJMTWithUserDelay correctly when both clickToInterstitialWithUserDelay is not a number', () => { + cdl.saveTimestamp({ + key: 'internal.client.interstitial-window.click.joinbutton', + value: 40, + }); + cdl.saveLatency( + 'internal.click.to.interstitial.with.user.delay', + 'eleven' as unknown as number + ); + cdl.saveTimestamp({ + key: 'client.locus.join.response', + value: 40, }); + assert.deepEqual(cdl.getTotalJMTWithUserDelay(), undefined); + }); - it('calculates getTotalJMTWithUserDelay correctly when it is greater than MAX_INTEGER', () => { - cdl.saveLatency('internal.click.to.interstitial.with.user.delay', 2147483648); - cdl.saveTimestamp({ - key: 'internal.client.interstitial-window.click.joinbutton', - value: 20, - }); - cdl.saveTimestamp({ - key: 'client.locus.join.response', - value: 40, - }); - assert.deepEqual(cdl.getTotalJMTWithUserDelay(), 2147483647); + it('calculates getTotalJMTWithUserDelay correctly when it is greater than MAX_INTEGER', () => { + cdl.saveLatency('internal.click.to.interstitial.with.user.delay', 2147483648); + cdl.saveTimestamp({ + key: 'internal.client.interstitial-window.click.joinbutton', + value: 20, }); + cdl.saveTimestamp({ + key: 'client.locus.join.response', + value: 40, + }); + assert.deepEqual(cdl.getTotalJMTWithUserDelay(), 2147483647); + }); it('calculates getTotalMediaJMT correctly', () => { cdl.saveTimestamp({ @@ -827,7 +830,7 @@ describe('internal-plugin-metrics', () => { value: 20, }); cdl.saveTimestamp({ - key: 'internal.host.meeting.participant.admitted', + key: 'client.lobby.exited', value: 24, }); cdl.saveTimestamp({ @@ -863,7 +866,7 @@ describe('internal-plugin-metrics', () => { value: 2147483700, }); cdl.saveTimestamp({ - key: 'internal.host.meeting.participant.admitted', + key: 'client.lobby.exited', value: 2147483800, }); cdl.saveTimestamp({ @@ -900,7 +903,7 @@ describe('internal-plugin-metrics', () => { value: 20, }); cdl.saveTimestamp({ - key: 'internal.host.meeting.participant.admitted', + key: 'client.lobby.exited', value: 24, }); cdl.saveTimestamp({ @@ -937,7 +940,7 @@ describe('internal-plugin-metrics', () => { value: 2147483700, }); cdl.saveTimestamp({ - key: 'internal.host.meeting.participant.admitted', + key: 'client.lobby.exited', value: 2147483800, }); cdl.saveTimestamp({ @@ -1041,20 +1044,20 @@ describe('internal-plugin-metrics', () => { // the maximum possible sum is 2400000, which is less than MAX_INTEGER (2147483647). // This test should verify that the final clamping works by mocking the intermediate methods // to return values that would sum to more than MAX_INTEGER. - + const originalGetJoinReqResp = cdl.getJoinReqResp; const originalGetICESetupTime = cdl.getICESetupTime; - + // Mock the methods to return large values that would exceed MAX_INTEGER when summed cdl.getJoinReqResp = () => 1500000000; cdl.getICESetupTime = () => 1000000000; - + const result = cdl.getJoinConfJMT(); - + // Restore original methods cdl.getJoinReqResp = originalGetJoinReqResp; cdl.getICESetupTime = originalGetICESetupTime; - + assert.deepEqual(result, 2147483647); }); @@ -1140,7 +1143,7 @@ describe('internal-plugin-metrics', () => { value: 10, }); cdl.saveTimestamp({ - key: 'internal.host.meeting.participant.admitted', + key: 'client.lobby.exited', value: 12, }); cdl.saveTimestamp({ @@ -1160,7 +1163,7 @@ describe('internal-plugin-metrics', () => { value: 10, }); cdl.saveTimestamp({ - key: 'internal.host.meeting.participant.admitted', + key: 'client.lobby.exited', value: 12, }); cdl.saveTimestamp({ diff --git a/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics.util.ts b/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics.util.ts index 406ea91019b..53e5009d15c 100644 --- a/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics.util.ts +++ b/packages/@webex/internal-plugin-metrics/test/unit/spec/call-diagnostic/call-diagnostic-metrics.util.ts @@ -311,7 +311,7 @@ describe('internal-plugin-metrics', () => { origin: { buildType: 'prod', networkType: 'unknown', - upgradeChannel: expectedUpgradeChannel + upgradeChannel: expectedUpgradeChannel, }, event: {name: eventName, ...expectedEvent}, }, @@ -393,7 +393,7 @@ describe('internal-plugin-metrics', () => { totalJmt: undefined, clientJmt: undefined, downloadTime: undefined, - clickToInterstitialWithUserDelay: undefined, + clickToInterstitialWithUserDelay: undefined, totalJMTWithUserDelay: undefined, }, }, @@ -430,7 +430,6 @@ describe('internal-plugin-metrics', () => { totalMediaJMT: undefined, interstitialToMediaOKJMT: undefined, callInitMediaEngineReady: undefined, - stayLobbyTime: undefined, totalMediaJMTWithUserDelay: undefined, totalJMTWithUserDelay: undefined, }, @@ -447,6 +446,14 @@ describe('internal-plugin-metrics', () => { }, }, ], + [ + 'client.lobby.exited', + { + joinTimes: { + stayLobbyTime: undefined, + }, + }, + ], ].forEach(([eventName, expectedEvent]) => { it(`returns expected result for ${eventName}`, () => { check(eventName as string, expectedEvent, 'gold'); From 66d47ae643259464559e433f33057901d56aa9bb Mon Sep 17 00:00:00 2001 From: Hem Dutt Date: Mon, 23 Mar 2026 15:33:54 +0530 Subject: [PATCH 15/28] feat(internal-call-ai-assistant): ai generated summaries in call history (#4753) Co-authored-by: Hem Dutt --- .../.eslintrc.js | 6 + .../internal-plugin-call-ai-summary/README.md | 257 ++++ .../ai-docs/AGENTS.md | 300 +++++ .../ai-docs/ARCHITECTURE.md | 1189 +++++++++++++++++ .../babel.config.js | 3 + .../jest.config.js | 3 + .../package.json | 46 + .../src/ai-summary.ts | 318 +++++ .../src/config.ts | 7 + .../src/constants.ts | 19 + .../src/index.ts | 13 + .../src/manual-integration-test.js | 129 ++ .../src/manual-pragya-api-test.js | 166 +++ .../src/types.ts | 154 +++ yarn.lock | 19 + 15 files changed, 2629 insertions(+) create mode 100644 packages/@webex/internal-plugin-call-ai-summary/.eslintrc.js create mode 100644 packages/@webex/internal-plugin-call-ai-summary/README.md create mode 100644 packages/@webex/internal-plugin-call-ai-summary/ai-docs/AGENTS.md create mode 100644 packages/@webex/internal-plugin-call-ai-summary/ai-docs/ARCHITECTURE.md create mode 100644 packages/@webex/internal-plugin-call-ai-summary/babel.config.js create mode 100644 packages/@webex/internal-plugin-call-ai-summary/jest.config.js create mode 100644 packages/@webex/internal-plugin-call-ai-summary/package.json create mode 100644 packages/@webex/internal-plugin-call-ai-summary/src/ai-summary.ts create mode 100644 packages/@webex/internal-plugin-call-ai-summary/src/config.ts create mode 100644 packages/@webex/internal-plugin-call-ai-summary/src/constants.ts create mode 100644 packages/@webex/internal-plugin-call-ai-summary/src/index.ts create mode 100644 packages/@webex/internal-plugin-call-ai-summary/src/manual-integration-test.js create mode 100644 packages/@webex/internal-plugin-call-ai-summary/src/manual-pragya-api-test.js create mode 100644 packages/@webex/internal-plugin-call-ai-summary/src/types.ts diff --git a/packages/@webex/internal-plugin-call-ai-summary/.eslintrc.js b/packages/@webex/internal-plugin-call-ai-summary/.eslintrc.js new file mode 100644 index 00000000000..a4fc83c6f44 --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/.eslintrc.js @@ -0,0 +1,6 @@ +const config = { + root: true, + extends: ['@webex/eslint-config-legacy'], +}; + +module.exports = config; diff --git a/packages/@webex/internal-plugin-call-ai-summary/README.md b/packages/@webex/internal-plugin-call-ai-summary/README.md new file mode 100644 index 00000000000..ef72be791a7 --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/README.md @@ -0,0 +1,257 @@ +# @webex/internal-plugin-call-ai-summary + +Internal Webex JS SDK plugin for retrieving AI-generated call summaries, notes, action items, and transcripts from completed calls. + +## Overview + +This plugin resolves AI summary containers via the **Pragya** service and fetches encrypted summary content from URLs returned by Pragya. All AI-generated content is decrypted using KMS keys provided in the container response. + +**Discovery flow:** + +1. **Janus** (call history) returns `extensionPayload.callingContainerIds` per call session +2. **Pragya** resolves a container ID into metadata including content URLs and encryption key +3. **Plugin** fetches content from those URLs and decrypts using `@webex/internal-plugin-encryption` + +## Install + +This plugin is part of the Webex JS SDK monorepo. It self-registers when imported — no changes to `packages/webex` are needed. + +```bash +# From the SDK monorepo root +yarn +``` + +To use in a consuming application: + +```javascript +// Importing the plugin auto-registers it on webex.internal.aisummary +import '@webex/internal-plugin-call-ai-summary'; +``` + +## Prerequisites + +- An authenticated Webex SDK instance with a registered device +- `@webex/internal-plugin-encryption` (pulled in automatically as a dependency) +- A valid Pragya container ID (obtained from Janus call history `extensionPayload.callingContainerIds`) + +## API + +All methods are accessible via `webex.internal.aisummary`. + +### `getContainer({ containerId })` + +Resolves a Pragya container by ID. Returns container metadata with summary content URLs and the KMS encryption key. + +The raw Pragya response nests URLs under `summaryData.data` — this method flattens it so you can access `summaryData.summaryUrl` directly. + +```typescript +const container = await webex.internal.aisummary.getContainer({ + containerId: '34125120-13b5-11f1-9b36-adb685725098', +}); + +// container.summaryData.summaryUrl — full summary URL +// container.summaryData.transcriptUrl — transcript URL +// container.summaryData.status — "Active" when ready +// container.encryptionKeyUrl — KMS key for decryption +``` + +**Returns:** `Promise` + +### `getSummary({ containerInfo })` + +Fetches and decrypts all summary content (note, short note, and action items) in a single request via `summaryUrl?fields=note,shortnote,actionitems`. + +This is the **recommended** method for retrieving summary content. + +```typescript +const summary = await webex.internal.aisummary.getSummary({ + containerInfo: container, +}); + +console.log(summary.note); // Decrypted full note +console.log(summary.shortNote); // Decrypted short note +summary.actionItems.forEach((item) => { + console.log(item.aiGeneratedContent); // Decrypted action item +}); +``` + +**Returns:** `Promise` — `{ id, note, shortNote, actionItems, feedbackUrl? }` + +### `getNotes({ containerInfo })` + +Fetches and decrypts notes from the standalone `notesUrl` endpoint. Only available if `notesUrl` is present in the Pragya response. + +```typescript +const notes = await webex.internal.aisummary.getNotes({ + containerInfo: container, +}); + +console.log(notes.content); // Decrypted notes +``` + +**Returns:** `Promise` — `{ id, content, feedbackUrl? }` + +### `getActionItems({ containerInfo })` + +Fetches and decrypts action items from the standalone `actionItemsUrl` endpoint. Only available if `actionItemsUrl` is present in the Pragya response. + +```typescript +const actionItems = await webex.internal.aisummary.getActionItems({ + containerInfo: container, +}); + +actionItems.snippets.forEach((snippet) => { + console.log(snippet.aiGeneratedContent); // Decrypted + console.log(snippet.editedContent); // User-edited version (if any) +}); +``` + +**Returns:** `Promise` — `{ id?, snippets[], feedbackUrl? }` + +### `getTranscriptUrl({ containerInfo })` + +Returns the transcript URL string without fetching or decrypting. Use this when you need the URL for downstream processing. + +```typescript +const url = webex.internal.aisummary.getTranscriptUrl({ + containerInfo: container, +}); +``` + +**Returns:** `string` + +### `getTranscript({ containerInfo })` + +Fetches and decrypts the full call transcript. + +```typescript +const transcript = await webex.internal.aisummary.getTranscript({ + containerInfo: container, +}); + +transcript.snippets.forEach((snippet) => { + console.log(`[${snippet.startTime}] ${snippet.speaker?.speakerName}: ${snippet.content}`); +}); +``` + +**Returns:** `Promise` — `{ id, totalCount, snippets[] }` + +## Full Usage Example + +```typescript +import '@webex/internal-plugin-call-ai-summary'; + +// 1. Get call history (existing SDK API) +const callHistory = await callHistoryInstance.getCallHistoryData(10, 50); +const sessions = callHistory.data.userSessions; + +// 2. Find a session with AI summary +const session = sessions.find( + (s) => s.extensionPayload?.callingContainerIds?.length > 0 +); +if (!session) return; + +// 3. Resolve the container +const containerId = session.extensionPayload.callingContainerIds[0]; +const container = await webex.internal.aisummary.getContainer({ containerId }); + +if (container.summaryData.status !== 'Active') { + console.log('Summary not ready yet'); + return; +} + +// 4. Fetch all summary content in one call +const summary = await webex.internal.aisummary.getSummary({ containerInfo: container }); +console.log('Note:', summary.note); +console.log('Short Note:', summary.shortNote); +summary.actionItems.forEach((item, i) => { + console.log(`Action ${i + 1}: ${item.aiGeneratedContent}`); +}); + +// 5. Fetch transcript +const transcript = await webex.internal.aisummary.getTranscript({ containerInfo: container }); +transcript.snippets.forEach((s) => { + console.log(`[${s.startTime}] ${s.speaker?.speakerName}: ${s.content}`); +}); +``` + +## Manual Testing + +A manual integration test is provided for verifying against live APIs: + +```bash +cd packages/@webex/internal-plugin-call-ai-summary + +# Provide a fresh token and container ID +WEBEX_TOKEN='' CONTAINER_ID='' node src/manual-integration-test.js +``` + +This script registers a device (WDM), resolves the Pragya service via the SDK service catalog, fetches the container, decrypts summary content via KMS, and prints the results. + +## Error Handling + +| Error | Cause | Recovery | +|-------|-------|----------| +| `containerId is required and must be a non-empty string` | Empty or missing containerId | Validate input before calling | +| `containerInfo with valid summaryData and encryptionKeyUrl is required` | Missing container info or URL field | Call `getContainer()` first | +| `Container not found` | 404 from Pragya | Verify containerId from Janus | +| `Summary content not available or expired` | 404 from content endpoint | Content may have been deleted | +| `Access denied: User not authorized to view this summary` | 403 | Check user permissions / org AI settings | +| `Authentication failed: Invalid or expired token` | 401 | Re-authenticate the user | + +## Encryption + +All AI-generated content (`aiGeneratedContent` fields) is encrypted with KMS. The plugin decrypts automatically using: + +- **Key source:** `encryptionKeyUrl` from the Pragya container response, with fallback to `keyUrl` from the content response body +- **Decryption method:** `webex.internal.encryption.decryptText(keyUrl, ciphertext)` + +This is the same pattern used by `@webex/internal-plugin-ai-assistant` and `@webex/internal-plugin-task`. + +## Development + +```bash +cd packages/@webex/internal-plugin-call-ai-summary + +# Build +yarn build + +# Lint +yarn test:style + +# Unit tests +yarn test:unit + +# All checks +yarn test +``` + +## Package Structure + +``` +src/ + index.ts # Self-registration via registerInternalPlugin('aisummary', ...) + ai-summary.ts # Plugin implementation (WebexPlugin.extend) + config.ts # Plugin config + constants.ts # Service name, error messages + types.ts # TypeScript interfaces +test/ + unit/ + spec/ + ai-summary.ts # Unit tests (26 tests) + data/ + responses.ts # Mock API response fixtures +ai-docs/ + ARCHITECTURE.md # Detailed architecture document +``` + +## Dependencies + +| Package | Purpose | +|---------|---------| +| `@webex/webex-core` | Plugin infrastructure (`WebexPlugin`, `registerInternalPlugin`) | +| `@webex/internal-plugin-encryption` | KMS content decryption via `decryptText()` | + +## Architecture + +See [ai-docs/ARCHITECTURE.md](ai-docs/ARCHITECTURE.md) for the full architecture document covering data flows, API request/response details, DTOs, security considerations, and testing strategy. diff --git a/packages/@webex/internal-plugin-call-ai-summary/ai-docs/AGENTS.md b/packages/@webex/internal-plugin-call-ai-summary/ai-docs/AGENTS.md new file mode 100644 index 00000000000..263c4265e40 --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/ai-docs/AGENTS.md @@ -0,0 +1,300 @@ +# @webex/internal-plugin-call-ai-summary + +This is an internal Cisco Webex plugin. As such, it does not strictly adhere to semantic versioning. Use at your own risk. If you're not working on one of our first party clients, please look at our developer api and stick to our public plugins. +Internal Webex JS SDK plugin for retrieving AI-generated call summaries, notes, action items, and transcript URLs from the Pragya and AI Bridge services. + +## Overview + +This plugin provides methods to: + +1. Resolve a **Pragya container** by ID (returns metadata, summary URLs, and encryption key) +2. Fetch and decrypt **AI-generated summaries** (note, short note, action items) in a single call +3. Fetch and decrypt **AI-generated notes** via a dedicated notes endpoint +4. Fetch and decrypt **AI-generated action items** via a dedicated action items endpoint +5. Retrieve the **transcript URL** for a call + +All AI-generated content is **JWE-encrypted** and decrypted via the KMS (Key Management Service) using `@webex/internal-plugin-encryption`. + +## Architecture + +``` +Pragya Service AI Bridge Service +(container metadata) (summary content) + | | + getContainer() getSummary() / getNotes() / getActionItems() + | | + v v + PragyaContainerResponse Encrypted JWE content + (summaryData, encryptionKeyUrl) | + v + KMS Decryption + (internal-plugin-encryption) + | + v + Decrypted plaintext (HTML) +``` + +**Note:** The Pragya API returns summary URLs nested under `summaryData.data`. The `getContainer()` method normalizes this automatically, flattening `summaryData.data` into `summaryData` so consumers can access `summaryData.summaryUrl` directly. + +## Registration + +The plugin registers itself as `aisummary` on the internal namespace: + +```typescript +import '@webex/internal-plugin-call-ai-summary'; + +// Accessed via: +webex.internal.aisummary.getContainer({ containerId: '...' }); +``` + +## Source Files + +| File | Description | +|------|-------------| +| `src/index.ts` | Entry point. Registers the plugin via `registerInternalPlugin('aisummary', ...)`. | +| `src/ai-summary.ts` | Main plugin class extending `WebexPlugin`. Contains all public and private methods. | +| `src/types.ts` | TypeScript interfaces for request/response DTOs. | +| `src/constants.ts` | Service name, resource path, and error message constants. | +| `src/config.ts` | Plugin configuration (currently empty). | + +## API Reference + +### `getContainer(options: GetContainerOptions): Promise` + +Resolves a Pragya container by ID. Returns container metadata including summary URLs and the KMS encryption key URL. Normalizes the response by flattening `summaryData.data` into `summaryData`. + +```typescript +const container = await webex.internal.aisummary.getContainer({ + containerId: '34125120-13b5-11f1-9b36-adb685725098', +}); + +// After normalization, URLs are directly on summaryData: +console.log(container.summaryData.summaryUrl); // https://aibridge-.../summaries/... +console.log(container.summaryData.transcriptUrl); // https://aibridge-.../transcripts/... +``` + +**Request**: `GET {pragya-service}/containers/{containerId}` + +**Response fields**: +- `summaryData` — Contains summary URLs (`summaryUrl`, `transcriptUrl`, `status`, `summarizeAfterCall`) +- `encryptionKeyUrl` — KMS key URL for decrypting content (e.g., `kms://kms-aore.wbx2.com/keys/...`) +- `kmsResourceObjectUrl`, `aclUrl`, `forkSessionId`, `callSessionId`, `ownerUserId`, `orgId`, `start`, `end` + +### `getSummary(options: GetSummaryContentOptions): Promise` + +Fetches all AI-generated summary content (note, short note, and action items) from a single request to the summary URL, and decrypts each field via KMS. This is the primary method for retrieving summary content. + +```typescript +const summary = await webex.internal.aisummary.getSummary({ + containerInfo: container, +}); + +console.log(summary.note); // Decrypted full note (HTML) +console.log(summary.shortNote); // Decrypted short note (HTML) +console.log(summary.actionItems); // Array of decrypted action item snippets +console.log(summary.feedbackUrl); // Feedback URL from links (if available) +``` + +**Request**: `GET {summaryUrl}?fields=note,shortnote,actionitems` + +**Response structure** (from AI Bridge, before decryption): +```json +{ + "id": "...", + "keyUrl": "kms://...", + "note": { "aiGeneratedContent": "" }, + "shortnote": { "aiGeneratedContent": "" }, + "actionitems": { + "snippets": [ + { "id": "...", "aiGeneratedContent": "" } + ] + }, + "links": [ + { "rel": "feedback", "href": "https://..." } + ] +} +``` + +**Return type** (`SummaryContent`): +- `id` — Summary identifier +- `note` — Decrypted full note (HTML string) +- `shortNote` — Decrypted short note (HTML string) +- `actionItems` — Array of `ActionItemSnippet` objects +- `feedbackUrl` — Extracted from `links` array (`rel: "feedback"`), if available + +### `getNotes(options: GetSummaryContentOptions): Promise` + +Fetches AI-generated notes from the dedicated notes endpoint and decrypts via KMS. Requires `notesUrl` to be present in the container's `summaryData`. + +```typescript +const notes = await webex.internal.aisummary.getNotes({ + containerInfo: container, +}); + +console.log(notes.content); // Decrypted notes content +``` + +**Request**: `GET {notesUrl}` + +> **Note:** The `notesUrl` may not be present in all API versions. Prefer `getSummary()` which returns notes, short notes, and action items in a single call. + +### `getActionItems(options: GetSummaryContentOptions): Promise` + +Fetches AI-generated action items from the dedicated action items endpoint and decrypts each snippet via KMS. Requires `actionItemsUrl` to be present in the container's `summaryData`. + +```typescript +const actionItems = await webex.internal.aisummary.getActionItems({ + containerInfo: container, +}); + +actionItems.snippets.forEach((item) => { + console.log(item.aiGeneratedContent); // Decrypted action item +}); +``` + +**Request**: `GET {actionItemsUrl}` + +> **Note:** The `actionItemsUrl` may not be present in all API versions. Prefer `getSummary()` which returns notes, short notes, and action items in a single call. + +### `getTranscriptUrl(options: GetSummaryContentOptions): string` + +Returns the transcript URL from the container info. Does not fetch or decrypt content. + +```typescript +const transcriptUrl = webex.internal.aisummary.getTranscriptUrl({ + containerInfo: container, +}); +``` + +## Types + +### Request Types + +```typescript +interface GetContainerOptions { + containerId: string; // Pragya container ID +} + +interface GetSummaryContentOptions { + containerInfo: PragyaContainerResponse; // Resolved container from getContainer() +} +``` + +### Response Types + +```typescript +interface PragyaContainerResponse { + summaryData: PragyaSummaryData; + encryptionKeyUrl: string; + kmsResourceObjectUrl: string; + aclUrl: string; + forkSessionId: string; + callSessionId: string; + ownerUserId: string; + orgId: string; + start: string; + end: string; +} + +interface PragyaSummaryData { + status: string; + summaryUrl: string; + transcriptUrl: string; + summarizeAfterCall: boolean; + notesUrl?: string; // May not be present in all API versions + actionItemsUrl?: string; // May not be present in all API versions +} + +interface SummaryContent { + id: string; + note: string; // Decrypted full note (HTML) + shortNote: string; // Decrypted short note (HTML) + actionItems: ActionItemSnippet[]; + feedbackUrl?: string; // From links array (rel="feedback") +} + +interface SummaryNotes { + id: string; + content: string; // Decrypted notes content + feedbackUrl?: string; +} + +interface SummaryActionItems { + id: string; + snippets: ActionItemSnippet[]; + feedbackUrl?: string; +} + +interface ActionItemSnippet { + id: string; + editedContent?: string; // User-edited version (if available) + aiGeneratedContent: string; // Decrypted AI-generated content +} +``` + +## Error Handling + +The plugin normalizes HTTP errors into descriptive messages: + +| Status Code | Error Message | +|-------------|---------------| +| 401 | `Authentication failed: Invalid or expired token` | +| 403 | `Access denied: User not authorized to view this summary` | +| 404 | `Container not found` | +| Other | `{methodName} failed: {error.message}` | + +Validation errors are thrown synchronously: +- Missing or empty `containerId` throws `containerId is required and must be a non-empty string` +- Missing `containerInfo`, `summaryData` URL, or `encryptionKeyUrl` throws `containerInfo with valid summaryData and encryptionKeyUrl is required` + +## Encryption / Decryption + +All AI-generated content from the AI Bridge service is JWE-encrypted. Decryption uses: + +``` +webex.internal.encryption.decryptText(encryptionKeyUrl, encryptedContent) +``` + +This requires: +1. A registered device (`webex.internal.device.register()`) +2. Mercury WebSocket connection (initiated automatically during KMS key fetch) +3. ECDHE key exchange with KMS +4. Key retrieval from KMS using the `encryptionKeyUrl` + +The SDK handles steps 1-4 automatically when `decryptText` is called. + +## Dependencies + +- `@webex/webex-core` — Base plugin class, request handling, auth interceptor +- `@webex/internal-plugin-encryption` — KMS decryption + +## Token Requirements + +The Pragya and AI Bridge APIs require a valid Webex access token. The SDK's auth interceptor automatically attaches the token for URLs in the service catalog or on allowed domains (e.g., `wbx2.com`, `webex.com`). + +## Manual Testing + +Two manual test scripts are provided in `src/`: + +### `manual-pragya-api-test.js` +Validates the Pragya container response structure (34 checks). + +```bash +cd packages/@webex/internal-plugin-call-ai-summary +WEBEX_TOKEN='' node src/manual-pragya-api-test.js +``` + +### `manual-integration-test.js` +Tests the full end-to-end flow using the SDK service catalog: +1. Device registration (WDM) to populate the service catalog +2. `getContainer` via plugin (resolves `service: 'pragya'` from the catalog) +3. `getSummary` via plugin (fetches + decrypts note, short note, and action items via KMS) +4. `getTranscriptUrl` via plugin +5. Transcript content fetch + +```bash +cd packages/@webex/internal-plugin-call-ai-summary +WEBEX_TOKEN='' CONTAINER_ID='' node src/manual-integration-test.js +``` + +Both scripts require a valid Webex access token. Set `WEBEX_TOKEN` and optionally `CONTAINER_ID` as environment variables, or update the placeholder values in the scripts. diff --git a/packages/@webex/internal-plugin-call-ai-summary/ai-docs/ARCHITECTURE.md b/packages/@webex/internal-plugin-call-ai-summary/ai-docs/ARCHITECTURE.md new file mode 100644 index 00000000000..0200ee89f2b --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/ai-docs/ARCHITECTURE.md @@ -0,0 +1,1189 @@ +# AI Call Summary Architecture + +## 1. Overview + +The Webex JS SDK will provide AI-generated call summary retrieval capabilities through a new **`internal-plugin-call-ai-summary`** internal plugin. This document describes the architecture for retrieving AI-generated notes, action items, and transcripts from completed calls. + +### 1.1 Summary Discovery Flow + +AI summary content is discovered through a two-step lookup: **Janus** (call history) provides container IDs, and **Pragya** (AI container service) resolves those IDs into direct URLs for summary content. + +**Step 1: Get container IDs from Janus call history** + +The Janus `UserSession` response includes an `extensionPayload` field containing container IDs for AI artifacts related to a call: + +```typescript +export type UserSession = { + id: string; + sessionId: string; + disposition: Disposition; + startTime: string; + endTime: string; + url: string; + durationSeconds: number; + joinedDurationSeconds: number; + participantCount: number; + isDeleted: boolean; + isPMR: boolean; + correlationIds: string[]; + links: CallRecordLink; + self: CallRecordSelf; + other: CallRecordListOther; + sessionType: SessionType; + direction: string; + callingSpecifics?: { redirectionDetails: RedirectionDetails }; + extensionPayload?: { + callingContainerIds?: string[]; + }; +}; +``` + +> **Note:** The `extensionPayload.callingContainerIds` field is already present in the Janus API response but is not yet in the SDK's `UserSession` type definition at `packages/calling/src/Events/types.ts`. However, this plugin does **not** modify that type. It accepts a plain `containerId` string as input, keeping the plugin self-contained. The `UserSession` type update can be handled separately by the calling package team when convenient. + +**Step 2: Resolve container IDs via Pragya** + +For each `containerId`, call the Pragya container API: + +``` +GET https://{pragya-host}/pragya/api/v1/containers/{containerId} +``` + +**Pragya response:** + +> **Note:** The raw Pragya response nests summary URLs under `summaryData.data`. The plugin's `getContainer()` method flattens this so consumers can access `summaryData.summaryUrl` directly. + +```json +{ + "id": "34125120-13b5-11f1-9b36-adb685725098", + "objectType": "callingAIContainer", + "memberships": { + "items": [ + { "id": "...", "roles": ["OWNER"], "objectType": "containerMembership" } + ] + }, + "summaryData": { + "extensionId": "...", + "objectType": "extension", + "extensionType": "callingAISummary", + "data": { + "id": "...", + "objectType": "callingAISummary", + "status": "Active", + "summaryUrl": "https://aibridge-url/summaries/c635e870-7b3b-4b3b-8b3b-7b3b7b3b7b3c", + "transcriptUrl": "https://aibridge-url/summaries/c635e870-7b3b-4b3b-8b3b-7b3b7b3b7b3c/transcripts", + "summarizeAfterCall": true, + "aclUrl": "https://acl-a.wbx2.com/...", + "kmsResourceObjectUrl": "kms://kms-cisco.wbx2.com/resources/...", + "contentRetention": { ... } + } + }, + "encryptionKeyUrl": "kms://kms-cisco.wbx2.com/keys/897e4d2d-6219-433d-be77-7ec73fe1c0db", + "kmsResourceObjectUrl": "kms://kms-cisco.wbx2.com/resources/f7316435-2147-4d23-bf4a-762d831cb58c", + "aclUrl": "https://acl-a.wbx2.com/acl/api/v1/acls/78c4cd90-f880-11ee-96e9-3932dce37910", + "forkSessionId": "123e4567-e89b-12d3-a456-426614174000", + "callSessionId": "123e4567-e89b-12d3-a456-426614174000", + "ownerUserId": "123e4567-e89b-12d3-a456-426614174000", + "orgId": "123e4567-e89b-12d3-a456-426614174000", + "start": "2023-10-01T12:00:00Z", + "end": "2023-10-01T12:00:00Z" +} +``` + +**Step 3: Fetch summary content from the URLs** + +The `summaryData` object provides direct, region-correct URLs to each content type. The plugin fetches content from these URLs and decrypts it using the `encryptionKeyUrl` from the same Pragya response. + +### 1.2 Key Design Decisions + +- **Self-contained plugin with zero changes to existing packages.** The plugin owns all of its types, constants, and logic. It does not modify `UserSession`, `CallHistory`, or any other existing code. Consumers pass a `containerId` string; how they obtain it (Janus, Mercury event, hard-coded for testing) is their concern. +- **No separate service discovery for summary endpoints.** Pragya returns fully-qualified URLs that already include the correct regional host. The SDK fetches from these URLs directly using `uri:` rather than `service:` + `resource:`. +- **Pragya is the source of truth** for both the content URLs and the encryption key. +- **Pragya is discoverable via U2C** as `serviceName: "pragya"` (validated: e.g., load-us resolves to `https://pragya-loada.ciscospark.com/pragya/api/v1`). + +### 1.3 Goals + +- Resolve AI summary container IDs from Janus call history via Pragya +- Retrieve AI-generated notes (full notes) for a call +- Retrieve AI-generated action items for a call +- Retrieve transcript download URLs for a call +- Handle encrypted content decryption via KMS +- Maintain consistency with existing Webex JS SDK internal plugin patterns +- Provide type-safe interfaces for all operations +- Support both browser and Node.js environments + +### 1.4 Non-Goals + +- Start/stop AI assistant during active calls (handled by Pragya start/stop APIs, out of scope) +- Generate or regenerate summaries (backend-managed during/after calls) +- Provide real-time in-call AI responses +- Handle recording storage or deletion +- Implement feedback UI components + +### 1.5 Prerequisites + +1. Janus API already returns `extensionPayload.callingContainerIds` in the response +2. Testing environment with AI-enabled calls that generate summaries + +## 2. High-Level Design + +### 2.1 Component Architecture + +``` ++---------------------------------------------------------------+ +| Client Application | ++-------------------------------+-------------------------------+ + | + | webex.internal.aisummary.* + | ++-------------------------------v-------------------------------+ +| internal-plugin-call-ai-summary | +| (Internal Plugin) | +| +----------------------------------------------------------+ | +| | Public API Methods | | +| | - getContainer(containerId) | | +| | - getSummary(containerInfo) | | +| | - getNotes(containerInfo) | | +| | - getActionItems(containerInfo) | | +| | - getTranscriptUrl(containerInfo) | | +| +----------------------------+-----------------------------+ | +| | | +| +----------------------------v-----------------------------+ | +| | Internal Logic | | +| | - Input validation | | +| | - Content decryption (KMS via encryptionKeyUrl) | | +| | - Response normalization | | +| | - Error handling & mapping | | +| +----------------------------+-----------------------------+ | ++-------------------------------+-------------------------------+ + | + +-----------------+-----------------+ + | | ++-------------v--------------+ +-----------------v--------------+ +| Pragya Service | | Summary Content URLs | +| (U2C: serviceName:pragya) | | (Direct URLs from Pragya) | +| GET /containers/{id} | | GET {summaryUrl} | ++----------------------------+ | GET {notesUrl} | + | GET {actionItemsUrl} | + +--------------------------------+ + | + +-----------v--------------------+ + | internal-plugin-encryption | + | decryptText(keyUrl, cipher) | + +--------------------------------+ +``` + +### 2.2 Key Components + +| Component | Responsibility | +|-----------|----------------| +| `internal-plugin-call-ai-summary` | Internal plugin; resolves Pragya containers, fetches and decrypts summary content | +| `internal-plugin-encryption` | KMS integration for decrypting AI-generated content using `encryptionKeyUrl` | +| `http-core` | HTTP transport; adds authorization headers, handles retries | +| Pragya Service | Container metadata; provides content URLs and encryption key | +| Summary Content Endpoints | Serve encrypted AI-generated content (notes, action items, transcripts) | + +## 3. Data Flow + +### 3.1 End-to-End Summary Retrieval Flow + +``` +Client + | + | 1. Get call history + +-> callHistory.getCallHistoryData() + | +-> Janus API: GET /history/userSessions + | +-> Response includes extensionPayload.callingContainerIds + | + | 2. Resolve container + +-> webex.internal.aisummary.getContainer(containerId) + | +-> Pragya API: GET /pragya/api/v1/containers/{containerId} + | +-> Response: { summaryData: { summaryUrl, notesUrl, ... }, encryptionKeyUrl } + | + | 3. Fetch all summary content in one call + +-> webex.internal.aisummary.getSummary({ containerInfo: container }) + +-> HTTP GET {summaryUrl}?fields=note,shortnote,actionitems + +-> Response: { note: {...}, shortnote: {...}, actionitems: {...} } + +-> Decrypt note, shortNote, and all action item snippets + +-> Return { id, note, shortNote, actionItems, feedbackUrl } +``` + +### 3.2 Get Container Info Flow + +``` +Client + +-> webex.internal.aisummary.getContainer({ containerId }) + +-> Validate containerId (non-empty string) + +-> webex.request({ + method: 'GET', + service: 'pragya', + resource: `containers/${containerId}`, + }) + +-> Flatten: if body.summaryData.data exists, set body.summaryData = body.summaryData.data + +-> Return PragyaContainerResponse (with flat summaryData) +``` + +### 3.3 Get Notes Flow + +``` +Client + +-> webex.internal.aisummary.getNotes(containerInfo) + +-> Validate containerInfo has summaryData.notesUrl and encryptionKeyUrl + +-> webex.request({ + method: 'GET', + uri: containerInfo.summaryData.notesUrl, + }) + +-> Response: { id, aiGeneratedContent: "", feedbackUrl?, keyUrl } + +-> Decrypt aiGeneratedContent using containerInfo.encryptionKeyUrl + +-> Return decrypted SummaryNotes +``` + +### 3.4 Get Action Items Flow + +``` +Client + +-> webex.internal.aisummary.getActionItems(containerInfo) + +-> Validate containerInfo has summaryData.actionItemsUrl and encryptionKeyUrl + +-> webex.request({ + method: 'GET', + uri: containerInfo.summaryData.actionItemsUrl, + }) + +-> Response: [{ id, keyUrl, snippets: [{ id, content, aiGeneratedContent }] }] + +-> Decrypt all aiGeneratedContent fields using containerInfo.encryptionKeyUrl + +-> Return decrypted SummaryActionItems +``` + +## 4. SDK Method Interfaces + +### 4.1 Internal API Methods + +```typescript +/** + * AISummary namespace accessible via webex.internal.aisummary + */ +interface AISummary { + /** + * Resolve a Pragya container by ID to get summary URLs and encryption key. + */ + getContainer(options: GetContainerOptions): Promise; + + /** + * Get AI-generated full summary for a call. + * Fetches from summaryUrl with ?fields=note,shortnote,actionitems and decrypts all content. + * Returns note, shortNote, and actionItems in a single response. + */ + getSummary(options: GetSummaryContentOptions): Promise; + + /** + * Get AI-generated notes for a call. + * Fetches from containerInfo.summaryData.notesUrl and decrypts content. + * Only available if notesUrl is present in the Pragya response. + */ + getNotes(options: GetSummaryContentOptions): Promise; + + /** + * Get AI-generated action items for a call. + * Fetches from containerInfo.summaryData.actionItemsUrl and decrypts content. + * Only available if actionItemsUrl is present in the Pragya response. + */ + getActionItems(options: GetSummaryContentOptions): Promise; + + /** + * Get the transcript URL for a call. + * Returns the URL from containerInfo.summaryData.transcriptUrl. + * Does not fetch or decrypt - the consumer uses this URL directly. + */ + getTranscriptUrl(options: GetSummaryContentOptions): string; + + /** + * Get decrypted transcript for a call. + * Fetches from containerInfo.summaryData.transcriptUrl and decrypts each snippet. + */ + getTranscript(options: GetSummaryContentOptions): Promise; +} +``` + +## 5. Data Transfer Objects (DTOs) + +### 5.1 Request DTOs + +```typescript +/** + * Options for resolving a Pragya container + */ +export interface GetContainerOptions { + /** Pragya container ID from Janus extensionPayload.callingContainerIds */ + containerId: string; +} + +/** + * Options for fetching summary content. + * Requires the resolved Pragya container info. + */ +export interface GetSummaryContentOptions { + /** The resolved Pragya container response */ + containerInfo: PragyaContainerResponse; +} +``` + +### 5.2 Pragya Response DTOs + +```typescript +/** + * Summary data URLs from a Pragya container + */ +export interface PragyaSummaryData { + /** Status of the summary (e.g., "Active") */ + status: string; + /** Full summary URL (AI Bridge) */ + summaryUrl: string; + /** Transcript URL (AI Bridge) */ + transcriptUrl: string; + /** Whether summarization runs after call ends */ + summarizeAfterCall: boolean; + /** Notes-specific URL (may not be present in all API versions) */ + notesUrl?: string; + /** Action items URL (may not be present in all API versions) */ + actionItemsUrl?: string; +} + +/** + * Complete Pragya container response + */ +export interface PragyaContainerResponse { + /** Summary data with content URLs */ + summaryData: PragyaSummaryData; + /** KMS encryption key URL for decrypting content */ + encryptionKeyUrl: string; + /** KMS resource object URL */ + kmsResourceObjectUrl: string; + /** ACL URL for access control */ + aclUrl: string; + /** Fork session ID */ + forkSessionId: string; + /** Call session ID */ + callSessionId: string; + /** Owner user ID */ + ownerUserId: string; + /** Organization ID */ + orgId: string; + /** Call start time */ + start: string; + /** Call end time */ + end: string; +} +``` + +### 5.3 Summary Response DTOs + +```typescript +/** + * Decrypted AI-generated summary content. + * Contains all three content types returned by the summary API. + */ +export interface SummaryContent { + /** Unique identifier */ + id: string; + /** Decrypted full note content */ + note: string; + /** Decrypted short note content */ + shortNote: string; + /** Decrypted action item snippets */ + actionItems: ActionItemSnippet[]; + /** Feedback URL (if available) */ + feedbackUrl?: string; +} + +/** + * Decrypted AI-generated notes + */ +export interface SummaryNotes { + /** Unique identifier */ + id: string; + /** Decrypted notes content */ + content: string; + /** Feedback URL (if available) */ + feedbackUrl?: string; +} + +/** + * Single action item snippet + */ +export interface ActionItemSnippet { + /** Unique identifier */ + id: string; + /** User-edited version (if available) */ + editedContent?: string; + /** Decrypted AI-generated content */ + aiGeneratedContent: string; +} + +/** + * Decrypted AI-generated action items + */ +export interface SummaryActionItems { + /** Unique identifier (absent when no action items exist) */ + id?: string; + /** Array of action item snippets */ + snippets: ActionItemSnippet[]; + /** Feedback URL (if available) */ + feedbackUrl?: string; +} + +/** + * Single decrypted transcript snippet + */ +export interface TranscriptSnippet { + /** Start time in milliseconds */ + startTime: string; + /** End time in milliseconds */ + endTime: string; + /** Decrypted transcript content */ + content: string; + /** Audio CSI identifier */ + audioCSI?: string; + /** Speaker information */ + speaker?: { + speakerName: string; + speakerId: string; + }; +} + +/** + * Decrypted transcript response + */ +export interface TranscriptContent { + /** Unique identifier */ + id: string; + /** Total number of snippets */ + totalCount: number; + /** Decrypted transcript snippets */ + snippets: TranscriptSnippet[]; +} +``` + +## 6. Low-Level Design & Pseudo Code + +### 6.1 Plugin Registration + +```typescript +// packages/@webex/internal-plugin-call-ai-summary/src/index.ts + +import '@webex/internal-plugin-encryption'; +import {registerInternalPlugin} from '@webex/webex-core'; + +import AISummary from './ai-summary'; +import config from './config'; + +registerInternalPlugin('aisummary', AISummary, {config}); + +export {default} from './ai-summary'; +``` + +### 6.2 Config + +```typescript +// packages/@webex/internal-plugin-call-ai-summary/src/config.ts + +export default { + aisummary: {}, +}; +``` + +### 6.3 Constants + +```typescript +// packages/@webex/internal-plugin-call-ai-summary/src/constants.ts + +export const AI_SUMMARY_SERVICE = 'pragya'; +export const AI_SUMMARY_CONTAINERS_RESOURCE = 'containers'; + +export const SUMMARY_STATUSES = { + ACTIVE: 'Active', +} as const; + +export const ERROR_MESSAGES = { + INVALID_CONTAINER_ID: 'containerId is required and must be a non-empty string', + INVALID_CONTAINER_INFO: 'containerInfo with valid summaryData and encryptionKeyUrl is required', + CONTAINER_NOT_FOUND: 'Container not found', + CONTENT_NOT_FOUND: 'Summary content not available or expired', + ACCESS_DENIED: 'Access denied: User not authorized to view this summary', + AUTHENTICATION_FAILED: 'Authentication failed: Invalid or expired token', +} as const; +``` + +### 6.4 Plugin Implementation + +```typescript +// packages/@webex/internal-plugin-call-ai-summary/src/ai-summary.ts + +import {WebexPlugin} from '@webex/webex-core'; + +import {AI_SUMMARY_SERVICE, AI_SUMMARY_CONTAINERS_RESOURCE, ERROR_MESSAGES} from './constants'; +import type { + GetContainerOptions, + GetSummaryContentOptions, + PragyaContainerResponse, + SummaryContent, + SummaryNotes, + SummaryActionItems, + TranscriptContent, +} from './types'; + +const AISummary = WebexPlugin.extend({ + namespace: 'AISummary', + + /** + * Resolve a Pragya container by ID. + * Flattens the nested summaryData.data structure for consumer convenience. + */ + getContainer(options: GetContainerOptions): Promise { + const {containerId} = options; + this._validateContainerId(containerId); + + return this.webex + .request({ + method: 'GET', + service: AI_SUMMARY_SERVICE, + resource: `${AI_SUMMARY_CONTAINERS_RESOURCE}/${containerId}`, + }) + .then(({body}) => { + // Pragya API nests summary URLs under summaryData.data — flatten + if (body.summaryData?.data) { + body.summaryData = body.summaryData.data; + } + return body; + }) + .catch((error) => { + this.logger.error('AISummary->getContainer failed', {error, containerId}); + throw this._handleError(error, 'getContainer'); + }); + }, + + /** + * Get AI-generated full summary for a call. + * Fetches note, shortNote, and actionItems in a single request via + * summaryUrl?fields=note,shortnote,actionitems, then decrypts all content. + */ + async getSummary(options: GetSummaryContentOptions): Promise { + const {containerInfo} = options; + this._validateContainerInfo(containerInfo, 'summaryUrl'); + + try { + const {body} = await this.webex.request({ + method: 'GET', + uri: `${containerInfo.summaryData.summaryUrl}?fields=note,shortnote,actionitems`, + }); + + const keyUrl = body.keyUrl || containerInfo.encryptionKeyUrl; + const decryptedNote = await this._decryptContent(body.note.aiGeneratedContent, keyUrl); + const decryptedShortNote = await this._decryptContent( + body.shortnote.aiGeneratedContent, keyUrl + ); + + const decryptedSnippets = await Promise.all( + (body.actionitems?.snippets || []).map(async (snippet: any) => { + const decryptedAiContent = await this._decryptContent( + snippet.aiGeneratedContent, keyUrl + ); + return { + id: snippet.id, + editedContent: snippet.content || undefined, + aiGeneratedContent: decryptedAiContent, + }; + }) + ); + + const feedbackLink = (body.links || []).find((link: any) => link.rel === 'feedback'); + + return { + id: body.id, + note: decryptedNote, + shortNote: decryptedShortNote, + actionItems: decryptedSnippets, + feedbackUrl: feedbackLink?.href, + }; + } catch (error) { + this.logger.error('AISummary->getSummary failed', {error}); + throw this._handleError(error, 'getSummary'); + } + }, + + /** + * Get AI-generated notes for a call (standalone endpoint). + * Uses body.keyUrl as decryption key with fallback to containerInfo.encryptionKeyUrl. + */ + async getNotes(options: GetSummaryContentOptions): Promise { + const {containerInfo} = options; + this._validateContainerInfo(containerInfo, 'notesUrl'); + + try { + const {body} = await this.webex.request({ + method: 'GET', + uri: containerInfo.summaryData.notesUrl, + }); + + const keyUrl = body.keyUrl || containerInfo.encryptionKeyUrl; + const decryptedContent = await this._decryptContent(body.aiGeneratedContent, keyUrl); + + return { id: body.id, content: decryptedContent, feedbackUrl: body.feedbackUrl }; + } catch (error) { + this.logger.error('AISummary->getNotes failed', {error}); + throw this._handleError(error, 'getNotes'); + } + }, + + /** + * Get AI-generated action items for a call (standalone endpoint). + * Response is an array; takes the first element and decrypts all snippets. + */ + async getActionItems(options: GetSummaryContentOptions): Promise { + const {containerInfo} = options; + this._validateContainerInfo(containerInfo, 'actionItemsUrl'); + + try { + const {body} = await this.webex.request({ + method: 'GET', + uri: containerInfo.summaryData.actionItemsUrl, + }); + + const actionItemsData = Array.isArray(body) ? body[0] : body; + if (!actionItemsData) return {id: undefined, snippets: []}; + + const keyUrl = actionItemsData.keyUrl || containerInfo.encryptionKeyUrl; + const decryptedSnippets = await Promise.all( + (actionItemsData.snippets || []).map(async (snippet: any) => { + const decryptedAiContent = await this._decryptContent( + snippet.aiGeneratedContent, keyUrl + ); + return { + id: snippet.id, + editedContent: snippet.content || undefined, + aiGeneratedContent: decryptedAiContent, + }; + }) + ); + + return { + id: actionItemsData.id, + snippets: decryptedSnippets, + feedbackUrl: actionItemsData.feedbackUrl, + }; + } catch (error) { + this.logger.error('AISummary->getActionItems failed', {error}); + throw this._handleError(error, 'getActionItems'); + } + }, + + /** Returns the transcript URL string from the container info. */ + getTranscriptUrl(options: GetSummaryContentOptions): string { + const {containerInfo} = options; + this._validateContainerInfo(containerInfo, 'transcriptUrl'); + return containerInfo.summaryData.transcriptUrl; + }, + + /** Fetches and decrypts the full transcript, returning all snippets. */ + async getTranscript(options: GetSummaryContentOptions): Promise { + const {containerInfo} = options; + this._validateContainerInfo(containerInfo, 'transcriptUrl'); + + try { + const {body} = await this.webex.request({ + method: 'GET', + uri: containerInfo.summaryData.transcriptUrl, + }); + + const keyUrl = body.keyUrl || containerInfo.encryptionKeyUrl; + const decryptedSnippets = await Promise.all( + (body.transcriptSnippetList || []).map(async (snippet: any) => { + const decryptedContent = await this._decryptContent(snippet.content, keyUrl); + return { + startTime: snippet.startTime, + endTime: snippet.endTime, + content: decryptedContent, + audioCSI: snippet.audioCSI, + speaker: snippet.speaker, + }; + }) + ); + + return { id: body.id, totalCount: body.totalCount, snippets: decryptedSnippets }; + } catch (error) { + this.logger.error('AISummary->getTranscript failed', {error}); + throw this._handleError(error, 'getTranscript'); + } + }, + + // --- Private helpers --- + + _validateContainerId(containerId: string): void { /* ... */ }, + _validateContainerInfo(containerInfo: PragyaContainerResponse, urlField: string): void { /* ... */ }, + _decryptContent(encryptedContent: string, encryptionKeyUrl: string): Promise { + return this.webex.internal.encryption.decryptText(encryptionKeyUrl, encryptedContent); + }, + _handleError(error: any, methodName: string): Error { + if (error.statusCode === 404) { + const msg = methodName === 'getContainer' + ? ERROR_MESSAGES.CONTAINER_NOT_FOUND + : ERROR_MESSAGES.CONTENT_NOT_FOUND; + return new Error(msg); + } + if (error.statusCode === 403) return new Error(ERROR_MESSAGES.ACCESS_DENIED); + if (error.statusCode === 401) return new Error(ERROR_MESSAGES.AUTHENTICATION_FAILED); + return new Error(`${methodName} failed: ${error.message || 'Unknown error'}`); + }, +}); + +export default AISummary; +``` + +### 6.5 Usage Examples + +```typescript +// Step 1: Get call history (existing SDK API) +const callHistory = await callHistoryInstance.getCallHistoryData(10, 50); +const sessions = callHistory.data.userSessions; + +// Step 2: Find sessions with AI summaries +const sessionWithSummary = sessions.find( + (session) => session.extensionPayload?.callingContainerIds?.length > 0 +); + +if (!sessionWithSummary) { + console.log('No AI summaries available'); + return; +} + +// Step 3: Resolve the container (plugin flattens summaryData.data automatically) +const containerId = sessionWithSummary.extensionPayload.callingContainerIds[0]; +const container = await webex.internal.aisummary.getContainer({ containerId }); + +// Check if summary is available +if (container.summaryData.status !== 'Active') { + console.log('Summary is not yet ready'); + return; +} + +// Step 4: Fetch all summary content (note + shortNote + actionItems) in one call +const summary = await webex.internal.aisummary.getSummary({ containerInfo: container }); +console.log('Note:', summary.note); +console.log('Short Note:', summary.shortNote); +summary.actionItems.forEach((item, i) => { + console.log(`Action Item ${i + 1}: ${item.aiGeneratedContent}`); +}); + +// Step 5: Get transcript URL (or fetch full transcript) +const transcriptUrl = webex.internal.aisummary.getTranscriptUrl({ containerInfo: container }); +console.log('Transcript URL:', transcriptUrl); + +// Step 6: Fetch and decrypt full transcript +const transcript = await webex.internal.aisummary.getTranscript({ containerInfo: container }); +transcript.snippets.forEach((snippet) => { + console.log(`[${snippet.startTime}] ${snippet.speaker?.speakerName}: ${snippet.content}`); +}); +``` + +## 7. API Request/Response Details + +### 7.1 Pragya Container Lookup + +**Request:** +```http +GET /pragya/api/v1/containers/{containerId} HTTP/1.1 +Authorization: Bearer {user_access_token} +Accept: application/json +``` + +**Success Response (200 OK):** + +> The raw response nests URLs under `summaryData.data`. The plugin's `getContainer()` flattens this automatically. + +```json +{ + "id": "34125120-13b5-11f1-9b36-adb685725098", + "objectType": "callingAIContainer", + "memberships": { + "items": [{ "id": "...", "roles": ["OWNER"], "objectType": "containerMembership" }] + }, + "summaryData": { + "extensionId": "...", + "objectType": "extension", + "extensionType": "callingAISummary", + "data": { + "id": "...", + "objectType": "callingAISummary", + "status": "Active", + "summaryUrl": "https://aibridge-url/summaries/c635e870-...", + "transcriptUrl": "https://aibridge-url/summaries/c635e870-.../transcripts", + "summarizeAfterCall": true, + "aclUrl": "https://acl-a.wbx2.com/...", + "kmsResourceObjectUrl": "kms://kms-cisco.wbx2.com/resources/..." + } + }, + "encryptionKeyUrl": "kms://kms-cisco.wbx2.com/keys/897e4d2d-...", + "kmsResourceObjectUrl": "kms://kms-cisco.wbx2.com/resources/f7316435-...", + "aclUrl": "https://acl-a.wbx2.com/acl/api/v1/acls/78c4cd90-...", + "forkSessionId": "123e4567-...", + "callSessionId": "123e4567-...", + "ownerUserId": "123e4567-...", + "orgId": "123e4567-...", + "start": "2023-10-01T12:00:00Z", + "end": "2023-10-01T12:00:00Z" +} +``` + +**Error Responses:** +- `401 Unauthorized` - Invalid or expired token +- `403 Forbidden` - User not authorized to access this container +- `404 Not Found` - Container not found + +### 7.2 Summary Content (fetched via summaryUrl with fields query) + +The primary way to fetch all summary content is via `getSummary()`, which appends `?fields=note,shortnote,actionitems` to the `summaryUrl`. + +**Request:** +```http +GET {summaryData.summaryUrl}?fields=note,shortnote,actionitems HTTP/1.1 +Authorization: Bearer {user_access_token} +Accept: application/json +``` + +**Success Response (200 OK):** +```json +{ + "id": "10293-dk93-ddie-odir-did932j3kdde", + "keyUrl": "kms://kms-us-int.wbx2.com/keys/f19d4d28-...", + "note": { + "aiGeneratedContent": "" + }, + "shortnote": { + "aiGeneratedContent": "" + }, + "actionitems": { + "snippets": [ + { + "id": "394r0087-...", + "content": "edited version", + "aiGeneratedContent": "" + } + ] + }, + "links": [ + { "rel": "feedback", "href": "https://summarizer-r.wbx2.com/summarizer/api/v1/feedback/..." } + ] +} +``` + +### 7.3 Notes (standalone, fetched via notesUrl) + +**Request:** +```http +GET {summaryData.notesUrl} HTTP/1.1 +Authorization: Bearer {user_access_token} +Accept: application/json +``` + +**Success Response (200 OK):** +```json +{ + "id": "10293-dk93-ddie-odir-did932j3kdde", + "aiGeneratedContent": "", + "feedbackUrl": "https://summarizer-r.wbx2.com/summarizer/api/v1/feedback/report/...", + "keyUrl": "kms://kms-us-int.wbx2.com/keys/f19d4d28-..." +} +``` + +### 7.4 Action Items (standalone, fetched via actionItemsUrl) + +**Request:** +```http +GET {summaryData.actionItemsUrl} HTTP/1.1 +Authorization: Bearer {user_access_token} +Accept: application/json +``` + +**Success Response (200 OK):** +```json +[ + { + "id": "1234-dk93-ddie-odir-dk93dj33", + "keyUrl": "kms://kms-us-int.wbx2.com/keys/f19d4d28-...", + "snippets": [ + { + "id": "394r0087-...", + "content": "edited version", + "aiGeneratedContent": "" + } + ] + } +] +``` + +## 8. Encryption & Decryption + +### 8.1 Content Encryption + +All AI-generated content is encrypted using KMS (Key Management Service): + +- **Encryption Key**: The `encryptionKeyUrl` from the Pragya container response (format: `kms://kms-{region}.wbx2.com/keys/{key-id}`) +- **Encrypted Fields**: `aiGeneratedContent` in notes and action item snippets +- **Decryption**: Uses `@webex/internal-plugin-encryption` via `decryptText()` + +### 8.2 Decryption Pattern + +The SDK uses the existing `@webex/internal-plugin-encryption` plugin: + +```typescript +// Decrypt using the encryptionKeyUrl from the Pragya container response +const decryptedContent = await this.webex.internal.encryption.decryptText( + containerInfo.encryptionKeyUrl, + body.aiGeneratedContent +); +``` + +This is the same pattern used by existing plugins: + +**AI Assistant Plugin** (`internal-plugin-ai-assistant/src/utils.ts`): +```typescript +const decryptedValue = await webex.internal.encryption.decryptText( + encryptionKeyUrl, + encryptedValue +); +``` + +**Task Plugin** (`internal-plugin-task/src/helpers/decrypt.helper.js`): +```javascript +ctx.webex.internal.encryption.decryptText(key.uri || key, object[name]) +``` + +## 9. Error Handling + +### 9.1 Error Scenarios + +| Error Type | HTTP Status | SDK Error Message | Recovery Action | +|------------|-------------|-------------------|-----------------| +| Invalid Container ID | N/A (client) | "containerId is required and must be a non-empty string" | Validate input | +| Invalid Container Info | N/A (client) | "containerInfo with valid summaryData and encryptionKeyUrl is required" | Ensure getContainer was called first | +| Authentication Failed | 401 | "Authentication failed: Invalid or expired token" | Re-authenticate user | +| Access Denied | 403 | "Access denied: User not authorized to view this summary" | Check user permissions | +| Container Not Found | 404 | "Container not found" | Verify containerId from Janus | +| Content Not Found | 404 (non-getContainer) | "Summary content not available or expired" | Content may have been deleted or expired | +| Summary Not Ready | N/A | summaryData.status !== "Active" | Retry after delay | + +## 10. Security Considerations + +### 10.1 Authentication +- All API calls (Pragya and content URLs) require a valid user bearer token +- Token is automatically attached by the SDK's HTTP layer + +### 10.2 Authorization +- Only call participants or authorized users can access containers and summaries +- Org-level AI features must be enabled +- Per-call consent: AI assistant must have been enabled during the call + +### 10.3 Content Protection +- All AI-generated content is encrypted at rest with KMS +- `encryptionKeyUrl` from Pragya container is the decryption key +- HTTPS required for all API calls + +## 11. Testing Strategy + +### 11.1 Unit Tests + +```typescript +import {assert, expect} from '@webex/test-helper-chai'; +import MockWebex from '@webex/test-helper-mock-webex'; +import sinon from 'sinon'; +import AISummary from '@webex/internal-plugin-call-ai-summary'; +import config from '@webex/internal-plugin-call-ai-summary/src/config'; + +describe('internal-plugin-call-ai-summary', () => { + let webex; + + beforeEach(() => { + webex = MockWebex({ + children: { + aisummary: AISummary, + }, + }); + webex.config.aisummary = config.aisummary; + webex.internal.encryption = { + decryptText: sinon.stub().resolves('decrypted content'), + }; + }); + + describe('#getContainer', () => { + it('should resolve a Pragya container by ID', async () => { + const mockContainer = { + summaryData: { + status: 'Active', + summaryUrl: 'https://aibridge-url/summaries/abc123', + notesUrl: 'https://aibridge-url/summaries/abc123/notes', + actionItemsUrl: 'https://aibridge-url/summaries/abc123/action-items', + transcriptUrl: 'https://aibridge-url/summaries/abc123/transcripts', + summarizeAfterCall: true, + }, + encryptionKeyUrl: 'kms://kms.url/keys/key-id', + }; + + webex.request = sinon.stub().resolves({body: mockContainer}); + + const result = await webex.internal.aisummary.getContainer({ + containerId: 'container-123', + }); + + expect(result.summaryData.status).to.equal('Active'); + assert.calledWith(webex.request, sinon.match({ + method: 'GET', + service: 'pragya', + resource: 'containers/container-123', + })); + }); + + it('should throw for empty containerId', async () => { + await expect( + webex.internal.aisummary.getContainer({containerId: ''}) + ).to.be.rejectedWith('containerId is required'); + }); + }); + + describe('#getNotes', () => { + const mockContainerInfo = { + summaryData: { + notesUrl: 'https://aibridge-url/summaries/abc123/notes', + }, + encryptionKeyUrl: 'kms://kms.url/keys/key-id', + }; + + it('should fetch and decrypt notes', async () => { + webex.request = sinon.stub().resolves({ + body: { + id: 'note-id', + aiGeneratedContent: 'encrypted-notes', + feedbackUrl: 'https://feedback.url', + }, + }); + + const result = await webex.internal.aisummary.getNotes({ + containerInfo: mockContainerInfo, + }); + + expect(result.id).to.equal('note-id'); + expect(result.content).to.equal('decrypted content'); + expect(result.feedbackUrl).to.equal('https://feedback.url'); + assert.calledWith( + webex.internal.encryption.decryptText, + 'kms://kms.url/keys/key-id', + 'encrypted-notes' + ); + }); + + it('should throw when containerInfo is missing notesUrl', async () => { + await expect( + webex.internal.aisummary.getNotes({containerInfo: {summaryData: {}}}) + ).to.be.rejectedWith('containerInfo with valid summaryData'); + }); + }); + + describe('#getActionItems', () => { + const mockContainerInfo = { + summaryData: { + actionItemsUrl: 'https://aibridge-url/summaries/abc123/action-items', + }, + encryptionKeyUrl: 'kms://kms.url/keys/key-id', + }; + + it('should fetch and decrypt all action item snippets', async () => { + webex.request = sinon.stub().resolves({ + body: [{ + id: 'action-items-id', + keyUrl: 'kms://kms.url/keys/key-id', + snippets: [ + {id: 's1', aiGeneratedContent: 'encrypted-1'}, + {id: 's2', content: 'edited', aiGeneratedContent: 'encrypted-2'}, + ], + }], + }); + + webex.internal.encryption.decryptText + .onFirstCall().resolves('Decrypted item 1') + .onSecondCall().resolves('Decrypted item 2'); + + const result = await webex.internal.aisummary.getActionItems({ + containerInfo: mockContainerInfo, + }); + + expect(result.snippets).to.have.lengthOf(2); + expect(result.snippets[0].aiGeneratedContent).to.equal('Decrypted item 1'); + expect(result.snippets[1].aiGeneratedContent).to.equal('Decrypted item 2'); + expect(result.snippets[1].editedContent).to.equal('edited'); + }); + }); + + describe('#getTranscriptUrl', () => { + it('should return the transcript URL', () => { + const containerInfo = { + summaryData: { + transcriptUrl: 'https://aibridge-url/summaries/abc123/transcripts', + }, + encryptionKeyUrl: 'kms://kms.url/keys/key-id', + }; + + const url = webex.internal.aisummary.getTranscriptUrl({containerInfo}); + + expect(url).to.equal('https://aibridge-url/summaries/abc123/transcripts'); + }); + }); +}); +``` + +## 12. Modularity & Existing Code Impact + +### 12.1 Zero Changes to Existing Packages + +This plugin is fully self-contained. It does **not** require modifications to any existing package: + +| Concern | Approach | +|---------|----------| +| `UserSession` type in `@webex/calling` | **Not modified.** The plugin accepts a plain `containerId: string`. Consumers extract it from the Janus response at the application layer. The `UserSession` type update is a separate, optional task for the calling package team. | +| `packages/webex` bundle | **Not modified.** Consumers import `@webex/internal-plugin-call-ai-summary` directly, which self-registers via `registerInternalPlugin()`. No changes to the webex package index are needed. | +| `@webex/internal-plugin-encryption` | **Not modified.** Used as a runtime dependency via `this.webex.internal.encryption.decryptText()`. | + +### 12.2 Plugin Package Structure + +``` +packages/@webex/internal-plugin-call-ai-summary/ + src/ + index.ts # registerInternalPlugin('aisummary', ...) + ai-summary.ts # WebexPlugin.extend({...}) + config.ts # { aisummary: {} } + constants.ts # Service name, error messages + types.ts # All TypeScript interfaces + test/ + unit/ + spec/ + ai-summary.ts + data/ + responses.ts # Mock Pragya and content responses + package.json + jest.config.js + babel.config.js + .eslintrc.js +``` + +## 13. Dependencies + +### 13.1 Internal Dependencies + +| Package | Purpose | +|---------|---------| +| `@webex/webex-core` | Plugin infrastructure (`WebexPlugin`, `registerInternalPlugin`) | +| `@webex/internal-plugin-encryption` | Content decryption via `decryptText()` | + +### 13.2 External Service Dependencies + +| Service | Purpose | Discovery | +|---------|---------|-----------| +| **Janus** | Call history; provides `extensionPayload.callingContainerIds` | U2C: `serviceName: "janus"` | +| **Pragya** | Container metadata; provides content URLs and encryption key | U2C: `serviceName: "pragya"` | +| **Summary Content Endpoints** | Serve encrypted AI-generated content | Direct URLs from Pragya response | +| **KMS** | Encryption key management | Via `encryptionKeyUrl` from Pragya | diff --git a/packages/@webex/internal-plugin-call-ai-summary/babel.config.js b/packages/@webex/internal-plugin-call-ai-summary/babel.config.js new file mode 100644 index 00000000000..71a8b034b1f --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/babel.config.js @@ -0,0 +1,3 @@ +const babelConfigLegacy = require('@webex/babel-config-legacy'); + +module.exports = babelConfigLegacy; diff --git a/packages/@webex/internal-plugin-call-ai-summary/jest.config.js b/packages/@webex/internal-plugin-call-ai-summary/jest.config.js new file mode 100644 index 00000000000..0e9d38b401c --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/jest.config.js @@ -0,0 +1,3 @@ +const config = require('@webex/jest-config-legacy'); + +module.exports = config; diff --git a/packages/@webex/internal-plugin-call-ai-summary/package.json b/packages/@webex/internal-plugin-call-ai-summary/package.json new file mode 100644 index 00000000000..eebb7f2bc71 --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/package.json @@ -0,0 +1,46 @@ +{ + "name": "@webex/internal-plugin-call-ai-summary", + "description": "A Webex internal plugin for AI-generated call summary retrieval", + "license": "MIT", + "author": "Webex JS SDK Team", + "main": "dist/index.js", + "devMain": "src/index.ts", + "repository": { + "type": "git", + "url": "https://github.com/webex/webex-js-sdk.git", + "directory": "packages/@webex/internal-plugin-call-ai-summary" + }, + "engines": { + "node": ">=16" + }, + "browserify": { + "transform": [ + "babelify", + "envify" + ] + }, + "dependencies": { + "@webex/internal-plugin-encryption": "workspace:*", + "@webex/webex-core": "workspace:*" + }, + "devDependencies": { + "@babel/core": "^7.17.10", + "@webex/babel-config-legacy": "workspace:*", + "@webex/eslint-config-legacy": "workspace:*", + "@webex/jest-config-legacy": "workspace:*", + "@webex/legacy-tools": "workspace:*", + "@webex/test-helper-chai": "workspace:*", + "@webex/test-helper-mock-webex": "workspace:*", + "eslint": "^8.24.0", + "prettier": "^2.7.1", + "sinon": "^9.2.4" + }, + "scripts": { + "build": "yarn build:src", + "build:src": "webex-legacy-tools build -dest \"./dist\" -src \"./src\" -js -ts -maps", + "deploy:npm": "yarn npm publish", + "test": "yarn test:style && yarn test:unit", + "test:style": "eslint ./src/**/*.*", + "test:unit": "webex-legacy-tools test --unit --runner jest" + } +} diff --git a/packages/@webex/internal-plugin-call-ai-summary/src/ai-summary.ts b/packages/@webex/internal-plugin-call-ai-summary/src/ai-summary.ts new file mode 100644 index 00000000000..c9e6a2a23cd --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/src/ai-summary.ts @@ -0,0 +1,318 @@ +/*! + * Copyright (c) 2015-2025 Cisco Systems, Inc. See LICENSE file. + */ + +import {WebexPlugin} from '@webex/webex-core'; + +import {AI_SUMMARY_SERVICE, AI_SUMMARY_CONTAINERS_RESOURCE, ERROR_MESSAGES} from './constants'; +import type { + GetContainerOptions, + GetSummaryContentOptions, + PragyaContainerResponse, + SummaryContent, + SummaryNotes, + SummaryActionItems, + TranscriptContent, +} from './types'; + +const AISummary = WebexPlugin.extend({ + namespace: 'AISummary', + + /** + * Resolve a Pragya container by ID. + * Returns container metadata including summary URLs and encryption key. + * + * @param {GetContainerOptions} options + * @returns {Promise} + */ + getContainer(options: GetContainerOptions): Promise { + const {containerId} = options; + + this._validateContainerId(containerId); + + return this.webex + .request({ + method: 'GET', + service: AI_SUMMARY_SERVICE, + resource: `${AI_SUMMARY_CONTAINERS_RESOURCE}/${containerId}`, + }) + .then(({body}) => { + // Pragya API nests summary URLs under summaryData.data — flatten + // so consumers can access summaryData.summaryUrl directly. + if (body.summaryData?.data) { + body.summaryData = body.summaryData.data; + } + + return body; + }) + .catch((error) => { + this.logger.error('AISummary->getContainer failed', {error, containerId}); + throw this._handleError(error, 'getContainer'); + }); + }, + + /** + * Get AI-generated full summary for a call. + * Fetches from containerInfo.summaryData.summaryUrl and decrypts content. + * + * @param {GetSummaryContentOptions} options + * @returns {Promise} + */ + async getSummary(options: GetSummaryContentOptions): Promise { + const {containerInfo} = options; + + this._validateContainerInfo(containerInfo, 'summaryUrl'); + + try { + const {body} = await this.webex.request({ + method: 'GET', + uri: `${containerInfo.summaryData.summaryUrl}?fields=note,shortnote,actionitems`, + }); + + const keyUrl = body.keyUrl || containerInfo.encryptionKeyUrl; + + const decryptedNote = await this._decryptContent(body.note.aiGeneratedContent, keyUrl); + + const decryptedShortNote = await this._decryptContent( + body.shortnote.aiGeneratedContent, + keyUrl + ); + + const decryptedSnippets = await Promise.all( + (body.actionitems?.snippets || []).map(async (snippet: any) => { + const decryptedAiContent = await this._decryptContent(snippet.aiGeneratedContent, keyUrl); + + return { + id: snippet.id, + editedContent: snippet.content || undefined, + aiGeneratedContent: decryptedAiContent, + }; + }) + ); + + // feedbackUrl may be in the links array as rel="feedback" + const feedbackLink = (body.links || []).find((link: any) => link.rel === 'feedback'); + + return { + id: body.id, + note: decryptedNote, + shortNote: decryptedShortNote, + actionItems: decryptedSnippets, + feedbackUrl: feedbackLink?.href, + }; + } catch (error) { + this.logger.error('AISummary->getSummary failed', {error}); + throw this._handleError(error, 'getSummary'); + } + }, + + /** + * Get AI-generated notes for a call. + * Fetches from containerInfo.summaryData.notesUrl and decrypts content. + * + * @param {GetSummaryContentOptions} options + * @returns {Promise} + */ + async getNotes(options: GetSummaryContentOptions): Promise { + const {containerInfo} = options; + + this._validateContainerInfo(containerInfo, 'notesUrl'); + + try { + const {body} = await this.webex.request({ + method: 'GET', + uri: containerInfo.summaryData.notesUrl, + }); + + const keyUrl = body.keyUrl || containerInfo.encryptionKeyUrl; + + const decryptedContent = await this._decryptContent(body.aiGeneratedContent, keyUrl); + + return { + id: body.id, + content: decryptedContent, + feedbackUrl: body.feedbackUrl, + }; + } catch (error) { + this.logger.error('AISummary->getNotes failed', {error}); + throw this._handleError(error, 'getNotes'); + } + }, + + /** + * Get AI-generated action items for a call. + * Fetches from containerInfo.summaryData.actionItemsUrl and decrypts content. + * + * @param {GetSummaryContentOptions} options + * @returns {Promise} + */ + async getActionItems(options: GetSummaryContentOptions): Promise { + const {containerInfo} = options; + + this._validateContainerInfo(containerInfo, 'actionItemsUrl'); + + try { + const {body} = await this.webex.request({ + method: 'GET', + uri: containerInfo.summaryData.actionItemsUrl, + }); + + // Action items response is an array; take the first element + const actionItemsData = Array.isArray(body) ? body[0] : body; + + if (!actionItemsData) { + return {id: undefined, snippets: []}; + } + + const keyUrl = actionItemsData.keyUrl || containerInfo.encryptionKeyUrl; + + const decryptedSnippets = await Promise.all( + (actionItemsData.snippets || []).map(async (snippet: any) => { + const decryptedAiContent = await this._decryptContent(snippet.aiGeneratedContent, keyUrl); + + return { + id: snippet.id, + editedContent: snippet.content || undefined, + aiGeneratedContent: decryptedAiContent, + }; + }) + ); + + return { + id: actionItemsData.id, + snippets: decryptedSnippets, + feedbackUrl: actionItemsData.feedbackUrl, + }; + } catch (error) { + this.logger.error('AISummary->getActionItems failed', {error}); + throw this._handleError(error, 'getActionItems'); + } + }, + + /** + * Get the transcript URL for a call. + * Returns the URL string from the container info. + * + * @param {GetSummaryContentOptions} options + * @returns {string} + */ + getTranscriptUrl(options: GetSummaryContentOptions): string { + const {containerInfo} = options; + + this._validateContainerInfo(containerInfo, 'transcriptUrl'); + + return containerInfo.summaryData.transcriptUrl; + }, + + /** + * Get decrypted transcript for a call. + * Fetches from containerInfo.summaryData.transcriptUrl and decrypts each snippet. + * + * @param {GetSummaryContentOptions} options + * @returns {Promise} + */ + async getTranscript(options: GetSummaryContentOptions): Promise { + const {containerInfo} = options; + + this._validateContainerInfo(containerInfo, 'transcriptUrl'); + + try { + const {body} = await this.webex.request({ + method: 'GET', + uri: containerInfo.summaryData.transcriptUrl, + }); + + const keyUrl = body.keyUrl || containerInfo.encryptionKeyUrl; + + const decryptedSnippets = await Promise.all( + (body.transcriptSnippetList || []).map(async (snippet: any) => { + const decryptedContent = await this._decryptContent(snippet.content, keyUrl); + + return { + startTime: snippet.startTime, + endTime: snippet.endTime, + content: decryptedContent, + audioCSI: snippet.audioCSI, + speaker: snippet.speaker, + }; + }) + ); + + return { + id: body.id, + totalCount: body.totalCount, + snippets: decryptedSnippets, + }; + } catch (error) { + this.logger.error('AISummary->getTranscript failed', {error}); + throw this._handleError(error, 'getTranscript'); + } + }, + + /** + * Validate containerId parameter. + * @param {string} containerId - The container ID to validate. + * @returns {void} + * @private + */ + _validateContainerId(containerId: string): void { + if (!containerId || typeof containerId !== 'string' || containerId.trim().length === 0) { + throw new Error(ERROR_MESSAGES.INVALID_CONTAINER_ID); + } + }, + + /** + * Validate containerInfo has the required URL field and encryption key. + * @param {PragyaContainerResponse} containerInfo - The container info to validate. + * @param {string} urlField - The summaryData field name to check. + * @returns {void} + * @private + */ + _validateContainerInfo(containerInfo: PragyaContainerResponse, urlField: string): void { + if (!containerInfo?.summaryData?.[urlField] || !containerInfo?.encryptionKeyUrl) { + throw new Error(ERROR_MESSAGES.INVALID_CONTAINER_INFO); + } + }, + + /** + * Decrypt encrypted content using KMS. + * Delegates to the internal encryption plugin. + * @param {string} encryptedContent - The encrypted text (JWE format). + * @param {string} encryptionKeyUrl - KMS key URL. + * @returns {Promise} Decrypted plaintext. + * @private + */ + _decryptContent(encryptedContent: string, encryptionKeyUrl: string): Promise { + return this.webex.internal.encryption.decryptText(encryptionKeyUrl, encryptedContent); + }, + + /** + * Handle and normalize errors. + * @param {object} error - The error object from the request. + * @param {string} methodName - The name of the calling method. + * @returns {Error} A normalized Error instance. + * @private + */ + _handleError(error: any, methodName: string): Error { + if (error.statusCode === 404) { + const message = + methodName === 'getContainer' + ? ERROR_MESSAGES.CONTAINER_NOT_FOUND + : ERROR_MESSAGES.CONTENT_NOT_FOUND; + + return new Error(message); + } + + if (error.statusCode === 403) { + return new Error(ERROR_MESSAGES.ACCESS_DENIED); + } + + if (error.statusCode === 401) { + return new Error(ERROR_MESSAGES.AUTHENTICATION_FAILED); + } + + return new Error(`${methodName} failed: ${error.message || 'Unknown error'}`); + }, +}); + +export default AISummary; diff --git a/packages/@webex/internal-plugin-call-ai-summary/src/config.ts b/packages/@webex/internal-plugin-call-ai-summary/src/config.ts new file mode 100644 index 00000000000..3285944abd9 --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/src/config.ts @@ -0,0 +1,7 @@ +/*! + * Copyright (c) 2015-2025 Cisco Systems, Inc. See LICENSE file. + */ + +export default { + aisummary: {}, +}; diff --git a/packages/@webex/internal-plugin-call-ai-summary/src/constants.ts b/packages/@webex/internal-plugin-call-ai-summary/src/constants.ts new file mode 100644 index 00000000000..cedeb65f39e --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/src/constants.ts @@ -0,0 +1,19 @@ +/*! + * Copyright (c) 2015-2025 Cisco Systems, Inc. See LICENSE file. + */ + +export const AI_SUMMARY_SERVICE = 'pragya'; +export const AI_SUMMARY_CONTAINERS_RESOURCE = 'containers'; + +export const SUMMARY_STATUSES = { + ACTIVE: 'Active', +} as const; + +export const ERROR_MESSAGES = { + INVALID_CONTAINER_ID: 'containerId is required and must be a non-empty string', + INVALID_CONTAINER_INFO: 'containerInfo with valid summaryData and encryptionKeyUrl is required', + CONTAINER_NOT_FOUND: 'Container not found', + CONTENT_NOT_FOUND: 'Summary content not available or expired', + ACCESS_DENIED: 'Access denied: User not authorized to view this summary', + AUTHENTICATION_FAILED: 'Authentication failed: Invalid or expired token', +} as const; diff --git a/packages/@webex/internal-plugin-call-ai-summary/src/index.ts b/packages/@webex/internal-plugin-call-ai-summary/src/index.ts new file mode 100644 index 00000000000..16c8f7347fb --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/src/index.ts @@ -0,0 +1,13 @@ +/*! + * Copyright (c) 2015-2025 Cisco Systems, Inc. See LICENSE file. + */ + +import '@webex/internal-plugin-encryption'; +import {registerInternalPlugin} from '@webex/webex-core'; + +import AISummary from './ai-summary'; +import config from './config'; + +registerInternalPlugin('aisummary', AISummary, {config}); + +export {default} from './ai-summary'; diff --git a/packages/@webex/internal-plugin-call-ai-summary/src/manual-integration-test.js b/packages/@webex/internal-plugin-call-ai-summary/src/manual-integration-test.js new file mode 100644 index 00000000000..c1cadc30ec9 --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/src/manual-integration-test.js @@ -0,0 +1,129 @@ +/*! + * Copyright (c) 2015-2025 Cisco Systems, Inc. See LICENSE file. + * + * Manual integration test for internal-plugin-call-ai-summary + * Tests the full flow using the SDK service catalog (WDM): + * device.register() -> getContainer -> getSummary (with KMS decryption) + * + * The SDK resolves `service: 'pragya'` to the correct base URL via the + * service catalog populated during device registration. + * + * Usage: + * WEBEX_TOKEN='' node src/manual-integration-test.js + */ + +/* eslint-disable no-console, require-jsdoc */ + +require('@webex/internal-plugin-call-ai-summary'); + +const WebexCore = require('@webex/webex-core').default; + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- +const WEBEX_TOKEN = process.env.WEBEX_TOKEN || ''; +const CONTAINER_ID = process.env.CONTAINER_ID || ''; + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- +async function main() { + console.log('=== Step 1: Create WebexCore ===\n'); + const webex = new WebexCore({ + credentials: { + access_token: WEBEX_TOKEN, + }, + }); + + // Step 2: Register device to populate service catalog + console.log('=== Step 2: Register device (WDM) ===\n'); + await webex.internal.device.register(); + console.log('Device registered successfully.'); + console.log('Device URL:', webex.internal.device.url); + + // Log the pragya service URL from the service catalog + try { + const pragyaUrl = webex.internal.services.get('pragya'); + console.log('Pragya service URL (from catalog):', pragyaUrl); + } catch (e) { + console.log('Could not resolve pragya from service catalog:', e.message); + } + + // Step 3: Get container via plugin (uses service: 'pragya' + resource) + console.log('\n=== Step 3: getContainer via plugin ===\n'); + const container = await webex.internal.aisummary.getContainer({ + containerId: CONTAINER_ID, + }); + console.log( + 'Container Info:', + JSON.stringify( + { + id: container.id, + objectType: container.objectType, + encryptionKeyUrl: container.encryptionKeyUrl, + summaryUrl: container.summaryData?.summaryUrl, + transcriptUrl: container.summaryData?.transcriptUrl, + }, + null, + 2 + ) + ); + + // Step 4: Call getSummary via plugin (fetches + decrypts all content) + console.log('\n=== Step 4: getSummary via plugin ===\n'); + const summaryResult = await webex.internal.aisummary.getSummary({ + containerInfo: container, + }); + + console.log('=== getSummary return structure ==='); + const noteStr = summaryResult.note || ''; + const shortNoteStr = summaryResult.shortNote || ''; + const truncNote = noteStr.length > 200 ? `${noteStr.substring(0, 200)}...` : noteStr; + const truncShort = + shortNoteStr.length > 200 ? `${shortNoteStr.substring(0, 200)}...` : shortNoteStr; + const truncated = { + id: summaryResult.id, + note: truncNote, + shortNote: truncShort, + actionItems: (summaryResult.actionItems || []).map((item) => { + const content = item.aiGeneratedContent || ''; + const truncContent = content.length > 100 ? `${content.substring(0, 100)}...` : content; + + return { + id: item.id, + aiGeneratedContent: truncContent, + editedContent: item.editedContent, + }; + }), + feedbackUrl: summaryResult.feedbackUrl, + }; + console.log(JSON.stringify(truncated, null, 2)); + + // Step 5: Get transcript URL via plugin + console.log('\n=== Step 5: getTranscriptUrl via plugin ===\n'); + const transcriptUrl = webex.internal.aisummary.getTranscriptUrl({ + containerInfo: container, + }); + console.log('Transcript URL:', transcriptUrl); + + // Step 6: Fetch transcript content + console.log('\n=== Step 6: Fetch transcript content ===\n'); + try { + const {body: transcriptBody} = await webex.request({ + method: 'GET', + uri: `${transcriptUrl}?fields=id,content`, + }); + console.log('Transcript response keys:', Object.keys(transcriptBody)); + console.log(JSON.stringify(transcriptBody, null, 2).substring(0, 500)); + } catch (err) { + console.error('Transcript fetch failed:', err.message); + } + + console.log('\nDone.'); + process.exit(0); +} + +main().catch((err) => { + console.error('FATAL:', err.message || err); + process.exit(1); +}); diff --git a/packages/@webex/internal-plugin-call-ai-summary/src/manual-pragya-api-test.js b/packages/@webex/internal-plugin-call-ai-summary/src/manual-pragya-api-test.js new file mode 100644 index 00000000000..08878f1772d --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/src/manual-pragya-api-test.js @@ -0,0 +1,166 @@ +/*! + * Copyright (c) 2015-2025 Cisco Systems, Inc. See LICENSE file. + * + * Manual test for internal-plugin-call-ai-summary + * + * Usage: + * WEBEX_TOKEN='' node manual-test.js + * + * Or paste your token directly into WEBEX_TOKEN below. + */ + +/* eslint-disable no-console, require-jsdoc */ + +require('@webex/internal-plugin-call-ai-summary'); + +const WebexCore = require('@webex/webex-core').default; + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- +const WEBEX_TOKEN = process.env.WEBEX_TOKEN || ''; +const CONTAINER_ID = ''; +const PRAGYA_BASE_URL = ''; + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- +async function main() { + if (WEBEX_TOKEN === '') { + console.error('ERROR: Set WEBEX_TOKEN env var or paste your token in the script.'); + process.exit(1); + } + + const webex = new WebexCore({ + credentials: { + access_token: WEBEX_TOKEN, + }, + }); + + console.log('--- Fetching container', CONTAINER_ID, '(SDK auth) ---\n'); + + const response = await webex.request({ + method: 'GET', + uri: `${PRAGYA_BASE_URL}/containers/${CONTAINER_ID}`, + headers: { + 'content-type': 'application/json', + }, + }); + + const container = response.body; + let passed = 0; + let failed = 0; + + function check(label, actual, expected) { + if (actual === expected) { + console.log(` PASS: ${label}`); + passed += 1; + } else { + console.log( + ` FAIL: ${label} — expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}` + ); + failed += 1; + } + } + + function checkExists(label, value) { + if (value !== undefined && value !== null) { + console.log(` PASS: ${label} exists`); + passed += 1; + } else { + console.log(` FAIL: ${label} is missing`); + failed += 1; + } + } + + function checkType(label, value, type) { + const actual = Array.isArray(value) ? 'array' : typeof value; + const expected = type; + if (type === 'array' ? Array.isArray(value) : actual === expected) { + console.log(` PASS: ${label} is ${type}`); + passed += 1; + } else { + console.log(` FAIL: ${label} — expected ${type}, got ${actual}`); + failed += 1; + } + } + + function checkMatch(label, value, regex) { + if (regex.test(value)) { + console.log(` PASS: ${label} matches ${regex}`); + passed += 1; + } else { + console.log(` FAIL: ${label} — ${JSON.stringify(value)} does not match ${regex}`); + failed += 1; + } + } + + // --- Top-level container --- + console.log('\n--- Top-level container ---'); + checkType('container', container, 'object'); + check('container.id', container.id, CONTAINER_ID); + check('container.objectType', container.objectType, 'callingAIContainer'); + checkExists('container.summaryData', container.summaryData); + checkExists('container.memberships', container.memberships); + checkMatch('container.encryptionKeyUrl', container.encryptionKeyUrl, /^kms:\/\//); + checkMatch('container.kmsResourceObjectUrl', container.kmsResourceObjectUrl, /^kms:\/\//); + checkMatch('container.aclUrl', container.aclUrl, /^https?:\/\//); + checkExists('container.start', container.start); + checkExists('container.end', container.end); + checkExists('container.forkSessionId', container.forkSessionId); + checkExists('container.callSessionId', container.callSessionId); + checkExists('container.ownerUserId', container.ownerUserId); + checkExists('container.orgId', container.orgId); + + // --- summaryData --- + console.log('\n--- summaryData ---'); + const {summaryData} = container; + checkType('summaryData', summaryData, 'object'); + checkExists('summaryData.extensionId', summaryData.extensionId); + check('summaryData.objectType', summaryData.objectType, 'extension'); + check('summaryData.extensionType', summaryData.extensionType, 'callingAISummary'); + + // --- summaryData.data --- + console.log('\n--- summaryData.data ---'); + const summaryDataData = summaryData.data; + checkType('summaryData.data', summaryDataData, 'object'); + checkExists('summaryData.data.id', summaryDataData.id); + check('summaryData.data.objectType', summaryDataData.objectType, 'callingAISummary'); + checkExists('summaryData.data.status', summaryDataData.status); + checkMatch('summaryData.data.summaryUrl', summaryDataData.summaryUrl, /^https?:\/\//); + checkMatch('summaryData.data.transcriptUrl', summaryDataData.transcriptUrl, /^https?:\/\//); + checkMatch('summaryData.data.aclUrl', summaryDataData.aclUrl, /^https?:\/\//); + checkMatch( + 'summaryData.data.kmsResourceObjectUrl', + summaryDataData.kmsResourceObjectUrl, + /^kms:\/\// + ); + checkType('summaryData.data.summarizeAfterCall', summaryDataData.summarizeAfterCall, 'boolean'); + checkType('summaryData.data.contentRetention', summaryDataData.contentRetention, 'object'); + + // --- memberships --- + console.log('\n--- memberships ---'); + const {memberships} = container; + checkType('memberships', memberships, 'object'); + checkType('memberships.items', memberships.items, 'array'); + if (memberships.items.length > 0) { + console.log(` PASS: memberships.items has ${memberships.items.length} member(s)`); + passed += 1; + const first = memberships.items[0]; + checkExists('memberships.items[0].id', first.id); + checkType('memberships.items[0].roles', first.roles, 'array'); + check('memberships.items[0].objectType', first.objectType, 'containerMembership'); + } else { + console.log(' FAIL: memberships.items is empty'); + failed += 1; + } + + // --- Summary --- + console.log(`\n=== ${passed} passed, ${failed} failed ===`); + process.exit(failed > 0 ? 1 : 0); +} + +main().catch((err) => { + console.error('FAILED:', err.message || err); + process.exit(1); +}); diff --git a/packages/@webex/internal-plugin-call-ai-summary/src/types.ts b/packages/@webex/internal-plugin-call-ai-summary/src/types.ts new file mode 100644 index 00000000000..ce4fdfc4c14 --- /dev/null +++ b/packages/@webex/internal-plugin-call-ai-summary/src/types.ts @@ -0,0 +1,154 @@ +/*! + * Copyright (c) 2015-2025 Cisco Systems, Inc. See LICENSE file. + */ + +// --- Pragya Response DTOs --- + +/** + * Summary data URLs from a Pragya container. + */ +export interface PragyaSummaryData { + /** Status of the summary (e.g., "Active") */ + status: string; + /** Full summary URL (AI Bridge) */ + summaryUrl: string; + /** Transcript URL (AI Bridge) */ + transcriptUrl: string; + /** Whether summarization runs after call ends */ + summarizeAfterCall: boolean; + /** Notes-specific URL (may not be present in all API versions) */ + notesUrl?: string; + /** Action items URL (may not be present in all API versions) */ + actionItemsUrl?: string; +} + +/** + * Complete Pragya container response. + */ +export interface PragyaContainerResponse { + /** Summary data with content URLs */ + summaryData: PragyaSummaryData; + /** KMS encryption key URL for decrypting content */ + encryptionKeyUrl: string; + /** KMS resource object URL */ + kmsResourceObjectUrl: string; + /** ACL URL for access control */ + aclUrl: string; + /** Fork session ID */ + forkSessionId: string; + /** Call session ID */ + callSessionId: string; + /** Owner user ID */ + ownerUserId: string; + /** Organization ID */ + orgId: string; + /** Call start time */ + start: string; + /** Call end time */ + end: string; +} + +// --- Request DTOs --- + +/** + * Options for resolving a Pragya container. + */ +export interface GetContainerOptions { + /** Pragya container ID (from Janus extensionPayload.callingContainerIds) */ + containerId: string; +} + +/** + * Options for fetching summary content. + * Requires the resolved Pragya container info. + */ +export interface GetSummaryContentOptions { + /** The resolved Pragya container response */ + containerInfo: PragyaContainerResponse; +} + +// --- Summary Response DTOs --- + +/** + * Decrypted AI-generated summary content. + * Contains all three content types returned by the summary API. + */ +export interface SummaryContent { + /** Unique identifier */ + id: string; + /** Decrypted full note content */ + note: string; + /** Decrypted short note content */ + shortNote: string; + /** Decrypted action item snippets */ + actionItems: ActionItemSnippet[]; + /** Feedback URL (if available) */ + feedbackUrl?: string; +} + +/** + * Decrypted AI-generated notes. + */ +export interface SummaryNotes { + /** Unique identifier */ + id: string; + /** Decrypted notes content */ + content: string; + /** Feedback URL (if available) */ + feedbackUrl?: string; +} + +/** + * Single action item snippet. + */ +export interface ActionItemSnippet { + /** Unique identifier */ + id: string; + /** User-edited version (if available) */ + editedContent?: string; + /** Decrypted AI-generated content */ + aiGeneratedContent: string; +} + +/** + * Decrypted AI-generated action items. + */ +export interface SummaryActionItems { + /** Unique identifier (absent when no action items exist) */ + id?: string; + /** Array of action item snippets */ + snippets: ActionItemSnippet[]; + /** Feedback URL (if available) */ + feedbackUrl?: string; +} + +/** + * Single decrypted transcript snippet. + */ +export interface TranscriptSnippet { + /** Start time in milliseconds */ + startTime: string; + /** End time in milliseconds */ + endTime: string; + /** Decrypted transcript content */ + content: string; + /** Audio CSI identifier */ + audioCSI?: string; + /** Speaker information */ + speaker?: { + speakerName: string; + speakerId: string; + }; +} + +/** + * Decrypted transcript response. + */ +export interface TranscriptContent { + /** Unique identifier */ + id: string; + /** Total number of snippets */ + totalCount: number; + /** Decrypted transcript snippets */ + snippets: TranscriptSnippet[]; +} diff --git a/yarn.lock b/yarn.lock index c00ce400761..a624aa069c0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7853,6 +7853,25 @@ __metadata: languageName: unknown linkType: soft +"@webex/internal-plugin-call-ai-summary@workspace:packages/@webex/internal-plugin-call-ai-summary": + version: 0.0.0-use.local + resolution: "@webex/internal-plugin-call-ai-summary@workspace:packages/@webex/internal-plugin-call-ai-summary" + dependencies: + "@babel/core": ^7.17.10 + "@webex/babel-config-legacy": "workspace:*" + "@webex/eslint-config-legacy": "workspace:*" + "@webex/internal-plugin-encryption": "workspace:*" + "@webex/jest-config-legacy": "workspace:*" + "@webex/legacy-tools": "workspace:*" + "@webex/test-helper-chai": "workspace:*" + "@webex/test-helper-mock-webex": "workspace:*" + "@webex/webex-core": "workspace:*" + eslint: ^8.24.0 + prettier: ^2.7.1 + sinon: ^9.2.4 + languageName: unknown + linkType: soft + "@webex/internal-plugin-conversation@workspace:*, @webex/internal-plugin-conversation@workspace:packages/@webex/internal-plugin-conversation": version: 0.0.0-use.local resolution: "@webex/internal-plugin-conversation@workspace:packages/@webex/internal-plugin-conversation" From c0725781878f1aff73d3dd6b1d84e55905a0ba51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edmond=20Vuji=C4=87i?= <67634227+edvujic@users.noreply.github.com> Date: Wed, 25 Mar 2026 10:13:39 +0100 Subject: [PATCH 16/28] fix(plugin-meetings): remove JMP deduplication logic (#4773) Co-authored-by: evujici --- .../src/multistream/mediaRequestManager.ts | 49 +------------------ .../src/reconnection-manager/index.ts | 1 - .../spec/multistream/mediaRequestManager.ts | 6 --- .../unit/spec/reconnection-manager/index.js | 12 ++--- 4 files changed, 6 insertions(+), 62 deletions(-) diff --git a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts index 9ebb7a9fae4..f745c8c2db8 100644 --- a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts +++ b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts @@ -7,7 +7,7 @@ import { RecommendedOpusBitrates, SupportedResolution, } from '@webex/internal-media-core'; -import {cloneDeepWith, debounce, isEmpty} from 'lodash'; +import {cloneDeepWith, debounce} from 'lodash'; import LoggerProxy from '../common/logs/logger-proxy'; @@ -49,8 +49,6 @@ export default class MediaRequestManager { private debouncedSourceUpdateListener: () => void; - private previousStreamRequests: Array = []; - private trimRequestsToNumOfSources: boolean; private numTotalSources: number; private numLiveSources: number; @@ -105,36 +103,6 @@ export default class MediaRequestManager { } } - /** - * Returns true if two stream requests are the same, false otherwise. - * - * @param {StreamRequest} streamRequestA - Stream request A for comparison. - * @param {StreamRequest} streamRequestB - Stream request B for comparison. - * @returns {boolean} - Whether they are equal. - */ - // eslint-disable-next-line class-methods-use-this - public isEqual(streamRequestA: StreamRequest, streamRequestB: StreamRequest) { - return ( - JSON.stringify(streamRequestA._toJmpStreamRequest()) === - JSON.stringify(streamRequestB._toJmpStreamRequest()) - ); - } - - /** - * Compares new stream requests to previous ones and determines - * if they are the same. - * - * @param {StreamRequest[]} newRequests - Array with new requests. - * @returns {boolean} - True if they are equal, false otherwise. - */ - private checkIsNewRequestsEqualToPrev(newRequests: StreamRequest[]) { - return ( - !isEmpty(this.previousStreamRequests) && - this.previousStreamRequests.length === newRequests.length && - this.previousStreamRequests.every((req, idx) => this.isEqual(req, newRequests[idx])) - ); - } - /** * Returns the maxPayloadBitsPerSecond per Stream * @@ -166,15 +134,6 @@ export default class MediaRequestManager { return 0; } - /** - * Clears the previous stream requests. - * - * @returns {void} - */ - public clearPreviousRequests(): void { - this.previousStreamRequests = []; - } - /** Modifies the passed in clientRequests and makes sure that in total they don't ask * for more streams than there are available. * @@ -307,12 +266,8 @@ export default class MediaRequestManager { streamRequests.push(streamRequest); }); - if (this.checkIsNewRequestsEqualToPrev(streamRequests)) { - return; - } - - this.previousStreamRequests = [...streamRequests]; this.sendMediaRequestsCallback(streamRequests); + LoggerProxy.logger.info(`multistream:sendRequests --> media requests sent. `); } public addRequest(mediaRequest: Omit, commit = true): MediaRequestId { diff --git a/packages/@webex/plugin-meetings/src/reconnection-manager/index.ts b/packages/@webex/plugin-meetings/src/reconnection-manager/index.ts index 77f1a0f0048..5dc6ca5b9d0 100644 --- a/packages/@webex/plugin-meetings/src/reconnection-manager/index.ts +++ b/packages/@webex/plugin-meetings/src/reconnection-manager/index.ts @@ -609,7 +609,6 @@ export default class ReconnectionManager { if (this.meeting.isMultistream) { Object.values(this.meeting.mediaRequestManagers).forEach( (mediaRequestManager: MediaRequestManager) => { - mediaRequestManager.clearPreviousRequests(); mediaRequestManager.commit(); } ); diff --git a/packages/@webex/plugin-meetings/test/unit/spec/multistream/mediaRequestManager.ts b/packages/@webex/plugin-meetings/test/unit/spec/multistream/mediaRequestManager.ts index 1a4a3409bae..71af39060d7 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/multistream/mediaRequestManager.ts +++ b/packages/@webex/plugin-meetings/test/unit/spec/multistream/mediaRequestManager.ts @@ -655,12 +655,6 @@ describe('MediaRequestManager', () => { }, ]); - // check that when calling commit() - // all requests are not re-sent again (avoid duplicate requests) - mediaRequestManager.commit(); - - assert.notCalled(sendMediaRequestsCallback); - // now reset everything mediaRequestManager.reset(); diff --git a/packages/@webex/plugin-meetings/test/unit/spec/reconnection-manager/index.js b/packages/@webex/plugin-meetings/test/unit/spec/reconnection-manager/index.js index a831fb8b71c..e72b16b4464 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/reconnection-manager/index.js +++ b/packages/@webex/plugin-meetings/test/unit/spec/reconnection-manager/index.js @@ -54,8 +54,8 @@ describe('plugin-meetings', () => { webrtcMediaConnection: fakeMediaConnection, }, mediaRequestManagers: { - audio: {commit: sinon.stub(), clearPreviousRequests: sinon.stub()}, - video: {commit: sinon.stub(), clearPreviousRequests: sinon.stub()}, + audio: {commit: sinon.stub()}, + video: {commit: sinon.stub()}, }, roap: { doTurnDiscovery: sinon.stub().resolves({ @@ -179,26 +179,22 @@ describe('plugin-meetings', () => { }); }); - it('does not clear previous requests and re-request media for non-multistream meetings', async () => { + it('does not re-request media for non-multistream meetings', async () => { fakeMeeting.isMultistream = false; const rm = new ReconnectionManager(fakeMeeting); await rm.reconnect(); - assert.notCalled(fakeMeeting.mediaRequestManagers.audio.clearPreviousRequests); - assert.notCalled(fakeMeeting.mediaRequestManagers.video.clearPreviousRequests); assert.notCalled(fakeMeeting.mediaRequestManagers.audio.commit); assert.notCalled(fakeMeeting.mediaRequestManagers.video.commit); }); - it('does clear previous requests and re-request media for multistream meetings', async () => { + it('does re-request media for multistream meetings', async () => { fakeMeeting.isMultistream = true; const rm = new ReconnectionManager(fakeMeeting); await rm.reconnect(); - assert.calledOnce(fakeMeeting.mediaRequestManagers.audio.clearPreviousRequests); - assert.calledOnce(fakeMeeting.mediaRequestManagers.video.clearPreviousRequests); assert.calledOnce(fakeMeeting.mediaRequestManagers.audio.commit); assert.calledOnce(fakeMeeting.mediaRequestManagers.video.commit); }); From b8e38bd399d83db239c61f93870e9aba37d6e253 Mon Sep 17 00:00:00 2001 From: Tianhui-Han <133636998+Tianhui-Han@users.noreply.github.com> Date: Wed, 25 Mar 2026 18:40:56 +0800 Subject: [PATCH 17/28] feat(datachannel): add update subchannel subscriptions function (#4749) Co-authored-by: mickelr <121160648+mickelr@users.noreply.github.com> --- packages/@webex/internal-plugin-llm/README.md | 6 +- .../internal-plugin-llm/src/constants.ts | 8 + .../@webex/internal-plugin-llm/src/llm.ts | 70 ++++- .../internal-plugin-llm/src/llm.types.ts | 4 +- .../internal-plugin-llm/test/unit/spec/llm.js | 285 ++++++++++++------ .../internal-plugin-voicea/src/voicea.ts | 71 +++++ .../test/unit/spec/voicea.js | 208 ++++++++++++- packages/@webex/plugin-meetings/package.json | 1 + .../src/interceptors/dataChannelAuthToken.ts | 28 ++ .../plugin-meetings/src/interceptors/utils.ts | 16 + .../plugin-meetings/src/meeting/index.ts | 10 +- .../plugin-meetings/src/meeting/request.ts | 8 +- .../plugin-meetings/src/webinar/index.ts | 5 + .../spec/interceptors/dataChannelAuthToken.ts | 69 +++++ .../test/unit/spec/interceptors/utils.ts | 75 +++++ .../test/unit/spec/meeting/index.js | 12 +- .../test/unit/spec/meeting/request.js | 30 +- .../test/unit/spec/webinar/index.ts | 18 ++ yarn.lock | 8 + 19 files changed, 793 insertions(+), 139 deletions(-) create mode 100644 packages/@webex/plugin-meetings/src/interceptors/utils.ts create mode 100644 packages/@webex/plugin-meetings/test/unit/spec/interceptors/utils.ts diff --git a/packages/@webex/internal-plugin-llm/README.md b/packages/@webex/internal-plugin-llm/README.md index de0a161c117..6ab304f4fb7 100644 --- a/packages/@webex/internal-plugin-llm/README.md +++ b/packages/@webex/internal-plugin-llm/README.md @@ -79,8 +79,8 @@ llm.on(`event:${sessionB}`, (envelope) => { }); // Optional: store/retrieve token by token type -webex.internal.llm.setDatachannelToken(datachannelToken, 'DEFAULT'); -webex.internal.llm.getDatachannelToken('DEFAULT'); +webex.internal.llm.setDatachannelToken(datachannelToken, 'llm-default-session'); +webex.internal.llm.getDatachannelToken('llm-default-session'); // Optional: inject token refresh handler webex.internal.llm.setRefreshHandler(async () => { @@ -88,7 +88,7 @@ webex.internal.llm.setRefreshHandler(async () => { return { body: { datachannelToken: '', - datachannelTokenType: 'DEFAULT', + datachannelTokenType: 'llm-default-session', }, }; }); diff --git a/packages/@webex/internal-plugin-llm/src/constants.ts b/packages/@webex/internal-plugin-llm/src/constants.ts index 45b0c5b43d7..7d267df4392 100644 --- a/packages/@webex/internal-plugin-llm/src/constants.ts +++ b/packages/@webex/internal-plugin-llm/src/constants.ts @@ -4,3 +4,11 @@ export const LLM = 'llm'; export const LLM_DEFAULT_SESSION = 'llm-default-session'; export const DATA_CHANNEL_WITH_JWT_TOKEN = 'data-channel-with-jwt-token'; + +export const SUBSCRIPTION_AWARE_SUBCHANNELS_PARAM = 'subscriptionAwareSubchannels'; + +export const DATA_CHNANEL_TYPE = { + TRANSCRIPTION: 'transcription', +}; + +export const AWARE_DATA_CHANNEL = [DATA_CHNANEL_TYPE.TRANSCRIPTION]; diff --git a/packages/@webex/internal-plugin-llm/src/llm.ts b/packages/@webex/internal-plugin-llm/src/llm.ts index 57ee462324e..0c1ac265e4e 100644 --- a/packages/@webex/internal-plugin-llm/src/llm.ts +++ b/packages/@webex/internal-plugin-llm/src/llm.ts @@ -2,8 +2,14 @@ import Mercury from '@webex/internal-plugin-mercury'; -import {LLM, DATA_CHANNEL_WITH_JWT_TOKEN, LLM_DEFAULT_SESSION} from './constants'; // eslint-disable-next-line no-unused-vars +import { + LLM, + DATA_CHANNEL_WITH_JWT_TOKEN, + AWARE_DATA_CHANNEL, + SUBSCRIPTION_AWARE_SUBCHANNELS_PARAM, + LLM_DEFAULT_SESSION, +} from './constants'; import {ILLMChannel, DataChannelTokenType} from './llm.types'; export const config = { @@ -118,7 +124,7 @@ export default class LLMChannel extends (Mercury as any) implements ILLMChannel datachannelToken?: string, sessionId: string = LLM_DEFAULT_SESSION ): Promise => - this.register(datachannelUrl, datachannelToken, sessionId).then(() => { + this.register(datachannelUrl, datachannelToken, sessionId).then(async () => { if (!locusUrl || !datachannelUrl) return undefined; // Get or create connection data @@ -128,7 +134,13 @@ export default class LLMChannel extends (Mercury as any) implements ILLMChannel sessionData.datachannelToken = datachannelToken; this.connections.set(sessionId, sessionData); - return this.connect(sessionData.webSocketUrl, sessionId); + const isDataChannelTokenEnabled = await this.isDataChannelTokenEnabled(); + + const connectUrl = isDataChannelTokenEnabled + ? LLMChannel.buildUrlWithAwareSubchannels(sessionData.webSocketUrl, AWARE_DATA_CHANNEL) + : sessionData.webSocketUrl; + + return this.connect(connectUrl, sessionId); }); /** @@ -180,7 +192,9 @@ export default class LLMChannel extends (Mercury as any) implements ILLMChannel * @param {DataChannelTokenType} dataChannelTokenType * @returns {string} data channel token */ - public getDatachannelToken = (dataChannelTokenType: DataChannelTokenType): string => { + public getDatachannelToken = ( + dataChannelTokenType: DataChannelTokenType = DataChannelTokenType.Default + ): string => { return this.datachannelTokens[dataChannelTokenType]; }; @@ -192,11 +206,23 @@ export default class LLMChannel extends (Mercury as any) implements ILLMChannel */ public setDatachannelToken = ( datachannelToken: string, - dataChannelTokenType: DataChannelTokenType + dataChannelTokenType: DataChannelTokenType = DataChannelTokenType.Default ): void => { this.datachannelTokens[dataChannelTokenType] = datachannelToken; }; + /** + * Resets all data‑channel tokens to their initial undefined values. + * Used when leaving or disconnecting from a meeting. + * @returns {void} + */ + private resetDatachannelTokens() { + this.datachannelTokens = { + [DataChannelTokenType.Default]: undefined, + [DataChannelTokenType.PracticeSession]: undefined, + }; + } + /** * Set the handler used to refresh the DataChannel token * @@ -219,9 +245,11 @@ export default class LLMChannel extends (Mercury as any) implements ILLMChannel */ public async refreshDataChannelToken() { if (!this.refreshHandler) { - const error = new Error('LLM refreshHandler is not set'); - this.logger.error(`Error refreshing DataChannel token: ${error.message}`); - throw error; + this.logger.warn( + 'llm#refreshDataChannelToken --> LLM refreshHandler is not set, skipping token refresh' + ); + + return null; } try { @@ -229,8 +257,13 @@ export default class LLMChannel extends (Mercury as any) implements ILLMChannel return res; } catch (error: any) { - this.logger.error(`Error refreshing DataChannel token: ${error}`); - throw error; + this.logger.warn( + `llm#refreshDataChannelToken --> DataChannel token refresh failed (likely locus changed or participant left): ${ + error?.message || error + }` + ); + + return null; } } @@ -247,6 +280,7 @@ export default class LLMChannel extends (Mercury as any) implements ILLMChannel this.disconnect(options, sessionId).then(() => { // Clean up sessions data this.connections.delete(sessionId); + this.datachannelTokens[sessionId] = undefined; }); /** @@ -258,6 +292,7 @@ export default class LLMChannel extends (Mercury as any) implements ILLMChannel this.disconnectAll(options).then(() => { // Clean up all connection data this.connections.clear(); + this.resetDatachannelTokens(); }); /** @@ -283,4 +318,19 @@ export default class LLMChannel extends (Mercury as any) implements ILLMChannel // @ts-ignore return this.webex.internal.feature.getFeature('developer', DATA_CHANNEL_WITH_JWT_TOKEN); } + + /** + * Builds a WebSocket URL with the `subscriptionAwareSubchannels` query parameter. + * + * @param {string} baseUrl - The original WebSocket URL. + * @param {string[]} subchannels - List of subchannels to declare as subscription-aware. + * @returns {string} The final URL with updated query parameters. + */ + + public static buildUrlWithAwareSubchannels = (baseUrl: string, subchannels: string[]) => { + const urlObj = new URL(baseUrl); + urlObj.searchParams.set(SUBSCRIPTION_AWARE_SUBCHANNELS_PARAM, subchannels.join(',')); + + return urlObj.toString(); + }; } diff --git a/packages/@webex/internal-plugin-llm/src/llm.types.ts b/packages/@webex/internal-plugin-llm/src/llm.types.ts index a708c2c5b76..0bdb261164c 100644 --- a/packages/@webex/internal-plugin-llm/src/llm.types.ts +++ b/packages/@webex/internal-plugin-llm/src/llm.types.ts @@ -24,8 +24,8 @@ interface ILLMChannel { } export enum DataChannelTokenType { - Default = 'default', - PracticeSession = 'practiceSession', + Default = 'llm-default-session', + PracticeSession = 'llm-practice-session', } // eslint-disable-next-line import/prefer-default-export diff --git a/packages/@webex/internal-plugin-llm/test/unit/spec/llm.js b/packages/@webex/internal-plugin-llm/test/unit/spec/llm.js index 055a1cb19a3..c1948325fb8 100644 --- a/packages/@webex/internal-plugin-llm/test/unit/spec/llm.js +++ b/packages/@webex/internal-plugin-llm/test/unit/spec/llm.js @@ -25,55 +25,77 @@ describe('plugin-llm', () => { }; llmService = webex.internal.llm; - llmService.connect = sinon.stub().callsFake(() => { - // Simulate a successful connection by stubbing getSocket to return connected: true - llmService.getSocket = sinon.stub().returns({connected: true}); - }); + llmService.webSocketUrl = 'wss://example.com/socket'; llmService.disconnect = sinon.stub().resolves(true); llmService.request = sinon.stub().resolves({ headers: {}, body: { binding: 'binding', - webSocketUrl: 'url', + webSocketUrl: 'wss://example.com/socket', }, }); + const sockets = new Map(); + + llmService.connect = sinon.stub().callsFake((url, sessionId) => { + sockets.set(sessionId, {connected: true}); + llmService.getSocket = sinon.stub().callsFake((sid) => sockets.get(sid)); + }); + llmService.connections.set('llm-default-session',{ + webSocketUrl: 'wss://example.com/socket', + }) }); + afterEach(() => sinon.restore()); + describe('#registerAndConnect', () => { it('registers connection', async () => { - llmService.register = sinon.stub().resolves({ - body: { - binding: 'binding', - webSocketUrl: 'url', - }, + llmService.register = sinon.stub().callsFake(async () => { + llmService.binding = 'binding'; + llmService.webSocketUrl = 'wss://example.com/socket'; + return { + body: { + binding: 'binding', + webSocketUrl: 'wss://example.com/socket', + }, + }; }); - assert.equal(llmService.isConnected(), false); - await llmService.registerAndConnect(locusUrl, datachannelUrl); - assert.equal(llmService.isConnected(), true); + + assert.equal(llmService.isConnected('llm-default-session'), false); + await llmService.registerAndConnect(locusUrl, datachannelUrl,undefined); + assert.equal(llmService.isConnected('llm-default-session'), true); }); - it("doesn't registers connection for invalid input", async () => { - llmService.register = sinon.stub().resolves({ - body: { - binding: 'binding', - webSocketUrl: 'url', - }, + it("doesn't register connection for invalid input", async () => { + llmService.register = sinon.stub().callsFake(async () => { + llmService.binding = 'binding'; + llmService.webSocketUrl = 'wss://example.com/socket'; + return { + body: { + binding: 'binding', + webSocketUrl: 'wss://example.com/socket', + }, + }; }); + await llmService.registerAndConnect(); assert.equal(llmService.isConnected(), false); }); it('registers connection with token', async () => { - llmService.register = sinon.stub().resolves({ - body: { - binding: 'binding', - webSocketUrl: 'url', - }, + llmService.register = sinon.stub().callsFake(async () => { + llmService.binding = 'binding'; + llmService.webSocketUrl = 'wss://example.com/socket'; + return { + body: { + binding: 'binding', + webSocketUrl: 'wss://example.com/socket', + }, + }; }); assert.equal(llmService.isConnected(), false); - await llmService.registerAndConnect(locusUrl, datachannelUrl, 'abc123'); + await llmService.registerAndConnect(locusUrl, datachannelUrl,'abc123'); sinon.assert.calledOnceWithExactly( llmService.register, @@ -84,6 +106,72 @@ describe('plugin-llm', () => { assert.equal(llmService.isConnected(), true); }); + + it('connects with subscriptionAwareSubchannels when token enabled', async () => { + llmService.isDataChannelTokenEnabled = sinon.stub().returns(true); + + llmService.register = sinon.stub().callsFake(async () => { + llmService.binding = 'binding'; + llmService.webSocketUrl = 'wss://example.com/socket'; + return { + body: { + binding: 'binding', + webSocketUrl: 'wss://example.com/socket', + }, + }; + }); + + const buildSpy = sinon.spy(LLMService, 'buildUrlWithAwareSubchannels'); + + await llmService.registerAndConnect(locusUrl, datachannelUrl,'abc123'); + + sinon.assert.calledOnce(buildSpy); + sinon.assert.calledOnce(llmService.connect); + + const calledUrl = llmService.connect.getCall(0).args[0]; + assert.include(calledUrl, 'subscriptionAwareSubchannels='); + }); + + it('connects without subscriptionAwareSubchannels when token disabled', async () => { + llmService.isDataChannelTokenEnabled = sinon.stub().returns(false); + + llmService.register = sinon.stub().callsFake(async () => { + llmService.binding = 'binding'; + llmService.webSocketUrl = 'wss://example.com/socket'; + return { + body: { + binding: 'binding', + webSocketUrl: 'wss://example.com/socket', + }, + }; + }); + + const buildSpy = sinon.spy(LLMService, 'buildUrlWithAwareSubchannels'); + + await llmService.registerAndConnect(locusUrl, datachannelUrl); + + sinon.assert.notCalled(buildSpy); + sinon.assert.calledOnce(llmService.connect); + + const calledUrl = llmService.connect.getCall(0).args[0]; + assert.equal(calledUrl, llmService.webSocketUrl); + }); + + it('connects without subscriptionAwareSubchannels when token enabled BUT token missing', async () => { + llmService.isDataChannelTokenEnabled = sinon.stub().resolves(true); + + const buildSpy = sinon.spy(LLMService, 'buildUrlWithAwareSubchannels'); + + await llmService.registerAndConnect(locusUrl, datachannelUrl, undefined); + + sinon.assert.calledOnce(buildSpy); + sinon.assert.calledOnce(llmService.connect); + + const calledUrl = llmService.connect.getCall(0).args[0]; + assert.include(calledUrl, 'subscriptionAwareSubchannels='); + + buildSpy.restore(); + }); }); describe('#register', () => { @@ -136,15 +224,19 @@ describe('plugin-llm', () => { }); }); - describe('#getLocusUrl', () => { it('gets LocusUrl', async () => { - llmService.register = sinon.stub().resolves({ - body: { - binding: 'binding', - webSocketUrl: 'url', - }, + llmService.register = sinon.stub().callsFake(async () => { + llmService.binding = 'binding'; + llmService.webSocketUrl = 'wss://example.com/socket'; + return { + body: { + binding: 'binding', + webSocketUrl: 'wss://example.com/socket', + }, + }; }); + await llmService.registerAndConnect(locusUrl, datachannelUrl); assert.equal(llmService.getLocusUrl(), locusUrl); }); @@ -152,11 +244,15 @@ describe('plugin-llm', () => { describe('#getDatachannelUrl', () => { it('gets dataChannel Url', async () => { - llmService.register = sinon.stub().resolves({ - body: { - binding: 'binding', - webSocketUrl: 'url', - }, + llmService.register = sinon.stub().callsFake(async () => { + llmService.binding = 'binding'; + llmService.webSocketUrl = 'wss://example.com/socket'; + return { + body: { + binding: 'binding', + webSocketUrl: 'wss://example.com/socket', + }, + }; }); await llmService.registerAndConnect(locusUrl, datachannelUrl); assert.equal(llmService.getDatachannelUrl(), datachannelUrl); @@ -174,41 +270,47 @@ describe('plugin-llm', () => { }); }); - describe('disconnectLLM', () => { + describe('#disconnectLLM', () => { let instance; beforeEach(() => { instance = { disconnect: jest.fn(() => Promise.resolve()), - locusUrl: 'someUrl', - datachannelUrl: 'someUrl', - binding: {}, - webSocketUrl: 'someUrl', - disconnectLLM: function (options) { - return this.disconnect(options).then(() => { - this.locusUrl = undefined; - this.datachannelUrl = undefined; - this.binding = undefined; - this.webSocketUrl = undefined; + connections: new Map([ + ['llm-default-session', { foo: 'bar' }], + ]), + datachannelTokens: { + 'llm-default-session': 'session-token', + }, + + disconnectLLM: function (options, sessionId = 'llm-default-session') { + return this.disconnect(options, sessionId).then(() => { + this.connections.delete(sessionId); + this.datachannelTokens[sessionId] = undefined; }); - } + }, }; }); - it('should call disconnect and clear relevant properties', async () => { - await instance.disconnectLLM({}); + it('calls disconnect and clears session connection + token', async () => { + await instance.disconnectLLM({ code: 3000, reason: 'bye' }); + + expect(instance.disconnect).toHaveBeenCalledWith( + { code: 3000, reason: 'bye' }, + 'llm-default-session' + ); + + expect(instance.connections.has('llm-default-session')).toBe(false); - expect(instance.disconnect).toHaveBeenCalledWith({}); - expect(instance.locusUrl).toBeUndefined(); - expect(instance.datachannelUrl).toBeUndefined(); - expect(instance.binding).toBeUndefined(); - expect(instance.webSocketUrl).toBeUndefined(); + expect(instance.datachannelTokens['llm-default-session']).toBeUndefined(); }); - it('should handle errors from disconnect gracefully', async () => { - instance.disconnect.mockRejectedValue(new Error('Disconnect failed')); + it('propagates disconnect errors', async () => { + instance.disconnect.mockRejectedValue(new Error('disconnect failed')); - await expect(instance.disconnectLLM({})).rejects.toThrow('Disconnect failed'); + await expect( + instance.disconnectLLM({ code: 3000, reason: 'bye' }) + ).rejects.toThrow('disconnect failed'); }); }); @@ -239,53 +341,56 @@ describe('plugin-llm', () => { }); describe('#refreshDataChannelToken', () => { - it('throws if no handler is set', async () => { - try { - await llmService.refreshDataChannelToken(); - assert.fail('Should have thrown'); - } catch (err) { - assert.match(err.message, 'LLM refreshHandler is not set'); - } + it('returns null and logs warn if no handler is set', async () => { + const warnSpy = llmService.logger.warn + + const result = await llmService.refreshDataChannelToken(); + + assert.equal(result, null); + + sinon.assert.calledOnce(warnSpy); + sinon.assert.calledWithMatch( + warnSpy, + sinon.match('LLM refreshHandler is not set') + ); }); it('returns token when handler resolves', async () => { - const mockToken = { body: { datachannelToken: 'newToken' ,isPracticeSession: false} } + const mockToken = { body: { datachannelToken: 'newToken', isPracticeSession: false } }; const handler = sinon.stub().resolves(mockToken); + llmService.setRefreshHandler(handler); const token = await llmService.refreshDataChannelToken(); + assert.equal(token, mockToken); sinon.assert.calledOnce(handler); }); - it('logs and rethrows when handler rejects', async () => { + it('logs warn and returns null when handler rejects', async () => { const handler = sinon.stub().rejects(new Error('throw error')); + llmService.setRefreshHandler(handler); - const loggerSpy = llmService.logger.error; + const warnSpy = llmService.logger.warn - llmService.setRefreshHandler(handler); + const result = await llmService.refreshDataChannelToken(); - try { - await llmService.refreshDataChannelToken(); - assert.fail('Should have thrown'); - } catch (err) { - assert.match(err.message, /throw error/); - } + assert.equal(result, null); - sinon.assert.calledOnce(loggerSpy); + sinon.assert.calledOnce(warnSpy); sinon.assert.calledWithMatch( - loggerSpy, - sinon.match("Error refreshing DataChannel token: Error: throw error") + warnSpy, + sinon.match('DataChannel token refresh failed'), ); }); }); describe('#getDatachannelToken / #setDatachannelToken', () => { it('sets and gets datachannel token', () => { - llmService.setDatachannelToken('abc123','default'); - assert.equal(llmService.getDatachannelToken('default'), 'abc123'); - llmService.setDatachannelToken('123abc','practiceSession'); - assert.equal(llmService.getDatachannelToken('practiceSession'), '123abc'); + llmService.setDatachannelToken('abc123','llm-default-session'); + assert.equal(llmService.getDatachannelToken('llm-default-session'), 'abc123'); + llmService.setDatachannelToken('123abc','llm-practice-session'); + assert.equal(llmService.getDatachannelToken('llm-practice-session'), '123abc'); }); }); @@ -293,15 +398,6 @@ describe('plugin-llm', () => { const locusUrl2 = 'locusUrl2'; const datachannelUrl2 = 'datachannelUrl2'; - beforeEach(() => { - const sockets = new Map(); - - llmService.connect = sinon.stub().callsFake((url, sessionId) => { - sockets.set(sessionId, {connected: true}); - llmService.getSocket = sinon.stub().callsFake((sid) => sockets.get(sid)); - }); - }); - it('tracks multiple sessions independently', async () => { await llmService.registerAndConnect(locusUrl, datachannelUrl, undefined, 's1'); await llmService.registerAndConnect(locusUrl2, datachannelUrl2, undefined, 's2'); @@ -314,7 +410,6 @@ describe('plugin-llm', () => { assert.equal(llmService.getDatachannelUrl('s2'), datachannelUrl2); const all = llmService.getAllConnections(); - assert.equal(all.size, 2); assert.equal(all.has('s1'), true); assert.equal(all.has('s2'), true); }); @@ -333,10 +428,13 @@ describe('plugin-llm', () => { const all = llmService.getAllConnections(); assert.equal(all.has('s1'), false); assert.equal(all.has('s2'), true); + + assert.equal(llmService.datachannelTokens['s1'], undefined); }); it('disconnectAllLLM clears all sessions', async () => { llmService.disconnectAll = sinon.stub().resolves(true); + sinon.spy(llmService, 'resetDatachannelTokens'); await llmService.registerAndConnect(locusUrl, datachannelUrl, undefined, 's1'); await llmService.registerAndConnect(locusUrl2, datachannelUrl2, undefined, 's2'); @@ -345,7 +443,10 @@ describe('plugin-llm', () => { sinon.assert.calledOnce(llmService.disconnectAll); assert.equal(llmService.getAllConnections().size, 0); + + sinon.assert.calledOnce(llmService.resetDatachannelTokens); }); }); + }); }); diff --git a/packages/@webex/internal-plugin-voicea/src/voicea.ts b/packages/@webex/internal-plugin-voicea/src/voicea.ts index c300eebf422..2e1424dbfc5 100644 --- a/packages/@webex/internal-plugin-voicea/src/voicea.ts +++ b/packages/@webex/internal-plugin-voicea/src/voicea.ts @@ -42,6 +42,8 @@ export class VoiceaChannel extends WebexPlugin implements IVoiceaChannel { private captionStatus: string; + private isCaptionBoxOn: boolean; + private toggleManualCaptionStatus: string; private currentSpokenLanguage?: string; @@ -102,6 +104,7 @@ export class VoiceaChannel extends WebexPlugin implements IVoiceaChannel { */ public deregisterEvents() { this.areCaptionsEnabled = false; + this.isCaptionBoxOn = false; this.captionServiceId = undefined; // @ts-ignore this.webex.internal.llm.off('event:relay.event', this.eventProcessor); @@ -272,6 +275,8 @@ export class VoiceaChannel extends WebexPlugin implements IVoiceaChannel { // @ts-ignore this.webex.internal.llm.isConnected(LLM_PRACTICE_SESSION); + public getIsCaptionBoxOn = (): boolean => this.isCaptionBoxOn; + /** * Resolves the active LLM publish transport, preferring the practice-session * connection only when that session is fully connected. @@ -286,6 +291,9 @@ export class VoiceaChannel extends WebexPlugin implements IVoiceaChannel { socket: (isPracticeSessionConnected && llm.getSocket(LLM_PRACTICE_SESSION)) || llm.socket, binding: (isPracticeSessionConnected && llm.getBinding(LLM_PRACTICE_SESSION)) || llm.getBinding(), + datachannelUrl: + (isPracticeSessionConnected && llm.getDatachannelUrl(LLM_PRACTICE_SESSION)) || + llm.getDatachannelUrl(), }; }; @@ -461,6 +469,7 @@ export class VoiceaChannel extends WebexPlugin implements IVoiceaChannel { this.areCaptionsEnabled = true; this.captionStatus = TURN_ON_CAPTION_STATUS.ENABLED; this.announce(); + this.updateSubchannelSubscriptionsAndSyncCaptionState({subscribe: ['transcription']}, true); }) .catch(() => { this.captionStatus = TURN_ON_CAPTION_STATUS.IDLE; @@ -620,6 +629,68 @@ export class VoiceaChannel extends WebexPlugin implements IVoiceaChannel { * @returns {string} */ public getAnnounceStatus = () => this.announceStatus; + /** + * update LLM sub‑channel subscriptions. + * + * sends a single `subchannelSubscriptionRequest` to LLM, + * allowing subscribe and unsubscribe subchannel. + * + * @param {string[]} options.subscribe Sub‑channels to subscribe to. + * @param {string[]} options.unsubscribe Sub‑channels to unsubscribe from. + * @returns {Promise} + */ + public updateSubchannelSubscriptions = async ({ + subscribe = [], + unsubscribe = [], + }: { + subscribe?: string[]; + unsubscribe?: string[]; + } = {}): Promise => { + // @ts-ignore + const isDataChannelTokenEnabled = await this.webex.internal.llm.isDataChannelTokenEnabled(); + // @ts-ignore + if (!this.isLLMConnected() || !isDataChannelTokenEnabled) return; + + const {socket, datachannelUrl} = this.getPublishTransport(); + + // @ts-ignore + socket.send({ + id: `${this.seqNum}`, + type: 'subchannelSubscriptionRequest', + data: { + // @ts-ignore + datachannelUri: datachannelUrl, + subscribe, + unsubscribe, + }, + trackingId: `${config.trackingIdPrefix}_${uuid.v4().toString()}`, + }); + + this.seqNum += 1; + }; + + /** + * Syncs the UI caption intent and updates transcription subchannel + * subscriptions accordingly. + * + * @param {Object} [options] - Subscription options. + * @param {string[]} [options.subscribe] - Subchannels to subscribe to. + * @param {string[]} [options.unsubscribe] - Subchannels to unsubscribe from. + * @param {boolean} [isCaptionBoxOn=false] - Whether captions are intended to be enabled. + * + * @returns {Promise} + */ + public updateSubchannelSubscriptionsAndSyncCaptionState = ( + options: { + subscribe?: string[]; + unsubscribe?: string[]; + } = {}, + isCaptionBoxOn = false + ): Promise => { + this.isCaptionBoxOn = isCaptionBoxOn; + + return this.updateSubchannelSubscriptions(options); + }; } export default VoiceaChannel; diff --git a/packages/@webex/internal-plugin-voicea/test/unit/spec/voicea.js b/packages/@webex/internal-plugin-voicea/test/unit/spec/voicea.js index 4ea914f3e72..70dbc483ad0 100644 --- a/packages/@webex/internal-plugin-voicea/test/unit/spec/voicea.js +++ b/packages/@webex/internal-plugin-voicea/test/unit/spec/voicea.js @@ -221,19 +221,17 @@ describe('plugin-voicea', () => { assert.notCalled(voiceaService.webex.internal.llm.socket.send); }); }); - describe('#deregisterEvents', () => { beforeEach(async () => { const mockWebSocket = new MockWebSocket(); - voiceaService.webex.internal.llm.socket = mockWebSocket; + voiceaService.isCaptionBoxOn = true; }); - it('deregisters voicea service', async () => { + it('deregisters voicea service and resets caption state', async () => { voiceaService.listenToEvents(); await voiceaService.toggleTranscribing(true); - // eslint-disable-next-line no-underscore-dangle voiceaService.webex.internal.llm._emit('event:relay.event', { headers: {from: 'ws'}, data: {relayType: 'voicea.annc', voiceaPayload: {}}, @@ -241,12 +239,14 @@ describe('plugin-voicea', () => { assert.equal(voiceaService.areCaptionsEnabled, true); assert.equal(voiceaService.captionServiceId, 'ws'); + assert.equal(voiceaService.isCaptionBoxOn, true); voiceaService.deregisterEvents(); assert.equal(voiceaService.areCaptionsEnabled, false); assert.equal(voiceaService.captionServiceId, undefined); assert.equal(voiceaService.announceStatus, 'idle'); assert.equal(voiceaService.captionStatus, 'idle'); + assert.equal(voiceaService.isCaptionBoxOn, false); }); }); describe('#processAnnouncementMessage', () => { @@ -408,6 +408,7 @@ describe('plugin-voicea', () => { it('turns on captions', async () => { const announcementSpy = sinon.spy(voiceaService, 'announce'); + const updateSubchannelSubscriptionsAndSyncCaptionStateSpy = sinon.spy(voiceaService, 'updateSubchannelSubscriptionsAndSyncCaptionState'); const triggerSpy = sinon.spy(); @@ -428,6 +429,11 @@ describe('plugin-voicea', () => { assert.calledOnceWithExactly(triggerSpy); assert.calledOnce(announcementSpy); + assert.calledOnceWithExactly( + updateSubchannelSubscriptionsAndSyncCaptionStateSpy, + { subscribe: ['transcription'] }, + true + ); }); it("should handle request fail", async () => { @@ -486,6 +492,28 @@ describe('plugin-voicea', () => { }); }); + describe('#getIsCaptionBoxOn', () => { + beforeEach(() => { + voiceaService.isCaptionBoxOn = false; + }); + + it('returns false when captions are disabled', () => { + voiceaService.isCaptionBoxOn = false; + + const result = voiceaService.getIsCaptionBoxOn(); + + assert.equal(result, false); + }); + + it('returns true when captions are enabled', () => { + voiceaService.isCaptionBoxOn = true; + + const result = voiceaService.getIsCaptionBoxOn(); + + assert.equal(result, true); + }); + }); + describe("#announce", () => { let isAnnounceProcessed, sendAnnouncement; beforeEach(() => { @@ -1256,6 +1284,177 @@ describe('plugin-voicea', () => { }); }); + describe('#updateSubchannelSubscriptions', () => { + beforeEach(() => { + const mockWebSocket = new MockWebSocket(); + + sinon.stub(voiceaService, 'getPublishTransport').returns({ + socket: mockWebSocket, + datachannelUrl: 'mock-datachannel-uri', + }); + + voiceaService.seqNum = 1; + + voiceaService.isLLMConnected = sinon.stub().returns(true); + voiceaService.webex.internal.llm.isDataChannelTokenEnabled = sinon.stub().resolves(true); + }); + + it('sends subchannelSubscriptionRequest with subscribe and unsubscribe lists', async () => { + await voiceaService.updateSubchannelSubscriptions({ + subscribe: ['transcription'], + unsubscribe: ['polls'], + }); + + const socket = voiceaService.getPublishTransport().socket; + + sinon.assert.calledOnceWithExactly( + socket.send, + { + id: '1', + type: 'subchannelSubscriptionRequest', + data: { + datachannelUri: 'mock-datachannel-uri', + subscribe: ['transcription'], + unsubscribe: ['polls'], + }, + trackingId: sinon.match.string, + } + ); + + sinon.assert.match(voiceaService.seqNum, 2); + }); + + it('sends empty arrays when no subscribe/unsubscribe provided', async () => { + await voiceaService.updateSubchannelSubscriptions({}); + + const socket = voiceaService.getPublishTransport().socket; + + sinon.assert.calledOnceWithExactly( + socket.send, + { + id: '1', + type: 'subchannelSubscriptionRequest', + data: { + datachannelUri: 'mock-datachannel-uri', + subscribe: [], + unsubscribe: [], + }, + trackingId: sinon.match.string, + } + ); + + sinon.assert.match(voiceaService.seqNum, 2); + }); + + it('does nothing when LLM is not connected', async () => { + voiceaService.isLLMConnected = sinon.stub().returns(false); + + await voiceaService.updateSubchannelSubscriptions({ + subscribe: ['transcription'], + }); + + const socket = voiceaService.getPublishTransport().socket; + + sinon.assert.notCalled(socket.send); + sinon.assert.match(voiceaService.seqNum, 1); + }); + + it('does nothing when dataChannelToken is not enabled', async () => { + voiceaService.webex.internal.llm.isDataChannelTokenEnabled = sinon.stub().resolves(false); + + await voiceaService.updateSubchannelSubscriptions({ + subscribe: ['transcription'], + }); + + const socket = voiceaService.getPublishTransport().socket; + + sinon.assert.notCalled(socket.send); + sinon.assert.match(voiceaService.seqNum, 1); + }); + }); + + + describe('#updateSubchannelSubscriptionsAndSyncCaptionState', () => { + beforeEach(() => { + const mockWebSocket = new MockWebSocket(); + voiceaService.webex.internal.llm.socket = mockWebSocket; + + voiceaService.webex.internal.llm.getDatachannelUrl = sinon.stub().returns('mock-datachannel-uri'); + + voiceaService.seqNum = 1; + + voiceaService.isLLMConnected = sinon.stub().returns(true); + voiceaService.webex.internal.llm.isDataChannelTokenEnabled = sinon.stub().resolves(true); + + sinon.spy(voiceaService, 'updateSubchannelSubscriptions'); + }); + + afterEach(() => { + sinon.restore(); + }); + + it('updates caption intent and forwards subscribe/unsubscribe to updateSubchannelSubscriptions', async () => { + await voiceaService.updateSubchannelSubscriptionsAndSyncCaptionState( + { + subscribe: ['transcription'], + unsubscribe: ['polls'], + }, + true + ); + + assert.equal(voiceaService.isCaptionBoxOn, true); + + assert.calledOnceWithExactly( + voiceaService.updateSubchannelSubscriptions, + { + subscribe: ['transcription'], + unsubscribe: ['polls'], + } + ); + }); + + it('sets caption intent to false when isCCBoxOpen is false', async () => { + await voiceaService.updateSubchannelSubscriptionsAndSyncCaptionState( + { subscribe: ['transcription'] }, + false + ); + + assert.equal(voiceaService.isCaptionBoxOn, false); + + assert.calledOnceWithExactly( + voiceaService.updateSubchannelSubscriptions, + { subscribe: ['transcription'] } + ); + }); + + it('defaults subscribe/unsubscribe to empty arrays when options is empty', async () => { + await voiceaService.updateSubchannelSubscriptionsAndSyncCaptionState({}, true); + + assert.equal(voiceaService.isCaptionBoxOn, true); + + assert.calledOnceWithExactly( + voiceaService.updateSubchannelSubscriptions, + {} + ); + }); + + it('still updates caption intent even if updateSubchannelSubscriptions does nothing (e.g., LLM not connected)', async () => { + voiceaService.isLLMConnected = sinon.stub().returns(false); + + await voiceaService.updateSubchannelSubscriptionsAndSyncCaptionState( + { subscribe: ['transcription'] }, + true + ); + + assert.equal(voiceaService.isCaptionBoxOn, true); + + assert.calledOnceWithExactly( + voiceaService.updateSubchannelSubscriptions, + { subscribe: ['transcription'] } + ); + }); + }); + describe('#multiple llm connections', () => { let defaultSocket; let practiceSocket; @@ -1380,6 +1579,5 @@ describe('plugin-voicea', () => { assert.equal(voiceaService.captionServiceId, 'svc-practice'); }); }); - }); }); diff --git a/packages/@webex/plugin-meetings/package.json b/packages/@webex/plugin-meetings/package.json index 3203b4f49c7..c4b17a17ea0 100644 --- a/packages/@webex/plugin-meetings/package.json +++ b/packages/@webex/plugin-meetings/package.json @@ -84,6 +84,7 @@ "global": "^4.4.0", "ip-anonymize": "^0.1.0", "javascript-state-machine": "^3.1.0", + "jose": "^5.8.0", "jwt-decode": "3.1.2", "lodash": "^4.17.21", "uuid": "^3.3.2", diff --git a/packages/@webex/plugin-meetings/src/interceptors/dataChannelAuthToken.ts b/packages/@webex/plugin-meetings/src/interceptors/dataChannelAuthToken.ts index 77a79273357..54659385956 100644 --- a/packages/@webex/plugin-meetings/src/interceptors/dataChannelAuthToken.ts +++ b/packages/@webex/plugin-meetings/src/interceptors/dataChannelAuthToken.ts @@ -5,6 +5,7 @@ import {Interceptor} from '@webex/http-core'; import LoggerProxy from '../common/logs/logger-proxy'; import {DATA_CHANNEL_AUTH_HEADER, MAX_RETRY, RETRY_INTERVAL, RETRY_KEY} from './constant'; +import {isJwtTokenExpired} from './utils'; /*! * Copyright (c) 2015-2026 Cisco Systems, Inc. See LICENSE file. @@ -69,6 +70,33 @@ export default class DataChannelAuthTokenInterceptor extends Interceptor { return key ? headers[key] : undefined; } + /** + * Intercepts outgoing requests and refreshes the Data-Channel-Auth-Token + * if the current JWT token is expired before the request is sent. + * + * @param {Object} options - The original request options. + * @returns {Promise} Updated request options with refreshed token if needed. + */ + async onRequest(options) { + const token = this.getHeader(options.headers, DATA_CHANNEL_AUTH_HEADER); + const enabled = await this._isDataChannelTokenEnabled(); + + if (!token || !enabled) { + return options; + } + + if (isJwtTokenExpired(token)) { + try { + const newToken = await this._refreshDataChannelToken(); + options.headers[DATA_CHANNEL_AUTH_HEADER] = newToken; + } catch (e) { + LoggerProxy.logger.warn(`DataChannelAuthTokenInterceptor: refresh failed: ${e.message}`); + } + } + + return options; + } + /** * Intercept responses and, on 401/403 with `Data-Channel-Auth-Token` header, * attempt to refresh the data channel token and retry the original request once. diff --git a/packages/@webex/plugin-meetings/src/interceptors/utils.ts b/packages/@webex/plugin-meetings/src/interceptors/utils.ts new file mode 100644 index 00000000000..396e291e823 --- /dev/null +++ b/packages/@webex/plugin-meetings/src/interceptors/utils.ts @@ -0,0 +1,16 @@ +import * as jose from 'jose'; + +const EXPIRY_BUFFER = 30 * 1000; + +// eslint-disable-next-line import/prefer-default-export +export const isJwtTokenExpired = (token: string): boolean => { + try { + const payload = jose.decodeJwt(token); + + if (!payload?.exp) return false; + + return payload.exp * 1000 < Date.now() + EXPIRY_BUFFER; + } catch { + return true; + } +}; diff --git a/packages/@webex/plugin-meetings/src/meeting/index.ts b/packages/@webex/plugin-meetings/src/meeting/index.ts index dee34ce1cb8..4d6d2db12f4 100644 --- a/packages/@webex/plugin-meetings/src/meeting/index.ts +++ b/packages/@webex/plugin-meetings/src/meeting/index.ts @@ -10317,12 +10317,12 @@ export default class Meeting extends StatelessWebexPlugin { } catch (e) { const msg = e?.message || String(e); - const err = Object.assign(new Error(`Failed to refresh data channel token: ${msg}`), { - statusCode: e?.statusCode, - original: e, - }); + LoggerProxy.logger.warn( + `Meeting:index#refreshDataChannelToken --> DataChannel token refresh failed (likely locus changed or participant left): ${msg}`, + {statusCode: e?.statusCode} + ); - throw err; + return null; } } diff --git a/packages/@webex/plugin-meetings/src/meeting/request.ts b/packages/@webex/plugin-meetings/src/meeting/request.ts index 97e70fd3c78..ca684ca1a65 100644 --- a/packages/@webex/plugin-meetings/src/meeting/request.ts +++ b/packages/@webex/plugin-meetings/src/meeting/request.ts @@ -1159,13 +1159,13 @@ export default class MeetingRequest extends StatelessWebexPlugin { method: HTTP_VERBS.GET, uri, }).catch((err) => { - LoggerProxy.logger.error( - `Meeting:request#fetchDatachannelToken --> Error retrieving ${ + LoggerProxy.logger.warn( + `Meeting:request#fetchDatachannelToken --> Failed to retrieve ${ isPracticeSession ? 'practice session ' : '' - }datachannel token, error ${err}` + }datachannel token: ${err?.message || err}` ); - throw err; + return null; }); } } diff --git a/packages/@webex/plugin-meetings/src/webinar/index.ts b/packages/@webex/plugin-meetings/src/webinar/index.ts index 6aedec1010b..4c918083c0d 100644 --- a/packages/@webex/plugin-meetings/src/webinar/index.ts +++ b/packages/@webex/plugin-meetings/src/webinar/index.ts @@ -177,6 +177,8 @@ const Webinar = WebexPlugin.extend({ const finalToken = currentToken ?? practiceSessionDatachannelToken; + const isCaptionBoxOn = this.webex.internal.voicea.getIsCaptionBoxOn(); + if (!currentToken && practiceSessionDatachannelToken) { // @ts-ignore this.webex.internal.llm.setDatachannelToken( @@ -219,6 +221,9 @@ const Webinar = WebexPlugin.extend({ ); // @ts-ignore - Fix type this.webex.internal.voicea?.announce?.(); + if (isCaptionBoxOn) { + this.webex.internal.voicea.updateSubchannelSubscriptions({subscribe: ['transcription']}); + } LoggerProxy.logger.info( `Webinar:index#updatePSDataChannel --> enabled to receive relay events for default session for ${LLM_PRACTICE_SESSION}!` ); diff --git a/packages/@webex/plugin-meetings/test/unit/spec/interceptors/dataChannelAuthToken.ts b/packages/@webex/plugin-meetings/test/unit/spec/interceptors/dataChannelAuthToken.ts index c09b247a057..d62e5285ddb 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/interceptors/dataChannelAuthToken.ts +++ b/packages/@webex/plugin-meetings/test/unit/spec/interceptors/dataChannelAuthToken.ts @@ -5,6 +5,7 @@ import MockWebex from '@webex/test-helper-mock-webex'; import {WebexHttpError} from '@webex/webex-core'; import DataChannelAuthTokenInterceptor from '@webex/plugin-meetings/src/interceptors/dataChannelAuthToken'; import LoggerProxy from '@webex/plugin-meetings/src/common/logs/logger-proxy'; +import * as utils from '@webex/plugin-meetings/src/interceptors/utils'; import {DATA_CHANNEL_AUTH_HEADER, MAX_RETRY} from '@webex/plugin-meetings/src/interceptors/constant'; describe('plugin-meetings', () => { @@ -14,6 +15,10 @@ describe('plugin-meetings', () => { beforeEach(() => { clock = sinon.useFakeTimers(); + sinon.stub(LoggerProxy, 'logger').value({ + error: sinon.stub(), + warn: sinon.stub(), + }); webex = new MockWebex({children: {}}); webex.request = sinon.stub().resolves({}); @@ -25,6 +30,7 @@ describe('plugin-meetings', () => { }); afterEach(() => { + sinon.restore(); clock.restore(); }); @@ -86,6 +92,69 @@ describe('plugin-meetings', () => { }); }); + describe('#onRequest', () => { + let isJwtTokenExpiredStub; + + beforeEach(() => { + isJwtTokenExpiredStub = sinon.stub(utils, 'isJwtTokenExpired').returns(false); + }); + + it('does nothing when token is missing', async () => { + const options = {headers: {}}; + + const res = await interceptor.onRequest(options); + + expect(res).to.equal(options); + sinon.assert.notCalled(isJwtTokenExpiredStub); + }); + + it('does nothing when feature is disabled', async () => { + interceptor._isDataChannelTokenEnabled.resolves(false); + + const options = {headers: {[DATA_CHANNEL_AUTH_HEADER]: 'old-token'}}; + const res = await interceptor.onRequest(options); + + expect(res).to.equal(options); + sinon.assert.notCalled(isJwtTokenExpiredStub); + }); + + it('does not refresh when token is not expired', async () => { + interceptor._isDataChannelTokenEnabled.resolves(true); + isJwtTokenExpiredStub.returns(false); + + const options = {headers: {[DATA_CHANNEL_AUTH_HEADER]: 'old-token'}}; + const res = await interceptor.onRequest(options); + + sinon.assert.notCalled(interceptor._refreshDataChannelToken); + expect(res.headers[DATA_CHANNEL_AUTH_HEADER]).to.equal('old-token'); + }); + + it('refreshes token when expired', async () => { + interceptor._isDataChannelTokenEnabled.resolves(true); + isJwtTokenExpiredStub.returns(true); + + interceptor._refreshDataChannelToken.resolves('new-token'); + + const options = {headers: {[DATA_CHANNEL_AUTH_HEADER]: 'old-token'}}; + const res = await interceptor.onRequest(options); + + sinon.assert.calledOnce(interceptor._refreshDataChannelToken); + expect(res.headers[DATA_CHANNEL_AUTH_HEADER]).to.equal('new-token'); + }); + + it('continues request when refresh fails', async () => { + interceptor._isDataChannelTokenEnabled.resolves(true); + isJwtTokenExpiredStub.returns(true); + + interceptor._refreshDataChannelToken.rejects(new Error('refresh failed')); + + const options = {headers: {[DATA_CHANNEL_AUTH_HEADER]: 'old-token'}}; + const res = await interceptor.onRequest(options); + + expect(res.headers[DATA_CHANNEL_AUTH_HEADER]).to.equal('old-token'); + }); + }); + describe('#refreshTokenAndRetryWithDelay', () => { const options = { headers: {[DATA_CHANNEL_AUTH_HEADER]: 'old-token'}, diff --git a/packages/@webex/plugin-meetings/test/unit/spec/interceptors/utils.ts b/packages/@webex/plugin-meetings/test/unit/spec/interceptors/utils.ts new file mode 100644 index 00000000000..1ad08a71fbf --- /dev/null +++ b/packages/@webex/plugin-meetings/test/unit/spec/interceptors/utils.ts @@ -0,0 +1,75 @@ +import 'jsdom-global/register'; +import {expect} from '@webex/test-helper-chai'; +import sinon from 'sinon'; +import {isJwtTokenExpired} from '@webex/plugin-meetings/src/interceptors/utils'; + +const makeJwt = (payload) => + [ + Buffer.from(JSON.stringify({alg: 'none', typ: 'JWT'})).toString('base64url'), + Buffer.from(JSON.stringify(payload)).toString('base64url'), + '' + ].join('.'); + +describe('plugin-meetings', () => { + describe('Interceptors', () => { + describe('utils - isJwtTokenExpired', () => { + let clock; + + beforeEach(() => { + clock = sinon.useFakeTimers(); + }); + + afterEach(() => { + sinon.restore(); + clock.restore(); + }); + + it('returns false when token has no exp', () => { + const token = makeJwt({}); // no exp + + const result = isJwtTokenExpired(token); + + expect(result).to.equal(false); + }); + + it('returns false when token is not expired', () => { + const now = Date.now(); + const futureExp = Math.floor((now + 60 * 1000) / 1000); + + const token = makeJwt({exp: futureExp}); + + const result = isJwtTokenExpired(token); + + expect(result).to.equal(false); + }); + + it('returns true when token is expired', () => { + const now = Date.now(); + const pastExp = Math.floor((now - 60 * 1000) / 1000); + + const token = makeJwt({exp: pastExp}); + + const result = isJwtTokenExpired(token); + + expect(result).to.equal(true); + }); + + it('returns true when token expires within EXPIRY_BUFFER', () => { + const now = Date.now(); + const expSoon = Math.floor((now + 10 * 1000) / 1000); + + const token = makeJwt({exp: expSoon}); + + const result = isJwtTokenExpired(token); + + expect(result).to.equal(true); + }); + + it('returns true when token is invalid', () => { + const result = isJwtTokenExpired('not-a-jwt'); + + expect(result).to.equal(true); + }); + }); + }); +}); diff --git a/packages/@webex/plugin-meetings/test/unit/spec/meeting/index.js b/packages/@webex/plugin-meetings/test/unit/spec/meeting/index.js index b9a85dad0e2..8befbfc403d 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/meeting/index.js +++ b/packages/@webex/plugin-meetings/test/unit/spec/meeting/index.js @@ -13029,7 +13029,7 @@ describe('plugin-meetings', () => { 'a datachannel url', 'token-123' ); - assert.calledWithExactly(webex.internal.llm.setDatachannelToken, 'token-123', 'default'); + assert.calledWithExactly(webex.internal.llm.setDatachannelToken, 'token-123', 'llm-default-session'); }); it('prefers refreshed token over locus self token', async () => { meeting.joinedWith = {state: 'JOINED'}; @@ -13039,7 +13039,7 @@ describe('plugin-meetings', () => { self: {datachannelToken: 'locus-token'}, }; - webex.internal.llm.getDatachannelToken.withArgs('default').returns('refreshed-token'); + webex.internal.llm.getDatachannelToken.withArgs('llm-default-session').returns('refreshed-token'); await meeting.updateLLMConnection(); @@ -13072,7 +13072,7 @@ describe('plugin-meetings', () => { 'a datachannel url', 'token-123' ); - assert.calledWithExactly(webex.internal.llm.setDatachannelToken, 'token-123', 'default'); + assert.calledWithExactly(webex.internal.llm.setDatachannelToken, 'token-123', 'llm-default-session'); }); describe('#clearMeetingData', () => { @@ -14735,7 +14735,7 @@ describe('plugin-meetings', () => { expect(result).to.deep.equal({ body: { datachannelToken: 'mock-token', - dataChannelTokenType: 'practiceSession', + dataChannelTokenType: 'llm-practice-session', }, }); }); @@ -14748,7 +14748,7 @@ describe('plugin-meetings', () => { const result = meeting.getDataChannelTokenType(); - expect(result).to.equal('practiceSession'); + expect(result).to.equal('llm-practice-session'); }); it('returns Default when not in practice session mode', () => { @@ -14758,7 +14758,7 @@ describe('plugin-meetings', () => { const result = meeting.getDataChannelTokenType(); - expect(result).to.equal('default'); + expect(result).to.equal('llm-default-session'); }); }); describe('#stopKeepAlive', () => { diff --git a/packages/@webex/plugin-meetings/test/unit/spec/meeting/request.js b/packages/@webex/plugin-meetings/test/unit/spec/meeting/request.js index 9cd9d9579e6..46094edaea4 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/meeting/request.js +++ b/packages/@webex/plugin-meetings/test/unit/spec/meeting/request.js @@ -924,7 +924,14 @@ describe('plugin-meetings', () => { const locusUrl = 'https://locus.example.com/locus/api/v1/loci/123'; const participantId = 'participant-123'; + beforeEach(() => { + sinon.restore(); + locusDeltaRequestSpy = sinon.stub(meetingsRequest, 'locusDeltaRequest'); + }); + it('sends GET request to regular datachannel token endpoint', async () => { + locusDeltaRequestSpy.resolves({body: {}}); + await meetingsRequest.fetchDatachannelToken({ locusUrl, requestingParticipantId: participantId, @@ -938,6 +945,8 @@ describe('plugin-meetings', () => { }); it('sends GET request to practice session datachannel token endpoint', async () => { + locusDeltaRequestSpy.resolves({body: {}}); + await meetingsRequest.fetchDatachannelToken({ locusUrl, requestingParticipantId: participantId, @@ -950,7 +959,7 @@ describe('plugin-meetings', () => { }); }); - it('throws if locusUrl or participantId is missing', async () => { + it('rejects when locusUrl or participantId is missing', async () => { await assert.isRejected( meetingsRequest.fetchDatachannelToken({ locusUrl: null, @@ -968,18 +977,15 @@ describe('plugin-meetings', () => { ); }); - it('logs and rethrows error when locusDeltaRequest fails', async () => { - const error = new Error('network error'); - locusDeltaRequestSpy.restore(); - sinon.stub(meetingsRequest, 'locusDeltaRequest').rejects(error); + it('returns null when locusDeltaRequest fails', async () => { + locusDeltaRequestSpy.rejects(new Error('network error')); - await assert.isRejected( - meetingsRequest.fetchDatachannelToken({ - locusUrl, - requestingParticipantId: participantId, - }), - /network error/ - ); + const result = await meetingsRequest.fetchDatachannelToken({ + locusUrl, + requestingParticipantId: participantId, + }); + + assert.equal(result, null); }); }); }); diff --git a/packages/@webex/plugin-meetings/test/unit/spec/webinar/index.ts b/packages/@webex/plugin-meetings/test/unit/spec/webinar/index.ts index 35c54b62478..4678f440a70 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/webinar/index.ts +++ b/packages/@webex/plugin-meetings/test/unit/spec/webinar/index.ts @@ -233,6 +233,8 @@ describe('plugin-meetings', () => { // Ensure connect path is eligible webinar.selfIsPanelist = true; webinar.practiceSessionEnabled = true; + webex.internal.voicea.getIsCaptionBoxOn = sinon.stub().returns(false); + webex.internal.voicea.updateSubchannelSubscriptions = sinon.stub(); }); it('no-ops when practice session join eligibility is false', async () => { @@ -342,6 +344,22 @@ describe('plugin-meetings', () => { processRelayEvent ); }); + + it('subscribes to transcription when caption intent is enabled', async () => { + webex.internal.voicea.getIsCaptionBoxOn = sinon.stub().returns(true); + + await webinar.updatePSDataChannel(); + + assert.calledOnceWithExactly(webex.internal.voicea.updateSubchannelSubscriptions, { subscribe: ['transcription'] }); + }); + + it('does not subscribe to transcription when caption intent is disabled', async () => { + webex.internal.voicea.getIsCaptionBoxOn = sinon.stub().returns(false); + + await webinar.updatePSDataChannel(); + + assert.notCalled(webex.internal.voicea.updateSubchannelSubscriptions); + }); }); describe('#updateStatusByRole', () => { diff --git a/yarn.lock b/yarn.lock index a624aa069c0..4d7e383a897 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8872,6 +8872,7 @@ __metadata: global: ^4.4.0 ip-anonymize: ^0.1.0 javascript-state-machine: ^3.1.0 + jose: ^5.8.0 jsdom: 19.0.0 jsdom-global: 3.0.2 jwt-decode: 3.1.2 @@ -22458,6 +22459,13 @@ __metadata: languageName: node linkType: hard +"jose@npm:^5.8.0": + version: 5.10.0 + resolution: "jose@npm:5.10.0" + checksum: e80965ef3ab47baafac3517f53fa9c74b948b57690de524f51320c314cd545ef51ec7b18761605d58fb5965b7c5e12b2bb6ddae87a6ccf55e3f4ad077347d8d7 + languageName: node + linkType: hard + "js-logger@npm:^1.6.1": version: 1.6.1 resolution: "js-logger@npm:1.6.1" From 4be524e73500d5829c39085e649faa935cba0168 Mon Sep 17 00:00:00 2001 From: Filip Nowakowski Date: Thu, 26 Mar 2026 11:01:12 +0100 Subject: [PATCH 18/28] refactor(metrics): update deprecated metric constants and improve media request handling --- .../@webex/plugin-meetings/src/metrics/constants.ts | 2 +- .../src/multistream/codec/mediaCodecHelper.h264.ts | 4 ++-- .../src/multistream/mediaRequestManager.ts | 8 +++----- .../plugin-meetings/src/multistream/receiveSlot.ts | 6 ++---- .../plugin-meetings/src/multistream/remoteMedia.ts | 13 +++++++++++-- .../unit/spec/multistream/mediaRequestManager.ts | 3 --- 6 files changed, 19 insertions(+), 17 deletions(-) diff --git a/packages/@webex/plugin-meetings/src/metrics/constants.ts b/packages/@webex/plugin-meetings/src/metrics/constants.ts index 0f313b43df2..a7e879e4469 100644 --- a/packages/@webex/plugin-meetings/src/metrics/constants.ts +++ b/packages/@webex/plugin-meetings/src/metrics/constants.ts @@ -91,7 +91,7 @@ const BEHAVIORAL_METRICS = { LOCUS_CLASSIC_VS_HASH_TREE_MISMATCH: 'js_sdk_locus_classic_vs_hash_tree_mismatch', LOCUS_HASH_TREE_UNSUPPORTED_OPERATION: 'js_sdk_locus_hash_tree_unsupported_operation', MEDIA_STILL_NOT_CONNECTED: 'js_sdk_media_still_not_connected', - DEPRECATED_SET_MAX_FS_USED: 'js_sdk_deprecated_set_max_fs_used', + DEPRECATED_GET_MAX_FS_USED: 'js_sdk_deprecated_get_max_fs_used', DEPRECATED_GET_EFFECTIVE_MAX_FS_USED: 'js_sdk_deprecated_get_effective_max_fs_used', DEPRECATED_RECEIVE_SLOT_SET_MAX_FS_USED: 'js_sdk_deprecated_receive_slot_set_max_fs_used', }; diff --git a/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts b/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts index 8adeeab9479..29223ec0ff7 100644 --- a/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts +++ b/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts @@ -18,7 +18,7 @@ export default class MediaCodecHelperH264 implements MediaCodecHelper { - totalMacroblocksRequested += Math.max( - ...mr.codecInfos.map((codecInfo) => - MediaCodecHelper.get(codecInfo.codec).degradeMediaRequest(mr, resolution) - ) - ); + totalMacroblocksRequested += mr.codecInfos.reduce((acc, codecInfo) => { + return acc + MediaCodecHelper.get(codecInfo.codec).degradeMediaRequest(mr, resolution); + }, 0); }); if (totalMacroblocksRequested <= this.degradationPreferences.maxMacroblocksLimit) { diff --git a/packages/@webex/plugin-meetings/src/multistream/receiveSlot.ts b/packages/@webex/plugin-meetings/src/multistream/receiveSlot.ts index 0a0591f6bf5..711a096ba71 100644 --- a/packages/@webex/plugin-meetings/src/multistream/receiveSlot.ts +++ b/packages/@webex/plugin-meetings/src/multistream/receiveSlot.ts @@ -87,10 +87,8 @@ export class ReceiveSlot extends EventsScope { } /** - * Supply the width and height of the video element - * to restrict the requested resolution to this size - * @param width width of the video element - * @param height height of the video element + * Emits a SizeHintUpdate event with the given size hint + * @param {SizeHint} sizeHint - The size hint to set */ public setSizeHint(sizeHint: SizeHint) { this.emit( diff --git a/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts b/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts index 3af714983df..7e81c7a349b 100644 --- a/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts +++ b/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts @@ -25,7 +25,7 @@ export function getMaxFs(paneSize: RemoteVideoResolution): number { LoggerProxy.logger.warn( 'RemoteMedia->getMaxFs --> [DEPRECATION WARNING]: getMaxFs has been deprecated; use size hints / resolution on RemoteMedia instead' ); - Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.DEPRECATED_SET_MAX_FS_USED, {paneSize}); + Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.DEPRECATED_GET_MAX_FS_USED, {paneSize}); return MediaCodecHelper.H264.getMaxFs(paneSize); } @@ -103,7 +103,16 @@ export class RemoteMedia extends EventsScope { // TODO: remove this once deprecation of getEffectiveMaxFs() is complete const maxFs = MediaCodecHelper.H264.getSizeHintMaxFs(this.sizeHint); if (maxFs !== undefined) { - this.receiveSlot?.setMaxFs(maxFs); + this.receiveSlot?.emit( + { + file: 'meeting/receiveSlot', + function: 'setMaxFs', + }, + ReceiveSlotEvents.MaxFsUpdate, + { + maxFs, + } + ); } } diff --git a/packages/@webex/plugin-meetings/test/unit/spec/multistream/mediaRequestManager.ts b/packages/@webex/plugin-meetings/test/unit/spec/multistream/mediaRequestManager.ts index a179b20bfeb..754c3d186b5 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/multistream/mediaRequestManager.ts +++ b/packages/@webex/plugin-meetings/test/unit/spec/multistream/mediaRequestManager.ts @@ -715,9 +715,6 @@ describe('MediaRequestManager', () => { }, ]); - // clear previous requests - mediaRequestManager.clearPreviousRequests(); - // commit same request mediaRequestManager.commit(); From ef47849a30fc723d878d9ab401c1dfcb3ab1f710 Mon Sep 17 00:00:00 2001 From: Filip Nowakowski Date: Fri, 27 Mar 2026 00:37:17 +0100 Subject: [PATCH 19/28] refactor(remoteMedia): update getSizeHint method --- .../src/multistream/remoteMedia.ts | 4 ++-- .../test/unit/spec/multistream/remoteMedia.ts | 15 +++++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts b/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts index 7e81c7a349b..58ab5c27899 100644 --- a/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts +++ b/packages/@webex/plugin-meetings/src/multistream/remoteMedia.ts @@ -118,9 +118,9 @@ export class RemoteMedia extends EventsScope { /** * Get the current size hint that would be used in media requests - * @returns {SizeHint | undefined} The size hint, or undefined if no size hint has been set + * @returns {SizeHint} The size hint */ - public getSizeHint(): SizeHint | undefined { + public getSizeHint(): SizeHint { return this.sizeHint; } diff --git a/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMedia.ts b/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMedia.ts index 487c93a5ff0..564d1cb830d 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMedia.ts +++ b/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMedia.ts @@ -288,10 +288,10 @@ describe('RemoteMedia', () => { } ); - it('also calls setMaxFs on the receive slot for backward compatibility', () => { - remoteMedia.setSizeHint(960, 540); + it('also emits MaxFsUpdate on the receive slot for backward compatibility', () => { + const emitSpy = sinon.spy(fakeReceiveSlot, 'emit'); - assert.calledOnce(fakeReceiveSlot.setMaxFs); + remoteMedia.setSizeHint(960, 540); const expectedMaxFs = MediaCodecHelper.H264.getSizeHintMaxFs({ resolution: 'medium', @@ -299,7 +299,14 @@ describe('RemoteMedia', () => { height: 540, }); - assert.calledWith(fakeReceiveSlot.setMaxFs, expectedMaxFs); + assert.calledWith( + emitSpy, + sinon.match({file: 'meeting/receiveSlot', function: 'setMaxFs'}), + ReceiveSlotEvents.MaxFsUpdate, + {maxFs: expectedMaxFs} + ); + + emitSpy.restore(); }); }); From 36ccf91d27618be38f23d281b2a5444b06e5e29e Mon Sep 17 00:00:00 2001 From: Filip Nowakowski Date: Fri, 27 Mar 2026 01:36:30 +0100 Subject: [PATCH 20/28] refactor(remoteMedia): update backward compatibility handling for setMaxFs method --- .../test/unit/spec/multistream/remoteMedia.ts | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMedia.ts b/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMedia.ts index 564d1cb830d..487c93a5ff0 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMedia.ts +++ b/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMedia.ts @@ -288,25 +288,18 @@ describe('RemoteMedia', () => { } ); - it('also emits MaxFsUpdate on the receive slot for backward compatibility', () => { - const emitSpy = sinon.spy(fakeReceiveSlot, 'emit'); - + it('also calls setMaxFs on the receive slot for backward compatibility', () => { remoteMedia.setSizeHint(960, 540); + assert.calledOnce(fakeReceiveSlot.setMaxFs); + const expectedMaxFs = MediaCodecHelper.H264.getSizeHintMaxFs({ resolution: 'medium', width: 960, height: 540, }); - assert.calledWith( - emitSpy, - sinon.match({file: 'meeting/receiveSlot', function: 'setMaxFs'}), - ReceiveSlotEvents.MaxFsUpdate, - {maxFs: expectedMaxFs} - ); - - emitSpy.restore(); + assert.calledWith(fakeReceiveSlot.setMaxFs, expectedMaxFs); }); }); From 5514bb23d67db7991ce3a275da79c492ac6e009f Mon Sep 17 00:00:00 2001 From: Filip Nowakowski Date: Fri, 27 Mar 2026 01:37:51 +0100 Subject: [PATCH 21/28] refactor(mediaRequestManager): improve macroblock calculation in degradeMediaRequest method --- .../plugin-meetings/src/multistream/mediaRequestManager.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts index 9a391e65612..dd913870cb7 100644 --- a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts +++ b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts @@ -82,7 +82,12 @@ export default class MediaRequestManager { Object.values(clientRequests).forEach((mr) => { totalMacroblocksRequested += mr.codecInfos.reduce((acc, codecInfo) => { - return acc + MediaCodecHelper.get(codecInfo.codec).degradeMediaRequest(mr, resolution); + const macroblocks = MediaCodecHelper.get(codecInfo.codec).degradeMediaRequest( + mr, + resolution + ); + + return Math.max(acc, macroblocks); }, 0); }); From 67fbb865c6bd1d84460da1a8feb4dc0d95ae2800 Mon Sep 17 00:00:00 2001 From: Filip Nowakowski Date: Fri, 27 Mar 2026 02:03:09 +0100 Subject: [PATCH 22/28] refactor(mediaRequestManager): streamline codecInfo handling and enhance video request processing --- .../src/multistream/mediaRequestManager.ts | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts index dd913870cb7..432a3915472 100644 --- a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts +++ b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts @@ -14,8 +14,6 @@ import LoggerProxy from '../common/logs/logger-proxy'; import {ReceiveSlotEvents} from './receiveSlot'; import {MediaRequest, MediaRequestId} from './types'; import MediaCodecHelper from './codec/mediaCodecHelper'; -import {CODEC_DEFAULTS} from './codec/constants'; -import type {CodecInfo} from './codec/types'; const DEBOUNCED_SOURCE_UPDATE_TIME = 1000; @@ -81,6 +79,7 @@ export default class MediaRequestManager { let totalMacroblocksRequested = 0; Object.values(clientRequests).forEach((mr) => { + // TODO: Instead of degrading based on codecInfos, degrade based on sizeHint totalMacroblocksRequested += mr.codecInfos.reduce((acc, codecInfo) => { const macroblocks = MediaCodecHelper.get(codecInfo.codec).degradeMediaRequest( mr, @@ -231,6 +230,14 @@ export default class MediaRequestManager { // clone the requests so that any modifications we do to them don't affect the original ones const clientRequests = this.cloneClientRequests(); + if (this.kind === 'video') { + Object.values(clientRequests).forEach((mr) => { + mr.codecInfos = [MediaCodecHelper.H264.getCodecInfo({sizeHint: mr.sizeHint})].filter( + (codecInfo) => codecInfo !== undefined + ); + }); + } + this.trimRequests(clientRequests); this.getDegradedClientRequests(clientRequests); @@ -277,20 +284,8 @@ export default class MediaRequestManager { // eslint-disable-next-line no-plusplus const newId = `${this.counter++}`; - const codecInfos: CodecInfo[] = - this.kind === 'audio' - ? [] - : (() => { - const info = MediaCodecHelper.H264.getCodecInfo({ - sizeHint: mediaRequest.sizeHint, - }); - - return info ? [info] : [{codec: 'h264', maxFs: CODEC_DEFAULTS.h264.maxFs}]; - })(); - const storedRequest: MediaRequest = { ...mediaRequest, - codecInfos, }; this.clientRequests[newId] = storedRequest; From cd5cc5d026e36d6cf2f146ed9228558dfd9b6fbb Mon Sep 17 00:00:00 2001 From: Filip Nowakowski Date: Fri, 27 Mar 2026 02:10:14 +0100 Subject: [PATCH 23/28] refactor(mediaRequestManager): enhance codecInfo handling and improve request processing logic --- .../src/multistream/mediaRequestManager.ts | 27 ++++++++++--------- .../test/unit/spec/multistream/remoteMedia.ts | 20 +++++++++++--- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts index 432a3915472..c8cfc4da5b9 100644 --- a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts +++ b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts @@ -80,14 +80,15 @@ export default class MediaRequestManager { Object.values(clientRequests).forEach((mr) => { // TODO: Instead of degrading based on codecInfos, degrade based on sizeHint - totalMacroblocksRequested += mr.codecInfos.reduce((acc, codecInfo) => { - const macroblocks = MediaCodecHelper.get(codecInfo.codec).degradeMediaRequest( - mr, - resolution - ); - - return Math.max(acc, macroblocks); - }, 0); + totalMacroblocksRequested += + mr.codecInfos?.reduce((acc, codecInfo) => { + const macroblocks = MediaCodecHelper.get(codecInfo.codec).degradeMediaRequest( + mr, + resolution + ); + + return Math.max(acc, macroblocks); + }, 0) ?? 0; }); if (totalMacroblocksRequested <= this.degradationPreferences.maxMacroblocksLimit) { @@ -230,13 +231,15 @@ export default class MediaRequestManager { // clone the requests so that any modifications we do to them don't affect the original ones const clientRequests = this.cloneClientRequests(); - if (this.kind === 'video') { - Object.values(clientRequests).forEach((mr) => { + Object.values(clientRequests).forEach((mr) => { + if (this.kind === 'video') { mr.codecInfos = [MediaCodecHelper.H264.getCodecInfo({sizeHint: mr.sizeHint})].filter( (codecInfo) => codecInfo !== undefined ); - }); - } + } else { + mr.codecInfos = []; + } + }); this.trimRequests(clientRequests); this.getDegradedClientRequests(clientRequests); diff --git a/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMedia.ts b/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMedia.ts index 487c93a5ff0..6a5ec161415 100644 --- a/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMedia.ts +++ b/packages/@webex/plugin-meetings/test/unit/spec/multistream/remoteMedia.ts @@ -288,10 +288,10 @@ describe('RemoteMedia', () => { } ); - it('also calls setMaxFs on the receive slot for backward compatibility', () => { - remoteMedia.setSizeHint(960, 540); + it('also emits MaxFsUpdate on the receive slot for backward compatibility', () => { + const emitSpy = sinon.spy(fakeReceiveSlot, 'emit'); - assert.calledOnce(fakeReceiveSlot.setMaxFs); + remoteMedia.setSizeHint(960, 540); const expectedMaxFs = MediaCodecHelper.H264.getSizeHintMaxFs({ resolution: 'medium', @@ -299,7 +299,19 @@ describe('RemoteMedia', () => { height: 540, }); - assert.calledWith(fakeReceiveSlot.setMaxFs, expectedMaxFs); + assert.calledWith( + emitSpy, + sinon.match({ + file: 'meeting/receiveSlot', + function: 'setMaxFs', + }), + ReceiveSlotEvents.MaxFsUpdate, + sinon.match({ + maxFs: expectedMaxFs, + }) + ); + + emitSpy.restore(); }); }); From a99d18f05efb89ef2d459a542c38e5bfa5bed8ad Mon Sep 17 00:00:00 2001 From: Filip Nowakowski Date: Fri, 27 Mar 2026 02:23:40 +0100 Subject: [PATCH 24/28] refactor(mediaRequestManager): optimize codecInfo processing --- .../src/multistream/mediaRequestManager.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts index c8cfc4da5b9..fc6688b7a57 100644 --- a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts +++ b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts @@ -123,11 +123,11 @@ export default class MediaRequestManager { } if (mediaRequest.codecInfos) { - const maxPbps = mediaRequest.codecInfos.map((codecInfo) => - MediaCodecHelper.get(codecInfo.codec).getMaxPayloadBitsPerSecond(mediaRequest) - ); + return mediaRequest.codecInfos.reduce((acc, codecInfo) => { + const mbps = MediaCodecHelper.get(codecInfo.codec).getMaxPayloadBitsPerSecond(mediaRequest); - return Math.max(...maxPbps); + return Math.max(acc, mbps); + }, 0); } LoggerProxy.logger.warn( From ecfd435a0b9d72f0546716c811df68314c552f05 Mon Sep 17 00:00:00 2001 From: Filip Nowakowski Date: Fri, 27 Mar 2026 02:23:55 +0100 Subject: [PATCH 25/28] refactor(mediaRequestManager): add maxMbps calculation and improve codecInfo mapping --- .../codec/mediaCodecHelper.h264.ts | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts b/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts index 29223ec0ff7..7347c6dbb18 100644 --- a/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts +++ b/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts @@ -87,18 +87,18 @@ export default class MediaCodecHelperH264 implements MediaCodecHelper - WcmeCodecInfo.fromH264( + ...mr.codecInfos.map((codecInfo) => { + return WcmeCodecInfo.fromH264( 0x80, // TODO: Fix this constant new H264Codec( codecInfo.maxFs, codecInfo.maxFps || CODEC_DEFAULTS.h264.maxFps, - codecInfo.maxMbps || CODEC_DEFAULTS.h264.maxMbps, + this.getMaxMbps(codecInfo), codecInfo.maxWidth, codecInfo.maxHeight ) - ) - ), + ); + }), ]; } @@ -123,6 +123,22 @@ export default class MediaCodecHelperH264 implements MediaCodecHelper Date: Fri, 27 Mar 2026 02:40:33 +0100 Subject: [PATCH 26/28] refactor(codec): rename and streamline WCME codec info retrieval methods --- .../codec/mediaCodecHelper.h264.ts | 36 +++++++++---------- .../src/multistream/codec/types.ts | 2 +- .../src/multistream/mediaRequestManager.ts | 18 ++++------ .../src/multistream/remoteMediaGroup.ts | 2 +- 4 files changed, 26 insertions(+), 32 deletions(-) diff --git a/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts b/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts index 7347c6dbb18..dad25431d9b 100644 --- a/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts +++ b/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts @@ -80,26 +80,22 @@ export default class MediaCodecHelperH264 implements MediaCodecHelper { - return WcmeCodecInfo.fromH264( - 0x80, // TODO: Fix this constant - new H264Codec( - codecInfo.maxFs, - codecInfo.maxFps || CODEC_DEFAULTS.h264.maxFps, - this.getMaxMbps(codecInfo), - codecInfo.maxWidth, - codecInfo.maxHeight - ) - ); - }), - ]; + getWCMECodecInfo(codecInfo: H264CodecInfo): WcmeCodecInfo { + return WcmeCodecInfo.fromH264( + 0x80, // TODO: Fix this constant + new H264Codec( + codecInfo.maxFs, + codecInfo.maxFps || CODEC_DEFAULTS.h264.maxFps, + this.getMaxMbps(codecInfo), + codecInfo.maxWidth, + codecInfo.maxHeight + ) + ); } /** @@ -133,6 +129,10 @@ export default class MediaCodecHelperH264 implements MediaCodecHelper { getCodecInfo(options: GetCodecInfoOptions): TCodecInfo | undefined; - getWCMECodecInfos(mediaRequest: MediaRequest): WcmeCodecInfo[]; + getWCMECodecInfo(codecInfo: TCodecInfo): WcmeCodecInfo; degradeMediaRequest(mediaRequest: MediaRequest, resolution: SupportedResolution): number; getMaxPayloadBitsPerSecond(mediaRequest: MediaRequest): number; } diff --git a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts index fc6688b7a57..d58e7cbaeba 100644 --- a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts +++ b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts @@ -265,8 +265,8 @@ export default class MediaRequestManager { const receiveSlots = mr.receiveSlots.map((receiveSlot) => receiveSlot.wcmeReceiveSlot); const maxPayloadBitsPerSecond = this.getMaxPayloadBitsPerSecond(mr); - const codecInfos = mr.codecInfos.flatMap((codecInfo) => - MediaCodecHelper.get(codecInfo.codec).getWCMECodecInfos(mr) + const codecInfos = mr.codecInfos.map((codecInfo) => + MediaCodecHelper.get(codecInfo.codec).getWCMECodecInfo(codecInfo) ); const streamRequest = new StreamRequest( @@ -287,28 +287,22 @@ export default class MediaRequestManager { // eslint-disable-next-line no-plusplus const newId = `${this.counter++}`; - const storedRequest: MediaRequest = { - ...mediaRequest, - }; - - this.clientRequests[newId] = storedRequest; + this.clientRequests[newId] = mediaRequest; const handleMaxFs = ({maxFs}: {maxFs: number}) => { - storedRequest.preferredMaxFs = maxFs; + mediaRequest.preferredMaxFs = maxFs; this.debouncedSourceUpdateListener(); }; const handleSizeHint = (sizeHint: MediaRequest['sizeHint']) => { - storedRequest.sizeHint = sizeHint; + mediaRequest.sizeHint = sizeHint; this.debouncedSourceUpdateListener(); }; - storedRequest.handleMaxFs = handleMaxFs; - storedRequest.handleSizeHint = handleSizeHint; mediaRequest.handleMaxFs = handleMaxFs; mediaRequest.handleSizeHint = handleSizeHint; - storedRequest.receiveSlots.forEach((rs) => { + mediaRequest.receiveSlots.forEach((rs) => { rs.on(ReceiveSlotEvents.SourceUpdate, this.sourceUpdateListener); rs.on(ReceiveSlotEvents.MaxFsUpdate, handleMaxFs); rs.on(ReceiveSlotEvents.SizeHintUpdate, handleSizeHint); diff --git a/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts b/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts index 420a8c350d7..b0bdc7937be 100644 --- a/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts +++ b/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts @@ -122,7 +122,7 @@ export class RemoteMediaGroup { } /** - * Pins a specific remote media instance to a specfic CSI, so the media will + * Pins a specific remote media instance to a specific CSI, so the media will * no longer come from active speaker, but from that CSI. * If no CSI is given, the current CSI value is used. * From 0ff5e8ab82f0432ee4581891891546d7a84f33cf Mon Sep 17 00:00:00 2001 From: Filip Nowakowski Date: Fri, 27 Mar 2026 02:47:53 +0100 Subject: [PATCH 27/28] refactor(remoteMediaGroup): simplify size hint filtering logic --- .../@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts b/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts index b0bdc7937be..82576cd5264 100644 --- a/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts +++ b/packages/@webex/plugin-meetings/src/multistream/remoteMediaGroup.ts @@ -316,7 +316,7 @@ export class RemoteMediaGroup { return undefined; } - const withPixels = sizeHints.filter((sh) => (sh.width ?? 0) > 0 && (sh.height ?? 0) > 0); + const withPixels = sizeHints.filter((sh) => sh.width > 0 && sh.height > 0); if (withPixels.length > 0) { // return the size hint with the largest area return withPixels.reduce((best, cur) => From a27120c505959d82af287f23b83e9b7ca0edd5ce Mon Sep 17 00:00:00 2001 From: Filip Nowakowski Date: Fri, 27 Mar 2026 12:13:13 +0100 Subject: [PATCH 28/28] fix: refactor in progress --- .../src/multistream/codec/constants.ts | 9 ++ .../codec/mediaCodecHelper.h264.ts | 80 ++++------------- .../src/multistream/codec/mediaCodecHelper.ts | 85 +++++++++++++++++++ .../src/multistream/codec/types.ts | 11 +-- .../src/multistream/mediaRequestManager.ts | 37 ++++---- 5 files changed, 134 insertions(+), 88 deletions(-) diff --git a/packages/@webex/plugin-meetings/src/multistream/codec/constants.ts b/packages/@webex/plugin-meetings/src/multistream/codec/constants.ts index 9b84e82e66a..7079214f29c 100644 --- a/packages/@webex/plugin-meetings/src/multistream/codec/constants.ts +++ b/packages/@webex/plugin-meetings/src/multistream/codec/constants.ts @@ -1,6 +1,15 @@ import {H264EncodingParams, SupportedResolution} from '@webex/internal-media-core'; import {RemoteVideoResolution} from '../types'; +export const DEGRADATION_FRAME_SIZE = { + '90p': 60, + '180p': 240, + '360p': 920, + '540p': 2040, + '720p': 3600, + '1080p': 8192, +} satisfies Record; + export const H264_CODEC_PARAMETERS = { '90p': { maxFs: 60, diff --git a/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts b/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts index dad25431d9b..1035c83b96a 100644 --- a/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts +++ b/packages/@webex/plugin-meetings/src/multistream/codec/mediaCodecHelper.h264.ts @@ -6,8 +6,8 @@ import { CodecInfo as WcmeCodecInfo, } from '@webex/internal-media-core'; import {CODEC_DEFAULTS, H264_CODEC_PARAMETERS, PANE_SIZE_TO_RESOLUTION} from './constants'; -import {MediaCodecHelper, H264CodecInfo, GetCodecInfoOptions} from './types'; -import {MediaRequest, RemoteVideoResolution, SizeHint} from '../types'; +import {MediaCodecHelper, H264CodecInfo, GetCodecInfoOptions, CodecInfo} from './types'; +import {RemoteVideoResolution, SizeHint} from '../types'; import LoggerProxy from '../../common/logs/logger-proxy'; /** @@ -33,50 +33,26 @@ export default class MediaCodecHelperH264 implements MediaCodecHelper codecInfo.codec === 'h264'); - if (codecInfos.length === 0) { - return 0; - } - - const maxFs = Math.min( - mr.preferredMaxFs || CODEC_DEFAULTS.h264.maxFs, - Math.min(...codecInfos.map((codecInfo) => codecInfo.maxFs)), - H264_CODEC_PARAMETERS[resolution].maxFs - ); - - codecInfos.forEach((codecInfo) => { - codecInfo.maxFs = maxFs; - }); - - // we only consider sources with "live" state - const slotsWithLiveSource = mr.receiveSlots.filter((rs) => rs.sourceState === 'live'); - - return maxFs * slotsWithLiveSource.length; - } - /** * Gets the max payload bits per second * - * @param {MediaRequest} mediaRequest - The media request to get the max payload bits per second from + * @param {CodecInfo[]} codecInfos - The codec infos to get the max payload bits per second from * @returns {number} The max payload bits per second */ - getMaxPayloadBitsPerSecond(mediaRequest: MediaRequest): number { - const codecInfos = mediaRequest.codecInfos.filter((codecInfo) => codecInfo.codec === 'h264'); - if (codecInfos.length === 0) { - return 0; - } - - return getRecommendedMaxBitrateForFrameSize( - Math.min(...codecInfos.map((codecInfo) => codecInfo.maxFs)) - ); + getMaxPayloadBitsPerSecond(codecInfos: CodecInfo[]): number { + return codecInfos + .filter((codecInfo) => codecInfo.codec === 'h264') + .reduce((acc, codecInfo) => { + let bitrate = 0; + // Legacy maxFs + if (codecInfo.maxFs) { + bitrate = getRecommendedMaxBitrateForFrameSize(codecInfo.maxFs); + } else { + bitrate = getRecommendedMaxBitrateForFrameSize(this.getMaxFs(codecInfo.resolution)) || 0; + } + + return Math.max(acc, bitrate); + }, 0); } /** @@ -91,7 +67,7 @@ export default class MediaCodecHelperH264 implements MediaCodecHelper 0 && height > 0) { + // we switch to the next resolution level when the height is 10% more than the current resolution height + // except for 1080p - we switch to it immediately when the height is more than 720p + const threshold = 1.1; + const getThresholdHeight = (h: number) => Math.round(h * threshold); + + if (height < getThresholdHeight(90)) { + return DEGRADATION_FRAME_SIZE['90p']; + } + if (height < getThresholdHeight(180)) { + return DEGRADATION_FRAME_SIZE['180p']; + } + if (height < getThresholdHeight(360)) { + return DEGRADATION_FRAME_SIZE['360p']; + } + if (height < getThresholdHeight(540)) { + return DEGRADATION_FRAME_SIZE['540p']; + } + if (height <= 720) { + return DEGRADATION_FRAME_SIZE['720p']; + } + + return DEGRADATION_FRAME_SIZE['1080p']; + } + + // Fall back to resolution option + if (resolution) { + return DEGRADATION_FRAME_SIZE[resolution]; + } + + return undefined; + }, + + /** + * Gets the max fs for the given width and height + * + * @param {number} frameSize - The frame size to get the resolution for + * @returns {number | undefined} The max fs for the given width and height, or undefined if the width or height is 0 + */ + getResolutionForFrameSize(frameSize: number): SupportedResolution { + const {width, height, resolution} = sizeHint ?? {}; + if (width > 0 && height > 0) { + // we switch to the next resolution level when the height is 10% more than the current resolution height + // except for 1080p - we switch to it immediately when the height is more than 720p + const threshold = 1.1; + const getThresholdHeight = (h: number) => Math.round(h * threshold); + + if (height < getThresholdHeight(90)) { + return H264_CODEC_PARAMETERS['90p'].maxFs; + } + if (height < getThresholdHeight(180)) { + return H264_CODEC_PARAMETERS['180p'].maxFs; + } + if (height < getThresholdHeight(360)) { + return H264_CODEC_PARAMETERS['360p'].maxFs; + } + if (height < getThresholdHeight(540)) { + return H264_CODEC_PARAMETERS['540p'].maxFs; + } + if (height <= 720) { + return H264_CODEC_PARAMETERS['720p'].maxFs; + } + + return H264_CODEC_PARAMETERS['1080p'].maxFs; + } + + // Fall back to resolution option + if (resolution) { + return this.getMaxFs(resolution); + } + + return undefined; + }, }; export default MediaCodecHelper; diff --git a/packages/@webex/plugin-meetings/src/multistream/codec/types.ts b/packages/@webex/plugin-meetings/src/multistream/codec/types.ts index 0f997d1e2d8..89cfc28ccad 100644 --- a/packages/@webex/plugin-meetings/src/multistream/codec/types.ts +++ b/packages/@webex/plugin-meetings/src/multistream/codec/types.ts @@ -1,9 +1,5 @@ -import { - H264EncodingParams, - SupportedResolution, - CodecInfo as WcmeCodecInfo, -} from '@webex/internal-media-core'; -import {MediaRequest, SizeHint} from '../types'; +import {H264EncodingParams, CodecInfo as WcmeCodecInfo} from '@webex/internal-media-core'; +import {SizeHint} from '../types'; export type H264CodecInfo = H264EncodingParams & { codec: 'h264'; @@ -16,6 +12,5 @@ export type GetCodecInfoOptions = {sizeHint?: SizeHint}; export interface MediaCodecHelper { getCodecInfo(options: GetCodecInfoOptions): TCodecInfo | undefined; getWCMECodecInfo(codecInfo: TCodecInfo): WcmeCodecInfo; - degradeMediaRequest(mediaRequest: MediaRequest, resolution: SupportedResolution): number; - getMaxPayloadBitsPerSecond(mediaRequest: MediaRequest): number; + getMaxPayloadBitsPerSecond(codecInfos: CodecInfo[]): number; } diff --git a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts index d58e7cbaeba..47d18e68be0 100644 --- a/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts +++ b/packages/@webex/plugin-meetings/src/multistream/mediaRequestManager.ts @@ -6,6 +6,7 @@ import { ReceiverSelectedInfo, RecommendedOpusBitrates, SupportedResolution, + getRecommendedMaxBitrateForFrameSize, } from '@webex/internal-media-core'; import {cloneDeepWith, debounce} from 'lodash'; @@ -14,6 +15,7 @@ import LoggerProxy from '../common/logs/logger-proxy'; import {ReceiveSlotEvents} from './receiveSlot'; import {MediaRequest, MediaRequestId} from './types'; import MediaCodecHelper from './codec/mediaCodecHelper'; +import {DEGRADATION_FRAME_SIZE} from './codec/constants'; const DEBOUNCED_SOURCE_UPDATE_TIME = 1000; @@ -76,25 +78,27 @@ export default class MediaRequestManager { const resolutions: SupportedResolution[] = ['1080p', '720p', '540p', '360p', '180p', '90p']; for (const resolution of resolutions) { - let totalMacroblocksRequested = 0; + let totalFrameSizeRequested = 0; Object.values(clientRequests).forEach((mr) => { - // TODO: Instead of degrading based on codecInfos, degrade based on sizeHint - totalMacroblocksRequested += - mr.codecInfos?.reduce((acc, codecInfo) => { - const macroblocks = MediaCodecHelper.get(codecInfo.codec).degradeMediaRequest( - mr, - resolution - ); - - return Math.max(acc, macroblocks); - }, 0) ?? 0; + // we only consider sources with "live" state + const slotsWithLiveSourceCount = mr.receiveSlots.filter( + (rs) => rs.sourceState === 'live' + ).length; + + const frameSize = Math.min( + MediaCodecHelper.getSizeHintMaxFs(mr.sizeHint) || Infinity, + mr.preferredMaxFs || Infinity, + DEGRADATION_FRAME_SIZE[resolution] + ); + + totalFrameSizeRequested += frameSize * slotsWithLiveSourceCount; }); - if (totalMacroblocksRequested <= this.degradationPreferences.maxMacroblocksLimit) { + if (totalFrameSizeRequested <= this.degradationPreferences.maxMacroblocksLimit) { if (resolution !== '1080p') { LoggerProxy.logger.warn( - `multistream:mediaRequestManager --> too many streams with high macroblocks requested, resolution will be limited to ${resolution}` + `multistream:mediaRequestManager --> too many streams with high frame size requested, resolution will be limited to ${resolution}` ); } break; @@ -123,11 +127,8 @@ export default class MediaRequestManager { } if (mediaRequest.codecInfos) { - return mediaRequest.codecInfos.reduce((acc, codecInfo) => { - const mbps = MediaCodecHelper.get(codecInfo.codec).getMaxPayloadBitsPerSecond(mediaRequest); - - return Math.max(acc, mbps); - }, 0); + // Default to H264 max payload bits per second + return MediaCodecHelper.H264.getMaxPayloadBitsPerSecond(mediaRequest.codecInfos); } LoggerProxy.logger.warn(