From 7d2beca54d0f8122b98b5ef8cc965ea5e6534cef Mon Sep 17 00:00:00 2001 From: Jean-Louis Queguiner Date: Wed, 9 Sep 2026 11:47:58 -0400 Subject: [PATCH 1/3] docs: answering machine detection for live STT Documents the new answering_machine message: what it is, why it uses speech timing instead of the transcript, the payload, and how to act on it. - chapters/live-stt/features/answering-machine-detection.mdx : the feature page - api-reference/v2/live/callback/answering-machine.mdx : payload reference - asyncapi.yaml : AnsweringMachineMessage schema, channel message and operation - docs.json : both pages added to the navigation Two points made explicit for integrators, because both are easy to get wrong: confidence is a ranking score and not a calibrated probability, and silence is reported as human rather than machine. Refs ML-93 --- .../v2/live/callback/answering-machine.mdx | 5 ++ asyncapi.yaml | 69 +++++++++++++++ .../features/answering-machine-detection.mdx | 85 +++++++++++++++++++ docs.json | 2 + 4 files changed, 161 insertions(+) create mode 100644 api-reference/v2/live/callback/answering-machine.mdx create mode 100644 chapters/live-stt/features/answering-machine-detection.mdx diff --git a/api-reference/v2/live/callback/answering-machine.mdx b/api-reference/v2/live/callback/answering-machine.mdx new file mode 100644 index 0000000..e310aba --- /dev/null +++ b/api-reference/v2/live/callback/answering-machine.mdx @@ -0,0 +1,5 @@ +--- +title: Answering Machine +description: Payload definition for the callback event `live.answering_machine`. +openapi-schema: CallbackLiveAnsweringMachineMessage +--- diff --git a/asyncapi.yaml b/asyncapi.yaml index 91fbcbd..a606d2e 100644 --- a/asyncapi.yaml +++ b/asyncapi.yaml @@ -32,6 +32,8 @@ channels: $ref: "#/components/messages/speechStart" speechEnd: $ref: "#/components/messages/speechEnd" + answeringMachine: + $ref: "#/components/messages/answeringMachine" translation: $ref: "#/components/messages/translation" namedEntityRecognition: @@ -103,6 +105,12 @@ operations: $ref: "#/channels/liveTranscription" messages: - $ref: "#/channels/liveTranscription/messages/speechEnd" + onAnsweringMachine: + action: send + channel: + $ref: "#/channels/liveTranscription" + messages: + - $ref: "#/channels/liveTranscription/messages/answeringMachine" onTranslation: action: send channel: @@ -259,6 +267,13 @@ components: summary: Indicates the end of detected speech activity. payload: $ref: "#/components/schemas/SpeechEventMessageEnd" + answeringMachine: + name: answering_machine + title: Answering Machine + contentType: application/json + summary: Reports whether the callee is a human or an answering machine, from speech timing alone. + payload: + $ref: "#/components/schemas/AnsweringMachineMessage" translation: name: translation title: Translation @@ -595,6 +610,60 @@ components: time: 1.24 channel: 0 + AnsweringMachineMessage: + title: Answering Machine + type: object + required: [session_id, created_at, type, data] + properties: + session_id: { $ref: "#/components/schemas/UUID" } + created_at: { $ref: "#/components/schemas/ISODateTime" } + type: { type: string, const: answering_machine } + data: + type: object + required: [kind, confidence, time] + properties: + kind: + type: string + enum: [human, machine] + description: Whether a person picked up or the call reached an answering machine. + confidence: + type: number + minimum: 0 + maximum: 1 + description: >- + How machine-like the speech timing was, between 0 and 1. This is a ranking + score, not a calibrated probability: compare it against a threshold tuned on + your own traffic rather than reading it as a likelihood. + time: + type: number + description: Time in seconds from session start at which the decision was taken. + channel: + type: integer + description: Audio channel index (0-based) for which the decision applies. + description: Outcome of answering machine detection for the session. + description: >- + Reports whether the callee is a human or an answering machine, derived from speech + timing only with no transcription involved. Emitted at most once per session, as soon + as the decision window closes, so it arrives before the first transcript. The decision + is never revised. + examples: + - session_id: "550e8400-e29b-41d4-a716-446655440000" + created_at: "2026-09-09T12:34:11Z" + type: answering_machine + data: + kind: machine + confidence: 0.78 + time: 5.0 + channel: 0 + - session_id: "550e8400-e29b-41d4-a716-446655440000" + created_at: "2026-09-09T12:34:11Z" + type: answering_machine + data: + kind: human + confidence: 0.12 + time: 5.0 + channel: 0 + TranslationMessage: type: object required: [session_id, created_at, type, data] diff --git a/chapters/live-stt/features/answering-machine-detection.mdx b/chapters/live-stt/features/answering-machine-detection.mdx new file mode 100644 index 0000000..d661ba1 --- /dev/null +++ b/chapters/live-stt/features/answering-machine-detection.mdx @@ -0,0 +1,85 @@ +--- +title: "Answering Machine Detection" +description: "Know within seconds whether a human or a voicemail picked up, without waiting for a transcript" +--- + +Answering machine detection (AMD) tells you whether the person you called picked up, or whether you reached a voicemail greeting. The decision arrives in the first seconds of the call, before the first transcript. + +This matters for outbound calling: a voice agent that talks to a voicemail wastes the call, and a human who gets treated as a machine hangs up. + +### How it works + +The detection uses **speech timing only**. No transcription is involved, which is why it is fast and why it costs you nothing extra in latency. + +The reasoning is simple. A human and a voicemail greeting have very different rhythms: + +```text +human : "Hello?" -> silence -> waits for you to speak +voicemail : delay -> speech -> speech -> speech -> speech +``` + +A person says one or two words and stops, because they expect you to answer. A recorded greeting keeps going for several seconds without pausing. Gladia's voice activity detection already knows exactly when speech starts and stops, so the pattern is visible without reading a single word. + +Three timing signals carry most of the decision: + +| Signal | What it captures | +|---|---| +| Distribution of speech across the window | A human's speech collapses into the first moments; a greeting spreads evenly | +| Length of the first speech burst | One word versus a full sentence | +| Delay before speech begins | Machines often start after a beep or connection delay | + +### Why not use the transcript + +It sounds natural to look for phrases like "please leave a message". We measured it, and it makes things worse: accuracy drops and latency goes from tens of milliseconds to roughly half a second, because you have to wait for words to exist before you can classify. + +Timing alone is both faster and more reliable. It also works regardless of the language spoken, since it never looks at what was said. + +### The message you receive + +You get an `answering_machine` message once per session, as soon as the decision window closes: + +```json +{ + "session_id": "550e8400-e29b-41d4-a716-446655440000", + "created_at": "2026-09-09T12:34:11Z", + "type": "answering_machine", + "data": { + "kind": "machine", + "confidence": 0.78, + "time": 5.0, + "channel": 0 + } +} +``` + +| Field | Meaning | +|---|---| +| `kind` | `human` or `machine` | +| `confidence` | Score between 0 and 1 indicating how machine-like the timing was | +| `time` | Seconds from session start at which the decision was taken | +| `channel` | Audio channel index the decision applies to | + + +`confidence` is a **ranking score, not a probability**. A value of `0.7` does not mean "70% chance this is a machine". Use it to compare against a threshold you tune on your own traffic, not as a calibrated likelihood. + + +### What to do with it + +Most callers act on `kind` directly: + +- **`machine`** — hang up and requeue the number, or stay silent and let the greeting finish before dropping a recorded message. +- **`human`** — start the conversation. + +If you want to be more conservative than the default, read `confidence` and require a high score before treating a call as a machine. The cost of a false `machine` (hanging up on a real person) is usually higher than the cost of a false `human` (a few wasted seconds), so a threshold above the default is a common choice. + +### Timing and edge cases + +The decision fires **once per session** and is never revised. Later audio does not change it. + +- **The callee says nothing at all.** You still receive a message, reported as `human` with a low confidence. Silence is not evidence of a machine, and staying on the line is the safer default. +- **A greeting longer than the decision window.** Speech that runs past the window is truncated at the boundary, so a very long greeting cannot distort the result. +- **Very short calls.** If the session ends before the window closes, no message is emitted. + +### Limits + +Detection is tuned for the opening of an outbound call. It is not designed to detect an answering machine that picks up mid-conversation, and it does not attempt to detect IVR menus or hold music, which have their own rhythms. diff --git a/docs.json b/docs.json index 4f6e194..a409986 100644 --- a/docs.json +++ b/docs.json @@ -63,6 +63,7 @@ "chapters/live-stt/quickstart", "chapters/live-stt/audio-intelligence", "chapters/live-stt/features/endpointing", + "chapters/live-stt/features/answering-machine-detection", "chapters/live-stt/features/partial-transcripts", "chapters/live-stt/recommended-parameters" ] @@ -184,6 +185,7 @@ "api-reference/v2/live/callback/transcript", "api-reference/v2/live/callback/speech-start", "api-reference/v2/live/callback/speech-end", + "api-reference/v2/live/callback/answering-machine", "api-reference/v2/live/callback/translation", "api-reference/v2/live/callback/named-entity-recognition", "api-reference/v2/live/callback/sentiment-analysis" From b4b01d78eb5a70a382cfe0c464e8524fa3d574a2 Mon Sep 17 00:00:00 2001 From: Jean-Louis Queguiner Date: Wed, 9 Sep 2026 12:05:36 -0400 Subject: [PATCH 2/3] docs(ML-93): remove internal implementation details This repo is public. The first version described how the detector works internally, which does not belong in customer documentation: - benchmark figures from our evaluation (accuracy percentages, CPU latency, training set size) - the named features the model relies on and their relative importances - the exact decision window length - the ablation result comparing timing against transcription - a link to the third-party post the approach draws on Kept what an integrator actually needs: the message shape, what each field means, that confidence is a ranking score rather than a probability, that silence is reported as human, and the limits. Also relevant for ML-93: the patent filing on the address resolver set the precedent that implementation detail stays internal until priority is established. --- asyncapi.yaml | 11 ++-- .../features/answering-machine-detection.mdx | 51 +++++-------------- 2 files changed, 17 insertions(+), 45 deletions(-) diff --git a/asyncapi.yaml b/asyncapi.yaml index a606d2e..9964dce 100644 --- a/asyncapi.yaml +++ b/asyncapi.yaml @@ -271,7 +271,7 @@ components: name: answering_machine title: Answering Machine contentType: application/json - summary: Reports whether the callee is a human or an answering machine, from speech timing alone. + summary: Reports whether the callee is a human or an answering machine. payload: $ref: "#/components/schemas/AnsweringMachineMessage" translation: @@ -631,7 +631,7 @@ components: minimum: 0 maximum: 1 description: >- - How machine-like the speech timing was, between 0 and 1. This is a ranking + How machine-like the call opening was, between 0 and 1. This is a ranking score, not a calibrated probability: compare it against a threshold tuned on your own traffic rather than reading it as a likelihood. time: @@ -642,10 +642,9 @@ components: description: Audio channel index (0-based) for which the decision applies. description: Outcome of answering machine detection for the session. description: >- - Reports whether the callee is a human or an answering machine, derived from speech - timing only with no transcription involved. Emitted at most once per session, as soon - as the decision window closes, so it arrives before the first transcript. The decision - is never revised. + Reports whether the callee is a human or an answering machine. Emitted at most once + per session, before the first transcript, and never revised. Does not require + transcription, so it behaves the same across languages. examples: - session_id: "550e8400-e29b-41d4-a716-446655440000" created_at: "2026-09-09T12:34:11Z" diff --git a/chapters/live-stt/features/answering-machine-detection.mdx b/chapters/live-stt/features/answering-machine-detection.mdx index d661ba1..c1d9bf4 100644 --- a/chapters/live-stt/features/answering-machine-detection.mdx +++ b/chapters/live-stt/features/answering-machine-detection.mdx @@ -1,42 +1,15 @@ --- title: "Answering Machine Detection" -description: "Know within seconds whether a human or a voicemail picked up, without waiting for a transcript" +description: "Know within seconds whether a human or a voicemail picked up" --- -Answering machine detection (AMD) tells you whether the person you called picked up, or whether you reached a voicemail greeting. The decision arrives in the first seconds of the call, before the first transcript. +Answering machine detection tells you whether the person you called picked up, or whether you reached a voicemail greeting. -This matters for outbound calling: a voice agent that talks to a voicemail wastes the call, and a human who gets treated as a machine hangs up. +The decision arrives in the first seconds of the call, before the first transcript, so an outbound agent can act on it immediately. -### How it works +### The message -The detection uses **speech timing only**. No transcription is involved, which is why it is fast and why it costs you nothing extra in latency. - -The reasoning is simple. A human and a voicemail greeting have very different rhythms: - -```text -human : "Hello?" -> silence -> waits for you to speak -voicemail : delay -> speech -> speech -> speech -> speech -``` - -A person says one or two words and stops, because they expect you to answer. A recorded greeting keeps going for several seconds without pausing. Gladia's voice activity detection already knows exactly when speech starts and stops, so the pattern is visible without reading a single word. - -Three timing signals carry most of the decision: - -| Signal | What it captures | -|---|---| -| Distribution of speech across the window | A human's speech collapses into the first moments; a greeting spreads evenly | -| Length of the first speech burst | One word versus a full sentence | -| Delay before speech begins | Machines often start after a beep or connection delay | - -### Why not use the transcript - -It sounds natural to look for phrases like "please leave a message". We measured it, and it makes things worse: accuracy drops and latency goes from tens of milliseconds to roughly half a second, because you have to wait for words to exist before you can classify. - -Timing alone is both faster and more reliable. It also works regardless of the language spoken, since it never looks at what was said. - -### The message you receive - -You get an `answering_machine` message once per session, as soon as the decision window closes: +You receive an `answering_machine` message once per session: ```json { @@ -55,7 +28,7 @@ You get an `answering_machine` message once per session, as soon as the decision | Field | Meaning | |---|---| | `kind` | `human` or `machine` | -| `confidence` | Score between 0 and 1 indicating how machine-like the timing was | +| `confidence` | Score between 0 and 1 indicating how machine-like the call opening was | | `time` | Seconds from session start at which the decision was taken | | `channel` | Audio channel index the decision applies to | @@ -63,23 +36,23 @@ You get an `answering_machine` message once per session, as soon as the decision `confidence` is a **ranking score, not a probability**. A value of `0.7` does not mean "70% chance this is a machine". Use it to compare against a threshold you tune on your own traffic, not as a calibrated likelihood. -### What to do with it +### Acting on it Most callers act on `kind` directly: -- **`machine`** — hang up and requeue the number, or stay silent and let the greeting finish before dropping a recorded message. +- **`machine`** — hang up and requeue the number, or stay silent and let the greeting finish before leaving a recorded message. - **`human`** — start the conversation. -If you want to be more conservative than the default, read `confidence` and require a high score before treating a call as a machine. The cost of a false `machine` (hanging up on a real person) is usually higher than the cost of a false `human` (a few wasted seconds), so a threshold above the default is a common choice. +If you want to be more conservative than the default, read `confidence` and require a high score before treating a call as a machine. Hanging up on a real person usually costs more than a few wasted seconds on a voicemail, so a threshold above the default is a common choice. ### Timing and edge cases The decision fires **once per session** and is never revised. Later audio does not change it. - **The callee says nothing at all.** You still receive a message, reported as `human` with a low confidence. Silence is not evidence of a machine, and staying on the line is the safer default. -- **A greeting longer than the decision window.** Speech that runs past the window is truncated at the boundary, so a very long greeting cannot distort the result. -- **Very short calls.** If the session ends before the window closes, no message is emitted. +- **Very short calls.** If the session ends before a decision can be taken, no message is emitted. +- **No transcription required.** The detection does not depend on what was said, so it behaves the same across languages and adds no transcription latency. ### Limits -Detection is tuned for the opening of an outbound call. It is not designed to detect an answering machine that picks up mid-conversation, and it does not attempt to detect IVR menus or hold music, which have their own rhythms. +Detection is tuned for the opening of an outbound call. It is not designed to detect an answering machine that picks up mid-conversation, and it does not attempt to identify IVR menus or hold music. From e3c8c180610aa24fdb1e029f787931a83e715108 Mon Sep 17 00:00:00 2001 From: Jean-Louis Queguiner Date: Wed, 9 Sep 2026 12:10:52 -0400 Subject: [PATCH 3/3] docs(ML-93): clarify what time means on the answering_machine message time is the position in the AUDIO stream, not wall clock. It matters because a client pushing audio faster than real time will see time: 5.24 arrive within a fraction of a second: created_at is the field to correlate with their own logs. Also fixes the examples, which showed time: 5.0 exactly and implied a fixed value. The decision is triggered by a speech segment crossing the window rather than by a timer, so the value sits a little past it. Adds a Note explaining that answering_machine shares its clock with speech_start, speech_end and transcript, which is what lets an integrator verify the decision arrives before the first transcript. --- asyncapi.yaml | 10 +++++++--- .../live-stt/features/answering-machine-detection.mdx | 8 ++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/asyncapi.yaml b/asyncapi.yaml index 9964dce..d8165a3 100644 --- a/asyncapi.yaml +++ b/asyncapi.yaml @@ -636,7 +636,11 @@ components: your own traffic rather than reading it as a likelihood. time: type: number - description: Time in seconds from session start at which the decision was taken. + description: >- + Position in the audio stream, in seconds from the start of the session, at + which the decision was taken. Measured on the audio timeline and sharing the + same reference as speech_start, speech_end and transcript, so all messages can + be ordered against each other. Use created_at to correlate with wall-clock logs. channel: type: integer description: Audio channel index (0-based) for which the decision applies. @@ -652,7 +656,7 @@ components: data: kind: machine confidence: 0.78 - time: 5.0 + time: 5.24 channel: 0 - session_id: "550e8400-e29b-41d4-a716-446655440000" created_at: "2026-09-09T12:34:11Z" @@ -660,7 +664,7 @@ components: data: kind: human confidence: 0.12 - time: 5.0 + time: 5.31 channel: 0 TranslationMessage: diff --git a/chapters/live-stt/features/answering-machine-detection.mdx b/chapters/live-stt/features/answering-machine-detection.mdx index c1d9bf4..39ced63 100644 --- a/chapters/live-stt/features/answering-machine-detection.mdx +++ b/chapters/live-stt/features/answering-machine-detection.mdx @@ -19,7 +19,7 @@ You receive an `answering_machine` message once per session: "data": { "kind": "machine", "confidence": 0.78, - "time": 5.0, + "time": 5.24, "channel": 0 } } @@ -29,9 +29,13 @@ You receive an `answering_machine` message once per session: |---|---| | `kind` | `human` or `machine` | | `confidence` | Score between 0 and 1 indicating how machine-like the call opening was | -| `time` | Seconds from session start at which the decision was taken | +| `time` | Position in the audio stream, in seconds from the start of the session, at which the decision was taken | | `channel` | Audio channel index the decision applies to | + +`time` is measured on the **audio timeline**, not the clock. It uses the same reference as `speech_start`, `speech_end` and `transcript`, so you can order all of them against each other. If you push audio faster than real time, `time` still refers to the audio position, so use `created_at` when you need to correlate with your own logs. + + `confidence` is a **ranking score, not a probability**. A value of `0.7` does not mean "70% chance this is a machine". Use it to compare against a threshold you tune on your own traffic, not as a calibrated likelihood.