Add ESP32-P4 support; fix voice sequence numbers and VAD cut-offs

README.md

@@ -51,8 +51,10 @@ Communicator mode is selectable from the **Mode** entity in Home Assistant (alon
 | Onju Voice | Arduino | SPH0645 I2S | MAX98357A I2S | Nest Mini form factor, touch, LEDs |
 | M5Stack Atom Echo | Arduino | PDM mic | External DAC | Compact |
 | Generic ESP32-S3 | Arduino | Any I2S mic | Any I2S DAC/amp | User-defined pins |
+| Waveshare ESP32-P4-WIFI6-POE-ETH | ESP-IDF | Analog mic via ES8311 | ES8311 DAC | ESP32-P4 (rev3), Ethernet/PoE, native esp-idf toolchain |
 
-All boards require an ESP32-S3 with PSRAM and Wi-Fi.
+The ESP32-S3 boards require PSRAM and Wi-Fi. The ESP32-P4 board connects over Ethernet and
+builds with ESPHome's native esp-idf toolchain (see `esphome/waveshare-esp32-p4-poe.yaml`).
 
 ## Architecture
 

components/mumble/__init__.py

@@ -22,6 +22,7 @@
 CONF_USERNAME = "username"
 CONF_MODE = "mode"
 CONF_PTT_PIN = "ptt_pin"
+CONF_MIC_WARMUP = "mic_warmup"
 CONF_MUTE_PIN = "mute_pin"
 CONF_CRYPTO = "crypto"
 CONF_CA_CERT = "ca_cert"
@@ -108,6 +109,8 @@ def _validate_connection_config(config):
             cv.Optional(CONF_SPEAKER): cv.use_id(speaker.Speaker),
             cv.Optional(CONF_MODE, default=CONF_ALWAYS_ON): cv.enum(MUMBLE_MODE, lower=True),
             cv.Optional(CONF_PTT_PIN): pins.gpio_input_pin_schema,
+            # Mic settle time after acquiring the I2S bus (the S3 boxes' ES7210 needs ~200 ms).
+            cv.Optional(CONF_MIC_WARMUP, default="200ms"): cv.positive_time_period_milliseconds,
             cv.Optional(CONF_MUTE_PIN): pins.gpio_input_pin_schema,
             cv.Optional(CONF_CRYPTO, default=CONF_LEGACY): cv.enum(MUMBLE_CRYPTO, lower=True),
             cv.Optional(CONF_CA_CERT, default=""): cv.string,
@@ -140,7 +143,20 @@ async def to_code(config):
         import os
 
         lib_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "lib", "micro-opus"))
-        cg.add_library("micro-opus", None, "symlink://" + lib_path)
+        if CORE.using_toolchain_esp_idf:
+            # Native esp-idf toolchain: lib/micro-opus/CMakeLists.txt registers it as an
+            # IDF component (and sets OPUS_XTENSA_LX7 on the S3 itself).
+            from esphome.components.esp32 import add_idf_component
+
+            add_idf_component(name="micro-opus", path=lib_path)
+        else:
+            cg.add_library("micro-opus", None, "symlink://" + lib_path)
+            # Xtensa LX7 DSP paths only exist on the ESP32-S3; RISC-V (ESP32-P4)
+            # uses Opus's generic C code.
+            from esphome.components.esp32 import VARIANT_ESP32S3, get_esp32_variant
+
+            if get_esp32_variant() == VARIANT_ESP32S3:
+                cg.add_build_flag("-DOPUS_XTENSA_LX7")
     var = cg.new_Pvariable(config[CONF_ID])
     await cg.register_component(var, config)
 
@@ -193,6 +209,7 @@ async def to_code(config):
         speaker_var = await cg.get_variable(config[CONF_SPEAKER])
         cg.add(var.set_speaker(speaker_var))
 
+    cg.add(var.set_mic_warmup_ms(config[CONF_MIC_WARMUP].total_milliseconds))
     if CONF_PTT_PIN in config:
         pin = await cg.gpio_pin_expression(config[CONF_PTT_PIN])
         cg.add(var.set_ptt_pin(pin))

components/mumble/mumble_component.cpp

@@ -4,7 +4,7 @@
 #include "mumble_diag.h"
 #include "mumble_socket.h"
 #include "esphome/core/log.h"
-#ifdef USE_ESP_IDF
+#if defined(USE_ESP_IDF) && defined(USE_WIFI)
 #include "esp_wifi.h"
 #endif
 #include <cmath>
@@ -85,7 +85,12 @@ uint16_t MumbleComponent::get_port() const {
 
 std::string MumbleComponent::get_mac_based_username() const {
   uint8_t mac[6];
-  if (esp_read_mac(mac, ESP_MAC_WIFI_STA) != ESP_OK) {
+#ifdef USE_WIFI
+  const esp_mac_type_t mac_type = ESP_MAC_WIFI_STA;
+#else
+  const esp_mac_type_t mac_type = ESP_MAC_ETH; // e.g. ESP32-P4: Ethernet only
+#endif
+  if (esp_read_mac(mac, mac_type) != ESP_OK) {
     return "";
   }
   char buf[32];
@@ -379,7 +384,7 @@ void MumbleComponent::log_connection_config() const {
 
 void MumbleComponent::setup() {
   mumble_diag_run_boot();
-#ifdef USE_ESP_IDF
+#if defined(USE_ESP_IDF) && defined(USE_WIFI)
   // Disable WiFi power save to improve UDP transmission. Modem sleep can cause
   // UDP packets to be lost or delayed (ESP-IDF issue tracker). For plugged-in
   // devices this has negligible impact.
@@ -774,7 +779,7 @@ void MumbleComponent::manage_i2s_bus() {
     if (desired == BusOwner::MIC) {
       uint32_t now = mumble_millis();
       if (mic_warmup_until_ms_ == 0) {
-        mic_warmup_until_ms_ = now + 200; // 200ms warmup for I2S/ES7210 (longer after bus re-acquire)
+        mic_warmup_until_ms_ = now + mic_warmup_ms_; // settle time for I2S/codec (configurable: mic_warmup)
       }
       if (now < mic_warmup_until_ms_) {
         return; // Wait for warmup
@@ -919,8 +924,13 @@ int32_t MumbleComponent::frame_rms(const int16_t *pcm, size_t samples) {
 void MumbleComponent::send_voice_packet(const uint8_t *opus_data, size_t opus_len, bool is_terminator) {
   if (opus_len > MumbleUdp::MAX_PACKET_SIZE - 32)
     return; // leave room for header/varints
-  size_t n = build_voice_packet(tx_packet_buf_, sizeof(tx_packet_buf_), tx_sequence_++, opus_data, opus_len,
-                                is_terminator, voice_target_id_);
+  // Mumble's voice sequence counts 10 ms frames, so a 20 ms packet advances it by 2. Clients
+  // (Mumla, desktop Mumble) time their jitter buffer by it; stepping by 1 made them drop audio.
+  static constexpr uint64_t SEQ_PER_PACKET = OpusAudioEncoder::FRAME_SAMPLES / (16000 / 100);
+  uint64_t seq = tx_sequence_;
+  tx_sequence_ += SEQ_PER_PACKET;
+  size_t n = build_voice_packet(tx_packet_buf_, sizeof(tx_packet_buf_), seq, opus_data, opus_len, is_terminator,
+                                voice_target_id_);
   if (n == 0) {
     ESP_LOGW(TAG, "build_voice_packet returned 0 (opus_len=%u, term=%d)", (unsigned)opus_len, is_terminator);
     return;
@@ -976,7 +986,9 @@ void MumbleComponent::audio_capture() {
 
   if (tx_state_ == TxState::IDLE) {
     tx_state_ = TxState::CAPTURING;
-    tx_sequence_ = 0;
+    preroll_count_ = 0;
+    // Keep tx_sequence_ running across utterances (like desktop Mumble): restarting at 0
+    // looks like stale packets to the receiver's jitter buffer.
     // Restart ambient calibration each time we begin capturing.
     vad_noise_floor_ = 0.0f;
     vad_calib_frames_ = 0;
@@ -994,7 +1006,8 @@ void MumbleComponent::audio_capture() {
     int32_t rms = frame_rms(frame_buf, OpusAudioEncoder::FRAME_SAMPLES);
 
     // Calibration window: estimate the ambient noise floor before allowing voice detection.
-    if (vad_calib_frames_ < VAD_CALIB_FRAMES) {
+    // Push-to-talk doesn't use VAD, so don't swallow the first word while calibrating.
+    if (get_mode() != 1 && vad_calib_frames_ < VAD_CALIB_FRAMES) {
       if (vad_noise_floor_ <= 0.0f)
         vad_noise_floor_ = static_cast<float>(rms);
       else
@@ -1008,14 +1021,17 @@ void MumbleComponent::audio_capture() {
     int32_t threshold = static_cast<int32_t>(vad_noise_floor_ * VAD_MARGIN_RATIO);
     if (threshold < VAD_MIN_THRESHOLD)
       threshold = VAD_MIN_THRESHOLD;
-    bool above = (rms > threshold);
+    // Push-to-talk: the held button is the voice gate, so send everything (no VAD).
+    bool above = (get_mode() == 1) || (rms > threshold);
 
     // Adapt the noise floor: fast down toward quieter ambient; slow up when the level is in the
     // noise range; a very slow creep under loud input so an erroneously low floor can recover.
     float frms = static_cast<float>(rms);
     if (frms < vad_noise_floor_)
       vad_noise_floor_ += (frms - vad_noise_floor_) * VAD_NOISE_DOWN;
-    else if (!above)
+    else if (!above && tx_state_ == TxState::CAPTURING)
+      // Only learn the ambient level between utterances: quiet syllables mid-sentence
+      // would otherwise drag the floor up and cut TX off.
       vad_noise_floor_ += (frms - vad_noise_floor_) * VAD_NOISE_UP_FAST;
     else
       vad_noise_floor_ += (frms - vad_noise_floor_) * VAD_NOISE_UP_SLOW;
@@ -1028,6 +1044,8 @@ void MumbleComponent::audio_capture() {
           tx_state_ = TxState::TRANSMITTING;
           voice_sending_ = true;
           ESP_LOGI(TAG, "Voice TX started");
+          if (!echo_suppress)
+            flush_preroll();
         }
       } else {
         vad_voice_frames_ = 0;
@@ -1052,15 +1070,30 @@ void MumbleComponent::audio_capture() {
     }
 
     if ((tx_state_ == TxState::TRANSMITTING || tx_state_ == TxState::TAIL) && !echo_suppress) {
-      int enc_len = opus_encoder_.encode(frame_buf, OpusAudioEncoder::FRAME_SAMPLES, opus_payload_buf_,
-                                         OpusAudioEncoder::MAX_PAYLOAD_BYTES);
-      if (enc_len > 0) {
-        send_voice_packet(opus_payload_buf_, static_cast<size_t>(enc_len), false);
-      }
+      encode_and_send(frame_buf);
+    } else if (tx_state_ == TxState::CAPTURING) {
+      memcpy(preroll_buf_[preroll_head_], frame_buf, sizeof(frame_buf));
+      preroll_head_ = (preroll_head_ + 1) % PREROLL_FRAMES;
+      if (preroll_count_ < PREROLL_FRAMES)
+        preroll_count_++;
     }
   }
 }
 
+void MumbleComponent::flush_preroll() {
+  size_t start = (preroll_head_ + PREROLL_FRAMES - preroll_count_) % PREROLL_FRAMES;
+  for (size_t i = 0; i < preroll_count_; i++)
+    encode_and_send(preroll_buf_[(start + i) % PREROLL_FRAMES]);
+  preroll_count_ = 0;
+}
+
+void MumbleComponent::encode_and_send(const int16_t *frame) {
+  int enc_len = opus_encoder_.encode(frame, OpusAudioEncoder::FRAME_SAMPLES, opus_payload_buf_,
+                                     OpusAudioEncoder::MAX_PAYLOAD_BYTES);
+  if (enc_len > 0)
+    send_voice_packet(opus_payload_buf_, static_cast<size_t>(enc_len), false);
+}
+
 void MumbleComponent::dump_config() {
   ESP_LOGCONFIG(TAG, "Mumble client:");
   log_connection_config();

components/mumble/mumble_component.h

@@ -102,6 +102,7 @@ class MumbleComponent : public Component {
   void set_bot_mode(bool bot) { bot_mode_ = bot; }
   /** Voice target id for TX (0 = channel, 1-30 = custom target, 31 = loopback). */
   void set_voice_target_id(uint8_t id) { voice_target_id_ = id; }
+  void set_mic_warmup_ms(uint32_t ms) { mic_warmup_ms_ = ms; }
   uint8_t get_voice_target_id() const { return voice_target_id_; }
   /** Register a voice target with the server (delegates to client). */
   bool send_voice_target(uint8_t id, const std::vector<MsgVoiceTargetTarget> &targets);
@@ -216,7 +217,7 @@ class MumbleComponent : public Component {
   static constexpr size_t CAPTURE_BUF_FRAMES = 8;
   static constexpr size_t CAPTURE_BUF_SAMPLES = CAPTURE_BUF_FRAMES * OpusAudioEncoder::FRAME_SAMPLES;
   static constexpr int VAD_ATTACK_FRAMES = 3;    // ~60ms of voice frames to start TX
-  static constexpr int VAD_HANGOVER_FRAMES = 15; // ~300ms of silence frames to stop TX
+  static constexpr int VAD_HANGOVER_FRAMES = 40; // ~800ms of silence frames to stop TX (bridges pauses between words)
   static constexpr uint32_t ECHO_SUPPRESS_TAIL_MS = 100;
   // Adaptive VAD: track the ambient noise floor and require speech to rise a margin above it,
   // so constant background noise no longer reads as continuous voice (communicator can auto-close).
@@ -225,8 +226,18 @@ class MumbleComponent : public Component {
   static constexpr float VAD_MARGIN_RATIO = 2.5f;   // speech must exceed noise_floor * this (~8dB)
   static constexpr float VAD_NOISE_DOWN = 0.25f;    // fast decay toward a lower ambient level
   static constexpr float VAD_NOISE_UP_FAST = 0.05f; // adapt up while in the noise range (~400ms)
-  static constexpr float VAD_NOISE_UP_SLOW = 0.01f; // creep up under loud input to recover a bad floor
+  // Creep up under loud input to recover a bad floor. Must be far slower than speech: at 0.01 the
+  // floor caught up with continuous speech within ~1s and cut TX mid-sentence.
+  static constexpr float VAD_NOISE_UP_SLOW = 0.0005f;
   static constexpr size_t TX_PACKET_BUF_SIZE = 1024;
+  // Always-on: keep the last 200 ms while waiting for VAD and send it when TX starts,
+  // so the start of the first word isn't lost to the VAD attack time.
+  static constexpr size_t PREROLL_FRAMES = 10;
+  int16_t preroll_buf_[PREROLL_FRAMES][OpusAudioEncoder::FRAME_SAMPLES];
+  size_t preroll_head_{0};  // next slot to write
+  size_t preroll_count_{0}; // valid frames in the ring
+  void flush_preroll();
+  void encode_and_send(const int16_t *frame);
 
   // Chime playback (bus-aware; uses speaker_sink_ via manage_i2s_bus)
   static constexpr float CHIME_VOLUME_SCALE = 0.25f; // reduce level to avoid clipping
@@ -240,6 +251,7 @@ class MumbleComponent : public Component {
   BusOwner bus_owner_{BusOwner::NONE};
   bool bus_releasing_{false};
   uint32_t mic_warmup_until_ms_{0}; // Delay before first mic start (fixes toggle-to-start)
+  uint32_t mic_warmup_ms_{200};     // mic settle time after (re)acquiring the I2S bus (mic_warmup option)
 
   uint8_t opus_payload_buf_[OpusAudioEncoder::MAX_PAYLOAD_BYTES];
   microphone::Microphone *microphone_{nullptr};