PoC: Speech Enhancement using DeepFilterNet

.gitignore

@@ -7,6 +7,7 @@ dist-ssr
 *.bkp
 .idea/
 public/config.json
+public/assets/deepfilternet3
 backend/synapse_tmp/*
 backend/synapse_tmp_othersite/*
 /coverage

locales/de/app.json

@@ -177,6 +177,12 @@
     "background_blur_header": "Hintergrund",
     "background_blur_label": "Unschärfeeffekt für den Hintergrund aktivieren",
     "blur_not_supported_by_browser": "(Hintergrundunschärfe wird von diesem Gerät nicht unterstützt.)",
+    "noise_suppression_header": "Audioverarbeitung",
+    "noise_suppression_label": "Störgeräuschreduktion",
+    "noise_suppression_description": "Reduziert Hintergrundgeräusche von Ihrem Mikrofon",
+    "noise_suppression_level_label": "Niveau der Störgeräuschreduktion",
+    "noise_suppression_level_description": "Höhere Werte unterdrücken mehr Rauschen, können aber die Sprachklarheit beeinflussen",
+    "noise_suppression_level_value": "Niveau: {{level}}",
     "developer_tab_title": "Entwickler",
     "devices": {
       "camera": "Kamera",

locales/en/app.json

@@ -221,6 +221,12 @@
     "feedback_tab_send_logs_label": "Include debug logs",
     "feedback_tab_thank_you": "Thanks, we received your feedback!",
     "feedback_tab_title": "Feedback",
+    "noise_suppression_description": "Reduces background noise from your microphone",
+    "noise_suppression_header": "Audio Processing",
+    "noise_suppression_label": "Noise suppression",
+    "noise_suppression_level_description": "Higher levels suppress more noise but may affect speech clarity",
+    "noise_suppression_level_label": "Noise suppression level",
+    "noise_suppression_level_value": "Level: {{level}}",
     "opt_in_description": "<0></0><1></1>You may withdraw consent by unchecking this box. If you are currently in a call, this setting will take effect at the end of the call.",
     "preferences_tab": {
       "developer_mode_label": "Developer mode",

package.json

@@ -7,7 +7,7 @@
     "dev:full": "vite",
     "dev:embedded": "vite --config vite-embedded.config.js",
     "build": "yarn build:full",
-    "build:full": "NODE_OPTIONS=--max-old-space-size=16384 vite build",
+    "build:full": "yarn setup:assets && NODE_OPTIONS=--max-old-space-size=16384 vite build",
     "build:full:production": "yarn build:full",
     "build:full:development": "yarn build:full --mode development",
     "build:embedded": "yarn build:full --config vite-embedded.config.js",
@@ -17,6 +17,7 @@
     "build:sdk": "yarn build:full --config vite-sdk.config.js",
     "build:sdk:production": "yarn build:sdk",
     "serve": "vite preview",
+    "setup:assets": "node scripts/setup-noise-suppression-assets.js",
     "prettier:check": "prettier -c .",
     "prettier:format": "prettier -w .",
     "lint": "yarn lint:types && yarn lint:eslint && yarn lint:knip",
@@ -68,6 +69,7 @@
     "@testing-library/user-event": "^14.5.1",
     "@types/content-type": "^1.1.5",
     "@types/grecaptcha": "^3.0.9",
+    "@types/jest": "^30.0.0",
     "@types/jsdom": "^21.1.7",
     "@types/lodash-es": "^4.17.12",
     "@types/node": "^24.0.0",
@@ -101,7 +103,6 @@
     "eslint-plugin-storybook": "^10.3.3",
     "eslint-plugin-unicorn": "^56.0.0",
     "fetch-mock": "11.1.5",
-    "global-jsdom": "^26.0.0",
     "i18next": "^25.0.0",
     "i18next-browser-languagedetector": "^8.0.0",
     "i18next-parser": "^9.1.0",
@@ -151,5 +152,8 @@
     "qs": "^6.14.1",
     "js-yaml": "^4.1.1"
   },
-  "packageManager": "yarn@4.7.0"
+  "packageManager": "yarn@4.7.0",
+  "dependencies": {
+    "deepfilternet3-noise-filter": "^1.2.1"
+  }
 }

scripts/setup-noise-suppression-assets.js

@@ -0,0 +1,156 @@
+#!/usr/bin/env node
+
+/**
+ * Setup script to download DeepFilterNet3 assets for local bundling.
+ * This downloads the WASM binary and AI model from Mezon's CDN
+ * and places them in public/assets/deepfilternet3/ for bundling.
+ *
+ * Usage:
+ *   node scripts/setup-noise-suppression-assets.js
+ *
+ * Environment variables:
+ *   DEEPFILTERNET3_CDN_URL: Override the default CDN URL (optional)
+ */
+
+import fs from "fs";
+import path from "path";
+import https from "https";
+import { fileURLToPath } from "url";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const projectRoot = path.join(__dirname, "..");
+
+const CDN_URL =
+  process.env.DEEPFILTERNET3_CDN_URL ||
+  "https://cdn.mezon.ai/AI/models/datas/noise_suppression/deepfilternet3";
+
+const ASSETS_DIR = path.join(projectRoot, "public", "assets", "deepfilternet3");
+const V2_DIR = path.join(ASSETS_DIR, "v2");
+const PKG_DIR = path.join(V2_DIR, "pkg");
+const MODELS_DIR = path.join(V2_DIR, "models");
+
+const FILES_TO_DOWNLOAD = [
+  {
+    url: `${CDN_URL}/v2/pkg/df_bg.wasm`,
+    path: path.join(PKG_DIR, "df_bg.wasm"),
+    description: "WASM binary",
+  },
+  {
+    url: `${CDN_URL}/v2/pkg/df_bg.wasm.d.ts`,
+    path: path.join(PKG_DIR, "df_bg.wasm.d.ts"),
+    description: "WASM TypeScript definitions",
+    optional: true,
+  },
+  {
+    url: `${CDN_URL}/v2/models/DeepFilterNet3_onnx.tar.gz`,
+    path: path.join(MODELS_DIR, "DeepFilterNet3_onnx.tar.gz"),
+    description: "AI Model (ONNX format)",
+  },
+];
+
+function ensureDir(dir) {
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true });
+    console.log(`✓ Created directory: ${dir}`);
+  }
+}
+
+function downloadFile(fileUrl, filePath, isOptional = false) {
+  return new Promise((resolve, reject) => {
+    const fileName = path.basename(filePath);
+
+    // Skip if already exists
+    if (fs.existsSync(filePath)) {
+      console.log(`✓ Already exists: ${fileName}`);
+      resolve();
+      return;
+    }
+
+    console.log(`⏳ Downloading ${fileName}...`);
+
+    https
+      .get(fileUrl, (response) => {
+        // Handle redirects
+        if (
+          response.statusCode === 301 ||
+          response.statusCode === 302 ||
+          response.statusCode === 307
+        ) {
+          const redirectUrl = response.headers.location;
+          console.log(`  Redirected to: ${redirectUrl}`);
+          downloadFile(redirectUrl, filePath, isOptional)
+            .then(resolve)
+            .catch(reject);
+          return;
+        }
+
+        if (response.statusCode !== 200) {
+          const error = new Error(
+            `Download failed: HTTP ${response.statusCode} for ${fileName}`,
+          );
+          if (isOptional) {
+            console.warn(`⚠ Optional file skipped: ${fileName}`);
+            resolve();
+          } else {
+            reject(error);
+          }
+          return;
+        }
+
+        const fileStream = fs.createWriteStream(filePath);
+
+        response.pipe(fileStream);
+
+        fileStream.on("finish", () => {
+          fileStream.close();
+          const sizeMB = (fs.statSync(filePath).size / 1024 / 1024).toFixed(2);
+          console.log(`✓ Downloaded: ${fileName} (${sizeMB} MB)`);
+          resolve();
+        });
+
+        fileStream.on("error", (err) => {
+          fs.unlink(filePath, () => {}); // Clean up partial file
+          reject(err);
+        });
+      })
+      .on("error", (err) => {
+        if (isOptional) {
+          console.warn(`⚠ Optional file skipped: ${fileName} (${err.message})`);
+          resolve();
+        } else {
+          reject(err);
+        }
+      });
+  });
+}
+
+async function main() {
+  try {
+    console.log("\n🚀 Setting up DeepFilterNet3 assets for bundling...\n");
+    console.log(`📦 CDN URL: ${CDN_URL}`);
+    console.log(`📁 Asset directory: ${ASSETS_DIR}\n`);
+
+    // Ensure directories exist
+    ensureDir(ASSETS_DIR);
+    ensureDir(V2_DIR);
+    ensureDir(PKG_DIR);
+    ensureDir(MODELS_DIR);
+
+    // Download files
+    for (const file of FILES_TO_DOWNLOAD) {
+      await downloadFile(file.url, file.path, file.optional);
+    }
+
+    console.log("\n✅ Asset setup complete!");
+    console.log(
+      "\nAssets are ready for bundling. Next build will include them.\n",
+    );
+    process.exit(0);
+  } catch (error) {
+    console.error("\n❌ Asset setup failed:", error.message);
+    process.exit(1);
+  }
+}
+
+main();

src/UrlParams.ts

@@ -245,6 +245,19 @@ export interface UrlConfiguration {
    */
   noiseSuppression?: boolean;
 
+  /**
+   * Whether to enable the advanced noise suppression filter (DeepFilterNet3).
+   * This can be used to override the user's noise suppression setting via URL parameter.
+   */
+  noiseSuppressionEnabled?: boolean;
+
+  /**
+   * The noise suppression level (30-80) when using the advanced DeepFilterNet3 filter.
+   * This can be used to override the user's setting via URL parameter.
+   * Defaults to 75 if not specified.
+   */
+  noiseSuppressionLevel?: number;
+
   callIntent?: RTCCallIntent;
 }
 
@@ -504,6 +517,11 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => {
     autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"),
     noiseSuppression: parser.getFlagParam("noiseSuppression", true),
     echoCancellation: parser.getFlagParam("echoCancellation", true),
+    noiseSuppressionEnabled: parser.getFlagParam("noiseSuppressionEnabled"),
+    noiseSuppressionLevel: ((): number | undefined => {
+      const val = parseInt(parser.getParam("noiseSuppressionLevel") ?? "", 10);
+      return isNaN(val) ? undefined : val / 100;
+    })(),
   };
 
   // Log the final configuration for debugging purposes.