diff --git a/app/src/index.html b/app/src/index.html index 454c08a..d371f71 100644 --- a/app/src/index.html +++ b/app/src/index.html @@ -7,288 +7,67 @@ Leon
+ + + +
- - - - - - - - - - - - - - - - - + + + - - - - - - - - - - - - - - - - - - + + - - - - - - - - - - - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - + +
It is recommended to use a headset for a better voice experience.
- Otherwise, if your microphone is too sensitive or speakers are too - loud, + Otherwise, if your microphone is too sensitive or speakers are too loud,
Leon may hear his own voice and get confused.
@@ -343,6 +122,57 @@
+ + + + - + \ No newline at end of file diff --git a/app/src/js/main.js b/app/src/js/main.js index ba63f54..a507632 100644 --- a/app/src/js/main.js +++ b/app/src/js/main.js @@ -172,3 +172,148 @@ document.addEventListener('DOMContentLoaded', async () => { console.error(e) } }) + +// Hotword detection error handling +class HotwordDetection { + constructor() { + this.isListening = false; + this.hotwordWorker = null; + this.init(); + } + + async init() { + try { + // Check if hotword detection is supported + if (!this.isHotwordSupported()) { + this.showHotwordError('Hotword detection is not supported in your browser. Please use Chrome or Edge for full functionality.'); + return; + } + + await this.loadHotwordModel(); + this.setupHotwordListener(); + } catch (error) { + console.error('Hotword detection failed:', error); + this.handleHotwordError(error); + } + } + + isHotwordSupported() { + return typeof Worker !== 'undefined' && + typeof AudioContext !== 'undefined' && + typeof navigator.mediaDevices !== 'undefined'; + } + + async loadHotwordModel() { + return new Promise((resolve, reject) => { + // Check if hotword files exist + fetch('/hotword/leon.pmdl') + .then(response => { + if (!response.ok) { + throw new Error('Hotword model file not found'); + } + resolve(); + }) + .catch(error => { + reject(new Error('Hotword model missing. Please run: npm run setup:hotword')); + }); + }); + } + + setupHotwordListener() { + try { + this.hotwordWorker = new Worker('/js/hotword-worker.js'); + + this.hotwordWorker.onmessage = (e) => { + if (e.data === 'hotword') { + this.onHotwordDetected(); + } + }; + + this.hotwordWorker.onerror = (error) => { + this.handleHotwordError(error); + }; + + this.startListening(); + } catch (error) { + this.handleHotwordError(error); + } + } + + handleHotwordError(error) { + console.error('Hotword detection error:', error); + + let errorMessage = 'Hotword detection temporarily unavailable. '; + + if (error.message.includes('model file not found') || + error.message.includes('Hotword model missing')) { + errorMessage += 'Please run: npm run setup:hotword'; + } else if (error.message.includes('Microphone access denied')) { + errorMessage += 'Please allow microphone access and refresh the page.'; + } else if (error.message.includes('not supported')) { + errorMessage += 'Please use Chrome or Edge browser.'; + } else { + errorMessage += 'Please refresh the page to try again.'; + } + + this.showHotwordError(errorMessage); + } + + showHotwordError(message) { + // Create error notification + const errorDiv = document.createElement('div'); + errorDiv.className = 'hotword-error'; + errorDiv.innerHTML = ` +
+ Hotword Issue +

${message}

+ +
+ `; + document.body.appendChild(errorDiv); + } + + async startListening() { + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + this.isListening = true; + // Initialize audio processing here + } catch (error) { + throw new Error('Microphone access denied: ' + error.message); + } + } + + onHotwordDetected() { + // Handle hotword detection + console.log('Hotword detected!'); + // Trigger voice listening mode + this.activateVoiceMode(); + } + + activateVoiceMode() { + document.getElementById('voice-overlay-bg').style.display = 'block'; + // Add your voice mode activation logic here + } +} + +// Initialize hotword detection when DOM is loaded +document.addEventListener('DOMContentLoaded', () => { + new HotwordDetection(); +}); \ No newline at end of file diff --git a/package.json b/package.json index ee03faa..e02db5f 100644 --- a/package.json +++ b/package.json @@ -1,4 +1,5 @@ { + "name": "leon", "version": "1.0.0-beta.10+dev", "description": "Server, skills and web app of the Leon personal assistant", @@ -165,3 +166,18 @@ "vite": "4.5.0" } } + +{ + "scripts": { + "build:server": "tsc --project server/tsconfig.json", + "build:python-bridge": "cd bridges/python && npm run build", + "build:tcp-server": "cd tcp_server && npm run build", + "prebuild": "npm run clean", + "build": "npm run build:server && npm run build:python-bridge && npm run build:tcp-server", + "prestart": "npm run build && node server/dist/pre-check.js", + "start": "cross-env LEON_NODE_ENV=production node server/dist/index.js", + "dev": "cross-env LEON_NODE_ENV=development concurrently \"npm run dev:server\" \"npm run dev:app\"", + "dev:server": "ts-node --project server/tsconfig.json server/src/index.ts", + "dev:app": "vite app --port 3000" + } +} \ No newline at end of file diff --git a/scripts/check-os.js b/scripts/check-os.js index 1037d91..6d05cac 100644 --- a/scripts/check-os.js +++ b/scripts/check-os.js @@ -57,3 +57,133 @@ export default () => } } }) + +const os = require('os'); +const path = require('path'); +const fs = require('fs'); + +class OSChecker { + constructor() { + this.platform = os.platform(); + this.arch = os.arch(); + this.distro = this.getLinuxDistro(); + } + + getLinuxDistro() { + if (this.platform !== 'linux') return null; + + try { + const release = fs.readFileSync('/etc/os-release', 'utf8'); + const match = release.match(/PRETTY_NAME="(.+)"/); + return match ? match[1] : 'Linux'; + } catch (error) { + return 'Linux'; + } + } + + getBinaryExtension() { + return this.platform === 'win32' ? '.exe' : ''; + } + + getBinaryDirectory() { + const baseDirs = { + 'python-bridge': 'bridges/python/dist', + 'tcp-server': 'tcp_server/dist' + }; + + // Platform and architecture specific subdirectories + const platformMap = { + 'win32-x64': 'win-amd64', + 'linux-x64': 'linux-amd64', + 'darwin-x64': 'darwin-amd64', + 'darwin-arm64': 'darwin-arm64' + }; + + const key = `${this.platform}-${this.arch}`; + const subdir = platformMap[key] || 'unknown'; + + return { + pythonBridge: path.join(baseDirs['python-bridge'], subdir), + tcpServer: path.join(baseDirs['tcp-server'], subdir) + }; + } + + getBinaryPaths() { + const dirs = this.getBinaryDirectory(); + const ext = this.getBinaryExtension(); + + return { + pythonBridge: path.join(dirs.pythonBridge, `leon-python-bridge${ext}`), + tcpServer: path.join(dirs.tcpServer, `leon-tcp-server${ext}`) + }; + } + + checkBinaryPermissions(binaryPath) { + if (this.platform === 'win32') { + return true; // Windows doesn't have executable permissions in the same way + } + + try { + fs.accessSync(binaryPath, fs.constants.X_OK); + return true; + } catch (error) { + try { + fs.chmodSync(binaryPath, 0o755); // Make executable + return true; + } catch (chmodError) { + return false; + } + } + } + + verifyBinaries() { + const paths = this.getBinaryPaths(); + const results = {}; + + for (const [name, binaryPath] of Object.entries(paths)) { + const exists = fs.existsSync(binaryPath); + const isExecutable = exists ? this.checkBinaryPermissions(binaryPath) : false; + + results[name] = { + path: binaryPath, + exists, + isExecutable, + directory: path.dirname(binaryPath) + }; + + if (!exists) { + console.error(`āŒ Missing binary: ${name}`); + console.error(` Expected at: ${binaryPath}`); + } else if (!isExecutable) { + console.error(`āŒ Binary not executable: ${name}`); + } + } + + return results; + } + + setupBinaryDirectories() { + const dirs = this.getBinaryDirectory(); + + // Create directories if they don't exist + Object.values(dirs).forEach(dir => { + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + console.log(`šŸ“ Created directory: ${dir}`); + } + }); + } +} + +module.exports = OSChecker; + +// Run if this script is called directly +if (require.main === module) { + const checker = new OSChecker(); + console.log('Platform:', checker.platform); + console.log('Architecture:', checker.arch); + console.log('Distribution:', checker.distro); + + const binaryInfo = checker.verifyBinaries(); + console.log('Binary verification:', binaryInfo); +} \ No newline at end of file diff --git a/scripts/setup-offline/stt-download.js b/scripts/setup-offline/stt-download.js new file mode 100644 index 0000000..9c99698 --- /dev/null +++ b/scripts/setup-offline/stt-download.js @@ -0,0 +1,186 @@ +const fs = require('fs'); +const path = require('path'); +const { execSync } = require('child_process'); +const https = require('https'); + +class STTDownloader { + constructor() { + this.baseUrl = 'https://github.com/coqui-ai/STT/releases/download'; + this.version = 'v1.4.0'; + this.platform = process.platform; + this.arch = process.arch; + } + + async downloadSTTFiles() { + try { + console.log('Checking for Coqui STT dependencies...'); + + const requiredFiles = [ + 'libstt.so', + 'stt.node', + 'coqui-stt.h', + 'libstt.dylib' + ]; + + const binDir = path.join(__dirname, '../../bin/coqui'); + + // Create bin directory if it doesn't exist + if (!fs.existsSync(binDir)) { + fs.mkdirSync(binDir, { recursive: true }); + } + + let missingFiles = requiredFiles.filter(file => + !fs.existsSync(path.join(binDir, file)) + ); + + if (missingFiles.length > 0) { + console.log('Downloading missing Coqui STT files...'); + await this.downloadFromGitHub(); + } else { + console.log('All Coqui STT files are present.'); + } + + // Download model files if missing + await this.downloadModelFiles(); + + } catch (error) { + console.error('Error downloading STT files:', error); + this.provideFallbackSolution(); + } + } + + async downloadFromGitHub() { + const assets = this.getPlatformAssets(); + + for (const asset of assets) { + const downloadUrl = `${this.baseUrl}/${this.version}/${asset.filename}`; + const destination = path.join(__dirname, '../../bin/coqui', asset.filename); + + console.log(`Downloading ${asset.filename}...`); + + try { + await this.downloadFile(downloadUrl, destination); + console.log(`āœ“ Downloaded ${asset.filename}`); + + // Extract if it's a zip file + if (asset.filename.endsWith('.zip')) { + await this.extractZip(destination, path.dirname(destination)); + } + } catch (error) { + console.error(`Failed to download ${asset.filename}:`, error); + } + } + } + + getPlatformAssets() { + const assets = []; + + if (this.platform === 'win32' && this.arch === 'x64') { + assets.push({ + filename: 'stt-1.4.0-windows-x64.zip', + type: 'library' + }); + } else if (this.platform === 'linux' && this.arch === 'x64') { + assets.push({ + filename: 'stt-1.4.0-linux-x64.tar.gz', + type: 'library' + }); + } else if (this.platform === 'darwin') { + assets.push({ + filename: 'stt-1.4.0-osx-x64.tar.gz', + type: 'library' + }); + } + + return assets; + } + + async downloadModelFiles() { + const modelFiles = [ + { + url: 'https://github.com/coqui-ai/STT-models/releases/download/english%2Fcoqui%2Fv1.0.0-huge-vocab/huge-vocabulary.scorer', + filename: 'huge-vocabulary.scorer' + }, + { + url: 'https://github.com/coqui-ai/STT-models/releases/download/english%2Fcoqui%2Fv1.0.0-huge-vocab/model.tflite', + filename: 'model.tflite' + } + ]; + + const modelDir = path.join(__dirname, '../../bin/coqui'); + + for (const file of modelFiles) { + const destination = path.join(modelDir, file.filename); + + if (!fs.existsSync(destination)) { + console.log(`Downloading ${file.filename}...`); + try { + await this.downloadFile(file.url, destination); + console.log(`āœ“ Downloaded ${file.filename}`); + } catch (error) { + console.error(`Failed to download ${file.filename}:`, error); + } + } + } + } + + downloadFile(url, destination) { + return new Promise((resolve, reject) => { + const file = fs.createWriteStream(destination); + + https.get(url, (response) => { + if (response.statusCode === 302 || response.statusCode === 301) { + // Follow redirect + this.downloadFile(response.headers.location, destination) + .then(resolve) + .catch(reject); + return; + } + + if (response.statusCode !== 200) { + reject(new Error(`HTTP ${response.statusCode}`)); + return; + } + + response.pipe(file); + + file.on('finish', () => { + file.close(); + resolve(); + }); + }).on('error', (error) => { + fs.unlink(destination, () => {}); // Delete the file async + reject(error); + }); + }); + } + + extractZip(zipPath, extractTo) { + return new Promise((resolve, reject) => { + // You would need to implement zip extraction here + // Using adm-zip or similar library + console.log(`Extracting ${zipPath}...`); + resolve(); + }); + } + + provideFallbackSolution() { + console.log('\n=== MANUAL SETUP INSTRUCTIONS ==='); + console.log('1. Download Coqui STT binaries manually:'); + console.log(' Windows: https://github.com/coqui-ai/STT/releases/download/v1.4.0/stt-1.4.0-windows-x64.zip'); + console.log('2. Extract to: bin/coqui/ directory'); + console.log('3. Download model files:'); + console.log(' - https://github.com/coqui-ai/STT-models/releases/download/english%2Fcoqui%2Fv1.0.0-huge-vocab/huge-vocabulary.scorer'); + console.log(' - https://github.com/coqui-ai/STT-models/releases/download/english%2Fcoqui%2Fv1.0.0-huge-vocab/model.tflite'); + console.log('4. Place them in: bin/coqui/ directory'); + console.log('5. Run: npm run setup:offline-stt\n'); + } +} + +// Run if this script is called directly +if (require.main === module) { + const downloader = new STTDownloader(); + downloader.downloadSTTFiles().catch(console.error); +} + +module.exports = STTDownloader; \ No newline at end of file diff --git a/scripts/setup.js b/scripts/setup.js new file mode 100644 index 0000000..7222117 --- /dev/null +++ b/scripts/setup.js @@ -0,0 +1,69 @@ +const { execSync } = require('child_process'); +const fs = require('fs'); +const path = require('path'); +const OSChecker = require('./check-os'); + +class SetupManager { + constructor() { + this.osChecker = new OSChecker(); + } + + async runFullSetup() { + console.log('šŸš€ Starting Leon AI full setup...\n'); + + try { + // Step 1: Build project + console.log('šŸ“¦ Step 1: Building project...'); + this.runCommand('npm run build'); + + // Step 2: Setup offline TTS + console.log('\nšŸ—£ļø Step 2: Setting up offline TTS...'); + this.runCommand('npm run setup:offline-tts'); + + // Step 3: Setup offline STT + console.log('\nšŸŽ¤ Step 3: Setting up offline STT...'); + this.runCommand('npm run setup:offline-stt'); + + // Step 4: Setup hotword + console.log('\nšŸ‘‚ Step 4: Setting up hotword detection...'); + this.runCommand('npm run setup:hotword'); + + // Step 5: Verify setup + console.log('\nāœ… Step 5: Verifying setup...'); + this.runCommand('npm run check'); + + console.log('\nšŸŽ‰ Setup completed successfully!'); + console.log('šŸ’” You can now start Leon with: npm start'); + + } catch (error) { + console.error('\nāŒ Setup failed:', error.message); + this.provideManualInstructions(); + } + } + + runCommand(command) { + try { + execSync(command, { stdio: 'inherit' }); + } catch (error) { + throw new Error(`Command failed: ${command}`); + } + } + + provideManualInstructions() { + console.log('\nšŸ“‹ MANUAL SETUP INSTRUCTIONS:'); + console.log('1. Build project: npm run build'); + console.log('2. Setup offline TTS: npm run setup:offline-tts'); + console.log('3. Setup offline STT: npm run setup:offline-stt'); + console.log('4. Setup hotword: npm run setup:hotword'); + console.log('5. Verify: npm run check'); + console.log('6. Start: npm start'); + } +} + +// Run if this script is called directly +if (require.main === module) { + const setupManager = new SetupManager(); + setupManager.runFullSetup().catch(console.error); +} + +module.exports = SetupManager; \ No newline at end of file diff --git a/server/src/pre-check.ts b/server/src/pre-check.ts index a377fdf..075e9b6 100644 --- a/server/src/pre-check.ts +++ b/server/src/pre-check.ts @@ -272,3 +272,102 @@ const GLOBAL_DATA_SCHEMAS = { process.exit(0) })() + +import { existsSync, readFileSync } from 'fs'; +import { join } from 'path'; +import { config } from 'dotenv'; + +// Load environment variables +config(); + +class PreCheck { + private requiredPaths = [ + 'bridges/python/dist', + 'tcp_server/dist', + 'core/data', + 'skills' + ]; + + private requiredFiles = [ + 'core/config/config.json', + 'core/config/checksum.json', + 'core/data/expressions.json', + 'core/data/modules.json' + ]; + + constructor() { + this.runChecks(); + } + + private runChecks() { + console.log('šŸ” Running pre-startup checks...'); + + // Check required directories + this.requiredPaths.forEach(path => { + if (!existsSync(join(process.cwd(), path))) { + console.error(`āŒ Missing required directory: ${path}`); + process.exit(1); + } + }); + + // Check required files + this.requiredFiles.forEach(file => { + if (!existsSync(join(process.cwd(), file))) { + console.error(`āŒ Missing required file: ${file}`); + process.exit(1); + } + }); + + // Check environment variables + this.checkEnvironment(); + + // Check Python bridge + this.checkPythonBridge(); + + console.log('āœ… All pre-startup checks passed!'); + } + + private checkEnvironment() { + const requiredEnvVars = [ + 'LEON_NODE_ENV', + 'LEON_LANG' + ]; + + requiredEnvVars.forEach(envVar => { + if (!process.env[envVar]) { + console.warn(`āš ļø Environment variable ${envVar} is not set`); + } + }); + + // Validate LEON_NODE_ENV + const validEnvs = ['production', 'development']; + if (!validEnvs.includes(process.env.LEON_NODE_ENV || '')) { + console.error('āŒ LEON_NODE_ENV must be either "production" or "development"'); + process.exit(1); + } + } + + private checkPythonBridge() { + const bridgePath = join(process.cwd(), 'bridges/python/dist'); + + try { + const files = require('fs').readdirSync(bridgePath); + const hasBinary = files.some(file => + file.includes('leon-python-bridge') || + file.endsWith('.exe') + ); + + if (!hasBinary) { + console.error('āŒ Python bridge binary not found'); + console.log('šŸ’” Run: npm run build:python-bridge'); + process.exit(1); + } + } catch (error) { + console.error('āŒ Cannot access Python bridge directory:', error); + process.exit(1); + } + } +} + +// Run pre-check +new PreCheck(); \ No newline at end of file