diff --git a/src/voice_assistant/audio_gateway/app/main.py b/src/voice_assistant/audio_gateway/app/main.py index ecfa135..e8c130d 100644 --- a/src/voice_assistant/audio_gateway/app/main.py +++ b/src/voice_assistant/audio_gateway/app/main.py @@ -281,12 +281,110 @@ async def audio_socket(websocket: WebSocket) -> None: stt: STTAdapter | None = None stt_events_task: asyncio.Task | None = None + + finalization_task: asyncio.Task[None] | None = None + turn_transition_lock = asyncio.Lock() + interrupting_turns: set[int] = set() + completed_turns: set[int] = set() + turn_manager = create_turn_manager() agent: AgentAdapter | None = None tts: TTSAdapter | None = None pipeline: VoicePipeline | None = None + async def finalize_turn( + turn_id: int, + ) -> None: + """ + Complete a turn without blocking WebSocket message reception. + + Normal completion must not race with interruption. + """ + if pipeline is None or session is None: + return + + try: + await pipeline.wait_turn_task() + + async with turn_transition_lock: + if turn_id in interrupting_turns: + return + + if turn_id in completed_turns: + return + + if not pipeline.is_active_turn(turn_id): + return + + pipeline.complete_turn(turn_id) + completed_turns.add(turn_id) + + await send_json( + websocket, + TurnEnded( + session_id=session.session_id, + turn_id=turn_id, + ), + ) + + except asyncio.CancelledError: + raise + + except Exception: + logger.exception( + "Turn finalization failed. session_id=%s turn_id=%s", + session.session_id, + turn_id, + ) + + async def cancel_finalization_task() -> None: + """ + Cancel and await the Gateway-owned finalization task. + + This task is separate from the execution task owned + by VoicePipeline. + """ + nonlocal finalization_task + + task = finalization_task + + if task is None: + return + + if task is asyncio.current_task(): + raise RuntimeError( + "Finalization task cannot cancel itself." + ) + + if not task.done(): + task.cancel() + + try: + await asyncio.wait_for( + asyncio.shield(task), + timeout=5.0, + ) + except asyncio.CancelledError: + current = asyncio.current_task() + + if current is not None and current.cancelling(): + raise + + if not task.cancelled(): + raise + + except TimeoutError: + logger.error( + "Finalization task cancellation timed out. " + "session_id=%s", + session.session_id if session else None, + ) + + finally: + if task.done() and finalization_task is task: + finalization_task = None + try: while True: message = await websocket.receive() @@ -729,7 +827,6 @@ async def audio_socket(websocket: WebSocket) -> None: # Finish the STT session normally. await stt.stop_session() - await pipeline.wait_turn_task() stt_events_task = None @@ -904,6 +1001,15 @@ async def audio_socket(websocket: WebSocket) -> None: pass finally: + # Stop Gateway-owned finalization before provider cleanup. + try: + await cancel_finalization_task() + except Exception: + logger.exception( + "Failed to cancel finalization task. session_id=%s", + session.session_id if session else None, + ) + # Cancel and await the task owned by VoicePipeline. if pipeline is not None: try: diff --git a/src/voice_assistant/audio_gateway/web/favicon.svg b/src/voice_assistant/audio_gateway/web/favicon.svg new file mode 100644 index 0000000..d1f7585 --- /dev/null +++ b/src/voice_assistant/audio_gateway/web/favicon.svg @@ -0,0 +1,5 @@ + + + + diff --git a/src/voice_assistant/audio_gateway/web/index.html b/src/voice_assistant/audio_gateway/web/index.html index 26e6799..32b9918 100644 --- a/src/voice_assistant/audio_gateway/web/index.html +++ b/src/voice_assistant/audio_gateway/web/index.html @@ -1,83 +1,517 @@ - - - - AgentVoice · Voice Console - + + + + +Oracle — assistente de voz + + + + + + -
-
-
-
◉
-
-

AgentVoice

-

Realtime Voice Console

-
-
- Desconectado -
+
+
+ + +
Assistente pessoal de IA
+
+

Assistente de voz Oracle

+
+

AI Data Platform

unifies your enterprise data

+

Oracle 26AI

Powers the AI for Data Revolution

+

GenAI

for building, deploying, and governing AI applications at scale.

+

A-Team

+
+
+
+ +
+ +
-
- VOICE ASSISTANT · V1 -

Uma conversa, naturalmente.

-

Fale pelo microfone e receba respostas - geradas por inteligência artificial.

-
+
+
+ +function releaseAudio(){ + // Close the controller first, before closing its AudioContext. + try{assistantIntegration.onVoiceEnd();}catch(e){console.warn('Integração de voz:',e);} + micSource?.disconnect();micSource=null; + const oldStream=stream;stream=null; + analyser=null;samples=null;spectrum=null;voice=0;audioBands.fill(0); + if(oldStream)oldStream.getTracks().forEach(track=>track.stop()); + const oldContext=audioContext;audioContext=null; + if(oldContext&&oldContext.state!=='closed')oldContext.close().catch(()=>{}); +} +function stopListening(){ + requestId++; + releaseAudio(); + setState('ended','Conversa encerrada.'); +} + +button.addEventListener('click',async()=>{ + if(activeStates.has(state)||voiceController!==null){ + stopListening(); + return; + } + if(state==='requesting')return; + + const currentRequest=++requestId; + setState('requesting','Permita o microfone no navegador para continuar.'); + try{ + if(!window.isSecureContext||!navigator.mediaDevices?.getUserMedia) + throw Object.assign(new Error(),{name:'UnsupportedError'}); + const AudioContextClass=window.AudioContext||window.webkitAudioContext; + if(!AudioContextClass)throw Object.assign(new Error(),{name:'UnsupportedError'}); + + audioContext=new AudioContextClass(); + await audioContext.resume(); + if(currentRequest!==requestId)return; + const acquired=await navigator.mediaDevices.getUserMedia({ + audio:{echoCancellation:true,noiseSuppression:true,autoGainControl:true},video:false + }); + if(currentRequest!==requestId){ + acquired.getTracks().forEach(track=>track.stop()); + return; + } + stream=acquired; + analyser=audioContext.createAnalyser(); + analyser.fftSize=1024; + analyser.smoothingTimeConstant=.65; + samples=new Float32Array(analyser.fftSize); + spectrum=new Uint8Array(analyser.frequencyBinCount); + noiseFloor=.003;voicePeak=.045; + micSource=audioContext.createMediaStreamSource(stream); + micSource.connect(analyser); + stream.getAudioTracks().forEach(track=>track.addEventListener('ended',()=>{ + if(currentRequest===requestId)stopListening(); + },{once:true})); + await assistantIntegration.onVoiceStart({stream,audioContext,analyser},currentRequest); + }catch(error){ + if(currentRequest!==requestId)return; + requestId++; + releaseAudio(); + const messages={ + NotAllowedError:'Acesso ao microfone negado. Permita nas configurações do navegador.', + NotFoundError:'Nenhum microfone encontrado.', + NotReadableError:'Microfone ocupado ou indisponível.', + UnsupportedError:'Use HTTPS ou localhost em um navegador compatível.' + }; + setState('error',messages[error.name]||error.message||'Não foi possível iniciar a conversa.',true); + } +}); +addEventListener('pagehide',()=>{requestId++;releaseAudio();}); +addEventListener('pageshow',event=>{ + if(event.persisted)setState('ended','Microfone encerrado. Você pode iniciar novamente.'); +}); + +// Órbita automática: pausa para leitura, seguida por uma transição em hélice. +const cards=[...document.querySelectorAll('.orbit-card')], dots=[...document.querySelectorAll('.card-progress > span')]; +const cardStage=$('card-stage'), pauseCards=$('card-pause'); +let cardFrame=0,cardLast=0,cardTime=0,cardsPaused=false,cardsHovered=false,cardsFocused=false; +const CARD_HOLD=4.2,CARD_TRANSITION=1.8,CARD_CYCLE=CARD_HOLD+CARD_TRANSITION; +function updateCards(now){ + cardFrame=0;if(document.hidden)return; + if(motion.matches){cards.forEach(card=>{card.style.transform='';card.style.opacity='';card.style.zIndex='';});return;} + const dt=cardLast?Math.min((now-cardLast)/1000,.05):0;cardLast=now; + if(!cardsPaused&&!cardsHovered&&!cardsFocused)cardTime+=dt; + const turn=Math.floor(cardTime/CARD_CYCLE), local=cardTime%CARD_CYCLE; + const t=Math.max(0,Math.min(1,(local-CARD_HOLD)/CARD_TRANSITION)); + const ease=t*t*t*(t*(t*6.-15.)+10.); + const phase=(turn+ease)%cards.length; + const rx=Math.min(cardStage.clientWidth*.29,290); + cards.forEach((card,i)=>{ + const q=((i-phase+cards.length*1.5)%cards.length)-cards.length*.5; + const angle=q*Math.PI/2, depth=Math.cos(angle), sc=.8+.2*depth; + const x=Math.sin(angle)*rx,y=q*62+(1-depth)*18; + card.style.transform=`translate(-50%,-50%) translate3d(${x}px,${y}px,${depth*60}px) scale(${sc}) rotateY(${-q*90}deg)`; + card.style.opacity=String(Math.max(0,depth)*.94+.06);card.style.zIndex=String(Math.round(sc*100)); + }); + dots.forEach((dot,i)=>dot.classList.toggle('active',i===Math.round(phase)%cards.length)); + if(!cardsPaused&&!cardsHovered&&!cardsFocused)cardFrame=requestAnimationFrame(updateCards); +} +function resumeCards(){if(!cardFrame){cardLast=0;cardFrame=requestAnimationFrame(updateCards);}} +pauseCards.addEventListener('click',()=>{ + cardsPaused=!cardsPaused;pauseCards.setAttribute('aria-pressed',String(cardsPaused)); + pauseCards.setAttribute('aria-label',cardsPaused?'Retomar transição automática dos cards':'Pausar transição automática dos cards'); + pauseCards.lastElementChild.textContent=cardsPaused?'Retomar cards':'Pausar cards'; + pauseCards.querySelector('path').setAttribute('d',cardsPaused?'M3 2l7 4-7 4z':'M3 2h2v8H3zm4 0h2v8H7z');resumeCards(); +}); +cardStage.addEventListener('pointerenter',event=>{if(event.pointerType==='mouse'||event.pointerType==='pen')cardsHovered=true;}); +cardStage.addEventListener('pointerleave',()=>{cardsHovered=false;resumeCards();}); +cardStage.addEventListener('focusin',()=>{cardsFocused=true;}); +cardStage.addEventListener('focusout',()=>{cardsFocused=false;resumeCards();}); +addEventListener('resize',resumeCards);motion.addEventListener('change',resumeCards); +document.addEventListener('visibilitychange',()=>{if(document.hidden){cancelAnimationFrame(cardFrame);cardFrame=0;}else resumeCards();}); +resumeCards(); + +// Three.js com versão fixa; carregamento isolado mantém todos os controles disponíveis se o CDN falhar. +let renderer,scene,camera,material,points,raf=0,last=0,time=0,energy=0,activity=0; +const pointer={x:0,y:0}, smoothPointer={x:0,y:0}; +function layout(){ + const w=hero.clientWidth,h=hero.clientHeight; + // O contorno conserva as retas e as extremidades semicirculares do “O”. + const contentHeight=document.querySelector('.content').offsetHeight; + const ringHeight=Math.max(contentHeight+155,Math.min(h*.79,660)); + const ringWidth=Math.max(ringHeight*1.72,Math.min(w*.85,1380)); + hero.style.setProperty('--ring-width',`${ringWidth}px`); + hero.style.setProperty('--ring-height',`${ringHeight}px`); + if(!renderer)return; + renderer.setSize(w,h);camera.left=-w/2;camera.right=w/2;camera.top=h/2;camera.bottom=-h/2;camera.updateProjectionMatrix(); + material.uniforms.uShape.value.set((ringWidth-ringHeight)/2,ringHeight/2); + resume(); +} +layout();const layoutObserver=new ResizeObserver(layout);layoutObserver.observe(hero);layoutObserver.observe(document.querySelector('.content'));document.fonts.ready.then(layout); +if(matchMedia('(hover:hover) and (pointer:fine)').matches){hero.addEventListener('pointermove',event=>{const rect=hero.getBoundingClientRect();pointer.x=(event.clientX-rect.left)/rect.width-.5;pointer.y=(event.clientY-rect.top)/rect.height-.5;},{passive:true});hero.addEventListener('pointerleave',()=>{pointer.x=pointer.y=0;});} + +function frame(now) { + raf = 0; + + if (document.hidden || !renderer) return; + + const dt = last + ? Math.min((now - last) / 1000, 0.05) + : 0.016; + + last = now; + + // TTS may continue playing after turn.ended. + const playbackActive = + voiceController?.player?.playing === true; + + const ttsSpeaking = + state === "speaking" || + ( + state === "waiting-playback" && + playbackActive + ); + + // Select the audio source according to conversation state. + const microphoneActive = [ + "listening", + "starting", + "recording", + "flushing" + ].includes(state); + + const activeAnalyser = ttsSpeaking + ? voiceController?.getPlaybackAnalyser() + : microphoneActive + ? analyser + : null; + + if (activeAnalyser && samples && spectrum) { + activeAnalyser.getFloatTimeDomainData(samples); + activeAnalyser.getByteFrequencyData(spectrum); + + let sum = 0; + + for (let i = 0; i < samples.length; i++) { + sum += samples[i] * samples[i]; + } + + const rms = Math.sqrt(sum / samples.length); + + noiseFloor += + (Math.min(rms, 0.012) - noiseFloor) * + (1 - Math.exp(-dt * 0.5)); + + voicePeak = Math.max( + 0.025, + rms, + voicePeak * Math.exp(-dt * 0.22) + ); + + const gate = Math.max(0.002, noiseFloor * 1.45); + + voice = Math.pow( + Math.min( + 1, + Math.max(0, rms - gate) / + Math.max(0.012, voicePeak - gate) + ), + 0.65 + ); + + const hzPerBin = + audioContext.sampleRate / activeAnalyser.fftSize; + + const limits = [100, 450, 1500, 5000]; + + for (let band = 0; band < 3; band++) { + const start = Math.max( + 1, + Math.floor(limits[band] / hzPerBin) + ); + + const end = Math.min( + spectrum.length, + Math.ceil(limits[band + 1] / hzPerBin) + ); + + let total = 0; + + for (let i = start; i < end; i++) { + total += spectrum[i] / 255; + } + + const target = + (total / Math.max(1, end - start)) * voice; + + audioBands[band] += + (target - audioBands[band]) * + (1 - Math.exp(-dt * 10)); + } + } else { + voice = 0; + audioBands.fill(0); + } + + energy += + (voice - energy) * + (1 - Math.exp(-dt * (voice > energy ? 19 : 4.5))); + + const targetActivity = + ttsSpeaking ? 1 : + state === "recording" ? 0.7 : + state === "listening" ? 0.35 : + state === "processing" ? 0.15 : + state === "requesting" ? 0.35 : 0; + + activity += + (targetActivity - activity) * + (1 - Math.exp(-dt * 3)); + + if (!motion.matches) { + time += dt; + } + + smoothPointer.x += + (pointer.x - smoothPointer.x) * 0.035; + + smoothPointer.y += + (pointer.y - smoothPointer.y) * 0.035; + + const u = material.uniforms; + + u.uTime.value = time; + u.uVoice.value = motion.matches ? 0 : energy; + u.uActivity.value = activity; + + u.uBands.value.set( + audioBands[0], + audioBands[1], + audioBands[2] + ); + + u.uPointer.value.set( + motion.matches ? 0 : smoothPointer.x, + motion.matches ? 0 : smoothPointer.y + ); + + renderer.render(scene, camera); + + const keepAnimating = + !motion.matches || + microphoneActive || + ttsSpeaking || + playbackActive || + state === "requesting" || + Math.abs(activity) > 0.005; + + if (keepAnimating) { + raf = requestAnimationFrame(frame); + } +} + +function resume(){if(!raf){last=0;raf=requestAnimationFrame(frame);}} +document.addEventListener('visibilitychange',()=>{if(document.hidden){cancelAnimationFrame(raf);raf=0;last=0;}else resume();});motion.addEventListener('change',resume); +new MutationObserver(resume).observe(hero,{attributes:true,attributeFilter:['data-state']}); +async function initVisual(){ + try{ + const THREE=await import('https://cdn.jsdelivr.net/npm/three@0.170.0/build/three.module.js'); + renderer=new THREE.WebGLRenderer({alpha:true,antialias:false,powerPreference:'low-power'}); + renderer.setPixelRatio(Math.min(devicePixelRatio||1,innerWidth<601?1.5:1.75)); + renderer.setClearColor(0x030305,0);scene=new THREE.Scene();camera=new THREE.OrthographicCamera(-1,1,1,-1,.1,100);camera.position.z=20; + const small=innerWidth<601||(navigator.hardwareConcurrency||4)<=4; + const filaments=small?38:60,steps=small?650:1250,count=filaments*steps; + const positions=new Float32Array(count*3),params=new Float32Array(count*3); + for(let j=0;j.5)discard;float light=exp(-d*d*18.);gl_FragColor=vec4(vColor,vAlpha*light);} + `}); + points=new THREE.Points(geometry,material);points.frustumCulled=false;scene.add(points); + // Halo compartilha a deformação dos filamentos, sem pós-processamento de tela inteira. + const glowMaterial=material.clone();glowMaterial.uniforms=material.uniforms; + glowMaterial.vertexShader=material.vertexShader.replace('gl_PointSize=(1.55+core*1.1+seed*.45+uVoice*1.3)*uDpr;','gl_PointSize=(15.+core*13.+uVoice*12.)*uDpr;').replace('vAlpha=(.28+core*.42)*(.65+seed*.35)*(1.+uVoice*.55);','vAlpha=.027*(.6+core)*(1.+uVoice*.7);'); + const glow=new THREE.Points(geometry,glowMaterial);glow.frustumCulled=false;glow.renderOrder=-1;scene.add(glow); + $('visual').appendChild(renderer.domElement);renderer.domElement.setAttribute('aria-hidden','true'); + renderer.domElement.addEventListener('webglcontextlost',event=>{event.preventDefault();cancelAnimationFrame(raf);raf=0;hero.classList.remove('webgl-ready');}); + renderer.domElement.addEventListener('webglcontextrestored',()=>{hero.classList.add('webgl-ready');resume();}); + layout();renderer.render(scene,camera);hero.classList.add('webgl-ready');resume(); + }catch(error){console.warn('Anel WebGL indisponível; composição CSS ativa.',error);} +} +initVisual(); +})(); + diff --git a/src/voice_assistant/audio_gateway/web/package.json b/src/voice_assistant/audio_gateway/web/package.json new file mode 100644 index 0000000..e986b24 --- /dev/null +++ b/src/voice_assistant/audio_gateway/web/package.json @@ -0,0 +1,4 @@ +{ + "private": true, + "type": "module" +} diff --git a/src/voice_assistant/audio_gateway/web/pcm-player.js b/src/voice_assistant/audio_gateway/web/pcm-player.js new file mode 100644 index 0000000..6597876 --- /dev/null +++ b/src/voice_assistant/audio_gateway/web/pcm-player.js @@ -0,0 +1,224 @@ + +/** + * PCM playback for AgentVoice. + * + * Responsibilities: + * - Decode raw PCM signed 16-bit little-endian. + * - Schedule audio chunks in arrival order. + * - Track actual playback completion. + * - Stop scheduled audio immediately. + * + * Non-responsibilities: + * - WebSocket communication. + * - Voice activity detection. + * - Session or turn lifecycle. + */ + +export class PCMPlayer { + constructor({ + audioContext, + sampleRate = 48000, + onPlaybackStart = () => {}, + onPlaybackEnd = () => {}, + } = {}) { + if (!audioContext) { + throw new Error("AudioContext is required."); + } + + if (!Number.isFinite(sampleRate) || sampleRate <= 0) { + throw new Error("Invalid PCM sample rate."); + } + + this.context = audioContext; + this.sampleRate = sampleRate; + this.analyser = this.context.createAnalyser(); + this.analyser.fftSize = 1024; + this.analyser.smoothingTimeConstant = 0.65; + + this.analyser.connect(this.context.destination); + + this.onPlaybackStart = onPlaybackStart; + this.onPlaybackEnd = onPlaybackEnd; + + this.sources = new Set(); + this.nextStartTime = 0; + this.playing = false; + this.finished = false; + this.generation = 0; + + this.waiters = []; + } + + createAnalyser() { + return { + fftSize: 1024, + smoothingTimeConstant: 0.65, + connect() {}, + disconnect() {}, + }; + } + + getAnalyser() { + return this.analyser; + } + + decodePCM(buffer) { + if (!(buffer instanceof ArrayBuffer)) { + throw new TypeError("PCM chunk must be an ArrayBuffer."); + } + + if (buffer.byteLength % 2 !== 0) { + throw new Error("Invalid PCM byte length."); + } + + const view = new DataView(buffer); + const samples = new Float32Array(buffer.byteLength / 2); + + for (let i = 0; i < samples.length; i++) { + samples[i] = view.getInt16(i * 2, true) / 32768; + } + + return samples; + } + + async enqueue(buffer) { + if (this.finished) { + throw new Error("Playback has already been finalized."); + } + + const samples = this.decodePCM(buffer); + + if (samples.length === 0) { + return; + } + + await this.context.resume(); + + if (this.finished) { + throw new Error("Playback was finalized during enqueue."); + } + + const audioBuffer = this.context.createBuffer( + 1, + samples.length, + this.sampleRate + ); + + audioBuffer.copyToChannel(samples, 0); + + const source = this.context.createBufferSource(); + source.buffer = audioBuffer; + source.connect(this.analyser); + + const startAt = Math.max( + this.context.currentTime + 0.01, + this.nextStartTime + ); + + this.nextStartTime = startAt + audioBuffer.duration; + + const generation = this.generation; + + this.sources.add(source); + + if (!this.playing) { + this.playing = true; + this.onPlaybackStart(); + } + + source.onended = () => { + source.disconnect(); + this.sources.delete(source); + + if (generation !== this.generation) { + return; + } + + this.checkCompletion(); + }; + + try { + source.start(startAt); + } catch (error) { + this.sources.delete(source); + source.disconnect(); + this.checkCompletion(); + throw error; + } + } + + finish() { + this.finished = true; + this.checkCompletion(); + } + + checkCompletion() { + if (!this.finished || this.sources.size > 0) { + return; + } + + const wasPlaying = this.playing; + this.playing = false; + this.nextStartTime = 0; + + if (wasPlaying) { + this.onPlaybackEnd(); + } + + this.resolveWaiters(); + } + + waitForCompletion() { + if (this.finished && this.sources.size === 0) { + return Promise.resolve(); + } + + return new Promise((resolve) => { + this.waiters.push(resolve); + }); + } + + resolveWaiters() { + const waiters = this.waiters.splice(0); + + for (const resolve of waiters) { + resolve(); + } + } + + stop() { + this.generation++; + + for (const source of this.sources) { + source.onended = null; + + try { + source.stop(); + } catch { + // Source may already have finished. + } + + source.disconnect(); + } + + this.sources.clear(); + this.nextStartTime = 0; + + const wasPlaying = this.playing; + this.playing = false; + this.finished = true; + + if (wasPlaying) { + this.onPlaybackEnd(); + } + + this.resolveWaiters(); + } + + reset() { + this.stop(); + + this.finished = false; + this.nextStartTime = 0; + } +} diff --git a/src/voice_assistant/audio_gateway/web/pcm-worklet.js b/src/voice_assistant/audio_gateway/web/pcm-worklet.js index aea2fb2..ff83ef5 100644 --- a/src/voice_assistant/audio_gateway/web/pcm-worklet.js +++ b/src/voice_assistant/audio_gateway/web/pcm-worklet.js @@ -2,21 +2,90 @@ class PCMInputProcessor extends AudioWorkletProcessor { constructor() { super(); + this.targetRate = 16000; this.step = sampleRate / this.targetRate; + this.chunkSamples = 1600; + this.buffer = []; this.position = 0; - this.sequence = 0; - this.chunkSamples = 1600; this.output = []; + this.sequence = 0; + + this.capturing = false; + + this.port.onmessage = ({ data }) => { + switch (data?.type) { + case "start": + this.reset(); + this.capturing = true; + break; + + case "flush": + this.capturing = false; + this.flush(); + break; + + case "stop": + this.capturing = false; + this.reset(); + break; + } + }; + } + + reset() { + this.buffer = []; + this.position = 0; + this.output = []; + this.sequence = 0; + } + + emitPCM() { + if (!this.output.length) return; + + const pcm = new Int16Array(this.output); + this.output = []; + + this.port.postMessage({ + type: "pcm", + sequence: this.sequence++, + buffer: pcm.buffer + }, [pcm.buffer]); + } + + flush() { + while (this.position + 1 < this.buffer.length) { + const index = Math.floor(this.position); + const fraction = this.position - index; + + const sample = + this.buffer[index] * (1 - fraction) + + this.buffer[index + 1] * fraction; + + const clipped = Math.max(-1, Math.min(1, sample)); + + this.output.push( + clipped < 0 + ? Math.round(clipped * 32768) + : Math.round(clipped * 32767) + ); + + this.position += this.step; + } + + this.emitPCM(); + this.reset(); + + this.port.postMessage({ type: "flushed" }); } process(inputs) { + if (!this.capturing) return true; + const channels = inputs[0]; - if (!channels || channels.length === 0) { - return true; - } + if (!channels?.length) return true; const frameLength = channels[0].length; @@ -49,14 +118,7 @@ class PCMInputProcessor extends AudioWorkletProcessor { this.position += this.step; if (this.output.length >= this.chunkSamples) { - const pcm = new Int16Array(this.output); - this.output = []; - - this.port.postMessage({ - type: "pcm", - sequence: this.sequence++, - buffer: pcm.buffer - }, [pcm.buffer]); + this.emitPCM(); } } diff --git a/src/voice_assistant/audio_gateway/web/vad.js b/src/voice_assistant/audio_gateway/web/vad.js new file mode 100644 index 0000000..05a82ff --- /dev/null +++ b/src/voice_assistant/audio_gateway/web/vad.js @@ -0,0 +1,92 @@ + +export class VoiceActivityDetector { + constructor({ + onSpeechStart, + onSpeechEnd, + minSpeechMs = 250, + silenceMs = 800, + threshold = 0.012, + noiseMultiplier = 3, + } = {}) { + this.onSpeechStart = onSpeechStart; + this.onSpeechEnd = onSpeechEnd; + + this.minSpeechMs = minSpeechMs; + this.silenceMs = silenceMs; + this.threshold = threshold; + this.noiseMultiplier = noiseMultiplier; + + this.noiseFloor = 0.003; + this.speaking = false; + this.speechStartedAt = null; + this.lastVoiceAt = null; + this.active = false; + } + + start() { + this.active = true; + this.reset(); + } + + stop() { + this.active = false; + this.reset(); + } + + reset() { + this.speaking = false; + this.speechStartedAt = null; + this.lastVoiceAt = null; + } + + process(rms, now = performance.now()) { + if (!this.active) return; + + if (!Number.isFinite(rms) || rms < 0) return; + + const threshold = Math.max( + this.threshold, + this.noiseFloor * this.noiseMultiplier + ); + + const voiced = rms >= threshold; + + if (!this.speaking) { + // Update noise estimation only outside detected speech. + if (!voiced) { + this.noiseFloor = + this.noiseFloor * 0.98 + rms * 0.02; + } + + if (voiced) { + if (this.speechStartedAt === null) { + this.speechStartedAt = now; + } + + if (now - this.speechStartedAt >= this.minSpeechMs) { + this.speaking = true; + this.lastVoiceAt = now; + this.onSpeechStart?.(); + } + } else { + this.speechStartedAt = null; + } + + return; + } + + if (voiced) { + this.lastVoiceAt = now; + return; + } + + if ( + this.lastVoiceAt !== null && + now - this.lastVoiceAt >= this.silenceMs + ) { + this.speaking = false; + this.onSpeechEnd?.(); + this.reset(); + } + } +} diff --git a/src/voice_assistant/audio_gateway/web/voice-session.js b/src/voice_assistant/audio_gateway/web/voice-session.js new file mode 100644 index 0000000..8c1da6e --- /dev/null +++ b/src/voice_assistant/audio_gateway/web/voice-session.js @@ -0,0 +1,479 @@ + +import { VoiceActivityDetector } from "./vad.js"; +import { PCMPlayer } from "./pcm-player.js"; + +export class VoiceSessionController { + constructor({ + onStateChange = () => {}, + onTranscript = () => {}, + onAgentText = () => {}, + onError = () => {}, + silenceMs = 800, + prebufferChunks = 4, + maxPendingChunks = 100, + } = {}) { + this.onStateChange = onStateChange; + this.onTranscript = onTranscript; + this.onAgentText = onAgentText; + this.onError = onError; + + this.prebufferChunks = prebufferChunks; + this.maxPendingChunks = maxPendingChunks; + + this.state = "closed"; + this.ws = null; + this.stream = null; + this.context = null; + this.analyser = null; + this.worklet = null; + this.source = null; + this.silentGain = null; + + this.turnId = null; + this.sequence = 0; + this.prebuffer = []; + this.pending = []; + this.stopRequested = false; + this.frameId = 0; + this.rmsSamples = null; + + this.vad = new VoiceActivityDetector({ + silenceMs, + onSpeechStart: () => this.beginTurn(), + onSpeechEnd: () => this.finishSpeech(), + }); + + this.player = null; + this.playbackPending = Promise.resolve(); + this.gatewayTurnEnded = false; + this.playbackGeneration = 0; + } + + getPlaybackAnalyser() { + return this.player?.getAnalyser() ?? null; + } + + setState(state) { + this.state = state; + this.onStateChange(state); + } + + send(event) { + if (this.ws?.readyState !== WebSocket.OPEN) { + throw new Error("WebSocket indisponível."); + } + this.ws.send(JSON.stringify(event)); + } + + async start({ stream, audioContext, analyser }) { + if (this.state !== "closed") { + throw new Error("Sessão já iniciada."); + } + + this.stream = stream; + this.context = audioContext; + this.analyser = analyser; + + this.player = new PCMPlayer({ + audioContext, + sampleRate: 48000, + + onPlaybackStart: () => { + if (this.state !== "closed") { + this.setState("speaking"); + } + }, + }); + this.rmsSamples = new Float32Array(analyser.fftSize); + + this.setState("connecting"); + + try { + await this.context.audioWorklet.addModule( + "/web/pcm-worklet.js" + ); + + this.source = this.context.createMediaStreamSource(stream); + this.worklet = new AudioWorkletNode( + this.context, + "pcm-input" + ); + + this.silentGain = this.context.createGain(); + this.silentGain.gain.value = 0; + + this.source.connect(this.worklet); + this.worklet.connect(this.silentGain); + this.silentGain.connect(this.context.destination); + + this.worklet.port.onmessage = ({ data }) => { + this.handleWorkletMessage(data); + }; + + const scheme = location.protocol === "https:" ? "wss" : "ws"; + const socket = new WebSocket( + `${scheme}://${location.host}/ws/audio` + ); + + this.ws = socket; + socket.binaryType = "arraybuffer"; + + socket.onopen = () => { + if (this.ws !== socket) return; + + this.send({ + type: "session.start", + audio: { + encoding: "pcm_s16le", + sample_rate: 16000, + channels: 1, + }, + }); + }; + + socket.onmessage = ({ data }) => { + if (this.ws !== socket) return; + + if (typeof data === "string") { + try { + this.handleGatewayEvent(JSON.parse(data)); + } catch (error) { + this.fail(error); + } + } else if (data instanceof ArrayBuffer) { + this.handleAudioChunk(data); + } + }; + + socket.onerror = () => { + if (this.ws === socket) { + this.fail(new Error("Falha na conexão WebSocket.")); + } + }; + + socket.onclose = () => { + if (this.ws === socket && this.state !== "closed") { + this.fail(new Error("Conexão WebSocket encerrada.")); + } + }; + } catch (error) { + this.close(); + throw error; + } + } + + startListening() { + this.playbackGeneration++; + + this.gatewayTurnEnded = false; + this.player?.reset(); + this.playbackPending = Promise.resolve(); + + this.turnId = null; + this.sequence = 0; + this.prebuffer = []; + this.pending = []; + this.stopRequested = false; + + this.worklet.port.postMessage({ type: "start" }); + this.vad.start(); + this.setState("listening"); + + this.monitorVoice(); + } + + monitorVoice() { + cancelAnimationFrame(this.frameId); + + const tick = (now) => { + if (this.state !== "listening" && + this.state !== "starting" && + this.state !== "recording") { + return; + } + + this.analyser.getFloatTimeDomainData(this.rmsSamples); + + let power = 0; + for (const sample of this.rmsSamples) { + power += sample * sample; + } + + const rms = Math.sqrt(power / this.rmsSamples.length); + + if (!this.stopRequested) { + this.vad.process(rms, now); + } + + this.frameId = requestAnimationFrame(tick); + }; + + this.frameId = requestAnimationFrame(tick); + } + + beginTurn() { + if (this.state !== "listening") return; + + this.pending = [...this.prebuffer]; + this.setState("starting"); + + try { + this.send({ type: "audio.start" }); + } catch (error) { + this.fail(error); + } + } + + handleWorkletMessage(data) { + if (data?.type === "pcm") { + this.handlePCM(data.buffer); + } else if (data?.type === "flushed") { + this.handleFlushed(); + } + } + + handlePCM(buffer) { + if (this.state === "listening") { + this.prebuffer.push(buffer); + + if (this.prebuffer.length > this.prebufferChunks) { + this.prebuffer.shift(); + } + return; + } + + if (this.state === "starting") { + this.pending.push(buffer); + + if (this.pending.length > this.maxPendingChunks) { + this.fail(new Error("Buffer de início de turno excedido.")); + } + return; + } + + if (this.state === "recording" || + this.state === "flushing") { + this.sendPCM(buffer); + } + } + + sendPCM(buffer) { + if (this.turnId === null) return; + + if (this.ws?.readyState !== WebSocket.OPEN) return; + + this.send({ + type: "audio.chunk", + turn_id: this.turnId, + sequence_number: this.sequence++, + }); + + this.ws.send(buffer); + } + + finishSpeech() { + if (this.stopRequested) return; + + if (this.state !== "starting" && + this.state !== "recording") return; + + this.stopRequested = true; + this.vad.stop(); + + if (this.state === "recording") { + this.flushTurn(); + } + // If still starting, flush after turn.started. + } + + flushTurn() { + if (this.state !== "recording") return; + + this.setState("flushing"); + this.worklet.port.postMessage({ type: "flush" }); + } + + handleFlushed() { + if (this.state !== "flushing") return; + + this.send({ + type: "audio.stop", + turn_id: this.turnId, + }); + + this.setState("processing"); + cancelAnimationFrame(this.frameId); + } + + handleAudioChunk(buffer) { + if ( + this.state === "closed" || + this.turnId === null || + this.gatewayTurnEnded + ) { + return; + } + + const generation = this.playbackGeneration; + + this.playbackPending = this.playbackPending + .then(async () => { + if (generation !== this.playbackGeneration) return; + await this.player.enqueue(buffer); + }) + .catch((error) => { + if (generation === this.playbackGeneration) { + this.fail(error); + } + }); + } + + async finishPlaybackAndResume() { + const generation = this.playbackGeneration; + + try { + await this.playbackPending; + + if (generation !== this.playbackGeneration) return; + if (this.state === "closed") return; + + this.player.finish(); + await this.player.waitForCompletion(); + + if (generation !== this.playbackGeneration) return; + if (this.state === "closed") return; + if (!this.gatewayTurnEnded) return; + + this.startListening(); + } catch (error) { + if (generation === this.playbackGeneration) { + this.fail(error); + } + } + } + + + + handleGatewayEvent(event) { + switch (event.type) { + case "session.ready": + this.startListening(); + break; + + case "turn.started": + if (this.state !== "starting") return; + + this.turnId = event.turn_id; + this.sequence = 0; + + this.setState("recording"); + + for (const buffer of this.pending) { + this.sendPCM(buffer); + } + + this.pending = []; + + if (this.stopRequested) { + this.flushTurn(); + } + break; + + case "stt.partial": + case "stt.final": + if (event.turn_id === this.turnId) { + this.onTranscript(event.text, event.type === "stt.final"); + } + break; + + case "agent.delta": + if (event.turn_id === this.turnId) { + this.onAgentText(event.text, false); + } + break; + + case "agent.response": + if (event.turn_id === this.turnId) { + this.onAgentText(event.text, true); + } + break; + + case "turn.ended": + if (event.turn_id === this.turnId) { + this.gatewayTurnEnded = true; + this.setState("waiting-playback"); + + void this.finishPlaybackAndResume(); + } + break; + + case "interrupted": + this.close(); + break; + + case "error": + this.fail( + new Error(`${event.code}: ${event.message}`) + ); + break; + + case "session.ended": + this.close(); + break; + } + } + + fail(error) { + this.onError(error); + this.close(); + } + + close() { + if (this.state === "closed") return; + + this.playbackGeneration++; + this.player?.stop(); + + this.vad.stop(); + cancelAnimationFrame(this.frameId); + + if (this.worklet) { + this.worklet.port.postMessage({ type: "stop" }); + this.worklet.port.onmessage = null; + this.worklet.disconnect(); + } + + this.source?.disconnect(); + this.silentGain?.disconnect(); + + const socket = this.ws; + this.ws = null; + + if (socket) { + socket.onclose = null; + socket.onerror = null; + socket.onmessage = null; + + if (socket.readyState === WebSocket.OPEN) { + if (this.turnId === null) { + socket.send(JSON.stringify({ type: "session.end" })); + } else { + socket.send(JSON.stringify({ + type: "interrupt", + turn_id: this.turnId, + })); + } + } + + socket.close(); + } + + this.worklet = null; + this.source = null; + this.silentGain = null; + this.turnId = null; + this.prebuffer = []; + this.pending = []; + this.setState("closed"); + } +} diff --git a/tests/js/pcm-player.test.mjs b/tests/js/pcm-player.test.mjs new file mode 100644 index 0000000..a4501a7 --- /dev/null +++ b/tests/js/pcm-player.test.mjs @@ -0,0 +1,204 @@ + +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { PCMPlayer } from + "../../src/voice_assistant/audio_gateway/web/pcm-player.js"; + +class FakeAudioSource { + constructor(context) { + this.context = context; + this.buffer = null; + this.onended = null; + this.startedAt = null; + this.stopped = false; + } + + connect() {} + + disconnect() {} + + start(time) { + this.startedAt = time; + this.context.sources.push(this); + } + + stop() { + this.stopped = true; + } + + end() { + this.onended?.(); + } +} + +class FakeAudioContext { + constructor() { + this.currentTime = 0; + this.destination = {}; + this.sources = []; + } + + async resume() {} + + createAnalyser() { + return { + fftSize: 1024, + smoothingTimeConstant: 0.65, + + connect() {}, + disconnect() {}, + + getFloatTimeDomainData(array) { + array.fill(0); + }, + + getByteFrequencyData(array) { + array.fill(0); + }, + }; + } + + createBuffer(channels, length, sampleRate) { + return { + duration: length / sampleRate, + copyToChannel(samples) { + this.samples = samples; + }, + }; + } + + createBufferSource() { + return new FakeAudioSource(this); + } +} + +function pcm(values) { + const buffer = new ArrayBuffer(values.length * 2); + const view = new DataView(buffer); + + values.forEach((value, index) => { + view.setInt16(index * 2, value, true); + }); + + return buffer; +} + +test("decodes signed PCM16 correctly", () => { + const player = new PCMPlayer({ + audioContext: new FakeAudioContext(), + }); + + const samples = player.decodePCM( + pcm([-32768, 0, 16384, 32767]) + ); + + assert.equal(samples[0], -1); + assert.equal(samples[1], 0); + assert.equal(samples[2], 0.5); + assert.ok(samples[3] > 0.99); +}); + +test("schedules chunks in order", async () => { + const context = new FakeAudioContext(); + const player = new PCMPlayer({ + audioContext: context, + }); + + await player.enqueue(pcm([100, 200, 300, 400])); + await player.enqueue(pcm([500, 600, 700, 800])); + + assert.equal(context.sources.length, 2); + + const first = context.sources[0]; + const second = context.sources[1]; + + assert.ok(second.startedAt >= + first.startedAt + first.buffer.duration); +}); + +test("waits for actual playback completion", async () => { + const context = new FakeAudioContext(); + const player = new PCMPlayer({ + audioContext: context, + }); + + await player.enqueue(pcm([100, 200])); + + player.finish(); + + let completed = false; + + const waiting = player.waitForCompletion().then(() => { + completed = true; + }); + + await Promise.resolve(); + + assert.equal(completed, false); + + context.sources[0].end(); + + await waiting; + + assert.equal(completed, true); +}); + +test("stop interrupts scheduled playback", async () => { + const context = new FakeAudioContext(); + const player = new PCMPlayer({ + audioContext: context, + }); + + await player.enqueue(pcm([100, 200])); + + player.stop(); + + assert.equal(context.sources[0].stopped, true); + assert.equal(player.sources.size, 0); + assert.equal(player.playing, false); +}); + +test("reset allows a new playback turn", async () => { + const context = new FakeAudioContext(); + const player = new PCMPlayer({ + audioContext: context, + }); + + await player.enqueue(pcm([100, 200])); + player.finish(); + context.sources[0].end(); + + await player.waitForCompletion(); + + player.reset(); + + await player.enqueue(pcm([300, 400])); + + assert.equal(context.sources.length, 2); + assert.equal(player.finished, false); +}); + +test("rejects invalid PCM buffers", () => { + const player = new PCMPlayer({ + audioContext: new FakeAudioContext(), + }); + + assert.throws( + () => player.decodePCM(new ArrayBuffer(3)), + /Invalid PCM byte length/ + ); +}); + +test("exposes playback analyser", () => { + const context = new FakeAudioContext(); + + const player = new PCMPlayer({ + audioContext: context, + }); + + assert.equal( + player.getAnalyser(), + player.analyser + ); +}); diff --git a/tests/js/vad.test.mjs b/tests/js/vad.test.mjs new file mode 100644 index 0000000..c3f269e --- /dev/null +++ b/tests/js/vad.test.mjs @@ -0,0 +1,83 @@ + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { VoiceActivityDetector } from "../../src/voice_assistant/audio_gateway/web/vad.js"; + +test("detects speech start after minimum duration", () => { + let starts = 0; + + const vad = new VoiceActivityDetector({ + minSpeechMs: 250, + onSpeechStart: () => starts++, + }); + + vad.start(); + + vad.process(0.04, 0); + vad.process(0.04, 100); + + assert.equal(starts, 0); + + vad.process(0.04, 260); + + assert.equal(starts, 1); +}); + +test("detects speech end after silence", () => { + let ends = 0; + + const vad = new VoiceActivityDetector({ + minSpeechMs: 100, + silenceMs: 800, + onSpeechEnd: () => ends++, + }); + + vad.start(); + + vad.process(0.04, 0); + vad.process(0.04, 120); + vad.process(0.04, 300); + + vad.process(0.001, 600); + vad.process(0.001, 1000); + + assert.equal(ends, 0); + + vad.process(0.001, 1101); + + assert.equal(ends, 1); +}); + +test("ignores short noise spikes", () => { + let starts = 0; + + const vad = new VoiceActivityDetector({ + minSpeechMs: 250, + onSpeechStart: () => starts++, + }); + + vad.start(); + + vad.process(0.05, 0); + vad.process(0.001, 80); + vad.process(0.001, 300); + + assert.equal(starts, 0); +}); + +test("stop disables detection", () => { + let starts = 0; + + const vad = new VoiceActivityDetector({ + minSpeechMs: 100, + onSpeechStart: () => starts++, + }); + + vad.start(); + vad.stop(); + + vad.process(0.05, 0); + vad.process(0.05, 200); + + assert.equal(starts, 0); +}); diff --git a/tests/js/voice-session.test.mjs b/tests/js/voice-session.test.mjs new file mode 100644 index 0000000..adf0d5d --- /dev/null +++ b/tests/js/voice-session.test.mjs @@ -0,0 +1,339 @@ + +import assert from "node:assert/strict"; +import { afterEach, test } from "node:test"; + +import { VoiceSessionController } from + "../../src/voice_assistant/audio_gateway/web/voice-session.js"; + +const originalWebSocket = globalThis.WebSocket; + +class FakeWebSocket { + static OPEN = 1; + static CLOSED = 3; + + constructor() { + this.readyState = FakeWebSocket.OPEN; + this.sent = []; + this.closed = false; + } + + send(data) { + this.sent.push(data); + } + + close() { + this.closed = true; + this.readyState = FakeWebSocket.CLOSED; + } +} + +globalThis.WebSocket = FakeWebSocket; +globalThis.cancelAnimationFrame = () => {}; + +afterEach(() => { + // Restore only state modified by individual tests. +}); + +function makeController() { + const states = []; + const errors = []; + const workletMessages = []; + + const controller = new VoiceSessionController({ + onStateChange: (state) => states.push(state), + onError: (error) => errors.push(error), + }); + + controller.ws = new FakeWebSocket(); + controller.worklet = { + port: { + postMessage: (data) => workletMessages.push(data), + onmessage: null, + }, + disconnect() {}, + }; + + controller.source = { disconnect() {} }; + controller.silentGain = { disconnect() {} }; + + return { + controller, + states, + errors, + workletMessages, + }; +} + +test("speech detection requests a new turn", () => { + const { controller } = makeController(); + + controller.setState("listening"); + controller.beginTurn(); + + assert.equal(controller.state, "starting"); + + assert.deepEqual( + JSON.parse(controller.ws.sent[0]), + { type: "audio.start" } + ); +}); + +test("PCM is buffered until turn.started", () => { + const { controller } = makeController(); + + controller.setState("listening"); + + controller.handlePCM(new ArrayBuffer(4)); + + assert.equal(controller.prebuffer.length, 1); + assert.equal(controller.ws.sent.length, 0); + + controller.beginTurn(); + controller.handlePCM(new ArrayBuffer(6)); + + assert.equal(controller.pending.length, 2); + assert.equal(controller.ws.sent.length, 1); + + controller.handleGatewayEvent({ + type: "turn.started", + turn_id: 1, + }); + + assert.equal(controller.state, "recording"); + assert.equal(controller.pending.length, 0); + assert.equal(controller.sequence, 2); + assert.equal(controller.ws.sent.length, 5); +}); + +test("PCM metadata precedes each binary frame", () => { + const { controller } = makeController(); + + controller.turnId = 7; + controller.setState("recording"); + + const pcm = new ArrayBuffer(8); + + controller.sendPCM(pcm); + + assert.deepEqual( + JSON.parse(controller.ws.sent[0]), + { + type: "audio.chunk", + turn_id: 7, + sequence_number: 0, + } + ); + + assert.equal(controller.ws.sent[1], pcm); + assert.equal(controller.sequence, 1); +}); + +test("speech end requests worklet flush", () => { + const { controller, workletMessages } = makeController(); + + controller.turnId = 1; + controller.setState("recording"); + + controller.finishSpeech(); + + assert.equal(controller.state, "flushing"); + assert.deepEqual( + workletMessages.at(-1), + { type: "flush" } + ); + + assert.equal(controller.ws.sent.length, 0); +}); + +test("audio.stop is sent after flushed", () => { + const { controller } = makeController(); + + controller.turnId = 1; + controller.setState("flushing"); + + controller.handlePCM(new ArrayBuffer(4)); + controller.handleFlushed(); + + assert.equal(controller.state, "processing"); + + const lastMessage = JSON.parse(controller.ws.sent.at(-1)); + + assert.deepEqual(lastMessage, { + type: "audio.stop", + turn_id: 1, + }); + + assert.equal(controller.ws.sent.length, 3); +}); + +test("speech ending before turn.started is handled", () => { + const { controller, workletMessages } = makeController(); + + controller.setState("listening"); + controller.beginTurn(); + controller.finishSpeech(); + + assert.equal(controller.stopRequested, true); + assert.equal(controller.state, "starting"); + + controller.handleGatewayEvent({ + type: "turn.started", + turn_id: 3, + }); + + assert.equal(controller.state, "flushing"); + assert.deepEqual( + workletMessages.at(-1), + { type: "flush" } + ); +}); + +test("closing a session releases resources", () => { + const { controller, workletMessages } = makeController(); + const socket = controller.ws; + + controller.setState("listening"); + controller.close(); + + assert.equal(controller.state, "closed"); + assert.equal(socket.closed, true); + assert.equal(controller.ws, null); + assert.deepEqual( + workletMessages.at(-1), + { type: "stop" } + ); +}); + +test("gateway errors close the session", () => { + const { controller, errors } = makeController(); + + controller.setState("processing"); + + controller.handleGatewayEvent({ + type: "error", + code: "STT_START_FAILED", + message: "Provider unavailable", + }); + + assert.equal(controller.state, "closed"); + assert.equal(errors.length, 1); + assert.match( + errors[0].message, + /STT_START_FAILED/ + ); +}); + + +test("turn.ended waits for playback before listening", async () => { + const { controller } = makeController(); + + let resolvePlayback; + + controller.player = { + finish() {}, + reset() {}, + stop() {}, + waitForCompletion() { + return new Promise((resolve) => { + resolvePlayback = resolve; + }); + }, + }; + + controller.turnId = 1; + controller.setState("processing"); + + let resumed = false; + + controller.startListening = () => { + resumed = true; + }; + + controller.handleGatewayEvent({ + type: "turn.ended", + turn_id: 1, + }); + + await Promise.resolve(); + await Promise.resolve(); + + assert.equal(resumed, false); + + resolvePlayback(); + + await Promise.resolve(); + await Promise.resolve(); + + assert.equal(resumed, true); +}); + +test("closing during playback prevents listening resume", async () => { + const { controller } = makeController(); + + let resolvePlayback; + + controller.player = { + finish() {}, + reset() {}, + stop() {}, + waitForCompletion() { + return new Promise((resolve) => { + resolvePlayback = resolve; + }); + }, + }; + + controller.turnId = 1; + controller.setState("processing"); + + let resumed = false; + + controller.startListening = () => { + resumed = true; + }; + + controller.handleGatewayEvent({ + type: "turn.ended", + turn_id: 1, + }); + + await Promise.resolve(); + await Promise.resolve(); + + controller.close(); + + resolvePlayback(); + + await Promise.resolve(); + await Promise.resolve(); + + assert.equal(resumed, false); + assert.equal(controller.state, "closed"); +}); + +test("binary audio is forwarded to PCMPlayer", async () => { + const { controller } = makeController(); + + const received = []; + + controller.player = { + async enqueue(buffer) { + received.push(buffer); + }, + finish() {}, + reset() {}, + stop() {}, + waitForCompletion: async () => {}, + }; + + controller.turnId = 1; + controller.setState("processing"); + + const pcm = new ArrayBuffer(8); + + controller.handleAudioChunk(pcm); + + await controller.playbackPending; + + assert.deepEqual(received, [pcm]); +}); diff --git a/tests/test_web_client.py b/tests/test_web_client.py index d532247..f86c63a 100644 --- a/tests/test_web_client.py +++ b/tests/test_web_client.py @@ -11,8 +11,10 @@ def test_web_client_page(): response = client.get("/ui") assert response.status_code == 200 - assert "AgentVoice" in response.text - assert "/web/app.js" in response.text + assert "Oracle — assistente de voz" in response.text + assert 'id="microphone"' in response.text + assert 'id="feedback"' in response.text + assert "assistantIntegration" in response.text def test_web_client_javascript():