implementacao de interrupcao em audio streming
This commit is contained in:
@@ -281,12 +281,110 @@ async def audio_socket(websocket: WebSocket) -> None:
|
||||
stt: STTAdapter | None = None
|
||||
stt_events_task: asyncio.Task | None = None
|
||||
|
||||
|
||||
finalization_task: asyncio.Task[None] | None = None
|
||||
turn_transition_lock = asyncio.Lock()
|
||||
interrupting_turns: set[int] = set()
|
||||
completed_turns: set[int] = set()
|
||||
|
||||
turn_manager = create_turn_manager()
|
||||
agent: AgentAdapter | None = None
|
||||
|
||||
tts: TTSAdapter | None = None
|
||||
pipeline: VoicePipeline | None = None
|
||||
|
||||
async def finalize_turn(
|
||||
turn_id: int,
|
||||
) -> None:
|
||||
"""
|
||||
Complete a turn without blocking WebSocket message reception.
|
||||
|
||||
Normal completion must not race with interruption.
|
||||
"""
|
||||
if pipeline is None or session is None:
|
||||
return
|
||||
|
||||
try:
|
||||
await pipeline.wait_turn_task()
|
||||
|
||||
async with turn_transition_lock:
|
||||
if turn_id in interrupting_turns:
|
||||
return
|
||||
|
||||
if turn_id in completed_turns:
|
||||
return
|
||||
|
||||
if not pipeline.is_active_turn(turn_id):
|
||||
return
|
||||
|
||||
pipeline.complete_turn(turn_id)
|
||||
completed_turns.add(turn_id)
|
||||
|
||||
await send_json(
|
||||
websocket,
|
||||
TurnEnded(
|
||||
session_id=session.session_id,
|
||||
turn_id=turn_id,
|
||||
),
|
||||
)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Turn finalization failed. session_id=%s turn_id=%s",
|
||||
session.session_id,
|
||||
turn_id,
|
||||
)
|
||||
|
||||
async def cancel_finalization_task() -> None:
|
||||
"""
|
||||
Cancel and await the Gateway-owned finalization task.
|
||||
|
||||
This task is separate from the execution task owned
|
||||
by VoicePipeline.
|
||||
"""
|
||||
nonlocal finalization_task
|
||||
|
||||
task = finalization_task
|
||||
|
||||
if task is None:
|
||||
return
|
||||
|
||||
if task is asyncio.current_task():
|
||||
raise RuntimeError(
|
||||
"Finalization task cannot cancel itself."
|
||||
)
|
||||
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
asyncio.shield(task),
|
||||
timeout=5.0,
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
current = asyncio.current_task()
|
||||
|
||||
if current is not None and current.cancelling():
|
||||
raise
|
||||
|
||||
if not task.cancelled():
|
||||
raise
|
||||
|
||||
except TimeoutError:
|
||||
logger.error(
|
||||
"Finalization task cancellation timed out. "
|
||||
"session_id=%s",
|
||||
session.session_id if session else None,
|
||||
)
|
||||
|
||||
finally:
|
||||
if task.done() and finalization_task is task:
|
||||
finalization_task = None
|
||||
|
||||
try:
|
||||
while True:
|
||||
message = await websocket.receive()
|
||||
@@ -729,7 +827,6 @@ async def audio_socket(websocket: WebSocket) -> None:
|
||||
|
||||
# Finish the STT session normally.
|
||||
await stt.stop_session()
|
||||
|
||||
await pipeline.wait_turn_task()
|
||||
stt_events_task = None
|
||||
|
||||
@@ -904,6 +1001,15 @@ async def audio_socket(websocket: WebSocket) -> None:
|
||||
pass
|
||||
|
||||
finally:
|
||||
# Stop Gateway-owned finalization before provider cleanup.
|
||||
try:
|
||||
await cancel_finalization_task()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to cancel finalization task. session_id=%s",
|
||||
session.session_id if session else None,
|
||||
)
|
||||
|
||||
# Cancel and await the task owned by VoicePipeline.
|
||||
if pipeline is not None:
|
||||
try:
|
||||
|
||||
5
src/voice_assistant/audio_gateway/web/favicon.svg
Normal file
5
src/voice_assistant/audio_gateway/web/favicon.svg
Normal file
@@ -0,0 +1,5 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64">
|
||||
<rect width="64" height="64" rx="14" fill="#030305"/>
|
||||
<circle cx="32" cy="32" r="20"
|
||||
fill="none" stroke="#ea171c" stroke-width="6"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 215 B |
@@ -1,83 +1,517 @@
|
||||
|
||||
<!doctype html>
|
||||
<html lang="pt-BR">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport"
|
||||
content="width=device-width, initial-scale=1">
|
||||
<title>AgentVoice · Voice Console</title>
|
||||
<link rel="stylesheet" href="/web/styles.css">
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="theme-color" content="#030305">
|
||||
<meta name="description" content="Assistente de voz para explorar dados e inteligência artificial Oracle.">
|
||||
<title>Oracle — assistente de voz</title>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
|
||||
<link
|
||||
href="https://fonts.googleapis.com/css2?family=Assistant:wght@200..800&family=Inter:wght@100..900&display=swap"
|
||||
rel="stylesheet"
|
||||
>
|
||||
<link rel="icon" type="image/svg+xml" href="/web/favicon.svg">
|
||||
<style>
|
||||
:root{color-scheme:dark;--background:#030305;--foreground:#fff;--oracle-red:#ea171c;--muted:#a0a0a0;--accent-warm:#ffa091;--accent-cool:#8da9fc;--surface-glass:#ffffff0a;--surface-button:#14192366;--border-glass:#ffffff14;--border-glass-strong:#ffffff26}
|
||||
*{box-sizing:border-box}body{margin:0;background:var(--background);color:var(--foreground);font-family:Inter,system-ui,sans-serif}button,input{font:inherit}button{cursor:pointer}button:focus-visible,input:focus-visible{outline:2px solid var(--accent-cool);outline-offset:5px}button:disabled{cursor:wait}button, input{-webkit-tap-highlight-color:transparent}[hidden]{display:none!important}
|
||||
.hero{position:relative;isolation:isolate;display:grid;place-items:center;min-height:100svh;overflow:hidden;padding:100px 24px 72px;background:radial-gradient(ellipse at 50% 3%,#421b182b,transparent 60%),radial-gradient(ellipse at 50% 96%,#42191420,transparent 60%),var(--background)}
|
||||
.ambient,.visual{position:absolute;inset:0;z-index:-2;pointer-events:none}.ambient{background:radial-gradient(ellipse at 50% 20%,#db080d1d,transparent 57%),radial-gradient(ellipse at 50% 85%,#85030910,transparent 53%)}.visual canvas{display:block;width:100%;height:100%;opacity:0;transition:opacity 1.2s}.webgl-ready canvas{opacity:1}
|
||||
/* O halo CSS permanece disponível se o CDN, a GPU ou o WebGL falhar. */
|
||||
.fallback{position:absolute;left:50%;top:calc(50% + 14px);width:var(--ring-width,1120px);height:var(--ring-height,600px);transform:translate(-50%,-50%);border-radius:999px;border:8px solid var(--oracle-red);box-shadow:0 0 20px #ea171cd0,0 0 65px #ea171c80,inset 0 0 35px #ea171c55;opacity:.8;transition:opacity 1.2s}.fallback:before,.fallback:after{content:'';position:absolute;inset:-15px;border:1px solid #ea171c80;border-radius:inherit}.fallback:after{inset:12px;border-color:#ea171c70}.webgl-ready .fallback{opacity:0}
|
||||
.headnote{position:absolute;right:32px;top:34px;font-size:10px;letter-spacing:.16em;text-transform:uppercase;color:#a5a2ae;display:flex;align-items:center;gap:9px}.headnote:before{content:'';width:4px;height:4px;background:var(--accent-warm);border-radius:50%;box-shadow:0 0 8px #ffa09177}
|
||||
.scroll-scene{position:relative}.content{text-align:center;width:100%;max-width:1000px;position:relative}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip-path:inset(50%);white-space:nowrap;border:0}
|
||||
.card-stage{position:relative;height:340px;perspective:1200px;overflow:hidden;isolation:isolate;mask-image:linear-gradient(transparent,#000 7%,#000 92%,transparent)}
|
||||
.orbit-card{position:absolute;left:50%;top:50%;width:clamp(220px,30vw,350px);min-height:256px;padding:32px 36px;text-align:left;border-radius:24px;border:1px solid var(--border-glass-strong);background:radial-gradient(ellipse at 0 0,#ffa09120,transparent 70%),radial-gradient(ellipse at 100% 100%,#8da9fc1c,transparent 72%),linear-gradient(135deg,#201b26d9,#0c0e16e8 65%,#141b2bd9);box-shadow:inset 0 1px 0 #ffffff0c,0 12px 36px #0004;backdrop-filter:blur(16px);font-family:Assistant,system-ui,sans-serif;will-change:transform,opacity;backface-visibility:hidden;transform:translate(-50%,-50%)}
|
||||
.orbit-card h2{margin:0 0 20px;font-size:clamp(36px,4vw,54px);line-height:1.05;font-weight:600;letter-spacing:-.035em;color:var(--oracle-red)}
|
||||
.orbit-card{display:flex;flex-direction:column;align-items:center;justify-content:center;text-align:center}.orbit-card.team-card{display:flex;align-items:center;justify-content:center;text-align:center}.orbit-card.team-card h2{margin:0}
|
||||
.orbit-card p{margin:0;color:#fff;font-size:18px;line-height:1.5;font-weight:400}.orbit-card:nth-child(2){transform:translate(calc(-50% + 220px),calc(-50% + 30px)) scale(.7);opacity:.12}.orbit-card:nth-child(3),.orbit-card:nth-child(4){transform:translate(calc(-50% - 220px),calc(-50% + 100px)) scale(.7);opacity:.12}
|
||||
.card-pause{display:inline-flex;align-items:center;gap:5px;padding:5px 8px;margin-left:8px;font:10px Inter,system-ui,sans-serif;color:#b9b1c2;background:transparent;border:1px solid transparent;border-radius:999px;cursor:pointer}.card-pause:hover{color:#fff;border-color:var(--border-glass-strong);background:var(--surface-glass)}.card-pause svg{width:10px;height:10px}
|
||||
.card-progress{display:flex;align-items:center;justify-content:center;gap:7px;margin:4px 0 23px}.card-progress > span{width:15px;height:2px;border-radius:2px;background:#ffffff26;transition:background .3s,width .3s}.card-progress > span.active{width:28px;background:var(--accent-warm)}
|
||||
.controls{display:flex;justify-content:center;align-items:center;gap:13px;margin-top:32px}.button{position:relative;display:inline-flex;align-items:center;justify-content:center;gap:12px;min-height:54px;padding:0 24px;border-radius:999px;color:#fff;transition:transform .38s cubic-bezier(.3,1.35,.5,1),background-color .3s,box-shadow .3s;font-size:13px;font-weight:500;white-space:nowrap}.button:hover{transform:translateY(-2px)}.button:active{transform:translateY(0) scale(.98)}.primary{border:1px solid transparent;background:linear-gradient(#151827,#11131d) padding-box,linear-gradient(120deg,var(--accent-cool),#d7c3db,var(--accent-warm)) border-box;box-shadow:inset 0 2px 10px #ffffff40,inset 0 -6px 16px #000000bf,inset 0 0 0 1px #ffffff0d,0 8px 24px #0006;padding-left:23px;padding-right:16px}.primary:hover{box-shadow:inset 0 3px 14px #ffffff60,inset 0 -6px 16px #000000bf,0 12px 32px #0009}.primary .icon{display:grid;place-items:center;width:29px;height:29px;border:1px solid #ffffff30;border-radius:50%;background:#ffffff05;color:#e6d8e7}.icon svg{width:14px;height:14px}.feedback{min-height:38px;max-width:410px;margin:17px auto 0;font-size:11px;line-height:1.6;color:#a7a2b0}.feedback.error{color:#ffc0b3}.listening .primary{border-color:#ffa09180}
|
||||
@media(max-width:600px){.hero{padding:75px 20px 65px}.headnote{right:22px;top:29px;font-size:8px;letter-spacing:.1em}.headnote:before{display:none}.card-stage{height:330px;margin:0 -20px}.orbit-card{width:min(74vw,310px);min-height:230px;padding:28px}.orbit-card h2{font-size:42px}.orbit-card p{font-size:17px}.controls{margin-top:0}.button{min-height:50px;min-width:218px}.feedback{max-width:290px;margin-top:16px}.card-progress{margin-bottom:24px}}
|
||||
@media(max-height:580px) and (min-width:601px){.hero{padding:55px 24px 35px}.card-stage{height:250px}.orbit-card{min-height:180px;padding:25px}.orbit-card h2{font-size:38px;margin-bottom:12px}.orbit-card p{font-size:16px}.controls{margin-top:0}.card-progress{margin-bottom:10px}.feedback{margin-top:10px}}
|
||||
@media(prefers-reduced-motion:reduce){*,*:before,*:after{animation:none!important;transition:none!important}.button:hover{transform:none}.scroll-scene{height:auto}.hero{position:relative}.card-stage{height:auto;display:grid;gap:12px;max-width:520px;margin:0 auto;overflow:visible;mask-image:none}.orbit-card,.orbit-card:nth-child(2),.orbit-card:nth-child(3),.orbit-card:nth-child(4){position:relative;left:auto;top:auto;width:100%;min-height:0;transform:none!important;opacity:1!important;will-change:auto;padding:22px 26px}.orbit-card h2{font-size:32px;margin-bottom:10px}.orbit-card p{font-size:16px}.card-progress{display:none}.controls{margin-top:26px}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main class="console">
|
||||
<header class="header">
|
||||
<div class="brand">
|
||||
<div class="brand-icon">◉</div>
|
||||
<div>
|
||||
<h1>AgentVoice</h1>
|
||||
<p>Realtime Voice Console</p>
|
||||
<main class="scroll-scene" id="scroll-scene">
|
||||
<section class="hero" id="hero" aria-label="Assistente de voz Oracle">
|
||||
<div class="ambient" aria-hidden="true"></div>
|
||||
<div class="visual" id="visual" aria-hidden="true"><div class="fallback"></div></div>
|
||||
<div class="headnote">Assistente pessoal de IA</div>
|
||||
<div class="content">
|
||||
<h1 class="sr-only">Assistente de voz Oracle</h1>
|
||||
<div class="card-stage" id="card-stage" aria-label="Dados e inteligência artificial Oracle">
|
||||
<article class="orbit-card" lang="en"><h2>AI Data Platform</h2><p>unifies your enterprise data</p></article>
|
||||
<article class="orbit-card" lang="en"><h2>Oracle 26AI</h2><p>Powers the AI for Data Revolution</p></article>
|
||||
<article class="orbit-card" lang="en"><h2>GenAI</h2><p>for building, deploying, and governing AI applications at scale.</p></article>
|
||||
<article class="orbit-card team-card" lang="en"><h2>A-Team</h2></article>
|
||||
</div>
|
||||
<div class="card-progress"><span class="active" aria-hidden="true"></span><span aria-hidden="true"></span><span aria-hidden="true"></span><span aria-hidden="true"></span><button id="card-pause" class="card-pause" type="button" aria-pressed="false" aria-label="Pausar transição automática dos cards"><svg viewBox="0 0 12 12" fill="currentColor" aria-hidden="true"><path d="M3 2h2v8H3zm4 0h2v8H7z"/></svg><span>Pausar cards</span></button></div>
|
||||
<div class="controls">
|
||||
<button class="button primary" id="microphone" type="button"><span id="mic-label">Iniciar conversa</span><span class="icon"><svg id="mic-icon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.6" stroke-linecap="round" aria-hidden="true"><rect x="9" y="3" width="6" height="11" rx="3"/><path d="M6 10v2a6 6 0 0 0 12 0v-2M12 18v3m-3 0h6"/></svg><svg id="stop-icon" viewBox="0 0 24 24" fill="currentColor" hidden aria-hidden="true"><rect x="6" y="6" width="12" height="12" rx="2"/></svg></span></button>
|
||||
</div>
|
||||
<span id="status" class="status">Desconectado</span>
|
||||
</header>
|
||||
|
||||
<section class="hero">
|
||||
<span class="eyebrow">VOICE ASSISTANT · V1</span>
|
||||
<h2>Uma conversa, naturalmente.</h2>
|
||||
<p>Fale pelo microfone e receba respostas
|
||||
geradas por inteligência artificial.</p>
|
||||
</section>
|
||||
|
||||
<section id="messages"
|
||||
class="messages"
|
||||
aria-live="polite">
|
||||
<div class="empty-state" id="emptyState">
|
||||
<div class="empty-icon">◎</div>
|
||||
<p>Sua conversa aparecerá aqui.</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="controls">
|
||||
<div class="mic-container">
|
||||
<button id="talkButton"
|
||||
class="talk-button"
|
||||
disabled
|
||||
aria-label="Iniciar gravação">
|
||||
<span id="micIcon">🎙</span>
|
||||
</button>
|
||||
<p id="talkLabel">Conecte para começar</p>
|
||||
<p id="feedback" class="feedback" role="status" aria-live="polite" aria-atomic="true">Pronto para conversar. No seu ritmo.</p>
|
||||
</div>
|
||||
|
||||
<div class="actions">
|
||||
<button id="connectButton" class="primary">
|
||||
Conectar
|
||||
</button>
|
||||
<button id="interruptButton"
|
||||
class="secondary" disabled>
|
||||
Interromper
|
||||
</button>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
<script>
|
||||
(() => {
|
||||
'use strict';
|
||||
const $ = id => document.getElementById(id);
|
||||
const hero=$('hero'), button=$('microphone'), feedback=$('feedback');
|
||||
const motion=matchMedia('(prefers-reduced-motion: reduce)');
|
||||
let state='ready', stream=null, audioContext=null, analyser=null, samples=null, spectrum=null, voice=0, requestId=0;
|
||||
let micSource=null;
|
||||
let noiseFloor=.003, voicePeak=.045;
|
||||
const audioBands=new Float32Array(3);
|
||||
let voiceController=null;
|
||||
|
||||
<div class="settings">
|
||||
<label for="outputRate">
|
||||
Frequência de reprodução PCM
|
||||
</label>
|
||||
<select id="outputRate">
|
||||
<option value="24000">24 kHz</option>
|
||||
<option value="16000">16 kHz</option>
|
||||
<option value="32000">32 kHz</option>
|
||||
<option value="48000">48 kHz</option>
|
||||
</select>
|
||||
</div>
|
||||
</section>
|
||||
const activeStates=new Set([
|
||||
'connecting','listening','starting','recording','flushing',
|
||||
'processing','speaking','waiting-playback'
|
||||
]);
|
||||
|
||||
<footer>
|
||||
<span>NVIDIA STT</span>
|
||||
<span>OCI Generative AI</span>
|
||||
<span>VoxCPM TTS</span>
|
||||
</footer>
|
||||
</main>
|
||||
function setState(next,message='',error=false){
|
||||
state=next;
|
||||
const active=activeStates.has(next);
|
||||
hero.dataset.state=next;
|
||||
hero.classList.toggle('listening',active);
|
||||
button.disabled=next==='requesting';
|
||||
button.setAttribute('aria-busy',String(next==='requesting'));
|
||||
$('mic-label').textContent=active?'Encerrar conversa':next==='requesting'?'Conectando...':'Iniciar conversa';
|
||||
$('mic-icon').toggleAttribute('hidden',active);
|
||||
$('stop-icon').toggleAttribute('hidden',!active);
|
||||
feedback.textContent=message;
|
||||
feedback.classList.toggle('error',error);
|
||||
}
|
||||
|
||||
<script type="module" src="/web/app.js"></script>
|
||||
const assistantIntegration={
|
||||
async onVoiceStart({stream,audioContext,analyser},token){
|
||||
const {VoiceSessionController}=await import('/web/voice-session.js');
|
||||
if(token!==requestId)return;
|
||||
|
||||
const controller=new VoiceSessionController({
|
||||
onStateChange(next){
|
||||
if(token!==requestId||voiceController!==controller)return;
|
||||
if(next==='closed'){
|
||||
requestId++;
|
||||
releaseAudio();
|
||||
setState('ended','Conversa encerrada.');
|
||||
return;
|
||||
}
|
||||
const messages={
|
||||
connecting:'Conectando ao AgentVoice...',
|
||||
listening:'Estou ouvindo. Pode falar.',
|
||||
starting:'Iniciando reconhecimento...',
|
||||
recording:'Estou ouvindo sua pergunta...',
|
||||
flushing:'Finalizando captura de áudio...',
|
||||
processing:'Preparando sua resposta...',
|
||||
speaking:'Reproduzindo resposta...',
|
||||
'waiting-playback':'Aguardando fim da reprodução.'
|
||||
};
|
||||
setState(next,messages[next]||next);
|
||||
},
|
||||
onTranscript(text,isFinal){
|
||||
if(token!==requestId)return;
|
||||
console.log(isFinal?'[STT FINAL]':'[STT PARTIAL]',text);
|
||||
},
|
||||
onAgentText(text,isFinal){
|
||||
if(token!==requestId)return;
|
||||
console.log(isFinal?'[AGENT RESPONSE]':'[AGENT DELTA]',text);
|
||||
},
|
||||
onError(error){
|
||||
if(token!==requestId||voiceController!==controller)return;
|
||||
console.error('AgentVoice:',error);
|
||||
requestId++;
|
||||
releaseAudio();
|
||||
setState('error',error.message||'Falha na conversa.',true);
|
||||
}
|
||||
});
|
||||
|
||||
voiceController=controller;
|
||||
await controller.start({stream,audioContext,analyser});
|
||||
if(token!==requestId){
|
||||
controller.close();
|
||||
}
|
||||
},
|
||||
onVoiceEnd(){
|
||||
const controller=voiceController;
|
||||
voiceController=null;
|
||||
controller?.close();
|
||||
}
|
||||
};
|
||||
|
||||
function releaseAudio(){
|
||||
// Close the controller first, before closing its AudioContext.
|
||||
try{assistantIntegration.onVoiceEnd();}catch(e){console.warn('Integração de voz:',e);}
|
||||
micSource?.disconnect();micSource=null;
|
||||
const oldStream=stream;stream=null;
|
||||
analyser=null;samples=null;spectrum=null;voice=0;audioBands.fill(0);
|
||||
if(oldStream)oldStream.getTracks().forEach(track=>track.stop());
|
||||
const oldContext=audioContext;audioContext=null;
|
||||
if(oldContext&&oldContext.state!=='closed')oldContext.close().catch(()=>{});
|
||||
}
|
||||
function stopListening(){
|
||||
requestId++;
|
||||
releaseAudio();
|
||||
setState('ended','Conversa encerrada.');
|
||||
}
|
||||
|
||||
button.addEventListener('click',async()=>{
|
||||
if(activeStates.has(state)||voiceController!==null){
|
||||
stopListening();
|
||||
return;
|
||||
}
|
||||
if(state==='requesting')return;
|
||||
|
||||
const currentRequest=++requestId;
|
||||
setState('requesting','Permita o microfone no navegador para continuar.');
|
||||
try{
|
||||
if(!window.isSecureContext||!navigator.mediaDevices?.getUserMedia)
|
||||
throw Object.assign(new Error(),{name:'UnsupportedError'});
|
||||
const AudioContextClass=window.AudioContext||window.webkitAudioContext;
|
||||
if(!AudioContextClass)throw Object.assign(new Error(),{name:'UnsupportedError'});
|
||||
|
||||
audioContext=new AudioContextClass();
|
||||
await audioContext.resume();
|
||||
if(currentRequest!==requestId)return;
|
||||
const acquired=await navigator.mediaDevices.getUserMedia({
|
||||
audio:{echoCancellation:true,noiseSuppression:true,autoGainControl:true},video:false
|
||||
});
|
||||
if(currentRequest!==requestId){
|
||||
acquired.getTracks().forEach(track=>track.stop());
|
||||
return;
|
||||
}
|
||||
stream=acquired;
|
||||
analyser=audioContext.createAnalyser();
|
||||
analyser.fftSize=1024;
|
||||
analyser.smoothingTimeConstant=.65;
|
||||
samples=new Float32Array(analyser.fftSize);
|
||||
spectrum=new Uint8Array(analyser.frequencyBinCount);
|
||||
noiseFloor=.003;voicePeak=.045;
|
||||
micSource=audioContext.createMediaStreamSource(stream);
|
||||
micSource.connect(analyser);
|
||||
stream.getAudioTracks().forEach(track=>track.addEventListener('ended',()=>{
|
||||
if(currentRequest===requestId)stopListening();
|
||||
},{once:true}));
|
||||
await assistantIntegration.onVoiceStart({stream,audioContext,analyser},currentRequest);
|
||||
}catch(error){
|
||||
if(currentRequest!==requestId)return;
|
||||
requestId++;
|
||||
releaseAudio();
|
||||
const messages={
|
||||
NotAllowedError:'Acesso ao microfone negado. Permita nas configurações do navegador.',
|
||||
NotFoundError:'Nenhum microfone encontrado.',
|
||||
NotReadableError:'Microfone ocupado ou indisponível.',
|
||||
UnsupportedError:'Use HTTPS ou localhost em um navegador compatível.'
|
||||
};
|
||||
setState('error',messages[error.name]||error.message||'Não foi possível iniciar a conversa.',true);
|
||||
}
|
||||
});
|
||||
addEventListener('pagehide',()=>{requestId++;releaseAudio();});
|
||||
addEventListener('pageshow',event=>{
|
||||
if(event.persisted)setState('ended','Microfone encerrado. Você pode iniciar novamente.');
|
||||
});
|
||||
|
||||
// Órbita automática: pausa para leitura, seguida por uma transição em hélice.
|
||||
const cards=[...document.querySelectorAll('.orbit-card')], dots=[...document.querySelectorAll('.card-progress > span')];
|
||||
const cardStage=$('card-stage'), pauseCards=$('card-pause');
|
||||
let cardFrame=0,cardLast=0,cardTime=0,cardsPaused=false,cardsHovered=false,cardsFocused=false;
|
||||
const CARD_HOLD=4.2,CARD_TRANSITION=1.8,CARD_CYCLE=CARD_HOLD+CARD_TRANSITION;
|
||||
function updateCards(now){
|
||||
cardFrame=0;if(document.hidden)return;
|
||||
if(motion.matches){cards.forEach(card=>{card.style.transform='';card.style.opacity='';card.style.zIndex='';});return;}
|
||||
const dt=cardLast?Math.min((now-cardLast)/1000,.05):0;cardLast=now;
|
||||
if(!cardsPaused&&!cardsHovered&&!cardsFocused)cardTime+=dt;
|
||||
const turn=Math.floor(cardTime/CARD_CYCLE), local=cardTime%CARD_CYCLE;
|
||||
const t=Math.max(0,Math.min(1,(local-CARD_HOLD)/CARD_TRANSITION));
|
||||
const ease=t*t*t*(t*(t*6.-15.)+10.);
|
||||
const phase=(turn+ease)%cards.length;
|
||||
const rx=Math.min(cardStage.clientWidth*.29,290);
|
||||
cards.forEach((card,i)=>{
|
||||
const q=((i-phase+cards.length*1.5)%cards.length)-cards.length*.5;
|
||||
const angle=q*Math.PI/2, depth=Math.cos(angle), sc=.8+.2*depth;
|
||||
const x=Math.sin(angle)*rx,y=q*62+(1-depth)*18;
|
||||
card.style.transform=`translate(-50%,-50%) translate3d(${x}px,${y}px,${depth*60}px) scale(${sc}) rotateY(${-q*90}deg)`;
|
||||
card.style.opacity=String(Math.max(0,depth)*.94+.06);card.style.zIndex=String(Math.round(sc*100));
|
||||
});
|
||||
dots.forEach((dot,i)=>dot.classList.toggle('active',i===Math.round(phase)%cards.length));
|
||||
if(!cardsPaused&&!cardsHovered&&!cardsFocused)cardFrame=requestAnimationFrame(updateCards);
|
||||
}
|
||||
function resumeCards(){if(!cardFrame){cardLast=0;cardFrame=requestAnimationFrame(updateCards);}}
|
||||
pauseCards.addEventListener('click',()=>{
|
||||
cardsPaused=!cardsPaused;pauseCards.setAttribute('aria-pressed',String(cardsPaused));
|
||||
pauseCards.setAttribute('aria-label',cardsPaused?'Retomar transição automática dos cards':'Pausar transição automática dos cards');
|
||||
pauseCards.lastElementChild.textContent=cardsPaused?'Retomar cards':'Pausar cards';
|
||||
pauseCards.querySelector('path').setAttribute('d',cardsPaused?'M3 2l7 4-7 4z':'M3 2h2v8H3zm4 0h2v8H7z');resumeCards();
|
||||
});
|
||||
cardStage.addEventListener('pointerenter',event=>{if(event.pointerType==='mouse'||event.pointerType==='pen')cardsHovered=true;});
|
||||
cardStage.addEventListener('pointerleave',()=>{cardsHovered=false;resumeCards();});
|
||||
cardStage.addEventListener('focusin',()=>{cardsFocused=true;});
|
||||
cardStage.addEventListener('focusout',()=>{cardsFocused=false;resumeCards();});
|
||||
addEventListener('resize',resumeCards);motion.addEventListener('change',resumeCards);
|
||||
document.addEventListener('visibilitychange',()=>{if(document.hidden){cancelAnimationFrame(cardFrame);cardFrame=0;}else resumeCards();});
|
||||
resumeCards();
|
||||
|
||||
// Three.js com versão fixa; carregamento isolado mantém todos os controles disponíveis se o CDN falhar.
|
||||
let renderer,scene,camera,material,points,raf=0,last=0,time=0,energy=0,activity=0;
|
||||
const pointer={x:0,y:0}, smoothPointer={x:0,y:0};
|
||||
function layout(){
|
||||
const w=hero.clientWidth,h=hero.clientHeight;
|
||||
// O contorno conserva as retas e as extremidades semicirculares do “O”.
|
||||
const contentHeight=document.querySelector('.content').offsetHeight;
|
||||
const ringHeight=Math.max(contentHeight+155,Math.min(h*.79,660));
|
||||
const ringWidth=Math.max(ringHeight*1.72,Math.min(w*.85,1380));
|
||||
hero.style.setProperty('--ring-width',`${ringWidth}px`);
|
||||
hero.style.setProperty('--ring-height',`${ringHeight}px`);
|
||||
if(!renderer)return;
|
||||
renderer.setSize(w,h);camera.left=-w/2;camera.right=w/2;camera.top=h/2;camera.bottom=-h/2;camera.updateProjectionMatrix();
|
||||
material.uniforms.uShape.value.set((ringWidth-ringHeight)/2,ringHeight/2);
|
||||
resume();
|
||||
}
|
||||
layout();const layoutObserver=new ResizeObserver(layout);layoutObserver.observe(hero);layoutObserver.observe(document.querySelector('.content'));document.fonts.ready.then(layout);
|
||||
if(matchMedia('(hover:hover) and (pointer:fine)').matches){hero.addEventListener('pointermove',event=>{const rect=hero.getBoundingClientRect();pointer.x=(event.clientX-rect.left)/rect.width-.5;pointer.y=(event.clientY-rect.top)/rect.height-.5;},{passive:true});hero.addEventListener('pointerleave',()=>{pointer.x=pointer.y=0;});}
|
||||
|
||||
function frame(now) {
|
||||
raf = 0;
|
||||
|
||||
if (document.hidden || !renderer) return;
|
||||
|
||||
const dt = last
|
||||
? Math.min((now - last) / 1000, 0.05)
|
||||
: 0.016;
|
||||
|
||||
last = now;
|
||||
|
||||
// TTS may continue playing after turn.ended.
|
||||
const playbackActive =
|
||||
voiceController?.player?.playing === true;
|
||||
|
||||
const ttsSpeaking =
|
||||
state === "speaking" ||
|
||||
(
|
||||
state === "waiting-playback" &&
|
||||
playbackActive
|
||||
);
|
||||
|
||||
// Select the audio source according to conversation state.
|
||||
const microphoneActive = [
|
||||
"listening",
|
||||
"starting",
|
||||
"recording",
|
||||
"flushing"
|
||||
].includes(state);
|
||||
|
||||
const activeAnalyser = ttsSpeaking
|
||||
? voiceController?.getPlaybackAnalyser()
|
||||
: microphoneActive
|
||||
? analyser
|
||||
: null;
|
||||
|
||||
if (activeAnalyser && samples && spectrum) {
|
||||
activeAnalyser.getFloatTimeDomainData(samples);
|
||||
activeAnalyser.getByteFrequencyData(spectrum);
|
||||
|
||||
let sum = 0;
|
||||
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
sum += samples[i] * samples[i];
|
||||
}
|
||||
|
||||
const rms = Math.sqrt(sum / samples.length);
|
||||
|
||||
noiseFloor +=
|
||||
(Math.min(rms, 0.012) - noiseFloor) *
|
||||
(1 - Math.exp(-dt * 0.5));
|
||||
|
||||
voicePeak = Math.max(
|
||||
0.025,
|
||||
rms,
|
||||
voicePeak * Math.exp(-dt * 0.22)
|
||||
);
|
||||
|
||||
const gate = Math.max(0.002, noiseFloor * 1.45);
|
||||
|
||||
voice = Math.pow(
|
||||
Math.min(
|
||||
1,
|
||||
Math.max(0, rms - gate) /
|
||||
Math.max(0.012, voicePeak - gate)
|
||||
),
|
||||
0.65
|
||||
);
|
||||
|
||||
const hzPerBin =
|
||||
audioContext.sampleRate / activeAnalyser.fftSize;
|
||||
|
||||
const limits = [100, 450, 1500, 5000];
|
||||
|
||||
for (let band = 0; band < 3; band++) {
|
||||
const start = Math.max(
|
||||
1,
|
||||
Math.floor(limits[band] / hzPerBin)
|
||||
);
|
||||
|
||||
const end = Math.min(
|
||||
spectrum.length,
|
||||
Math.ceil(limits[band + 1] / hzPerBin)
|
||||
);
|
||||
|
||||
let total = 0;
|
||||
|
||||
for (let i = start; i < end; i++) {
|
||||
total += spectrum[i] / 255;
|
||||
}
|
||||
|
||||
const target =
|
||||
(total / Math.max(1, end - start)) * voice;
|
||||
|
||||
audioBands[band] +=
|
||||
(target - audioBands[band]) *
|
||||
(1 - Math.exp(-dt * 10));
|
||||
}
|
||||
} else {
|
||||
voice = 0;
|
||||
audioBands.fill(0);
|
||||
}
|
||||
|
||||
energy +=
|
||||
(voice - energy) *
|
||||
(1 - Math.exp(-dt * (voice > energy ? 19 : 4.5)));
|
||||
|
||||
const targetActivity =
|
||||
ttsSpeaking ? 1 :
|
||||
state === "recording" ? 0.7 :
|
||||
state === "listening" ? 0.35 :
|
||||
state === "processing" ? 0.15 :
|
||||
state === "requesting" ? 0.35 : 0;
|
||||
|
||||
activity +=
|
||||
(targetActivity - activity) *
|
||||
(1 - Math.exp(-dt * 3));
|
||||
|
||||
if (!motion.matches) {
|
||||
time += dt;
|
||||
}
|
||||
|
||||
smoothPointer.x +=
|
||||
(pointer.x - smoothPointer.x) * 0.035;
|
||||
|
||||
smoothPointer.y +=
|
||||
(pointer.y - smoothPointer.y) * 0.035;
|
||||
|
||||
const u = material.uniforms;
|
||||
|
||||
u.uTime.value = time;
|
||||
u.uVoice.value = motion.matches ? 0 : energy;
|
||||
u.uActivity.value = activity;
|
||||
|
||||
u.uBands.value.set(
|
||||
audioBands[0],
|
||||
audioBands[1],
|
||||
audioBands[2]
|
||||
);
|
||||
|
||||
u.uPointer.value.set(
|
||||
motion.matches ? 0 : smoothPointer.x,
|
||||
motion.matches ? 0 : smoothPointer.y
|
||||
);
|
||||
|
||||
renderer.render(scene, camera);
|
||||
|
||||
const keepAnimating =
|
||||
!motion.matches ||
|
||||
microphoneActive ||
|
||||
ttsSpeaking ||
|
||||
playbackActive ||
|
||||
state === "requesting" ||
|
||||
Math.abs(activity) > 0.005;
|
||||
|
||||
if (keepAnimating) {
|
||||
raf = requestAnimationFrame(frame);
|
||||
}
|
||||
}
|
||||
|
||||
function resume(){if(!raf){last=0;raf=requestAnimationFrame(frame);}}
|
||||
document.addEventListener('visibilitychange',()=>{if(document.hidden){cancelAnimationFrame(raf);raf=0;last=0;}else resume();});motion.addEventListener('change',resume);
|
||||
new MutationObserver(resume).observe(hero,{attributes:true,attributeFilter:['data-state']});
|
||||
async function initVisual(){
|
||||
try{
|
||||
const THREE=await import('https://cdn.jsdelivr.net/npm/three@0.170.0/build/three.module.js');
|
||||
renderer=new THREE.WebGLRenderer({alpha:true,antialias:false,powerPreference:'low-power'});
|
||||
renderer.setPixelRatio(Math.min(devicePixelRatio||1,innerWidth<601?1.5:1.75));
|
||||
renderer.setClearColor(0x030305,0);scene=new THREE.Scene();camera=new THREE.OrthographicCamera(-1,1,1,-1,.1,100);camera.position.z=20;
|
||||
const small=innerWidth<601||(navigator.hardwareConcurrency||4)<=4;
|
||||
const filaments=small?38:60,steps=small?650:1250,count=filaments*steps;
|
||||
const positions=new Float32Array(count*3),params=new Float32Array(count*3);
|
||||
for(let j=0;j<filaments;j++)for(let i=0;i<steps;i++){const n=(j*steps+i)*3;params[n]=i/steps*Math.PI*2;params[n+1]=j/(filaments-1);params[n+2]=Math.random();}
|
||||
const geometry=new THREE.BufferGeometry();geometry.setAttribute('position',new THREE.BufferAttribute(positions,3));geometry.setAttribute('aParam',new THREE.BufferAttribute(params,3));
|
||||
material=new THREE.ShaderMaterial({transparent:true,depthWrite:false,depthTest:false,blending:THREE.NormalBlending,uniforms:{uTime:{value:0},uShape:{value:new THREE.Vector2(220,300)},uBands:{value:new THREE.Vector3()},uVoice:{value:0},uActivity:{value:0},uDpr:{value:renderer.getPixelRatio()},uPointer:{value:new THREE.Vector2()}},
|
||||
vertexShader:`
|
||||
attribute vec3 aParam;
|
||||
uniform float uTime,uVoice,uActivity,uDpr;
|
||||
uniform vec2 uShape,uPointer;
|
||||
uniform vec3 uBands;
|
||||
varying vec3 vColor;varying float vAlpha;
|
||||
const float PI=3.14159265359;
|
||||
// Comprimento de arco mantém densidade regular nas retas e nas curvas.
|
||||
void capsule(float phase,out vec2 p,out vec2 n){
|
||||
float halfLine=uShape.x;float r=uShape.y;
|
||||
float line=2.*halfLine;float arc=PI*r;
|
||||
float d=fract(phase/(2.*PI))*(2.*line+2.*arc);
|
||||
if(d<line){p=vec2(-halfLine+d,r);n=vec2(0.,1.);}
|
||||
else if(d<line+arc){float a=PI*.5-(d-line)/r;n=vec2(cos(a),sin(a));p=vec2(halfLine,0.)+n*r;}
|
||||
else if(d<2.*line+arc){p=vec2(halfLine-(d-line-arc),-r);n=vec2(0.,-1.);}
|
||||
else{float a=-PI*.5-(d-2.*line-arc)/r;n=vec2(cos(a),sin(a));p=vec2(-halfLine,0.)+n*r;}
|
||||
}
|
||||
void main(){
|
||||
float t=uTime;float f=aParam.y;float seed=aParam.z;
|
||||
float a=aParam.x+t*.065+f*.007+seed*.0007;
|
||||
vec2 base,normal;capsule(a,base,normal);
|
||||
float spread=f-.5;
|
||||
// Respiração, dobras e correntes em velocidades diferentes.
|
||||
float breath=sin(t*1.15)*4.+sin(t*.53+1.)*3.;
|
||||
float wave=sin(a*5.-t*1.35+f*4.)*9.+sin(a*9.+t*1.8+f*7.)*5.;
|
||||
float ripple=sin(a*21.-t*2.6+f*8.)*2.5+sin(a*37.+t*2.+f*11.)*1.3;
|
||||
float voiceWave=uVoice*(13.+sin(a*6.-t*4.5+f*5.)*19.+sin(a*13.+t*5.+f*8.)*9.);
|
||||
float spectral=sin(a*4.-t*3.+f*6.)*uBands.x*17.+sin(a*18.+t*4.)*uBands.z*12.;
|
||||
float thickness=30.+10.*sin(a*3.+t*.9)+uActivity*9.+uVoice*40.;
|
||||
float displacement=spread*thickness+breath+wave+ripple+voiceWave+spectral;
|
||||
vec2 pos=base+normal*displacement;
|
||||
pos+=vec2(uPointer.x*12.,-uPointer.y*12.-14.);
|
||||
float z=sin(a*3.+f*7.+t)*2.;
|
||||
float core=pow(max(0.,1.-abs(spread)*2.),2.5);
|
||||
float current=.5+.5*sin(a*3.-t*1.6+f*3.);
|
||||
// Mesma cor sólida dos títulos (#ea171c); a profundidade varia só a intensidade.
|
||||
vec3 red=vec3(.917647,.090196,.109804);
|
||||
float shade=.70+.30*(.5+.5*sin(a*3.-t*.7+f*2.));
|
||||
vColor=red*shade;
|
||||
vAlpha=(.28+core*.42)*(.65+seed*.35)*(1.+uVoice*.55);
|
||||
gl_Position=projectionMatrix*modelViewMatrix*vec4(pos,z,1.);
|
||||
gl_PointSize=(1.55+core*1.1+seed*.45+uVoice*1.3)*uDpr;
|
||||
}`,
|
||||
fragmentShader:`
|
||||
varying vec3 vColor;varying float vAlpha;
|
||||
void main(){float d=length(gl_PointCoord-.5);if(d>.5)discard;float light=exp(-d*d*18.);gl_FragColor=vec4(vColor,vAlpha*light);}
|
||||
`});
|
||||
points=new THREE.Points(geometry,material);points.frustumCulled=false;scene.add(points);
|
||||
// Halo compartilha a deformação dos filamentos, sem pós-processamento de tela inteira.
|
||||
const glowMaterial=material.clone();glowMaterial.uniforms=material.uniforms;
|
||||
glowMaterial.vertexShader=material.vertexShader.replace('gl_PointSize=(1.55+core*1.1+seed*.45+uVoice*1.3)*uDpr;','gl_PointSize=(15.+core*13.+uVoice*12.)*uDpr;').replace('vAlpha=(.28+core*.42)*(.65+seed*.35)*(1.+uVoice*.55);','vAlpha=.027*(.6+core)*(1.+uVoice*.7);');
|
||||
const glow=new THREE.Points(geometry,glowMaterial);glow.frustumCulled=false;glow.renderOrder=-1;scene.add(glow);
|
||||
$('visual').appendChild(renderer.domElement);renderer.domElement.setAttribute('aria-hidden','true');
|
||||
renderer.domElement.addEventListener('webglcontextlost',event=>{event.preventDefault();cancelAnimationFrame(raf);raf=0;hero.classList.remove('webgl-ready');});
|
||||
renderer.domElement.addEventListener('webglcontextrestored',()=>{hero.classList.add('webgl-ready');resume();});
|
||||
layout();renderer.render(scene,camera);hero.classList.add('webgl-ready');resume();
|
||||
}catch(error){console.warn('Anel WebGL indisponível; composição CSS ativa.',error);}
|
||||
}
|
||||
initVisual();
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
4
src/voice_assistant/audio_gateway/web/package.json
Normal file
4
src/voice_assistant/audio_gateway/web/package.json
Normal file
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"private": true,
|
||||
"type": "module"
|
||||
}
|
||||
224
src/voice_assistant/audio_gateway/web/pcm-player.js
Normal file
224
src/voice_assistant/audio_gateway/web/pcm-player.js
Normal file
@@ -0,0 +1,224 @@
|
||||
|
||||
/**
|
||||
* PCM playback for AgentVoice.
|
||||
*
|
||||
* Responsibilities:
|
||||
* - Decode raw PCM signed 16-bit little-endian.
|
||||
* - Schedule audio chunks in arrival order.
|
||||
* - Track actual playback completion.
|
||||
* - Stop scheduled audio immediately.
|
||||
*
|
||||
* Non-responsibilities:
|
||||
* - WebSocket communication.
|
||||
* - Voice activity detection.
|
||||
* - Session or turn lifecycle.
|
||||
*/
|
||||
|
||||
export class PCMPlayer {
|
||||
constructor({
|
||||
audioContext,
|
||||
sampleRate = 48000,
|
||||
onPlaybackStart = () => {},
|
||||
onPlaybackEnd = () => {},
|
||||
} = {}) {
|
||||
if (!audioContext) {
|
||||
throw new Error("AudioContext is required.");
|
||||
}
|
||||
|
||||
if (!Number.isFinite(sampleRate) || sampleRate <= 0) {
|
||||
throw new Error("Invalid PCM sample rate.");
|
||||
}
|
||||
|
||||
this.context = audioContext;
|
||||
this.sampleRate = sampleRate;
|
||||
this.analyser = this.context.createAnalyser();
|
||||
this.analyser.fftSize = 1024;
|
||||
this.analyser.smoothingTimeConstant = 0.65;
|
||||
|
||||
this.analyser.connect(this.context.destination);
|
||||
|
||||
this.onPlaybackStart = onPlaybackStart;
|
||||
this.onPlaybackEnd = onPlaybackEnd;
|
||||
|
||||
this.sources = new Set();
|
||||
this.nextStartTime = 0;
|
||||
this.playing = false;
|
||||
this.finished = false;
|
||||
this.generation = 0;
|
||||
|
||||
this.waiters = [];
|
||||
}
|
||||
|
||||
createAnalyser() {
|
||||
return {
|
||||
fftSize: 1024,
|
||||
smoothingTimeConstant: 0.65,
|
||||
connect() {},
|
||||
disconnect() {},
|
||||
};
|
||||
}
|
||||
|
||||
getAnalyser() {
|
||||
return this.analyser;
|
||||
}
|
||||
|
||||
decodePCM(buffer) {
|
||||
if (!(buffer instanceof ArrayBuffer)) {
|
||||
throw new TypeError("PCM chunk must be an ArrayBuffer.");
|
||||
}
|
||||
|
||||
if (buffer.byteLength % 2 !== 0) {
|
||||
throw new Error("Invalid PCM byte length.");
|
||||
}
|
||||
|
||||
const view = new DataView(buffer);
|
||||
const samples = new Float32Array(buffer.byteLength / 2);
|
||||
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
samples[i] = view.getInt16(i * 2, true) / 32768;
|
||||
}
|
||||
|
||||
return samples;
|
||||
}
|
||||
|
||||
async enqueue(buffer) {
|
||||
if (this.finished) {
|
||||
throw new Error("Playback has already been finalized.");
|
||||
}
|
||||
|
||||
const samples = this.decodePCM(buffer);
|
||||
|
||||
if (samples.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.context.resume();
|
||||
|
||||
if (this.finished) {
|
||||
throw new Error("Playback was finalized during enqueue.");
|
||||
}
|
||||
|
||||
const audioBuffer = this.context.createBuffer(
|
||||
1,
|
||||
samples.length,
|
||||
this.sampleRate
|
||||
);
|
||||
|
||||
audioBuffer.copyToChannel(samples, 0);
|
||||
|
||||
const source = this.context.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(this.analyser);
|
||||
|
||||
const startAt = Math.max(
|
||||
this.context.currentTime + 0.01,
|
||||
this.nextStartTime
|
||||
);
|
||||
|
||||
this.nextStartTime = startAt + audioBuffer.duration;
|
||||
|
||||
const generation = this.generation;
|
||||
|
||||
this.sources.add(source);
|
||||
|
||||
if (!this.playing) {
|
||||
this.playing = true;
|
||||
this.onPlaybackStart();
|
||||
}
|
||||
|
||||
source.onended = () => {
|
||||
source.disconnect();
|
||||
this.sources.delete(source);
|
||||
|
||||
if (generation !== this.generation) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.checkCompletion();
|
||||
};
|
||||
|
||||
try {
|
||||
source.start(startAt);
|
||||
} catch (error) {
|
||||
this.sources.delete(source);
|
||||
source.disconnect();
|
||||
this.checkCompletion();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
finish() {
|
||||
this.finished = true;
|
||||
this.checkCompletion();
|
||||
}
|
||||
|
||||
checkCompletion() {
|
||||
if (!this.finished || this.sources.size > 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const wasPlaying = this.playing;
|
||||
this.playing = false;
|
||||
this.nextStartTime = 0;
|
||||
|
||||
if (wasPlaying) {
|
||||
this.onPlaybackEnd();
|
||||
}
|
||||
|
||||
this.resolveWaiters();
|
||||
}
|
||||
|
||||
waitForCompletion() {
|
||||
if (this.finished && this.sources.size === 0) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
return new Promise((resolve) => {
|
||||
this.waiters.push(resolve);
|
||||
});
|
||||
}
|
||||
|
||||
resolveWaiters() {
|
||||
const waiters = this.waiters.splice(0);
|
||||
|
||||
for (const resolve of waiters) {
|
||||
resolve();
|
||||
}
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.generation++;
|
||||
|
||||
for (const source of this.sources) {
|
||||
source.onended = null;
|
||||
|
||||
try {
|
||||
source.stop();
|
||||
} catch {
|
||||
// Source may already have finished.
|
||||
}
|
||||
|
||||
source.disconnect();
|
||||
}
|
||||
|
||||
this.sources.clear();
|
||||
this.nextStartTime = 0;
|
||||
|
||||
const wasPlaying = this.playing;
|
||||
this.playing = false;
|
||||
this.finished = true;
|
||||
|
||||
if (wasPlaying) {
|
||||
this.onPlaybackEnd();
|
||||
}
|
||||
|
||||
this.resolveWaiters();
|
||||
}
|
||||
|
||||
reset() {
|
||||
this.stop();
|
||||
|
||||
this.finished = false;
|
||||
this.nextStartTime = 0;
|
||||
}
|
||||
}
|
||||
@@ -2,21 +2,90 @@
|
||||
class PCMInputProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
|
||||
this.targetRate = 16000;
|
||||
this.step = sampleRate / this.targetRate;
|
||||
this.chunkSamples = 1600;
|
||||
|
||||
this.buffer = [];
|
||||
this.position = 0;
|
||||
this.sequence = 0;
|
||||
this.chunkSamples = 1600;
|
||||
this.output = [];
|
||||
this.sequence = 0;
|
||||
|
||||
this.capturing = false;
|
||||
|
||||
this.port.onmessage = ({ data }) => {
|
||||
switch (data?.type) {
|
||||
case "start":
|
||||
this.reset();
|
||||
this.capturing = true;
|
||||
break;
|
||||
|
||||
case "flush":
|
||||
this.capturing = false;
|
||||
this.flush();
|
||||
break;
|
||||
|
||||
case "stop":
|
||||
this.capturing = false;
|
||||
this.reset();
|
||||
break;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
reset() {
|
||||
this.buffer = [];
|
||||
this.position = 0;
|
||||
this.output = [];
|
||||
this.sequence = 0;
|
||||
}
|
||||
|
||||
emitPCM() {
|
||||
if (!this.output.length) return;
|
||||
|
||||
const pcm = new Int16Array(this.output);
|
||||
this.output = [];
|
||||
|
||||
this.port.postMessage({
|
||||
type: "pcm",
|
||||
sequence: this.sequence++,
|
||||
buffer: pcm.buffer
|
||||
}, [pcm.buffer]);
|
||||
}
|
||||
|
||||
flush() {
|
||||
while (this.position + 1 < this.buffer.length) {
|
||||
const index = Math.floor(this.position);
|
||||
const fraction = this.position - index;
|
||||
|
||||
const sample =
|
||||
this.buffer[index] * (1 - fraction) +
|
||||
this.buffer[index + 1] * fraction;
|
||||
|
||||
const clipped = Math.max(-1, Math.min(1, sample));
|
||||
|
||||
this.output.push(
|
||||
clipped < 0
|
||||
? Math.round(clipped * 32768)
|
||||
: Math.round(clipped * 32767)
|
||||
);
|
||||
|
||||
this.position += this.step;
|
||||
}
|
||||
|
||||
this.emitPCM();
|
||||
this.reset();
|
||||
|
||||
this.port.postMessage({ type: "flushed" });
|
||||
}
|
||||
|
||||
process(inputs) {
|
||||
if (!this.capturing) return true;
|
||||
|
||||
const channels = inputs[0];
|
||||
|
||||
if (!channels || channels.length === 0) {
|
||||
return true;
|
||||
}
|
||||
if (!channels?.length) return true;
|
||||
|
||||
const frameLength = channels[0].length;
|
||||
|
||||
@@ -49,14 +118,7 @@ class PCMInputProcessor extends AudioWorkletProcessor {
|
||||
this.position += this.step;
|
||||
|
||||
if (this.output.length >= this.chunkSamples) {
|
||||
const pcm = new Int16Array(this.output);
|
||||
this.output = [];
|
||||
|
||||
this.port.postMessage({
|
||||
type: "pcm",
|
||||
sequence: this.sequence++,
|
||||
buffer: pcm.buffer
|
||||
}, [pcm.buffer]);
|
||||
this.emitPCM();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
92
src/voice_assistant/audio_gateway/web/vad.js
Normal file
92
src/voice_assistant/audio_gateway/web/vad.js
Normal file
@@ -0,0 +1,92 @@
|
||||
|
||||
export class VoiceActivityDetector {
|
||||
constructor({
|
||||
onSpeechStart,
|
||||
onSpeechEnd,
|
||||
minSpeechMs = 250,
|
||||
silenceMs = 800,
|
||||
threshold = 0.012,
|
||||
noiseMultiplier = 3,
|
||||
} = {}) {
|
||||
this.onSpeechStart = onSpeechStart;
|
||||
this.onSpeechEnd = onSpeechEnd;
|
||||
|
||||
this.minSpeechMs = minSpeechMs;
|
||||
this.silenceMs = silenceMs;
|
||||
this.threshold = threshold;
|
||||
this.noiseMultiplier = noiseMultiplier;
|
||||
|
||||
this.noiseFloor = 0.003;
|
||||
this.speaking = false;
|
||||
this.speechStartedAt = null;
|
||||
this.lastVoiceAt = null;
|
||||
this.active = false;
|
||||
}
|
||||
|
||||
start() {
|
||||
this.active = true;
|
||||
this.reset();
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.active = false;
|
||||
this.reset();
|
||||
}
|
||||
|
||||
reset() {
|
||||
this.speaking = false;
|
||||
this.speechStartedAt = null;
|
||||
this.lastVoiceAt = null;
|
||||
}
|
||||
|
||||
process(rms, now = performance.now()) {
|
||||
if (!this.active) return;
|
||||
|
||||
if (!Number.isFinite(rms) || rms < 0) return;
|
||||
|
||||
const threshold = Math.max(
|
||||
this.threshold,
|
||||
this.noiseFloor * this.noiseMultiplier
|
||||
);
|
||||
|
||||
const voiced = rms >= threshold;
|
||||
|
||||
if (!this.speaking) {
|
||||
// Update noise estimation only outside detected speech.
|
||||
if (!voiced) {
|
||||
this.noiseFloor =
|
||||
this.noiseFloor * 0.98 + rms * 0.02;
|
||||
}
|
||||
|
||||
if (voiced) {
|
||||
if (this.speechStartedAt === null) {
|
||||
this.speechStartedAt = now;
|
||||
}
|
||||
|
||||
if (now - this.speechStartedAt >= this.minSpeechMs) {
|
||||
this.speaking = true;
|
||||
this.lastVoiceAt = now;
|
||||
this.onSpeechStart?.();
|
||||
}
|
||||
} else {
|
||||
this.speechStartedAt = null;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (voiced) {
|
||||
this.lastVoiceAt = now;
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
this.lastVoiceAt !== null &&
|
||||
now - this.lastVoiceAt >= this.silenceMs
|
||||
) {
|
||||
this.speaking = false;
|
||||
this.onSpeechEnd?.();
|
||||
this.reset();
|
||||
}
|
||||
}
|
||||
}
|
||||
479
src/voice_assistant/audio_gateway/web/voice-session.js
Normal file
479
src/voice_assistant/audio_gateway/web/voice-session.js
Normal file
@@ -0,0 +1,479 @@
|
||||
|
||||
import { VoiceActivityDetector } from "./vad.js";
|
||||
import { PCMPlayer } from "./pcm-player.js";
|
||||
|
||||
export class VoiceSessionController {
|
||||
constructor({
|
||||
onStateChange = () => {},
|
||||
onTranscript = () => {},
|
||||
onAgentText = () => {},
|
||||
onError = () => {},
|
||||
silenceMs = 800,
|
||||
prebufferChunks = 4,
|
||||
maxPendingChunks = 100,
|
||||
} = {}) {
|
||||
this.onStateChange = onStateChange;
|
||||
this.onTranscript = onTranscript;
|
||||
this.onAgentText = onAgentText;
|
||||
this.onError = onError;
|
||||
|
||||
this.prebufferChunks = prebufferChunks;
|
||||
this.maxPendingChunks = maxPendingChunks;
|
||||
|
||||
this.state = "closed";
|
||||
this.ws = null;
|
||||
this.stream = null;
|
||||
this.context = null;
|
||||
this.analyser = null;
|
||||
this.worklet = null;
|
||||
this.source = null;
|
||||
this.silentGain = null;
|
||||
|
||||
this.turnId = null;
|
||||
this.sequence = 0;
|
||||
this.prebuffer = [];
|
||||
this.pending = [];
|
||||
this.stopRequested = false;
|
||||
this.frameId = 0;
|
||||
this.rmsSamples = null;
|
||||
|
||||
this.vad = new VoiceActivityDetector({
|
||||
silenceMs,
|
||||
onSpeechStart: () => this.beginTurn(),
|
||||
onSpeechEnd: () => this.finishSpeech(),
|
||||
});
|
||||
|
||||
this.player = null;
|
||||
this.playbackPending = Promise.resolve();
|
||||
this.gatewayTurnEnded = false;
|
||||
this.playbackGeneration = 0;
|
||||
}
|
||||
|
||||
getPlaybackAnalyser() {
|
||||
return this.player?.getAnalyser() ?? null;
|
||||
}
|
||||
|
||||
setState(state) {
|
||||
this.state = state;
|
||||
this.onStateChange(state);
|
||||
}
|
||||
|
||||
send(event) {
|
||||
if (this.ws?.readyState !== WebSocket.OPEN) {
|
||||
throw new Error("WebSocket indisponível.");
|
||||
}
|
||||
this.ws.send(JSON.stringify(event));
|
||||
}
|
||||
|
||||
async start({ stream, audioContext, analyser }) {
|
||||
if (this.state !== "closed") {
|
||||
throw new Error("Sessão já iniciada.");
|
||||
}
|
||||
|
||||
this.stream = stream;
|
||||
this.context = audioContext;
|
||||
this.analyser = analyser;
|
||||
|
||||
this.player = new PCMPlayer({
|
||||
audioContext,
|
||||
sampleRate: 48000,
|
||||
|
||||
onPlaybackStart: () => {
|
||||
if (this.state !== "closed") {
|
||||
this.setState("speaking");
|
||||
}
|
||||
},
|
||||
});
|
||||
this.rmsSamples = new Float32Array(analyser.fftSize);
|
||||
|
||||
this.setState("connecting");
|
||||
|
||||
try {
|
||||
await this.context.audioWorklet.addModule(
|
||||
"/web/pcm-worklet.js"
|
||||
);
|
||||
|
||||
this.source = this.context.createMediaStreamSource(stream);
|
||||
this.worklet = new AudioWorkletNode(
|
||||
this.context,
|
||||
"pcm-input"
|
||||
);
|
||||
|
||||
this.silentGain = this.context.createGain();
|
||||
this.silentGain.gain.value = 0;
|
||||
|
||||
this.source.connect(this.worklet);
|
||||
this.worklet.connect(this.silentGain);
|
||||
this.silentGain.connect(this.context.destination);
|
||||
|
||||
this.worklet.port.onmessage = ({ data }) => {
|
||||
this.handleWorkletMessage(data);
|
||||
};
|
||||
|
||||
const scheme = location.protocol === "https:" ? "wss" : "ws";
|
||||
const socket = new WebSocket(
|
||||
`${scheme}://${location.host}/ws/audio`
|
||||
);
|
||||
|
||||
this.ws = socket;
|
||||
socket.binaryType = "arraybuffer";
|
||||
|
||||
socket.onopen = () => {
|
||||
if (this.ws !== socket) return;
|
||||
|
||||
this.send({
|
||||
type: "session.start",
|
||||
audio: {
|
||||
encoding: "pcm_s16le",
|
||||
sample_rate: 16000,
|
||||
channels: 1,
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
socket.onmessage = ({ data }) => {
|
||||
if (this.ws !== socket) return;
|
||||
|
||||
if (typeof data === "string") {
|
||||
try {
|
||||
this.handleGatewayEvent(JSON.parse(data));
|
||||
} catch (error) {
|
||||
this.fail(error);
|
||||
}
|
||||
} else if (data instanceof ArrayBuffer) {
|
||||
this.handleAudioChunk(data);
|
||||
}
|
||||
};
|
||||
|
||||
socket.onerror = () => {
|
||||
if (this.ws === socket) {
|
||||
this.fail(new Error("Falha na conexão WebSocket."));
|
||||
}
|
||||
};
|
||||
|
||||
socket.onclose = () => {
|
||||
if (this.ws === socket && this.state !== "closed") {
|
||||
this.fail(new Error("Conexão WebSocket encerrada."));
|
||||
}
|
||||
};
|
||||
} catch (error) {
|
||||
this.close();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
startListening() {
|
||||
this.playbackGeneration++;
|
||||
|
||||
this.gatewayTurnEnded = false;
|
||||
this.player?.reset();
|
||||
this.playbackPending = Promise.resolve();
|
||||
|
||||
this.turnId = null;
|
||||
this.sequence = 0;
|
||||
this.prebuffer = [];
|
||||
this.pending = [];
|
||||
this.stopRequested = false;
|
||||
|
||||
this.worklet.port.postMessage({ type: "start" });
|
||||
this.vad.start();
|
||||
this.setState("listening");
|
||||
|
||||
this.monitorVoice();
|
||||
}
|
||||
|
||||
monitorVoice() {
|
||||
cancelAnimationFrame(this.frameId);
|
||||
|
||||
const tick = (now) => {
|
||||
if (this.state !== "listening" &&
|
||||
this.state !== "starting" &&
|
||||
this.state !== "recording") {
|
||||
return;
|
||||
}
|
||||
|
||||
this.analyser.getFloatTimeDomainData(this.rmsSamples);
|
||||
|
||||
let power = 0;
|
||||
for (const sample of this.rmsSamples) {
|
||||
power += sample * sample;
|
||||
}
|
||||
|
||||
const rms = Math.sqrt(power / this.rmsSamples.length);
|
||||
|
||||
if (!this.stopRequested) {
|
||||
this.vad.process(rms, now);
|
||||
}
|
||||
|
||||
this.frameId = requestAnimationFrame(tick);
|
||||
};
|
||||
|
||||
this.frameId = requestAnimationFrame(tick);
|
||||
}
|
||||
|
||||
beginTurn() {
|
||||
if (this.state !== "listening") return;
|
||||
|
||||
this.pending = [...this.prebuffer];
|
||||
this.setState("starting");
|
||||
|
||||
try {
|
||||
this.send({ type: "audio.start" });
|
||||
} catch (error) {
|
||||
this.fail(error);
|
||||
}
|
||||
}
|
||||
|
||||
handleWorkletMessage(data) {
|
||||
if (data?.type === "pcm") {
|
||||
this.handlePCM(data.buffer);
|
||||
} else if (data?.type === "flushed") {
|
||||
this.handleFlushed();
|
||||
}
|
||||
}
|
||||
|
||||
handlePCM(buffer) {
|
||||
if (this.state === "listening") {
|
||||
this.prebuffer.push(buffer);
|
||||
|
||||
if (this.prebuffer.length > this.prebufferChunks) {
|
||||
this.prebuffer.shift();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.state === "starting") {
|
||||
this.pending.push(buffer);
|
||||
|
||||
if (this.pending.length > this.maxPendingChunks) {
|
||||
this.fail(new Error("Buffer de início de turno excedido."));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.state === "recording" ||
|
||||
this.state === "flushing") {
|
||||
this.sendPCM(buffer);
|
||||
}
|
||||
}
|
||||
|
||||
sendPCM(buffer) {
|
||||
if (this.turnId === null) return;
|
||||
|
||||
if (this.ws?.readyState !== WebSocket.OPEN) return;
|
||||
|
||||
this.send({
|
||||
type: "audio.chunk",
|
||||
turn_id: this.turnId,
|
||||
sequence_number: this.sequence++,
|
||||
});
|
||||
|
||||
this.ws.send(buffer);
|
||||
}
|
||||
|
||||
finishSpeech() {
|
||||
if (this.stopRequested) return;
|
||||
|
||||
if (this.state !== "starting" &&
|
||||
this.state !== "recording") return;
|
||||
|
||||
this.stopRequested = true;
|
||||
this.vad.stop();
|
||||
|
||||
if (this.state === "recording") {
|
||||
this.flushTurn();
|
||||
}
|
||||
// If still starting, flush after turn.started.
|
||||
}
|
||||
|
||||
flushTurn() {
|
||||
if (this.state !== "recording") return;
|
||||
|
||||
this.setState("flushing");
|
||||
this.worklet.port.postMessage({ type: "flush" });
|
||||
}
|
||||
|
||||
handleFlushed() {
|
||||
if (this.state !== "flushing") return;
|
||||
|
||||
this.send({
|
||||
type: "audio.stop",
|
||||
turn_id: this.turnId,
|
||||
});
|
||||
|
||||
this.setState("processing");
|
||||
cancelAnimationFrame(this.frameId);
|
||||
}
|
||||
|
||||
handleAudioChunk(buffer) {
|
||||
if (
|
||||
this.state === "closed" ||
|
||||
this.turnId === null ||
|
||||
this.gatewayTurnEnded
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const generation = this.playbackGeneration;
|
||||
|
||||
this.playbackPending = this.playbackPending
|
||||
.then(async () => {
|
||||
if (generation !== this.playbackGeneration) return;
|
||||
await this.player.enqueue(buffer);
|
||||
})
|
||||
.catch((error) => {
|
||||
if (generation === this.playbackGeneration) {
|
||||
this.fail(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async finishPlaybackAndResume() {
|
||||
const generation = this.playbackGeneration;
|
||||
|
||||
try {
|
||||
await this.playbackPending;
|
||||
|
||||
if (generation !== this.playbackGeneration) return;
|
||||
if (this.state === "closed") return;
|
||||
|
||||
this.player.finish();
|
||||
await this.player.waitForCompletion();
|
||||
|
||||
if (generation !== this.playbackGeneration) return;
|
||||
if (this.state === "closed") return;
|
||||
if (!this.gatewayTurnEnded) return;
|
||||
|
||||
this.startListening();
|
||||
} catch (error) {
|
||||
if (generation === this.playbackGeneration) {
|
||||
this.fail(error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
handleGatewayEvent(event) {
|
||||
switch (event.type) {
|
||||
case "session.ready":
|
||||
this.startListening();
|
||||
break;
|
||||
|
||||
case "turn.started":
|
||||
if (this.state !== "starting") return;
|
||||
|
||||
this.turnId = event.turn_id;
|
||||
this.sequence = 0;
|
||||
|
||||
this.setState("recording");
|
||||
|
||||
for (const buffer of this.pending) {
|
||||
this.sendPCM(buffer);
|
||||
}
|
||||
|
||||
this.pending = [];
|
||||
|
||||
if (this.stopRequested) {
|
||||
this.flushTurn();
|
||||
}
|
||||
break;
|
||||
|
||||
case "stt.partial":
|
||||
case "stt.final":
|
||||
if (event.turn_id === this.turnId) {
|
||||
this.onTranscript(event.text, event.type === "stt.final");
|
||||
}
|
||||
break;
|
||||
|
||||
case "agent.delta":
|
||||
if (event.turn_id === this.turnId) {
|
||||
this.onAgentText(event.text, false);
|
||||
}
|
||||
break;
|
||||
|
||||
case "agent.response":
|
||||
if (event.turn_id === this.turnId) {
|
||||
this.onAgentText(event.text, true);
|
||||
}
|
||||
break;
|
||||
|
||||
case "turn.ended":
|
||||
if (event.turn_id === this.turnId) {
|
||||
this.gatewayTurnEnded = true;
|
||||
this.setState("waiting-playback");
|
||||
|
||||
void this.finishPlaybackAndResume();
|
||||
}
|
||||
break;
|
||||
|
||||
case "interrupted":
|
||||
this.close();
|
||||
break;
|
||||
|
||||
case "error":
|
||||
this.fail(
|
||||
new Error(`${event.code}: ${event.message}`)
|
||||
);
|
||||
break;
|
||||
|
||||
case "session.ended":
|
||||
this.close();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
fail(error) {
|
||||
this.onError(error);
|
||||
this.close();
|
||||
}
|
||||
|
||||
close() {
|
||||
if (this.state === "closed") return;
|
||||
|
||||
this.playbackGeneration++;
|
||||
this.player?.stop();
|
||||
|
||||
this.vad.stop();
|
||||
cancelAnimationFrame(this.frameId);
|
||||
|
||||
if (this.worklet) {
|
||||
this.worklet.port.postMessage({ type: "stop" });
|
||||
this.worklet.port.onmessage = null;
|
||||
this.worklet.disconnect();
|
||||
}
|
||||
|
||||
this.source?.disconnect();
|
||||
this.silentGain?.disconnect();
|
||||
|
||||
const socket = this.ws;
|
||||
this.ws = null;
|
||||
|
||||
if (socket) {
|
||||
socket.onclose = null;
|
||||
socket.onerror = null;
|
||||
socket.onmessage = null;
|
||||
|
||||
if (socket.readyState === WebSocket.OPEN) {
|
||||
if (this.turnId === null) {
|
||||
socket.send(JSON.stringify({ type: "session.end" }));
|
||||
} else {
|
||||
socket.send(JSON.stringify({
|
||||
type: "interrupt",
|
||||
turn_id: this.turnId,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
socket.close();
|
||||
}
|
||||
|
||||
this.worklet = null;
|
||||
this.source = null;
|
||||
this.silentGain = null;
|
||||
this.turnId = null;
|
||||
this.prebuffer = [];
|
||||
this.pending = [];
|
||||
this.setState("closed");
|
||||
}
|
||||
}
|
||||
204
tests/js/pcm-player.test.mjs
Normal file
204
tests/js/pcm-player.test.mjs
Normal file
@@ -0,0 +1,204 @@
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
|
||||
import { PCMPlayer } from
|
||||
"../../src/voice_assistant/audio_gateway/web/pcm-player.js";
|
||||
|
||||
class FakeAudioSource {
|
||||
constructor(context) {
|
||||
this.context = context;
|
||||
this.buffer = null;
|
||||
this.onended = null;
|
||||
this.startedAt = null;
|
||||
this.stopped = false;
|
||||
}
|
||||
|
||||
connect() {}
|
||||
|
||||
disconnect() {}
|
||||
|
||||
start(time) {
|
||||
this.startedAt = time;
|
||||
this.context.sources.push(this);
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.stopped = true;
|
||||
}
|
||||
|
||||
end() {
|
||||
this.onended?.();
|
||||
}
|
||||
}
|
||||
|
||||
class FakeAudioContext {
|
||||
constructor() {
|
||||
this.currentTime = 0;
|
||||
this.destination = {};
|
||||
this.sources = [];
|
||||
}
|
||||
|
||||
async resume() {}
|
||||
|
||||
createAnalyser() {
|
||||
return {
|
||||
fftSize: 1024,
|
||||
smoothingTimeConstant: 0.65,
|
||||
|
||||
connect() {},
|
||||
disconnect() {},
|
||||
|
||||
getFloatTimeDomainData(array) {
|
||||
array.fill(0);
|
||||
},
|
||||
|
||||
getByteFrequencyData(array) {
|
||||
array.fill(0);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
createBuffer(channels, length, sampleRate) {
|
||||
return {
|
||||
duration: length / sampleRate,
|
||||
copyToChannel(samples) {
|
||||
this.samples = samples;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
createBufferSource() {
|
||||
return new FakeAudioSource(this);
|
||||
}
|
||||
}
|
||||
|
||||
function pcm(values) {
|
||||
const buffer = new ArrayBuffer(values.length * 2);
|
||||
const view = new DataView(buffer);
|
||||
|
||||
values.forEach((value, index) => {
|
||||
view.setInt16(index * 2, value, true);
|
||||
});
|
||||
|
||||
return buffer;
|
||||
}
|
||||
|
||||
test("decodes signed PCM16 correctly", () => {
|
||||
const player = new PCMPlayer({
|
||||
audioContext: new FakeAudioContext(),
|
||||
});
|
||||
|
||||
const samples = player.decodePCM(
|
||||
pcm([-32768, 0, 16384, 32767])
|
||||
);
|
||||
|
||||
assert.equal(samples[0], -1);
|
||||
assert.equal(samples[1], 0);
|
||||
assert.equal(samples[2], 0.5);
|
||||
assert.ok(samples[3] > 0.99);
|
||||
});
|
||||
|
||||
test("schedules chunks in order", async () => {
|
||||
const context = new FakeAudioContext();
|
||||
const player = new PCMPlayer({
|
||||
audioContext: context,
|
||||
});
|
||||
|
||||
await player.enqueue(pcm([100, 200, 300, 400]));
|
||||
await player.enqueue(pcm([500, 600, 700, 800]));
|
||||
|
||||
assert.equal(context.sources.length, 2);
|
||||
|
||||
const first = context.sources[0];
|
||||
const second = context.sources[1];
|
||||
|
||||
assert.ok(second.startedAt >=
|
||||
first.startedAt + first.buffer.duration);
|
||||
});
|
||||
|
||||
test("waits for actual playback completion", async () => {
|
||||
const context = new FakeAudioContext();
|
||||
const player = new PCMPlayer({
|
||||
audioContext: context,
|
||||
});
|
||||
|
||||
await player.enqueue(pcm([100, 200]));
|
||||
|
||||
player.finish();
|
||||
|
||||
let completed = false;
|
||||
|
||||
const waiting = player.waitForCompletion().then(() => {
|
||||
completed = true;
|
||||
});
|
||||
|
||||
await Promise.resolve();
|
||||
|
||||
assert.equal(completed, false);
|
||||
|
||||
context.sources[0].end();
|
||||
|
||||
await waiting;
|
||||
|
||||
assert.equal(completed, true);
|
||||
});
|
||||
|
||||
test("stop interrupts scheduled playback", async () => {
|
||||
const context = new FakeAudioContext();
|
||||
const player = new PCMPlayer({
|
||||
audioContext: context,
|
||||
});
|
||||
|
||||
await player.enqueue(pcm([100, 200]));
|
||||
|
||||
player.stop();
|
||||
|
||||
assert.equal(context.sources[0].stopped, true);
|
||||
assert.equal(player.sources.size, 0);
|
||||
assert.equal(player.playing, false);
|
||||
});
|
||||
|
||||
test("reset allows a new playback turn", async () => {
|
||||
const context = new FakeAudioContext();
|
||||
const player = new PCMPlayer({
|
||||
audioContext: context,
|
||||
});
|
||||
|
||||
await player.enqueue(pcm([100, 200]));
|
||||
player.finish();
|
||||
context.sources[0].end();
|
||||
|
||||
await player.waitForCompletion();
|
||||
|
||||
player.reset();
|
||||
|
||||
await player.enqueue(pcm([300, 400]));
|
||||
|
||||
assert.equal(context.sources.length, 2);
|
||||
assert.equal(player.finished, false);
|
||||
});
|
||||
|
||||
test("rejects invalid PCM buffers", () => {
|
||||
const player = new PCMPlayer({
|
||||
audioContext: new FakeAudioContext(),
|
||||
});
|
||||
|
||||
assert.throws(
|
||||
() => player.decodePCM(new ArrayBuffer(3)),
|
||||
/Invalid PCM byte length/
|
||||
);
|
||||
});
|
||||
|
||||
test("exposes playback analyser", () => {
|
||||
const context = new FakeAudioContext();
|
||||
|
||||
const player = new PCMPlayer({
|
||||
audioContext: context,
|
||||
});
|
||||
|
||||
assert.equal(
|
||||
player.getAnalyser(),
|
||||
player.analyser
|
||||
);
|
||||
});
|
||||
83
tests/js/vad.test.mjs
Normal file
83
tests/js/vad.test.mjs
Normal file
@@ -0,0 +1,83 @@
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { VoiceActivityDetector } from "../../src/voice_assistant/audio_gateway/web/vad.js";
|
||||
|
||||
test("detects speech start after minimum duration", () => {
|
||||
let starts = 0;
|
||||
|
||||
const vad = new VoiceActivityDetector({
|
||||
minSpeechMs: 250,
|
||||
onSpeechStart: () => starts++,
|
||||
});
|
||||
|
||||
vad.start();
|
||||
|
||||
vad.process(0.04, 0);
|
||||
vad.process(0.04, 100);
|
||||
|
||||
assert.equal(starts, 0);
|
||||
|
||||
vad.process(0.04, 260);
|
||||
|
||||
assert.equal(starts, 1);
|
||||
});
|
||||
|
||||
test("detects speech end after silence", () => {
|
||||
let ends = 0;
|
||||
|
||||
const vad = new VoiceActivityDetector({
|
||||
minSpeechMs: 100,
|
||||
silenceMs: 800,
|
||||
onSpeechEnd: () => ends++,
|
||||
});
|
||||
|
||||
vad.start();
|
||||
|
||||
vad.process(0.04, 0);
|
||||
vad.process(0.04, 120);
|
||||
vad.process(0.04, 300);
|
||||
|
||||
vad.process(0.001, 600);
|
||||
vad.process(0.001, 1000);
|
||||
|
||||
assert.equal(ends, 0);
|
||||
|
||||
vad.process(0.001, 1101);
|
||||
|
||||
assert.equal(ends, 1);
|
||||
});
|
||||
|
||||
test("ignores short noise spikes", () => {
|
||||
let starts = 0;
|
||||
|
||||
const vad = new VoiceActivityDetector({
|
||||
minSpeechMs: 250,
|
||||
onSpeechStart: () => starts++,
|
||||
});
|
||||
|
||||
vad.start();
|
||||
|
||||
vad.process(0.05, 0);
|
||||
vad.process(0.001, 80);
|
||||
vad.process(0.001, 300);
|
||||
|
||||
assert.equal(starts, 0);
|
||||
});
|
||||
|
||||
test("stop disables detection", () => {
|
||||
let starts = 0;
|
||||
|
||||
const vad = new VoiceActivityDetector({
|
||||
minSpeechMs: 100,
|
||||
onSpeechStart: () => starts++,
|
||||
});
|
||||
|
||||
vad.start();
|
||||
vad.stop();
|
||||
|
||||
vad.process(0.05, 0);
|
||||
vad.process(0.05, 200);
|
||||
|
||||
assert.equal(starts, 0);
|
||||
});
|
||||
339
tests/js/voice-session.test.mjs
Normal file
339
tests/js/voice-session.test.mjs
Normal file
@@ -0,0 +1,339 @@
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { afterEach, test } from "node:test";
|
||||
|
||||
import { VoiceSessionController } from
|
||||
"../../src/voice_assistant/audio_gateway/web/voice-session.js";
|
||||
|
||||
const originalWebSocket = globalThis.WebSocket;
|
||||
|
||||
class FakeWebSocket {
|
||||
static OPEN = 1;
|
||||
static CLOSED = 3;
|
||||
|
||||
constructor() {
|
||||
this.readyState = FakeWebSocket.OPEN;
|
||||
this.sent = [];
|
||||
this.closed = false;
|
||||
}
|
||||
|
||||
send(data) {
|
||||
this.sent.push(data);
|
||||
}
|
||||
|
||||
close() {
|
||||
this.closed = true;
|
||||
this.readyState = FakeWebSocket.CLOSED;
|
||||
}
|
||||
}
|
||||
|
||||
globalThis.WebSocket = FakeWebSocket;
|
||||
globalThis.cancelAnimationFrame = () => {};
|
||||
|
||||
afterEach(() => {
|
||||
// Restore only state modified by individual tests.
|
||||
});
|
||||
|
||||
function makeController() {
|
||||
const states = [];
|
||||
const errors = [];
|
||||
const workletMessages = [];
|
||||
|
||||
const controller = new VoiceSessionController({
|
||||
onStateChange: (state) => states.push(state),
|
||||
onError: (error) => errors.push(error),
|
||||
});
|
||||
|
||||
controller.ws = new FakeWebSocket();
|
||||
controller.worklet = {
|
||||
port: {
|
||||
postMessage: (data) => workletMessages.push(data),
|
||||
onmessage: null,
|
||||
},
|
||||
disconnect() {},
|
||||
};
|
||||
|
||||
controller.source = { disconnect() {} };
|
||||
controller.silentGain = { disconnect() {} };
|
||||
|
||||
return {
|
||||
controller,
|
||||
states,
|
||||
errors,
|
||||
workletMessages,
|
||||
};
|
||||
}
|
||||
|
||||
test("speech detection requests a new turn", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
controller.setState("listening");
|
||||
controller.beginTurn();
|
||||
|
||||
assert.equal(controller.state, "starting");
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(controller.ws.sent[0]),
|
||||
{ type: "audio.start" }
|
||||
);
|
||||
});
|
||||
|
||||
test("PCM is buffered until turn.started", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
controller.setState("listening");
|
||||
|
||||
controller.handlePCM(new ArrayBuffer(4));
|
||||
|
||||
assert.equal(controller.prebuffer.length, 1);
|
||||
assert.equal(controller.ws.sent.length, 0);
|
||||
|
||||
controller.beginTurn();
|
||||
controller.handlePCM(new ArrayBuffer(6));
|
||||
|
||||
assert.equal(controller.pending.length, 2);
|
||||
assert.equal(controller.ws.sent.length, 1);
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "recording");
|
||||
assert.equal(controller.pending.length, 0);
|
||||
assert.equal(controller.sequence, 2);
|
||||
assert.equal(controller.ws.sent.length, 5);
|
||||
});
|
||||
|
||||
test("PCM metadata precedes each binary frame", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
controller.turnId = 7;
|
||||
controller.setState("recording");
|
||||
|
||||
const pcm = new ArrayBuffer(8);
|
||||
|
||||
controller.sendPCM(pcm);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(controller.ws.sent[0]),
|
||||
{
|
||||
type: "audio.chunk",
|
||||
turn_id: 7,
|
||||
sequence_number: 0,
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(controller.ws.sent[1], pcm);
|
||||
assert.equal(controller.sequence, 1);
|
||||
});
|
||||
|
||||
test("speech end requests worklet flush", () => {
|
||||
const { controller, workletMessages } = makeController();
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("recording");
|
||||
|
||||
controller.finishSpeech();
|
||||
|
||||
assert.equal(controller.state, "flushing");
|
||||
assert.deepEqual(
|
||||
workletMessages.at(-1),
|
||||
{ type: "flush" }
|
||||
);
|
||||
|
||||
assert.equal(controller.ws.sent.length, 0);
|
||||
});
|
||||
|
||||
test("audio.stop is sent after flushed", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("flushing");
|
||||
|
||||
controller.handlePCM(new ArrayBuffer(4));
|
||||
controller.handleFlushed();
|
||||
|
||||
assert.equal(controller.state, "processing");
|
||||
|
||||
const lastMessage = JSON.parse(controller.ws.sent.at(-1));
|
||||
|
||||
assert.deepEqual(lastMessage, {
|
||||
type: "audio.stop",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
assert.equal(controller.ws.sent.length, 3);
|
||||
});
|
||||
|
||||
test("speech ending before turn.started is handled", () => {
|
||||
const { controller, workletMessages } = makeController();
|
||||
|
||||
controller.setState("listening");
|
||||
controller.beginTurn();
|
||||
controller.finishSpeech();
|
||||
|
||||
assert.equal(controller.stopRequested, true);
|
||||
assert.equal(controller.state, "starting");
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 3,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "flushing");
|
||||
assert.deepEqual(
|
||||
workletMessages.at(-1),
|
||||
{ type: "flush" }
|
||||
);
|
||||
});
|
||||
|
||||
test("closing a session releases resources", () => {
|
||||
const { controller, workletMessages } = makeController();
|
||||
const socket = controller.ws;
|
||||
|
||||
controller.setState("listening");
|
||||
controller.close();
|
||||
|
||||
assert.equal(controller.state, "closed");
|
||||
assert.equal(socket.closed, true);
|
||||
assert.equal(controller.ws, null);
|
||||
assert.deepEqual(
|
||||
workletMessages.at(-1),
|
||||
{ type: "stop" }
|
||||
);
|
||||
});
|
||||
|
||||
test("gateway errors close the session", () => {
|
||||
const { controller, errors } = makeController();
|
||||
|
||||
controller.setState("processing");
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "error",
|
||||
code: "STT_START_FAILED",
|
||||
message: "Provider unavailable",
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "closed");
|
||||
assert.equal(errors.length, 1);
|
||||
assert.match(
|
||||
errors[0].message,
|
||||
/STT_START_FAILED/
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
test("turn.ended waits for playback before listening", async () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
let resolvePlayback;
|
||||
|
||||
controller.player = {
|
||||
finish() {},
|
||||
reset() {},
|
||||
stop() {},
|
||||
waitForCompletion() {
|
||||
return new Promise((resolve) => {
|
||||
resolvePlayback = resolve;
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("processing");
|
||||
|
||||
let resumed = false;
|
||||
|
||||
controller.startListening = () => {
|
||||
resumed = true;
|
||||
};
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.ended",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
assert.equal(resumed, false);
|
||||
|
||||
resolvePlayback();
|
||||
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
assert.equal(resumed, true);
|
||||
});
|
||||
|
||||
test("closing during playback prevents listening resume", async () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
let resolvePlayback;
|
||||
|
||||
controller.player = {
|
||||
finish() {},
|
||||
reset() {},
|
||||
stop() {},
|
||||
waitForCompletion() {
|
||||
return new Promise((resolve) => {
|
||||
resolvePlayback = resolve;
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("processing");
|
||||
|
||||
let resumed = false;
|
||||
|
||||
controller.startListening = () => {
|
||||
resumed = true;
|
||||
};
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.ended",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
controller.close();
|
||||
|
||||
resolvePlayback();
|
||||
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
assert.equal(resumed, false);
|
||||
assert.equal(controller.state, "closed");
|
||||
});
|
||||
|
||||
test("binary audio is forwarded to PCMPlayer", async () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
const received = [];
|
||||
|
||||
controller.player = {
|
||||
async enqueue(buffer) {
|
||||
received.push(buffer);
|
||||
},
|
||||
finish() {},
|
||||
reset() {},
|
||||
stop() {},
|
||||
waitForCompletion: async () => {},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("processing");
|
||||
|
||||
const pcm = new ArrayBuffer(8);
|
||||
|
||||
controller.handleAudioChunk(pcm);
|
||||
|
||||
await controller.playbackPending;
|
||||
|
||||
assert.deepEqual(received, [pcm]);
|
||||
});
|
||||
@@ -11,8 +11,10 @@ def test_web_client_page():
|
||||
response = client.get("/ui")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "AgentVoice" in response.text
|
||||
assert "/web/app.js" in response.text
|
||||
assert "Oracle — assistente de voz" in response.text
|
||||
assert 'id="microphone"' in response.text
|
||||
assert 'id="feedback"' in response.text
|
||||
assert "assistantIntegration" in response.text
|
||||
|
||||
|
||||
def test_web_client_javascript():
|
||||
|
||||
Reference in New Issue
Block a user