feat: integra VAD ao controle de turnos da conversa
This commit is contained in:
alex alves
93
tests/test_vad_session_composition.py
Normal file
93
tests/test_vad_session_composition.py
Normal file
@@ -0,0 +1,93 @@
|
||||
"""Composition selects VAD input and routes application events to the channel."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from voice_assistant.agent.mock import MockAgentAdapter
|
||||
from voice_assistant.application.events import TurnOpened, TurnInterrupted
|
||||
from voice_assistant.audio_gateway.app import main
|
||||
from voice_assistant.config.loader import load_config
|
||||
from voice_assistant.tts.mock import MockTTSAdapter
|
||||
from voice_assistant.vad.adapter import VADAdapter
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def anyio_backend():
|
||||
return 'asyncio'
|
||||
|
||||
|
||||
class VAD(VADAdapter):
|
||||
async def process(self, audio):
|
||||
return ()
|
||||
|
||||
async def flush(self):
|
||||
return ()
|
||||
|
||||
async def reset(self):
|
||||
pass
|
||||
|
||||
async def aclose(self):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
@pytest.mark.parametrize('server_vad', [False, True])
|
||||
async def test_composition_selects_vad_and_accepts_environment_policy(monkeypatch, server_vad):
|
||||
config = load_config('development', config_dir=Path(__file__).resolve().parents[1] / 'config')
|
||||
assert config.voice_input == {
|
||||
'pre_roll_ms': 200, 'history_ms': 2000, 'max_input_chunk_bytes': 65536,
|
||||
}
|
||||
monkeypatch.setattr(main, 'app_config', config)
|
||||
monkeypatch.setattr(main, 'create_agent_adapter', lambda **kwargs: MockAgentAdapter())
|
||||
monkeypatch.setattr(main, 'create_tts_adapter', lambda **kwargs: MockTTSAdapter())
|
||||
detector = VAD()
|
||||
calls = []
|
||||
|
||||
def factory(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return detector
|
||||
|
||||
monkeypatch.setattr(main, 'create_vad_adapter', factory)
|
||||
session = await main.create_voice_session(AsyncMock(), None, server_vad=server_vad)
|
||||
try:
|
||||
if server_vad:
|
||||
assert len(calls) == 1
|
||||
assert calls[0]['config'] is config
|
||||
await session.receive_audio(bytes(512))
|
||||
else:
|
||||
assert calls == []
|
||||
with pytest.raises(Exception, match='requires VAD'):
|
||||
await session.receive_audio(bytes(512))
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_bad_policy_releases_all_created_providers(monkeypatch):
|
||||
config = main.app_config.model_copy(update={'voice_input': {'history_ms': 0}})
|
||||
agent, tts, vad = MockAgentAdapter(), MockTTSAdapter(), VAD()
|
||||
for provider in [agent, tts, vad]:
|
||||
provider.aclose = AsyncMock()
|
||||
monkeypatch.setattr(main, 'app_config', config)
|
||||
monkeypatch.setattr(main, 'create_agent_adapter', lambda **kwargs: agent)
|
||||
monkeypatch.setattr(main, 'create_tts_adapter', lambda **kwargs: tts)
|
||||
monkeypatch.setattr(main, 'create_vad_adapter', lambda **kwargs: vad)
|
||||
with pytest.raises(ValueError, match='history_ms'):
|
||||
await main.create_voice_session(AsyncMock(), None, server_vad=True)
|
||||
for provider in [agent, tts, vad]:
|
||||
provider.aclose.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_automatic_turn_events_map_to_existing_protocol():
|
||||
socket = AsyncMock()
|
||||
await main.send_voice_event(socket, TurnOpened('session', 2))
|
||||
await main.send_voice_event(socket, TurnInterrupted('session', 1))
|
||||
events = [json.loads(call.args[0]) for call in socket.send_text.await_args_list]
|
||||
assert events == [
|
||||
{'type': 'turn.started', 'session_id': 'session', 'turn_id': 2},
|
||||
{'type': 'interrupted', 'session_id': 'session', 'turn_id': 1},
|
||||
]
|
||||
370
tests/test_voice_session_vad.py
Normal file
370
tests/test_voice_session_vad.py
Normal file
@@ -0,0 +1,370 @@
|
||||
"""Continuous, channel-independent PCM controls conversation turns."""
|
||||
|
||||
import asyncio
|
||||
import struct
|
||||
|
||||
import pytest
|
||||
|
||||
from voice_assistant.agent.mock import MockAgentAdapter
|
||||
from voice_assistant.application.events import TurnCompleted, TurnOpened, TurnInterrupted, SpeechChunk
|
||||
from voice_assistant.application.voice_session import VoiceSession, VoiceSessionError
|
||||
from voice_assistant.domain.session import SessionState
|
||||
from voice_assistant.stt.mock import MockSTTAdapter
|
||||
from voice_assistant.tts.mock import MockTTSAdapter
|
||||
from voice_assistant.turn_manager import TurnManager
|
||||
from voice_assistant.vad.adapter import VADAdapter
|
||||
from voice_assistant.vad.models import SpeechStarted, SpeechEnded
|
||||
from voice_assistant.voice_pipeline import VoicePipeline
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def anyio_backend():
|
||||
return "asyncio"
|
||||
|
||||
|
||||
class ScriptedVAD(VADAdapter):
|
||||
def __init__(self, script=(), tail=()):
|
||||
self.script = list(script)
|
||||
self.tail = tail
|
||||
self.received = []
|
||||
self.closed = False
|
||||
|
||||
async def process(self, audio):
|
||||
self.received.append(audio)
|
||||
return self.script.pop(0) if self.script else ()
|
||||
|
||||
async def flush(self):
|
||||
return self.tail
|
||||
|
||||
async def reset(self):
|
||||
pass
|
||||
|
||||
async def aclose(self):
|
||||
self.closed = True
|
||||
|
||||
|
||||
class RecordingSTT(MockSTTAdapter):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.received = []
|
||||
self.aborted = False
|
||||
|
||||
async def send_audio(self, audio):
|
||||
await super().send_audio(audio)
|
||||
self.received.append(audio)
|
||||
|
||||
async def stop_session(self):
|
||||
await self.emit_final("Olá")
|
||||
await super().stop_session()
|
||||
|
||||
async def abort_session(self):
|
||||
self.aborted = True
|
||||
await super().abort_session()
|
||||
|
||||
|
||||
def build(vad, factory=RecordingSTT, event_sink=None, **options):
|
||||
providers = []
|
||||
events = []
|
||||
completed = asyncio.Event()
|
||||
|
||||
def create_stt():
|
||||
provider = factory()
|
||||
providers.append(provider)
|
||||
return provider
|
||||
|
||||
async def emit(event):
|
||||
events.append(event)
|
||||
if event_sink is not None:
|
||||
await event_sink(event)
|
||||
if isinstance(event, TurnCompleted):
|
||||
completed.set()
|
||||
|
||||
session = VoiceSession(
|
||||
pipeline=VoicePipeline(
|
||||
session=SessionState(), turn_manager=TurnManager(),
|
||||
agent=MockAgentAdapter(), tts=MockTTSAdapter(),
|
||||
),
|
||||
create_stt=create_stt, emit=emit, vad=vad, **options,
|
||||
)
|
||||
return session, providers, events, completed
|
||||
|
||||
|
||||
def pcm(value, samples=256):
|
||||
return struct.pack(f'<{samples}h', *([value] * samples))
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_delayed_start_replays_pre_roll_without_duplicating_audio():
|
||||
vad = ScriptedVAD([(), (), (SpeechStarted(400),), ()])
|
||||
session, providers, events, _ = build(vad, pre_roll_ms=10)
|
||||
chunks = [pcm(i) for i in [1, 2, 3, 4]]
|
||||
try:
|
||||
await session.receive_audio(chunks[0])
|
||||
await session.receive_audio(chunks[1])
|
||||
assert providers == []
|
||||
await session.receive_audio(chunks[2])
|
||||
await session.receive_audio(chunks[3])
|
||||
assert len(providers) == 1
|
||||
assert b''.join(providers[0].received) == b''.join(chunks)[240 * 2:]
|
||||
assert isinstance(events[0], TurnOpened)
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_speech_end_finalizes_in_background_while_silence_continues():
|
||||
vad = ScriptedVAD([(SpeechStarted(0),), (SpeechEnded(400, 'silence'),), ()])
|
||||
session, providers, events, completed = build(vad, pre_roll_ms=0)
|
||||
try:
|
||||
await session.receive_audio(pcm(1))
|
||||
await session.receive_audio(pcm(2))
|
||||
await session.receive_audio(pcm(3))
|
||||
await asyncio.wait_for(completed.wait(), timeout=1.0)
|
||||
assert b''.join(providers[0].received) == pcm(1) + pcm(2)[:144 * 2]
|
||||
assert isinstance(events[-1], TurnCompleted)
|
||||
assert session.active_turn_id is None
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_multiple_speech_intervals_in_one_chunk_keep_audio_separate():
|
||||
vad = ScriptedVAD([(
|
||||
SpeechStarted(0), SpeechEnded(160, 'silence'),
|
||||
SpeechStarted(240), SpeechEnded(400, 'silence'),
|
||||
)])
|
||||
session, providers, events, completed = build(vad, pre_roll_ms=0)
|
||||
audio = struct.pack('<512h', *range(512))
|
||||
try:
|
||||
await session.receive_audio(audio)
|
||||
await asyncio.wait_for(completed.wait(), timeout=1.0)
|
||||
assert len(providers) == 2
|
||||
assert b''.join(providers[0].received) == audio[:320]
|
||||
assert b''.join(providers[1].received) == audio[480:800]
|
||||
assert any(isinstance(event, TurnInterrupted) and event.turn_id == 1 for event in events)
|
||||
assert [event.turn_id for event in events if isinstance(event, TurnOpened)] == [1, 2]
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_new_speech_cancels_stt_finalization_and_opens_fresh_turn():
|
||||
started = asyncio.Event()
|
||||
cancelled = asyncio.Event()
|
||||
|
||||
class WaitingSTT(RecordingSTT):
|
||||
async def stop_session(self):
|
||||
started.set()
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
except asyncio.CancelledError:
|
||||
cancelled.set()
|
||||
raise
|
||||
|
||||
vad = ScriptedVAD([
|
||||
(SpeechStarted(0),), (SpeechEnded(400, 'silence'),), (SpeechStarted(600),),
|
||||
])
|
||||
session, providers, events, _ = build(vad, factory=WaitingSTT, pre_roll_ms=0)
|
||||
try:
|
||||
await session.receive_audio(pcm(1))
|
||||
await session.receive_audio(pcm(2))
|
||||
await asyncio.wait_for(started.wait(), timeout=1.0)
|
||||
await session.receive_audio(pcm(3))
|
||||
assert cancelled.is_set()
|
||||
assert providers[0].aborted
|
||||
assert len(providers) == 2
|
||||
assert session.active_turn_id == 2
|
||||
assert b''.join(providers[1].received) == pcm(3)[88 * 2:]
|
||||
assert any(isinstance(event, TurnInterrupted) for event in events)
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
@pytest.mark.parametrize('audio', [b'', b'\x00', bytes(65538)])
|
||||
async def test_invalid_input_does_not_reach_vad_or_advance_history(audio):
|
||||
vad = ScriptedVAD([(SpeechStarted(0),)])
|
||||
session, providers, _, _ = build(vad, pre_roll_ms=0)
|
||||
try:
|
||||
with pytest.raises(ValueError):
|
||||
await session.receive_audio(audio)
|
||||
assert vad.received == []
|
||||
await session.receive_audio(pcm(1))
|
||||
assert b''.join(providers[0].received) == pcm(1)
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_manual_commands_cannot_compete_with_vad_turn_control():
|
||||
session, _, _, _ = build(ScriptedVAD())
|
||||
try:
|
||||
for operation in [session.start_turn, lambda: session.send_audio(1, 0, pcm(1))]:
|
||||
with pytest.raises(VoiceSessionError) as error:
|
||||
await operation()
|
||||
assert error.value.code == 'TURN_CONTROLLED_BY_VAD'
|
||||
with pytest.raises(VoiceSessionError):
|
||||
session.stop_turn(1)
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_input_error_prevents_unsafe_retry_and_cleanup_closes_vad():
|
||||
class FailingVAD(ScriptedVAD):
|
||||
async def process(self, audio):
|
||||
raise RuntimeError('Inference failed')
|
||||
|
||||
vad = FailingVAD()
|
||||
session, _, _, _ = build(vad)
|
||||
try:
|
||||
with pytest.raises(VoiceSessionError, match='Inference failed'):
|
||||
await session.receive_audio(pcm(1))
|
||||
with pytest.raises(VoiceSessionError, match='new session'):
|
||||
await session.receive_audio(pcm(1))
|
||||
finally:
|
||||
await session.aclose()
|
||||
assert vad.closed
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_speech_older_than_history_is_reported_instead_of_silently_truncated():
|
||||
vad = ScriptedVAD([(), (SpeechStarted(0),)])
|
||||
session, providers, _, _ = build(vad, pre_roll_ms=0, history_ms=1)
|
||||
try:
|
||||
await session.receive_audio(pcm(1))
|
||||
with pytest.raises(VoiceSessionError, match='increase history_ms'):
|
||||
await session.receive_audio(pcm(2))
|
||||
assert providers == []
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_normal_end_of_input_flushes_active_speech():
|
||||
vad = ScriptedVAD([(SpeechStarted(0),)], tail=(SpeechEnded(256, 'end_of_input'),))
|
||||
session, _, _, completed = build(vad, pre_roll_ms=0)
|
||||
try:
|
||||
await session.receive_audio(pcm(1))
|
||||
await session.finish_audio_input()
|
||||
await asyncio.wait_for(completed.wait(), timeout=1.0)
|
||||
with pytest.raises(VoiceSessionError, match='closed'):
|
||||
await session.receive_audio(pcm(2))
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_session_end_during_vad_processing_does_not_open_late_turn():
|
||||
started = asyncio.Event()
|
||||
release = asyncio.Event()
|
||||
|
||||
class WaitingVAD(ScriptedVAD):
|
||||
async def process(self, audio):
|
||||
started.set()
|
||||
await release.wait()
|
||||
return (SpeechStarted(0),)
|
||||
|
||||
vad = WaitingVAD()
|
||||
session, providers, events, _ = build(vad)
|
||||
receiving = asyncio.create_task(session.receive_audio(pcm(1)))
|
||||
ending = None
|
||||
try:
|
||||
await asyncio.wait_for(started.wait(), timeout=1.0)
|
||||
ending = asyncio.create_task(session.end())
|
||||
await asyncio.sleep(0)
|
||||
release.set()
|
||||
with pytest.raises(VoiceSessionError, match='shutdown'):
|
||||
await receiving
|
||||
await asyncio.wait_for(ending, timeout=1.0)
|
||||
assert session.closed
|
||||
assert providers == []
|
||||
assert events == []
|
||||
finally:
|
||||
release.set()
|
||||
await asyncio.gather(receiving, return_exceptions=True)
|
||||
if ending:
|
||||
await asyncio.gather(ending, return_exceptions=True)
|
||||
await session.aclose()
|
||||
|
||||
|
||||
@pytest.mark.parametrize('options', [
|
||||
{'pre_roll_ms': -1}, {'pre_roll_ms': True}, {'history_ms': 0},
|
||||
{'history_ms': 100, 'pre_roll_ms': 200}, {'max_input_chunk_bytes': 0},
|
||||
])
|
||||
def test_invalid_audio_history_policy_is_rejected(options):
|
||||
with pytest.raises(ValueError):
|
||||
build(ScriptedVAD(), **options)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_empty_stt_result_releases_turn_without_stopping_continuous_input():
|
||||
interrupted = asyncio.Event()
|
||||
|
||||
async def sink(event):
|
||||
if isinstance(event, TurnInterrupted):
|
||||
interrupted.set()
|
||||
|
||||
vad = ScriptedVAD([
|
||||
(SpeechStarted(0),), (SpeechEnded(400, 'silence'),), (SpeechStarted(600),),
|
||||
])
|
||||
session, providers, _, _ = build(vad, factory=MockSTTAdapter, event_sink=sink)
|
||||
try:
|
||||
await session.receive_audio(pcm(1))
|
||||
await session.receive_audio(pcm(2))
|
||||
await asyncio.wait_for(interrupted.wait(), timeout=1.0)
|
||||
assert session.active_turn_id is None
|
||||
await session.receive_audio(pcm(3))
|
||||
assert session.active_turn_id == 2
|
||||
assert len(providers) == 2
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_new_speech_cancels_response_delivery_before_opening_next_turn():
|
||||
delivering = asyncio.Event()
|
||||
cancelled = asyncio.Event()
|
||||
|
||||
async def sink(event):
|
||||
if isinstance(event, SpeechChunk) and event.turn_id == 1:
|
||||
delivering.set()
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
except asyncio.CancelledError:
|
||||
cancelled.set()
|
||||
raise
|
||||
|
||||
vad = ScriptedVAD([
|
||||
(SpeechStarted(0),), (SpeechEnded(400, 'silence'),), (SpeechStarted(600),),
|
||||
])
|
||||
session, providers, events, _ = build(vad, event_sink=sink, pre_roll_ms=0)
|
||||
try:
|
||||
await session.receive_audio(pcm(1))
|
||||
await session.receive_audio(pcm(2))
|
||||
await asyncio.wait_for(delivering.wait(), timeout=1.0)
|
||||
await session.receive_audio(pcm(3))
|
||||
assert cancelled.is_set()
|
||||
assert providers[0].aborted
|
||||
assert session.active_turn_id == 2
|
||||
opened = next(i for i, event in enumerate(events) if isinstance(event, TurnOpened) and event.turn_id == 2)
|
||||
assert not any(isinstance(event, SpeechChunk) and event.turn_id == 1 for event in events[opened:])
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_pre_roll_never_replays_previous_utterance_into_next_stt():
|
||||
vad = ScriptedVAD([(
|
||||
SpeechStarted(0), SpeechEnded(160, 'silence'),
|
||||
SpeechStarted(240), SpeechEnded(400, 'silence'),
|
||||
)])
|
||||
session, providers, _, completed = build(vad, pre_roll_ms=200)
|
||||
audio = struct.pack('<512h', *range(512))
|
||||
try:
|
||||
await session.receive_audio(audio)
|
||||
await asyncio.wait_for(completed.wait(), timeout=1.0)
|
||||
assert b''.join(providers[0].received) == audio[:320]
|
||||
assert b''.join(providers[1].received) == audio[320:800]
|
||||
finally:
|
||||
await session.aclose()
|
||||
Reference in New Issue
Block a user