Files
2026-08-21 08:37:51 -03:00

4803 lines
194 KiB
Python

from __future__ import annotations
import asyncio
import importlib
import json
import os
import sys
import time
import types
import unittest
import uuid
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Any
from unittest import mock
from types import SimpleNamespace
def _install_fake_livekit() -> None:
try:
importlib.import_module("livekit.agents")
return
except ImportError:
pass
livekit_module = types.ModuleType("livekit")
agents_module = types.ModuleType("livekit.agents")
class UserInputTranscribedEvent:
def __init__(self, transcript: str = "", is_final: bool = True) -> None:
self.transcript = transcript
self.is_final = is_final
class _AudioInputOptions:
def __init__(self, **kwargs) -> None:
self.kwargs = kwargs
class _AudioOutputOptions:
def __init__(self, **kwargs) -> None:
self.kwargs = kwargs
class _RoomOptions:
def __init__(self, **kwargs) -> None:
self.kwargs = kwargs
agents_module.UserInputTranscribedEvent = UserInputTranscribedEvent
agents_module.room_io = SimpleNamespace(
AudioInputOptions=_AudioInputOptions,
AudioOutputOptions=_AudioOutputOptions,
RoomOptions=_RoomOptions,
)
livekit_module.agents = agents_module
sys.modules["livekit"] = livekit_module
sys.modules["livekit.agents"] = agents_module
_install_fake_livekit()
from app.livekit.policies.finalization_policy import FinalizationPolicy
from app.livekit.policies.idle_policy import IdlePolicy, IdlePolicyConfig
from app.livekit.policies.interrupt_policy import InterruptPolicy
from app.livekit.adapters.speech_service import SpeechService
from app.livekit.adapters.agent_backend import BackendReply
from app.livekit.runtime.call_runtime import (
CallRuntime,
InflightBackendWaitNoticeResult,
InflightBackendWaitTimedOut,
)
from app.livekit.runtime.command_executor import RuntimeCommandExecutor
from app.livekit.runtime.commands import (
EndServiceOnce,
StartSpeech,
ExportSession,
ExtractSpokenText,
InjectIdleNudge,
InterruptSpeech,
NotifyBridgeDone,
NotifyBridgeStop,
RunPipelineInput,
SetPendingInterrupt,
SetPipelineInterruption,
SetPipelineProcessingInterruption,
StartSession,
)
from app.livekit.runtime.scheduler import TimerScheduler
from app.livekit.runtime.state import RuntimeConfig
from app.livekit.vad_dynamic_threshold import (
DynamicVADThresholdConfig,
DynamicVADThresholdController,
)
from app.providers.stt_internal_livekit import stt_text_with_single_word_threshold
from app.utils.logging import StructuredLogContext
from app.utils.turn_ids import (
peek_started_turn_message_id,
register_started_turn_message_id,
register_transcribed_turn,
reset_turn_message_sequence,
)
WAIT_LONG_AUDIO_PATH = str(
Path(__file__).resolve().parents[2]
/ "src"
/ "app"
/ "livekit"
/ "assets"
/ "comfort"
/ "long"
/ "01.wav"
)
DEFAULT_WAIT_TEXT = "Um momento, ainda estou consultando para te ajudar."
def _expected_wait_text_for_audio(path: str | Path, fallback: str = DEFAULT_WAIT_TEXT) -> str:
text_path = Path(path).with_suffix(".txt")
if not text_path.is_file():
return fallback
for encoding in ("utf-8-sig", "utf-8", "latin-1"):
try:
text = " ".join(text_path.read_text(encoding=encoding).split())
except UnicodeDecodeError:
continue
except OSError:
return fallback
return text or fallback
return fallback
class _NullLogger:
def info(self, *args, **kwargs) -> None:
return None
def debug(self, *args, **kwargs) -> None:
return None
def warning(self, *args, **kwargs) -> None:
return None
def exception(self, *args, **kwargs) -> None:
return None
class _AsyncTestCase(unittest.IsolatedAsyncioTestCase):
async def asyncSetUp(self) -> None:
asyncio.get_running_loop().set_debug(False)
class _FakeAgent:
def __init__(self) -> None:
self._ready = asyncio.Event()
self._ready.set()
self._run_lock = asyncio.Lock()
self.pipeline = object()
self.pending_interrupts: list[tuple[str, bool, str]] = []
self._consumed_interrupt = (None, "", False)
self.end_calls = 0
async def set_pending_interrupt(
self,
*,
listened_text: str = "",
skipped: bool = False,
speech_id: str = "",
) -> None:
self.pending_interrupts.append((listened_text, skipped, speech_id))
async def consume_pending_interrupt(self):
return self._consumed_interrupt
async def end_service_once(self):
self.end_calls += 1
return BackendReply(stage="DONE", done=True, export_payload=[{"status": "ok"}])
class _FakePushPipeline:
def __init__(self, replies) -> None:
self._replies = asyncio.Queue()
for reply in replies:
self._replies.put_nowait(reply)
def supports_server_push(self) -> bool:
return True
async def wait_for_server_push(self):
if self._replies.empty():
return None
return await self._replies.get()
class _FakeInflightPushPipeline:
def __init__(self) -> None:
self._replies = asyncio.Queue()
self._closed = asyncio.Event()
def supports_inflight_backend_push(self) -> bool:
return True
def supports_server_push(self) -> bool:
return False
async def push(self, reply) -> None:
await self._replies.put(reply)
def close(self) -> None:
self._closed.set()
async def wait_for_server_push(self):
if not self._replies.empty():
return await self._replies.get()
if self._closed.is_set():
return None
get_task = asyncio.create_task(self._replies.get())
close_task = asyncio.create_task(self._closed.wait())
done, _pending = await asyncio.wait(
{get_task, close_task},
return_when=asyncio.FIRST_COMPLETED,
)
if get_task in done:
close_task.cancel()
await asyncio.gather(close_task, return_exceptions=True)
return get_task.result()
get_task.cancel()
await asyncio.gather(get_task, return_exceptions=True)
return None
class _FakeAgentInput:
def __init__(self) -> None:
self.audio_enabled = True
self.calls = []
def set_audio_enabled(self, enabled: bool) -> None:
self.audio_enabled = bool(enabled)
self.calls.append(bool(enabled))
class _FakeSession:
def __init__(self) -> None:
self.handlers = {}
self.start_calls = []
self.input = _FakeAgentInput()
def on(self, event_name: str):
def _decorator(fn):
self.handlers[event_name] = fn
return fn
return _decorator
async def start(self, **kwargs) -> None:
self.start_calls.append(kwargs)
class _FakeRoom:
def __init__(self) -> None:
self.name = "room-test"
self.remote_participants = {}
self.handlers = {}
def on(self, event_name: str):
def _decorator(fn):
self.handlers[event_name] = fn
return fn
return _decorator
class _FakeTimeline:
def __init__(self) -> None:
self.events = []
def emit(self, event: str, **fields) -> None:
self.events.append((event, fields))
class _FakeContext:
def __init__(self) -> None:
self.room = _FakeRoom()
self.shutdown_callbacks = []
def add_shutdown_callback(self, callback) -> None:
self.shutdown_callbacks.append(callback)
class _FakeCommandExecutor:
def __init__(self) -> None:
self.commands = []
self.pipeline_result = BackendReply(stage="PRESENTATION", text="resposta")
self.end_reply = BackendReply(stage="DONE", done=True, export_payload=[{"status": "ok"}])
self.spoken_text = "trecho falado"
self.speech_handle = SimpleNamespace(
id="speech-test",
interrupted=False,
)
self.speech_handles = None
self.run_pipeline_side_effect = None
self.on_start_session: Any = None
self.wait_for_playout_delay_s = 0.0
self.wait_for_playout_delays_s = []
self.wait_for_playout_exceptions = []
self.wait_for_playout_callbacks = []
async def execute(self, command):
self.commands.append(command)
if isinstance(command, RunPipelineInput):
if self.run_pipeline_side_effect is not None:
await self.run_pipeline_side_effect()
return self.pipeline_result
if isinstance(command, EndServiceOnce):
return self.end_reply
if isinstance(command, ExportSession):
return None
if isinstance(command, NotifyBridgeDone):
return None
if isinstance(command, NotifyBridgeStop):
return None
if isinstance(command, SetPendingInterrupt):
return None
if command.__class__.__name__ == "StartSpeech":
if self.speech_handles:
return self.speech_handles.pop(0)
return self.speech_handle
if command.__class__.__name__ == "WaitForSpeechPlayout":
if self.wait_for_playout_callbacks:
callback = self.wait_for_playout_callbacks.pop(0)
if callback is not None:
result = callback(command)
if asyncio.iscoroutine(result):
await result
if self.wait_for_playout_exceptions:
exc = self.wait_for_playout_exceptions.pop(0)
if exc is not None:
raise exc
delay_s = (
self.wait_for_playout_delays_s.pop(0)
if self.wait_for_playout_delays_s
else self.wait_for_playout_delay_s
)
if delay_s > 0:
await asyncio.sleep(delay_s)
return None
if isinstance(command, InterruptSpeech):
setattr(command.handle, "interrupted", True)
return None
if command.__class__.__name__ == "SetPipelineInterruption":
return None
if command.__class__.__name__ == "InjectIdleNudge":
return None
if command.__class__.__name__ == "StartSession":
if self.on_start_session is not None:
self.on_start_session(command)
return None
raise AssertionError(f"Unsupported command in fake executor: {command!r}")
def execute_now(self, command):
self.commands.append(command)
if isinstance(command, ExtractSpokenText):
return self.spoken_text
raise AssertionError(f"Unsupported synchronous command: {command!r}")
class _FakeVADThresholdController:
def __init__(self) -> None:
self.calls = []
self.active = False
def activate_agent_wait_timeout_retry(self, *, attempt: int, reason: str) -> bool:
if self.active:
return False
self.calls.append(("activate", attempt, reason))
self.active = True
return True
def restore(self, *, reason: str) -> bool:
if not self.active:
return False
self.calls.append(("restore", reason))
self.active = False
return True
class _FakeVAD:
def __init__(self) -> None:
self.update_options_calls = []
def update_options(self, **kwargs) -> None:
self.update_options_calls.append(kwargs)
class _AwaitableSpeechHandle:
def __init__(self) -> None:
self.awaited = False
async def wait_for_playout(self) -> None:
return None
def interrupt(self, *, force: bool = False) -> None:
return None
def __await__(self):
async def _wait():
self.awaited = True
return self
return _wait().__await__()
class SpeechServiceTests(_AsyncTestCase):
async def test_start_returns_livekit_speech_handle_without_waiting_for_playout(self) -> None:
handle = _AwaitableSpeechHandle()
session = SimpleNamespace(
say=lambda *args, **kwargs: handle,
)
service = SpeechService(session)
result = await service.start(
"texto",
allow_interruptions=True,
add_to_chat_ctx=True,
)
self.assertIs(result, handle)
self.assertFalse(handle.awaited)
async def test_start_passes_provided_audio_to_livekit_session(self) -> None:
handle = _AwaitableSpeechHandle()
calls = []
audio = object()
session = SimpleNamespace(
say=lambda *args, **kwargs: calls.append((args, kwargs)) or handle,
)
service = SpeechService(session)
result = await service.start(
"texto",
allow_interruptions=True,
add_to_chat_ctx=False,
audio=audio,
)
self.assertIs(result, handle)
self.assertIs(calls[0][1]["audio"], audio)
self.assertFalse(calls[0][1]["add_to_chat_ctx"])
class DynamicVADThresholdControllerTests(unittest.TestCase):
def test_activate_and_restore_updates_vad_options(self) -> None:
vad = _FakeVAD()
controller = DynamicVADThresholdController(
vad,
config=DynamicVADThresholdConfig(
enabled=True,
baseline_activation_threshold=0.30,
baseline_deactivation_threshold=0.15,
retry_activation_threshold=0.20,
retry_deactivation_threshold=0.05,
),
)
self.assertEqual(
controller.current_thresholds(),
{
"mode": "baseline",
"activation_threshold": 0.30,
"deactivation_threshold": 0.15,
},
)
self.assertTrue(
controller.activate_agent_wait_timeout_retry(
attempt=1,
reason="agent_wait_timeout_retry",
)
)
self.assertEqual(
controller.current_thresholds(),
{
"mode": "agent_wait_timeout_retry",
"activation_threshold": 0.20,
"deactivation_threshold": 0.05,
},
)
self.assertTrue(controller.restore(reason="user_final"))
self.assertEqual(
controller.current_thresholds(),
{
"mode": "baseline",
"activation_threshold": 0.30,
"deactivation_threshold": 0.15,
},
)
self.assertEqual(
vad.update_options_calls,
[
{
"activation_threshold": 0.20,
"deactivation_threshold": 0.05,
},
{
"activation_threshold": 0.30,
"deactivation_threshold": 0.15,
},
],
)
class InterruptPolicyTests(unittest.TestCase):
def test_allow_stage_and_backchannel(self) -> None:
policy = InterruptPolicy()
self.assertTrue(policy.allow_stage("ARGUMENTATION"))
self.assertFalse(policy.allow_stage("FORMALIZATION"))
self.assertFalse(policy.allow_stage("IDLE_NUDGE"))
self.assertTrue(policy.allow_stage("AGENT_WAIT_TIMEOUT_RETRY"))
self.assertTrue(policy.allow_stage("unknown-stage"))
self.assertTrue(policy.should_ignore_backchannel(speaking=True, text="uhum"))
self.assertFalse(policy.should_ignore_backchannel(speaking=False, text="uhum"))
class IdlePolicyTests(unittest.TestCase):
def test_idle_policy_decisions(self) -> None:
policy = IdlePolicy(
IdlePolicyConfig(
enabled=True,
delay_s=10.0,
join_delay_s=15.0,
close_delay_s=20.0,
max_tries=3,
end_reason="no_user_response",
)
)
self.assertTrue(policy.should_arm_nudge(delay_s=10.0, nudge_text="alo"))
self.assertFalse(policy.should_arm_nudge(delay_s=0.0, nudge_text="alo"))
self.assertTrue(
policy.should_fire_nudge(
token_is_current=True,
seq_matches=True,
speaking=False,
gap_active=False,
finalized=False,
current_stage="PRESENTATION",
)
)
self.assertFalse(
policy.should_fire_nudge(
token_is_current=True,
seq_matches=True,
speaking=False,
gap_active=False,
finalized=False,
current_stage="DONE",
)
)
self.assertTrue(policy.should_arm_close_after_nudge(stage="IDLE_NUDGE", nudge_count=3))
def test_idle_policy_is_disabled_by_default(self) -> None:
policy = IdlePolicy(
IdlePolicyConfig(
enabled=False,
delay_s=10.0,
join_delay_s=15.0,
close_delay_s=20.0,
max_tries=3,
end_reason="no_user_response",
)
)
self.assertFalse(policy.should_arm_nudge(delay_s=10.0, nudge_text="alo"))
self.assertFalse(policy.should_arm_close_before_fire(nudge_count=3))
self.assertFalse(policy.should_arm_close_after_nudge(stage="IDLE_NUDGE", nudge_count=3))
class FinalizationPolicyTests(unittest.TestCase):
def test_finalization_decisions(self) -> None:
policy = FinalizationPolicy(call_end_grace_s=2.0)
self.assertTrue(policy.should_skip(finalized=True))
self.assertFalse(policy.should_skip(finalized=False))
self.assertTrue(policy.should_finalize_room_empty(remote_participants=0, finalized=False))
self.assertFalse(policy.should_finalize_room_empty(remote_participants=1, finalized=False))
self.assertTrue(policy.is_done_stage("DONE"))
self.assertFalse(policy.is_done_stage("PRESENTATION"))
class TimerSchedulerTests(_AsyncTestCase):
async def test_arm_updates_token_and_cancel_stops_task(self) -> None:
created_tasks = []
logger = _NullLogger()
def _create_task_logged(coro, *, name: str):
task = asyncio.create_task(coro, name=name)
created_tasks.append(task)
return task
scheduler = TimerScheduler(create_task_logged=_create_task_logged, logger=logger)
gate = asyncio.Event()
async def _wait_forever():
await gate.wait()
token1 = scheduler.arm("idle", task_name="idle_1", coro=_wait_forever())
token2 = scheduler.arm("idle", task_name="idle_2", coro=_wait_forever())
self.assertFalse(scheduler.is_current("idle", token1))
self.assertTrue(scheduler.is_current("idle", token2))
scheduler.cancel("idle", reason="test", log_name="IDLE_TIMER_CANCEL")
await asyncio.sleep(0)
self.assertTrue(created_tasks[-1].cancelled())
gate.set()
await asyncio.gather(*created_tasks, return_exceptions=True)
class RuntimeCommandExecutorTests(_AsyncTestCase):
async def test_routes_commands_to_underlying_services(self) -> None:
agent = _FakeAgent()
bridge_gateway = SimpleNamespace(notify_stage_done=self._make_async_noop())
bridge_gateway.notify_stop = self._make_async_noop()
export_service = SimpleNamespace(export_session=self._make_async_noop())
session = _FakeSession()
speech_service = SimpleNamespace(
start=self._make_async_return("speech-handle"),
wait_for_playout=self._make_async_noop(),
extract_spoken_text=lambda handle: "spoken",
)
agent.pipeline = SimpleNamespace(
inject_idle_nudge=self._make_async_noop(),
set_interruption=self._make_async_noop(),
run=self._make_async_return(BackendReply(stage="DONE", text="encerrando", done=True)),
)
executor = RuntimeCommandExecutor(
agent=agent,
bridge_gateway=bridge_gateway,
export_service=export_service,
session=session,
speech_service=speech_service,
)
result = await executor.execute(RunPipelineInput("payload"))
self.assertEqual(result, BackendReply(stage="DONE", text="encerrando", done=True))
self.assertEqual(executor.execute_now(ExtractSpokenText(object())), "spoken")
@staticmethod
def _make_async_noop():
async def _noop(*args, **kwargs):
return None
return _noop
@staticmethod
def _make_async_return(value):
async def _return(*args, **kwargs):
return value
return _return
class CallRuntimeTests(_AsyncTestCase):
def setUp(self) -> None:
self.created_tasks = []
def _create_task_logged(self, coro, *, name: str):
task = asyncio.create_task(coro, name=name)
self.created_tasks.append(task)
return task
async def _drain_tasks(self) -> None:
while True:
pending = [task for task in self.created_tasks if not task.done()]
if not pending:
return
await asyncio.gather(*pending)
async def _cancel_pending_tasks(self) -> None:
pending = [task for task in self.created_tasks if not task.done()]
for task in pending:
task.cancel()
if pending:
await asyncio.gather(*pending, return_exceptions=True)
def test_inflight_backend_wait_message_id_uses_uuid_when_original_is_missing(self) -> None:
generated = uuid.UUID("12345678-1234-4234-9234-123456789abc")
with mock.patch("app.livekit.runtime.call_runtime.uuid.uuid4", return_value=generated):
message_id = CallRuntime._inflight_backend_wait_message_id(attempt=1)
self.assertEqual(
message_id,
"12345678-1234-4234-9234-123456789abc_conforto_12345678123442349234123456789abc",
)
def test_feedback_message_id_keeps_the_agent_speech_id(self) -> None:
message_id = CallRuntime._feedback_message_id(
message_id="turno-1",
speech_id="speech-feedback-1",
)
self.assertEqual(message_id, "turno-1_feedback_speech-feedback-1")
self.assertEqual(CallRuntime._base_message_id(message_id), "turno-1")
def test_auxiliary_message_ids_are_distinct_and_keep_the_base(self) -> None:
generated = uuid.UUID("12345678-1234-4234-9234-123456789abc")
with mock.patch("app.livekit.runtime.call_runtime.uuid.uuid4", return_value=generated):
interruption = CallRuntime._interruption_comfort_message_id(message_id="turno-1")
tts_error = CallRuntime._tts_error_message_id(message_id="turno-1")
self.assertEqual(CallRuntime._idle_nudge_message_id(message_id="turno-1", attempt=2), "turno-1_idle_nudge_2")
self.assertEqual(interruption, "turno-1_interruption_confort_12345678123442349234123456789abc")
self.assertEqual(tts_error, "turno-1_tts_error_12345678123442349234123456789abc")
self.assertEqual(CallRuntime._base_message_id(interruption), "turno-1")
self.assertEqual(CallRuntime._base_message_id(tts_error), "turno-1")
self.assertTrue(
CallRuntime._transfer_message_id(message_id="turno-1").startswith(
"turno-1_transferencia_"
)
)
self.assertTrue(
CallRuntime._resource_error_message_id(
message_id="turno-1",
terminal=True,
tipo_evento="envio msg",
).startswith("turno-1_erro_terminal_envio_")
)
async def test_backend_feedback_uses_a_suffixed_message_id(self) -> None:
runtime, _agent, _executor = self._make_runtime()
runtime._ctx.room.remote_participants = {"user-1": object()}
with mock.patch.object(runtime, "say_stage", new_callable=mock.AsyncMock) as say_stage:
await runtime._speak_backend_reply(
BackendReply(
stage="PRESENTATION",
text="Ainda estou consultando.",
metadata={
"message_id": "turno-feedback",
"speech_id": "speech-feedback-1",
"agent_message_type": "feedback",
},
),
source="backend_push",
add_to_chat_ctx=True,
)
self.assertEqual(
say_stage.await_args.kwargs["message_id"],
"turno-feedback_feedback_speech-feedback-1",
)
self.assertEqual(say_stage.await_args.kwargs["speech_id"], "speech-feedback-1")
def _make_runtime(
self,
*,
agent_starts_conversation: bool = False,
push_replies=None,
runtime_config_overrides=None,
idle_policy_overrides=None,
vad_threshold_controller=None,
):
agent = _FakeAgent()
if push_replies is not None:
agent.pipeline = _FakePushPipeline(push_replies)
session = _FakeSession()
ctx = _FakeContext()
executor = _FakeCommandExecutor()
config_kwargs = {
"call_end_grace_s": 0.0,
"final_grace_s": 0.0,
"idle_nudge_delay_s": 0.0,
"idle_nudge_join_delay_s": 0.0,
"idle_nudge_close_delay_s": 0.0,
"idle_nudge_max_tries": 0,
"idle_nudge_end_reason": "no_user_response",
}
if runtime_config_overrides:
config_kwargs.update(runtime_config_overrides)
idle_policy_kwargs = {
"enabled": False,
"delay_s": 0.0,
"join_delay_s": 0.0,
"close_delay_s": 0.0,
"max_tries": 0,
"end_reason": "no_user_response",
}
if idle_policy_overrides:
idle_policy_kwargs.update(idle_policy_overrides)
runtime = CallRuntime(
ctx=ctx,
session=session,
agent=agent,
command_executor=executor,
call_logger=_NullLogger(),
protocol="PRT-1",
session_id="S-1",
bridge_identity="bridge-1",
agent_starts_conversation=agent_starts_conversation,
nudge_text="alo",
call_t0=0.0,
extract_text_from_transcript=lambda transcript: transcript,
interrupt_policy=InterruptPolicy(),
idle_policy=IdlePolicy(IdlePolicyConfig(**idle_policy_kwargs)),
finalization_policy=FinalizationPolicy(call_end_grace_s=0.0),
create_task_logged=self._create_task_logged,
config=RuntimeConfig(**config_kwargs),
logger=_NullLogger(),
vad_threshold_controller=vad_threshold_controller,
)
return runtime, agent, executor
def test_inflight_backend_wait_uses_short_audio_then_long_audio(self) -> None:
with TemporaryDirectory() as tmp_dir:
base_dir = Path(tmp_dir)
short_dir = base_dir / "short"
long_dir = base_dir / "long"
short_dir.mkdir()
long_dir.mkdir()
short_audio = short_dir / "curto.wav"
long_audio = long_dir / "longo.wav"
short_audio.write_bytes(b"short")
long_audio.write_bytes(b"long")
runtime, _agent, _executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_short_audio_dir": str(short_dir),
"inflight_backend_wait_long_audio_dir": str(long_dir),
}
)
self.assertEqual(
runtime._inflight_backend_wait_audio_path(attempt=1),
short_audio,
)
self.assertEqual(
runtime._inflight_backend_wait_audio_path(attempt=2),
long_audio,
)
def test_inflight_backend_wait_prewarms_audio_duration_cache(self) -> None:
with TemporaryDirectory() as tmp_dir:
short_dir = Path(tmp_dir) / "short"
short_dir.mkdir()
short_audio = short_dir / "curto.wav"
short_audio.write_bytes(b"fake-wav")
with mock.patch(
"app.livekit.runtime.call_runtime.wav_duration_ms",
return_value=321,
) as duration_ms:
runtime, _agent, _executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_short_audio_dir": str(short_dir),
}
)
duration_ms.assert_called_once_with(str(short_audio))
with mock.patch(
"app.livekit.runtime.call_runtime.wav_duration_ms",
side_effect=AssertionError("duration should come from cache"),
):
self.assertEqual(
runtime._inflight_backend_wait_audio_duration_ms(short_audio),
321,
)
def test_inflight_backend_wait_keeps_legacy_long_audio_path_fallback(self) -> None:
runtime, _agent, _executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
self.assertEqual(
runtime._inflight_backend_wait_audio_path(),
Path(WAIT_LONG_AUDIO_PATH),
)
async def test_idle_nudge_does_not_activate_retry_vad_threshold(self) -> None:
controller = _FakeVADThresholdController()
runtime, _agent, executor = self._make_runtime(
idle_policy_overrides={
"enabled": True,
"delay_s": 10.0,
"join_delay_s": 10.0,
"close_delay_s": 10.0,
"max_tries": 2,
"end_reason": "no_user_response",
},
vad_threshold_controller=controller,
)
try:
runtime.arm_idle_timer(reason="test", delay_s=0.001)
for _ in range(50):
if any(cmd.__class__.__name__ == "StartSpeech" for cmd in executor.commands):
break
await asyncio.sleep(0.001)
self.assertNotIn(("activate", 1, "idle_nudge_fire"), controller.calls)
start_speech = next(
cmd for cmd in executor.commands if cmd.__class__.__name__ == "StartSpeech"
)
self.assertEqual(start_speech.text, "alo")
self.assertFalse(start_speech.allow_interruptions)
runtime.on_user_input_transcribed(SimpleNamespace(transcript="sim", is_final=True))
self.assertNotIn(("restore", "user_final"), controller.calls)
finally:
await self._cancel_pending_tasks()
async def test_idle_nudge_uses_a_suffixed_message_id(self) -> None:
runtime, _agent, _executor = self._make_runtime(
idle_policy_overrides={
"enabled": True,
"delay_s": 10.0,
"join_delay_s": 10.0,
"close_delay_s": 10.0,
"max_tries": 2,
"end_reason": "no_user_response",
}
)
runtime._last_agent_message_id = "turno-idle"
try:
with mock.patch.object(runtime, "say_stage", new_callable=mock.AsyncMock) as say_stage:
runtime.arm_idle_timer(reason="test", delay_s=0.001)
for _ in range(50):
if say_stage.await_count:
break
await asyncio.sleep(0.001)
say_stage.assert_awaited_once()
self.assertEqual(
say_stage.await_args.kwargs["message_id"],
"turno-idle_idle_nudge_1",
)
finally:
await self._cancel_pending_tasks()
async def test_on_user_input_transcribed_stashes_when_stage_is_not_interruptible(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.state.current_stage = "FORMALIZATION"
runtime.speaking.set()
event = SimpleNamespace(transcript="quero continuar", is_final=True)
with mock.patch("app.livekit.runtime.call_runtime.log_flow_event") as flow_log:
runtime.on_user_input_transcribed(event)
await self._drain_tasks()
self.assertIsNotNone(runtime.state.pending_user_final)
self.assertEqual(runtime.state.pending_user_final.text, "quero continuar")
self.assertFalse(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
self.assertTrue(
any(
call.args[1] == "interrupt_ignored"
and call.kwargs["reason"] == "stage_not_interruptible"
for call in flow_log.call_args_list
)
)
async def test_on_user_input_transcribed_runs_when_argumentation_is_speaking(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.state.current_stage = "ARGUMENTATION"
runtime.state.current_speech.allow_interruptions = True
runtime.speaking.set()
event = SimpleNamespace(transcript="quero continuar", is_final=True)
runtime.on_user_input_transcribed(event)
await self._drain_tasks()
self.assertIsNone(runtime.state.pending_user_final)
self.assertTrue(any(isinstance(cmd, SetPendingInterrupt) for cmd in executor.commands))
self.assertTrue(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
async def test_short_vad_utterance_with_text_is_discarded_while_r1_is_processing(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.state.user_final_seq = 1
runtime.state.deferred_interruption.backend_in_flight = True
register_transcribed_turn(
runtime._structured_log_context,
message_id="short-utterance",
transcription="sim",
text="sim",
)
runtime.note_vad_speech_end(999)
runtime.on_user_input_transcribed(SimpleNamespace(transcript="sim", is_final=True))
await self._drain_tasks()
self.assertEqual(runtime.state.user_final_seq, 1)
self.assertEqual(runtime.state.deferred_interruption.long_turns, [])
self.assertFalse(runtime.state.deferred_interruption.special_comfort_sent)
self.assertFalse(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
async def test_first_long_vad_plays_special_before_empty_stt(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.state.user_final_seq = 1
runtime.state.deferred_interruption.backend_in_flight = True
with mock.patch.object(runtime, "_play_deferred_interruption_comfort") as comfort:
runtime.note_vad_speech_end(1000)
comfort.assert_called_once_with()
runtime.on_user_input_transcribed(SimpleNamespace(transcript="", is_final=True))
await self._drain_tasks()
self.assertEqual(runtime.state.user_final_seq, 1)
self.assertEqual(runtime.state.deferred_interruption.long_turns, [])
self.assertTrue(runtime.state.deferred_interruption.special_comfort_sent)
self.assertFalse(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
async def test_special_processing_comfort_starts_protected(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.state.deferred_interruption.special_comfort_sent = True
runtime._play_deferred_interruption_comfort()
await self._drain_tasks()
speech = next(cmd for cmd in executor.commands if isinstance(cmd, StartSpeech))
self.assertEqual(speech.text, "Um instante")
self.assertFalse(speech.allow_interruptions)
async def test_short_backend_wait_comfort_starts_protected(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._ctx.room.remote_participants = {"user-1": object()}
with mock.patch.object(
runtime,
"_select_inflight_backend_wait_audio",
return_value=(Path(WAIT_LONG_AUDIO_PATH), "short"),
):
played = await runtime._play_inflight_backend_wait_notice_audio(
user_seq=None,
attempt=1,
)
self.assertTrue(played)
speech = next(cmd for cmd in executor.commands if isinstance(cmd, StartSpeech))
self.assertFalse(speech.allow_interruptions)
async def test_short_comfort_is_not_rearmed_by_its_originating_user_final(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.speaking.set()
runtime.state.current_speech.stage = "AGENT_BACKEND_WAIT"
runtime.state.current_speech.handle = executor.speech_handle
runtime.state.current_speech.allow_interruptions = False
runtime.state.current_speech.rearm_interruptions_on_next_user_speech = True
executor.speech_handle.allow_interruptions = False
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="fala original", is_final=True)
)
await self._drain_tasks()
self.assertFalse(runtime.state.current_speech.allow_interruptions)
self.assertFalse(executor.speech_handle.allow_interruptions)
self.assertTrue(
runtime.state.current_speech.rearm_interruptions_on_next_user_speech
)
async def test_short_comfort_is_rearmed_on_new_user_speech(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.speaking.set()
runtime.state.current_speech.stage = "INTERRUPTION_COMFORT"
runtime.state.current_speech.handle = executor.speech_handle
runtime.state.current_speech.allow_interruptions = False
runtime.state.current_speech.rearm_interruptions_on_next_user_speech = True
executor.speech_handle.allow_interruptions = False
runtime._rearm_current_speech_interruptions_on_new_user_speech()
self.assertTrue(runtime.state.current_speech.allow_interruptions)
self.assertFalse(
runtime.state.current_speech.rearm_interruptions_on_next_user_speech
)
self.assertTrue(executor.speech_handle.allow_interruptions)
async def test_feedback_compat_mode_drops_processing_speech_without_replay(self) -> None:
runtime, _agent, executor = self._make_runtime(
runtime_config_overrides={"deferred_interruption_enabled": False}
)
runtime.state.user_final_seq = 1
runtime.state.deferred_interruption.backend_in_flight = True
register_transcribed_turn(
runtime._structured_log_context,
message_id="processing-drop",
transcription="quero interromper",
text="quero interromper",
)
with (
mock.patch.object(runtime, "_play_deferred_interruption_comfort") as special,
mock.patch.object(runtime, "_play_deferred_short_comfort") as short,
mock.patch("app.livekit.runtime.call_runtime.log_flow_event") as flow_log,
):
runtime.note_vad_speech_end(2500)
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="quero interromper", is_final=True)
)
await self._drain_tasks()
special.assert_not_called()
short.assert_not_called()
self.assertEqual(runtime.state.user_final_seq, 1)
self.assertEqual(runtime.state.deferred_interruption.long_turns, [])
self.assertEqual(len(runtime.state.deferred_interruption.pending_vad_utterances), 0)
self.assertFalse(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
self.assertTrue(
any(
call.args[1] == "backend_processing_vad_discarded"
and call.kwargs["mode"] == "feedback_compat"
for call in flow_log.call_args_list
)
)
self.assertTrue(
any(
call.args[1] == "user_input_dropped"
and call.kwargs["reason"] == "backend_processing_feedback_compat"
and call.kwargs["agent_message_type"] == "feedback"
for call in flow_log.call_args_list
)
)
async def test_vad_end_outside_backend_window_is_not_deferred(self) -> None:
runtime, _agent, _executor = self._make_runtime()
runtime.state.deferred_interruption.backend_in_flight = False
runtime.speaking.set()
with mock.patch.object(runtime, "_play_deferred_interruption_comfort") as comfort:
runtime.note_vad_speech_end(4000)
comfort.assert_not_called()
self.assertEqual(len(runtime.state.deferred_interruption.pending_vad_utterances), 0)
self.assertFalse(runtime.state.deferred_interruption.special_comfort_sent)
async def test_barge_in_during_agent_playout_still_runs_a_new_pipeline(self) -> None:
runtime, agent, executor = self._make_runtime()
runtime.state.current_stage = "ARGUMENTATION"
runtime.state.current_speech.allow_interruptions = True
runtime.speaking.set()
# O playout segura o run_lock, mas a janela do ciclo diferido ja fechou:
# a fala precisa voltar a ser barge-in comum.
await agent._run_lock.acquire()
try:
runtime.note_vad_speech_end(4000)
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="muda de assunto", is_final=True)
)
await asyncio.sleep(0)
finally:
agent._run_lock.release()
await self._drain_tasks()
self.assertEqual(runtime.state.deferred_interruption.long_turns, [])
self.assertTrue(any(isinstance(cmd, SetPendingInterrupt) for cmd in executor.commands))
self.assertTrue(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
async def test_long_vad_transcripts_are_concatenated_once_and_r2_blocks_replay(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.state.user_final_seq = 1
runtime.state.deferred_interruption.backend_in_flight = True
with (
mock.patch.object(runtime, "_play_deferred_interruption_comfort") as comfort,
mock.patch.object(runtime, "_play_deferred_short_comfort") as short_comfort,
mock.patch.object(runtime, "run_pipeline", new_callable=mock.AsyncMock) as run_pipeline,
):
runtime.note_vad_speech_end(1200)
runtime.note_vad_speech_end(300)
runtime.note_vad_speech_end(1400)
register_transcribed_turn(
runtime._structured_log_context,
message_id="long-1",
transcription="primeira fala longa",
text="primeira fala longa",
)
register_transcribed_turn(
runtime._structured_log_context,
message_id="short-1",
transcription="fala curta",
text="fala curta",
)
register_transcribed_turn(
runtime._structured_log_context,
message_id="long-2",
transcription="segunda fala longa",
text="segunda fala longa",
)
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="primeira fala longa", is_final=True)
)
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="fala curta", is_final=True)
)
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="segunda fala longa", is_final=True)
)
comfort.assert_called_once_with()
short_comfort.assert_called_once_with(source="repeated_interruption")
self.assertEqual(
[turn.text for turn in runtime.state.deferred_interruption.long_turns],
["primeira fala longa", "segunda fala longa"],
)
dispatched = await runtime._dispatch_deferred_interruption(
BackendReply(stage="PRESENTATION", text="resposta R1")
)
await self._drain_tasks()
self.assertTrue(dispatched)
self.assertTrue(runtime.state.deferred_interruption.replay_in_flight)
run_pipeline.assert_awaited_once_with(
"primeira fala longa. segunda fala longa",
"primeira fala longa. segunda fala longa",
is_deferred_replay=True,
user_seq=3,
message_id="long-2",
inflight_initial_notice_consumed=True,
)
runtime.note_vad_speech_end(1500)
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="nao deve gerar R3", is_final=True)
)
self.assertEqual(runtime.state.user_final_seq, 3)
self.assertEqual(runtime.state.deferred_interruption.long_turns, [])
self.assertFalse(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
async def test_vad_ended_before_r1_is_promoted_when_stt_finishes_during_r1(self) -> None:
runtime, _agent, executor = self._make_runtime()
backend_started = asyncio.Event()
release_backend = asyncio.Event()
runtime.state.user_final_seq = 0
async def _wait_for_backend() -> None:
backend_started.set()
await release_backend.wait()
executor.run_pipeline_side_effect = _wait_for_backend
with (
mock.patch.object(runtime, "_play_deferred_interruption_comfort"),
mock.patch.object(
runtime,
"_dispatch_deferred_interruption",
new_callable=mock.AsyncMock,
return_value=True,
) as dispatch,
):
# O segundo fim de fala acontece antes de R1, mas seu STT chega
# somente enquanto R1 está em voo.
runtime.note_vad_speech_end(1500)
runtime.note_vad_speech_end(1500)
register_transcribed_turn(
runtime._structured_log_context,
message_id="r1-turn",
transcription="primeiro turno",
text="primeiro turno",
)
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="primeiro turno", is_final=True)
)
await backend_started.wait()
register_transcribed_turn(
runtime._structured_log_context,
message_id="late-turn",
transcription="fala que terminou antes de R1",
text="fala que terminou antes de R1",
)
runtime.on_user_input_transcribed(
SimpleNamespace(
transcript="fala que terminou antes de R1", is_final=True
)
)
release_backend.set()
# O task criado pelo callback pode precisar de um ciclo extra para
# concluir a espera e tentar o dispatch.
await self._drain_tasks()
dispatch.assert_awaited_once()
self.assertEqual(len(runtime.state.deferred_interruption.pre_backend_vad_utterances), 0)
async def test_r1_waits_for_pending_long_stt_before_dispatching(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.state.user_final_seq = 1
backend_started = asyncio.Event()
release_backend = asyncio.Event()
async def _wait_for_backend() -> None:
backend_started.set()
await release_backend.wait()
executor.run_pipeline_side_effect = _wait_for_backend
with (
mock.patch.object(runtime, "_play_deferred_interruption_comfort"),
mock.patch.object(runtime, "_dispatch_deferred_interruption", new_callable=mock.AsyncMock, return_value=True) as dispatch,
):
task = asyncio.create_task(runtime.run_pipeline("original", "original", user_seq=1))
await backend_started.wait()
runtime.note_vad_speech_end(1200)
release_backend.set()
await asyncio.sleep(0)
self.assertFalse(task.done())
self.assertFalse(any(isinstance(cmd, StartSpeech) for cmd in executor.commands))
register_transcribed_turn(
runtime._structured_log_context,
message_id="long-wait",
transcription="fala longa",
text="fala longa",
)
runtime.on_user_input_transcribed(SimpleNamespace(transcript="fala longa", is_final=True))
await task
dispatch.assert_awaited_once()
async def test_r2_discards_input_until_its_playout_finishes(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 1
runtime.state.deferred_interruption.replay_in_flight = True
executor.pipeline_result = BackendReply(stage="PRESENTATION", text="resposta R2")
played = []
def _during_r2_playout(_command) -> None:
played.append(True)
self.assertTrue(runtime.state.deferred_interruption.replay_in_flight)
runtime.note_vad_speech_end(1500)
runtime.on_user_input_transcribed(SimpleNamespace(transcript="fala ignorada", is_final=True))
executor.wait_for_playout_callbacks.append(_during_r2_playout)
with mock.patch.object(runtime, "_play_deferred_short_comfort") as short_comfort:
await runtime.run_pipeline("texto R2", "texto R2", user_seq=1, is_deferred_replay=True)
self.assertTrue(played, "o playout de R2 precisa acontecer de verdade")
short_comfort.assert_not_called()
self.assertFalse(runtime.state.deferred_interruption.replay_in_flight)
self.assertEqual(runtime.state.user_final_seq, 1)
self.assertEqual(runtime.state.deferred_interruption.long_turns, [])
async def test_r2_reply_is_not_interruptible(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.current_stage = "ARGUMENTATION"
runtime.state.user_final_seq = 1
runtime.state.deferred_interruption.replay_in_flight = True
executor.pipeline_result = BackendReply(stage="ARGUMENTATION", text="resposta R2")
await runtime.run_pipeline("texto R2", "texto R2", user_seq=1, is_deferred_replay=True)
speech_commands = [cmd for cmd in executor.commands if isinstance(cmd, StartSpeech)]
self.assertEqual([cmd.text for cmd in speech_commands], ["resposta R2"])
# Descartar a fala no runtime nao basta: com allow_interruptions=True o
# barge-in do proprio LiveKit corta o playout e R2 sai pela metade.
self.assertFalse(speech_commands[0].allow_interruptions)
async def test_r2_logs_and_ignores_short_voice_without_playing_comfort(self) -> None:
runtime, _agent, _executor = self._make_runtime()
runtime.state.deferred_interruption.replay_in_flight = True
with (
mock.patch.object(runtime, "_play_deferred_short_comfort") as short_comfort,
mock.patch("app.livekit.runtime.call_runtime.log_flow_event") as flow_log,
):
runtime.note_vad_speech_end(999)
short_comfort.assert_not_called()
self.assertTrue(
any(
call.args[1] == "deferred_replay_vad_discarded"
and call.kwargs["speech_duration_ms"] == 999
and not call.kwargs["short_comfort_scheduled"]
for call in flow_log.call_args_list
)
)
async def test_normal_reply_stays_interruptible_outside_the_replay(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.current_stage = "ARGUMENTATION"
runtime.state.user_final_seq = 1
executor.pipeline_result = BackendReply(stage="ARGUMENTATION", text="resposta normal")
await runtime.run_pipeline("texto", "texto", user_seq=1)
speech_commands = [cmd for cmd in executor.commands if isinstance(cmd, StartSpeech)]
self.assertEqual([cmd.text for cmd in speech_commands], ["resposta normal"])
self.assertTrue(speech_commands[0].allow_interruptions)
async def test_r1_waits_for_the_user_turn_to_end_before_deciding(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.state.user_final_seq = 1
backend_started = asyncio.Event()
release_backend = asyncio.Event()
async def _wait_for_backend() -> None:
backend_started.set()
await release_backend.wait()
executor.run_pipeline_side_effect = _wait_for_backend
with (
mock.patch.object(runtime, "_play_deferred_interruption_comfort"),
mock.patch.object(
runtime,
"_dispatch_deferred_interruption",
new_callable=mock.AsyncMock,
return_value=True,
) as dispatch,
):
task = asyncio.create_task(runtime.run_pipeline("original", "original", user_seq=1))
await backend_started.wait()
# R1 volta com o cliente no meio da fala: nem o VAD fechou, nem ha
# STT pendente ainda.
runtime._user_not_speaking.clear()
release_backend.set()
await asyncio.sleep(0)
self.assertFalse(task.done())
self.assertFalse(any(isinstance(cmd, StartSpeech) for cmd in executor.commands))
runtime.note_vad_speech_end(1300)
runtime._user_not_speaking.set()
register_transcribed_turn(
runtime._structured_log_context,
message_id="late-long",
transcription="fala que comecou antes de R1 voltar",
text="fala que comecou antes de R1 voltar",
)
runtime.on_user_input_transcribed(
SimpleNamespace(
transcript="fala que comecou antes de R1 voltar", is_final=True
)
)
await task
dispatch.assert_awaited_once()
self.assertFalse(any(isinstance(cmd, StartSpeech) for cmd in executor.commands))
async def test_r1_gives_up_waiting_when_the_stt_final_never_arrives(self) -> None:
runtime, _agent, executor = self._make_runtime(
runtime_config_overrides={
"deferred_interruption_stt_settle_timeout_s": 0.01,
}
)
runtime.state.user_final_seq = 1
backend_started = asyncio.Event()
release_backend = asyncio.Event()
async def _wait_for_backend() -> None:
backend_started.set()
await release_backend.wait()
executor.run_pipeline_side_effect = _wait_for_backend
executor.pipeline_result = BackendReply(stage="PRESENTATION", text="resposta R1")
with (
mock.patch.object(runtime, "_play_deferred_interruption_comfort"),
mock.patch.object(
runtime, "_speak_backend_reply", new_callable=mock.AsyncMock
) as speak,
):
task = asyncio.create_task(runtime.run_pipeline("original", "original", user_seq=1))
await backend_started.wait()
runtime.note_vad_speech_end(1200)
release_backend.set()
# Sem teto na espera esta chamada nunca retornaria.
await asyncio.wait_for(task, timeout=5)
deferred = runtime.state.deferred_interruption
self.assertEqual(deferred.pending_long_stt_finals, 0)
self.assertEqual(len(deferred.pending_vad_utterances), 0)
self.assertFalse(deferred.backend_in_flight)
# A resposta de R1 e liberada em vez de ficar presa a um final que nao vem.
speak.assert_awaited_once()
async def test_special_comfort_flag_is_cleared_when_r1_answers_empty(self) -> None:
runtime, _agent, _executor = self._make_runtime()
runtime.state.user_final_seq = 1
backend_started = asyncio.Event()
release_backend = asyncio.Event()
async def _wait_for_backend() -> None:
backend_started.set()
await release_backend.wait()
_executor.run_pipeline_side_effect = _wait_for_backend
_executor.pipeline_result = BackendReply(stage="PRESENTATION", text="")
with mock.patch.object(runtime, "_play_deferred_interruption_comfort"):
task = asyncio.create_task(runtime.run_pipeline("original", "original", user_seq=1))
await backend_started.wait()
runtime.note_vad_speech_end(1200)
runtime.on_user_input_transcribed(SimpleNamespace(transcript="", is_final=True))
release_backend.set()
await task
# Sem a limpeza aqui o conforto especial nunca mais tocaria na ligacao.
self.assertFalse(runtime.state.deferred_interruption.special_comfort_sent)
async def test_empty_final_is_counted_once_when_provider_and_livekit_both_report(self) -> None:
runtime, _agent, _executor = self._make_runtime()
# Em raw_json o transcript cru nao e vazio, so o texto extraido dele e.
runtime._extract_text_from_transcript = lambda _transcript: ""
deferred = runtime.state.deferred_interruption
deferred.backend_in_flight = True
runtime.note_vad_speech_end(300)
runtime.note_vad_speech_end(1500)
with mock.patch.object(runtime, "_play_deferred_interruption_comfort"):
# Provider avisa o final vazio da fala curta...
runtime.handle_empty_stt_final(source="stt_provider")
# ...e o LiveKit repassa o mesmo resultado como evento.
runtime.on_user_input_transcribed(
SimpleNamespace(transcript='{"data":{"text":""}}', is_final=True)
)
await self._drain_tasks()
# A fala longa continua na fila, aguardando o proprio final.
self.assertEqual(len(deferred.pending_vad_utterances), 1)
self.assertEqual(deferred.pending_long_stt_finals, 1)
async def test_vad_utterance_without_final_does_not_leak_into_the_next_turn(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.state.user_final_seq = 1
backend_started = asyncio.Event()
release_backend = asyncio.Event()
async def _wait_for_backend() -> None:
backend_started.set()
await release_backend.wait()
executor.run_pipeline_side_effect = _wait_for_backend
with (
mock.patch.object(runtime, "_play_deferred_interruption_comfort"),
mock.patch.object(runtime, "_speak_backend_reply", new_callable=mock.AsyncMock),
):
task = asyncio.create_task(runtime.run_pipeline("original", "original", user_seq=1))
await backend_started.wait()
# Ruido curto que o STT nunca finaliza.
runtime.note_vad_speech_end(200)
release_backend.set()
await asyncio.wait_for(task, timeout=5)
deferred = runtime.state.deferred_interruption
self.assertEqual(len(deferred.pending_vad_utterances), 0)
# No turno seguinte a fala do cliente nao pode ser consumida pela sobra.
deferred.backend_in_flight = True
with mock.patch.object(runtime, "_play_deferred_interruption_comfort"):
runtime.note_vad_speech_end(1500)
register_transcribed_turn(
runtime._structured_log_context,
message_id="turno-seguinte",
transcription="agora eu quero falar",
text="agora eu quero falar",
)
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="agora eu quero falar", is_final=True)
)
await self._drain_tasks()
self.assertEqual(
[turn.text for turn in deferred.long_turns], ["agora eu quero falar"]
)
async def test_protected_speech_drop_wins_over_the_deferred_cycle(self) -> None:
runtime, _agent, _executor = self._make_runtime()
runtime.state.user_final_seq = 1
deferred = runtime.state.deferred_interruption
deferred.backend_in_flight = True
runtime._mark_drop_next_user_final(
{"reason": "protected_speech", "stage": "PRESENTATION"}
)
with mock.patch.object(runtime, "_play_deferred_interruption_comfort"):
runtime.note_vad_speech_end(1500)
register_transcribed_turn(
runtime._structured_log_context,
message_id="protegida",
transcription="fala descartada",
text="fala descartada",
)
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="fala descartada", is_final=True)
)
await self._drain_tasks()
self.assertEqual(deferred.long_turns, [])
self.assertEqual(runtime.state.user_final_seq, 1)
# A fala VAD tem de sair da fila junto, senao o proximo final consumiria
# a entrada errada.
self.assertEqual(len(deferred.pending_vad_utterances), 0)
self.assertEqual(deferred.pending_long_stt_finals, 0)
async def test_special_comfort_defers_the_periodic_backend_wait_notice(self) -> None:
runtime, _agent, _executor = self._make_runtime()
runtime.state.deferred_interruption.backend_in_flight = True
runtime._inflight_backend_activity_at = 0.0
with mock.patch.object(runtime, "_play_deferred_interruption_comfort"):
runtime.note_vad_speech_end(1500)
self.assertGreater(runtime._inflight_backend_activity_at, 0.0)
def test_combined_interruption_transcription_concatenates_text_and_drops_word_timestamps(self) -> None:
turns = [
SimpleNamespace(
transcription='{"data":{"text":"primeira", "words":[{"word":"primeira"}]}}',
text="primeira",
),
SimpleNamespace(transcription="segunda", text="segunda"),
]
transcription, text = CallRuntime._combined_interruption_transcription(turns)
payload = __import__("json").loads(transcription)
self.assertEqual(text, "primeira. segunda")
self.assertEqual(payload["data"]["text"], "primeira. segunda")
self.assertNotIn("words", payload["data"])
async def test_on_user_input_transcribed_drops_during_protected_feedback(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._timeline = _FakeTimeline()
runtime.state.current_stage = "PRESENTATION"
runtime.state.current_speech.stage = "PRESENTATION"
runtime.state.current_speech.drop_user_input_while_speaking = True
runtime.state.current_speech.agent_message_type = "feedback"
runtime.speaking.set()
with mock.patch("app.livekit.runtime.call_runtime.log_flow_event") as flow_log:
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="quero falar agora", is_final=True)
)
await self._drain_tasks()
self.assertEqual(runtime.state.user_final_seq, 0)
self.assertIsNone(runtime.state.pending_user_final)
self.assertFalse(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
self.assertTrue(
any(
call.args[1] == "user_input_dropped"
and call.kwargs["agent_message_type"] == "feedback"
and call.kwargs["text"] == "quero falar agora"
for call in flow_log.call_args_list
)
)
self.assertTrue(
any(event == "user_input_dropped" for event, _fields in runtime._timeline.events)
)
async def test_dropped_user_final_does_not_leak_message_id_to_repeated_text(self) -> None:
runtime, _agent, executor = self._make_runtime()
structured_context = StructuredLogContext(
callid="call-1",
session_id="session-drop-repeated-text",
num_telefone="5511999999999",
cod_ani="1234",
nome_agente="conta",
)
runtime._structured_log_context = structured_context
reset_turn_message_sequence(structured_context, clear_pending=True)
register_started_turn_message_id(structured_context, "MSG-drop-0001")
register_transcribed_turn(
structured_context,
message_id="MSG-drop-0001",
transcription="mesmo texto",
text="mesmo texto",
)
runtime._drop_next_user_final_context = {
"reason": "test_drop",
"stage": "PRESENTATION",
"agent_message_type": "ready",
"agent_result_type": "",
}
try:
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="mesmo texto", is_final=True)
)
await self._drain_tasks()
self.assertFalse(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
register_started_turn_message_id(structured_context, "MSG-current-0002")
register_transcribed_turn(
structured_context,
message_id="MSG-current-0002",
transcription="mesmo texto",
text="mesmo texto",
)
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="mesmo texto", is_final=True)
)
await self._drain_tasks()
pipeline_command = next(cmd for cmd in executor.commands if isinstance(cmd, RunPipelineInput))
self.assertEqual(pipeline_command.user_input["message_id"], "MSG-current-0002")
finally:
reset_turn_message_sequence(structured_context, clear_pending=True)
async def test_on_user_input_transcribed_drops_after_feedback_while_backend_is_processing(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._timeline = _FakeTimeline()
runtime._ctx.room.remote_participants = {"user-1": object()}
await runtime._speak_backend_reply(
BackendReply(
stage="PRESENTATION",
text="Ainda estou consultando sua fatura.",
metadata={
"event": "feedback",
"agent_message_type": "feedback",
"expects_user_response": False,
"drop_user_input_while_speaking": True,
"is_interruptible": False,
},
),
source="backend_push",
add_to_chat_ctx=True,
)
self.assertFalse(runtime.speaking.is_set())
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="posso falar?", is_final=True)
)
await self._drain_tasks()
self.assertEqual(runtime.state.user_final_seq, 0)
self.assertFalse(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
self.assertTrue(
any(
event == "user_input_dropped"
and fields["agent_message_type"] == "feedback"
for event, fields in runtime._timeline.events
)
)
async def test_on_user_input_transcribed_drops_after_result_feedback_while_backend_is_processing(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._timeline = _FakeTimeline()
runtime._ctx.room.remote_participants = {"user-1": object()}
await runtime._speak_backend_reply(
BackendReply(
stage="PRESENTATION",
text="Aguarde um instante, por favor.",
metadata={
"agent_message_type": "result",
"agent_result_type": "feedback",
"expects_user_response": False,
"drop_user_input_while_speaking": True,
"is_interruptible": False,
},
),
source="run_pipeline",
add_to_chat_ctx=True,
)
self.assertFalse(runtime.speaking.is_set())
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="voce esta cancelando?", is_final=True)
)
await self._drain_tasks()
self.assertEqual(runtime.state.user_final_seq, 0)
self.assertIsNone(runtime.state.pending_user_final)
self.assertFalse(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
self.assertTrue(
any(
event == "user_input_dropped"
and fields["agent_message_type"] == "result"
and fields["agent_result_type"] == "feedback"
for event, fields in runtime._timeline.events
)
)
async def test_on_user_input_transcribed_drops_during_terminal_speech(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.state.current_stage = "DONE"
runtime.state.current_speech.stage = "DONE"
runtime.state.current_speech.drop_user_input_while_speaking = True
runtime.state.current_speech.agent_message_type = "result"
runtime.state.current_speech.agent_result_type = "resolvido"
runtime.speaking.set()
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="tenho outra pergunta", is_final=True)
)
await self._drain_tasks()
self.assertEqual(runtime.state.user_final_seq, 0)
self.assertIsNone(runtime.state.pending_user_final)
self.assertFalse(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
async def test_user_state_speaking_during_ready_drops_late_final_but_arms_timeout(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._timeline = _FakeTimeline()
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 1
executor.pipeline_result = BackendReply(
stage="PRESENTATION",
text="Posso verificar sua fatura.",
metadata={
"agent_message_type": "ready",
"expects_user_response": True,
"drop_user_input_while_speaking": True,
"is_interruptible": False,
"wait_timeout_seconds": 60.0,
},
)
executor.wait_for_playout_delay_s = 0.03
runtime.register_callbacks()
task = asyncio.create_task(runtime.run_pipeline("", "", user_seq=1))
try:
while not any(cmd.__class__.__name__ == "StartSpeech" for cmd in executor.commands):
await asyncio.sleep(0)
runtime._session.handlers["user_state_changed"](
SimpleNamespace(old_state="listening", new_state="speaking")
)
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="ja estou respondendo", is_final=True)
)
await task
pipeline_commands = [cmd for cmd in executor.commands if isinstance(cmd, RunPipelineInput)]
self.assertEqual(len(pipeline_commands), 1)
self.assertEqual(runtime.state.user_final_seq, 1)
self.assertTrue(
any(
event == "user_input_dropped"
and fields["agent_message_type"] == "ready"
for event, fields in runtime._timeline.events
)
)
self.assertTrue(
any(event == "agent_wait_timeout_armed" for event, _fields in runtime._timeline.events)
)
finally:
if not task.done():
task.cancel()
await asyncio.gather(task, return_exceptions=True)
await self._cancel_pending_tasks()
async def test_user_state_speaking_during_noninterruptible_presentation_drops_late_final(self) -> None:
runtime, _agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 60.0,
"inflight_backend_wait_timeout_s": 300.0,
"inflight_backend_wait_max_notices": 6,
"inflight_backend_wait_text": DEFAULT_WAIT_TEXT,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
runtime._timeline = _FakeTimeline()
runtime._ctx.room.remote_participants = {"user-1": object()}
_agent.pipeline = _FakeInflightPushPipeline()
runtime.state.current_stage = "PRESENTATION"
runtime.state.current_speech.stage = "PRESENTATION"
runtime.state.current_speech.allow_interruptions = False
runtime.speaking.set()
runtime.register_callbacks()
runtime._session.handlers["user_state_changed"](
SimpleNamespace(old_state="listening", new_state="speaking")
)
runtime.schedule_pre_backend_wait_notice(reason="vad_end_of_speech")
runtime.speaking.clear()
runtime.state.current_speech.stage = ""
runtime.state.current_speech.allow_interruptions = False
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="oi", is_final=True)
)
await self._drain_tasks()
self.assertEqual(runtime.state.user_final_seq, 0)
self.assertIsNone(runtime.state.pending_user_final)
self.assertFalse(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
self.assertFalse(any(cmd.__class__.__name__ == "StartSpeech" for cmd in executor.commands))
self.assertTrue(
any(
event == "pre_backend_wait_notice_skipped"
and fields["skipped_reason"] == "user_input_drop_pending"
for event, fields in runtime._timeline.events
)
)
self.assertTrue(
any(
event == "user_input_dropped"
and fields["agent_message_type"] == "ready"
and fields["text"] == "oi"
for event, fields in runtime._timeline.events
)
)
async def test_user_state_speaking_in_presentation_post_playout_grace_drops_late_final(self) -> None:
runtime, _agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 60.0,
"inflight_backend_wait_timeout_s": 300.0,
"inflight_backend_wait_max_notices": 6,
"inflight_backend_wait_text": DEFAULT_WAIT_TEXT,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
runtime._timeline = _FakeTimeline()
runtime._ctx.room.remote_participants = {"user-1": object()}
_agent.pipeline = _FakeInflightPushPipeline()
runtime.register_callbacks()
await runtime.say_stage(
"Ola, sou a assistente virtual da TIM.",
"PRESENTATION",
allow_interruptions=False,
schedule_idle_after=False,
)
self.assertFalse(runtime.speaking.is_set())
runtime._session.handlers["user_state_changed"](
SimpleNamespace(old_state="listening", new_state="speaking")
)
runtime.schedule_pre_backend_wait_notice(reason="vad_end_of_speech")
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="estou falando junto", is_final=True)
)
await self._drain_tasks()
self.assertEqual(runtime.state.user_final_seq, 0)
self.assertIsNone(runtime.state.pending_user_final)
self.assertFalse(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
self.assertTrue(
any(
event == "user_input_drop_grace_armed"
and fields["grace_ms"] == 250
for event, fields in runtime._timeline.events
)
)
self.assertTrue(
any(
event == "pre_backend_wait_notice_skipped"
and fields["skipped_reason"] == "user_input_drop_pending"
for event, fields in runtime._timeline.events
)
)
self.assertTrue(
any(
event == "user_input_dropped"
and fields["reason"] == "user_state_speaking_protected_post_playout_grace"
and fields["agent_message_type"] == "ready"
and fields["text"] == "estou falando junto"
for event, fields in runtime._timeline.events
)
)
async def test_presentation_post_playout_grace_expires_before_next_user_final(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._ctx.room.remote_participants = {"user-1": object()}
with mock.patch(
"app.livekit.runtime.call_runtime.PROTECTED_SPEECH_POST_PLAYOUT_DROP_GRACE_S",
0.001,
):
await runtime.say_stage(
"Ola, sou a assistente virtual da TIM.",
"PRESENTATION",
allow_interruptions=False,
schedule_idle_after=False,
)
await asyncio.sleep(0.01)
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="agora posso responder", is_final=True)
)
await self._drain_tasks()
self.assertEqual(runtime.state.user_final_seq, 1)
self.assertTrue(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
async def test_user_state_speaking_interrupts_current_tts(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.state.current_stage = "ARGUMENTATION"
runtime.state.current_speech.stage = "ARGUMENTATION"
runtime.state.current_speech.allow_interruptions = True
runtime.state.current_speech.handle = executor.speech_handle
runtime.state.current_speech.speech_id = "7f3a0000-0000-4000-8000-000000000000"
runtime.speaking.set()
runtime.register_callbacks()
runtime._session.handlers["user_state_changed"](
SimpleNamespace(old_state="listening", new_state="speaking")
)
await self._drain_tasks()
interrupt_commands = [
cmd for cmd in executor.commands if isinstance(cmd, InterruptSpeech)
]
self.assertEqual(len(interrupt_commands), 1)
self.assertIs(interrupt_commands[0].handle, executor.speech_handle)
async def test_user_state_speaking_interrupts_inflight_wait_notice(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.state.current_stage = "AGENT_BACKEND_WAIT"
runtime.state.current_speech.stage = "AGENT_BACKEND_WAIT"
runtime.state.current_speech.allow_interruptions = False
runtime.state.current_speech.handle = executor.speech_handle
runtime.speaking.set()
runtime.register_callbacks()
runtime._session.handlers["user_state_changed"](
SimpleNamespace(old_state="listening", new_state="speaking")
)
await self._drain_tasks()
interrupt_commands = [
cmd for cmd in executor.commands if isinstance(cmd, InterruptSpeech)
]
self.assertEqual(len(interrupt_commands), 1)
self.assertIs(interrupt_commands[0].handle, executor.speech_handle)
self.assertTrue(interrupt_commands[0].force)
async def test_user_state_speaking_cancels_agent_wait_timeout(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.register_callbacks()
runtime.arm_agent_wait_timeout_from_reply(
BackendReply(
stage="PRESENTATION",
text="Ola, como posso ajudar?",
metadata={"wait_timeout_seconds": 0.01},
),
user_seq=0,
source="test",
)
runtime._session.handlers["user_state_changed"](
SimpleNamespace(old_state="listening", new_state="speaking")
)
await asyncio.sleep(0.03)
await self._drain_tasks()
self.assertFalse(any(isinstance(cmd, NotifyBridgeStop) for cmd in executor.commands))
self.assertFalse(runtime.finalized.is_set())
async def test_user_state_listening_does_not_speak_wait_notice_before_stt_final(self) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
runtime, agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 60.0,
"inflight_backend_wait_timeout_s": 300.0,
"inflight_backend_wait_max_notices": 6,
"inflight_backend_wait_text": wait_text,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
runtime._ctx.room.remote_participants = {"user-1": object()}
agent.pipeline = _FakeInflightPushPipeline()
runtime.register_callbacks()
runtime._session.handlers["user_state_changed"](
SimpleNamespace(old_state="speaking", new_state="listening")
)
await self._drain_tasks()
speech_texts = [
cmd.text
for cmd in executor.commands
if cmd.__class__.__name__ == "StartSpeech"
]
self.assertEqual(speech_texts, [])
self.assertFalse(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
async def test_vad_wait_notice_starts_without_waiting_for_user_state_listening(self) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
runtime, agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 60.0,
"inflight_backend_wait_timeout_s": 300.0,
"inflight_backend_wait_max_notices": 6,
"inflight_backend_wait_text": wait_text,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
runtime._ctx.room.remote_participants = {"user-1": object()}
agent.pipeline = _FakeInflightPushPipeline()
runtime.register_callbacks()
runtime.schedule_pre_backend_wait_notice(reason="vad_end_of_speech")
await self._drain_tasks()
speech_texts = [
cmd.text
for cmd in executor.commands
if cmd.__class__.__name__ == "StartSpeech"
]
self.assertEqual(speech_texts, [_expected_wait_text_for_audio(WAIT_LONG_AUDIO_PATH, wait_text)])
async def test_vad_wait_notice_fast_path_uses_vad_pause_without_user_state(self) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
runtime, agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 60.0,
"inflight_backend_wait_timeout_s": 300.0,
"inflight_backend_wait_max_notices": 6,
"inflight_backend_wait_text": wait_text,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
"pre_backend_wait_notice_fast_on_vad_pause": True,
}
)
runtime._timeline = _FakeTimeline()
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime._user_not_speaking.clear()
agent.pipeline = _FakeInflightPushPipeline()
with mock.patch(
"app.livekit.runtime.call_runtime.asyncio.sleep",
new=mock.AsyncMock(),
) as sleep:
runtime.schedule_pre_backend_wait_notice(reason="vad_end_of_speech")
await self._drain_tasks()
sleep.assert_not_awaited()
speech_texts = [
cmd.text
for cmd in executor.commands
if cmd.__class__.__name__ == "StartSpeech"
]
self.assertEqual(speech_texts, [_expected_wait_text_for_audio(WAIT_LONG_AUDIO_PATH, wait_text)])
self.assertTrue(
any(
event == "pre_backend_wait_notice_started"
and fields["fast_on_vad_pause"] is True
for event, fields in runtime._timeline.events
)
)
async def test_vad_wait_notice_default_path_skips_while_user_state_speaking(self) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
runtime, agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 60.0,
"inflight_backend_wait_timeout_s": 300.0,
"inflight_backend_wait_max_notices": 6,
"inflight_backend_wait_text": wait_text,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
runtime._timeline = _FakeTimeline()
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime._user_not_speaking.clear()
agent.pipeline = _FakeInflightPushPipeline()
with mock.patch("app.livekit.runtime.call_runtime.PRE_BACKEND_WAIT_NOTICE_GUARD_S", 0):
runtime.schedule_pre_backend_wait_notice(reason="vad_end_of_speech")
await self._drain_tasks()
self.assertFalse(
any(cmd.__class__.__name__ == "StartSpeech" for cmd in executor.commands)
)
self.assertTrue(
any(
event == "pre_backend_wait_notice_skipped"
and fields["skipped_reason"] == "user_speaking"
for event, fields in runtime._timeline.events
)
)
def _wait_notice_config(self, wait_text: str) -> dict:
return {
"inflight_backend_wait_interval_s": 60.0,
"inflight_backend_wait_timeout_s": 300.0,
"inflight_backend_wait_max_notices": 6,
"inflight_backend_wait_text": wait_text,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
async def test_vad_wait_notice_skipped_while_agent_is_speaking(self) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
runtime, agent, executor = self._make_runtime(
runtime_config_overrides=self._wait_notice_config(wait_text)
)
runtime._timeline = _FakeTimeline()
runtime._ctx.room.remote_participants = {"user-1": object()}
agent.pipeline = _FakeInflightPushPipeline()
runtime.state.current_stage = "PRESENTATION"
runtime.state.current_speech.stage = "PRESENTATION"
runtime.speaking.set()
runtime.schedule_pre_backend_wait_notice(reason="vad_end_of_speech")
await self._drain_tasks()
self.assertFalse(
any(cmd.__class__.__name__ == "StartSpeech" for cmd in executor.commands)
)
self.assertTrue(
any(
event == "pre_backend_wait_notice_skipped"
and fields["skipped_reason"] == "agent_speaking"
for event, fields in runtime._timeline.events
)
)
self.assertFalse(runtime._pre_backend_wait_notice_reserved)
async def test_vad_wait_notice_skipped_while_backend_is_processing(self) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
runtime, agent, executor = self._make_runtime(
runtime_config_overrides=self._wait_notice_config(wait_text)
)
runtime._timeline = _FakeTimeline()
runtime._ctx.room.remote_participants = {"user-1": object()}
agent.pipeline = _FakeInflightPushPipeline()
await agent._run_lock.acquire()
try:
runtime.schedule_pre_backend_wait_notice(reason="vad_end_of_speech")
await self._drain_tasks()
finally:
agent._run_lock.release()
self.assertFalse(
any(cmd.__class__.__name__ == "StartSpeech" for cmd in executor.commands)
)
self.assertTrue(
any(
event == "pre_backend_wait_notice_skipped"
and fields["skipped_reason"] == "backend_processing"
for event, fields in runtime._timeline.events
)
)
self.assertFalse(runtime._pre_backend_wait_notice_reserved)
async def test_wait_notice_audio_aborts_when_another_speech_took_the_lock(self) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
runtime, agent, executor = self._make_runtime(
runtime_config_overrides=self._wait_notice_config(wait_text)
)
runtime._ctx.room.remote_participants = {"user-1": object()}
agent.pipeline = _FakeInflightPushPipeline()
say_stage_seq_at_schedule = runtime._say_stage_seq
await runtime.say_stage("Resposta do agente.", "PRESENTATION")
played = await runtime._play_inflight_backend_wait_notice_audio(
user_seq=None,
attempt=1,
abort_if_agent_spoke_since=say_stage_seq_at_schedule,
)
await self._drain_tasks()
self.assertFalse(played)
speech_texts = [
cmd.text
for cmd in executor.commands
if cmd.__class__.__name__ == "StartSpeech"
]
self.assertEqual(speech_texts, ["Resposta do agente."])
async def test_vad_wait_notice_does_not_queue_behind_an_agent_speech(self) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
runtime, agent, executor = self._make_runtime(
runtime_config_overrides=self._wait_notice_config(wait_text)
)
runtime._ctx.room.remote_participants = {"user-1": object()}
agent.pipeline = _FakeInflightPushPipeline()
# O aviso e agendado no fim de fala do VAD e so pega o say_lock depois
# que a fala do agente libera: nesse ponto ele nao deve mais sair.
runtime.schedule_pre_backend_wait_notice(reason="vad_end_of_speech")
await runtime.say_stage("Resposta do agente.", "PRESENTATION")
await self._drain_tasks()
speech_texts = [
cmd.text
for cmd in executor.commands
if cmd.__class__.__name__ == "StartSpeech"
]
self.assertEqual(speech_texts, ["Resposta do agente."])
async def test_vad_wait_notice_returns_reservation_when_it_does_not_play(self) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
runtime, agent, executor = self._make_runtime(
runtime_config_overrides=self._wait_notice_config(wait_text)
)
runtime._ctx.room.remote_participants = {"user-1": object()}
agent.pipeline = _FakeInflightPushPipeline()
runtime.schedule_pre_backend_wait_notice(reason="vad_end_of_speech")
await runtime.say_stage("Resposta do agente.", "PRESENTATION")
await self._drain_tasks()
self.assertFalse(runtime._pre_backend_wait_notice_reserved)
# A reserva devolvida nao pode bloquear o proximo agendamento.
runtime.schedule_pre_backend_wait_notice(reason="vad_end_of_speech")
await self._drain_tasks()
speech_texts = [
cmd.text
for cmd in executor.commands
if cmd.__class__.__name__ == "StartSpeech"
]
self.assertEqual(
speech_texts,
[
"Resposta do agente.",
_expected_wait_text_for_audio(WAIT_LONG_AUDIO_PATH, wait_text),
],
)
async def test_vad_wait_notice_uses_started_turn_message_id(self) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
runtime, agent, _executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 60.0,
"inflight_backend_wait_timeout_s": 300.0,
"inflight_backend_wait_max_notices": 6,
"inflight_backend_wait_text": wait_text,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
structured_context = StructuredLogContext(
callid="call-1",
session_id="session-started-turn-runtime",
num_telefone="5511999999999",
cod_ani="1234",
nome_agente="conta",
)
runtime._structured_log_context = structured_context
runtime._timeline = _FakeTimeline()
runtime._ctx.room.remote_participants = {"user-1": object()}
agent.pipeline = _FakeInflightPushPipeline()
reset_turn_message_sequence(structured_context, clear_pending=True)
register_started_turn_message_id(structured_context, "GED-test-0002")
try:
runtime.schedule_pre_backend_wait_notice(reason="vad_end_of_speech")
await self._drain_tasks()
wait_events = [
fields
for event, fields in runtime._timeline.events
if event == "inflight_backend_wait_audio"
]
self.assertEqual(len(wait_events), 1)
self.assertTrue(wait_events[0]["message_id"].startswith("GED-test-0002_conforto_"))
finally:
reset_turn_message_sequence(structured_context, clear_pending=True)
async def test_vad_wait_notice_uses_last_agent_message_id_without_started_turn(self) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
runtime, agent, _executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 60.0,
"inflight_backend_wait_timeout_s": 300.0,
"inflight_backend_wait_max_notices": 6,
"inflight_backend_wait_text": wait_text,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
structured_context = StructuredLogContext(
callid="call-1",
session_id="session-last-agent-turn-runtime",
num_telefone="5511999999999",
cod_ani="1234",
nome_agente="conta",
)
runtime._structured_log_context = structured_context
runtime._timeline = _FakeTimeline()
runtime._ctx.room.remote_participants = {"user-1": object()}
agent.pipeline = _FakeInflightPushPipeline()
reset_turn_message_sequence(structured_context, clear_pending=True)
runtime._remember_agent_base_message_id("MSG-agent-0001")
try:
runtime.schedule_pre_backend_wait_notice(reason="vad_end_of_speech")
await self._drain_tasks()
wait_events = [
fields
for event, fields in runtime._timeline.events
if event == "inflight_backend_wait_audio"
]
self.assertEqual(len(wait_events), 1)
self.assertTrue(wait_events[0]["message_id"].startswith("MSG-agent-0001_conforto_"))
finally:
reset_turn_message_sequence(structured_context, clear_pending=True)
async def test_inflight_backend_wait_notice_skips_post_user_final_grace(self) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
runtime, agent, _executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 60.0,
"inflight_backend_wait_timeout_s": 300.0,
"inflight_backend_wait_max_notices": 6,
"inflight_backend_wait_text": wait_text,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.last_user_final_at = time.monotonic()
agent.pipeline = _FakeInflightPushPipeline()
with mock.patch.object(
runtime,
"_wait_post_user_final_grace",
new=mock.AsyncMock(),
) as wait_grace:
await runtime._play_inflight_backend_wait_notice_audio(
user_seq=None,
attempt=1,
)
wait_grace.assert_not_awaited()
async def test_empty_stt_final_logs_and_clears_pre_backend_wait_notice(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._timeline = _FakeTimeline()
runtime._pre_backend_wait_notice_reserved = True
with (
mock.patch("app.livekit.runtime.call_runtime.log_flow_event") as flow_log,
mock.patch.object(runtime, "arm_idle_timer") as arm_idle,
):
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="", is_final=True)
)
await self._drain_tasks()
self.assertFalse(runtime._pre_backend_wait_notice_reserved)
arm_idle.assert_called_once_with(reason="empty_stt_final")
self.assertFalse(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
self.assertTrue(
any(
call.args[1] == "stt_final_empty"
and call.kwargs["reason"] == "empty_text"
and call.kwargs["pre_backend_wait_reserved"] is True
for call in flow_log.call_args_list
)
)
self.assertIn(
(
"user_transcript_final_empty",
{
"current_stage": "INTRO",
"speaking": False,
"transcript_len": 0,
"pre_backend_wait_reserved": True,
},
),
runtime._timeline.events,
)
async def test_empty_stt_final_rearms_agent_wait_timeout_when_available(self) -> None:
runtime, _agent, _executor = self._make_runtime()
reply = BackendReply(
stage="PRESENTATION",
text="Ainda esta ai?",
metadata={"wait_timeout_seconds": 60.0},
)
runtime.arm_agent_wait_timeout_from_reply(reply, user_seq=0, source="test")
runtime.cancel_agent_wait_timeout("user_state_speaking")
try:
with (
mock.patch.object(
runtime,
"_arm_agent_wait_timeout",
wraps=runtime._arm_agent_wait_timeout,
) as arm_wait,
mock.patch.object(runtime, "arm_idle_timer") as arm_idle,
):
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="", is_final=True)
)
arm_wait.assert_called_once()
self.assertEqual(arm_wait.call_args.kwargs["user_seq"], 0)
self.assertEqual(arm_wait.call_args.kwargs["source"], "empty_stt_final")
arm_idle.assert_not_called()
finally:
await self._cancel_pending_tasks()
async def test_empty_stt_final_during_comfort_rearms_after_playout(self) -> None:
runtime, _agent, executor = self._make_runtime()
reply = BackendReply(
stage="PRESENTATION",
text="Ainda esta ai?",
metadata={"wait_timeout_seconds": 60.0},
)
runtime.arm_agent_wait_timeout_from_reply(reply, user_seq=0, source="test")
runtime.cancel_agent_wait_timeout("user_state_speaking")
executor.wait_for_playout_callbacks.append(
lambda _command: runtime.handle_empty_stt_final(source="stt_provider")
)
try:
with mock.patch.object(
runtime,
"_arm_agent_wait_timeout",
wraps=runtime._arm_agent_wait_timeout,
) as arm_wait:
await runtime.say_stage(
"Um instantinho",
"AGENT_BACKEND_WAIT",
add_to_chat_ctx=False,
allow_interruptions=False,
schedule_idle_after=False,
)
arm_wait.assert_called_once()
self.assertEqual(arm_wait.call_args.kwargs["user_seq"], 0)
self.assertEqual(arm_wait.call_args.kwargs["source"], "empty_stt_final")
self.assertFalse(runtime._empty_stt_recovery_pending)
finally:
await self._cancel_pending_tasks()
async def test_inflight_backend_wait_audio_leaves_tts_metrics_empty_without_exporting_original_id(
self,
) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
runtime, agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 60.0,
"inflight_backend_wait_timeout_s": 300.0,
"inflight_backend_wait_max_notices": 6,
"inflight_backend_wait_text": wait_text,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 1
runtime._timeline = _FakeTimeline()
agent.pipeline = _FakeInflightPushPipeline()
executor.speech_handle = SimpleNamespace(id="speech-no-metric", interrupted=False)
structured_events = []
def _capture_event(*args, **kwargs):
structured_events.append(kwargs)
with mock.patch.dict(os.environ, {"TTS_TTFB_METRIC_WAIT_MS": "0"}), mock.patch(
"app.livekit.runtime.call_runtime.log_structured_event",
side_effect=_capture_event,
):
await runtime._play_inflight_backend_wait_notice_audio(
user_seq=1,
attempt=2,
message_id="GED-test-0002",
)
self.assertEqual(len(structured_events), 1)
self.assertIsNone(structured_events[0]["latencia_total_ms"])
self.assertIsNone(structured_events[0]["latencia_tffb_ms"])
self.assertIsNone(structured_events[0]["duracao_audio_ms"])
self.assertNotIn("original_message_id", structured_events[0])
wait_events = [
fields
for event, fields in runtime._timeline.events
if event == "inflight_backend_wait_audio"
]
self.assertEqual(len(wait_events), 1)
self.assertTrue(wait_events[0]["message_id"].startswith("GED-test-0002_conforto_"))
self.assertEqual(wait_events[0]["text"], _expected_wait_text_for_audio(WAIT_LONG_AUDIO_PATH, wait_text))
self.assertNotIn("original_message_id", wait_events[0])
async def test_inflight_backend_wait_audio_uses_sidecar_text_for_logged_event(self) -> None:
with TemporaryDirectory() as tmp_dir:
long_dir = Path(tmp_dir) / "long"
long_dir.mkdir()
audio_path = long_dir / "01.wav"
text_path = long_dir / "01.txt"
audio_path.write_bytes(b"fake-wav")
text_path.write_text(
"Estou verificando as informacoes para te ajudar.\nSo um momentinho.",
encoding="utf-8",
)
expected_text = "Estou verificando as informacoes para te ajudar. So um momentinho."
runtime, agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 60.0,
"inflight_backend_wait_timeout_s": 300.0,
"inflight_backend_wait_max_notices": 6,
"inflight_backend_wait_text": "Texto padrao que nao deve aparecer.",
"inflight_backend_wait_long_audio_dir": str(long_dir),
}
)
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 1
runtime._timeline = _FakeTimeline()
agent.pipeline = _FakeInflightPushPipeline()
with (
mock.patch("app.livekit.runtime.call_runtime.wav_duration_ms", return_value=100),
mock.patch("app.livekit.runtime.call_runtime.wav_audio_frames", return_value="audio-frames"),
):
await runtime._play_inflight_backend_wait_notice_audio(
user_seq=1,
attempt=2,
message_id="GED-test-0002",
)
speech_commands = [
cmd
for cmd in executor.commands
if cmd.__class__.__name__ == "StartSpeech"
]
self.assertEqual([cmd.text for cmd in speech_commands], [expected_text])
wait_events = [
fields
for event, fields in runtime._timeline.events
if event == "inflight_backend_wait_audio"
]
self.assertEqual(len(wait_events), 1)
self.assertEqual(wait_events[0]["text"], expected_text)
self.assertEqual(wait_events[0]["path"], str(audio_path))
async def test_backend_wait_notice_starts_immediately_after_stt_final(self) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
runtime, agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 60.0,
"inflight_backend_wait_timeout_s": 0.01,
"inflight_backend_wait_max_notices": 1,
"inflight_backend_wait_text": wait_text,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
runtime._ctx.room.remote_participants = {"user-1": object()}
agent.pipeline = _FakeInflightPushPipeline()
runtime.state.user_final_seq = 1
never_finishes = asyncio.Event()
async def _hang_backend() -> None:
await never_finishes.wait()
executor.run_pipeline_side_effect = _hang_backend
await runtime.run_pipeline("texto", "texto", user_seq=1)
await self._drain_tasks()
speech_texts = [
cmd.text
for cmd in executor.commands
if cmd.__class__.__name__ == "StartSpeech"
]
self.assertEqual(speech_texts, [_expected_wait_text_for_audio(WAIT_LONG_AUDIO_PATH, wait_text)])
speech_commands = [
cmd
for cmd in executor.commands
if cmd.__class__.__name__ == "StartSpeech"
]
self.assertTrue(speech_commands[0].allow_interruptions)
self.assertIsNotNone(speech_commands[0].audio)
async def test_pre_stt_wait_notice_activity_controls_next_backend_notice_interval(self) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
runtime, agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 60.0,
"inflight_backend_wait_timeout_s": 0.01,
"inflight_backend_wait_max_notices": 6,
"inflight_backend_wait_text": wait_text,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 1
agent.pipeline = _FakeInflightPushPipeline()
runtime._pre_backend_wait_notice_reserved = True
runtime._inflight_backend_activity_at = time.monotonic() - 120.0
never_finishes = asyncio.Event()
async def _hang_backend() -> None:
await never_finishes.wait()
executor.run_pipeline_side_effect = _hang_backend
await runtime.run_pipeline("texto", "texto", user_seq=1)
await self._drain_tasks()
speech_commands = [
cmd
for cmd in executor.commands
if cmd.__class__.__name__ == "StartSpeech"
]
self.assertEqual(
[cmd.text for cmd in speech_commands],
[_expected_wait_text_for_audio(WAIT_LONG_AUDIO_PATH, wait_text)],
)
self.assertTrue(speech_commands[0].allow_interruptions)
async def test_inflight_backend_wait_notice_skips_when_user_is_speaking(self) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
runtime, agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 60.0,
"inflight_backend_wait_timeout_s": 300.0,
"inflight_backend_wait_max_notices": 6,
"inflight_backend_wait_text": wait_text,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
runtime._ctx.room.remote_participants = {"user-1": object()}
agent.pipeline = _FakeInflightPushPipeline()
runtime._user_not_speaking.clear()
await runtime._play_inflight_backend_wait_notice_audio(
user_seq=1,
attempt=1,
message_id="GED-test-0001",
)
await self._drain_tasks()
self.assertFalse(
any(cmd.__class__.__name__ == "StartSpeech" for cmd in executor.commands)
)
async def test_pre_stt_wait_notice_is_not_restarted_while_pending(self) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
runtime, agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 60.0,
"inflight_backend_wait_timeout_s": 300.0,
"inflight_backend_wait_max_notices": 6,
"inflight_backend_wait_text": wait_text,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
runtime._ctx.room.remote_participants = {"user-1": object()}
agent.pipeline = _FakeInflightPushPipeline()
runtime.schedule_pre_backend_wait_notice(reason="vad_end_of_speech")
runtime.schedule_pre_backend_wait_notice(reason="user_state_listening")
await self._drain_tasks()
speech_texts = [
cmd.text
for cmd in executor.commands
if cmd.__class__.__name__ == "StartSpeech"
]
self.assertEqual(speech_texts, [_expected_wait_text_for_audio(WAIT_LONG_AUDIO_PATH, wait_text)])
async def test_stt_final_does_not_interrupt_pre_backend_wait_notice(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._pre_backend_wait_notice_reserved = True
runtime.speaking.set()
runtime.state.current_stage = "PRESENTATION"
runtime.state.current_speech.stage = "AGENT_BACKEND_WAIT"
runtime.state.current_speech.allow_interruptions = True
runtime.state.current_speech.handle = executor.speech_handle
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="Por que veio mais caro?", is_final=True)
)
await self._drain_tasks()
self.assertFalse(any(isinstance(cmd, InterruptSpeech) for cmd in executor.commands))
self.assertTrue(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
async def test_sim_during_pre_backend_wait_notice_runs_pipeline(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._pre_backend_wait_notice_reserved = True
runtime.speaking.set()
runtime.state.current_stage = "PRESENTATION"
runtime.state.current_speech.stage = "AGENT_BACKEND_WAIT"
runtime.state.current_speech.allow_interruptions = False
runtime.state.current_speech.handle = executor.speech_handle
with mock.patch("app.livekit.runtime.call_runtime.log_flow_event") as flow_log:
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="sim", is_final=True)
)
await self._drain_tasks()
self.assertFalse(any(isinstance(cmd, InterruptSpeech) for cmd in executor.commands))
self.assertTrue(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
self.assertFalse(
any(
call.args[1] == "interrupt_ignored"
and call.kwargs.get("reason") == "backchannel"
and call.kwargs.get("text") == "sim"
for call in flow_log.call_args_list
)
)
async def test_run_pipeline_forwards_pending_speech_interruption(self) -> None:
runtime, agent, executor = self._make_runtime()
runtime.state.user_final_seq = 1
agent._consumed_interrupt = (
"Claro, vou te explicar",
"7f3a0000-0000-4000-8000-000000000000",
False,
)
await runtime.run_pipeline("texto", "texto", user_seq=1)
interruption_commands = [
cmd for cmd in executor.commands if isinstance(cmd, SetPipelineInterruption)
]
self.assertEqual(len(interruption_commands), 1)
self.assertTrue(interruption_commands[0].interrupted)
self.assertEqual(interruption_commands[0].listened_text, "Claro, vou te explicar")
self.assertEqual(
interruption_commands[0].speech_id,
"7f3a0000-0000-4000-8000-000000000000",
)
async def test_deferred_replay_marks_processing_interruption(self) -> None:
runtime, agent, executor = self._make_runtime()
runtime.state.user_final_seq = 1
agent._consumed_interrupt = (
"",
"7f3a0000-0000-4000-8000-000000000001",
True,
)
await runtime.run_pipeline(
"texto interrompido",
"texto interrompido",
user_seq=1,
is_deferred_replay=True,
)
command = next(
cmd
for cmd in executor.commands
if isinstance(cmd, SetPipelineProcessingInterruption)
)
self.assertEqual(command.listened_text, "")
self.assertEqual(
command.speech_id,
"7f3a0000-0000-4000-8000-000000000001",
)
async def test_run_pipeline_uses_registered_turn_message_id(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.state.user_final_seq = 1
structured_context = StructuredLogContext(
callid="call-1",
session_id="session-1",
num_telefone="5511999999999",
cod_ani="1234",
nome_agente="conta",
)
runtime._structured_log_context = structured_context
reset_turn_message_sequence(structured_context, clear_pending=True)
register_transcribed_turn(
structured_context,
message_id="MSG-call-1-0001",
transcription="texto",
text="texto",
)
await runtime.run_pipeline("texto", "texto", user_seq=1)
pipeline_command = next(cmd for cmd in executor.commands if isinstance(cmd, RunPipelineInput))
self.assertEqual(pipeline_command.user_input["message_id"], "MSG-call-1-0001")
enriched_reply = runtime._backend_reply_with_message_id(
BackendReply(stage="PRESENTATION", text="resposta"),
"MSG-call-1-0001",
)
self.assertEqual(runtime._message_id_from_reply(enriched_reply), "MSG-call-1-0001")
async def test_on_user_input_transcribed_clears_started_turn_message_id(self) -> None:
runtime, _agent, executor = self._make_runtime()
structured_context = StructuredLogContext(
callid="call-1",
session_id="session-user-final-started-turn",
num_telefone="5511999999999",
cod_ani="1234",
nome_agente="conta",
)
runtime._structured_log_context = structured_context
reset_turn_message_sequence(structured_context, clear_pending=True)
register_started_turn_message_id(structured_context, "MSG-call-1-0002")
register_transcribed_turn(
structured_context,
message_id="MSG-call-1-0002",
transcription="texto",
text="texto",
)
try:
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="texto", is_final=True)
)
await self._drain_tasks()
pipeline_command = next(cmd for cmd in executor.commands if isinstance(cmd, RunPipelineInput))
self.assertEqual(pipeline_command.user_input["message_id"], "MSG-call-1-0002")
self.assertEqual(peek_started_turn_message_id(structured_context), "")
finally:
reset_turn_message_sequence(structured_context, clear_pending=True)
async def test_say_stage_marks_interruption_with_speech_id(self) -> None:
runtime, _agent, executor = self._make_runtime()
executor.speech_handle = SimpleNamespace(interrupted=True)
executor.spoken_text = "Claro, vou te explicar"
await runtime.say_stage(
"Claro, vou te explicar sua fatura.",
"ARGUMENTATION",
allow_interruptions=True,
speech_id="7f3a0000-0000-4000-8000-000000000000",
)
pending_commands = [cmd for cmd in executor.commands if isinstance(cmd, SetPendingInterrupt)]
self.assertEqual(len(pending_commands), 1)
self.assertEqual(pending_commands[0].listened_text, "Claro, vou te explicar")
self.assertEqual(
pending_commands[0].speech_id,
"7f3a0000-0000-4000-8000-000000000000",
)
async def test_backend_reply_metadata_controls_tts_interruptibility(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 1
executor.pipeline_result = BackendReply(
stage="ARGUMENTATION",
text="Ola, como posso ajudar?",
metadata={
"speech_id": "11111111-1111-4111-8111-111111111111",
"is_interruptible": False,
},
)
await runtime.run_pipeline("texto", "texto", user_seq=1)
speech_commands = [cmd for cmd in executor.commands if cmd.__class__.__name__ == "StartSpeech"]
self.assertEqual(len(speech_commands), 1)
self.assertFalse(speech_commands[0].allow_interruptions)
async def test_run_pipeline_skips_stale_turn_before_running_pipeline(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.state.user_final_seq = 2
await runtime.run_pipeline("texto", "texto", user_seq=1)
self.assertFalse(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
async def test_run_pipeline_forwards_short_acknowledgement_while_speaking(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.speaking.set()
with mock.patch("app.livekit.runtime.call_runtime.log_flow_event") as flow_log:
await runtime.run_pipeline("ok", "ok", user_seq=0)
self.assertTrue(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
self.assertFalse(
any(
call.args[1] == "interrupt_ignored"
and call.kwargs.get("reason") == "backchannel"
for call in flow_log.call_args_list
)
)
async def test_low_confidence_sim_after_tts_runs_pipeline(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._ctx.room.remote_participants = {"user-1": object()}
await runtime.say_stage(
"Consegui esclarecer sua dúvida?",
"PRESENTATION",
allow_interruptions=True,
)
self.assertFalse(runtime.speaking.is_set())
low_confidence_sim = stt_text_with_single_word_threshold(
{
"data": {
"text": "sim",
"words": [{"word": "sim", "probability": 0.009}],
}
},
min_prob_single_word=0.03,
)
self.assertEqual(low_confidence_sim, "sim")
with mock.patch("app.livekit.runtime.call_runtime.log_flow_event") as flow_log:
runtime.on_user_input_transcribed(
SimpleNamespace(transcript=low_confidence_sim, is_final=True)
)
await self._drain_tasks()
pipeline_commands = [
cmd for cmd in executor.commands if isinstance(cmd, RunPipelineInput)
]
self.assertTrue(pipeline_commands)
self.assertEqual(pipeline_commands[-1].user_input, "sim")
self.assertFalse(
any(
call.args[1] == "interrupt_ignored"
and call.kwargs.get("reason") == "backchannel"
for call in flow_log.call_args_list
)
)
async def test_run_pipeline_drops_output_if_user_speaks_during_pipeline(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.state.user_final_seq = 1
async def _mutate_seq():
runtime.state.user_final_seq = 2
executor.pipeline_result = BackendReply(stage="PRESENTATION", text="resposta")
executor.run_pipeline_side_effect = _mutate_seq
await runtime.run_pipeline("texto", "texto", user_seq=1)
self.assertEqual(runtime.state.current_stage, "INTRO")
self.assertFalse(any(cmd.__class__.__name__ == "StartSpeech" for cmd in executor.commands))
async def test_say_stage_waits_briefly_after_user_final(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.state.last_user_final_at = time.monotonic()
sleep_calls = []
async def _fake_sleep(delay: float) -> None:
sleep_calls.append(delay)
with mock.patch("app.livekit.runtime.call_runtime.asyncio.sleep", side_effect=_fake_sleep):
await runtime.say_stage("resposta", "ARGUMENTATION")
self.assertTrue(sleep_calls)
self.assertGreater(sleep_calls[0], 0.0)
self.assertLessEqual(sleep_calls[0], 0.35)
self.assertTrue(any(cmd.__class__.__name__ == "StartSpeech" for cmd in executor.commands))
async def test_say_stage_logs_tts_metrics_and_success_status(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._record_xai_tts_turn_timing(
{
"segment_id": "segment-test",
"max_audio_delta_gap_ms": 840,
"max_playout_underrun_0ms": 125,
"xai_micro_underflows": 2,
"xai_avg_underrun_ms": 80,
}
)
runtime._record_tts_metric(
SimpleNamespace(
metrics=SimpleNamespace(
type="tts_metrics",
ttfb=0.123,
duration=0.456,
audio_duration=1.500,
speech_id="speech-test",
segment_id="segment-test",
label="provider",
)
)
)
structured_events = []
def _capture_event(*args, **kwargs):
structured_events.append(kwargs)
with mock.patch("app.livekit.runtime.call_runtime.log_structured_event", side_effect=_capture_event):
await runtime.say_stage("resposta", "ARGUMENTATION", message_id="GED-555-0002")
self.assertTrue(any(cmd.__class__.__name__ == "StartSpeech" for cmd in executor.commands))
self.assertEqual(len(structured_events), 1)
event = structured_events[0]
self.assertEqual(event["tipo_evento"], "envio msg")
self.assertIsNotNone(event["inicio_ns"])
self.assertIsNotNone(event["fim_ns"])
self.assertEqual(event["latencia_total_ms"], 456)
self.assertEqual(event["latencia_tffb_ms"], 123)
self.assertEqual(event["duracao_audio_ms"], 1500)
self.assertEqual(event["tts_max_gap_ms"], 840)
self.assertEqual(event["tts_max_underrun_0ms"], 125)
self.assertEqual(event["tts_underflow_count"], 2)
self.assertEqual(event["tts_avg_underflow_ms"], 80)
self.assertEqual(event["message_id"], "GED-555-0002")
self.assertEqual(event["http_cod_status"], 200)
self.assertEqual(event["http_cod_desc"], "OK")
async def test_say_stage_leaves_tts_metrics_empty_when_metric_is_missing(self) -> None:
runtime, _agent, executor = self._make_runtime()
executor.speech_handle = SimpleNamespace(id="speech-without-metric", interrupted=False)
structured_events = []
def _capture_event(*args, **kwargs):
structured_events.append(kwargs)
with mock.patch.dict(os.environ, {"TTS_TTFB_METRIC_WAIT_MS": "0"}), mock.patch(
"app.livekit.runtime.call_runtime.log_structured_event",
side_effect=_capture_event,
):
await runtime.say_stage("resposta", "ARGUMENTATION", message_id="GED-555-0003")
self.assertEqual(len(structured_events), 1)
self.assertIsNone(structured_events[0]["latencia_total_ms"])
self.assertIsNone(structured_events[0]["latencia_tffb_ms"])
self.assertIsNone(structured_events[0]["duracao_audio_ms"])
async def test_say_stage_logs_audio_duration_from_tts_metrics(self) -> None:
runtime, _agent, executor = self._make_runtime()
executor.speech_handle = SimpleNamespace(id="speech-with-audio-metric", interrupted=False)
runtime._record_tts_metric(
SimpleNamespace(
metrics=SimpleNamespace(
type="tts_metrics",
ttfb=0.123,
duration=0.400,
audio_duration=1.500,
speech_id="speech-with-audio-metric",
label="provider",
)
)
)
structured_events = []
flow_events = []
def _capture_event(*args, **kwargs):
structured_events.append(kwargs)
def _capture_flow_event(_logger, event_name, **kwargs):
flow_events.append((event_name, kwargs))
with mock.patch(
"app.livekit.runtime.call_runtime.log_structured_event",
side_effect=_capture_event,
), mock.patch(
"app.livekit.runtime.call_runtime.log_flow_event",
side_effect=_capture_flow_event,
):
await runtime.say_stage("resposta", "ARGUMENTATION", message_id="GED-555-0004")
self.assertEqual(len(structured_events), 1)
self.assertEqual(structured_events[0]["latencia_total_ms"], 400)
self.assertEqual(structured_events[0]["latencia_tffb_ms"], 123)
self.assertEqual(structured_events[0]["duracao_audio_ms"], 1500)
tts_done_events = [fields for event, fields in flow_events if event == "tts_done"]
self.assertEqual(tts_done_events[0]["duration_ms"], 400)
self.assertEqual(tts_done_events[0]["provider_duration_ms"], 400)
self.assertEqual(tts_done_events[0]["metrics_source"], "livekit_tts_metrics")
async def test_say_stage_marks_interruption_with_cancelled_tts_metric(self) -> None:
runtime, _agent, executor = self._make_runtime()
executor.speech_handle = SimpleNamespace(id="speech-cancelled", interrupted=True)
executor.spoken_text = "preciso falar"
runtime._record_tts_metric(
SimpleNamespace(
metrics=SimpleNamespace(
type="tts_metrics",
ttfb=0.150,
duration=0.700,
audio_duration=0.250,
speech_id="speech-cancelled",
label="provider",
cancelled=True,
)
)
)
structured_events = []
flow_events = []
def _capture_event(*args, **kwargs):
structured_events.append(kwargs)
def _capture_flow_event(_logger, event_name, **kwargs):
flow_events.append((event_name, kwargs))
with mock.patch(
"app.livekit.runtime.call_runtime.log_structured_event",
side_effect=_capture_event,
), mock.patch(
"app.livekit.runtime.call_runtime.log_flow_event",
side_effect=_capture_flow_event,
):
await runtime.say_stage(
"resposta interrompida",
"ARGUMENTATION",
message_id="GED-555-0005",
allow_interruptions=True,
)
self.assertEqual(len(structured_events), 1)
self.assertEqual(structured_events[0]["latencia_total_ms"], 700)
self.assertEqual(structured_events[0]["latencia_tffb_ms"], 150)
self.assertEqual(structured_events[0]["duracao_audio_ms"], 250)
self.assertIs(structured_events[0]["interrupcao"], True)
tts_done_events = [fields for event, fields in flow_events if event == "tts_done"]
self.assertIs(tts_done_events[0]["interruption"], True)
pending_commands = [cmd for cmd in executor.commands if isinstance(cmd, SetPendingInterrupt)]
self.assertEqual(len(pending_commands), 1)
self.assertEqual(pending_commands[0].listened_text, "preciso falar")
async def test_say_stage_keeps_interruption_duration_without_first_tts_audio(self) -> None:
runtime, _agent, executor = self._make_runtime()
executor.speech_handle = SimpleNamespace(id="speech-cancelled-no-audio", interrupted=True)
runtime._record_tts_metric(
SimpleNamespace(
metrics=SimpleNamespace(
type="tts_metrics",
ttfb=-1.0,
duration=0.700,
audio_duration=0.0,
speech_id="speech-cancelled-no-audio",
label="provider",
cancelled=True,
)
)
)
structured_events = []
def _capture_event(*args, **kwargs):
structured_events.append(kwargs)
with mock.patch(
"app.livekit.runtime.call_runtime.log_structured_event",
side_effect=_capture_event,
):
await runtime.say_stage(
"resposta interrompida",
"ARGUMENTATION",
message_id="GED-555-0006",
allow_interruptions=True,
)
self.assertEqual(len(structured_events), 1)
self.assertEqual(structured_events[0]["latencia_total_ms"], 700)
self.assertIsNone(structured_events[0]["latencia_tffb_ms"])
self.assertIsNone(structured_events[0]["duracao_audio_ms"])
self.assertIs(structured_events[0]["interrupcao"], True)
async def test_say_stage_returns_to_listening_when_no_first_audio_is_seen(self) -> None:
runtime, _agent, executor = self._make_runtime()
first_handle = SimpleNamespace(id="speech-first", interrupted=False)
executor.speech_handles = [first_handle]
executor.wait_for_playout_delays_s = [0.05]
flow_events = []
def capture(_logger, event_name, **kwargs):
flow_events.append((event_name, kwargs))
with mock.patch.dict(os.environ, {"TTS_PLAYOUT_START_TIMEOUT_S": "0.01", "TTS_TTFB_METRIC_WAIT_MS": "0"}), mock.patch("app.livekit.runtime.call_runtime.log_flow_event", side_effect=capture):
result = await runtime.say_stage("resposta", "ARGUMENTATION", message_id="GED-555-0005")
self.assertFalse(result)
speech_commands = [cmd for cmd in executor.commands if cmd.__class__.__name__ == "StartSpeech"]
self.assertEqual(len(speech_commands), 1)
self.assertFalse(runtime.finalized.is_set())
self.assertTrue(first_handle.interrupted)
self.assertTrue(any(event == "tts_failure_return_to_listening" for event, _ in flow_events))
async def test_say_stage_returns_to_listening_after_partial_audio_failure(self) -> None:
runtime, _agent, executor = self._make_runtime()
first_handle = SimpleNamespace(id="speech-gap", interrupted=False)
executor.speech_handles = [first_handle]
executor.wait_for_playout_exceptions = [RuntimeError("xAI TTS partial audio failure: underflow_error; socket_resynchronized=0; xai_underrun_estimado_ms=1001; xai_micro_underflows=1; xai_avg_underrun_ms=1001; underflow_error_ms=1000; max_audio_delta_gap_ms=1200; pcm_duration_ms=500; pcm_bytes=24000")]
structured_events = []
def capture_event(*_args, **kwargs):
structured_events.append(kwargs)
with mock.patch.dict(os.environ, {"TTS_TTFB_METRIC_WAIT_MS": "0"}), mock.patch("app.livekit.runtime.call_runtime.log_structured_event", side_effect=capture_event):
result = await runtime.say_stage("resposta com falha no meio", "ARGUMENTATION", message_id="GED-555-0006")
self.assertFalse(result)
speech_commands = [cmd for cmd in executor.commands if cmd.__class__.__name__ == "StartSpeech"]
self.assertEqual(len(speech_commands), 1)
self.assertFalse(runtime.finalized.is_set())
self.assertTrue(first_handle.interrupted)
self.assertEqual(structured_events[0]["erro_msg"], "Falha TTS")
self.assertIn("underflow_error", structured_events[0]["erro_detalhe"])
self.assertIn("underflow_error_ms=1000", structured_events[0]["erro_detalhe"])
self.assertEqual(structured_events[0]["tts_max_gap_ms"], 1200)
self.assertEqual(structured_events[0]["tts_max_underrun_0ms"], 1001)
self.assertEqual(structured_events[0]["tts_underflow_count"], 1)
self.assertEqual(structured_events[0]["tts_avg_underflow_ms"], 1001)
async def test_say_stage_publishes_partial_failure_without_livekit_metrics(self) -> None:
runtime, _agent, executor = self._make_runtime()
first_handle = SimpleNamespace(id="speech-partial-event", interrupted=False)
executor.speech_handles = [first_handle]
structured_events = []
def emit_partial_failure(_command) -> None:
runtime._record_xai_tts_turn_failed(
{
"segment_id": "segment-partial-event",
"reason": "underflow_error",
"max_audio_delta_gap_ms": 920,
"max_playout_underrun_0ms": 1001,
"xai_micro_underflows": 3,
"xai_avg_underrun_ms": 611,
"provider_synthesis_ms": 2450,
"pcm_duration_ms": 860,
}
)
executor.wait_for_playout_callbacks = [emit_partial_failure]
def capture_event(*_args, **kwargs):
structured_events.append(kwargs)
with mock.patch.dict(os.environ, {"TTS_TTFB_METRIC_WAIT_MS": "0"}), mock.patch(
"app.livekit.runtime.call_runtime.log_structured_event",
side_effect=capture_event,
):
result = await runtime.say_stage(
"resposta com falha parcial",
"ARGUMENTATION",
message_id="GED-555-0008",
)
self.assertFalse(result)
self.assertTrue(first_handle.interrupted)
self.assertEqual(len(structured_events), 1)
event = structured_events[0]
self.assertEqual(event["erro_msg"], "Falha TTS")
self.assertIn("underflow_error", event["erro_detalhe"])
self.assertEqual(event["latencia_total_ms"], 2450)
self.assertEqual(event["duracao_audio_ms"], 860)
self.assertEqual(event["tts_max_gap_ms"], 920)
self.assertEqual(event["tts_max_underrun_0ms"], 1001)
self.assertEqual(event["tts_underflow_count"], 3)
self.assertEqual(event["tts_avg_underflow_ms"], 611)
self.assertIsNone(event["http_cod_status"])
async def test_say_stage_returns_to_listening_after_swallowed_tts_error(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.register_callbacks()
first_handle = SimpleNamespace(id="speech-swallowed", interrupted=False)
executor.speech_handles = [first_handle]
def emit_error(_command):
error = SimpleNamespace(type="tts_error", error=RuntimeError("no audio frames were pushed for text: resposta"), recoverable=False)
runtime._session.handlers["error"](SimpleNamespace(error=error))
executor.wait_for_playout_callbacks = [emit_error]
with mock.patch.dict(os.environ, {"TTS_TTFB_METRIC_WAIT_MS": "0"}):
result = await runtime.say_stage("resposta", "PRESENTATION", message_id="GED-555-0007")
self.assertFalse(result)
speech_commands = [cmd for cmd in executor.commands if cmd.__class__.__name__ == "StartSpeech"]
self.assertEqual(len(speech_commands), 1)
self.assertFalse(runtime.finalized.is_set())
async def test_tts_metric_zero_is_not_used_as_ttfb(self) -> None:
runtime, _agent, _executor = self._make_runtime()
runtime._record_tts_metric(
SimpleNamespace(
metrics=SimpleNamespace(
type="tts_metrics",
ttfb=0,
speech_id="speech-zero",
label="provider",
)
)
)
self.assertNotIn("speech-zero", runtime._tts_tffb_ms_by_speech_handle_id)
async def test_finalize_is_idempotent(self) -> None:
runtime, _agent, executor = self._make_runtime()
await asyncio.gather(runtime.finalize("first"), runtime.finalize("second"))
self.assertTrue(runtime.finalized.is_set())
self.assertEqual(sum(isinstance(cmd, EndServiceOnce) for cmd in executor.commands), 1)
self.assertEqual(sum(isinstance(cmd, ExportSession) for cmd in executor.commands), 1)
async def test_run_pipeline_done_triggers_done_notification_and_finalize(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.state.user_final_seq = 1
executor.pipeline_result = BackendReply(stage="DONE", text="encerrando", done=True)
await runtime.run_pipeline("texto", "texto", user_seq=1)
await self._drain_tasks()
self.assertTrue(any(isinstance(cmd, NotifyBridgeDone) for cmd in executor.commands))
self.assertTrue(any(isinstance(cmd, EndServiceOnce) for cmd in executor.commands))
self.assertTrue(any(isinstance(cmd, ExportSession) for cmd in executor.commands))
async def test_run_pipeline_non_terminal_reply_does_not_finalize_from_global_done_stage(
self,
) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.state.user_final_seq = 1
executor.pipeline_result = BackendReply(
stage="PRESENTATION",
text="Perfeito! Seguiremos com o cancelamento.",
done=False,
)
def _mark_global_stage_done(_command) -> None:
runtime.state.current_stage = "DONE"
executor.wait_for_playout_callbacks = [_mark_global_stage_done]
await runtime.run_pipeline("texto", "texto", user_seq=1)
await self._drain_tasks()
self.assertFalse(any(isinstance(cmd, NotifyBridgeDone) for cmd in executor.commands))
self.assertFalse(any(isinstance(cmd, EndServiceOnce) for cmd in executor.commands))
self.assertFalse(runtime.finalized.is_set())
async def test_run_pipeline_agent_final_result_speaks_before_terminal_stop(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 1
executor.pipeline_result = BackendReply(
stage="DONE",
text="Atendimento encerrado com sucesso.",
done=True,
export_payload={
"type": "nao_resolvido",
"content": "Atendimento encerrado com sucesso.",
"tool_calls": [],
},
)
executor.end_reply = BackendReply(
stage="DONE",
text="Nao deveria falar de novo.",
done=True,
export_payload=[{"status": "ok"}],
)
await runtime.run_pipeline("texto", "texto", user_seq=1)
await self._drain_tasks()
speech_commands = [cmd for cmd in executor.commands if cmd.__class__.__name__ == "StartSpeech"]
stop_commands = [cmd for cmd in executor.commands if isinstance(cmd, NotifyBridgeStop)]
self.assertEqual([cmd.text for cmd in speech_commands], ["Atendimento encerrado com sucesso."])
self.assertEqual(len(stop_commands), 1)
self.assertEqual(stop_commands[0].status, "stop_nao_resolvido")
self.assertEqual(stop_commands[0].reason, "nao_resolvido")
self.assertEqual(stop_commands[0].phase, "in_session")
self.assertFalse(any(isinstance(cmd, NotifyBridgeDone) for cmd in executor.commands))
self.assertLess(
executor.commands.index(speech_commands[0]),
executor.commands.index(stop_commands[0]),
)
self.assertTrue(runtime.finalized.is_set())
self.assertFalse(any(isinstance(cmd, EndServiceOnce) for cmd in executor.commands))
self.assertFalse(any(isinstance(cmd, ExportSession) for cmd in executor.commands))
async def test_terminal_reply_waits_until_user_stops_speaking(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 1
runtime._user_not_speaking.clear()
executor.pipeline_result = BackendReply(
stage="DONE",
text="Atendimento encerrado.",
done=True,
export_payload={"type": "nao_resolvido", "content": "Atendimento encerrado."},
)
with mock.patch(
"app.livekit.runtime.call_runtime.POST_USER_FINAL_SPEECH_GRACE_S",
0.01,
):
task = asyncio.create_task(runtime.run_pipeline("texto", "texto", user_seq=1))
await asyncio.sleep(0.01)
self.assertFalse(any(isinstance(cmd, StartSpeech) for cmd in executor.commands))
runtime._user_not_speaking.set()
await task
self.assertEqual(
[cmd.text for cmd in executor.commands if isinstance(cmd, StartSpeech)],
["Atendimento encerrado."],
)
async def test_terminal_reply_is_spoken_when_speaking_becomes_new_user_turn(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 1
runtime._user_not_speaking.clear()
executor.pipeline_result = BackendReply(
stage="DONE",
text="Atendimento encerrado.",
done=True,
export_payload={"type": "nao_resolvido", "content": "Atendimento encerrado."},
)
with mock.patch(
"app.livekit.runtime.call_runtime.POST_USER_FINAL_SPEECH_GRACE_S",
0.01,
):
task = asyncio.create_task(runtime.run_pipeline("texto", "texto", user_seq=1))
await asyncio.sleep(0.01)
runtime._user_not_speaking.set()
runtime.state.user_final_seq = 2
await task
speech_commands = [cmd for cmd in executor.commands if isinstance(cmd, StartSpeech)]
self.assertEqual([cmd.text for cmd in speech_commands], ["Atendimento encerrado."])
self.assertFalse(speech_commands[0].allow_interruptions)
self.assertTrue(any(isinstance(cmd, NotifyBridgeStop) for cmd in executor.commands))
self.assertTrue(runtime.finalized.is_set())
async def test_run_pipeline_skips_new_turn_after_done_stage(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.state.current_stage = "DONE"
runtime.state.user_final_seq = 2
await runtime.run_pipeline("Ok,", "Ok,", user_seq=2)
self.assertFalse(any(isinstance(cmd, RunPipelineInput) for cmd in executor.commands))
self.assertFalse(any(isinstance(cmd, StartSpeech) for cmd in executor.commands))
async def test_run_pipeline_ready_wait_timeout_sends_long_silence_stop(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 0
executor.pipeline_result = BackendReply(
stage="PRESENTATION",
text="Ola, como posso ajudar com sua fatura?",
metadata={"wait_timeout_seconds": 0.01},
)
structured_events = []
def _capture_event(*args, **kwargs):
structured_events.append(kwargs)
with mock.patch("app.livekit.runtime.call_runtime.log_structured_event", side_effect=_capture_event):
await runtime.run_pipeline("", "", user_seq=0)
await self._drain_tasks()
speech_commands = [cmd for cmd in executor.commands if cmd.__class__.__name__ == "StartSpeech"]
stop_commands = [cmd for cmd in executor.commands if isinstance(cmd, NotifyBridgeStop)]
self.assertEqual([cmd.text for cmd in speech_commands], ["Ola, como posso ajudar com sua fatura?"])
self.assertEqual(len(stop_commands), 1)
self.assertEqual(stop_commands[0].status, "stop_silencio_longo")
self.assertEqual(stop_commands[0].reason, "no_user_response")
self.assertEqual(stop_commands[0].phase, "in_session")
self.assertTrue(
any(
event["tipo_evento"] == "recebimento msg"
and event.get("erro_msg") == "Silencio Longo"
for event in structured_events
)
)
self.assertTrue(runtime.finalized.is_set())
self.assertFalse(any(isinstance(cmd, EndServiceOnce) for cmd in executor.commands))
self.assertFalse(any(isinstance(cmd, ExportSession) for cmd in executor.commands))
async def test_run_pipeline_ready_wait_timeout_sends_retry_messages_before_stop(self) -> None:
controller = _FakeVADThresholdController()
runtime, _agent, executor = self._make_runtime(vad_threshold_controller=controller)
runtime._timeline = _FakeTimeline()
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 0
retry_messages = [
"Voce ainda esta na linha?",
"Sigo por aqui.",
"Vou confirmar mais uma vez.",
]
executor.pipeline_result = BackendReply(
stage="PRESENTATION",
text="Ola, como posso ajudar com sua fatura?",
metadata={
"message_id": "MSG-timeout-0001",
"wait_timeout_seconds": 0.01,
"wait_retry_messages": retry_messages,
},
)
structured_events = []
def _capture_structured_event(*args, **kwargs):
structured_events.append(kwargs)
with mock.patch(
"app.livekit.runtime.call_runtime.log_structured_event",
side_effect=_capture_structured_event,
):
await runtime.run_pipeline("", "", user_seq=0)
await self._drain_tasks()
speech_commands = [cmd for cmd in executor.commands if cmd.__class__.__name__ == "StartSpeech"]
stop_commands = [cmd for cmd in executor.commands if isinstance(cmd, NotifyBridgeStop)]
self.assertEqual(
[cmd.text for cmd in speech_commands],
["Ola, como posso ajudar com sua fatura?", *retry_messages],
)
self.assertEqual(
[cmd.add_to_chat_ctx for cmd in speech_commands],
[True, False, False, False],
)
self.assertEqual(
[cmd.allow_interruptions for cmd in speech_commands[1:]],
[True, True, True],
)
self.assertIn(("activate", 1, "agent_wait_timeout_retry"), controller.calls)
self.assertIn(("restore", "agent_wait_timeout:no_user_response"), controller.calls)
event_ids = [event["message_id"] for event in structured_events]
self.assertEqual(
event_ids[:4],
[
"MSG-timeout-0001",
"MSG-timeout-0001_inatividade_1",
"MSG-timeout-0001_inatividade_2",
"MSG-timeout-0001_inatividade_3",
],
)
self.assertEqual(len(event_ids), 5)
self.assertIn("_erro_terminal_recebimento_", event_ids[4])
retry_events = [
fields
for event, fields in runtime._timeline.events
if event == "agent_wait_timeout_retry"
]
self.assertEqual(
[event["message_id"] for event in retry_events],
[
"MSG-timeout-0001_inatividade_1",
"MSG-timeout-0001_inatividade_2",
"MSG-timeout-0001_inatividade_3",
],
)
self.assertEqual(len(stop_commands), 1)
self.assertEqual(stop_commands[0].status, "stop_silencio_longo")
self.assertEqual(stop_commands[0].reason, "no_user_response")
self.assertEqual(stop_commands[0].phase, "in_session")
self.assertLess(
executor.commands.index(speech_commands[-1]),
executor.commands.index(stop_commands[0]),
)
self.assertTrue(runtime.finalized.is_set())
self.assertFalse(any(isinstance(cmd, EndServiceOnce) for cmd in executor.commands))
self.assertFalse(any(isinstance(cmd, ExportSession) for cmd in executor.commands))
async def test_agent_wait_timeout_retry_registers_idle_nudge_event(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._timeline = _FakeTimeline()
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 0
retry_messages = [
"Voce ainda esta na linha?",
"Sigo por aqui.",
"Vou confirmar mais uma vez.",
]
executor.pipeline_result = BackendReply(
stage="PRESENTATION",
text="Ola, como posso ajudar com sua fatura?",
metadata={
"message_id": "MSG-timeout-0001",
"wait_timeout_seconds": 0.01,
"wait_retry_messages": retry_messages,
},
)
await runtime.run_pipeline("", "", user_seq=0)
await self._drain_tasks()
nudge_commands = [cmd for cmd in executor.commands if isinstance(cmd, InjectIdleNudge)]
self.assertEqual([cmd.text for cmd in nudge_commands], retry_messages)
# O evento e bufferizado ANTES da fala, para sobreviver a uma interrupcao
# do cliente em cima do retry.
speech_commands = [cmd for cmd in executor.commands if cmd.__class__.__name__ == "StartSpeech"]
for nudge, speech in zip(nudge_commands, speech_commands[1:]):
self.assertLess(
executor.commands.index(nudge),
executor.commands.index(speech),
)
async def test_agent_wait_timeout_metadata_persists_for_later_replies(self) -> None:
runtime, _agent, _executor = self._make_runtime()
runtime._timeline = _FakeTimeline()
try:
runtime.arm_agent_wait_timeout_from_reply(
BackendReply(
stage="PRESENTATION",
text="Ola, como posso ajudar?",
metadata={
"wait_timeout_seconds": 60.0,
"wait_retry_messages": ["Voce ainda esta na linha?"],
},
),
user_seq=0,
source="initial",
)
runtime.cancel_agent_wait_timeout("user_final")
runtime._clear_agent_wait_timeout_memory()
runtime.state.user_final_seq = 1
runtime.arm_agent_wait_timeout_from_reply(
BackendReply(
stage="PRESENTATION",
text="Pode me responder?",
),
user_seq=1,
source="next_reply",
)
armed_events = [
fields
for event, fields in runtime._timeline.events
if event == "agent_wait_timeout_armed"
]
self.assertEqual(len(armed_events), 2)
self.assertEqual(armed_events[-1]["wait_timeout_s"], 60.0)
self.assertEqual(armed_events[-1]["retry_messages"], 1)
self.assertEqual(armed_events[-1]["source"], "next_reply")
self.assertEqual(armed_events[-1]["user_seq"], 1)
finally:
await self._cancel_pending_tasks()
async def test_agent_wait_timeout_metadata_rewrites_persisted_config(self) -> None:
runtime, _agent, _executor = self._make_runtime()
runtime._timeline = _FakeTimeline()
try:
runtime.arm_agent_wait_timeout_from_reply(
BackendReply(
stage="PRESENTATION",
text="Primeira pergunta",
metadata={
"wait_timeout_seconds": 60.0,
"wait_retry_messages": ["Mensagem antiga"],
},
),
user_seq=0,
source="initial",
)
runtime.cancel_agent_wait_timeout("user_final")
runtime._clear_agent_wait_timeout_memory()
runtime.state.user_final_seq = 1
runtime.arm_agent_wait_timeout_from_reply(
BackendReply(
stage="PRESENTATION",
text="Nova pergunta",
metadata={
"wait_timeout_seconds": 30.0,
"wait_retry_messages": ["Nova mensagem", "Ultimo aviso"],
},
),
user_seq=1,
source="rewrite",
)
runtime.cancel_agent_wait_timeout("user_final")
runtime._clear_agent_wait_timeout_memory()
runtime.state.user_final_seq = 2
runtime.arm_agent_wait_timeout_from_reply(
BackendReply(
stage="PRESENTATION",
text="Pergunta sem metadata",
),
user_seq=2,
source="after_rewrite",
)
armed_events = [
fields
for event, fields in runtime._timeline.events
if event == "agent_wait_timeout_armed"
]
self.assertEqual(len(armed_events), 3)
self.assertEqual(armed_events[-2]["wait_timeout_s"], 30.0)
self.assertEqual(armed_events[-2]["retry_messages"], 2)
self.assertEqual(armed_events[-1]["wait_timeout_s"], 30.0)
self.assertEqual(armed_events[-1]["retry_messages"], 2)
self.assertEqual(armed_events[-1]["source"], "after_rewrite")
self.assertEqual(armed_events[-1]["user_seq"], 2)
finally:
await self._cancel_pending_tasks()
async def test_feedback_reply_does_not_arm_client_wait_timeout(self) -> None:
runtime, _agent, _executor = self._make_runtime()
runtime._timeline = _FakeTimeline()
try:
runtime.arm_agent_wait_timeout_from_reply(
BackendReply(
stage="PRESENTATION",
text="Posso verificar sua fatura.",
metadata={
"agent_message_type": "ready",
"expects_user_response": True,
"drop_user_input_while_speaking": True,
"is_interruptible": False,
"wait_timeout_seconds": 60.0,
"wait_retry_messages": ["Voce esta ai?"],
},
),
user_seq=0,
source="ready",
)
runtime.cancel_agent_wait_timeout("test")
runtime._clear_agent_wait_timeout_memory()
runtime.arm_agent_wait_timeout_from_reply(
BackendReply(
stage="PRESENTATION",
text="Ainda estou consultando sua fatura.",
metadata={
"agent_message_type": "feedback",
"expects_user_response": False,
"drop_user_input_while_speaking": True,
"is_interruptible": False,
"event": "feedback",
},
),
user_seq=0,
source="feedback",
)
armed_events = [
fields
for event, fields in runtime._timeline.events
if event == "agent_wait_timeout_armed"
]
self.assertEqual(len(armed_events), 1)
runtime.arm_agent_wait_timeout_from_reply(
BackendReply(
stage="PRESENTATION",
text="Consegui consultar. Posso te ajudar em mais algo?",
metadata={
"agent_message_type": "result",
"agent_result_type": "final",
"expects_user_response": True,
"drop_user_input_while_speaking": False,
},
),
user_seq=0,
source="final",
)
armed_events = [
fields
for event, fields in runtime._timeline.events
if event == "agent_wait_timeout_armed"
]
self.assertEqual(len(armed_events), 2)
self.assertEqual(armed_events[-1]["source"], "final")
self.assertEqual(armed_events[-1]["retry_messages"], 1)
finally:
await self._cancel_pending_tasks()
async def test_agent_wait_timeout_zero_metadata_clears_persisted_config(self) -> None:
runtime, _agent, _executor = self._make_runtime()
runtime._timeline = _FakeTimeline()
try:
runtime.arm_agent_wait_timeout_from_reply(
BackendReply(
stage="PRESENTATION",
text="Primeira pergunta",
metadata={"wait_timeout_seconds": 60.0},
),
user_seq=0,
source="initial",
)
runtime.arm_agent_wait_timeout_from_reply(
BackendReply(
stage="PRESENTATION",
text="Sem timeout daqui em diante",
metadata={"wait_timeout_seconds": 0},
),
user_seq=0,
source="disable",
)
runtime.state.user_final_seq = 1
runtime.arm_agent_wait_timeout_from_reply(
BackendReply(
stage="PRESENTATION",
text="Pergunta sem metadata",
),
user_seq=1,
source="after_disable",
)
armed_events = [
fields
for event, fields in runtime._timeline.events
if event == "agent_wait_timeout_armed"
]
self.assertEqual(len(armed_events), 1)
finally:
await self._cancel_pending_tasks()
async def test_run_pipeline_backend_failure_triggers_terminal_stop_and_finalize(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.state.user_final_seq = 1
async def _fail_backend() -> None:
raise RuntimeError("backend closed connection")
executor.run_pipeline_side_effect = _fail_backend
structured_events = []
def _capture_event(*args, **kwargs):
structured_events.append(kwargs)
with mock.patch("app.livekit.runtime.call_runtime.log_structured_event", side_effect=_capture_event):
await runtime.run_pipeline("texto", "texto", user_seq=1)
stop_commands = [cmd for cmd in executor.commands if isinstance(cmd, NotifyBridgeStop)]
self.assertEqual(len(stop_commands), 1)
self.assertEqual(stop_commands[0].status, "stop_agent_backend_unavailable")
self.assertEqual(stop_commands[0].reason, "resource_unhealthy")
self.assertEqual(stop_commands[0].resource, "agent_backend")
self.assertEqual(stop_commands[0].failed_resources, ("agent_backend",))
self.assertTrue(
any(
event["tipo_evento"] == "envio msg"
and event.get("erro_msg") == "Falha comunicacao"
and "_erro_terminal_envio_" in event.get("message_id", "")
for event in structured_events
)
)
self.assertTrue(any(isinstance(cmd, EndServiceOnce) for cmd in executor.commands))
self.assertTrue(any(isinstance(cmd, ExportSession) for cmd in executor.commands))
async def test_run_pipeline_transfer_reply_uses_suffixed_message_id(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.state.user_final_seq = 1
executor.pipeline_result = BackendReply(
stage="PRESENTATION",
text="",
done=False,
export_payload={
"type": "transferred",
"status": "transferred",
"additionalInformations": {"changeAgentId": "ATH-001"},
},
)
structured_events = []
def _capture_event(*args, **kwargs):
structured_events.append(kwargs)
with mock.patch("app.livekit.runtime.call_runtime.log_structured_event", side_effect=_capture_event):
await runtime.run_pipeline(
"texto",
"texto",
user_seq=1,
message_id="GED-test-0002",
)
self.assertTrue(
any(
event["tipo_evento"] == "envio msg"
and event.get("erro_msg") == "Transferido"
and event.get("message_id", "").startswith("GED-test-0002_transferencia_")
for event in structured_events
)
)
async def test_finalize_speaks_end_reply_when_remote_listener_exists(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._ctx.room.remote_participants = {"user-1": object()}
executor.end_reply = BackendReply(
stage="DONE",
text="Mensagem final do backend.",
done=True,
export_payload=[{"status": "ok"}],
)
await runtime.finalize("manual")
self.assertTrue(any(cmd.__class__.__name__ == "StartSpeech" for cmd in executor.commands))
self.assertTrue(any(isinstance(cmd, ExportSession) for cmd in executor.commands))
async def test_finalize_skips_end_reply_tts_without_remote_listener(self) -> None:
runtime, _agent, executor = self._make_runtime()
executor.end_reply = BackendReply(
stage="DONE",
text="Mensagem final do backend.",
done=True,
export_payload=[{"status": "ok"}],
)
await runtime.finalize("manual")
self.assertFalse(any(cmd.__class__.__name__ == "StartSpeech" for cmd in executor.commands))
self.assertTrue(any(isinstance(cmd, ExportSession) for cmd in executor.commands))
async def test_register_callbacks_wires_session_and_shutdown(self) -> None:
runtime, _agent, _executor = self._make_runtime()
runtime.register_callbacks()
self.assertIn("user_input_transcribed", runtime._session.handlers)
self.assertIn("close", runtime._session.handlers)
self.assertIn("data_received", runtime._ctx.room.handlers)
self.assertEqual(len(runtime._ctx.shutdown_callbacks), 1)
async def test_session_close_error_triggers_resource_stop(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime.register_callbacks()
runtime._session.handlers["close"](
SimpleNamespace(
reason=SimpleNamespace(value="error"),
error=SimpleNamespace(type="stt_error"),
)
)
await self._drain_tasks()
stop_commands = [cmd for cmd in executor.commands if isinstance(cmd, NotifyBridgeStop)]
self.assertEqual(len(stop_commands), 1)
self.assertEqual(stop_commands[0].status, "stop_stt_unavailable")
self.assertEqual(stop_commands[0].resource, "stt")
async def test_bridge_control_client_audio_enabled_triggers_initial_agent_turn(self) -> None:
runtime, _agent, executor = self._make_runtime(agent_starts_conversation=True)
runtime._ctx.room.remote_participants = {"user-1": object()}
executor.pipeline_result = BackendReply(stage="PRESENTATION", text="Ola, sou o agent.")
runtime.register_callbacks()
runtime._ctx.room.handlers["data_received"](
SimpleNamespace(
topic="bridge.control",
data=b'{"type":"client_audio_enabled","room":"room-test","protocol":"PRT-1"}',
)
)
await self._drain_tasks()
run_commands = [cmd for cmd in executor.commands if isinstance(cmd, RunPipelineInput)]
self.assertEqual(len(run_commands), 1)
self.assertEqual(run_commands[0].user_input, "")
self.assertTrue(any(cmd.__class__.__name__ == "StartSpeech" for cmd in executor.commands))
async def test_run_holds_user_audio_input_before_starting_session(self) -> None:
runtime, _agent, executor = self._make_runtime(agent_starts_conversation=True)
runtime._ctx.room.remote_participants = {"user-1": object()}
audio_enabled_on_start = []
executor.on_start_session = lambda _cmd: audio_enabled_on_start.append(
runtime._session.input.audio_enabled
)
await runtime.run()
await asyncio.sleep(0)
await self._cancel_pending_tasks()
# O RoomIO le o estado do input ao anexar a track: se o hold viesse depois
# do StartSession, o audio do setup ja teria chegado no VAD/STT.
self.assertEqual(audio_enabled_on_start, [False])
self.assertFalse(runtime._session.input.audio_enabled)
self.assertEqual(runtime._session.input.calls, [False])
async def test_run_keeps_user_audio_input_when_user_starts_conversation(self) -> None:
runtime, _agent, _executor = self._make_runtime(agent_starts_conversation=False)
runtime._ctx.room.remote_participants = {"user-1": object()}
await runtime.run()
await asyncio.sleep(0)
await self._cancel_pending_tasks()
self.assertTrue(runtime._session.input.audio_enabled)
self.assertEqual(runtime._session.input.calls, [])
async def test_initial_agent_turn_releases_user_audio_input_after_playout(self) -> None:
runtime, _agent, executor = self._make_runtime(agent_starts_conversation=True)
runtime._ctx.room.remote_participants = {"user-1": object()}
executor.pipeline_result = BackendReply(stage="PRESENTATION", text="Ola, sou o agent.")
audio_enabled_during_playout = []
executor.wait_for_playout_callbacks = [
lambda _cmd: audio_enabled_during_playout.append(
runtime._session.input.audio_enabled
)
]
runtime.hold_user_audio_input(reason="room_setup")
runtime.register_callbacks()
runtime._ctx.room.handlers["data_received"](
SimpleNamespace(
topic="bridge.control",
data=b'{"type":"client_audio_enabled","room":"room-test","protocol":"PRT-1"}',
)
)
await self._drain_tasks()
self.assertEqual(audio_enabled_during_playout, [False])
self.assertTrue(runtime._session.input.audio_enabled)
self.assertEqual(runtime._session.input.calls, [False, True])
async def test_initial_agent_turn_releases_user_audio_input_when_pipeline_fails(self) -> None:
runtime, _agent, executor = self._make_runtime(agent_starts_conversation=True)
runtime._ctx.room.remote_participants = {"user-1": object()}
async def _boom() -> None:
raise RuntimeError("backend fora do ar")
executor.run_pipeline_side_effect = _boom
runtime.hold_user_audio_input(reason="room_setup")
runtime.register_callbacks()
runtime._ctx.room.handlers["data_received"](
SimpleNamespace(
topic="bridge.control",
data=b'{"type":"client_audio_enabled","room":"room-test","protocol":"PRT-1"}',
)
)
await self._drain_tasks()
# Falha no turno inicial nao pode deixar a chamada surda.
self.assertTrue(runtime._session.input.audio_enabled)
async def test_user_audio_input_gate_timeout_releases_when_initial_turn_never_starts(
self,
) -> None:
runtime, _agent, _executor = self._make_runtime(agent_starts_conversation=True)
runtime._ctx.room.remote_participants = {"user-1": object()}
with mock.patch.dict(os.environ, {"USER_AUDIO_INPUT_SETUP_TIMEOUT_S": "0.01"}):
runtime.hold_user_audio_input(reason="room_setup")
runtime._arm_user_audio_input_gate_timeout()
await self._drain_tasks()
# client_audio_enabled nunca chegou: o gate abre sozinho.
self.assertTrue(runtime._session.input.audio_enabled)
self.assertEqual(runtime._session.input.calls, [False, True])
async def test_backend_push_message_is_spoken_without_new_user_turn(self) -> None:
runtime, _agent, executor = self._make_runtime(
agent_starts_conversation=True,
push_replies=[BackendReply(stage="PRESENTATION", text="Segunda mensagem do backend.")],
)
runtime._ctx.room.remote_participants = {"user-1": object()}
executor.pipeline_result = BackendReply(stage="PRESENTATION", text="Primeira mensagem do backend.")
runtime.register_callbacks()
runtime._ctx.room.handlers["data_received"](
SimpleNamespace(
topic="bridge.control",
data=b'{"type":"client_audio_enabled","room":"room-test","protocol":"PRT-1"}',
)
)
await self._drain_tasks()
speech_texts = [
cmd.text
for cmd in executor.commands
if cmd.__class__.__name__ == "StartSpeech"
]
self.assertEqual(
speech_texts,
[
"Primeira mensagem do backend.",
"Segunda mensagem do backend.",
],
)
async def test_inflight_backend_feedback_is_spoken_before_final_reply(self) -> None:
runtime, agent, executor = self._make_runtime()
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 1
pipeline = _FakeInflightPushPipeline()
agent.pipeline = pipeline
executor.pipeline_result = BackendReply(
stage="PRESENTATION",
text="Resposta final do backend.",
)
async def _emit_feedback_then_finish() -> None:
await pipeline.push(
BackendReply(
stage="PRESENTATION",
text="Ainda estou consultando sua fatura.",
metadata={
"event": "feedback",
"agent_message_type": "feedback",
"expects_user_response": False,
"drop_user_input_while_speaking": True,
"is_interruptible": False,
},
)
)
await asyncio.sleep(0.01)
pipeline.close()
executor.run_pipeline_side_effect = _emit_feedback_then_finish
await runtime.run_pipeline("texto", "texto", user_seq=1)
await self._drain_tasks()
speech_texts = [
cmd.text
for cmd in executor.commands
if cmd.__class__.__name__ == "StartSpeech"
]
self.assertEqual(
speech_texts,
[
"Ainda estou consultando sua fatura.",
"Resposta final do backend.",
],
)
speech_commands = [
cmd
for cmd in executor.commands
if cmd.__class__.__name__ == "StartSpeech"
]
self.assertFalse(speech_commands[0].allow_interruptions)
async def test_inflight_backend_wait_notice_is_spoken_immediately_after_stt_final(self) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
runtime, agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 60.0,
"inflight_backend_wait_timeout_s": 0.01,
"inflight_backend_wait_max_notices": 1,
"inflight_backend_wait_text": wait_text,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 1
agent.pipeline = _FakeInflightPushPipeline()
never_finishes = asyncio.Event()
async def _hang_backend() -> None:
await never_finishes.wait()
executor.run_pipeline_side_effect = _hang_backend
await runtime.run_pipeline("texto", "texto", user_seq=1)
await self._drain_tasks()
speech_texts = [
cmd.text
for cmd in executor.commands
if cmd.__class__.__name__ == "StartSpeech"
]
self.assertEqual(speech_texts, [_expected_wait_text_for_audio(WAIT_LONG_AUDIO_PATH, wait_text)])
async def test_inflight_backend_wait_audio_sequence_starts_short_then_long(self) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
with TemporaryDirectory() as tmp_dir:
base_dir = Path(tmp_dir)
short_dir = base_dir / "short"
long_dir = base_dir / "long"
short_dir.mkdir()
long_dir.mkdir()
short_audio = short_dir / "curto.wav"
long_audio = long_dir / "longo.wav"
short_audio.write_bytes(b"short")
long_audio.write_bytes(b"long")
runtime, agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 0.005,
"inflight_backend_wait_timeout_s": 0.6,
"inflight_backend_wait_max_notices": 2,
"inflight_backend_wait_text": wait_text,
"inflight_backend_wait_short_audio_dir": str(short_dir),
"inflight_backend_wait_long_audio_dir": str(long_dir),
}
)
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 1
runtime._timeline = _FakeTimeline()
agent.pipeline = _FakeInflightPushPipeline()
never_finishes = asyncio.Event()
async def _hang_backend() -> None:
await never_finishes.wait()
def _audio_frames(path: str):
return f"audio:{Path(path).name}"
executor.run_pipeline_side_effect = _hang_backend
with (
mock.patch("app.livekit.runtime.call_runtime.wav_duration_ms", return_value=100),
mock.patch(
"app.livekit.runtime.call_runtime.wav_audio_frames",
side_effect=_audio_frames,
),
mock.patch.object(runtime, "_log_resource_error_events") as structured_error,
):
await runtime.run_pipeline("texto", "texto", user_seq=1)
await self._drain_tasks()
speech_commands = [
cmd
for cmd in executor.commands
if cmd.__class__.__name__ == "StartSpeech"
]
self.assertEqual(
[cmd.audio for cmd in speech_commands],
["audio:curto.wav", "audio:longo.wav"],
)
max_notice_events = [
fields
for event, fields in runtime._timeline.events
if event == "inflight_backend_wait_max_notices_reached"
]
self.assertEqual(len(max_notice_events), 1)
self.assertEqual(max_notice_events[0]["notices_sent"], 2)
self.assertEqual(max_notice_events[0]["max_notices"], 2)
max_notice_structured_calls = [
call
for call in structured_error.call_args_list
if call.kwargs.get("reason")
== "inflight_backend_wait_max_notices_reached"
]
self.assertEqual(len(max_notice_structured_calls), 1)
self.assertEqual(
max_notice_structured_calls[0].kwargs["resource"],
"agent_backend",
)
async def test_inflight_backend_wait_zero_max_notices_is_unlimited(self) -> None:
runtime, agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 0.005,
"inflight_backend_wait_timeout_s": 0.2,
"inflight_backend_wait_max_notices": 0,
"inflight_backend_wait_text": DEFAULT_WAIT_TEXT,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 1
runtime._timeline = _FakeTimeline()
agent.pipeline = _FakeInflightPushPipeline()
backend_done = asyncio.Event()
attempts: list[int] = []
async def _hang_backend() -> None:
await backend_done.wait()
async def _play_notice(**kwargs):
attempts.append(kwargs["attempt"])
if len(attempts) == 3:
backend_done.set()
return InflightBackendWaitNoticeResult(started=True)
executor.run_pipeline_side_effect = _hang_backend
with mock.patch.object(
runtime,
"_play_inflight_backend_wait_notice_audio",
side_effect=_play_notice,
):
await runtime.run_pipeline("texto", "texto", user_seq=1)
self.assertEqual(attempts, [1, 2, 3])
self.assertFalse(
any(
event == "inflight_backend_wait_max_notices_reached"
for event, _fields in runtime._timeline.events
)
)
async def test_inflight_backend_wait_skipped_notice_does_not_consume_attempt(self) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
with TemporaryDirectory() as tmp_dir:
base_dir = Path(tmp_dir)
short_dir = base_dir / "short"
short_dir.mkdir()
short_audio = short_dir / "curto.wav"
short_audio.write_bytes(b"short")
runtime, agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 0.005,
"inflight_backend_wait_timeout_s": 0.06,
"inflight_backend_wait_max_notices": 1,
"inflight_backend_wait_text": wait_text,
"inflight_backend_wait_short_audio_dir": str(short_dir),
}
)
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 1
runtime._user_not_speaking.clear()
agent.pipeline = _FakeInflightPushPipeline()
never_finishes = asyncio.Event()
async def _hang_backend() -> None:
await never_finishes.wait()
async def _mark_user_not_speaking() -> None:
await asyncio.sleep(0.015)
runtime._user_not_speaking.set()
def _audio_frames(path: str):
return f"audio:{Path(path).name}"
executor.run_pipeline_side_effect = _hang_backend
restore_task = asyncio.create_task(_mark_user_not_speaking())
try:
with (
mock.patch("app.livekit.runtime.call_runtime.wav_duration_ms", return_value=100),
mock.patch(
"app.livekit.runtime.call_runtime.wav_audio_frames",
side_effect=_audio_frames,
),
):
await runtime.run_pipeline("texto", "texto", user_seq=1)
finally:
restore_task.cancel()
await asyncio.gather(restore_task, return_exceptions=True)
await self._drain_tasks()
speech_commands = [
cmd
for cmd in executor.commands
if cmd.__class__.__name__ == "StartSpeech"
]
self.assertEqual([cmd.audio for cmd in speech_commands], ["audio:curto.wav"])
async def test_inflight_backend_wait_interval_counts_after_audio_playout(self) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
runtime, agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 0.025,
"inflight_backend_wait_timeout_s": 0.04,
"inflight_backend_wait_max_notices": 6,
"inflight_backend_wait_text": wait_text,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 1
agent.pipeline = _FakeInflightPushPipeline()
executor.wait_for_playout_delay_s = 0.02
never_finishes = asyncio.Event()
async def _hang_backend() -> None:
await never_finishes.wait()
executor.run_pipeline_side_effect = _hang_backend
await runtime.run_pipeline("texto", "texto", user_seq=1)
await self._drain_tasks()
speech_texts = [
cmd.text
for cmd in executor.commands
if cmd.__class__.__name__ == "StartSpeech"
]
self.assertEqual(speech_texts, [_expected_wait_text_for_audio(WAIT_LONG_AUDIO_PATH, wait_text)])
async def test_short_processing_interruption_rearms_periodic_notice_deadline(self) -> None:
interval_s = 0.05
runtime, agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": interval_s,
"inflight_backend_wait_timeout_s": 0.3,
"inflight_backend_wait_max_notices": 2,
"inflight_backend_wait_text": DEFAULT_WAIT_TEXT,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 1
agent.pipeline = _FakeInflightPushPipeline()
backend_done = asyncio.Event()
attempts: list[int] = []
attempted_at: list[float] = []
async def _hang_backend() -> None:
await backend_done.wait()
async def _finish_short_speech() -> None:
await asyncio.sleep(0.01)
runtime.note_vad_speech_end(300)
runtime._user_not_speaking.set()
async def _play_notice(**kwargs):
attempt = kwargs["attempt"]
attempts.append(attempt)
attempted_at.append(time.monotonic())
if attempts == [1]:
return InflightBackendWaitNoticeResult(started=True)
if attempts == [1, 2]:
runtime._user_not_speaking.clear()
asyncio.create_task(_finish_short_speech())
return InflightBackendWaitNoticeResult(
started=True,
interrupted_by_user=True,
)
backend_done.set()
return InflightBackendWaitNoticeResult(started=True)
executor.run_pipeline_side_effect = _hang_backend
with mock.patch.object(
runtime,
"_play_inflight_backend_wait_notice_audio",
side_effect=_play_notice,
):
await runtime.run_pipeline("texto", "texto", user_seq=1)
self.assertEqual(attempts, [1, 2, 2])
self.assertGreaterEqual(attempted_at[2] - attempted_at[1], interval_s)
async def test_short_speech_during_silent_backend_wait_does_not_rearm_notice_deadline(self) -> None:
interval_s = 0.05
runtime, agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": interval_s,
"inflight_backend_wait_timeout_s": 0.3,
"inflight_backend_wait_max_notices": 2,
"inflight_backend_wait_text": DEFAULT_WAIT_TEXT,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 1
agent.pipeline = _FakeInflightPushPipeline()
backend_done = asyncio.Event()
attempts: list[int] = []
attempted_at: list[float] = []
short_speech_ended_at = 0.0
async def _hang_backend() -> None:
await backend_done.wait()
async def _emit_short_speech_during_wait() -> None:
nonlocal short_speech_ended_at
await asyncio.sleep(interval_s / 2)
runtime.note_vad_speech_end(300)
short_speech_ended_at = time.monotonic()
async def _play_notice(**kwargs):
attempts.append(kwargs["attempt"])
attempted_at.append(time.monotonic())
if attempts == [1]:
asyncio.create_task(_emit_short_speech_during_wait())
else:
backend_done.set()
return InflightBackendWaitNoticeResult(started=True)
executor.run_pipeline_side_effect = _hang_backend
with mock.patch.object(
runtime,
"_play_inflight_backend_wait_notice_audio",
side_effect=_play_notice,
):
await runtime.run_pipeline("texto", "texto", user_seq=1)
self.assertEqual(attempts, [1, 2])
self.assertGreater(short_speech_ended_at, 0.0)
self.assertLess(attempted_at[1] - short_speech_ended_at, interval_s)
async def test_valid_processing_interruption_rearms_periodic_notice_deadline(self) -> None:
interval_s = 0.04
runtime, agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": interval_s,
"inflight_backend_wait_timeout_s": 0.3,
"inflight_backend_wait_max_notices": 2,
"inflight_backend_wait_text": DEFAULT_WAIT_TEXT,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 1
agent.pipeline = _FakeInflightPushPipeline()
backend_done = asyncio.Event()
attempts: list[int] = []
attempted_at: list[float] = []
async def _hang_backend() -> None:
await backend_done.wait()
async def _finish_valid_speech() -> None:
await asyncio.sleep(0.01)
runtime.note_vad_speech_end(1000)
runtime._user_not_speaking.set()
async def _play_notice(**kwargs):
attempt = kwargs["attempt"]
attempts.append(attempt)
attempted_at.append(time.monotonic())
if attempts == [1]:
return InflightBackendWaitNoticeResult(started=True)
if attempts == [1, 2]:
runtime._user_not_speaking.clear()
asyncio.create_task(_finish_valid_speech())
return InflightBackendWaitNoticeResult(
started=True,
interrupted_by_user=True,
)
backend_done.set()
return InflightBackendWaitNoticeResult(started=True)
executor.run_pipeline_side_effect = _hang_backend
with (
mock.patch.object(
runtime,
"_play_inflight_backend_wait_notice_audio",
side_effect=_play_notice,
),
mock.patch.object(runtime, "_play_deferred_interruption_comfort"),
):
await runtime.run_pipeline("texto", "texto", user_seq=1)
self.assertEqual(attempts, [1, 2, 2])
self.assertGreaterEqual(
attempted_at[2] - attempted_at[1],
interval_s,
)
async def test_long_processing_interruption_keeps_periodic_notices_active(self) -> None:
runtime, agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 0.01,
"inflight_backend_wait_timeout_s": 0.2,
"inflight_backend_wait_max_notices": 2,
"inflight_backend_wait_text": DEFAULT_WAIT_TEXT,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 1
runtime.state.deferred_interruption.backend_in_flight = True
agent.pipeline = _FakeInflightPushPipeline()
backend_done = asyncio.Event()
attempts: list[int] = []
async def _hang_backend() -> None:
await backend_done.wait()
async def _interrupt_after_first_notice() -> None:
await asyncio.sleep(0)
runtime.note_vad_speech_end(1500)
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="e nao resolve", is_final=True)
)
async def _play_notice(**kwargs):
attempts.append(kwargs["attempt"])
if attempts == [1]:
asyncio.create_task(_interrupt_after_first_notice())
else:
backend_done.set()
return InflightBackendWaitNoticeResult(started=True)
executor.run_pipeline_side_effect = _hang_backend
with (
mock.patch.object(
runtime,
"_play_inflight_backend_wait_notice_audio",
side_effect=_play_notice,
),
mock.patch.object(runtime, "_play_deferred_interruption_comfort"),
):
await runtime._execute_pipeline_with_inflight_backend_wait(
"pedido original",
user_seq=1,
)
self.assertEqual(runtime.state.user_final_seq, 2)
self.assertEqual(
[turn.text for turn in runtime.state.deferred_interruption.long_turns],
["e nao resolve"],
)
self.assertEqual(attempts, [1, 2])
async def test_long_processing_interruption_does_not_disable_backend_timeout(self) -> None:
runtime, agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 60.0,
"inflight_backend_wait_timeout_s": 0.03,
"inflight_backend_wait_max_notices": 1,
"inflight_backend_wait_text": DEFAULT_WAIT_TEXT,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 1
runtime.state.deferred_interruption.backend_in_flight = True
agent.pipeline = _FakeInflightPushPipeline()
never_finishes = asyncio.Event()
async def _hang_backend() -> None:
await never_finishes.wait()
async def _play_notice(**_kwargs):
runtime.note_vad_speech_end(1500)
runtime.on_user_input_transcribed(
SimpleNamespace(transcript="continua demorando", is_final=True)
)
return InflightBackendWaitNoticeResult(started=True)
executor.run_pipeline_side_effect = _hang_backend
with (
mock.patch.object(
runtime,
"_play_inflight_backend_wait_notice_audio",
side_effect=_play_notice,
),
mock.patch.object(runtime, "_play_deferred_interruption_comfort"),
):
with self.assertRaises(InflightBackendWaitTimedOut):
await runtime._execute_pipeline_with_inflight_backend_wait(
"pedido original",
user_seq=1,
)
self.assertEqual(runtime.state.user_final_seq, 2)
self.assertTrue(runtime.finalized.is_set())
async def test_inflight_backend_wait_notice_then_backend_unavailable_stop(self) -> None:
wait_text = "Um momento, ainda estou consultando para te ajudar."
runtime, agent, executor = self._make_runtime(
runtime_config_overrides={
"inflight_backend_wait_interval_s": 0.01,
"inflight_backend_wait_timeout_s": 0.035,
"inflight_backend_wait_max_notices": 6,
"inflight_backend_wait_text": wait_text,
"inflight_backend_wait_long_audio_path": WAIT_LONG_AUDIO_PATH,
}
)
runtime._ctx.room.remote_participants = {"user-1": object()}
runtime.state.user_final_seq = 1
agent.pipeline = _FakeInflightPushPipeline()
never_finishes = asyncio.Event()
async def _hang_backend() -> None:
await never_finishes.wait()
executor.run_pipeline_side_effect = _hang_backend
structured_events = []
def _capture_event(*args, **kwargs):
structured_events.append(kwargs)
with mock.patch("app.livekit.runtime.call_runtime.log_structured_event", side_effect=_capture_event):
await runtime.run_pipeline(
"texto",
"texto",
user_seq=1,
message_id="GED-test-0002",
)
await self._drain_tasks()
speech_texts = [
cmd.text
for cmd in executor.commands
if cmd.__class__.__name__ == "StartSpeech"
]
stop_commands = [cmd for cmd in executor.commands if isinstance(cmd, NotifyBridgeStop)]
self.assertGreaterEqual(len(speech_texts), 1)
self.assertTrue(
all(
text == _expected_wait_text_for_audio(WAIT_LONG_AUDIO_PATH, wait_text)
for text in speech_texts
)
)
comfort_events = [
event
for event in structured_events
if str(event.get("message_id") or "").startswith("GED-test-0002_conforto_")
]
self.assertGreaterEqual(len(comfort_events), 1)
self.assertTrue(
any(
event["tipo_evento"] == "envio msg"
and event.get("erro_msg") == "Falha comunicacao"
and "_erro_terminal_envio_" in event.get("message_id", "")
for event in structured_events
)
)
self.assertEqual(len(stop_commands), 1)
self.assertEqual(stop_commands[0].status, "stop_agent_backend_unavailable")
self.assertEqual(stop_commands[0].reason, "resource_unhealthy")
self.assertEqual(stop_commands[0].resource, "agent_backend")
self.assertEqual(stop_commands[0].failed_resources, ("agent_backend",))
self.assertTrue(runtime.finalized.is_set())
async def test_run_starts_session_with_bridge_identity(self) -> None:
runtime, _agent, executor = self._make_runtime()
runtime._ctx.room.remote_participants = {"user-1": object()}
await runtime.run()
await asyncio.sleep(0)
await self._cancel_pending_tasks()
start_commands = [cmd for cmd in executor.commands if isinstance(cmd, StartSession)]
self.assertEqual(len(start_commands), 1)
room_options = start_commands[0].room_options
participant_identity = getattr(room_options, "participant_identity", None)
if participant_identity is None:
participant_identity = room_options.kwargs["participant_identity"]
self.assertEqual(participant_identity, "bridge-1")
self.assertIn("user_input_transcribed", runtime._session.handlers)
if __name__ == "__main__":
unittest.main()
#só pra fazer um novo PR