From f153d7fee78a3bf656320630abd7e95679266930 Mon Sep 17 00:00:00 2001 From: Andriamanampisoa Date: Tue, 14 Jul 2026 15:02:57 +0200 Subject: [PATCH 01/16] feat: AI is now proactive during the interview --- .../simulation/dto/appendAssistantTurn.dto.ts | 7 ++++ .../simulation/simulation-context.service.ts | 40 ++++++++++++++++++- .../simulation-internal.controller.ts | 13 ++++++ .../modules/simulation/simulation.module.ts | 4 +- 4 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 server/src/modules/simulation/dto/appendAssistantTurn.dto.ts diff --git a/server/src/modules/simulation/dto/appendAssistantTurn.dto.ts b/server/src/modules/simulation/dto/appendAssistantTurn.dto.ts new file mode 100644 index 00000000..90b03572 --- /dev/null +++ b/server/src/modules/simulation/dto/appendAssistantTurn.dto.ts @@ -0,0 +1,7 @@ +import { IsString, MinLength } from "class-validator"; + +export class AppendAssistantTurnDto { + @IsString() + @MinLength(1) + assistantText: string; +} diff --git a/server/src/modules/simulation/simulation-context.service.ts b/server/src/modules/simulation/simulation-context.service.ts index f68d1d9d..83ffaa3c 100644 --- a/server/src/modules/simulation/simulation-context.service.ts +++ b/server/src/modules/simulation/simulation-context.service.ts @@ -21,7 +21,22 @@ export type SimulationSessionContext = { history: SimulationChatTurn[]; }; -const BASE_RECRUITER_PERSONA = `Tu es Sophie Martin, recruteuse senior IT chez une ESN francaise. Tu es chaleureuse, professionnelle, patiente et humaine. Tu parles de facon naturelle comme dans une vraie conversation. Tu dois repondre uniquement a la derniere prise de parole du candidat, en une seule reponse courte et naturelle. N'ecris jamais un dialogue multi-tours, n'imite jamais des balises comme system: ou user:, et ne recopie jamais l'historique de conversation.`; +const BASE_RECRUITER_PERSONA = `Tu es Sophie Martin, recruteuse senior IT chez une ESN francaise. Tu conduis un entretien d'embauche professionnel en visioconference. Tu es la meneuse de l'entretien : c'est TOI qui guides la conversation, poses les questions, fais les transitions entre les themes et conclus l'echange. Ne laisse jamais le candidat diriger seul l'entretien. + +Style : chaleureuse, professionnelle, humaine et naturelle. Reponses orales concises (2 a 4 phrases en general, un peu plus pour l'accueil initial). Une seule prise de parole a la fois — ne simule pas plusieurs tours. N'ecris jamais de balises (system:, user:, assistant:) ni un dialogue multi-tours. Utilise l'historique pour rester coherent : reprends le prenom et les elements deja mentionnes, ne repose pas une question deja posee, ne contredis pas le candidat. + +Structure de l'entretien (~15-20 minutes) — respecte cet ordre strict : +1. Accueil : salutations, remerciements, presentation de l'entreprise et du poste +2. Presentation du candidat : invite-le a se presenter AVANT toute question sur l'experience +3. Parcours : experiences passees, formations, evolutions de carriere +4. Competences : questions techniques ou metier selon le type de simulation +5. Motivation : pourquoi ce poste, projet professionnel, soft skills +6. Echange : propose au candidat de poser ses questions +7. Cloture : remerciements, prochaines etapes, au revoir chaleureux + +Comportement proactif : pose TOUJOURS une question ou annonce clairement la prochaine etape a la fin de chaque reponse. Transitionne activement entre les phases. Rebondis sur les reponses du candidat avant d'enchaîner. Ne reponds jamais par un simple accord sans question de suivi. + +Regles sur le prenom du candidat : n'utilise JAMAIS de placeholder entre crochets (ex. [Prenom du candidat], [Nom]). Si le prenom n'est pas explicitement connu dans le contexte, dis simplement « Bonjour » sans nom — le candidat se presentera ensuite.`; @Injectable() export class SimulationContextService { @@ -107,6 +122,29 @@ export class SimulationContextService { await this.capacity.touchHeartbeat(interviewId, ctx.userId); } + async appendAssistantTurn( + interviewId: string, + assistantText: string, + ): Promise { + const ctx = await this.getContextForSts(interviewId); + const { contextTtlSec, historyMaxTurns } = loadSimulationConfig(); + + ctx.history.push({ role: "assistant", content: assistantText }); + + if (ctx.history.length > historyMaxTurns * 2) { + ctx.history = ctx.history.slice(-historyMaxTurns * 2); + } + + await this.redis.set( + SimRedisKeys.context(interviewId), + JSON.stringify(ctx), + "EX", + contextTtlSec, + ); + + await this.capacity.touchHeartbeat(interviewId, ctx.userId); + } + async deleteContext(interviewId: string): Promise { await this.redis.del(SimRedisKeys.context(interviewId)); } diff --git a/server/src/modules/simulation/simulation-internal.controller.ts b/server/src/modules/simulation/simulation-internal.controller.ts index cc46c011..3c62b5c9 100644 --- a/server/src/modules/simulation/simulation-internal.controller.ts +++ b/server/src/modules/simulation/simulation-internal.controller.ts @@ -7,6 +7,7 @@ import { InternalApiKeyGuard } from "./guards/internal-api-key.guard"; import { SimulationContextService } from "./simulation-context.service"; import { SimulationVerbalAnalysisService } from "./simulation-verbal-analysis.service"; import { AppendSimulationHistoryDto } from "./dto/appendSimulationHistory.dto"; +import { AppendAssistantTurnDto } from "./dto/appendAssistantTurn.dto"; import { SaveVerbalAnalysisDto } from "./dto/saveVerbalAnalysis.dto"; @ApiExcludeController() @@ -36,6 +37,18 @@ export class SimulationInternalController { ); } + @UsePipes(new PostValidationPipe()) + @Post("sessions/:interviewId/assistant-turn") + appendAssistantTurn( + @Param("interviewId") interviewId: string, + @Body() dto: AppendAssistantTurnDto, + ) { + return this.contextService.appendAssistantTurn( + interviewId, + dto.assistantText, + ); + } + @UsePipes(new PostValidationPipe()) @Post("sessions/:interviewId/verbal-analysis") saveVerbalAnalysis( diff --git a/server/src/modules/simulation/simulation.module.ts b/server/src/modules/simulation/simulation.module.ts index 960d9bf9..7bc11471 100644 --- a/server/src/modules/simulation/simulation.module.ts +++ b/server/src/modules/simulation/simulation.module.ts @@ -18,7 +18,9 @@ import { InternalApiKeyGuard } from "./guards/internal-api-key.guard"; @Module({ imports: [ RedisModule, - HttpModule.register({ timeout: 5000 }), + HttpModule.register({ + timeout: parseInt(process.env.SIM_AI_INIT_TIMEOUT_MS ?? "30000", 10), + }), TypeOrmModule.forFeature([ai_interview, ai_verbal_analysis]), ], controllers: [SimulationInternalController], From 22d23833cfd01e74aa64c0b04e48dbcf8824d83b Mon Sep 17 00:00:00 2001 From: Andriamanampisoa Date: Tue, 14 Jul 2026 15:03:48 +0200 Subject: [PATCH 02/16] feat: Setup the structure of the interview (greatings, experience...) --- ai/docker-compose.yml | 2 +- ai/microservices/speech-to-speech/config.json | 4 +- .../speech-to-speech/engine/interview_flow.py | 206 ++++++++++++++++++ .../speech-to-speech/engine/models.py | 11 +- .../speech-to-speech/engine/pipeline.py | 76 ++++++- .../engine/session_context.py | 51 +++++ .../speech-to-speech/engine/settings.py | 4 +- .../engine/simulation_brief.py | 116 ++++++++-- .../speech-to-speech/engine/stsProcess.py | 97 +++++++++ .../tests/test_interview_flow.py | 88 ++++++++ 10 files changed, 630 insertions(+), 25 deletions(-) create mode 100644 ai/microservices/speech-to-speech/engine/interview_flow.py create mode 100644 ai/microservices/speech-to-speech/tests/test_interview_flow.py diff --git a/ai/docker-compose.yml b/ai/docker-compose.yml index c6339fa9..ba13c371 100644 --- a/ai/docker-compose.yml +++ b/ai/docker-compose.yml @@ -55,7 +55,7 @@ services: - BACKEND_URL=${BACKEND_URL:-http://server:3000/v1/api} - SIM_INTERNAL_API_KEY=${SIM_INTERNAL_API_KEY:-talkup-dev-internal-key} - LLM_BACKEND=${LLM_BACKEND:-openrouter} - - LLM_MAX_NEW_TOKENS=${LLM_MAX_NEW_TOKENS:-160} + - LLM_MAX_NEW_TOKENS=${LLM_MAX_NEW_TOKENS:-320} - VA_SERVICE_URL=http://verbal_analyzer:8006 - VA_ENABLED=true - VA_TIMEOUT_SEC=3 diff --git a/ai/microservices/speech-to-speech/config.json b/ai/microservices/speech-to-speech/config.json index 7453f4fb..3a03f782 100644 --- a/ai/microservices/speech-to-speech/config.json +++ b/ai/microservices/speech-to-speech/config.json @@ -1,11 +1,11 @@ { "LLM_PATH": "/app/llm/sup-it-v3-merged", - "SYSTEM_PROMPT": "Tu es Sophie Martin, recruteuse senior IT chez une ESN francaise. Tu es chaleureuse, professionnelle, patiente et humaine. Tu parles de facon naturelle comme dans une vraie conversation. Tu dois repondre uniquement a la derniere prise de parole du candidat, en une seule reponse courte et naturelle. N'ecris jamais un dialogue multi-tours, n'imite jamais des balises comme system: ou user:, et ne recopie jamais l'historique de conversation.", + "SYSTEM_PROMPT": "Tu es Sophie Martin, recruteuse senior IT chez une ESN francaise. Tu conduis un entretien d'embauche professionnel en visioconference. Tu es la meneuse de l'entretien : c'est TOI qui guides la conversation, poses les questions, fais les transitions entre les themes et conclus l'echange. Ne laisse jamais le candidat diriger seul l'entretien. Style : chaleureuse, professionnelle, humaine et naturelle. Reponses orales concises (2 a 4 phrases en general). Une seule prise de parole a la fois. N'ecris jamais de balises ni un dialogue multi-tours. Structure : accueil, parcours, competences, motivation, echange, cloture (~15-20 min). Comportement proactif : pose toujours une question ou annonce la prochaine etape.", "PIPER_VOICE_PATH": "/app/models/piper/fr_FR-siwis-medium.onnx", "WHISPER_MODEL_PATH": "/app/models/whisper/faster-whisper-large-v3", "LLM_BACKEND": "openrouter", "LLM_GPU_MAX_MEMORY": "3GiB", "LLM_CPU_MAX_MEMORY": "3GiB", - "LLM_MAX_NEW_TOKENS": 160, + "LLM_MAX_NEW_TOKENS": 320, "QUEUE_MAXSIZE": 32 } diff --git a/ai/microservices/speech-to-speech/engine/interview_flow.py b/ai/microservices/speech-to-speech/engine/interview_flow.py new file mode 100644 index 00000000..3de5b798 --- /dev/null +++ b/ai/microservices/speech-to-speech/engine/interview_flow.py @@ -0,0 +1,206 @@ +## +## Talkup Project, 2026 +## TalkUp.AI +## Interview flow state and dynamic phase instructions. +## + +from __future__ import annotations + +import re +from dataclasses import dataclass +from threading import Lock + +# ~15-20 min at ~40-50 s per exchange. +CLOSING_TURN_THRESHOLD = 22 + +MEMORY_INSTRUCTION = ( + "Consigne de coherence : tu as acces a l'historique complet de la conversation. " + "Utilise-le pour enchainer logiquement : reprends le prenom et les elements deja " + "mentionnes par le candidat, ne repose pas une question deja posee, " + "ne contredis pas ce qui a ete dit, et rebondis sur sa derniere reponse " + "avant de poser la suivante." +) + +OPENING_TRIGGER = ( + "L'entretien commence. Le candidat vient de rejoindre la visioconference. " + "Accueille-le chaleureusement avec un « Bonjour » (sans prenom — interdiction " + "d'utiliser des crochets ou placeholders). Remercie-le pour sa presence, " + "presente-toi brievement, presente l'entreprise et le poste, " + "explique le deroulement (~15-20 minutes). " + "IMPORTANT : ne pose AUCUNE question sur le parcours professionnel, " + "l'experience ou les competences techniques. " + "Ta seule question finale doit inviter le candidat a se presenter " + "(par exemple : « Pour commencer, pourriez-vous vous presenter ? »)." +) + +FAREWELL_ACK_TRIGGER = ( + "Le candidat vient de repondre a tes salutations de fin d'entretien. " + "Remercie-le une derniere fois brievement et dis-lui au revoir. " + "L'entretien est termine." +) + +_NAME_ONLY_PATTERNS = ( + re.compile( + r"^(?:bonjour|salut|hello|bonsoir)[,!.]?\s*(?:je\s+(?:m'appelle|m'|me\s+nomme)\s+)?" + r"([a-zàâäéèêëïîôùûüç\-']+)[.!]?$", + re.IGNORECASE, + ), + re.compile( + r"^(?:je\s+(?:m'appelle|m'|me\s+nomme)\s+)([a-zàâäéèêëïîôùûüç\-']+)[.!]?$", + re.IGNORECASE, + ), +) + +_EXPERIENCE_KEYWORDS = frozenset({ + "experience", "experiences", "travaille", "travaille", "poste", "entreprise", + "stage", "alternance", "diplome", "formation", "projet", "developpeur", + "developpement", "ingenieur", "ans", "annee", "annees", "carriere", "parcours", + "competence", "competences", "mission", "missions", "cdi", "cdd", +}) + + +@dataclass +class InterviewFlowState: + greeting_sent: bool = False + farewell_sent: bool = False + presentation_done: bool = False + + +class InterviewFlowStore: + _lock = Lock() + _sessions: dict[str, InterviewFlowState] = {} + + @classmethod + def get(cls, interview_id: str) -> InterviewFlowState: + with cls._lock: + state = cls._sessions.get(interview_id) + if state is None: + state = InterviewFlowState() + cls._sessions[interview_id] = state + return state + + @classmethod + def clear(cls, interview_id: str) -> None: + with cls._lock: + cls._sessions.pop(interview_id, None) + + +def count_user_turns(history: list[dict[str, str]]) -> int: + return sum( + 1 + for turn in history + if turn.get("role") == "user" and isinstance(turn.get("content"), str) + ) + + +def _normalize_text(text: str) -> str: + decomposed = text.lower().strip() + return re.sub(r"\s+", " ", decomposed) + + +def looks_like_name_only(text: str) -> bool: + return _looks_like_name_only(text) + + +def _looks_like_name_only(text: str) -> bool: + normalized = _normalize_text(text) + if len(normalized.split()) > 12: + return False + + for pattern in _NAME_ONLY_PATTERNS: + if pattern.match(normalized): + return True + + if len(normalized.split()) <= 6 and not any( + keyword in normalized for keyword in _EXPERIENCE_KEYWORDS + ): + return True + + return False + + +def _user_has_substantial_intro(text: str) -> bool: + normalized = _normalize_text(text) + if len(normalized.split()) >= 20: + return True + return any(keyword in normalized for keyword in _EXPERIENCE_KEYWORDS) + + +def mark_presentation_done(interview_id: str, user_text: str) -> None: + if _user_has_substantial_intro(user_text) or ( + not _looks_like_name_only(user_text) and len(_normalize_text(user_text).split()) >= 10 + ): + state = InterviewFlowStore.get(interview_id) + state.presentation_done = True + + +def get_phase_instruction( + user_turn_count: int, + presentation_done: bool, + latest_user_text: str = "", +) -> str: + if user_turn_count <= 0: + return "" + + if not presentation_done: + if _looks_like_name_only(latest_user_text): + return ( + "Phase actuelle : PRESENTATION. Le candidat vient de donner son prenom " + "ou une presentation tres breve. Remercie-le chaleureusement, " + "utilise son prenom, et invite-le a se presenter un peu plus " + "(qui il est, sa formation, sa situation actuelle en quelques phrases). " + "Ne pose pas encore de questions sur le detail de ses experiences professionnelles." + ) + if user_turn_count <= 2: + return ( + "Phase actuelle : PRESENTATION. Le candidat est en train de se presenter. " + "Rebondis sur ce qu'il vient de dire. Si sa presentation est incomplete, " + "pose une question de relance douce pour en savoir plus sur lui. " + "N'abord pas encore les experiences professionnelles en detail." + ) + + if user_turn_count <= 6: + return ( + "Phase actuelle : PARCOURS. Le candidat s'est presente. " + "Explore maintenant ses experiences passees, formations et evolutions de carriere. " + "Pose des questions de suivi sur ses reponses." + ) + if user_turn_count <= 12: + return ( + "Phase actuelle : COMPETENCES. Pose des questions techniques ou metier " + "adaptees au poste. Challenge gentiment avec des cas concrets." + ) + if user_turn_count <= 16: + return ( + "Phase actuelle : MOTIVATION. Explore les motivations du candidat, " + "son projet professionnel et ses soft skills." + ) + if user_turn_count <= 20: + return ( + "Phase actuelle : ECHANGE. Propose au candidat de poser ses questions " + "sur le poste, l'equipe ou l'entreprise." + ) + return ( + "Phase actuelle : CONCLUSION. Prepare la fin de l'entretien : " + "resume brievement, remercie le candidat et annonce les prochaines etapes." + ) + + +def get_turn_instruction( + history: list[dict[str, str]], + user_turn_count: int, + latest_user_text: str, + farewell_sent: bool, + presentation_done: bool, +) -> str: + if farewell_sent: + return f"{MEMORY_INSTRUCTION}\n\n{FAREWELL_ACK_TRIGGER}" + if user_turn_count >= CLOSING_TURN_THRESHOLD: + return ( + f"{MEMORY_INSTRUCTION}\n\n" + "C'est le moment de conclure l'entretien. Remercie le candidat pour son temps, " + "resume brievement les prochaines etapes du processus de recrutement " + "et dis-lui au revoir chaleureusement." + ) + phase = get_phase_instruction(user_turn_count, presentation_done, latest_user_text) + return f"{MEMORY_INSTRUCTION}\n\n{phase}" diff --git a/ai/microservices/speech-to-speech/engine/models.py b/ai/microservices/speech-to-speech/engine/models.py index ad5d207d..ac166652 100644 --- a/ai/microservices/speech-to-speech/engine/models.py +++ b/ai/microservices/speech-to-speech/engine/models.py @@ -51,6 +51,15 @@ class STSModels: _ROLE_LABEL_PATTERN = re.compile(r"(?im)(?:^|[\n\t])\s*(system|user|assistant)\s*:\s*") +_BRACKET_PLACEHOLDER_PATTERN = re.compile(r"\[[^\]]+\]") + + +def _strip_bracket_placeholders(text: str) -> str: + """Remove LLM placeholders like [Prenom du candidat] from spoken output.""" + cleaned = _BRACKET_PLACEHOLDER_PATTERN.sub("", text) + cleaned = re.sub(r"\s{2,}", " ", cleaned) + cleaned = re.sub(r"Bonjour\s+!", "Bonjour !", cleaned, flags=re.IGNORECASE) + return cleaned.strip() def _sanitize_llm_response(text: str) -> str: @@ -75,7 +84,7 @@ def _sanitize_llm_response(text: str) -> str: cleaned = " ".join(filtered_lines).strip() cleaned = re.sub(r"\s{2,}", " ", cleaned) - return cleaned + return _strip_bracket_placeholders(cleaned) def _is_valid_vllm_model_dir(model_path: str) -> tuple[bool, str]: diff --git a/ai/microservices/speech-to-speech/engine/pipeline.py b/ai/microservices/speech-to-speech/engine/pipeline.py index 6cc3e480..dfe50355 100644 --- a/ai/microservices/speech-to-speech/engine/pipeline.py +++ b/ai/microservices/speech-to-speech/engine/pipeline.py @@ -16,8 +16,15 @@ from dataclasses import dataclass from .audio_decode import decode_audio_to_float32 from .models import STSModels, generate_ai_response, synthesize_tts_chunks -from .simulation_brief import build_messages_for_turn -from .session_context import append_session_history +from .simulation_brief import build_messages_for_turn, build_messages_for_opening +from .session_context import append_session_history, append_assistant_turn, fetch_session_history +from .interview_flow import ( + InterviewFlowStore, + CLOSING_TURN_THRESHOLD, + count_user_turns, + get_turn_instruction, + mark_presentation_done, +) # Canned phrases faster-whisper frequently hallucinates on silence/noise (FR). # These are only discarded when the model is also unsure speech was present, so @@ -82,6 +89,7 @@ class STSResult: transcription: str ai_response: str audio_chunks: list[bytes] + simulation_complete: bool = False def process_sts_request( models: STSModels, @@ -128,16 +136,78 @@ def process_sts_request( if not user_text or len(user_text) < 2: return STSResult(transcription="", ai_response="", audio_chunks=[]) + flow = InterviewFlowStore.get(interview_id) if interview_id else None + history = fetch_session_history(interview_id) if interview_id else [] + user_turn_count = count_user_turns(history) + 1 + farewell_sent = flow.farewell_sent if flow else False + presentation_done = flow.presentation_done if flow else False + extra_instruction = get_turn_instruction( + history, + user_turn_count, + user_text, + farewell_sent, + presentation_done, + ) + messages = build_messages_for_turn( models.settings.system_prompt, interview_id, user_text, + extra_instruction=extra_instruction, ) ai_response = generate_ai_response(models, messages) audio_chunks = list(synthesize_tts_chunks(models, ai_response)) + simulation_complete = False if interview_id and ai_response: + if farewell_sent: + simulation_complete = True + if flow: + InterviewFlowStore.clear(interview_id) + elif user_turn_count >= CLOSING_TURN_THRESHOLD and flow: + flow.farewell_sent = True + + if flow: + mark_presentation_done(interview_id, user_text) + append_session_history(interview_id, user_text, ai_response) - return STSResult(transcription=user_text, ai_response=ai_response, audio_chunks=audio_chunks) + return STSResult( + transcription=user_text, + ai_response=ai_response, + audio_chunks=audio_chunks, + simulation_complete=simulation_complete, + ) + + +def generate_opening_greeting( + models: STSModels, + interview_id: str | None = None, +) -> STSResult: + """Generate the proactive opening greeting when a session starts.""" + if not interview_id: + return STSResult(transcription="", ai_response="", audio_chunks=[]) + + flow = InterviewFlowStore.get(interview_id) + if flow.greeting_sent: + return STSResult(transcription="", ai_response="", audio_chunks=[]) + + messages = build_messages_for_opening( + models.settings.system_prompt, + interview_id, + ) + + ai_response = generate_ai_response(models, messages) + if not ai_response: + return STSResult(transcription="", ai_response="", audio_chunks=[]) + + audio_chunks = list(synthesize_tts_chunks(models, ai_response)) + flow.greeting_sent = True + append_assistant_turn(interview_id, ai_response) + + return STSResult( + transcription="", + ai_response=ai_response, + audio_chunks=audio_chunks, + ) diff --git a/ai/microservices/speech-to-speech/engine/session_context.py b/ai/microservices/speech-to-speech/engine/session_context.py index 96ef8ab7..c28322ac 100644 --- a/ai/microservices/speech-to-speech/engine/session_context.py +++ b/ai/microservices/speech-to-speech/engine/session_context.py @@ -101,12 +101,63 @@ def append_session_history( ) +def append_assistant_turn(interview_id: str, assistant_text: str) -> None: + if not interview_id or interview_id == "unknown": + return + + SimulationBriefStore.append_assistant(interview_id, assistant_text) + + key = _internal_key() + if not key: + return + + url = f"{_backend_base()}/ai/internal/sessions/{interview_id}/assistant-turn" + payload = json.dumps({"assistantText": assistant_text}).encode("utf-8") + req = urllib.request.Request( + url, + data=payload, + headers={ + "X-Internal-Api-Key": key, + "Content-Type": "application/json", + }, + method="POST", + ) + + try: + with urllib.request.urlopen(req, timeout=10) as resp: + resp.read() + except Exception as err: + NOTIFIER.send_notification( + EnumMcs.MicroservicesNames.STS, + 1, + f"Failed to append assistant turn for {interview_id}: {err}", + ) + + +def fetch_session_history(interview_id: str) -> list[dict[str, str]]: + from .simulation_brief import SimulationBriefStore + + state = SimulationBriefStore.get(interview_id) + if state is not None: + return list(state.history) + + session = fetch_session_context(interview_id) + if session is not None: + history = session.get("history") + if isinstance(history, list): + return list(history) + return [] + + def build_messages_from_nest_session( default_system_prompt: str, session: dict[str, Any], user_text: str, + extra_instruction: str = "", ) -> list[dict[str, str]]: system_prompt = session.get("systemPrompt") or default_system_prompt + if extra_instruction: + system_prompt = f"{system_prompt}\n\n{extra_instruction}" history = session.get("history") or [] messages: list[dict[str, str]] = [{"role": "system", "content": system_prompt}] diff --git a/ai/microservices/speech-to-speech/engine/settings.py b/ai/microservices/speech-to-speech/engine/settings.py index a078b095..51d742b0 100644 --- a/ai/microservices/speech-to-speech/engine/settings.py +++ b/ai/microservices/speech-to-speech/engine/settings.py @@ -38,13 +38,13 @@ class STSSettings: DEFAULT_SETTINGS = STSSettings( llm_path="/app/llm/sup-it-v3-merged", - system_prompt="Tu es Sophie Martin, recruteuse senior IT chez une ESN francaise. Tu es chaleureuse, professionnelle, patiente et humaine. Tu parles de facon naturelle comme dans une vraie conversation. Tu dois repondre uniquement a la derniere prise de parole du candidat, en une seule reponse courte et naturelle. N'ecris jamais un dialogue multi-tours, n'imite jamais des balises comme system: ou user:, et ne recopie jamais l'historique de conversation.", + system_prompt="Tu es Sophie Martin, recruteuse senior IT chez une ESN francaise. Tu conduis un entretien d'embauche professionnel en visioconference. Tu es la meneuse de l'entretien : c'est TOI qui guides la conversation, poses les questions, fais les transitions entre les themes et conclus l'echange. Ne laisse jamais le candidat diriger seul l'entretien. Style : chaleureuse, professionnelle, humaine et naturelle. Reponses orales concises (2 a 4 phrases en general). Une seule prise de parole a la fois. N'ecris jamais de balises ni un dialogue multi-tours. Structure : accueil, parcours, competences, motivation, echange, cloture (~15-20 min). Comportement proactif : pose toujours une question ou annonce la prochaine etape.", piper_voice_path="/app/models/piper/fr_FR-siwis-medium.onnx", whisper_model_path="/app/models/whisper/faster-whisper-large-v3", llm_backend="auto", llm_gpu_max_memory="2GiB", llm_cpu_max_memory="16GiB", - llm_max_new_tokens=160, + llm_max_new_tokens=320, queue_maxsize=32, openrouter_api_key="", openrouter_model="mistralai/mistral-small-3.2-24b-instruct", diff --git a/ai/microservices/speech-to-speech/engine/simulation_brief.py b/ai/microservices/speech-to-speech/engine/simulation_brief.py index 1ce19ddc..2a2f4b6b 100644 --- a/ai/microservices/speech-to-speech/engine/simulation_brief.py +++ b/ai/microservices/speech-to-speech/engine/simulation_brief.py @@ -17,12 +17,32 @@ BASE_RECRUITER_PERSONA = ( "Tu es Sophie Martin, recruteuse senior IT chez une ESN francaise. " - "Tu es chaleureuse, professionnelle, patiente et humaine. " - "Tu parles de facon naturelle comme dans une vraie conversation. " - "Tu dois repondre uniquement a la derniere prise de parole du candidat, " - "en une seule reponse courte et naturelle. " - "N'ecris jamais un dialogue multi-tours, n'imite jamais des balises comme system: ou user:, " - "et ne recopie jamais l'historique de conversation." + "Tu conduis un entretien d'embauche professionnel en visioconference. " + "Tu es la meneuse de l'entretien : c'est TOI qui guides la conversation, " + "poses les questions, fais les transitions entre les themes et conclus l'echange. " + "Ne laisse jamais le candidat diriger seul l'entretien.\n\n" + "Style : chaleureuse, professionnelle, humaine et naturelle. " + "Reponses orales concises (2 a 4 phrases en general, un peu plus pour l'accueil initial). " + "Une seule prise de parole a la fois — ne simule pas plusieurs tours. " + "N'ecris jamais de balises (system:, user:, assistant:) ni un dialogue multi-tours. " + "Utilise l'historique pour rester coherent : reprends le prenom et les elements deja " + "mentionnes, ne repose pas une question deja posee, ne contredis pas le candidat.\n\n" + "Structure de l'entretien (~15-20 minutes) — respecte cet ordre strict :\n" + "1. Accueil : salutations, remerciements, presentation de l'entreprise et du poste\n" + "2. Presentation du candidat : invite-le a se presenter AVANT toute question sur l'experience\n" + "3. Parcours : experiences passees, formations, evolutions de carriere\n" + "4. Competences : questions techniques ou metier selon le type de simulation\n" + "5. Motivation : pourquoi ce poste, projet professionnel, soft skills\n" + "6. Echange : propose au candidat de poser ses questions\n" + "7. Cloture : remerciements, prochaines etapes, au revoir chaleureux\n\n" + "Comportement proactif : pose TOUJOURS une question ou annonce clairement " + "la prochaine etape a la fin de chaque reponse. " + "Transitionne activement entre les phases. " + "Rebondis sur les reponses du candidat avant d'enchaîner. " + "Ne reponds jamais par un simple accord sans question de suivi.\n\n" + "Regles sur le prenom du candidat : n'utilise JAMAIS de placeholder entre crochets " + "(ex. [Prenom du candidat], [Nom]). Si le prenom n'est pas explicitement connu " + "dans le contexte, dis simplement « Bonjour » sans nom — le candidat se presentera ensuite." ) @@ -115,6 +135,16 @@ def append_turn(cls, interview_id: str, user_text: str, assistant_text: str) -> if len(state.history) > 60: state.history = state.history[-60:] + @classmethod + def append_assistant(cls, interview_id: str, assistant_text: str) -> None: + with cls._lock: + state = cls._sessions.get(interview_id) + if state is None: + return + state.history.append({"role": "assistant", "content": assistant_text}) + if len(state.history) > 60: + state.history = state.history[-60:] + @classmethod def clear(cls, interview_id: str) -> None: with cls._lock: @@ -166,31 +196,85 @@ def build_system_prompt_from_brief(brief: SimulationBrief, fallback: str) -> str return "\n".join(sections) +def _append_history_turns( + messages: list[dict[str, str]], + history: list[dict[str, str]], +) -> None: + for turn in history: + role = turn.get("role") + content = turn.get("content") + if role in ("user", "assistant") and isinstance(content, str) and content.strip(): + messages.append({"role": role, "content": content}) + + +def _resolve_session( + default_system_prompt: str, + interview_id: str | None, +) -> tuple[str, list[dict[str, str]]] | None: + from .session_context import fetch_session_context + + if not interview_id: + return None + + state = SimulationBriefStore.get(interview_id) + if state is not None: + system_prompt = build_system_prompt_from_brief(state.brief, default_system_prompt) + return system_prompt, list(state.history) + + nest_session = fetch_session_context(interview_id) + if nest_session is not None: + system_prompt = nest_session.get("systemPrompt") or default_system_prompt + history = nest_session.get("history") or [] + return system_prompt, list(history) + + return None + + def build_messages_for_turn( default_system_prompt: str, interview_id: str | None, user_text: str, + extra_instruction: str = "", ) -> list[dict[str, str]]: from .session_context import build_messages_from_nest_session, fetch_session_context if interview_id: - state = SimulationBriefStore.get(interview_id) - if state is not None: - system_prompt = build_system_prompt_from_brief(state.brief, default_system_prompt) + resolved = _resolve_session(default_system_prompt, interview_id) + if resolved is not None: + system_prompt, history = resolved + if extra_instruction: + system_prompt = f"{system_prompt}\n\n{extra_instruction}" messages: list[dict[str, str]] = [{"role": "system", "content": system_prompt}] - for turn in state.history: - role = turn.get("role") - content = turn.get("content") - if role in ("user", "assistant") and isinstance(content, str) and content.strip(): - messages.append({"role": role, "content": content}) + _append_history_turns(messages, history) messages.append({"role": "user", "content": user_text}) return messages nest_session = fetch_session_context(interview_id) if nest_session is not None: - return build_messages_from_nest_session(default_system_prompt, nest_session, user_text) + return build_messages_from_nest_session( + default_system_prompt, + nest_session, + user_text, + extra_instruction=extra_instruction, + ) + system_prompt = default_system_prompt + if extra_instruction: + system_prompt = f"{system_prompt}\n\n{extra_instruction}" return [ - {"role": "system", "content": default_system_prompt}, + {"role": "system", "content": system_prompt}, {"role": "user", "content": user_text}, ] + + +def build_messages_for_opening( + default_system_prompt: str, + interview_id: str | None, +) -> list[dict[str, str]]: + from .interview_flow import OPENING_TRIGGER + + return build_messages_for_turn( + default_system_prompt, + interview_id, + OPENING_TRIGGER, + ) diff --git a/ai/microservices/speech-to-speech/engine/stsProcess.py b/ai/microservices/speech-to-speech/engine/stsProcess.py index 82c0f40d..5903ece6 100644 --- a/ai/microservices/speech-to-speech/engine/stsProcess.py +++ b/ai/microservices/speech-to-speech/engine/stsProcess.py @@ -21,6 +21,7 @@ from .queueService import StsQueueService from .settings import load_settings from .simulation_brief import SimulationBrief, SimulationBriefStore +from .interview_flow import InterviewFlowStore from .verbal_analyzer_client import analyze_transcription, finalize_session from .ws_auth import WsSessionClaims, authenticate_websocket, interview_id_allowed @@ -171,6 +172,8 @@ async def _process_stream_and_reply( "response": result.ai_response, "audio_chunks": encoded_chunks, } + if result.simulation_complete: + payload["simulation_complete"] = True if request_id is not None: payload["request_id"] = request_id @@ -207,6 +210,67 @@ async def _process_stream_and_reply( NOTIFIER.send_notification(EnumMcs.MicroservicesNames.STS, 1, "Client disconnected while sending STS warning") +async def _process_session_start_and_reply( + websocket: WebSocket, + send_lock: asyncio.Lock, + interview_id: str, + request_id: int | None = None, +) -> None: + """Generate and send the proactive opening greeting for a new session.""" + try: + from .pipeline import generate_opening_greeting + + result = await asyncio.to_thread( + generate_opening_greeting, + models, + interview_id, + ) + + if not result.ai_response: + return + + NOTIFIER.send_notification( + EnumMcs.MicroservicesNames.STS, + 0, + f"AI greeting: {result.ai_response[:100]}...", + ) + encoded_chunks = [base64.b64encode(chunk).decode("ascii") for chunk in result.audio_chunks] + payload: dict = { + "type": "sts_result", + "transcription": "", + "response": result.ai_response, + "audio_chunks": encoded_chunks, + } + if request_id is not None: + payload["request_id"] = request_id + + await _ws_send_json(websocket, send_lock, payload) + except (WebSocketDisconnect, ConnectionClosed): + NOTIFIER.send_notification( + EnumMcs.MicroservicesNames.STS, + 1, + "Client disconnected while sending session_start greeting", + ) + except Exception as err: + NOTIFIER.send_notification( + EnumMcs.MicroservicesNames.STS, + 2, + f"session_start greeting error: {err}", + ) + NOTIFIER.send_notification(EnumMcs.MicroservicesNames.STS, 2, traceback.format_exc().strip()) + try: + await _ws_send_json( + websocket, + send_lock, + { + "type": "warning", + "text": "Erreur temporaire lors de l'accueil. Vous pouvez commencer a parler.", + }, + ) + except (WebSocketDisconnect, ConnectionClosed): + pass + + @app.get("/health") async def health() -> dict[str, str]: return {"status": "ok"} @@ -282,6 +346,8 @@ async def audio_worker() -> None: if payload.get("type") == "session_end": end_interview_id = payload.get("interview_id") or payload.get("stream_id") if isinstance(end_interview_id, str) and end_interview_id.strip(): + InterviewFlowStore.clear(end_interview_id.strip()) + SimulationBriefStore.clear(end_interview_id.strip()) asyncio.create_task( asyncio.to_thread(finalize_session, end_interview_id.strip()), ) @@ -292,6 +358,37 @@ async def audio_worker() -> None: ) continue + if payload.get("type") == "session_start": + start_interview_id = payload.get("interview_id") or payload.get("stream_id") + start_request_id = payload.get("request_id") + if isinstance(start_interview_id, str): + start_interview_id = start_interview_id.strip() or None + else: + start_interview_id = None + + if not interview_id_allowed(claims, start_interview_id): + await _ws_send_json( + websocket, + send_lock, + {"type": "error", "text": "interview_id does not match session token"}, + ) + continue + + if start_request_id is not None and not isinstance(start_request_id, int): + try: + start_request_id = int(start_request_id) + except (TypeError, ValueError): + start_request_id = None + + if start_interview_id: + await _process_session_start_and_reply( + websocket, + send_lock, + start_interview_id, + start_request_id, + ) + continue + if payload.get("type") == "ping": await _ws_send_json( websocket, diff --git a/ai/microservices/speech-to-speech/tests/test_interview_flow.py b/ai/microservices/speech-to-speech/tests/test_interview_flow.py new file mode 100644 index 00000000..98939019 --- /dev/null +++ b/ai/microservices/speech-to-speech/tests/test_interview_flow.py @@ -0,0 +1,88 @@ +## +## Talkup Project, 2026 +## TalkUp.AI +## Interview flow unit tests. +## + +from __future__ import annotations + +from engine.interview_flow import ( + CLOSING_TURN_THRESHOLD, + InterviewFlowStore, + count_user_turns, + get_phase_instruction, + get_turn_instruction, + looks_like_name_only, + mark_presentation_done, +) + + +def test_count_user_turns() -> None: + history = [ + {"role": "assistant", "content": "Bonjour"}, + {"role": "user", "content": "Bonjour"}, + {"role": "assistant", "content": "Parlez-moi de vous"}, + {"role": "user", "content": "Je suis dev"}, + ] + assert count_user_turns(history) == 2 + + +def test_looks_like_name_only() -> None: + assert looks_like_name_only("Bonjour, je m'appelle Mathéo.") is True + assert looks_like_name_only("Je m'appelle Mathéo") is True + assert looks_like_name_only( + "Je suis Mathéo, diplômé en informatique à Nantes, " + "je travaille depuis 3 ans en développement web." + ) is False + + +def test_presentation_phase_before_experience() -> None: + instruction = get_phase_instruction( + user_turn_count=1, + presentation_done=False, + latest_user_text="Bonjour, je m'appelle Mathéo.", + ) + assert "PRESENTATION" in instruction + assert "experiences professionnelles" in instruction.lower() + + parcours = get_phase_instruction( + user_turn_count=3, + presentation_done=True, + latest_user_text="Je suis développeur depuis 3 ans.", + ) + assert "PARCOURS" in parcours + + +def test_closing_instruction_after_threshold() -> None: + instruction = get_turn_instruction( + [], + CLOSING_TURN_THRESHOLD, + "Merci beaucoup", + farewell_sent=False, + presentation_done=True, + ) + assert "conclure" in instruction.lower() + assert "historique" in instruction.lower() + + +def test_farewell_ack_when_farewell_sent() -> None: + instruction = get_turn_instruction( + [], + 5, + "Au revoir", + farewell_sent=True, + presentation_done=True, + ) + assert "termine" in instruction.lower() + + +def test_mark_presentation_done() -> None: + InterviewFlowStore.clear("int-test") + state = InterviewFlowStore.get("int-test") + assert state.presentation_done is False + mark_presentation_done( + "int-test", + "Je suis Mathéo, développeur .NET avec 4 ans d'expérience en React et C#.", + ) + assert InterviewFlowStore.get("int-test").presentation_done is True + InterviewFlowStore.clear("int-test") From 70171ba2082a0816d9793d345e9143675cf86db4 Mon Sep 17 00:00:00 2001 From: Andriamanampisoa Date: Tue, 14 Jul 2026 15:04:33 +0200 Subject: [PATCH 03/16] feat: relay session_start to STS for the initial handshake --- .../inc/network/MicroservicesManager.hpp | 14 ++ .../server/inc/network/WebsocketManager.hpp | 3 + .../src/network/MicroservicesManager.cpp | 149 ++++++++++++++++++ .../server/src/network/WebsocketManager.cpp | 73 +++++++++ 4 files changed, 239 insertions(+) diff --git a/ai/core/server/inc/network/MicroservicesManager.hpp b/ai/core/server/inc/network/MicroservicesManager.hpp index b25f35bd..d47eaf18 100644 --- a/ai/core/server/inc/network/MicroservicesManager.hpp +++ b/ai/core/server/inc/network/MicroservicesManager.hpp @@ -99,6 +99,14 @@ namespace talkup_network { */ static void send_session_end_to_sts(const std::string &interview_id); + /** + * @brief Request STS to generate the proactive opening greeting. + */ + static void send_session_start_to_sts( + const nlohmann::json &data, + ResponseCallback callback, + const std::shared_ptr &client_session = nullptr); + /** * @brief Initialize WebSocket connections to all registered microservices. * It's establishes persistent WebSocket connections to each microservice @@ -136,6 +144,7 @@ namespace talkup_network { enum class StsJobKind { StreamChunk, SimulationContext, + SessionStart, }; struct StsJob { @@ -207,6 +216,11 @@ namespace talkup_network { ResponseCallback callback, const std::weak_ptr &client_session = {}); + static void process_session_start_job( + const nlohmann::json &data, + ResponseCallback callback, + const std::weak_ptr &client_session = {}); + static void process_simulation_context_job( const std::string &interview_id, const nlohmann::json &context_data, diff --git a/ai/core/server/inc/network/WebsocketManager.hpp b/ai/core/server/inc/network/WebsocketManager.hpp index 2a3a8d48..2d0b2d5b 100644 --- a/ai/core/server/inc/network/WebsocketManager.hpp +++ b/ai/core/server/inc/network/WebsocketManager.hpp @@ -92,6 +92,9 @@ namespace talkup_network { void handle_session_end(const nlohmann::json& json, crow::websocket::connection& conn, std::shared_ptr microservices_manager); + void handle_session_start(const nlohmann::json& json, crow::websocket::connection& conn, + std::shared_ptr microservices_manager); + private: std::unordered_map)>> _type_handlers; diff --git a/ai/core/server/src/network/MicroservicesManager.cpp b/ai/core/server/src/network/MicroservicesManager.cpp index b40c0cd0..837a035b 100644 --- a/ai/core/server/src/network/MicroservicesManager.cpp +++ b/ai/core/server/src/network/MicroservicesManager.cpp @@ -239,6 +239,8 @@ void talkup_network::MicroservicesManager::create_service_worker( job.interview_id, job.context_data, job.context_promise); + } else if (job.kind == WebSocketConnection::StsJobKind::SessionStart) { + process_session_start_job(job.data, std::move(job.callback), job.client_session); } else { process_sts_job(job.data, std::move(job.callback), job.client_session); } @@ -497,6 +499,42 @@ void talkup_network::MicroservicesManager::send_to_sts_microservice( } } +void talkup_network::MicroservicesManager::send_session_start_to_sts( + const nlohmann::json &data, + ResponseCallback callback, + const std::shared_ptr &client_session) +{ + nlohmann::json err_response; + bool enqueue_ok = false; + + { + std::lock_guard lock(__ws_mutex); + auto it = __ws_connections.find("sts"); + if (it == __ws_connections.end() || !it->second.is_connected) { + std::cerr << "[MicroservicesManager] STS connection not available" << std::endl; + err_response = {{"error", "STS connection not available"}}; + } else { + { + std::lock_guard qlock(it->second.queue_mutex); + WebSocketConnection::StsJob job; + job.kind = WebSocketConnection::StsJobKind::SessionStart; + job.data = data; + job.callback = std::move(callback); + if (client_session) + job.client_session = client_session; + it->second.job_queue.push(std::move(job)); + } + it->second.queue_cv.notify_one(); + enqueue_ok = true; + std::cout << "[MicroservicesManager] Enqueued STS session_start job" << std::endl; + } + } + + if (!enqueue_ok && callback) { + callback(err_response); + } +} + void talkup_network::MicroservicesManager::send_session_end_to_sts( const std::string &interview_id) { @@ -864,6 +902,115 @@ void talkup_network::MicroservicesManager::process_sts_job( } } +void talkup_network::MicroservicesManager::process_session_start_job( + const nlohmann::json &data, + ResponseCallback callback, + const std::weak_ptr &client_session) +{ + try { + std::shared_ptr> ws; + std::mutex *io_mutex = nullptr; + + { + std::lock_guard lock(__ws_mutex); + auto it = __ws_connections.find("sts"); + if (it == __ws_connections.end() || !it->second.is_connected || + !it->second.ws || !it->second.ws->is_open()) { + if (it != __ws_connections.end()) + it->second.is_connected = false; + } else { + ws = it->second.ws; + io_mutex = &it->second.io_mutex; + } + } + + if (!ws || !io_mutex) { + if (!reconnect_service_connection("sts")) { + if (callback) + callback(nlohmann::json{{"error", "STS connection not available"}}); + return; + } + std::lock_guard lock(__ws_mutex); + auto it = __ws_connections.find("sts"); + if (it == __ws_connections.end() || !it->second.ws) { + if (callback) + callback(nlohmann::json{{"error", "STS connection not available"}}); + return; + } + ws = it->second.ws; + io_mutex = &it->second.io_mutex; + } + + std::unique_lock io_lock(*io_mutex); + const uint64_t request_id = ++g_sts_request_id; + std::string interview_id; + if (data.contains("stream_id") && data["stream_id"].is_string()) + interview_id = data["stream_id"].get(); + if (data.contains("interview_id") && data["interview_id"].is_string()) + interview_id = data["interview_id"].get(); + + nlohmann::json start_json = { + {"services", {"STS"}}, + {"type", "session_start"}, + {"request_id", request_id}, + {"timestamp", std::time(nullptr)}, + {"data", nlohmann::json::object()}, + }; + if (!interview_id.empty()) { + start_json["interview_id"] = interview_id; + start_json["stream_id"] = interview_id; + } + ws->write(boost::asio::buffer(start_json.dump())); + std::cout << "[MicroservicesManager] Sent STS session_start request_id=" << request_id << std::endl; + + const int timeout_ms = 120000; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout_ms); + + while (std::chrono::steady_clock::now() < deadline) { + const int remaining_ms = static_cast(std::chrono::duration_cast( + deadline - std::chrono::steady_clock::now()).count()); + if (remaining_ms <= 0) + break; + + nlohmann::json msg_json; + if (!read_sts_json_message(*ws, msg_json, std::min(remaining_ms, 30000))) + continue; + + const std::string msg_type = msg_json.value("type", ""); + if (msg_type == "pong" || msg_type == "simulation_context_ack") + continue; + + if (!msg_json.contains("request_id")) + continue; + + const uint64_t response_id = msg_json.value("request_id", static_cast(0)); + if (response_id != request_id) + continue; + + if (msg_type == "sts_result") { + if (callback) + callback(msg_json); + io_lock.unlock(); + return; + } + + io_lock.unlock(); + if (callback) + callback(msg_json); + return; + } + + std::cerr << "[MicroservicesManager] session_start read timed out" << std::endl; + io_lock.unlock(); + if (callback) + callback(nlohmann::json{{"type", "error"}, {"error", "Read timeout"}}); + } catch (const std::exception &e) { + std::cerr << "[MicroservicesManager] session_start exception: " << e.what() << std::endl; + if (callback) + callback(nlohmann::json{{"type", "error"}, {"error", std::string("Exception: ") + e.what()}}); + } +} + void talkup_network::MicroservicesManager::start_service_worker(const std::string &service_name) { std::lock_guard lock(__ws_mutex); @@ -908,6 +1055,8 @@ void talkup_network::MicroservicesManager::start_service_worker(const std::strin job.interview_id, job.context_data, job.context_promise); + } else if (job.kind == WebSocketConnection::StsJobKind::SessionStart) { + process_session_start_job(job.data, std::move(job.callback), job.client_session); } else { process_sts_job(job.data, std::move(job.callback), job.client_session); } diff --git a/ai/core/server/src/network/WebsocketManager.cpp b/ai/core/server/src/network/WebsocketManager.cpp index acdb3d63..48bb418e 100644 --- a/ai/core/server/src/network/WebsocketManager.cpp +++ b/ai/core/server/src/network/WebsocketManager.cpp @@ -39,6 +39,10 @@ talkup_network::WsManager::WsManager() crow::websocket::connection& conn, std::shared_ptr microservices_manager) { handle_session_end(json, conn, microservices_manager); }; + _type_handlers["session_start"] = [this](const nlohmann::json& json, + crow::websocket::connection& conn, std::shared_ptr microservices_manager) { + handle_session_start(json, conn, microservices_manager); + }; } void talkup_network::WsManager::connection_type_manager(nlohmann::json &json, crow::websocket::connection &conn, @@ -316,6 +320,75 @@ void talkup_network::WsManager::handle_simulation_context(const nlohmann::json& }).dump()); } +void talkup_network::WsManager::handle_session_start(const nlohmann::json& json, + crow::websocket::connection& conn, std::shared_ptr microservices_manager) +{ + const std::string key = json.value("key", ""); + const int64_t timestamp = json.value("timestamp", static_cast( + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count())); + + std::string interview_id = json.value("stream_id", ""); + if (json.contains("interview_id") && json["interview_id"].is_string()) + interview_id = json["interview_id"].get(); + + if (interview_id.empty()) { + conn.send_text(set_respond_json_format({ + .type = "error", + .key = key, + .stream_id = "", + .format = "text", + .timestamp = timestamp, + .data = "session_start requires stream_id or interview_id" + }).dump()); + return; + } + + if (!microservices_manager) { + conn.send_text(set_respond_json_format({ + .type = "error", + .key = key, + .stream_id = interview_id, + .format = "text", + .timestamp = timestamp, + .data = "microservices manager unavailable" + }).dump()); + return; + } + + auto client_session = WsClientSession::bind(conn); + microservices_manager->send_session_start_to_sts(json, + [this, client_session, key, interview_id](const nlohmann::json& sts_resp) { + if (!client_session->is_open()) + return; + + const std::string sts_type = sts_resp.value("type", ""); + if (sts_resp.contains("error") || sts_type == "error" || sts_type == "warning") { + client_session->send_text(set_respond_json_format({ + .type = "error", + .key = key, + .stream_id = interview_id, + .format = "text", + .timestamp = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(), + .data = sts_resp.dump() + }).dump()); + return; + } + + client_session->send_text(set_respond_json_format({ + .type = "sts_result", + .key = key, + .stream_id = interview_id, + .format = "audio", + .timestamp = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(), + .data = sts_resp.dump() + }).dump()); + }, + client_session); +} + void talkup_network::WsManager::handle_session_end(const nlohmann::json& json, crow::websocket::connection& conn, std::shared_ptr microservices_manager) { From a89c1685982d770f54b9e3a68923ecfc0739bf5a Mon Sep 17 00:00:00 2001 From: Andriamanampisoa Date: Tue, 14 Jul 2026 15:05:04 +0200 Subject: [PATCH 04/16] feat: Automatic AI startup, end of conversation, and session resumption --- .../organisms/simulation-workspace/index.tsx | 31 ++++- web/src/hooks/simulation/useAudioPlayback.ts | 25 +++- .../hooks/simulation/useInterviewSession.ts | 121 +++++++++++++++--- .../simulation/useSimulationWebSocket.ts | 16 +++ 4 files changed, 169 insertions(+), 24 deletions(-) diff --git a/web/src/components/organisms/simulation-workspace/index.tsx b/web/src/components/organisms/simulation-workspace/index.tsx index 5a97707e..f03d0840 100644 --- a/web/src/components/organisms/simulation-workspace/index.tsx +++ b/web/src/components/organisms/simulation-workspace/index.tsx @@ -15,6 +15,7 @@ import { useVerbalAnalysis, } from '@/hooks/simulation'; import { useCallback, useEffect, useRef, useState } from 'react'; +import toast from 'react-hot-toast'; import { ReadyState } from 'react-use-websocket'; export interface SimulationWorkspaceProps { @@ -42,6 +43,7 @@ export function SimulationWorkspace({ const sendJsonMessageRef = useRef<(message: object) => void>(() => {}); const readyStateRef = useRef(ReadyState.CONNECTING); const interviewIDRef = useRef(null); + const greetingSentRef = useRef(false); const handleResumeStream = useCallback(() => { if (videoStreamToggleRef.current) { @@ -87,6 +89,7 @@ export function SimulationWorkspace({ sendMessage, sendJsonMessage, sendPing, + sendSessionStart, lastMessage, lastJsonMessage, readyState, @@ -98,7 +101,6 @@ export function SimulationWorkspace({ onOpen: () => { setWsError(null); setConnectionAttempts(0); - sendPing(); }, onClose: (event) => { if (event.code !== 1000 && event.code !== 1001) { @@ -124,14 +126,33 @@ export function SimulationWorkspace({ readyStateRef.current = readyState; }, [sendJsonMessage, readyState]); + useEffect(() => { + greetingSentRef.current = false; + }, [interviewID]); + + useEffect(() => { + if ( + !isCallActive || + !interviewID || + readyState !== ReadyState.OPEN || + greetingSentRef.current + ) { + return; + } + greetingSentRef.current = true; + sendPing(); + sendSessionStart(); + }, [isCallActive, interviewID, readyState, sendPing, sendSessionStart]); + const handleAudioPacket = useCallback((packet: WebSocketPacket) => { if (readyStateRef.current === ReadyState.OPEN) { sendJsonMessageRef.current(packet); } }, []); - const { isAiSpeaking, transcript } = useAudioPlayback({ + const { isAiSpeaking, transcript, simulationComplete } = useAudioPlayback({ message: lastJsonMessage, + interviewID, }); const { analysis } = useVerbalAnalysis({ @@ -166,6 +187,12 @@ export function SimulationWorkspace({ } }, [transcript]); + useEffect(() => { + if (!simulationComplete || isAiSpeaking) return; + toast.success('Entretien terminé. Merci pour votre participation !'); + void handleStreamToggle(false); + }, [simulationComplete, isAiSpeaking, handleStreamToggle]); + const { isListening, isSpeaking, diff --git a/web/src/hooks/simulation/useAudioPlayback.ts b/web/src/hooks/simulation/useAudioPlayback.ts index 3d294b4f..0d71f914 100644 --- a/web/src/hooks/simulation/useAudioPlayback.ts +++ b/web/src/hooks/simulation/useAudioPlayback.ts @@ -7,6 +7,7 @@ export interface AiAnswer { transcription?: string; response?: string; audio_chunks?: string[]; + simulation_complete?: boolean; } export interface AiTranscript { @@ -18,6 +19,7 @@ export interface AiTranscript { export interface UseAudioPlaybackProps { message: unknown; + interviewID?: string | null; } export interface UseAudioPlaybackReturn { @@ -26,6 +28,8 @@ export interface UseAudioPlaybackReturn { error: string | null; /** Transcript text from the latest sts_result message, or null. */ transcript: AiTranscript | null; + /** True when the AI has signaled the interview is complete. */ + simulationComplete: boolean; } function asOuterPacket( @@ -48,10 +52,12 @@ function base64ToArrayBuffer(base64: string): ArrayBuffer { export function useAudioPlayback({ message, + interviewID = null, }: UseAudioPlaybackProps): UseAudioPlaybackReturn { const [isAiSpeaking, setIsAiSpeaking] = useState(false); const [error, setError] = useState(null); const [transcript, setTranscript] = useState(null); + const [simulationComplete, setSimulationComplete] = useState(false); const audioContextRef = useRef(null); const activeSourcesRef = useRef([]); @@ -135,6 +141,12 @@ export function useAudioPlayback({ [getAudioContext, stopPlayback], ); + useEffect(() => { + setSimulationComplete(false); + setTranscript(null); + lastHandledRef.current = null; + }, [interviewID]); + useEffect(() => { if (!message || message === lastHandledRef.current) return; lastHandledRef.current = message; @@ -166,8 +178,17 @@ export function useAudioPlayback({ ); } + if (answer.simulation_complete) { + setSimulationComplete(true); + } + const chunks = answer.audio_chunks ?? []; - if (chunks.length === 0) return; + if (chunks.length === 0) { + if (answer.simulation_complete) { + setIsAiSpeaking(false); + } + return; + } void playAnswer(chunks); }, [message, playAnswer]); @@ -190,5 +211,5 @@ export function useAudioPlayback({ }; }, []); - return { isAiSpeaking, stopPlayback, error, transcript }; + return { isAiSpeaking, stopPlayback, error, transcript, simulationComplete }; } diff --git a/web/src/hooks/simulation/useInterviewSession.ts b/web/src/hooks/simulation/useInterviewSession.ts index 01404fca..b3fd5380 100644 --- a/web/src/hooks/simulation/useInterviewSession.ts +++ b/web/src/hooks/simulation/useInterviewSession.ts @@ -20,6 +20,12 @@ const QUEUE_POLL_INTERVAL_MS = 3000; const QUEUE_POLL_MAX_MS = 20 * 60 * 1000; const HEARTBEAT_INTERVAL_MS = 60 * 1000; +function clearInterviewStorage(): void { + localStorage.removeItem(STORAGE_KEYS.INTERVIEW_ID); + localStorage.removeItem(STORAGE_KEYS.INTERVIEW_URL); + localStorage.removeItem(STORAGE_KEYS.IS_STREAMING); +} + /** * Props for the useInterviewSession hook. */ @@ -121,25 +127,87 @@ export function useInterviewSession({ if (hasResumedRef.current) return; hasResumedRef.current = true; - const savedInterviewID = localStorage.getItem(STORAGE_KEYS.INTERVIEW_ID); - const savedInterviewURL = localStorage.getItem(STORAGE_KEYS.INTERVIEW_URL); - const wasStreaming = - localStorage.getItem(STORAGE_KEYS.IS_STREAMING) === 'true'; - - if (savedInterviewID && savedInterviewURL) { - setInputUrl(savedInterviewURL); - setInterviewID(savedInterviewID); - setIsCallActive(true); - onConnect(savedInterviewURL); - - if (wasStreaming && onResumeStream) { - setTimeout(() => { - onResumeStream(); - }, STREAM_RESUME_DELAY_MS); + const restoreSavedSession = async () => { + const savedInterviewID = localStorage.getItem(STORAGE_KEYS.INTERVIEW_ID); + const savedInterviewURL = localStorage.getItem(STORAGE_KEYS.INTERVIEW_URL); + const wasStreaming = + localStorage.getItem(STORAGE_KEYS.IS_STREAMING) === 'true'; + + if (!savedInterviewID) return; + + try { + const session = await getInterviewSession(savedInterviewID); + + if (session.sessionStatus === 'ended') { + clearInterviewStorage(); + return; + } + + if (session.sessionStatus === 'queued') { + setInterviewID(savedInterviewID); + setIsQueued(true); + setQueuePosition(session.queuePosition); + setEstimatedWaitSec(session.estimatedWaitSec); + toast.loading("Reprise de la file d'attente…", { id: 'sim-queue' }); + + queueAbortRef.current?.abort(); + queueAbortRef.current = new AbortController(); + + const entrypoint = await waitForReadyEntrypoint( + savedInterviewID, + queueAbortRef.current.signal, + ({ queuePosition, estimatedWaitSec }) => { + setQueuePosition(queuePosition); + setEstimatedWaitSec(estimatedWaitSec); + }, + ); + + toast.dismiss('sim-queue'); + setIsQueued(false); + setQueuePosition(0); + localStorage.setItem(STORAGE_KEYS.INTERVIEW_URL, entrypoint); + setInputUrl(entrypoint); + setIsCallActive(true); + onConnect(entrypoint); + toast.success('Simulation reprise'); + return; + } + + const entrypoint = session.entrypoint ?? savedInterviewURL; + if (!entrypoint) { + await cancelInterview(savedInterviewID); + clearInterviewStorage(); + return; + } + + localStorage.setItem(STORAGE_KEYS.INTERVIEW_URL, entrypoint); + setInputUrl(entrypoint); + setInterviewID(savedInterviewID); + setIsCallActive(true); + onConnect(entrypoint); + + if (wasStreaming && onResumeStream) { + setTimeout(() => { + onResumeStream(); + }, STREAM_RESUME_DELAY_MS); + } + + toast.success('Simulation reprise'); + } catch (error) { + console.warn('Failed to restore simulation session:', error); + try { + await cancelInterview(savedInterviewID); + } catch { + /* ignore cleanup errors */ + } + clearInterviewStorage(); + toast.error( + 'La session précédente a expiré. Vous pouvez démarrer un nouvel entretien.', + ); } + }; - toast.success('Resumed your interview session'); - } + void restoreSavedSession(); }, [onConnect, onResumeStream]); useEffect(() => { @@ -174,6 +242,21 @@ export function useInterviewSession({ if (streaming) { try { + const staleInterviewId = localStorage.getItem(STORAGE_KEYS.INTERVIEW_ID); + if (staleInterviewId) { + onBeforeDisconnect?.(); + onDisconnect(WEBSOCKET_CLOSE_CODE_NORMAL, 'Replacing stale session'); + try { + await cancelInterview(staleInterviewId); + } catch { + /* slot may already be released */ + } + clearInterviewStorage(); + setIsCallActive(false); + setInputUrl(''); + setInterviewID(null); + } + const created = await createInterview({ type: 'technical', language: 'French', @@ -253,9 +336,7 @@ export function useInterviewSession({ if (interviewId) { try { await updateInterview(interviewId, { status: 'completed' }); - localStorage.removeItem(STORAGE_KEYS.INTERVIEW_ID); - localStorage.removeItem(STORAGE_KEYS.INTERVIEW_URL); - localStorage.removeItem(STORAGE_KEYS.IS_STREAMING); + clearInterviewStorage(); } catch (error) { console.error('Failed to update interview status:', error); } diff --git a/web/src/hooks/simulation/useSimulationWebSocket.ts b/web/src/hooks/simulation/useSimulationWebSocket.ts index 00a5efed..a4e0f4f3 100644 --- a/web/src/hooks/simulation/useSimulationWebSocket.ts +++ b/web/src/hooks/simulation/useSimulationWebSocket.ts @@ -28,6 +28,7 @@ export interface UseSimulationWebSocketReturn { sendMessage: (message: string) => void; sendJsonMessage: (message: object) => void; sendPing: (payload?: unknown) => void; + sendSessionStart: () => void; lastMessage: MessageEvent | null; lastJsonMessage: unknown; getWebSocket: () => ReturnType< @@ -156,6 +157,20 @@ export function useSimulationWebSocket( sendJsonMessage(pingMessage); }, [readyState, sendJsonMessage]); + const sendSessionStart = useCallback(() => { + if (readyState !== ReadyState.OPEN || !interviewIDRef.current) return; + + sendJsonMessage({ + type: 'session_start', + key: import.meta.env.VITE_WEBSOCKET_KEY, + interview_id: interviewIDRef.current, + stream_id: interviewIDRef.current, + format: 'text', + data: '{}', + timestamp: Date.now(), + }); + }, [readyState, sendJsonMessage]); + return { isConnected: readyState === ReadyState.OPEN, readyState, @@ -165,6 +180,7 @@ export function useSimulationWebSocket( sendMessage, sendJsonMessage, sendPing, + sendSessionStart, lastMessage, lastJsonMessage, getWebSocket, From 31f1c73e7a18a9959507b19905f496ad1ea9b4eb Mon Sep 17 00:00:00 2001 From: Andriamanampisoa Date: Tue, 14 Jul 2026 16:30:17 +0200 Subject: [PATCH 05/16] build: fix timeout while installing the dependencies --- ai/core/server/Dockerfile | 43 +++++++++++++++---- ai/microservices/behavior-analyzer/Dockerfile | 11 ++++- .../emotional-analyzer/Dockerfile | 11 ++++- ai/microservices/speech-to-speech/Dockerfile | 4 +- ai/microservices/verbal-analyzer/Dockerfile | 6 ++- 5 files changed, 63 insertions(+), 12 deletions(-) diff --git a/ai/core/server/Dockerfile b/ai/core/server/Dockerfile index 7eb48991..9efc317c 100644 --- a/ai/core/server/Dockerfile +++ b/ai/core/server/Dockerfile @@ -1,10 +1,27 @@ FROM debian:bullseye-slim AS builder -RUN apt-get update && apt-get install -y \ - g++ cmake git curl libcurl4-openssl-dev wget libboost-all-dev libssl-dev && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -RUN wget https://github.com/Kitware/CMake/releases/download/v3.31.6/cmake-3.31.6-linux-x86_64.tar.gz && \ +# Reduce apt failures on slow/unstable networks (mirrors, large metapackages). +RUN printf '%s\n' \ + 'Acquire::Retries "5";' \ + 'Acquire::http::Timeout "120";' \ + 'Acquire::https::Timeout "120";' \ + > /etc/apt/apt.conf.d/80-retries + +RUN apt-get update && apt-get install -y --no-install-recommends \ + g++ \ + make \ + git \ + curl \ + wget \ + ca-certificates \ + libcurl4-openssl-dev \ + libssl-dev \ + libboost-system-dev \ + libboost-date-time-dev \ + libboost1.74-dev \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +RUN wget -q https://github.com/Kitware/CMake/releases/download/v3.31.6/cmake-3.31.6-linux-x86_64.tar.gz && \ tar -xvzf cmake-3.31.6-linux-x86_64.tar.gz --strip-components=1 -C /usr/local && \ rm cmake-3.31.6-linux-x86_64.tar.gz @@ -15,9 +32,19 @@ RUN rm -rf build && cmake -B build -S . && cmake --build build FROM debian:bullseye-slim -RUN apt-get update && apt-get install -y \ - libcurl4-openssl-dev libssl-dev && \ - apt-get clean && rm -rf /var/lib/apt/lists/* +RUN printf '%s\n' \ + 'Acquire::Retries "5";' \ + 'Acquire::http::Timeout "120";' \ + 'Acquire::https::Timeout "120";' \ + > /etc/apt/apt.conf.d/80-retries + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libcurl4 \ + libssl1.1 \ + libboost-system1.74.0 \ + libboost-date-time1.74.0 \ + ca-certificates \ + && apt-get clean && rm -rf /var/lib/apt/lists/* WORKDIR /app COPY --from=builder /app/build/talkup_ai_mic_server /app/talkup_ai_mic_server diff --git a/ai/microservices/behavior-analyzer/Dockerfile b/ai/microservices/behavior-analyzer/Dockerfile index 74889e4b..051019e3 100644 --- a/ai/microservices/behavior-analyzer/Dockerfile +++ b/ai/microservices/behavior-analyzer/Dockerfile @@ -1,5 +1,14 @@ FROM python:3.10-slim + WORKDIR /app + +ENV PIP_DEFAULT_TIMEOUT=120 \ + PIP_RETRIES=5 + +COPY requirements.txt . +RUN pip install --upgrade pip && \ + pip install --no-cache-dir -r requirements.txt + COPY . . -RUN pip install -r requirements.txt + CMD ["python", "main.py"] diff --git a/ai/microservices/emotional-analyzer/Dockerfile b/ai/microservices/emotional-analyzer/Dockerfile index 74889e4b..051019e3 100644 --- a/ai/microservices/emotional-analyzer/Dockerfile +++ b/ai/microservices/emotional-analyzer/Dockerfile @@ -1,5 +1,14 @@ FROM python:3.10-slim + WORKDIR /app + +ENV PIP_DEFAULT_TIMEOUT=120 \ + PIP_RETRIES=5 + +COPY requirements.txt . +RUN pip install --upgrade pip && \ + pip install --no-cache-dir -r requirements.txt + COPY . . -RUN pip install -r requirements.txt + CMD ["python", "main.py"] diff --git a/ai/microservices/speech-to-speech/Dockerfile b/ai/microservices/speech-to-speech/Dockerfile index 28ae8fba..f034bf6e 100644 --- a/ai/microservices/speech-to-speech/Dockerfile +++ b/ai/microservices/speech-to-speech/Dockerfile @@ -1,7 +1,9 @@ FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04 WORKDIR /app -ENV PYTHONPATH=/app:$PYTHONPATH +ENV PYTHONPATH=/app:$PYTHONPATH \ + PIP_DEFAULT_TIMEOUT=120 \ + PIP_RETRIES=5 RUN apt-get update && apt-get install -y \ python3.11 python3.11-dev python3-pip git espeak-ng wget ffmpeg \ diff --git a/ai/microservices/verbal-analyzer/Dockerfile b/ai/microservices/verbal-analyzer/Dockerfile index dbf349c1..34ba3cbb 100644 --- a/ai/microservices/verbal-analyzer/Dockerfile +++ b/ai/microservices/verbal-analyzer/Dockerfile @@ -1,7 +1,11 @@ FROM python:3.10-slim WORKDIR /app +ENV PIP_DEFAULT_TIMEOUT=120 \ + PIP_RETRIES=5 + COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --upgrade pip && \ + pip install --no-cache-dir -r requirements.txt COPY . . EXPOSE 8006 HEALTHCHECK --interval=10s --timeout=5s --retries=5 --start-period=10s \ From b9b9925cb2096692ca587d96087c40b451ddf46b Mon Sep 17 00:00:00 2001 From: Andriamanampisoa Date: Tue, 14 Jul 2026 16:30:49 +0200 Subject: [PATCH 06/16] tests: add tests for the sts pipeline --- .../tests/test_pipeline_validation.py | 59 ++++++++++++++++++ .../speech-to-speech/tests/test_ws_auth.py | 60 +++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 ai/microservices/speech-to-speech/tests/test_pipeline_validation.py create mode 100644 ai/microservices/speech-to-speech/tests/test_ws_auth.py diff --git a/ai/microservices/speech-to-speech/tests/test_pipeline_validation.py b/ai/microservices/speech-to-speech/tests/test_pipeline_validation.py new file mode 100644 index 00000000..10fe7ff2 --- /dev/null +++ b/ai/microservices/speech-to-speech/tests/test_pipeline_validation.py @@ -0,0 +1,59 @@ +"""Unit tests for STT hallucination filtering.""" + +from __future__ import annotations + +from engine.transcription_validation import ( + contains_hallucination_pattern, + should_drop_segment_text, + validate_transcription, +) + + +class TestHallucinationPatterns: + def test_detects_sous_titrage_st_501(self) -> None: + assert contains_hallucination_pattern("Sous-titrage ST' 501") + + def test_detects_amara_subtitles(self) -> None: + assert contains_hallucination_pattern( + "Sous-titres réalisés par la communauté d'Amara.org", + ) + + def test_accepts_normal_interview_answer(self) -> None: + assert not contains_hallucination_pattern( + "J'ai cinq ans d'expérience en développement backend avec Python.", + ) + + +class TestValidateTranscription: + def test_rejects_obvious_subtitle_hallucination(self) -> None: + valid, reason = validate_transcription("Sous-titrage ST' 501") + assert not valid + assert reason == "hallucination" + + def test_rejects_repetitive_gibberish(self) -> None: + valid, reason = validate_transcription("euh euh euh") + assert not valid + assert reason == "repetition" + + def test_accepts_substantive_answer(self) -> None: + valid, reason = validate_transcription( + "Je travaille principalement sur des APIs REST et des microservices.", + ) + assert valid + assert reason == "" + + +class TestShouldDropSegment: + def test_drops_subtitle_segment_even_when_confident(self) -> None: + assert should_drop_segment_text( + "Sous-titrage ST' 501", + no_speech_prob=0.1, + avg_logprob=-0.1, + ) + + def test_keeps_confident_interview_speech(self) -> None: + assert not should_drop_segment_text( + "Je suis développeur full stack depuis trois ans.", + no_speech_prob=0.1, + avg_logprob=-0.2, + ) diff --git a/ai/microservices/speech-to-speech/tests/test_ws_auth.py b/ai/microservices/speech-to-speech/tests/test_ws_auth.py new file mode 100644 index 00000000..adc66318 --- /dev/null +++ b/ai/microservices/speech-to-speech/tests/test_ws_auth.py @@ -0,0 +1,60 @@ +## +## Talkup Project, 2026 +## TalkUp.AI +## STS WebSocket auth tests. +## + +from __future__ import annotations + +import os +import time + +import jwt +import pytest + +from engine.ws_auth import ( + interview_id_allowed, + verify_ws_token, +) + + +@pytest.fixture +def jwt_secret(monkeypatch: pytest.MonkeyPatch) -> str: + secret = "test-secret-key" + monkeypatch.setenv("JWT_SECRET", secret) + return secret + + +def test_verify_ws_token_valid(jwt_secret: str) -> None: + token = jwt.encode( + { + "purpose": "simulation_ws", + "interviewId": "int-abc", + "userId": "user-1", + "exp": int(time.time()) + 3600, + }, + jwt_secret, + algorithm="HS256", + ) + claims = verify_ws_token(token) + assert claims is not None + assert claims.interview_id == "int-abc" + assert claims.user_id == "user-1" + + +def test_verify_ws_token_wrong_purpose(jwt_secret: str) -> None: + token = jwt.encode( + {"purpose": "other", "interviewId": "x", "userId": "y"}, + jwt_secret, + algorithm="HS256", + ) + assert verify_ws_token(token) is None + + +def test_interview_id_allowed() -> None: + from engine.ws_auth import WsSessionClaims + + claims = WsSessionClaims("int-1", "user-1", "simulation_ws") + assert interview_id_allowed(claims, "int-1") is True + assert interview_id_allowed(claims, "int-2") is False + assert interview_id_allowed(None, "int-2") is True From dc6cd2a941bc924edf186cd55664b083c591867d Mon Sep 17 00:00:00 2001 From: Andriamanampisoa Date: Tue, 14 Jul 2026 16:40:21 +0200 Subject: [PATCH 07/16] fix: the duplicate session_start hang --- .../speech-to-speech/engine/stsProcess.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ai/microservices/speech-to-speech/engine/stsProcess.py b/ai/microservices/speech-to-speech/engine/stsProcess.py index f4c119c5..6847a669 100644 --- a/ai/microservices/speech-to-speech/engine/stsProcess.py +++ b/ai/microservices/speech-to-speech/engine/stsProcess.py @@ -245,6 +245,18 @@ async def _process_session_start_and_reply( ) if not result.ai_response: + if request_id is not None: + await _ws_send_json( + websocket, + send_lock, + { + "type": "sts_result", + "transcription": "", + "response": "", + "audio_chunks": [], + "request_id": request_id, + }, + ) return NOTIFIER.send_notification( From 04a65cf4430f17f1d27dc6bc50b9f6fa10e420c9 Mon Sep 17 00:00:00 2001 From: Andriamanampisoa Date: Tue, 14 Jul 2026 16:41:58 +0200 Subject: [PATCH 08/16] fix: the early PARCOURS transition --- .../speech-to-speech/engine/interview_flow.py | 13 ++++++------- .../speech-to-speech/tests/test_interview_flow.py | 8 ++++++++ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/ai/microservices/speech-to-speech/engine/interview_flow.py b/ai/microservices/speech-to-speech/engine/interview_flow.py index 3de5b798..2ec92145 100644 --- a/ai/microservices/speech-to-speech/engine/interview_flow.py +++ b/ai/microservices/speech-to-speech/engine/interview_flow.py @@ -151,13 +151,12 @@ def get_phase_instruction( "(qui il est, sa formation, sa situation actuelle en quelques phrases). " "Ne pose pas encore de questions sur le detail de ses experiences professionnelles." ) - if user_turn_count <= 2: - return ( - "Phase actuelle : PRESENTATION. Le candidat est en train de se presenter. " - "Rebondis sur ce qu'il vient de dire. Si sa presentation est incomplete, " - "pose une question de relance douce pour en savoir plus sur lui. " - "N'abord pas encore les experiences professionnelles en detail." - ) + return ( + "Phase actuelle : PRESENTATION. Le candidat est en train de se presenter. " + "Rebondis sur ce qu'il vient de dire. Si sa presentation est incomplete, " + "pose une question de relance douce pour en savoir plus sur lui. " + "N'abord pas encore les experiences professionnelles en detail." + ) if user_turn_count <= 6: return ( diff --git a/ai/microservices/speech-to-speech/tests/test_interview_flow.py b/ai/microservices/speech-to-speech/tests/test_interview_flow.py index 98939019..d5c1cd80 100644 --- a/ai/microservices/speech-to-speech/tests/test_interview_flow.py +++ b/ai/microservices/speech-to-speech/tests/test_interview_flow.py @@ -45,6 +45,14 @@ def test_presentation_phase_before_experience() -> None: assert "PRESENTATION" in instruction assert "experiences professionnelles" in instruction.lower() + still_presenting = get_phase_instruction( + user_turn_count=3, + presentation_done=False, + latest_user_text="Je suis en master informatique cette annee.", + ) + assert "PRESENTATION" in still_presenting + assert "PARCOURS" not in still_presenting + parcours = get_phase_instruction( user_turn_count=3, presentation_done=True, From 3252fbef26d93fcd7e9e3668b3820e39c5185851 Mon Sep 17 00:00:00 2001 From: Andriamanampisoa Date: Tue, 14 Jul 2026 16:43:17 +0200 Subject: [PATCH 09/16] fix: the flow state revival after clear --- .../speech-to-speech/engine/pipeline.py | 2 +- .../tests/test_interview_flow.py | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/ai/microservices/speech-to-speech/engine/pipeline.py b/ai/microservices/speech-to-speech/engine/pipeline.py index 8455b9d4..f4a8f85d 100644 --- a/ai/microservices/speech-to-speech/engine/pipeline.py +++ b/ai/microservices/speech-to-speech/engine/pipeline.py @@ -158,7 +158,7 @@ def process_sts_request( elif user_turn_count >= CLOSING_TURN_THRESHOLD and flow: flow.farewell_sent = True - if flow: + if flow and not farewell_sent: mark_presentation_done(interview_id, user_text) append_session_history(interview_id, user_text, ai_response) diff --git a/ai/microservices/speech-to-speech/tests/test_interview_flow.py b/ai/microservices/speech-to-speech/tests/test_interview_flow.py index d5c1cd80..ea62fec6 100644 --- a/ai/microservices/speech-to-speech/tests/test_interview_flow.py +++ b/ai/microservices/speech-to-speech/tests/test_interview_flow.py @@ -94,3 +94,25 @@ def test_mark_presentation_done() -> None: ) assert InterviewFlowStore.get("int-test").presentation_done is True InterviewFlowStore.clear("int-test") + + +def test_farewell_ack_does_not_revive_cleared_flow() -> None: + interview_id = "int-farewell-clear" + InterviewFlowStore.clear(interview_id) + flow = InterviewFlowStore.get(interview_id) + flow.farewell_sent = True + flow.presentation_done = True + + farewell_sent = flow.farewell_sent + InterviewFlowStore.clear(interview_id) + + if not farewell_sent: + mark_presentation_done( + interview_id, + "Je suis Mathéo, développeur .NET avec 4 ans d'expérience en React et C#.", + ) + + revived = InterviewFlowStore.get(interview_id) + assert revived.presentation_done is False + assert revived.farewell_sent is False + InterviewFlowStore.clear(interview_id) From 36e1b9bf71fc3c7ea30031e3f65c665cb317a35a Mon Sep 17 00:00:00 2001 From: Andriamanampisoa Date: Tue, 14 Jul 2026 16:52:54 +0200 Subject: [PATCH 10/16] fix: Include simulation_complete in the success payload when the pipeline marks the interview as done --- ai/microservices/speech-to-speech/engine/stsProcess.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ai/microservices/speech-to-speech/engine/stsProcess.py b/ai/microservices/speech-to-speech/engine/stsProcess.py index 6847a669..4d72b6d1 100644 --- a/ai/microservices/speech-to-speech/engine/stsProcess.py +++ b/ai/microservices/speech-to-speech/engine/stsProcess.py @@ -179,6 +179,8 @@ async def _process_stream_and_reply( "response": result.ai_response, "audio_chunks": encoded_chunks, } + if result.simulation_complete: + payload["simulation_complete"] = True if request_id is not None: payload["request_id"] = request_id From 0f1ad9fa15b844107cc33ca45279172e503d08e3 Mon Sep 17 00:00:00 2001 From: Andriamanampisoa Date: Tue, 14 Jul 2026 16:55:21 +0200 Subject: [PATCH 11/16] fix: reset greetingSentRef in onClose so reconnects trigger another session_start --- .../speech-to-speech/engine/pipeline.py | 17 +++++++++++++++-- .../speech-to-speech/engine/session_context.py | 9 +++++++++ .../organisms/simulation-workspace/index.tsx | 1 + 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/ai/microservices/speech-to-speech/engine/pipeline.py b/ai/microservices/speech-to-speech/engine/pipeline.py index f4a8f85d..2a8f3930 100644 --- a/ai/microservices/speech-to-speech/engine/pipeline.py +++ b/ai/microservices/speech-to-speech/engine/pipeline.py @@ -16,7 +16,12 @@ from .audio_decode import decode_audio_to_float32 from .models import STSModels, generate_ai_response, synthesize_tts_chunks from .simulation_brief import build_messages_for_turn, build_messages_for_opening -from .session_context import append_session_history, append_assistant_turn, fetch_session_history +from .session_context import ( + append_session_history, + append_assistant_turn, + fetch_session_history, + find_stored_opening_greeting, +) from .interview_flow import ( InterviewFlowStore, CLOSING_TURN_THRESHOLD, @@ -181,7 +186,15 @@ def generate_opening_greeting( flow = InterviewFlowStore.get(interview_id) if flow.greeting_sent: - return STSResult(transcription="", ai_response="", audio_chunks=[]) + stored_greeting = find_stored_opening_greeting(interview_id) + if stored_greeting: + audio_chunks = list(synthesize_tts_chunks(models, stored_greeting)) + return STSResult( + transcription="", + ai_response=stored_greeting, + audio_chunks=audio_chunks, + ) + flow.greeting_sent = False messages = build_messages_for_opening( models.settings.system_prompt, diff --git a/ai/microservices/speech-to-speech/engine/session_context.py b/ai/microservices/speech-to-speech/engine/session_context.py index c28322ac..10085727 100644 --- a/ai/microservices/speech-to-speech/engine/session_context.py +++ b/ai/microservices/speech-to-speech/engine/session_context.py @@ -149,6 +149,15 @@ def fetch_session_history(interview_id: str) -> list[dict[str, str]]: return [] +def find_stored_opening_greeting(interview_id: str) -> str | None: + for turn in fetch_session_history(interview_id): + if turn.get("role") == "assistant" and isinstance(turn.get("content"), str): + content = turn["content"].strip() + if content: + return content + return None + + def build_messages_from_nest_session( default_system_prompt: str, session: dict[str, Any], diff --git a/web/src/components/organisms/simulation-workspace/index.tsx b/web/src/components/organisms/simulation-workspace/index.tsx index 7b425287..75e5cbb5 100644 --- a/web/src/components/organisms/simulation-workspace/index.tsx +++ b/web/src/components/organisms/simulation-workspace/index.tsx @@ -116,6 +116,7 @@ export function SimulationWorkspace({ setConnectionAttempts(0); }, onClose: (event) => { + greetingSentRef.current = false; if (event.code !== 1000 && event.code !== 1001) { setWsError( `Connection closed: ${event.code} - ${event.reason || 'Unknown reason'}`, From acb33eb3a536bad2592ce3c2f7133dc0c52b1c7a Mon Sep 17 00:00:00 2001 From: BhuvanArn Date: Tue, 14 Jul 2026 18:38:02 +0200 Subject: [PATCH 12/16] fix: address PR #170 review - greeting replay on reconnect, keyword substring match, prettier, tsc, test gaps --- .../speech-to-speech/engine/interview_flow.py | 18 ++++-- .../speech-to-speech/engine/pipeline.py | 11 ++++ .../tests/test_interview_flow.py | 20 +++++++ .../simulation-context.service.spec.ts | 55 +++++++++++++++++++ .../simulation-internal.controller.spec.ts | 20 ++++++- .../hooks/simulation/useInterviewSession.ts | 13 ++++- web/src/routes/-simulations.spec.tsx | 1 + 7 files changed, 129 insertions(+), 9 deletions(-) diff --git a/ai/microservices/speech-to-speech/engine/interview_flow.py b/ai/microservices/speech-to-speech/engine/interview_flow.py index 2ec92145..3ba4713d 100644 --- a/ai/microservices/speech-to-speech/engine/interview_flow.py +++ b/ai/microservices/speech-to-speech/engine/interview_flow.py @@ -52,12 +52,22 @@ ) _EXPERIENCE_KEYWORDS = frozenset({ - "experience", "experiences", "travaille", "travaille", "poste", "entreprise", + "experience", "experiences", "travaille", "poste", "entreprise", "stage", "alternance", "diplome", "formation", "projet", "developpeur", "developpement", "ingenieur", "ans", "annee", "annees", "carriere", "parcours", "competence", "competences", "mission", "missions", "cdi", "cdd", }) +# Split on any non-letter so keywords are matched as whole words, not +# substrings. Without this, "ans" would match inside common words like +# "dans"/"sans" and wrongly flag a trivial reply as an experience intro. +_WORD_TOKEN_PATTERN = re.compile(r"[^a-zàâäéèêëïîôùûüç]+") + + +def _has_experience_keyword(normalized: str) -> bool: + tokens = {tok for tok in _WORD_TOKEN_PATTERN.split(normalized) if tok} + return not tokens.isdisjoint(_EXPERIENCE_KEYWORDS) + @dataclass class InterviewFlowState: @@ -111,9 +121,7 @@ def _looks_like_name_only(text: str) -> bool: if pattern.match(normalized): return True - if len(normalized.split()) <= 6 and not any( - keyword in normalized for keyword in _EXPERIENCE_KEYWORDS - ): + if len(normalized.split()) <= 6 and not _has_experience_keyword(normalized): return True return False @@ -123,7 +131,7 @@ def _user_has_substantial_intro(text: str) -> bool: normalized = _normalize_text(text) if len(normalized.split()) >= 20: return True - return any(keyword in normalized for keyword in _EXPERIENCE_KEYWORDS) + return _has_experience_keyword(normalized) def mark_presentation_done(interview_id: str, user_text: str) -> None: diff --git a/ai/microservices/speech-to-speech/engine/pipeline.py b/ai/microservices/speech-to-speech/engine/pipeline.py index 2a8f3930..7ea21a1c 100644 --- a/ai/microservices/speech-to-speech/engine/pipeline.py +++ b/ai/microservices/speech-to-speech/engine/pipeline.py @@ -184,8 +184,19 @@ def generate_opening_greeting( if not interview_id: return STSResult(transcription="", ai_response="", audio_chunks=[]) + # A reconnect (or restored session) re-sends session_start while the + # interview is already under way. Once the candidate has spoken at least + # once, the opening greeting is stale: replaying or regenerating it would + # make the recruiter say "welcome, please introduce yourself" mid-interview. + # Stay silent in that case and let the normal turn flow resume. + if count_user_turns(fetch_session_history(interview_id)) > 0: + return STSResult(transcription="", ai_response="", audio_chunks=[]) + flow = InterviewFlowStore.get(interview_id) if flow.greeting_sent: + # Greeting already delivered but no user turn yet (e.g. a refresh right + # after the welcome): replay the stored opening so the candidate isn't + # left in silence, rather than generating a second, different one. stored_greeting = find_stored_opening_greeting(interview_id) if stored_greeting: audio_chunks = list(synthesize_tts_chunks(models, stored_greeting)) diff --git a/ai/microservices/speech-to-speech/tests/test_interview_flow.py b/ai/microservices/speech-to-speech/tests/test_interview_flow.py index ea62fec6..47fa8807 100644 --- a/ai/microservices/speech-to-speech/tests/test_interview_flow.py +++ b/ai/microservices/speech-to-speech/tests/test_interview_flow.py @@ -96,6 +96,26 @@ def test_mark_presentation_done() -> None: InterviewFlowStore.clear("int-test") +def test_short_reply_does_not_mark_presentation_done() -> None: + # Experience keywords are matched as whole words, not substrings, so the + # "ans" keyword must NOT fire inside "dans"/"sans" and a trivial reply must + # leave the candidate in the PRESENTATION phase. + for reply in ("Sans souci.", "J'habite dans le sud.", "Oui bien sur.", "De Nantes."): + interview_id = "int-short-" + reply[:5] + InterviewFlowStore.clear(interview_id) + mark_presentation_done(interview_id, reply) + assert ( + InterviewFlowStore.get(interview_id).presentation_done is False + ), reply + InterviewFlowStore.clear(interview_id) + + # A genuine experience keyword ("ans" as a standalone token) still counts. + InterviewFlowStore.clear("int-short-exp") + mark_presentation_done("int-short-exp", "J'ai 3 ans d'experience en developpement.") + assert InterviewFlowStore.get("int-short-exp").presentation_done is True + InterviewFlowStore.clear("int-short-exp") + + def test_farewell_ack_does_not_revive_cleared_flow() -> None: interview_id = "int-farewell-clear" InterviewFlowStore.clear(interview_id) diff --git a/server/src/modules/simulation/simulation-context.service.spec.ts b/server/src/modules/simulation/simulation-context.service.spec.ts index ba1e0b31..8d149f0a 100644 --- a/server/src/modules/simulation/simulation-context.service.spec.ts +++ b/server/src/modules/simulation/simulation-context.service.spec.ts @@ -151,6 +151,61 @@ describe("SimulationContextService", () => { }); }); + describe("appendAssistantTurn", () => { + const baseCtx = { + interviewId: "int-1", + userId: "user-1", + systemPrompt: "SYSTEM", + history: [] as { role: string; content: string }[], + }; + + it("appends a single assistant turn, persists with a TTL, and touches heartbeat", async () => { + redis.get.mockResolvedValueOnce(JSON.stringify(baseCtx)); + + await service.appendAssistantTurn("int-1", "Bonjour et bienvenue"); + + expect(redis.set).toHaveBeenCalledWith( + SimRedisKeys.context("int-1"), + expect.any(String), + "EX", + expect.any(Number), + ); + const saved = JSON.parse(redis.set.mock.calls[0][1]); + expect(saved.history).toEqual([ + { role: "assistant", content: "Bonjour et bienvenue" }, + ]); + expect(capacity.touchHeartbeat).toHaveBeenCalledWith("int-1", "user-1"); + }); + + it("trims history to the last historyMaxTurns*2 entries", async () => { + const longHistory = Array.from({ length: 60 }, (_, i) => ({ + role: i % 2 === 0 ? "user" : "assistant", + content: `m${i}`, + })); + redis.get.mockResolvedValueOnce( + JSON.stringify({ ...baseCtx, history: longHistory }), + ); + + await service.appendAssistantTurn("int-1", "latest-assistant"); + + const saved = JSON.parse(redis.set.mock.calls[0][1]); + // historyMaxTurns defaults to 30 -> keep last 60 entries. + expect(saved.history.length).toBe(60); + expect(saved.history[saved.history.length - 1]).toEqual({ + role: "assistant", + content: "latest-assistant", + }); + }); + + it("propagates NotFoundException when no context is stored", async () => { + redis.get.mockResolvedValueOnce(null); + + await expect(service.appendAssistantTurn("int-1", "x")).rejects.toThrow( + NotFoundException, + ); + }); + }); + describe("deleteContext", () => { it("deletes the context key", async () => { await service.deleteContext("int-1"); diff --git a/server/src/modules/simulation/simulation-internal.controller.spec.ts b/server/src/modules/simulation/simulation-internal.controller.spec.ts index 5ff630d3..6ee55371 100644 --- a/server/src/modules/simulation/simulation-internal.controller.spec.ts +++ b/server/src/modules/simulation/simulation-internal.controller.spec.ts @@ -6,7 +6,11 @@ import { SimulationVerbalAnalysisService } from "./simulation-verbal-analysis.se describe("SimulationInternalController", () => { let controller: SimulationInternalController; - let context: { getContextForSts: jest.Mock; appendTurn: jest.Mock }; + let context: { + getContextForSts: jest.Mock; + appendTurn: jest.Mock; + appendAssistantTurn: jest.Mock; + }; let verbalAnalysis: { saveForInterview: jest.Mock; getForInterview: jest.Mock; @@ -16,6 +20,7 @@ describe("SimulationInternalController", () => { context = { getContextForSts: jest.fn(), appendTurn: jest.fn(), + appendAssistantTurn: jest.fn(), }; verbalAnalysis = { @@ -60,4 +65,17 @@ describe("SimulationInternalController", () => { expect(context.appendTurn).toHaveBeenCalledWith("int-1", "hello", "hi"); }); + + it("appends a standalone assistant turn from the DTO", async () => { + context.appendAssistantTurn.mockResolvedValueOnce(undefined); + + await controller.appendAssistantTurn("int-1", { + assistantText: "Bonjour et bienvenue", + }); + + expect(context.appendAssistantTurn).toHaveBeenCalledWith( + "int-1", + "Bonjour et bienvenue", + ); + }); }); diff --git a/web/src/hooks/simulation/useInterviewSession.ts b/web/src/hooks/simulation/useInterviewSession.ts index b3fd5380..ccbde441 100644 --- a/web/src/hooks/simulation/useInterviewSession.ts +++ b/web/src/hooks/simulation/useInterviewSession.ts @@ -129,7 +129,9 @@ export function useInterviewSession({ const restoreSavedSession = async () => { const savedInterviewID = localStorage.getItem(STORAGE_KEYS.INTERVIEW_ID); - const savedInterviewURL = localStorage.getItem(STORAGE_KEYS.INTERVIEW_URL); + const savedInterviewURL = localStorage.getItem( + STORAGE_KEYS.INTERVIEW_URL, + ); const wasStreaming = localStorage.getItem(STORAGE_KEYS.IS_STREAMING) === 'true'; @@ -242,10 +244,15 @@ export function useInterviewSession({ if (streaming) { try { - const staleInterviewId = localStorage.getItem(STORAGE_KEYS.INTERVIEW_ID); + const staleInterviewId = localStorage.getItem( + STORAGE_KEYS.INTERVIEW_ID, + ); if (staleInterviewId) { onBeforeDisconnect?.(); - onDisconnect(WEBSOCKET_CLOSE_CODE_NORMAL, 'Replacing stale session'); + onDisconnect( + WEBSOCKET_CLOSE_CODE_NORMAL, + 'Replacing stale session', + ); try { await cancelInterview(staleInterviewId); } catch { diff --git a/web/src/routes/-simulations.spec.tsx b/web/src/routes/-simulations.spec.tsx index 2b201209..2d3fd7f5 100644 --- a/web/src/routes/-simulations.spec.tsx +++ b/web/src/routes/-simulations.spec.tsx @@ -70,6 +70,7 @@ const mockHandleStreamToggle = vi.fn(); const defaultAudioPlaybackReturn = { isAiSpeaking: false, + simulationComplete: false, stopPlayback: vi.fn(), error: null, transcript: null, From 0f2cd211721b2c5ebfca81526065010d7a6d8dad Mon Sep 17 00:00:00 2001 From: Andriamanampisoa Date: Tue, 14 Jul 2026 19:10:26 +0200 Subject: [PATCH 13/16] fix: session start drops VA followups --- ai/core/server/src/network/MicroservicesManager.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ai/core/server/src/network/MicroservicesManager.cpp b/ai/core/server/src/network/MicroservicesManager.cpp index 837a035b..e6c06c28 100644 --- a/ai/core/server/src/network/MicroservicesManager.cpp +++ b/ai/core/server/src/network/MicroservicesManager.cpp @@ -980,12 +980,18 @@ void talkup_network::MicroservicesManager::process_session_start_job( if (msg_type == "pong" || msg_type == "simulation_context_ack") continue; + if (try_dispatch_sts_va_followup(msg_json)) + continue; + if (!msg_json.contains("request_id")) continue; const uint64_t response_id = msg_json.value("request_id", static_cast(0)); - if (response_id != request_id) + if (response_id != request_id) { + if (try_dispatch_sts_va_followup(msg_json)) + continue; continue; + } if (msg_type == "sts_result") { if (callback) From 3845012fe9954a23ae40f7eac8fc72b3ac061a5e Mon Sep 17 00:00:00 2001 From: Andriamanampisoa Date: Tue, 14 Jul 2026 19:11:18 +0200 Subject: [PATCH 14/16] fix: restore failure cancels interview --- .../hooks/simulation/useInterviewSession.ts | 85 ++++++++++++++----- 1 file changed, 64 insertions(+), 21 deletions(-) diff --git a/web/src/hooks/simulation/useInterviewSession.ts b/web/src/hooks/simulation/useInterviewSession.ts index ccbde441..ebcc44db 100644 --- a/web/src/hooks/simulation/useInterviewSession.ts +++ b/web/src/hooks/simulation/useInterviewSession.ts @@ -5,6 +5,7 @@ import { heartbeatInterview, updateInterview, } from '@/services/ai/http'; +import axios from 'axios'; import { useCallback, useEffect, useRef, useState } from 'react'; import toast from 'react-hot-toast'; @@ -26,6 +27,25 @@ function clearInterviewStorage(): void { localStorage.removeItem(STORAGE_KEYS.IS_STREAMING); } +function isTerminalSessionRestoreError(error: unknown): boolean { + if (error instanceof Error) { + if (error.message === 'Simulation session ended before start') { + return true; + } + if ( + error.message === 'Queue wait aborted' || + error.message === 'Queue wait timed out' + ) { + return false; + } + } + + return ( + axios.isAxiosError(error) && + (error.response?.status === 404 || error.response?.status === 403) + ); +} + /** * Props for the useInterviewSession hook. */ @@ -80,20 +100,27 @@ async function waitForReadyEntrypoint( throw new Error('Queue wait aborted'); } - const session = await getInterviewSession(interviewId); - if (session.sessionStatus === 'ready' && session.entrypoint) { - return session.entrypoint; - } - if (session.sessionStatus === 'ended') { - throw new Error('Simulation session ended before start'); - } + try { + const session = await getInterviewSession(interviewId); + if (session.sessionStatus === 'ready' && session.entrypoint) { + return session.entrypoint; + } + if (session.sessionStatus === 'ended') { + throw new Error('Simulation session ended before start'); + } - // Surface the fresh position/wait so the queue banner counts down instead - // of showing the stale value captured at enqueue time. - onProgress?.({ - queuePosition: session.queuePosition, - estimatedWaitSec: session.estimatedWaitSec, - }); + // Surface the fresh position/wait so the queue banner counts down instead + // of showing the stale value captured at enqueue time. + onProgress?.({ + queuePosition: session.queuePosition, + estimatedWaitSec: session.estimatedWaitSec, + }); + } catch (error) { + if (isTerminalSessionRestoreError(error)) { + throw error; + } + console.warn('Transient queue poll failure, retrying:', error); + } await new Promise((resolve) => setTimeout(resolve, QUEUE_POLL_INTERVAL_MS)); } @@ -169,8 +196,9 @@ export function useInterviewSession({ setQueuePosition(0); localStorage.setItem(STORAGE_KEYS.INTERVIEW_URL, entrypoint); setInputUrl(entrypoint); - setIsCallActive(true); + setInterviewID(savedInterviewID); onConnect(entrypoint); + setIsCallActive(true); toast.success('Simulation reprise'); return; } @@ -185,8 +213,8 @@ export function useInterviewSession({ localStorage.setItem(STORAGE_KEYS.INTERVIEW_URL, entrypoint); setInputUrl(entrypoint); setInterviewID(savedInterviewID); - setIsCallActive(true); onConnect(entrypoint); + setIsCallActive(true); if (wasStreaming && onResumeStream) { setTimeout(() => { @@ -197,15 +225,30 @@ export function useInterviewSession({ toast.success('Simulation reprise'); } catch (error) { console.warn('Failed to restore simulation session:', error); - try { - await cancelInterview(savedInterviewID); - } catch { - /* ignore cleanup errors */ + + if (isTerminalSessionRestoreError(error)) { + try { + await cancelInterview(savedInterviewID); + } catch { + /* ignore cleanup errors */ + } + clearInterviewStorage(); + toast.error( + 'La session précédente a expiré. Vous pouvez démarrer un nouvel entretien.', + ); + return; } - clearInterviewStorage(); + toast.error( - 'La session précédente a expiré. Vous pouvez démarrer un nouvel entretien.', + 'Impossible de reprendre la simulation pour le moment. Rechargez la page pour réessayer.', ); + toast.dismiss('sim-queue'); + setIsCallActive(false); + setIsQueued(false); + setQueuePosition(0); + setEstimatedWaitSec(undefined); + setInterviewID(null); + setInputUrl(''); } }; From e7024c01b893608a0c867de88ec633cf5b7a1e7b Mon Sep 17 00:00:00 2001 From: Andriamanampisoa Date: Tue, 14 Jul 2026 19:44:16 +0200 Subject: [PATCH 15/16] refactor: changes required for the pr --- .../speech-to-speech/engine/interview_flow.py | 27 ++- .../tests/test_interview_flow.py | 14 ++ .../simulation/simulation.config.spec.ts | 33 ++++ .../modules/simulation/simulation.config.ts | 50 ++++-- .../modules/simulation/simulation.module.ts | 3 +- .../simulation/useInterviewSession.spec.ts | 164 ++++++++++++++++++ 6 files changed, 276 insertions(+), 15 deletions(-) create mode 100644 web/src/hooks/simulation/useInterviewSession.spec.ts diff --git a/ai/microservices/speech-to-speech/engine/interview_flow.py b/ai/microservices/speech-to-speech/engine/interview_flow.py index 3ba4713d..6c0b142d 100644 --- a/ai/microservices/speech-to-speech/engine/interview_flow.py +++ b/ai/microservices/speech-to-speech/engine/interview_flow.py @@ -63,12 +63,33 @@ # "dans"/"sans" and wrongly flag a trivial reply as an experience intro. _WORD_TOKEN_PATTERN = re.compile(r"[^a-zàâäéèêëïîôùûüç]+") +_NAME_LIKE_TOKEN = re.compile(r"^[a-zàâäéèêëïîôùûüç\-']{2,}$", re.IGNORECASE) + +# Common French function words and short acknowledgements that are not names. +_SHORT_REPLY_STOPWORDS = frozenset({ + "a", "ai", "au", "aux", "avec", "bien", "bonjour", "bonsoir", "ce", "cette", + "dans", "de", "des", "du", "en", "et", "habite", "hello", "ici", "il", "je", + "la", "le", "les", "mais", "me", "merci", "mon", "non", "nos", "notre", "oui", + "par", "pas", "peut", "pour", "pense", "que", "qui", "quoi", "salut", "sans", + "se", "si", "son", "sont", "souci", "suis", "sur", "tes", "toi", "ton", "tu", + "un", "une", "vos", "votre", "vous", +}) + def _has_experience_keyword(normalized: str) -> bool: tokens = {tok for tok in _WORD_TOKEN_PATTERN.split(normalized) if tok} return not tokens.isdisjoint(_EXPERIENCE_KEYWORDS) +def _has_name_like_token(normalized: str) -> bool: + for token in _WORD_TOKEN_PATTERN.split(normalized): + if not token or token in _SHORT_REPLY_STOPWORDS: + continue + if _NAME_LIKE_TOKEN.match(token): + return True + return False + + @dataclass class InterviewFlowState: greeting_sent: bool = False @@ -121,7 +142,11 @@ def _looks_like_name_only(text: str) -> bool: if pattern.match(normalized): return True - if len(normalized.split()) <= 6 and not _has_experience_keyword(normalized): + if ( + len(normalized.split()) <= 6 + and not _has_experience_keyword(normalized) + and _has_name_like_token(normalized) + ): return True return False diff --git a/ai/microservices/speech-to-speech/tests/test_interview_flow.py b/ai/microservices/speech-to-speech/tests/test_interview_flow.py index 47fa8807..e8a49e17 100644 --- a/ai/microservices/speech-to-speech/tests/test_interview_flow.py +++ b/ai/microservices/speech-to-speech/tests/test_interview_flow.py @@ -30,12 +30,26 @@ def test_count_user_turns() -> None: def test_looks_like_name_only() -> None: assert looks_like_name_only("Bonjour, je m'appelle Mathéo.") is True assert looks_like_name_only("Je m'appelle Mathéo") is True + assert looks_like_name_only("Mathéo") is True + assert looks_like_name_only("Oui bien sur.") is False + assert looks_like_name_only("Je pense que oui") is False + assert looks_like_name_only("Sans souci.") is False assert looks_like_name_only( "Je suis Mathéo, diplômé en informatique à Nantes, " "je travaille depuis 3 ans en développement web." ) is False +def test_short_acknowledgement_stays_in_presentation_not_name_only() -> None: + instruction = get_phase_instruction( + user_turn_count=1, + presentation_done=False, + latest_user_text="Oui bien sur.", + ) + assert "PRESENTATION" in instruction + assert "prenom" not in instruction.lower() + + def test_presentation_phase_before_experience() -> None: instruction = get_phase_instruction( user_turn_count=1, diff --git a/server/src/modules/simulation/simulation.config.spec.ts b/server/src/modules/simulation/simulation.config.spec.ts index 995f2f1e..f4e51f15 100644 --- a/server/src/modules/simulation/simulation.config.spec.ts +++ b/server/src/modules/simulation/simulation.config.spec.ts @@ -1,6 +1,9 @@ import { + loadAiInitTimeoutMs, loadSimulationConfig, + parsePositiveIntEnv, resolveAiWsPublicBase, + SIM_AI_INIT_TIMEOUT_MS_DEFAULT, } from "./simulation.config"; describe("simulation.config", () => { @@ -10,6 +13,36 @@ describe("simulation.config", () => { process.env = { ...ORIGINAL_ENV }; }); + describe("parsePositiveIntEnv", () => { + it("returns fallback for missing, invalid, zero, or negative values", () => { + expect(parsePositiveIntEnv(undefined, 30)).toBe(30); + expect(parsePositiveIntEnv("", 30)).toBe(30); + expect(parsePositiveIntEnv("abc", 30)).toBe(30); + expect(parsePositiveIntEnv("0", 30)).toBe(30); + expect(parsePositiveIntEnv("-5", 30)).toBe(30); + }); + + it("parses valid positive integers", () => { + expect(parsePositiveIntEnv("45", 30)).toBe(45); + expect(parsePositiveIntEnv("90.9", 30)).toBe(90); + }); + }); + + describe("loadAiInitTimeoutMs", () => { + it("defaults to 30s when unset or invalid", () => { + delete process.env.SIM_AI_INIT_TIMEOUT_MS; + expect(loadAiInitTimeoutMs()).toBe(SIM_AI_INIT_TIMEOUT_MS_DEFAULT); + + process.env.SIM_AI_INIT_TIMEOUT_MS = "not-a-number"; + expect(loadAiInitTimeoutMs()).toBe(SIM_AI_INIT_TIMEOUT_MS_DEFAULT); + }); + + it("parses a valid override", () => { + process.env.SIM_AI_INIT_TIMEOUT_MS = "45000"; + expect(loadAiInitTimeoutMs()).toBe(45_000); + }); + }); + describe("loadSimulationConfig", () => { it("returns defaults when no env vars are set", () => { delete process.env.SIM_MAX_CONCURRENT; diff --git a/server/src/modules/simulation/simulation.config.ts b/server/src/modules/simulation/simulation.config.ts index 964e04a0..0243eddc 100644 --- a/server/src/modules/simulation/simulation.config.ts +++ b/server/src/modules/simulation/simulation.config.ts @@ -10,22 +10,46 @@ export type SimulationConfig = { heartbeatIntervalSec: number; }; +export const SIM_AI_INIT_TIMEOUT_MS_DEFAULT = 30_000; + +/** Parse a positive integer env var; invalid or non-positive values use fallback. */ +export function parsePositiveIntEnv( + raw: string | undefined, + fallback: number, +): number { + const value = Number(raw); + return Number.isFinite(value) && value > 0 ? Math.trunc(value) : fallback; +} + +export function loadAiInitTimeoutMs(): number { + return parsePositiveIntEnv( + process.env.SIM_AI_INIT_TIMEOUT_MS, + SIM_AI_INIT_TIMEOUT_MS_DEFAULT, + ); +} + export function loadSimulationConfig(): SimulationConfig { return { - maxConcurrent: parseInt(process.env.SIM_MAX_CONCURRENT ?? "2", 10), - queueMaxSize: parseInt(process.env.SIM_QUEUE_MAX_SIZE ?? "20", 10), - slotTtlSec: parseInt(process.env.SIM_SLOT_TTL_SEC ?? "900", 10), - queueEntryTtlSec: parseInt( - process.env.SIM_QUEUE_ENTRY_TTL_SEC ?? "3600", - 10, + maxConcurrent: parsePositiveIntEnv(process.env.SIM_MAX_CONCURRENT, 2), + queueMaxSize: parsePositiveIntEnv(process.env.SIM_QUEUE_MAX_SIZE, 20), + slotTtlSec: parsePositiveIntEnv(process.env.SIM_SLOT_TTL_SEC, 900), + queueEntryTtlSec: parsePositiveIntEnv( + process.env.SIM_QUEUE_ENTRY_TTL_SEC, + 3600, + ), + wsTokenTtlSec: parsePositiveIntEnv(process.env.SIM_WS_TOKEN_TTL_SEC, 900), + estimatedTurnSec: parsePositiveIntEnv( + process.env.SIM_ESTIMATED_TURN_SEC, + 90, + ), + historyMaxTurns: parsePositiveIntEnv( + process.env.SIM_HISTORY_MAX_TURNS, + 30, ), - wsTokenTtlSec: parseInt(process.env.SIM_WS_TOKEN_TTL_SEC ?? "900", 10), - estimatedTurnSec: parseInt(process.env.SIM_ESTIMATED_TURN_SEC ?? "90", 10), - historyMaxTurns: parseInt(process.env.SIM_HISTORY_MAX_TURNS ?? "30", 10), - contextTtlSec: parseInt(process.env.SIM_CONTEXT_TTL_SEC ?? "7200", 10), - heartbeatIntervalSec: parseInt( - process.env.SIM_HEARTBEAT_INTERVAL_SEC ?? "60", - 10, + contextTtlSec: parsePositiveIntEnv(process.env.SIM_CONTEXT_TTL_SEC, 7200), + heartbeatIntervalSec: parsePositiveIntEnv( + process.env.SIM_HEARTBEAT_INTERVAL_SEC, + 60, ), }; } diff --git a/server/src/modules/simulation/simulation.module.ts b/server/src/modules/simulation/simulation.module.ts index 7bc11471..c9471a75 100644 --- a/server/src/modules/simulation/simulation.module.ts +++ b/server/src/modules/simulation/simulation.module.ts @@ -14,12 +14,13 @@ import { SimulationVerbalAnalysisService } from "./simulation-verbal-analysis.se import { SimulationCronService } from "./simulation.cron"; import { SimulationInternalController } from "./simulation-internal.controller"; import { InternalApiKeyGuard } from "./guards/internal-api-key.guard"; +import { loadAiInitTimeoutMs } from "./simulation.config"; @Module({ imports: [ RedisModule, HttpModule.register({ - timeout: parseInt(process.env.SIM_AI_INIT_TIMEOUT_MS ?? "30000", 10), + timeout: loadAiInitTimeoutMs(), }), TypeOrmModule.forFeature([ai_interview, ai_verbal_analysis]), ], diff --git a/web/src/hooks/simulation/useInterviewSession.spec.ts b/web/src/hooks/simulation/useInterviewSession.spec.ts new file mode 100644 index 00000000..cb0a2904 --- /dev/null +++ b/web/src/hooks/simulation/useInterviewSession.spec.ts @@ -0,0 +1,164 @@ +import { renderHook, waitFor } from '@testing-library/react'; +import axios from 'axios'; +import toast from 'react-hot-toast'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { useInterviewSession } from './useInterviewSession'; + +const mockGetInterviewSession = vi.fn(); +const mockCancelInterview = vi.fn(); +const mockCreateInterview = vi.fn(); +const mockUpdateInterview = vi.fn(); +const mockHeartbeatInterview = vi.fn(); + +vi.mock('@/services/ai/http', () => ({ + getInterviewSession: (...args: unknown[]) => mockGetInterviewSession(...args), + cancelInterview: (...args: unknown[]) => mockCancelInterview(...args), + createInterview: (...args: unknown[]) => mockCreateInterview(...args), + updateInterview: (...args: unknown[]) => mockUpdateInterview(...args), + heartbeatInterview: (...args: unknown[]) => mockHeartbeatInterview(...args), +})); + +vi.mock('react-hot-toast', () => ({ + default: { + success: vi.fn(), + error: vi.fn(), + loading: vi.fn(), + dismiss: vi.fn(), + }, +})); + +const localStorageMock = { + store: new Map(), + getItem: vi.fn((key: string) => localStorageMock.store.get(key) ?? null), + setItem: vi.fn((key: string, value: string) => { + localStorageMock.store.set(key, value); + }), + removeItem: vi.fn((key: string) => { + localStorageMock.store.delete(key); + }), + clear: vi.fn(() => { + localStorageMock.store.clear(); + }), +}; + +Object.defineProperty(window, 'localStorage', { + value: localStorageMock, +}); + +function makeAxiosError(status?: number) { + const error = new Error('request failed') as Error & { + isAxiosError: boolean; + response?: { status: number }; + }; + error.isAxiosError = true; + if (status !== undefined) { + error.response = { status }; + } + return error; +} + +describe('useInterviewSession restore', () => { + const onConnect = vi.fn(); + const onDisconnect = vi.fn(); + + beforeEach(() => { + localStorageMock.store.clear(); + vi.clearAllMocks(); + mockHeartbeatInterview.mockResolvedValue(undefined); + vi.spyOn(axios, 'isAxiosError').mockImplementation( + (error: unknown) => + typeof error === 'object' && + error !== null && + 'isAxiosError' in error && + (error as { isAxiosError?: boolean }).isAxiosError === true, + ); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('keeps saved session on transient restore failures', async () => { + localStorageMock.store.set('currentInterviewID', 'interview-1'); + localStorageMock.store.set('currentInterviewURL', 'wss://example.test/ws'); + mockGetInterviewSession.mockRejectedValue(makeAxiosError(503)); + + renderHook(() => + useInterviewSession({ + onConnect, + onDisconnect, + }), + ); + + await waitFor(() => { + expect(mockGetInterviewSession).toHaveBeenCalledWith('interview-1'); + }); + + expect(mockCancelInterview).not.toHaveBeenCalled(); + expect(localStorageMock.removeItem).not.toHaveBeenCalledWith( + 'currentInterviewID', + ); + expect(toast.error).toHaveBeenCalledWith( + 'Impossible de reprendre la simulation pour le moment. Rechargez la page pour réessayer.', + ); + }); + + it('cancels and clears storage when the saved session is gone', async () => { + localStorageMock.store.set('currentInterviewID', 'interview-1'); + mockGetInterviewSession.mockRejectedValue(makeAxiosError(404)); + mockCancelInterview.mockResolvedValue(undefined); + + renderHook(() => + useInterviewSession({ + onConnect, + onDisconnect, + }), + ); + + await waitFor(() => { + expect(mockCancelInterview).toHaveBeenCalledWith('interview-1'); + }); + + expect(localStorageMock.removeItem).toHaveBeenCalledWith( + 'currentInterviewID', + ); + expect(toast.error).toHaveBeenCalledWith( + 'La session précédente a expiré. Vous pouvez démarrer un nouvel entretien.', + ); + }); + + it('does not cancel when reconnect fails after a valid session lookup', async () => { + localStorageMock.store.set('currentInterviewID', 'interview-1'); + localStorageMock.store.set('currentInterviewURL', 'wss://example.test/ws'); + mockGetInterviewSession.mockResolvedValue({ + interviewID: 'interview-1', + dbStatus: 'in_progress', + sessionStatus: 'active', + queuePosition: 0, + entrypoint: 'wss://example.test/ws', + }); + onConnect.mockImplementation(() => { + throw new Error('WebSocket reconnect failed'); + }); + + renderHook(() => + useInterviewSession({ + onConnect, + onDisconnect, + }), + ); + + await waitFor(() => { + expect(onConnect).toHaveBeenCalledWith('wss://example.test/ws'); + }); + + expect(mockCancelInterview).not.toHaveBeenCalled(); + expect(localStorageMock.removeItem).not.toHaveBeenCalledWith( + 'currentInterviewID', + ); + expect(toast.error).toHaveBeenCalledWith( + 'Impossible de reprendre la simulation pour le moment. Rechargez la page pour réessayer.', + ); + }); +}); From a0842adff8f98fe01341eda11bf6f799bf4e0907 Mon Sep 17 00:00:00 2001 From: BhuvanArn Date: Tue, 14 Jul 2026 20:02:34 +0200 Subject: [PATCH 16/16] fix: defer interview completion until farewell audio finishes; format config + hook spec --- .../modules/simulation/simulation.config.ts | 5 +- .../hooks/simulation/useAudioPlayback.spec.ts | 62 ++++ web/src/hooks/simulation/useAudioPlayback.ts | 22 +- .../simulation/useInterviewSession.spec.ts | 328 +++++++++--------- 4 files changed, 243 insertions(+), 174 deletions(-) diff --git a/server/src/modules/simulation/simulation.config.ts b/server/src/modules/simulation/simulation.config.ts index 0243eddc..5c0b54f9 100644 --- a/server/src/modules/simulation/simulation.config.ts +++ b/server/src/modules/simulation/simulation.config.ts @@ -42,10 +42,7 @@ export function loadSimulationConfig(): SimulationConfig { process.env.SIM_ESTIMATED_TURN_SEC, 90, ), - historyMaxTurns: parsePositiveIntEnv( - process.env.SIM_HISTORY_MAX_TURNS, - 30, - ), + historyMaxTurns: parsePositiveIntEnv(process.env.SIM_HISTORY_MAX_TURNS, 30), contextTtlSec: parsePositiveIntEnv(process.env.SIM_CONTEXT_TTL_SEC, 7200), heartbeatIntervalSec: parsePositiveIntEnv( process.env.SIM_HEARTBEAT_INTERVAL_SEC, diff --git a/web/src/hooks/simulation/useAudioPlayback.spec.ts b/web/src/hooks/simulation/useAudioPlayback.spec.ts index 45f6f9b7..b689a5bb 100644 --- a/web/src/hooks/simulation/useAudioPlayback.spec.ts +++ b/web/src/hooks/simulation/useAudioPlayback.spec.ts @@ -100,6 +100,68 @@ describe('useAudioPlayback', () => { expect(result.current.isAiSpeaking).toBe(false); }); + it('defers simulationComplete until the closing farewell audio finishes', async () => { + const sources: MockAudioBufferSourceNode[] = []; + const ctx = new MockAudioContext(); + ctx.createBufferSource = vi.fn(() => { + const s = new MockAudioBufferSourceNode(); + sources.push(s); + return s; + }); + global.AudioContext = vi.fn(() => ctx) as unknown as typeof AudioContext; + + const completePacket = { + type: 'sts_result', + data: JSON.stringify({ + type: 'sts_result', + transcription: '', + response: 'Merci, au revoir.', + audio_chunks: ['QUFB'], + simulation_complete: true, + }), + }; + + const { result, rerender } = renderHook( + ({ message }) => useAudioPlayback({ message }), + { initialProps: { message: null as unknown } }, + ); + + await act(async () => { + rerender({ message: completePacket as unknown }); + }); + + // While the farewell audio is still playing, completion must NOT be signaled + // yet — otherwise the workspace tears down the socket mid-speech. + await waitFor(() => expect(result.current.isAiSpeaking).toBe(true)); + expect(result.current.simulationComplete).toBe(false); + + await act(async () => { + sources[sources.length - 1].onended?.(); + }); + expect(result.current.isAiSpeaking).toBe(false); + expect(result.current.simulationComplete).toBe(true); + }); + + it('signals simulationComplete immediately when the closing turn has no audio', () => { + const noAudioComplete = { + type: 'sts_result', + data: JSON.stringify({ + type: 'sts_result', + transcription: '', + response: '', + audio_chunks: [], + simulation_complete: true, + }), + }; + + const { result } = renderHook(() => + useAudioPlayback({ message: noAudioComplete as unknown }), + ); + + expect(result.current.simulationComplete).toBe(true); + expect(result.current.isAiSpeaking).toBe(false); + }); + it('stopPlayback stops sources and clears isAiSpeaking', async () => { const sources: MockAudioBufferSourceNode[] = []; const ctx = new MockAudioContext(); diff --git a/web/src/hooks/simulation/useAudioPlayback.ts b/web/src/hooks/simulation/useAudioPlayback.ts index 62c4e954..36c45017 100644 --- a/web/src/hooks/simulation/useAudioPlayback.ts +++ b/web/src/hooks/simulation/useAudioPlayback.ts @@ -127,7 +127,7 @@ export function useAudioPlayback({ }, [setAvatarSpeaking]); const playAnswer = useCallback( - async (chunks: string[]) => { + async (chunks: string[], completeAfterPlayback = false) => { const generation = ++playGenRef.current; const audioContext = getAudioContext(); if (audioContext.state === 'suspended') { @@ -148,6 +148,11 @@ export function useAudioPlayback({ if (buffers.length === 0) { stopPlayback(); + // Nothing decodable to play, so complete now rather than waiting for an + // onended that will never fire. + if (completeAfterPlayback) { + setSimulationComplete(true); + } return; } @@ -168,6 +173,11 @@ export function useAudioPlayback({ source.onended = () => { activeSourcesRef.current = []; setIsAiSpeaking(false); + // Only signal completion once the farewell audio has finished + // playing, so the workspace doesn't tear down the socket mid-speech. + if (completeAfterPlayback) { + setSimulationComplete(true); + } }; } scheduled.push(source); @@ -228,14 +238,12 @@ export function useAudioPlayback({ ); } - if (answer.simulation_complete) { - setSimulationComplete(true); - } - const chunks = answer.audio_chunks ?? []; if (chunks.length === 0) { + // No audio: the interview (if complete) can end right away. if (answer.simulation_complete) { setIsAiSpeaking(false); + setSimulationComplete(true); } return; } @@ -243,7 +251,9 @@ export function useAudioPlayback({ publishSpeechTurn(answer.response ?? '', chunks); // Always play Piper audio directly — the 3D avatar handles visuals only. - void playAnswer(chunks); + // When this is the closing turn, defer the completion signal until the + // farewell audio finishes so the workspace doesn't cut it off mid-speech. + void playAnswer(chunks, answer.simulation_complete === true); }, [message, playAnswer, publishSpeechTurn]); const replaySpeechTurnDirect = useCallback( diff --git a/web/src/hooks/simulation/useInterviewSession.spec.ts b/web/src/hooks/simulation/useInterviewSession.spec.ts index cb0a2904..1c6f3ea2 100644 --- a/web/src/hooks/simulation/useInterviewSession.spec.ts +++ b/web/src/hooks/simulation/useInterviewSession.spec.ts @@ -1,164 +1,164 @@ -import { renderHook, waitFor } from '@testing-library/react'; -import axios from 'axios'; -import toast from 'react-hot-toast'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; - -import { useInterviewSession } from './useInterviewSession'; - -const mockGetInterviewSession = vi.fn(); -const mockCancelInterview = vi.fn(); -const mockCreateInterview = vi.fn(); -const mockUpdateInterview = vi.fn(); -const mockHeartbeatInterview = vi.fn(); - -vi.mock('@/services/ai/http', () => ({ - getInterviewSession: (...args: unknown[]) => mockGetInterviewSession(...args), - cancelInterview: (...args: unknown[]) => mockCancelInterview(...args), - createInterview: (...args: unknown[]) => mockCreateInterview(...args), - updateInterview: (...args: unknown[]) => mockUpdateInterview(...args), - heartbeatInterview: (...args: unknown[]) => mockHeartbeatInterview(...args), -})); - -vi.mock('react-hot-toast', () => ({ - default: { - success: vi.fn(), - error: vi.fn(), - loading: vi.fn(), - dismiss: vi.fn(), - }, -})); - -const localStorageMock = { - store: new Map(), - getItem: vi.fn((key: string) => localStorageMock.store.get(key) ?? null), - setItem: vi.fn((key: string, value: string) => { - localStorageMock.store.set(key, value); - }), - removeItem: vi.fn((key: string) => { - localStorageMock.store.delete(key); - }), - clear: vi.fn(() => { - localStorageMock.store.clear(); - }), -}; - -Object.defineProperty(window, 'localStorage', { - value: localStorageMock, -}); - -function makeAxiosError(status?: number) { - const error = new Error('request failed') as Error & { - isAxiosError: boolean; - response?: { status: number }; - }; - error.isAxiosError = true; - if (status !== undefined) { - error.response = { status }; - } - return error; -} - -describe('useInterviewSession restore', () => { - const onConnect = vi.fn(); - const onDisconnect = vi.fn(); - - beforeEach(() => { - localStorageMock.store.clear(); - vi.clearAllMocks(); - mockHeartbeatInterview.mockResolvedValue(undefined); - vi.spyOn(axios, 'isAxiosError').mockImplementation( - (error: unknown) => - typeof error === 'object' && - error !== null && - 'isAxiosError' in error && - (error as { isAxiosError?: boolean }).isAxiosError === true, - ); - }); - - afterEach(() => { - vi.restoreAllMocks(); - }); - - it('keeps saved session on transient restore failures', async () => { - localStorageMock.store.set('currentInterviewID', 'interview-1'); - localStorageMock.store.set('currentInterviewURL', 'wss://example.test/ws'); - mockGetInterviewSession.mockRejectedValue(makeAxiosError(503)); - - renderHook(() => - useInterviewSession({ - onConnect, - onDisconnect, - }), - ); - - await waitFor(() => { - expect(mockGetInterviewSession).toHaveBeenCalledWith('interview-1'); - }); - - expect(mockCancelInterview).not.toHaveBeenCalled(); - expect(localStorageMock.removeItem).not.toHaveBeenCalledWith( - 'currentInterviewID', - ); - expect(toast.error).toHaveBeenCalledWith( - 'Impossible de reprendre la simulation pour le moment. Rechargez la page pour réessayer.', - ); - }); - - it('cancels and clears storage when the saved session is gone', async () => { - localStorageMock.store.set('currentInterviewID', 'interview-1'); - mockGetInterviewSession.mockRejectedValue(makeAxiosError(404)); - mockCancelInterview.mockResolvedValue(undefined); - - renderHook(() => - useInterviewSession({ - onConnect, - onDisconnect, - }), - ); - - await waitFor(() => { - expect(mockCancelInterview).toHaveBeenCalledWith('interview-1'); - }); - - expect(localStorageMock.removeItem).toHaveBeenCalledWith( - 'currentInterviewID', - ); - expect(toast.error).toHaveBeenCalledWith( - 'La session précédente a expiré. Vous pouvez démarrer un nouvel entretien.', - ); - }); - - it('does not cancel when reconnect fails after a valid session lookup', async () => { - localStorageMock.store.set('currentInterviewID', 'interview-1'); - localStorageMock.store.set('currentInterviewURL', 'wss://example.test/ws'); - mockGetInterviewSession.mockResolvedValue({ - interviewID: 'interview-1', - dbStatus: 'in_progress', - sessionStatus: 'active', - queuePosition: 0, - entrypoint: 'wss://example.test/ws', - }); - onConnect.mockImplementation(() => { - throw new Error('WebSocket reconnect failed'); - }); - - renderHook(() => - useInterviewSession({ - onConnect, - onDisconnect, - }), - ); - - await waitFor(() => { - expect(onConnect).toHaveBeenCalledWith('wss://example.test/ws'); - }); - - expect(mockCancelInterview).not.toHaveBeenCalled(); - expect(localStorageMock.removeItem).not.toHaveBeenCalledWith( - 'currentInterviewID', - ); - expect(toast.error).toHaveBeenCalledWith( - 'Impossible de reprendre la simulation pour le moment. Rechargez la page pour réessayer.', - ); - }); -}); +import { renderHook, waitFor } from '@testing-library/react'; +import axios from 'axios'; +import toast from 'react-hot-toast'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { useInterviewSession } from './useInterviewSession'; + +const mockGetInterviewSession = vi.fn(); +const mockCancelInterview = vi.fn(); +const mockCreateInterview = vi.fn(); +const mockUpdateInterview = vi.fn(); +const mockHeartbeatInterview = vi.fn(); + +vi.mock('@/services/ai/http', () => ({ + getInterviewSession: (...args: unknown[]) => mockGetInterviewSession(...args), + cancelInterview: (...args: unknown[]) => mockCancelInterview(...args), + createInterview: (...args: unknown[]) => mockCreateInterview(...args), + updateInterview: (...args: unknown[]) => mockUpdateInterview(...args), + heartbeatInterview: (...args: unknown[]) => mockHeartbeatInterview(...args), +})); + +vi.mock('react-hot-toast', () => ({ + default: { + success: vi.fn(), + error: vi.fn(), + loading: vi.fn(), + dismiss: vi.fn(), + }, +})); + +const localStorageMock = { + store: new Map(), + getItem: vi.fn((key: string) => localStorageMock.store.get(key) ?? null), + setItem: vi.fn((key: string, value: string) => { + localStorageMock.store.set(key, value); + }), + removeItem: vi.fn((key: string) => { + localStorageMock.store.delete(key); + }), + clear: vi.fn(() => { + localStorageMock.store.clear(); + }), +}; + +Object.defineProperty(window, 'localStorage', { + value: localStorageMock, +}); + +function makeAxiosError(status?: number) { + const error = new Error('request failed') as Error & { + isAxiosError: boolean; + response?: { status: number }; + }; + error.isAxiosError = true; + if (status !== undefined) { + error.response = { status }; + } + return error; +} + +describe('useInterviewSession restore', () => { + const onConnect = vi.fn(); + const onDisconnect = vi.fn(); + + beforeEach(() => { + localStorageMock.store.clear(); + vi.clearAllMocks(); + mockHeartbeatInterview.mockResolvedValue(undefined); + vi.spyOn(axios, 'isAxiosError').mockImplementation( + (error: unknown) => + typeof error === 'object' && + error !== null && + 'isAxiosError' in error && + (error as { isAxiosError?: boolean }).isAxiosError === true, + ); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('keeps saved session on transient restore failures', async () => { + localStorageMock.store.set('currentInterviewID', 'interview-1'); + localStorageMock.store.set('currentInterviewURL', 'wss://example.test/ws'); + mockGetInterviewSession.mockRejectedValue(makeAxiosError(503)); + + renderHook(() => + useInterviewSession({ + onConnect, + onDisconnect, + }), + ); + + await waitFor(() => { + expect(mockGetInterviewSession).toHaveBeenCalledWith('interview-1'); + }); + + expect(mockCancelInterview).not.toHaveBeenCalled(); + expect(localStorageMock.removeItem).not.toHaveBeenCalledWith( + 'currentInterviewID', + ); + expect(toast.error).toHaveBeenCalledWith( + 'Impossible de reprendre la simulation pour le moment. Rechargez la page pour réessayer.', + ); + }); + + it('cancels and clears storage when the saved session is gone', async () => { + localStorageMock.store.set('currentInterviewID', 'interview-1'); + mockGetInterviewSession.mockRejectedValue(makeAxiosError(404)); + mockCancelInterview.mockResolvedValue(undefined); + + renderHook(() => + useInterviewSession({ + onConnect, + onDisconnect, + }), + ); + + await waitFor(() => { + expect(mockCancelInterview).toHaveBeenCalledWith('interview-1'); + }); + + expect(localStorageMock.removeItem).toHaveBeenCalledWith( + 'currentInterviewID', + ); + expect(toast.error).toHaveBeenCalledWith( + 'La session précédente a expiré. Vous pouvez démarrer un nouvel entretien.', + ); + }); + + it('does not cancel when reconnect fails after a valid session lookup', async () => { + localStorageMock.store.set('currentInterviewID', 'interview-1'); + localStorageMock.store.set('currentInterviewURL', 'wss://example.test/ws'); + mockGetInterviewSession.mockResolvedValue({ + interviewID: 'interview-1', + dbStatus: 'in_progress', + sessionStatus: 'active', + queuePosition: 0, + entrypoint: 'wss://example.test/ws', + }); + onConnect.mockImplementation(() => { + throw new Error('WebSocket reconnect failed'); + }); + + renderHook(() => + useInterviewSession({ + onConnect, + onDisconnect, + }), + ); + + await waitFor(() => { + expect(onConnect).toHaveBeenCalledWith('wss://example.test/ws'); + }); + + expect(mockCancelInterview).not.toHaveBeenCalled(); + expect(localStorageMock.removeItem).not.toHaveBeenCalledWith( + 'currentInterviewID', + ); + expect(toast.error).toHaveBeenCalledWith( + 'Impossible de reprendre la simulation pour le moment. Rechargez la page pour réessayer.', + ); + }); +});