From 964d4b7e9765196b4abe81185b1bb4c09c251c79 Mon Sep 17 00:00:00 2001 From: Heeyaa Date: Sun, 17 May 2026 17:37:22 +0900 Subject: [PATCH] =?UTF-8?q?Feat:=20OpenAI=20fallback=20=EA=B4=80=EC=B8=A1?= =?UTF-8?q?=EC=84=B1=20=EA=B0=95=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../application/consensus/ConsensusDtos.kt | 2 + .../application/consensus/ConsensusService.kt | 11 +- .../application/consensus/LlmService.kt | 19 +- .../ScheduleGenerationPersistenceService.kt | 7 + .../schedule/ScheduleResponseMapper.kt | 41 +++- .../application/schedule/ScheduleService.kt | 2 +- .../infrastructure/llm/OpenAiClient.kt | 176 ++++++++++++++++-- .../consensus/ConsensusServiceTest.kt | 8 + .../schedule/ScheduleResponseMapperTest.kt | 85 +++++++++ .../infrastructure/llm/OpenAiClientTest.kt | 160 ++++++++++++++++ 10 files changed, 484 insertions(+), 27 deletions(-) create mode 100644 src/test/kotlin/com/tripsync/application/schedule/ScheduleResponseMapperTest.kt create mode 100644 src/test/kotlin/com/tripsync/infrastructure/llm/OpenAiClientTest.kt diff --git a/src/main/kotlin/com/tripsync/application/consensus/ConsensusDtos.kt b/src/main/kotlin/com/tripsync/application/consensus/ConsensusDtos.kt index b71a333..39665a3 100644 --- a/src/main/kotlin/com/tripsync/application/consensus/ConsensusDtos.kt +++ b/src/main/kotlin/com/tripsync/application/consensus/ConsensusDtos.kt @@ -71,8 +71,10 @@ data class ScheduleOptionDraft( val slots: List, val satisfactionByUser: List, val llmProvider: String, + val llmAttemptedProvider: String, val llmLatencyMs: Long?, val fallbackUsed: Boolean, + val llmFallbackReason: String?, ) data class OptionContext( diff --git a/src/main/kotlin/com/tripsync/application/consensus/ConsensusService.kt b/src/main/kotlin/com/tripsync/application/consensus/ConsensusService.kt index 0b9b595..90d17ff 100644 --- a/src/main/kotlin/com/tripsync/application/consensus/ConsensusService.kt +++ b/src/main/kotlin/com/tripsync/application/consensus/ConsensusService.kt @@ -488,7 +488,7 @@ class ConsensusService( ) } - val llmRefined = llmService.refineScheduleOption( + val llmAttempt = llmService.refineScheduleOption( optionType = optionType, label = label, summary = summary, @@ -498,6 +498,7 @@ class ConsensusService( members = members.map { MemberRef(it.userId, it.nickname) }, slotPlan = shortlistedPerSlot, ) + val llmRefined = llmAttempt.result val finalSummary = llmRefined?.summary ?: summary val placesById = places.associateBy { it.id } @@ -531,7 +532,7 @@ class ConsensusService( val groupSatisfaction = maxOf(threshold, satisfactionByUser.minOf { it.score }) logger.info { - "schedule_option optionType=$optionType roomId=${context.roomId} provider=${llmRefined?.provider ?: DETERMINISTIC_PROVIDER} latencyMs=${llmRefined?.latencyMs ?: 0} fallbackUsed=${llmRefined == null} groupSatisfaction=$groupSatisfaction" + "schedule_option optionType=$optionType roomId=${context.roomId} provider=${llmRefined?.provider ?: DETERMINISTIC_PROVIDER} attemptedProvider=${llmAttempt.attemptedProvider} latencyMs=${llmAttempt.latencyMs ?: 0} fallbackUsed=${llmAttempt.fallbackUsed} fallbackReason=${llmAttempt.fallbackReason?.code ?: "none"} groupSatisfaction=$groupSatisfaction" } return ScheduleOptionDraft( @@ -542,8 +543,10 @@ class ConsensusService( slots = finalSlots, satisfactionByUser = satisfactionByUser, llmProvider = llmRefined?.provider ?: DETERMINISTIC_PROVIDER, - llmLatencyMs = llmRefined?.latencyMs, - fallbackUsed = llmRefined == null, + llmAttemptedProvider = llmAttempt.attemptedProvider, + llmLatencyMs = llmAttempt.latencyMs, + fallbackUsed = llmAttempt.fallbackUsed, + llmFallbackReason = llmAttempt.fallbackReason?.code, ) } diff --git a/src/main/kotlin/com/tripsync/application/consensus/LlmService.kt b/src/main/kotlin/com/tripsync/application/consensus/LlmService.kt index ac8db62..099c8e1 100644 --- a/src/main/kotlin/com/tripsync/application/consensus/LlmService.kt +++ b/src/main/kotlin/com/tripsync/application/consensus/LlmService.kt @@ -15,6 +15,14 @@ class LlmService( val reason: String, ) + enum class FallbackReason(val code: String) { + API_KEY_MISSING("api_key_missing"), + API_HTTP_ERROR("api_http_error"), + API_CALL_FAILED("api_call_failed"), + RESPONSE_PARSE_FAILED("response_parse_failed"), + RESPONSE_SCHEMA_INVALID("response_schema_invalid"), + } + data class RefinementResult( val summary: String, val provider: String, @@ -22,6 +30,15 @@ class LlmService( val slots: List, ) + data class RefinementAttempt( + val result: RefinementResult?, + val attemptedProvider: String, + val latencyMs: Long?, + val fallbackUsed: Boolean, + val fallbackReason: FallbackReason?, + val failureDetail: String? = null, + ) + suspend fun refineScheduleOption( optionType: ScheduleOptionType, label: String, @@ -31,7 +48,7 @@ class LlmService( priorityAxes: List, members: List, slotPlan: List, - ): RefinementResult? { + ): RefinementAttempt { return openAiClient.refineSchedule( optionType = optionType, label = label, diff --git a/src/main/kotlin/com/tripsync/application/schedule/ScheduleGenerationPersistenceService.kt b/src/main/kotlin/com/tripsync/application/schedule/ScheduleGenerationPersistenceService.kt index b3cccb6..ff2ac5b 100644 --- a/src/main/kotlin/com/tripsync/application/schedule/ScheduleGenerationPersistenceService.kt +++ b/src/main/kotlin/com/tripsync/application/schedule/ScheduleGenerationPersistenceService.kt @@ -106,6 +106,13 @@ class ScheduleGenerationPersistenceService( "endTime" to dto.endTime, "tripStartDate" to (dto.tripStartDate ?: dto.tripDate), "tripEndDate" to (dto.tripEndDate ?: dto.tripDate), + "llm" to mapOf( + "provider" to option.llmProvider, + "attemptedProvider" to option.llmAttemptedProvider, + "latencyMs" to option.llmLatencyMs, + "fallbackUsed" to option.fallbackUsed, + "fallbackReason" to option.llmFallbackReason, + ), ), summary = option.summary, groupSatisfaction = option.groupSatisfaction, diff --git a/src/main/kotlin/com/tripsync/application/schedule/ScheduleResponseMapper.kt b/src/main/kotlin/com/tripsync/application/schedule/ScheduleResponseMapper.kt index dc04023..3a5ee4a 100644 --- a/src/main/kotlin/com/tripsync/application/schedule/ScheduleResponseMapper.kt +++ b/src/main/kotlin/com/tripsync/application/schedule/ScheduleResponseMapper.kt @@ -25,8 +25,10 @@ class ScheduleResponseMapper( "groupSatisfaction" to option.groupSatisfaction, "personaValidation" to personaValidation, "llmProvider" to option.llmProvider, + "llmAttemptedProvider" to option.llmAttemptedProvider, "llmLatencyMs" to option.llmLatencyMs, "fallbackUsed" to option.fallbackUsed, + "llmFallbackReason" to option.llmFallbackReason, "slots" to option.slots.sortedBy { it.orderIndex }.map { slot -> val place = placesById[slot.placeId] mapOf( @@ -55,6 +57,7 @@ class ScheduleResponseMapper( fun formatStoredSchedule(schedule: Schedule): Map { val memberNicknames = roomMemberProfileRepository.findAllByRoomIdAndDelYn(schedule.room.id, YnFlag.N) .associate { it.user.id to it.user.nickname } + val llmMetadata = formatLlmMetadata(schedule) return mapOf( "id" to schedule.id, "roomId" to schedule.room.id, @@ -66,7 +69,12 @@ class ScheduleResponseMapper( "groupSatisfaction" to schedule.groupSatisfaction, "summary" to (schedule.summary ?: ""), "personaValidation" to schedule.personaValidation, - "slots" to schedule.slots.filter { it.delYn == YnFlag.N }.sortedBy { it.orderIndex }.map { slot -> + "llmProvider" to llmMetadata["provider"], + "llmAttemptedProvider" to llmMetadata["attemptedProvider"], + "llmLatencyMs" to llmMetadata["latencyMs"], + "fallbackUsed" to llmMetadata["fallbackUsed"], + "llmFallbackReason" to llmMetadata["fallbackReason"], + "slots" to schedule.slots.filter { it.delYn == YnFlag.N }.sortedBy { slot -> slot.orderIndex }.map { slot -> mapOf( "slotId" to slot.id, "orderIndex" to slot.orderIndex, @@ -91,6 +99,37 @@ class ScheduleResponseMapper( ) } + fun formatPublicShareSchedule(schedule: Schedule): Map { + val publicKeys = setOf( + "id", + "roomId", + "destination", + "tripDate", + "version", + "optionType", + "isConfirmed", + "groupSatisfaction", + "summary", + "personaValidation", + "slots", + "satisfactionByUser", + ) + return formatStoredSchedule(schedule).filterKeys { it in publicKeys } + } + + @Suppress("UNCHECKED_CAST") + private fun formatLlmMetadata(schedule: Schedule): Map { + val metadata = schedule.generationInput["llm"] as? Map ?: emptyMap() + val provider = schedule.llmProvider ?: metadata["provider"] + return mapOf( + "provider" to provider, + "attemptedProvider" to (metadata["attemptedProvider"] ?: provider), + "latencyMs" to metadata["latencyMs"], + "fallbackUsed" to (metadata["fallbackUsed"] ?: (provider == "deterministic-consensus")), + "fallbackReason" to metadata["fallbackReason"], + ) + } + fun formatPlace(place: Place?, id: Long, name: String, address: String): Map = mapOf( "id" to id, "name" to name, diff --git a/src/main/kotlin/com/tripsync/application/schedule/ScheduleService.kt b/src/main/kotlin/com/tripsync/application/schedule/ScheduleService.kt index 843b93e..d96d45a 100644 --- a/src/main/kotlin/com/tripsync/application/schedule/ScheduleService.kt +++ b/src/main/kotlin/com/tripsync/application/schedule/ScheduleService.kt @@ -224,7 +224,7 @@ class ScheduleService( @Transactional(readOnly = true) fun getPublicShareSchedule(scheduleId: Long): ApiResponse> { val schedule = accessPolicy.getActiveSchedule(scheduleId) - return ApiResponse.ok(responseMapper.formatStoredSchedule(schedule)) + return ApiResponse.ok(responseMapper.formatPublicShareSchedule(schedule)) } private fun safePersonaValidationByType( diff --git a/src/main/kotlin/com/tripsync/infrastructure/llm/OpenAiClient.kt b/src/main/kotlin/com/tripsync/infrastructure/llm/OpenAiClient.kt index ced4c80..0a3aba8 100644 --- a/src/main/kotlin/com/tripsync/infrastructure/llm/OpenAiClient.kt +++ b/src/main/kotlin/com/tripsync/infrastructure/llm/OpenAiClient.kt @@ -2,16 +2,21 @@ package com.tripsync.infrastructure.llm import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper +import io.micrometer.core.instrument.MeterRegistry +import io.micrometer.core.instrument.Timer import com.tripsync.application.consensus.ConsensusService import com.tripsync.application.consensus.LlmService import com.tripsync.domain.enums.ScheduleOptionType +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.reactive.awaitSingle import mu.KotlinLogging import org.springframework.beans.factory.annotation.Value import org.springframework.http.MediaType import org.springframework.stereotype.Component import org.springframework.web.reactive.function.client.WebClient +import org.springframework.web.reactive.function.client.WebClientResponseException import org.springframework.web.reactive.function.client.bodyToMono +import java.util.concurrent.TimeUnit @Component class OpenAiClient( @@ -21,8 +26,11 @@ class OpenAiClient( private val apiKey: String, @Value("\${openai.model:gpt-4o-mini}") private val model: String, + private val meterRegistry: MeterRegistry, ) { private val logger = KotlinLogging.logger {} + private val providerName: String + get() = "openai/$model" suspend fun refineSchedule( optionType: ScheduleOptionType, @@ -33,16 +41,21 @@ class OpenAiClient( priorityAxes: List, members: List, slotPlan: List, - ): LlmService.RefinementResult? { + ): LlmService.RefinementAttempt { if (apiKey.isBlank()) { - logger.warn { "OpenAI API key not configured" } - return null + recordMetrics(optionType, "fallback", LlmService.FallbackReason.API_KEY_MISSING.code, 0) + logger.warn { + "llm_refinement outcome=fallback reason=${LlmService.FallbackReason.API_KEY_MISSING.code} provider=$providerName optionType=$optionType roomId=${room.roomId}" + } + return fallbackAttempt( + reason = LlmService.FallbackReason.API_KEY_MISSING, + latencyMs = 0, + detail = "OpenAI API key not configured", + ) } - val start = System.currentTimeMillis() - + val startNanos = System.nanoTime() val prompt = buildPrompt(optionType, label, summary, room, commonAxes, priorityAxes, members, slotPlan) - val requestBody = mapOf( "model" to model, "messages" to listOf( @@ -66,14 +79,83 @@ class OpenAiClient( .bodyToMono() .awaitSingle() - val latencyMs = System.currentTimeMillis() - start - parseResponse(response, latencyMs) + val latencyMs = elapsedMillis(startNanos) + val parsed = parseResponse(response, latencyMs) + validateRefinement(parsed, slotPlan) + recordMetrics(optionType, "success", "none", latencyMs) + logger.info { + "llm_refinement outcome=success provider=$providerName optionType=$optionType roomId=${room.roomId} latencyMs=$latencyMs slots=${parsed.slots.size}" + } + LlmService.RefinementAttempt( + result = parsed, + attemptedProvider = providerName, + latencyMs = latencyMs, + fallbackUsed = false, + fallbackReason = null, + ) + } catch (e: LlmParseException) { + val latencyMs = elapsedMillis(startNanos) + recordMetrics(optionType, "fallback", e.reason.code, latencyMs) + logger.warn(e) { + "llm_refinement outcome=fallback reason=${e.reason.code} provider=$providerName optionType=$optionType roomId=${room.roomId} latencyMs=$latencyMs" + } + fallbackAttempt(e.reason, latencyMs, e.message) + } catch (e: WebClientResponseException) { + val latencyMs = elapsedMillis(startNanos) + recordMetrics(optionType, "fallback", LlmService.FallbackReason.API_HTTP_ERROR.code, latencyMs) + logger.error(e) { + "llm_refinement outcome=fallback reason=${LlmService.FallbackReason.API_HTTP_ERROR.code} provider=$providerName optionType=$optionType roomId=${room.roomId} latencyMs=$latencyMs status=${e.statusCode.value()}" + } + fallbackAttempt(LlmService.FallbackReason.API_HTTP_ERROR, latencyMs, "HTTP ${e.statusCode.value()}") + } catch (e: CancellationException) { + throw e } catch (e: Exception) { - logger.error(e) { "LLM refinement failed" } - null + val latencyMs = elapsedMillis(startNanos) + recordMetrics(optionType, "fallback", LlmService.FallbackReason.API_CALL_FAILED.code, latencyMs) + logger.error(e) { + "llm_refinement outcome=fallback reason=${LlmService.FallbackReason.API_CALL_FAILED.code} provider=$providerName optionType=$optionType roomId=${room.roomId} latencyMs=$latencyMs" + } + fallbackAttempt(LlmService.FallbackReason.API_CALL_FAILED, latencyMs, e.javaClass.simpleName) } } + private fun elapsedMillis(startNanos: Long): Long { + return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos).coerceAtLeast(0) + } + + private fun recordMetrics( + optionType: ScheduleOptionType, + outcome: String, + reason: String, + latencyMs: Long, + ) { + val tags = listOf( + "provider", providerName, + "optionType", optionType.name.lowercase(), + "outcome", outcome, + "reason", reason, + ) + val tagArray = tags.toTypedArray() + meterRegistry.counter("tripsync.llm.refinement.calls", *tagArray).increment() + Timer.builder("tripsync.llm.refinement.latency") + .tags(*tagArray) + .register(meterRegistry) + .record(latencyMs, TimeUnit.MILLISECONDS) + } + + private fun fallbackAttempt( + reason: LlmService.FallbackReason, + latencyMs: Long, + detail: String?, + ): LlmService.RefinementAttempt = LlmService.RefinementAttempt( + result = null, + attemptedProvider = providerName, + latencyMs = latencyMs, + fallbackUsed = true, + fallbackReason = reason, + failureDetail = detail, + ) + private fun buildPrompt( optionType: ScheduleOptionType, label: String, @@ -116,27 +198,81 @@ class OpenAiClient( """.trimIndent() } - private fun parseResponse(response: String, latencyMs: Long): LlmService.RefinementResult? { - val root = objectMapper.readTree(response) - val content = root.path("choices").get(0)?.path("message")?.path("content")?.asText() - ?: return null + internal fun parseResponse(response: String, latencyMs: Long): LlmService.RefinementResult { + val root = readJson(response, LlmService.FallbackReason.RESPONSE_PARSE_FAILED) + val choices = root.path("choices") + if (!choices.isArray || choices.size() == 0) { + throw LlmParseException(LlmService.FallbackReason.RESPONSE_SCHEMA_INVALID, "OpenAI response choices missing") + } + + val content = choices.get(0)?.path("message")?.path("content")?.asText()?.trim().orEmpty() + if (content.isBlank()) { + throw LlmParseException(LlmService.FallbackReason.RESPONSE_SCHEMA_INVALID, "OpenAI response content missing") + } + + val result = readJson(content, LlmService.FallbackReason.RESPONSE_PARSE_FAILED) + val summary = result.path("summary").asText().trim() + if (summary.isBlank()) { + throw LlmParseException(LlmService.FallbackReason.RESPONSE_SCHEMA_INVALID, "LLM summary missing") + } + + val rawSlots = result.path("slots") + if (!rawSlots.isArray) { + throw LlmParseException(LlmService.FallbackReason.RESPONSE_SCHEMA_INVALID, "LLM slots missing") + } - val result = objectMapper.readTree(content) - val summary = result.path("summary").asText() - val slots = result.path("slots").mapNotNull { slot -> + val slots = rawSlots.mapNotNull { slot -> val orderIndex = slot.path("orderIndex").asInt() val placeId = slot.path("placeId").asLong() - val reason = slot.path("reason").asText() + val reason = slot.path("reason").asText().trim() if (orderIndex > 0 && placeId > 0) { - LlmService.RefinedSlot(orderIndex, placeId, reason) + LlmService.RefinedSlot(orderIndex, placeId, reason.ifBlank { "LLM 추천" }) } else null } + if (slots.isEmpty()) { + throw LlmParseException(LlmService.FallbackReason.RESPONSE_SCHEMA_INVALID, "LLM slots contain no valid place selection") + } return LlmService.RefinementResult( summary = summary, - provider = "openai/$model", + provider = providerName, latencyMs = latencyMs, slots = slots, ) } + + internal fun validateRefinement( + result: LlmService.RefinementResult, + slotPlan: List, + ) { + val validPlaceIdsByOrder = slotPlan.associate { slot -> + slot.orderIndex to slot.candidatePlaces.map { it.id }.toSet() + } + val expectedOrders = validPlaceIdsByOrder.keys + val actualOrders = result.slots.map { it.orderIndex } + if (actualOrders.toSet() != expectedOrders || actualOrders.size != expectedOrders.size) { + throw LlmParseException(LlmService.FallbackReason.RESPONSE_SCHEMA_INVALID, "LLM slots must contain exactly one selection for every shortlisted slot") + } + + val hasInvalidSelection = result.slots.any { slot -> + slot.placeId !in validPlaceIdsByOrder.getOrDefault(slot.orderIndex, emptySet()) + } + if (hasInvalidSelection) { + throw LlmParseException(LlmService.FallbackReason.RESPONSE_SCHEMA_INVALID, "LLM slots contain place selection outside shortlist") + } + } + + private fun readJson(value: String, reason: LlmService.FallbackReason): JsonNode { + return try { + objectMapper.readTree(value) + } catch (e: Exception) { + throw LlmParseException(reason, "LLM JSON parse failed", e) + } + } + + internal class LlmParseException( + val reason: LlmService.FallbackReason, + message: String, + cause: Throwable? = null, + ) : RuntimeException(message, cause) } diff --git a/src/test/kotlin/com/tripsync/application/consensus/ConsensusServiceTest.kt b/src/test/kotlin/com/tripsync/application/consensus/ConsensusServiceTest.kt index a1f3301..ce80df0 100644 --- a/src/test/kotlin/com/tripsync/application/consensus/ConsensusServiceTest.kt +++ b/src/test/kotlin/com/tripsync/application/consensus/ConsensusServiceTest.kt @@ -1,6 +1,7 @@ package com.tripsync.application.consensus import com.fasterxml.jackson.databind.ObjectMapper +import io.micrometer.core.instrument.simple.SimpleMeterRegistry import com.tripsync.common.exception.DomainException import com.tripsync.domain.entity.AxisScores import com.tripsync.infrastructure.llm.OpenAiClient @@ -20,6 +21,7 @@ class ConsensusServiceTest { objectMapper = ObjectMapper(), apiKey = "", model = "gpt-4o-mini", + meterRegistry = SimpleMeterRegistry(), ) ) ) @@ -44,6 +46,12 @@ class ConsensusServiceTest { assertTrue(option.slots.all { it.placeAddress.contains("전라북도") }) } assertTrue(options.last().summary.contains("전북")) + options.forEach { option -> + assertEquals("deterministic-consensus", option.llmProvider) + assertEquals("openai/gpt-4o-mini", option.llmAttemptedProvider) + assertTrue(option.fallbackUsed) + assertEquals("api_key_missing", option.llmFallbackReason) + } } @Test diff --git a/src/test/kotlin/com/tripsync/application/schedule/ScheduleResponseMapperTest.kt b/src/test/kotlin/com/tripsync/application/schedule/ScheduleResponseMapperTest.kt new file mode 100644 index 0000000..ffd6331 --- /dev/null +++ b/src/test/kotlin/com/tripsync/application/schedule/ScheduleResponseMapperTest.kt @@ -0,0 +1,85 @@ +package com.tripsync.application.schedule + +import com.tripsync.domain.entity.Schedule +import com.tripsync.domain.entity.TripRoom +import com.tripsync.domain.entity.User +import com.tripsync.domain.enums.AuthProvider +import com.tripsync.domain.enums.ScheduleOptionType +import com.tripsync.domain.enums.TripRoomStatus +import com.tripsync.domain.enums.YnFlag +import com.tripsync.domain.repository.RoomMemberProfileRepository +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Test +import org.mockito.Mockito.`when` +import org.mockito.Mockito.mock +import java.time.LocalDate + +class ScheduleResponseMapperTest { + private val roomMemberProfileRepository = mock(RoomMemberProfileRepository::class.java) + private val mapper = ScheduleResponseMapper(roomMemberProfileRepository) + + @Test + fun `stored schedule response includes persisted llm metadata`() { + val schedule = scheduleWithLlmMetadata() + `when`(roomMemberProfileRepository.findAllByRoomIdAndDelYn(schedule.room.id, YnFlag.N)).thenReturn(emptyList()) + + val response = mapper.formatStoredSchedule(schedule) + + assertEquals("deterministic-consensus", response["llmProvider"]) + assertEquals("openai/gpt-4o-mini", response["llmAttemptedProvider"]) + assertEquals(35L, response["llmLatencyMs"]) + assertEquals(true, response["fallbackUsed"]) + assertEquals("response_schema_invalid", response["llmFallbackReason"]) + } + + @Test + fun `public share response omits llm operational metadata`() { + val schedule = scheduleWithLlmMetadata() + `when`(roomMemberProfileRepository.findAllByRoomIdAndDelYn(schedule.room.id, YnFlag.N)).thenReturn(emptyList()) + + val response = mapper.formatPublicShareSchedule(schedule) + + assertFalse(response.containsKey("llmProvider")) + assertFalse(response.containsKey("llmAttemptedProvider")) + assertFalse(response.containsKey("llmLatencyMs")) + assertFalse(response.containsKey("fallbackUsed")) + assertFalse(response.containsKey("llmFallbackReason")) + } + + private fun scheduleWithLlmMetadata(): Schedule { + val host = User( + id = 10L, + nickname = "host", + authProvider = AuthProvider.LOCAL, + ) + val room = TripRoom( + id = 1L, + hostUser = host, + shareCode = "ABC123456789", + destination = "충남", + tripDate = LocalDate.parse("2026-06-01"), + status = TripRoomStatus.COMPLETED, + ) + return Schedule( + id = 100L, + room = room, + version = 1, + optionType = ScheduleOptionType.BALANCED, + generationInput = mapOf( + "destination" to "충남", + "llm" to mapOf( + "provider" to "deterministic-consensus", + "attemptedProvider" to "openai/gpt-4o-mini", + "latencyMs" to 35L, + "fallbackUsed" to true, + "fallbackReason" to "response_schema_invalid", + ), + ), + summary = "요약", + groupSatisfaction = 72, + personaValidation = null, + llmProvider = "deterministic-consensus", + ) + } +} diff --git a/src/test/kotlin/com/tripsync/infrastructure/llm/OpenAiClientTest.kt b/src/test/kotlin/com/tripsync/infrastructure/llm/OpenAiClientTest.kt new file mode 100644 index 0000000..4e9c1f4 --- /dev/null +++ b/src/test/kotlin/com/tripsync/infrastructure/llm/OpenAiClientTest.kt @@ -0,0 +1,160 @@ +package com.tripsync.infrastructure.llm + +import com.fasterxml.jackson.databind.ObjectMapper +import io.micrometer.core.instrument.simple.SimpleMeterRegistry +import com.tripsync.application.consensus.ConsensusService +import com.tripsync.application.consensus.LlmService +import com.tripsync.domain.enums.ScheduleOptionType +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertThrows +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.springframework.web.reactive.function.client.WebClient + +class OpenAiClientTest { + private val client = OpenAiClient( + webClient = WebClient.create(), + objectMapper = ObjectMapper(), + apiKey = "test-key", + model = "gpt-test", + meterRegistry = SimpleMeterRegistry(), + ) + + @Test + fun `refine schedule records fallback metrics when api key is missing`() = runBlocking { + val meterRegistry = SimpleMeterRegistry() + val blankKeyClient = OpenAiClient( + webClient = WebClient.create(), + objectMapper = ObjectMapper(), + apiKey = "", + model = "gpt-test", + meterRegistry = meterRegistry, + ) + + val attempt = blankKeyClient.refineSchedule( + optionType = ScheduleOptionType.BALANCED, + label = "균형형", + summary = "요약", + room = ConsensusService.RoomRef(roomId = 1, destination = "충남", tripDate = "2026-06-01"), + commonAxes = emptyList(), + priorityAxes = emptyList(), + members = emptyList(), + slotPlan = emptyList(), + ) + + assertTrue(attempt.fallbackUsed) + assertEquals(LlmService.FallbackReason.API_KEY_MISSING, attempt.fallbackReason) + assertEquals(1.0, meterRegistry.counter( + "tripsync.llm.refinement.calls", + "provider", "openai/gpt-test", + "optionType", "balanced", + "outcome", "fallback", + "reason", "api_key_missing", + ).count()) + } + + @Test + fun `parse response returns refinement result with latency and provider`() { + val response = """ + { + "choices": [ + { + "message": { + "content": "{\"summary\":\"차분한 하루\",\"slots\":[{\"orderIndex\":1,\"placeId\":10,\"reason\":\"공통 취향\"}]}" + } + } + ] + } + """.trimIndent() + + val result = client.parseResponse(response, latencyMs = 123) + + assertEquals("차분한 하루", result.summary) + assertEquals("openai/gpt-test", result.provider) + assertEquals(123, result.latencyMs) + assertEquals(1, result.slots.size) + assertEquals(10, result.slots.first().placeId) + } + + @Test + fun `validate refinement rejects partial slot selection`() { + val result = LlmService.RefinementResult( + summary = "요약", + provider = "openai/gpt-test", + latencyMs = 10, + slots = listOf(LlmService.RefinedSlot(orderIndex = 1, placeId = 10, reason = "첫 슬롯만")), + ) + val slotPlan = listOf( + shortlist(orderIndex = 1, placeId = 10), + shortlist(orderIndex = 2, placeId = 20), + ) + + val error = assertThrows(OpenAiClient.LlmParseException::class.java) { + client.validateRefinement(result, slotPlan) + } + + assertEquals(LlmService.FallbackReason.RESPONSE_SCHEMA_INVALID, error.reason) + } + + @Test + fun `validate refinement rejects place outside shortlist`() { + val result = LlmService.RefinementResult( + summary = "요약", + provider = "openai/gpt-test", + latencyMs = 10, + slots = listOf(LlmService.RefinedSlot(orderIndex = 1, placeId = 999, reason = "후보 밖")), + ) + val slotPlan = listOf(shortlist(orderIndex = 1, placeId = 10)) + + val error = assertThrows(OpenAiClient.LlmParseException::class.java) { + client.validateRefinement(result, slotPlan) + } + + assertEquals(LlmService.FallbackReason.RESPONSE_SCHEMA_INVALID, error.reason) + } + + @Test + fun `parse response classifies malformed json as parse failure`() { + val error = assertThrows(OpenAiClient.LlmParseException::class.java) { + client.parseResponse("not-json", latencyMs = 1) + } + + assertEquals(LlmService.FallbackReason.RESPONSE_PARSE_FAILED, error.reason) + } + + @Test + fun `parse response classifies missing slots as schema failure`() { + val response = """ + { + "choices": [ + { + "message": { + "content": "{\"summary\":\"요약\",\"slots\":[]}" + } + } + ] + } + """.trimIndent() + + val error = assertThrows(OpenAiClient.LlmParseException::class.java) { + client.parseResponse(response, latencyMs = 1) + } + + assertEquals(LlmService.FallbackReason.RESPONSE_SCHEMA_INVALID, error.reason) + } + + private fun shortlist(orderIndex: Int, placeId: Long): ConsensusService.SlotShortlist { + return ConsensusService.SlotShortlist( + orderIndex = orderIndex, + startTime = "09:00", + endTime = "11:00", + slotType = com.tripsync.domain.enums.SlotType.COMMON, + targetUserId = null, + reasonAxis = com.tripsync.domain.enums.ReasonAxis.COMMON, + candidatePlaces = listOf(ConsensusService.CandidatePlace(placeId, "장소", "카테고리", "주소")), + deterministicPlaceId = placeId, + deterministicReason = "기본", + ) + } +}