diff --git a/.pipelines/v2/templates/steps-build-js.yml b/.pipelines/v2/templates/steps-build-js.yml index 657704bd9..cf18c80c4 100644 --- a/.pipelines/v2/templates/steps-build-js.yml +++ b/.pipelines/v2/templates/steps-build-js.yml @@ -156,6 +156,14 @@ steps: throw "WinML runtime DLL not found in native artifact: $winmlDll" } + $directMlDll = Join-Path "${{ parameters.nativeArtifactDir }}" 'DirectML.dll' + if (Test-Path $directMlDll) { + Copy-Item $directMlDll -Destination $dst -Force + Write-Host " staged DirectML.dll" + } else { + throw "DirectML.dll not found in native artifact: $directMlDll" + } + Get-ChildItem $dst | ForEach-Object { Write-Host " $($_.Name) $($_.Length) bytes" } displayName: 'Stage prebuild directory' diff --git a/.pipelines/v2/templates/steps-build-linux.yml b/.pipelines/v2/templates/steps-build-linux.yml index f092abc41..7d9e76ef6 100644 --- a/.pipelines/v2/templates/steps-build-linux.yml +++ b/.pipelines/v2/templates/steps-build-linux.yml @@ -75,24 +75,42 @@ steps: displayName: 'Dump vcpkg error logs' condition: failed() -# Stage the redistributable native artifact. Vcpkg uses static linkage on -# Linux (see sdk_v2/cpp/triplets/x64-linux.cmake), so libfoundry_local.so -# carries its transitive deps (azure-*, spdlog, fmt, libcurl, libssl/libcrypto, -# zlib, brotli*) inside itself — the only file we need to forward downstream -# is libfoundry_local.so. ORT/GenAI .so files are also present in bin/ but are -# supplied to consumers separately (pip on the Python side, NuGet on the C# -# side); test/example binaries are not part of the redistributable surface. +# Stage the redistributable native artifacts for standalone C++ consumers. - bash: | set -euo pipefail src='$(Build.SourcesDirectory)/sdk_v2/cpp/build/Linux/${{ parameters.buildConfig }}/bin' dst='$(Build.ArtifactStagingDirectory)/native' mkdir -p "$dst" - primary="$src/libfoundry_local.so" - if [ ! -f "$primary" ]; then - echo "ERROR: libfoundry_local.so not found at $primary" >&2 - exit 1 - fi - cp -P "$primary" "$dst/" - echo " staged libfoundry_local.so" + required=( + "$src/libfoundry_local.so" + "$src/libonnxruntime-genai.so" + "$src/libonnxruntime.so" + ) + for f in "${required[@]}"; do + if [ ! -e "$f" ]; then + echo "ERROR: required native artifact not found: $f" >&2 + exit 1 + fi + cp -P "$f" "$dst/" + echo " staged $(basename "$f")" + done + for f in "$src"/libonnxruntime.so.*; do + if [ -e "$f" ]; then + cp -P "$f" "$dst/" + echo " staged $(basename "$f")" + fi + done + for f in "$src"/libonnxruntime-genai*.so; do + if [ -e "$f" ] && [ "$(basename "$f")" != "libonnxruntime-genai.so" ]; then + cp -P "$f" "$dst/" + echo " staged $(basename "$f")" + fi + done + for f in "$src"/libonnxruntime_providers_*.so; do + if [ -e "$f" ]; then + cp -P "$f" "$dst/" + echo " staged $(basename "$f")" + fi + done displayName: 'Stage native artifacts' diff --git a/.pipelines/v2/templates/steps-build-macos.yml b/.pipelines/v2/templates/steps-build-macos.yml index 92f9825e9..a3decb433 100644 --- a/.pipelines/v2/templates/steps-build-macos.yml +++ b/.pipelines/v2/templates/steps-build-macos.yml @@ -81,23 +81,36 @@ steps: displayName: 'Dump vcpkg error logs' condition: failed() -# Stage the redistributable native artifact. macOS has no triplet overlay so -# vcpkg uses its default arm64-osx triplet, which is static — libfoundry_local.dylib -# carries its transitive deps (azure-*, spdlog, fmt, libcurl, libssl/libcrypto, -# zlib, brotli*) inside itself. ORT/GenAI .dylib files are supplied to consumers -# separately (pip on the Python side, NuGet on the C# side); test/example binaries -# are not part of the redistributable surface. +# Stage the redistributable native artifacts for standalone C++ consumers. - bash: | set -euo pipefail src='$(Build.SourcesDirectory)/sdk_v2/cpp/build/macOS/${{ parameters.buildConfig }}/bin' dst='$(Build.ArtifactStagingDirectory)/native' mkdir -p "$dst" - primary="$src/libfoundry_local.dylib" - if [ ! -f "$primary" ]; then - echo "ERROR: libfoundry_local.dylib not found at $primary" >&2 - exit 1 - fi - cp -P "$primary" "$dst/" - echo " staged libfoundry_local.dylib" + required=( + "$src/libfoundry_local.dylib" + "$src/libonnxruntime-genai.dylib" + "$src/libonnxruntime.dylib" + ) + for f in "${required[@]}"; do + if [ ! -e "$f" ]; then + echo "ERROR: required native artifact not found: $f" >&2 + exit 1 + fi + cp -P "$f" "$dst/" + echo " staged $(basename "$f")" + done + for f in "$src"/libonnxruntime.*.dylib; do + if [ -e "$f" ]; then + cp -P "$f" "$dst/" + echo " staged $(basename "$f")" + fi + done + for f in "$src"/libonnxruntime_providers_*.dylib; do + if [ -e "$f" ]; then + cp -P "$f" "$dst/" + echo " staged $(basename "$f")" + fi + done displayName: 'Stage native artifacts' diff --git a/.pipelines/v2/templates/steps-build-python.yml b/.pipelines/v2/templates/steps-build-python.yml index 50f86c9da..84201a682 100644 --- a/.pipelines/v2/templates/steps-build-python.yml +++ b/.pipelines/v2/templates/steps-build-python.yml @@ -106,10 +106,32 @@ steps: # deps (onnxruntime-{core,gpu} / onnxruntime-genai-{core,cuda}). Test # and example binaries are also filtered out upstream. We just copy # the curated artifact contents straight in. - $files = Get-ChildItem -Path "${{ parameters.nativeArtifactDir }}" -Recurse -File + $rid = '${{ parameters.rid }}' + if ($rid -like 'win-*') { + $allowed = @('foundry_local.dll', 'foundry_local.lib', 'Microsoft.Windows.AI.MachineLearning.dll', 'DirectML.dll') + } elseif ($rid -like 'linux-*') { + $allowed = @('libfoundry_local.so') + } elseif ($rid -like 'osx-*') { + $allowed = @('libfoundry_local.dylib') + } else { + throw "Unsupported Python wheel RID: $rid" + } + $files = foreach ($name in $allowed) { + Get-ChildItem -Path "${{ parameters.nativeArtifactDir }}" -Filter $name -File -ErrorAction SilentlyContinue + } if ($files.Count -eq 0) { throw "No native artifacts found under ${{ parameters.nativeArtifactDir }}" } + if ($rid -like 'win-*') { + $found = @{} + foreach ($f in $files) { + $found[$f.Name] = $true + } + $missing = @($allowed | Where-Object { -not $found.ContainsKey($_) }) + if ($missing.Count -gt 0) { + throw "Missing required Windows native artifact(s): $($missing -join ', ') under ${{ parameters.nativeArtifactDir }}" + } + } foreach ($f in $files) { Copy-Item $f.FullName -Destination $dest -Force } diff --git a/.pipelines/v2/templates/steps-build-windows.yml b/.pipelines/v2/templates/steps-build-windows.yml index 5f1d0f761..e72fde515 100644 --- a/.pipelines/v2/templates/steps-build-windows.yml +++ b/.pipelines/v2/templates/steps-build-windows.yml @@ -138,7 +138,13 @@ steps: (Join-Path $binDir 'foundry_local.dll'), (Join-Path $binDir 'foundry_local.pdb'), (Join-Path $linkDir 'foundry_local.lib'), - (Join-Path $binDir 'Microsoft.Windows.AI.MachineLearning.dll') + (Join-Path $binDir 'Microsoft.Windows.AI.MachineLearning.dll'), + (Join-Path $binDir 'DirectML.dll'), + (Join-Path $binDir 'onnxruntime.dll'), + (Join-Path $binDir 'onnxruntime-genai.dll') + ) + $optionalPatterns = @( + 'onnxruntime_providers_*.dll' ) foreach ($s in $sources) { @@ -149,6 +155,12 @@ steps: Copy-Item -Path $s -Destination $dst -Force Write-Host " staged $(Split-Path -Leaf $s)" } + foreach ($pattern in $optionalPatterns) { + Get-ChildItem -Path $binDir -Filter $pattern -File -ErrorAction SilentlyContinue | ForEach-Object { + Copy-Item -Path $_.FullName -Destination $dst -Force + Write-Host " staged $($_.Name)" + } + } - ${{ if eq(parameters.stageHeaders, true) }}: - task: CopyFiles@2 diff --git a/.pipelines/v2/templates/steps-pack-cpp-sdk.yml b/.pipelines/v2/templates/steps-pack-cpp-sdk.yml index d18bfa8b7..ab5a94415 100644 --- a/.pipelines/v2/templates/steps-pack-cpp-sdk.yml +++ b/.pipelines/v2/templates/steps-pack-cpp-sdk.yml @@ -57,9 +57,8 @@ steps: Copy-Item -Path (Join-Path $includeSrc '*') -Destination $includeDst -Recurse -Force - # The SDK tgz should include only public wrapper headers. + # The SDK tgz should include public wrapper headers and their header-only dependencies. $pathsToPrune = @( - (Join-Path $includeDst 'gsl'), (Join-Path $includeDst '_manifest') ) foreach ($pathToPrune in $pathsToPrune) { @@ -68,13 +67,17 @@ steps: } } - foreach ($f in $Files) { - $src = Join-Path $nativeSrc $f - if (-not (Test-Path $src)) { - throw "Required native file not found: $src" + foreach ($entry in $Files) { + $optional = $entry.StartsWith('?') + $pattern = if ($optional) { $entry.Substring(1) } else { $entry } + $matches = @(Get-ChildItem -Path $nativeSrc -Filter $pattern -File -ErrorAction SilentlyContinue) + if ($matches.Count -eq 0 -and -not $optional) { + throw "Required native file not found: $(Join-Path $nativeSrc $pattern)" + } + foreach ($match in $matches) { + $dst = Join-Path $libDst $match.Name + Copy-Item -Path $match.FullName -Destination $dst -Force } - $dst = Join-Path $libDst (Split-Path -Leaf $src) - Copy-Item -Path $src -Destination $dst -Force } $archive = Join-Path $outDir ("cpp-sdk-v2-{0}.tgz" -f $Rid) @@ -96,21 +99,38 @@ steps: New-SdkArchive -Rid 'win-x64' -NativeArtifact 'cpp-native-win-x64' -Files @( 'foundry_local.dll', 'foundry_local.pdb', - 'foundry_local.lib' + 'foundry_local.lib', + 'Microsoft.Windows.AI.MachineLearning.dll', + 'DirectML.dll', + 'onnxruntime.dll', + 'onnxruntime-genai.dll', + '?onnxruntime_providers_*.dll' ) New-SdkArchive -Rid 'win-arm64' -NativeArtifact 'cpp-native-win-arm64' -Files @( 'foundry_local.dll', 'foundry_local.pdb', - 'foundry_local.lib' + 'foundry_local.lib', + 'Microsoft.Windows.AI.MachineLearning.dll', + 'DirectML.dll', + 'onnxruntime.dll', + 'onnxruntime-genai.dll', + '?onnxruntime_providers_*.dll' ) New-SdkArchive -Rid 'linux-x64' -NativeArtifact 'cpp-native-linux-x64' -Files @( - 'libfoundry_local.so' + 'libfoundry_local.so', + 'libonnxruntime-genai.so', + '?libonnxruntime-genai*.so', + '?libonnxruntime_providers_*.so', + 'libonnxruntime.so*' ) New-SdkArchive -Rid 'osx-arm64' -NativeArtifact 'cpp-native-osx-arm64' -Files @( - 'libfoundry_local.dylib' + 'libfoundry_local.dylib', + 'libonnxruntime-genai.dylib', + 'libonnxruntime*.dylib', + '?libonnxruntime_providers_*.dylib' ) Write-Host "Staged archives:" diff --git a/.pipelines/v2/templates/steps-pack-js.yml b/.pipelines/v2/templates/steps-pack-js.yml index 439ab41fd..9d7a4aff1 100644 --- a/.pipelines/v2/templates/steps-pack-js.yml +++ b/.pipelines/v2/templates/steps-pack-js.yml @@ -74,6 +74,7 @@ steps: $dst = '${{ parameters.outputDir }}' New-Item -ItemType Directory -Force -Path $dst | Out-Null Set-Location "$(Build.SourcesDirectory)/sdk_v2/js" + Copy-Item "$(Build.SourcesDirectory)/sdk_v2/deps_versions.json" -Destination "deps_versions.json" -Force npm pack --pack-destination $dst if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } Get-ChildItem $dst | ForEach-Object { Write-Host " $($_.Name) $($_.Length) bytes" } diff --git a/sdk_v2/cpp/CMakeLists.txt b/sdk_v2/cpp/CMakeLists.txt index 0bf0f7a5f..d0677dd75 100644 --- a/sdk_v2/cpp/CMakeLists.txt +++ b/sdk_v2/cpp/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright (c) Microsoft. All rights reserved. -cmake_minimum_required(VERSION 3.20) +cmake_minimum_required(VERSION 3.21) # Android: web service not applicable on device. # Detect Android early via the vcpkg triplet name because the ANDROID variable @@ -13,6 +13,9 @@ endif() if(NOT DEFINED FOUNDRY_LOCAL_BUILD_SERVICE OR FOUNDRY_LOCAL_BUILD_SERVICE) list(APPEND VCPKG_MANIFEST_FEATURES "service") endif() +if(NOT DEFINED FOUNDRY_LOCAL_USE_TELEMETRY OR FOUNDRY_LOCAL_USE_TELEMETRY) + list(APPEND VCPKG_MANIFEST_FEATURES "telemetry") +endif() project(foundry_local VERSION 0.1.0 LANGUAGES CXX C) @@ -43,8 +46,27 @@ endif() option(FOUNDRY_LOCAL_BUILD_TESTS "Build unit tests" ON) option(FOUNDRY_LOCAL_BUILD_EXAMPLES "Build example programs" ON) option(FOUNDRY_LOCAL_BUILD_SERVICE "Build web service support (requires oat++)" ON) +option(FOUNDRY_LOCAL_USE_TELEMETRY + "Build with 1DS telemetry uploads (requires cpp-client-telemetry vcpkg port)" ON) option(FOUNDRY_LOCAL_ENABLE_ASAN "Enable AddressSanitizer + UndefinedBehaviorSanitizer (Linux only)" OFF) +# 1DS ingestion token. Treat as a secret: pass via the +# FOUNDRY_LOCAL_TELEMETRY_TOKEN environment variable so it does not land in +# process argv or CMakeCache.txt. Defaults to empty so developer builds and forks +# don't accidentally bind to a real tenant. +get_property(_FL_TELEMETRY_TOKEN_IN_CACHE CACHE FOUNDRY_LOCAL_TELEMETRY_TOKEN PROPERTY TYPE SET) +if(_FL_TELEMETRY_TOKEN_IN_CACHE) + get_property(_FL_TELEMETRY_TOKEN_CACHE_VALUE CACHE FOUNDRY_LOCAL_TELEMETRY_TOKEN PROPERTY VALUE) + unset(FOUNDRY_LOCAL_TELEMETRY_TOKEN CACHE) + if(_FL_TELEMETRY_TOKEN_CACHE_VALUE) + message(FATAL_ERROR "FOUNDRY_LOCAL_TELEMETRY_TOKEN must be supplied as an environment variable, not via -D.") + endif() +endif() +set(FOUNDRY_LOCAL_TELEMETRY_TOKEN "$ENV{FOUNDRY_LOCAL_TELEMETRY_TOKEN}") +if(NOT FOUNDRY_LOCAL_USE_TELEMETRY) + set(FOUNDRY_LOCAL_TELEMETRY_TOKEN "") +endif() + # Android: interactive examples and host tools don't run on device if(ANDROID) set(FOUNDRY_LOCAL_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) @@ -87,6 +109,17 @@ if(WIN32) find_package(WinMLEpCatalog) endif() +# 1DS C++ client telemetry — provided by the cpp-client-telemetry vcpkg port. +# Enabled via FOUNDRY_LOCAL_USE_TELEMETRY (default ON) and the vcpkg +# manifest feature "telemetry". When off, the ITelemetry interface is satisfied +# by TelemetryLogger only (local diagnostic log only — no upload). +if(FOUNDRY_LOCAL_USE_TELEMETRY) + find_package(MSTelemetry CONFIG REQUIRED) + message(STATUS "1DS telemetry: enabled (cpp-client-telemetry found)") +else() + message(STATUS "1DS telemetry: disabled (FOUNDRY_LOCAL_USE_TELEMETRY=OFF)") +endif() + # -------------------------------------------------------------------------- # Library target # -------------------------------------------------------------------------- @@ -196,7 +229,12 @@ set(FOUNDRY_LOCAL_SOURCES src/service/web_service.cc src/telemetry/telemetry.cc src/telemetry/telemetry_action_tracker.cc + src/telemetry/invocation_context.cc + src/telemetry/telemetry_environment.cc src/telemetry/telemetry_logger.cc + src/telemetry/telemetry_metadata.cc + src/telemetry/ep_download_tracker.cc + src/telemetry/download_tracker.cc src/utils.cc src/util/file_lock.cc src/http/http_download.cc @@ -208,6 +246,13 @@ set(FOUNDRY_LOCAL_SOURCES ${FOUNDRY_LOCAL_INTERNAL_HEADERS} ) +# 1DS bridge — compiled only when the cpp-client-telemetry port is available. +# When disabled, Manager falls back to TelemetryLogger and the OneDsTelemetry +# symbol is never referenced. +if(FOUNDRY_LOCAL_USE_TELEMETRY) + list(APPEND FOUNDRY_LOCAL_SOURCES src/telemetry/one_ds_telemetry.cc) +endif() + # Organize headers into filters matching the directory structure in Visual Studio source_group(TREE "${CMAKE_CURRENT_SOURCE_DIR}" PREFIX "Source" FILES ${FOUNDRY_LOCAL_SOURCES}) source_group(TREE "${CMAKE_CURRENT_SOURCE_DIR}/include" PREFIX "Public Headers" FILES ${FOUNDRY_LOCAL_PUBLIC_HEADERS}) @@ -273,6 +318,14 @@ function(foundry_local_configure_target TARGET LINK_SCOPE) else() target_compile_definitions(${TARGET} ${LINK_SCOPE} FOUNDRY_LOCAL_HAS_EP_CATALOG=0) endif() + + if(FOUNDRY_LOCAL_USE_TELEMETRY) + target_link_libraries(${TARGET} ${LINK_SCOPE} MSTelemetry::mat) + target_compile_definitions(${TARGET} ${LINK_SCOPE} FOUNDRY_LOCAL_HAS_1DS=1) + else() + target_compile_definitions(${TARGET} ${LINK_SCOPE} FOUNDRY_LOCAL_HAS_1DS=0) + endif() + endfunction() # -------------------------------------------------------------------------- @@ -291,6 +344,16 @@ configure_file( @ONLY ) +# Generate the 1DS tenant-token header. Always generated (even when +# FOUNDRY_LOCAL_USE_TELEMETRY=OFF) so includes of one_ds_tenant_token.h compile +# in both configurations; consumers must still guard MAT::LogManager calls +# behind FOUNDRY_LOCAL_HAS_1DS. +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/src/telemetry/one_ds_tenant_token.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/generated/one_ds_tenant_token.h" + @ONLY +) + # -------------------------------------------------------------------------- # Object library — compiles all sources once. Both the shared (DLL) and # static library targets re-use these object files, avoiding a double build. @@ -355,6 +418,27 @@ if(WIN32) /DELAYLOAD:Microsoft.Windows.AI.MachineLearning.dll ) endif() + + # Strip unreferenced symbols and fold identical COMDATs in Release builds. + # MSVC's /DEBUG (needed for PDBs) silently disables /OPT:REF and /OPT:ICF + # unless they are explicitly re-enabled. /INCREMENTAL:NO is required because + # incremental linking also prevents dead-code elimination. + target_link_options(foundry_local PRIVATE + $<$:/OPT:REF> + $<$:/OPT:ICF> + $<$:/INCREMENTAL:NO> + ) +endif() + +# Strip unreferenced sections on Linux (GCC/Clang) and macOS (Apple ld). +if(UNIX AND NOT APPLE) + target_link_options(foundry_local PRIVATE + $<$:-Wl,--gc-sections> + ) +elseif(APPLE) + target_link_options(foundry_local PRIVATE + $<$:-Wl,-dead_strip> + ) endif() # -------------------------------------------------------------------------- @@ -404,6 +488,13 @@ if(TARGET OnnxRuntime::OnnxRuntime) $ ) endif() + if(WinMLEpCatalog_FOUND AND EXISTS "${WINML_EP_CATALOG_DLL_DIR}/DirectML.dll") + add_custom_command(TARGET foundry_local POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${WINML_EP_CATALOG_DLL_DIR}/DirectML.dll" + $ + ) + endif() elseif(APPLE) # macOS: copy dylibs so consumers that only link libfoundry_local.dylib (e.g. cache_only_tests) find the correct # ORT version instead of any system-installed ORT, which would cause an Ort::InitApi() version mismatch. @@ -433,6 +524,9 @@ if(TARGET OnnxRuntime::OnnxRuntime) COMMAND ${CMAKE_COMMAND} -E copy_if_different "${ORT_LIB_DIR}/libonnxruntime.so" $ + COMMAND ${CMAKE_COMMAND} -E create_symlink + libonnxruntime.so + $/libonnxruntime.so.1 ) if(EXISTS "${ORT_LIB_DIR}/libonnxruntime_providers_shared.so") @@ -442,6 +536,21 @@ if(TARGET OnnxRuntime::OnnxRuntime) $ ) endif() + + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + file(GLOB _ORT_GENAI_RUNTIME_LIBS CONFIGURE_DEPENDS + "${ORT_GENAI_LIB_DIR}/libonnxruntime-genai*.so") + file(GLOB _ORT_PROVIDER_RUNTIME_LIBS CONFIGURE_DEPENDS + "${ORT_LIB_DIR}/libonnxruntime_providers_*.so") + set(_LINUX_OPTIONAL_RUNTIME_LIBS ${_ORT_GENAI_RUNTIME_LIBS} ${_ORT_PROVIDER_RUNTIME_LIBS}) + if(_LINUX_OPTIONAL_RUNTIME_LIBS) + add_custom_command(TARGET foundry_local POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + ${_LINUX_OPTIONAL_RUNTIME_LIBS} + $ + ) + endif() + endif() endif() endif() diff --git a/sdk_v2/cpp/build.py b/sdk_v2/cpp/build.py index b1274cd60..8817c3b70 100644 --- a/sdk_v2/cpp/build.py +++ b/sdk_v2/cpp/build.py @@ -57,10 +57,25 @@ def _path_from_env_var(var: str) -> Path | None: # Subprocess runner # --------------------------------------------------------------------------- +def _redact_command_arg(arg: str) -> str: + sensitive_words = ("token", "secret", "password", "passwd", "apikey", "api_key", "accesskey", "sig") + if "=" in arg: + key, value = arg.split("=", 1) + normalized_key = key.lstrip("-").lower() + if "://" in value: + return f"{key}=" + if any(word in normalized_key for word in sensitive_words): + return f"{key}=" + return arg + + def run(cmd: list[str], cwd: Path | str | None = None, env: dict[str, str] | None = None) -> None: """Run a command, logging and streaming output. Raises on non-zero exit.""" - log.info("Running: %s", " ".join(str(c) for c in cmd)) - subprocess.run(cmd, cwd=cwd, env=env, check=True) + redacted = " ".join(_redact_command_arg(str(c)) for c in cmd) + log.info("Running: %s", redacted) + result = subprocess.run(cmd, cwd=cwd, env=env, check=False) + if result.returncode != 0: + raise RuntimeError(f"Command failed with exit code {result.returncode}: {redacted}") # --------------------------------------------------------------------------- @@ -165,7 +180,13 @@ class HelpFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescript help="Override the Microsoft.Windows.AI.MachineLearning NuGet version for the WinML EP " "catalog (Windows only). Defaults to the version pinned in deps_versions.json.", ) - + parser.add_argument( + "--no_telemetry", action="store_true", + help="Skip building the 1DS (cpp-client-telemetry) bridge. Useful while the vcpkg " + "port (microsoft/vcpkg#52316) is unmerged or when building forks that should " + "not link any telemetry transport. Local diagnostic logging via TelemetryLogger " + "still works.", + ) # Cross-compilation (mutually exclusive targets) cross_group = parser.add_mutually_exclusive_group() cross_group.add_argument( @@ -330,6 +351,12 @@ def _validate_args(args: argparse.Namespace) -> None: args.cmake_extra_defines = ( [f"-D{d}" for j in args.cmake_extra_defines for d in j] if args.cmake_extra_defines else [] ) + for define in args.cmake_extra_defines: + if define.startswith("-DFOUNDRY_LOCAL_TELEMETRY_TOKEN"): + raise ValueError( + "FOUNDRY_LOCAL_TELEMETRY_TOKEN must be supplied as an environment variable, " + "not via --cmake_extra_defines or command-line arguments." + ) # Cross-compilation if args.arm64: @@ -451,10 +478,20 @@ def configure(args: argparse.Namespace) -> None: f"-DFOUNDRY_LOCAL_BUILD_SERVICE={build_service}", ] - # Enable vcpkg manifest features for tests + # Enable vcpkg manifest features as needed. Multiple features are passed as a + # semicolon-separated list in a single -D flag. + manifest_features = [] if build_tests == "ON": - command += ["-DVCPKG_MANIFEST_FEATURES=tests"] - + manifest_features.append("tests") + if not args.no_telemetry: + manifest_features.append("telemetry") + if manifest_features: + command += [f"-DVCPKG_MANIFEST_FEATURES={';'.join(manifest_features)}"] + + if args.no_telemetry: + command += ["-DFOUNDRY_LOCAL_USE_TELEMETRY=OFF"] + else: + command += ["-DFOUNDRY_LOCAL_USE_TELEMETRY=ON"] # WinML EP catalog is enabled automatically on Windows by CMake. Allow an # optional version override for the Microsoft.Windows.AI.MachineLearning NuGet. if args.winml_sdk_version: diff --git a/sdk_v2/cpp/cmake/FindOnnxRuntime.cmake b/sdk_v2/cpp/cmake/FindOnnxRuntime.cmake index f11385873..5ed855cf1 100644 --- a/sdk_v2/cpp/cmake/FindOnnxRuntime.cmake +++ b/sdk_v2/cpp/cmake/FindOnnxRuntime.cmake @@ -97,7 +97,7 @@ else() message(STATUS "Downloading ${ORT_PACKAGE_NAME} ${ORT_VERSION} from nuget.org") endif() else() - message(STATUS "Using pre-configured ORT_FETCH_URL: ${ORT_FETCH_URL}") + message(STATUS "Using pre-configured ORT_FETCH_URL") endif() # Normalize backslashes (Windows paths) and handle .nupkg extension @@ -106,11 +106,15 @@ else() set(_ORT_ZIP_PATH "${CMAKE_BINARY_DIR}/_deps/ortlib-download/ort.zip") get_filename_component(_ORT_ZIP_DIR "${_ORT_ZIP_PATH}" DIRECTORY) file(MAKE_DIRECTORY "${_ORT_ZIP_DIR}") - file(COPY_FILE "${ORT_FETCH_URL}" "${_ORT_ZIP_PATH}") + configure_file("${ORT_FETCH_URL}" "${_ORT_ZIP_PATH}" COPYONLY) set(ORT_FETCH_URL "${_ORT_ZIP_PATH}") endif() - FetchContent_Declare(ortlib URL ${ORT_FETCH_URL} DOWNLOAD_EXTRACT_TIMESTAMP TRUE DOWNLOAD_NAME ort.zip) + set(_ORT_FETCH_ARGS URL ${ORT_FETCH_URL} DOWNLOAD_NAME ort.zip) + if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.24") + list(APPEND _ORT_FETCH_ARGS DOWNLOAD_EXTRACT_TIMESTAMP TRUE) + endif() + FetchContent_Declare(ortlib ${_ORT_FETCH_ARGS}) FetchContent_MakeAvailable(ortlib) set(_ORT_HEADER_DIR "${ortlib_SOURCE_DIR}/build/native/include") @@ -149,7 +153,7 @@ else() message(STATUS "Downloading ${_ORT_GPU_LINUX_PACKAGE} ${ORT_VERSION} from nuget.org") endif() else() - message(STATUS "Using pre-configured ORT_GPU_LINUX_FETCH_URL: ${ORT_GPU_LINUX_FETCH_URL}") + message(STATUS "Using pre-configured ORT_GPU_LINUX_FETCH_URL") endif() # Normalize backslashes and handle .nupkg extension @@ -158,11 +162,15 @@ else() set(_ORT_GPU_ZIP_PATH "${CMAKE_BINARY_DIR}/_deps/ort_gpu_linux-download/ort_gpu_linux.zip") get_filename_component(_ORT_GPU_ZIP_DIR "${_ORT_GPU_ZIP_PATH}" DIRECTORY) file(MAKE_DIRECTORY "${_ORT_GPU_ZIP_DIR}") - file(COPY_FILE "${ORT_GPU_LINUX_FETCH_URL}" "${_ORT_GPU_ZIP_PATH}") + configure_file("${ORT_GPU_LINUX_FETCH_URL}" "${_ORT_GPU_ZIP_PATH}" COPYONLY) set(ORT_GPU_LINUX_FETCH_URL "${_ORT_GPU_ZIP_PATH}") endif() - FetchContent_Declare(ort_gpu_linux URL ${ORT_GPU_LINUX_FETCH_URL} DOWNLOAD_EXTRACT_TIMESTAMP TRUE DOWNLOAD_NAME ort_gpu_linux.zip) + set(_ORT_GPU_FETCH_ARGS URL ${ORT_GPU_LINUX_FETCH_URL} DOWNLOAD_NAME ort_gpu_linux.zip) + if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.24") + list(APPEND _ORT_GPU_FETCH_ARGS DOWNLOAD_EXTRACT_TIMESTAMP TRUE) + endif() + FetchContent_Declare(ort_gpu_linux ${_ORT_GPU_FETCH_ARGS}) FetchContent_MakeAvailable(ort_gpu_linux) set(_ORT_LIB_DIR "${ort_gpu_linux_SOURCE_DIR}/runtimes/${_ORT_PLATFORM}/native") diff --git a/sdk_v2/cpp/cmake/FindOnnxRuntimeGenAI.cmake b/sdk_v2/cpp/cmake/FindOnnxRuntimeGenAI.cmake index f83002d64..f8bdc4b10 100644 --- a/sdk_v2/cpp/cmake/FindOnnxRuntimeGenAI.cmake +++ b/sdk_v2/cpp/cmake/FindOnnxRuntimeGenAI.cmake @@ -160,7 +160,7 @@ else() message(STATUS "Downloading ${_GENAI_PACKAGE_NAME} ${ORT_GENAI_VERSION} from nuget.org") endif() else() - message(STATUS "Using caller-provided GENAI_FETCH_URL: ${GENAI_FETCH_URL}") + message(STATUS "Using caller-provided GENAI_FETCH_URL") endif() # Normalize to forward slashes — backslashes from Windows paths cause CMake @@ -173,15 +173,19 @@ else() set(_GENAI_ZIP_PATH "${CMAKE_BINARY_DIR}/_deps/genai-download/genai.zip") get_filename_component(_GENAI_ZIP_DIR "${_GENAI_ZIP_PATH}" DIRECTORY) file(MAKE_DIRECTORY "${_GENAI_ZIP_DIR}") - file(COPY_FILE "${GENAI_FETCH_URL}" "${_GENAI_ZIP_PATH}") + configure_file("${GENAI_FETCH_URL}" "${_GENAI_ZIP_PATH}" COPYONLY) set(GENAI_FETCH_URL "${_GENAI_ZIP_PATH}") - message(STATUS "Copied .nupkg to .zip for CMake extraction: ${GENAI_FETCH_URL}") + message(STATUS "Copied local GenAI .nupkg to .zip for CMake extraction") endif() - FetchContent_Declare(genailib + set(_GENAI_FETCH_ARGS URL ${GENAI_FETCH_URL} - DOWNLOAD_EXTRACT_TIMESTAMP TRUE - DOWNLOAD_NAME genai.zip # .nupkg is a ZIP; force CMake to recognize the format + DOWNLOAD_NAME genai.zip) + if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.24") + list(APPEND _GENAI_FETCH_ARGS DOWNLOAD_EXTRACT_TIMESTAMP TRUE) + endif() + FetchContent_Declare(genailib + ${_GENAI_FETCH_ARGS} ) FetchContent_MakeAvailable(genailib) diff --git a/sdk_v2/cpp/cmake/FindWinMLEpCatalog.cmake b/sdk_v2/cpp/cmake/FindWinMLEpCatalog.cmake index dc732a4ff..4bf4eb0f7 100644 --- a/sdk_v2/cpp/cmake/FindWinMLEpCatalog.cmake +++ b/sdk_v2/cpp/cmake/FindWinMLEpCatalog.cmake @@ -60,30 +60,32 @@ if(NOT _WINML_PLATFORM_UPPER MATCHES "^(AMD64|X64|X86_64|ARM64|AARCH64)$") return() endif() -include(cmake/nuget.cmake) - # WINML_EP_CATALOG_FETCH_URL can be set externally (e.g. for CI where nuget.org is blocked). set(WINML_EP_CATALOG_FETCH_URL "" CACHE STRING "Override URL or local path for the WinML EP Catalog NuGet package") -if(WINML_EP_CATALOG_FETCH_URL) - # Use FetchContent to download/extract the pre-downloaded package - include(FetchContent) - string(REPLACE "\\" "/" WINML_EP_CATALOG_FETCH_URL "${WINML_EP_CATALOG_FETCH_URL}") - if(WINML_EP_CATALOG_FETCH_URL MATCHES "\\.nupkg$" AND NOT WINML_EP_CATALOG_FETCH_URL MATCHES "^https?://") - set(_WINML_ZIP_PATH "${CMAKE_BINARY_DIR}/_deps/winml_ep_catalog-download/winml_ep_catalog.zip") - get_filename_component(_WINML_ZIP_DIR "${_WINML_ZIP_PATH}" DIRECTORY) - file(MAKE_DIRECTORY "${_WINML_ZIP_DIR}") - file(COPY_FILE "${WINML_EP_CATALOG_FETCH_URL}" "${_WINML_ZIP_PATH}") - set(WINML_EP_CATALOG_FETCH_URL "${_WINML_ZIP_PATH}") - endif() - FetchContent_Declare(winml_ep_catalog URL ${WINML_EP_CATALOG_FETCH_URL} DOWNLOAD_EXTRACT_TIMESTAMP TRUE DOWNLOAD_NAME winml_ep_catalog.zip) - FetchContent_MakeAvailable(winml_ep_catalog) - set(_WINML_EP_ROOT "${winml_ep_catalog_SOURCE_DIR}") - message(STATUS "WinML EP Catalog via FetchContent: ${_WINML_EP_ROOT}") -else() - install_nuget_package(Microsoft.Windows.AI.MachineLearning ${WINML_EP_CATALOG_VERSION} _WINML_EP_ROOT - SOURCE https://api.nuget.org/v3/index.json) +include(FetchContent) +set(_WINML_EP_CATALOG_URL "${WINML_EP_CATALOG_FETCH_URL}") +if(NOT _WINML_EP_CATALOG_URL) + set(_WINML_EP_CATALOG_URL + "https://api.nuget.org/v3-flatcontainer/microsoft.windows.ai.machinelearning/${WINML_EP_CATALOG_VERSION}/microsoft.windows.ai.machinelearning.${WINML_EP_CATALOG_VERSION}.nupkg") +endif() + +string(REPLACE "\\" "/" _WINML_EP_CATALOG_URL "${_WINML_EP_CATALOG_URL}") +if(_WINML_EP_CATALOG_URL MATCHES "\\.nupkg$" AND NOT _WINML_EP_CATALOG_URL MATCHES "^https?://") + set(_WINML_ZIP_PATH "${CMAKE_BINARY_DIR}/_deps/winml_ep_catalog-download/winml_ep_catalog.zip") + get_filename_component(_WINML_ZIP_DIR "${_WINML_ZIP_PATH}" DIRECTORY) + file(MAKE_DIRECTORY "${_WINML_ZIP_DIR}") + configure_file("${_WINML_EP_CATALOG_URL}" "${_WINML_ZIP_PATH}" COPYONLY) + set(_WINML_EP_CATALOG_URL "${_WINML_ZIP_PATH}") +endif() +set(_WINML_FETCH_ARGS URL ${_WINML_EP_CATALOG_URL} DOWNLOAD_NAME winml_ep_catalog.zip) +if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.24") + list(APPEND _WINML_FETCH_ARGS DOWNLOAD_EXTRACT_TIMESTAMP TRUE) endif() +FetchContent_Declare(winml_ep_catalog ${_WINML_FETCH_ARGS}) +FetchContent_MakeAvailable(winml_ep_catalog) +set(_WINML_EP_ROOT "${winml_ep_catalog_SOURCE_DIR}") +message(STATUS "WinML EP Catalog via FetchContent: ${_WINML_EP_ROOT}") # Load the package's first-party CMake config for target discovery and layout # resolution. The config lives at build/cmake/-config.cmake diff --git a/sdk_v2/cpp/include/foundry_local/foundry_local_c.h b/sdk_v2/cpp/include/foundry_local/foundry_local_c.h index 034348e99..3974d75d4 100644 --- a/sdk_v2/cpp/include/foundry_local/foundry_local_c.h +++ b/sdk_v2/cpp/include/foundry_local/foundry_local_c.h @@ -60,7 +60,7 @@ * Incremented with each release. * Used to request the API function table via FoundryLocalGetApi. * ----------------------------------------------------------------------- */ -#define FOUNDRY_LOCAL_API_VERSION 1 +#define FOUNDRY_LOCAL_API_VERSION 2 /* ----------------------------------------------------------------------- * Platform export macros (C version) @@ -816,7 +816,7 @@ struct flItemApi { void FL_API_T(ItemQueue_MarkFinished, _In_ flItemQueue* queue); ///< Producer is done bool FL_API_T(ItemQueue_IsFinished, _In_ const flItemQueue* queue); - // End V1 + // End V2 }; struct flInferenceApi { @@ -889,7 +889,7 @@ struct flInferenceApi { /// If all turns are undone, the cached generator is destroyed. FL_API_STATUS(Session_UndoTurns, _In_ flSession* session, size_t count); - // End V1 + // End V2 }; /* --- Configuration API ------------------------------------------------- */ diff --git a/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h b/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h index fe6281302..19c4cef31 100644 --- a/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h +++ b/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.h @@ -61,39 +61,80 @@ inline const char* Version() noexcept; namespace detail { /// Get the API function table (cached on first call). -/// Returns nullptr if the library does not support the requested API version. inline const flApi* api() { - static const flApi* p = FoundryLocalGetApi(FOUNDRY_LOCAL_API_VERSION); + static const flApi* p = [] { + const flApi* root = FoundryLocalGetApi(FOUNDRY_LOCAL_API_VERSION); + if (!root) { + throw Error("Foundry Local native library does not support the requested API version", + FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + } + return root; + }(); return p; } /// Get the Catalog sub-API (cached on first call). inline const flCatalogApi* catalog_api() { - static const flCatalogApi* p = api()->GetCatalogApi(); + static const flCatalogApi* p = [] { + const flCatalogApi* catalog = api()->GetCatalogApi(); + if (!catalog) { + throw Error("Foundry Local native library returned a null Catalog API table", + FOUNDRY_LOCAL_ERROR_INTERNAL); + } + return catalog; + }(); return p; } /// Get the Configuration sub-API (cached on first call). inline const flConfigurationApi* config_api() { - static const flConfigurationApi* p = api()->GetConfigurationApi(); + static const flConfigurationApi* p = [] { + const flConfigurationApi* config = api()->GetConfigurationApi(); + if (!config) { + throw Error("Foundry Local native library returned a null Configuration API table", + FOUNDRY_LOCAL_ERROR_INTERNAL); + } + return config; + }(); return p; } /// Get the Inference sub-API (cached on first call). inline const flInferenceApi* inference_api() { - static const flInferenceApi* p = api()->GetInferenceApi(); + static const flInferenceApi* p = [] { + const flInferenceApi* inference = api()->GetInferenceApi(); + if (!inference) { + throw Error("Foundry Local native library returned a null Inference API table", + FOUNDRY_LOCAL_ERROR_INTERNAL); + } + return inference; + }(); return p; } /// Get the Item sub-API (cached on first call). inline const flItemApi* item_api() { - static const flItemApi* p = api()->GetItemApi(); + static const flItemApi* p = [] { + const flItemApi* item = api()->GetItemApi(); + if (!item) { + throw Error("Foundry Local native library returned a null Item API table", + FOUNDRY_LOCAL_ERROR_INTERNAL); + } + return item; + }(); return p; } /// Get the Model sub-API (cached on first call). inline const flModelApi* model_api() { - static const flModelApi* p = api()->GetModelApi(); + static const flModelApi* p = [] { + const flModelApi* model = api()->GetModelApi(); + if (!model) { + throw Error("Foundry Local native library returned a null Model API table", + FOUNDRY_LOCAL_ERROR_INTERNAL); + } + return model; + }(); return p; } @@ -205,7 +246,7 @@ class KeyValuePairs { KeyValuePairs& operator=(KeyValuePairs&&) noexcept = default; /// Get a value by key. Returns nullopt if not found. - std::optional Get(const char* key) const noexcept; + std::optional Get(const char* key) const; /// Get all key/value pairs. std::vector GetAll() const; @@ -305,71 +346,71 @@ class ModelInfo { explicit ModelInfo(const flModelInfo& info) noexcept : info_(&info) {} // Core identity. - std::string_view Id() const noexcept; - std::string_view Name() const noexcept; - int Version() const noexcept; - std::string_view Alias() const noexcept; - std::string_view Uri() const noexcept; - flDeviceType DeviceType() const noexcept; - std::optional ExecutionProvider() const noexcept; + std::string_view Id() const; + std::string_view Name() const; + int Version() const; + std::string_view Alias() const; + std::string_view Uri() const; + flDeviceType DeviceType() const; + std::optional ExecutionProvider() const; /// Returns the device + execution provider pair, or nullopt if device_type is unknown/invalid. - std::optional GetRuntime() const noexcept; + std::optional GetRuntime() const; // Key-value lookups - std::optional GetPromptTemplate(const char* key) const noexcept; - std::optional GetModelSetting(const char* key) const noexcept; + std::optional GetPromptTemplate(const char* key) const; + std::optional GetModelSetting(const char* key) const; /// Default/recommended inference settings declared by the model author. /// Returns a non-owning read-only view; nullopt if no settings are declared. /// Useful for discovering which parameters a model supports overriding. - std::optional GetModelSettings() const noexcept; + std::optional GetModelSettings() const; /// Get a string property by key. Known keys are defined by the FOUNDRY_LOCAL_MODEL_PROP_*_STR constants. - std::optional GetStringProperty(const char* key) const noexcept; + std::optional GetStringProperty(const char* key) const; /// Get an int property by key. Known keys are defined by the FOUNDRY_LOCAL_MODEL_PROP_*_INT constants. /// Returns default_value if key is not set. - int64_t GetIntProperty(const char* key, int64_t default_value = -1) const noexcept; + int64_t GetIntProperty(const char* key, int64_t default_value = -1) const; // --- Typed property accessors (convenience wrappers over Get{String,Int}Property) --- /// Display name shown in UIs. May differ from name(). - std::optional DisplayName() const noexcept; + std::optional DisplayName() const; /// Model format, e.g. "onnx". - std::optional ModelType() const noexcept; + std::optional ModelType() const; /// Publisher / organization. - std::optional Publisher() const noexcept; + std::optional Publisher() const; /// SPDX license identifier. - std::optional License() const noexcept; + std::optional License() const; /// Human-readable license description. - std::optional LicenseDescription() const noexcept; + std::optional LicenseDescription() const; /// Task the model is designed for, e.g. "chat", "text-generation". - std::optional Task() const noexcept; + std::optional Task() const; /// Source of the model, e.g. "AzureCatalog". - std::optional ModelProvider() const noexcept; + std::optional ModelProvider() const; /// Minimum Foundry Local version required to run this model. - std::optional MinFlVersion() const noexcept; + std::optional MinFlVersion() const; /// URI of the parent model (for derived/quantized variants). - std::optional ParentUri() const noexcept; + std::optional ParentUri() const; /// Whether the model supports tool/function calling. nullopt if unspecified. - std::optional SupportsToolCalling() const noexcept; + std::optional SupportsToolCalling() const; /// Download size in megabytes. nullopt if unspecified. - std::optional FilesizeMb() const noexcept; + std::optional FilesizeMb() const; /// Maximum output tokens the model can generate. nullopt if unspecified. - std::optional MaxOutputTokens() const noexcept; + std::optional MaxOutputTokens() const; /// Unix timestamp when the model entry was created. 0 if unset. - int64_t CreatedAtUnix() const noexcept; + int64_t CreatedAtUnix() const; /// Whether this is a test/synthetic model (not for production). - bool IsTestModel() const noexcept; + bool IsTestModel() const; /// Maximum context length in tokens. nullopt if unspecified. - std::optional ContextLength() const noexcept; + std::optional ContextLength() const; /// Comma-separated list of supported input modalities (e.g. "text,image"). nullopt if unspecified. - std::optional InputModalities() const noexcept; + std::optional InputModalities() const; /// Comma-separated list of supported output modalities. nullopt if unspecified. - std::optional OutputModalities() const noexcept; + std::optional OutputModalities() const; /// Comma-separated list of model capabilities. nullopt if unspecified. - std::optional Capabilities() const noexcept; + std::optional Capabilities() const; private: static std::string_view safe(const char* s) noexcept { return s ? s : ""; } @@ -510,7 +551,7 @@ class Item { // --- Read accessors (always work) --- - flItemType GetType() const noexcept; + flItemType GetType() const; BytesContent GetBytes() const; TextContent GetText() const; TensorContent GetTensor() const; @@ -776,7 +817,13 @@ class ICatalog { /// Queries for different aliases do not invalidate each other's results. virtual ModelList GetModelVersions(const std::string& model_alias, const std::string& variant_name = {}, - int max_versions = 50) = 0; + int max_versions = 50) { + (void)model_alias; + (void)variant_name; + (void)max_versions; + throw Error("GetModelVersions is not implemented by this catalog", + FOUNDRY_LOCAL_ERROR_NOT_IMPLEMENTED); + } }; // =========================================================================== @@ -942,7 +989,7 @@ class Request { Request& AddItem(Item&& item) { return AddItem(item, true); } - size_t GetItemCount() const noexcept; + size_t GetItemCount() const; Item GetItem(size_t idx) const; /// Options for this request. Overrides session options for the duration of this request. @@ -964,7 +1011,7 @@ class Response { /// Lifetime is tied to this Response object. const std::vector& GetItems() const noexcept { return items_; } - flFinishReason GetFinishReason() const noexcept; + flFinishReason GetFinishReason() const; flUsage GetUsage() const; private: diff --git a/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.inline.h b/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.inline.h index 08c8f594f..3fc4d9a66 100644 --- a/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.inline.h +++ b/sdk_v2/cpp/include/foundry_local/foundry_local_cpp.inline.h @@ -91,7 +91,7 @@ inline KeyValuePairs::KeyValuePairs( } } -inline std::optional KeyValuePairs::Get(const char* key) const noexcept { +inline std::optional KeyValuePairs::Get(const char* key) const { if (!handle_.get()) { return std::nullopt; } @@ -289,36 +289,36 @@ inline flManager* detail::CreateManager(const Configuration& config) { // ModelInfo // =========================================================================== -inline std::string_view ModelInfo::Id() const noexcept { +inline std::string_view ModelInfo::Id() const { return safe(detail::model_api()->Info_GetId(info_)); } -inline std::string_view ModelInfo::Name() const noexcept { +inline std::string_view ModelInfo::Name() const { return safe(detail::model_api()->Info_GetName(info_)); } -inline int ModelInfo::Version() const noexcept { +inline int ModelInfo::Version() const { return detail::model_api()->Info_GetVersion(info_); } -inline std::string_view ModelInfo::Alias() const noexcept { +inline std::string_view ModelInfo::Alias() const { return safe(detail::model_api()->Info_GetAlias(info_)); } -inline std::string_view ModelInfo::Uri() const noexcept { +inline std::string_view ModelInfo::Uri() const { return safe(detail::model_api()->Info_GetUri(info_)); } -inline flDeviceType ModelInfo::DeviceType() const noexcept { +inline flDeviceType ModelInfo::DeviceType() const { return detail::model_api()->Info_GetDeviceType(info_); } -inline std::optional ModelInfo::ExecutionProvider() const noexcept { +inline std::optional ModelInfo::ExecutionProvider() const { const char* v = detail::model_api()->Info_GetExecutionProvider(info_); return v ? std::optional{v} : std::nullopt; } -inline std::optional ModelInfo::GetRuntime() const noexcept { +inline std::optional ModelInfo::GetRuntime() const { flDeviceType dt = DeviceType(); if (dt == FOUNDRY_LOCAL_DEVICE_NOTSET) { return std::nullopt; @@ -326,7 +326,7 @@ inline std::optional ModelInfo::GetRuntime() const noexcept { return Runtime{dt, ExecutionProvider()}; } -inline std::optional ModelInfo::GetPromptTemplate(const char* key) const noexcept { +inline std::optional ModelInfo::GetPromptTemplate(const char* key) const { const flKeyValuePairs* kvps = detail::model_api()->Info_GetPromptTemplates(info_); if (!kvps) { return std::nullopt; @@ -335,7 +335,7 @@ inline std::optional ModelInfo::GetPromptTemplate(const char* return v ? std::optional{v} : std::nullopt; } -inline std::optional ModelInfo::GetModelSetting(const char* key) const noexcept { +inline std::optional ModelInfo::GetModelSetting(const char* key) const { const flKeyValuePairs* kvps = detail::model_api()->Info_GetModelSettings(info_); if (!kvps) { return std::nullopt; @@ -344,7 +344,7 @@ inline std::optional ModelInfo::GetModelSetting(const char* ke return v ? std::optional{v} : std::nullopt; } -inline std::optional ModelInfo::GetModelSettings() const noexcept { +inline std::optional ModelInfo::GetModelSettings() const { const flKeyValuePairs* kvps = detail::model_api()->Info_GetModelSettings(info_); if (!kvps) { return std::nullopt; @@ -352,54 +352,54 @@ inline std::optional ModelInfo::GetModelSettings() const noexcept return KeyValuePairs(*kvps); } -inline std::optional ModelInfo::GetStringProperty(const char* key) const noexcept { +inline std::optional ModelInfo::GetStringProperty(const char* key) const { const char* v = detail::model_api()->Info_GetStringProperty(info_, key); return v ? std::optional{v} : std::nullopt; } -inline int64_t ModelInfo::GetIntProperty(const char* key, int64_t default_value) const noexcept { +inline int64_t ModelInfo::GetIntProperty(const char* key, int64_t default_value) const { return detail::model_api()->Info_GetIntProperty(info_, key, default_value); } // --- Typed property accessors --- -inline std::optional ModelInfo::DisplayName() const noexcept { +inline std::optional ModelInfo::DisplayName() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_DISPLAY_NAME_STR); } -inline std::optional ModelInfo::ModelType() const noexcept { +inline std::optional ModelInfo::ModelType() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_MODEL_TYPE_STR); } -inline std::optional ModelInfo::Publisher() const noexcept { +inline std::optional ModelInfo::Publisher() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_PUBLISHER_STR); } -inline std::optional ModelInfo::License() const noexcept { +inline std::optional ModelInfo::License() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_LICENSE_STR); } -inline std::optional ModelInfo::LicenseDescription() const noexcept { +inline std::optional ModelInfo::LicenseDescription() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_LICENSE_DESCRIPTION_STR); } -inline std::optional ModelInfo::Task() const noexcept { +inline std::optional ModelInfo::Task() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_TASK_STR); } -inline std::optional ModelInfo::ModelProvider() const noexcept { +inline std::optional ModelInfo::ModelProvider() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_MODEL_PROVIDER_STR); } -inline std::optional ModelInfo::MinFlVersion() const noexcept { +inline std::optional ModelInfo::MinFlVersion() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_MIN_FL_VERSION_STR); } -inline std::optional ModelInfo::ParentUri() const noexcept { +inline std::optional ModelInfo::ParentUri() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_PARENT_URI_STR); } -inline std::optional ModelInfo::SupportsToolCalling() const noexcept { +inline std::optional ModelInfo::SupportsToolCalling() const { int64_t v = GetIntProperty(FOUNDRY_LOCAL_MODEL_PROP_SUPPORTS_TOOL_CALLING_INT); if (v < 0) { return std::nullopt; @@ -407,7 +407,7 @@ inline std::optional ModelInfo::SupportsToolCalling() const noexcept { return v != 0; } -inline std::optional ModelInfo::FilesizeMb() const noexcept { +inline std::optional ModelInfo::FilesizeMb() const { int64_t v = GetIntProperty(FOUNDRY_LOCAL_MODEL_PROP_FILESIZE_MB_INT); if (v < 0) { return std::nullopt; @@ -415,7 +415,7 @@ inline std::optional ModelInfo::FilesizeMb() const noexcept { return v; } -inline std::optional ModelInfo::MaxOutputTokens() const noexcept { +inline std::optional ModelInfo::MaxOutputTokens() const { int64_t v = GetIntProperty(FOUNDRY_LOCAL_MODEL_PROP_MAX_OUTPUT_TOKENS_INT); if (v < 0) { return std::nullopt; @@ -423,15 +423,15 @@ inline std::optional ModelInfo::MaxOutputTokens() const noexcept { return v; } -inline int64_t ModelInfo::CreatedAtUnix() const noexcept { +inline int64_t ModelInfo::CreatedAtUnix() const { return GetIntProperty(FOUNDRY_LOCAL_MODEL_PROP_CREATED_AT_UNIX_INT, 0); } -inline bool ModelInfo::IsTestModel() const noexcept { +inline bool ModelInfo::IsTestModel() const { return GetIntProperty(FOUNDRY_LOCAL_MODEL_PROP_IS_TEST_MODEL_INT, 0) != 0; } -inline std::optional ModelInfo::ContextLength() const noexcept { +inline std::optional ModelInfo::ContextLength() const { int64_t v = GetIntProperty(FOUNDRY_LOCAL_MODEL_PROP_CONTEXT_LENGTH_INT, -1); if (v < 0) { return std::nullopt; @@ -439,15 +439,15 @@ inline std::optional ModelInfo::ContextLength() const noexcept { return v; } -inline std::optional ModelInfo::InputModalities() const noexcept { +inline std::optional ModelInfo::InputModalities() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_INPUT_MODALITIES_STR); } -inline std::optional ModelInfo::OutputModalities() const noexcept { +inline std::optional ModelInfo::OutputModalities() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_OUTPUT_MODALITIES_STR); } -inline std::optional ModelInfo::Capabilities() const noexcept { +inline std::optional ModelInfo::Capabilities() const { return GetStringProperty(FOUNDRY_LOCAL_MODEL_PROP_CAPABILITIES_STR); } @@ -634,7 +634,7 @@ inline Item::Item(flItem& raw) inline Item::Item(flItemType type) : handle_(detail::CreateItem(type), detail::item_api()->Item_Release) {} -inline flItemType Item::GetType() const noexcept { +inline flItemType Item::GetType() const { return detail::item_api()->GetType(handle_.get()); } @@ -1076,7 +1076,7 @@ inline Request& Request::AddItem(Item& item, bool take_ownership) { return *this; } -inline size_t Request::GetItemCount() const noexcept { +inline size_t Request::GetItemCount() const { return detail::inference_api()->Request_GetItemCount(handle_.get()); } @@ -1118,7 +1118,7 @@ inline Response::Response(flResponse* response) } } -inline flFinishReason Response::GetFinishReason() const noexcept { +inline flFinishReason Response::GetFinishReason() const { return detail::inference_api()->Response_GetFinishReason(handle_.get()); } diff --git a/sdk_v2/cpp/nuget/pack.py b/sdk_v2/cpp/nuget/pack.py index 902ba288a..0ef856e1a 100644 --- a/sdk_v2/cpp/nuget/pack.py +++ b/sdk_v2/cpp/nuget/pack.py @@ -60,6 +60,7 @@ # foundry_local.dll; other platforms don't, so presence alone drives inclusion. OPTIONAL_SIBLINGS: tuple[str, ...] = ( "Microsoft.Windows.AI.MachineLearning.dll", + "DirectML.dll", ) log = logging.getLogger("pack") diff --git a/sdk_v2/cpp/src/c_api.cc b/sdk_v2/cpp/src/c_api.cc index dbf49cc03..71b385322 100644 --- a/sdk_v2/cpp/src/c_api.cc +++ b/sdk_v2/cpp/src/c_api.cc @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -70,9 +71,18 @@ struct flCatalog { // --- Manager --- struct flManager { + explicit flManager(fl::Manager& manager) : impl(manager) {} + + struct UrlSnapshot { + std::vector strings; + std::vector pointers; + }; + fl::Manager& impl; std::unique_ptr catalog; // stores the flCatalog wrapper around impl.GetCatalog() - mutable std::vector urls_cache; + mutable std::mutex urls_cache_mutex; + mutable std::vector> urls_snapshots; + mutable UrlSnapshot* current_urls_snapshot = nullptr; }; // ======================================================================== @@ -327,7 +337,7 @@ FL_API_STATUS_IMPL(Manager_CreateImpl, const flConfiguration* config, flManager* } auto& mgr = fl::Manager::Create(*cfg); - auto wrapper = std::make_unique(flManager{mgr, nullptr, {}}); + auto wrapper = std::make_unique(mgr); wrapper->catalog = std::make_unique(flCatalog{mgr.GetCatalog()}); *out_manager = wrapper.release(); return nullptr; @@ -372,15 +382,29 @@ FL_API_STATUS_IMPL(Manager_WebServiceUrlsImpl, const flManager* manager, return MakeStatus(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "null argument"); } - const auto& urls = manager->impl.GetWebServiceUrls(); - manager->urls_cache.clear(); - manager->urls_cache.reserve(urls.size()); - for (const auto& u : urls) { - manager->urls_cache.push_back(u.c_str()); + std::lock_guard lock(manager->urls_cache_mutex); + auto urls = manager->impl.GetWebServiceUrls(); + if (urls.empty()) { + *out_urls = nullptr; + *out_num_urls = 0; + manager->current_urls_snapshot = nullptr; + return nullptr; } - *out_urls = manager->urls_cache.data(); - *out_num_urls = manager->urls_cache.size(); + if (manager->current_urls_snapshot == nullptr || manager->current_urls_snapshot->strings != urls) { + auto snapshot = std::make_unique(); + snapshot->strings = std::move(urls); + snapshot->pointers.reserve(snapshot->strings.size()); + for (const auto& u : snapshot->strings) { + snapshot->pointers.push_back(u.c_str()); + } + manager->current_urls_snapshot = snapshot.get(); + manager->urls_snapshots.push_back(std::move(snapshot)); + } + + auto* snapshot_ptr = manager->current_urls_snapshot; + *out_urls = snapshot_ptr->pointers.data(); + *out_num_urls = snapshot_ptr->pointers.size(); return nullptr; API_IMPL_END } @@ -390,8 +414,9 @@ FL_API_STATUS_IMPL(Manager_WebServiceStopImpl, flManager* manager) { if (!manager) { return MakeStatus(FOUNDRY_LOCAL_ERROR_INVALID_ARGUMENT, "null manager"); } - + std::lock_guard lock(manager->urls_cache_mutex); manager->impl.StopWebService(); + manager->current_urls_snapshot = nullptr; return nullptr; API_IMPL_END } @@ -1844,10 +1869,10 @@ static const flModelApi* FL_API_CALL GetModelApiImpl() FL_NO_EXCEPTION { } // ======================================================================== -// Root API function table (version 1) +// Root API function table (version 2) // ======================================================================== -static const flApi g_api_v1 = { +static const flApi g_api_v2 = { /* Status */ Status_CreateImpl, Status_ReleaseImpl, @@ -1898,7 +1923,7 @@ extern "C" { FL_EXPORT const flApi* FL_API_CALL FoundryLocalGetApi(uint32_t version) FL_NO_EXCEPTION { if (version == 0 || version <= FOUNDRY_LOCAL_API_VERSION) { - return &g_api_v1; + return &g_api_v2; } return nullptr; diff --git a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc index 39afcae37..2135a6fe7 100644 --- a/sdk_v2/cpp/src/catalog/azure_model_catalog.cc +++ b/sdk_v2/cpp/src/catalog/azure_model_catalog.cc @@ -6,6 +6,9 @@ #include "catalog/local_model_scanner.h" #include "model.h" #include "model_info.h" +#include "telemetry/invocation_context.h" +#include "telemetry/telemetry.h" +#include "telemetry/telemetry_redaction.h" #include "utils.h" #include @@ -16,6 +19,72 @@ namespace fl { +namespace { + +/// Split a catalog URL into structured telemetry dimensions. The Azure Foundry +/// catalog URL looks like "https://ai.azure.com/api//", e.g. +/// "https://ai.azure.com/api/eastus/ux/v1.0" -> {ai.azure.com, eastus, ux/v1.0}. +/// Custom/private URLs are bucketed so telemetry does not disclose hostnames or paths. +struct ParsedCatalogUrl { + std::string endpoint; + std::string region; + std::string format; +}; + +ParsedCatalogUrl ParseCatalogUrl(const std::string& url) { + if (url == "static") { + return {"static", "", ""}; + } + + std::string rest = url; + if (auto scheme = rest.find("://"); scheme != std::string::npos) { + rest = rest.substr(scheme + 3); + } + + ParsedCatalogUrl out; + std::string path; + if (auto slash = rest.find('/'); slash == std::string::npos) { + out.endpoint = rest; + } else { + out.endpoint = rest.substr(0, slash); + path = rest.substr(slash + 1); + } + + if (auto q = path.find_first_of("?#"); q != std::string::npos) { + path = path.substr(0, q); + } + + std::vector segments; + size_t pos = 0; + while (pos < path.size()) { + auto next = path.find('/', pos); + if (next == std::string::npos) { + next = path.size(); + } + if (next > pos) { + segments.push_back(path.substr(pos, next - pos)); + } + pos = next + 1; + } + + if (out.endpoint != "ai.azure.com" || segments.size() < 4 || segments[0] != "api" || + segments[2] != "ux" || segments[3].empty() || segments[3][0] != 'v') { + return {"custom", "", ""}; + } + + out.region = segments[1]; + for (size_t i = 2; i < segments.size(); ++i) { + if (!out.format.empty()) { + out.format += '/'; + } + out.format += segments[i]; + } + + return out; +} + +} // namespace + AzureModelCatalog::AzureModelCatalog(std::vector>> catalog_urls, std::string cache_dir, ModelFactory model_factory, @@ -23,7 +92,8 @@ AzureModelCatalog::AzureModelCatalog(std::vector(kDefaultCatalogFilter)); } @@ -77,6 +148,9 @@ std::vector AzureModelCatalog::FetchModels() const { std::vector fetched_infos; const std::string& cache_dir = cache_dir_; + // One correlation id groups every catalog access made by this refresh. + const std::string correlation_id = MakeGuidV4Hex(); + logger_.Log(LogLevel::Information, "Getting latest info from the Azure catalog and for locally cached models."); @@ -96,7 +170,16 @@ std::vector AzureModelCatalog::FetchModels() const { // while letting callers explicitly request "" as a real filter override. auto client = MakeCatalogClient(url, filter.value_or(""), ep_detector_, logger_, cache_dir, catalog_region_, disable_region_fallback_); - auto model_infos = FetchAllModelInfosWithCachedModels(*client, cached_model_ids, logger_); + + auto parsed = ParseCatalogUrl(url); + CatalogFetchInfo base_info; + base_info.endpoint = parsed.endpoint; + base_info.region = parsed.region; + base_info.format = parsed.format; + base_info.correlation_id = correlation_id; + + auto model_infos = FetchAllModelInfosWithCachedModels(*client, cached_model_ids, logger_, + telemetry_, &base_info); for (const auto& info : model_infos) { // Check if the model is locally cached and pass the path if so. @@ -111,16 +194,24 @@ std::vector AzureModelCatalog::FetchModels() const { } }; + bool fetched_any_catalog = false; for (const auto& [url, filter] : catalog_urls_) { try { fetch_from(url, filter); + fetched_any_catalog = true; } catch (const std::exception& ex) { // One failing URL shouldn't block others — skip and continue. + auto parsed = ParseCatalogUrl(url); logger_.Log(LogLevel::Error, - fmt::format("failed to fetch catalog from {}: {}", url, ex.what())); + fmt::format("failed to fetch catalog from {}: {}", parsed.endpoint, + ScrubTelemetryErrorMessage(ex.what()))); } } + if (!fetched_any_catalog) { + FL_THROW(FOUNDRY_LOCAL_ERROR_NETWORK, "failed to fetch any configured catalog source"); + } + logger_.Log(LogLevel::Information, fmt::format("Populated model info for {} models.", models.size())); @@ -157,8 +248,10 @@ std::vector AzureModelCatalog::FetchModelVersions( out.push_back(model_factory_(std::move(info), /*local_path=*/"")); } } catch (const std::exception& ex) { + auto parsed = ParseCatalogUrl(url); logger_.Log(LogLevel::Error, - fmt::format("FetchModelVersions: failed to query {} — {}", url, ex.what())); + fmt::format("FetchModelVersions: failed to query {} — {}", parsed.endpoint, + ScrubTelemetryErrorMessage(ex.what()))); } } @@ -213,8 +306,10 @@ std::vector AzureModelCatalog::FetchModelsByIds(const std::vector #include "exception.h" +#include "telemetry/telemetry_redaction.h" #include #include @@ -20,6 +21,11 @@ BaseModelCatalog::BaseModelCatalog(std::string name, ILogger& logger) BaseModelCatalog::~BaseModelCatalog() = default; void BaseModelCatalog::PopulateModels(std::vector variants) const { + if (populated_) { + IntegrateVariantsLocked(std::move(variants)); + return; + } + // Group variants by alias into Model containers. // Matches C# Catalog.UpdateModels() pattern: // foreach (modelInfo) { find or create Model by alias, add variant } @@ -47,50 +53,25 @@ void BaseModelCatalog::PopulateModels(std::vector variants) const { model.SelectDefaultVariant(); } - // On refresh: merge new models into stable storage. Existing models keep their addresses. - // New aliases are appended. Existing aliases are left unchanged (their Model* stays valid). - if (populated_) { - // Build a set of existing aliases for fast lookup. - std::unordered_map existing_aliases; - for (auto& m : models_) { - existing_aliases[m->Alias()] = m.get(); - } - - size_t new_count = 0; - for (auto& [alias, model] : alias_to_model) { - if (!existing_aliases.contains(alias)) { - models_.push_back(std::make_unique(std::move(model))); - ++new_count; - } - } - - if (new_count > 0) { - logger_.Log(LogLevel::Information, - fmt::format("Catalog '{}' refresh: {} new model(s) added, {} total.", - name_, new_count, models_.size())); - } else { - logger_.Log(LogLevel::Debug, - fmt::format("Catalog '{}' refresh: no new models. {} total.", - name_, models_.size())); - } - } else { - // Initial population: move all models into stable storage. - models_.reserve(alias_to_model.size()); - for (auto& [alias, model] : alias_to_model) { - models_.push_back(std::make_unique(std::move(model))); - } - - logger_.Log(LogLevel::Debug, - fmt::format("Catalog '{}' populated with {} model(s).", name_, models_.size())); + // Initial population: move all models into stable storage. + models_.reserve(alias_to_model.size()); + for (auto& [alias, model] : alias_to_model) { + models_.push_back(std::make_unique(std::move(model))); } + logger_.Log(LogLevel::Debug, + fmt::format("Catalog '{}' populated with {} model(s).", name_, models_.size())); + RebuildIndex(); populated_ = true; } void BaseModelCatalog::IntegrateVariants(std::vector variants) const { std::lock_guard lock(mutex_); + IntegrateVariantsLocked(std::move(variants)); +} +void BaseModelCatalog::IntegrateVariantsLocked(std::vector variants) const { if (variants.empty()) { return; } @@ -141,6 +122,9 @@ void BaseModelCatalog::IntegrateVariants(std::vector variants) const { it->second->AddVariant(std::move(v)); ++added_variants; } + if (!it->second->HasExplicitVariantSelection()) { + it->second->SelectDefaultVariant(); + } } else { // New alias: build a container and choose default after all variants are added. auto first = std::move(alias_variants.front()); @@ -214,6 +198,7 @@ void BaseModelCatalog::InvalidateCache() { // This is called after EP registration changes — the catalog needs to // re-query with updated device/EP filters. std::lock_guard lock(mutex_); + force_refresh_ = true; next_refresh_at_ = std::chrono::steady_clock::time_point{}; } @@ -224,8 +209,9 @@ void BaseModelCatalog::EnsurePopulated(bool allow_refresh) const { // not worth the complexity to optimise.) std::lock_guard lock(mutex_); - bool needs_refresh = allow_refresh && - std::chrono::steady_clock::now() >= next_refresh_at_; + auto now = std::chrono::steady_clock::now(); + bool needs_refresh = (force_refresh_ && now >= next_refresh_at_) || + (allow_refresh && now >= next_refresh_at_); if (populated_ && !needs_refresh) { return; @@ -236,8 +222,21 @@ void BaseModelCatalog::EnsurePopulated(bool allow_refresh) const { fmt::format("Catalog '{}' refreshing (cache expired).", name_)); } - auto variants = FetchModels(); + std::vector variants; + try { + variants = FetchModels(); + } catch (const std::exception& ex) { + if (populated_) { + logger_.Log(LogLevel::Warning, + fmt::format("Catalog '{}' refresh failed; keeping existing catalog: {}", + name_, ScrubTelemetryErrorMessage(ex.what()))); + next_refresh_at_ = std::chrono::steady_clock::now() + std::chrono::minutes(1); + return; + } + throw; + } PopulateModels(std::move(variants)); + force_refresh_ = false; next_refresh_at_ = std::chrono::steady_clock::now() + kCacheDuration; } @@ -333,8 +332,10 @@ Model* BaseModelCatalog::GetLatestVersion(const Model* model) const { auto alias_it = idx->alias_index.find(std::string(model->Info().alias)); if (alias_it != idx->alias_index.end()) { auto variants = alias_it->second->Variants(); - if (!variants.empty()) { - return variants.front(); + for (auto* variant : variants) { + if (variant != nullptr && variant->Info().name == model->Info().name) { + return variant; + } } } diff --git a/sdk_v2/cpp/src/catalog/base_model_catalog.h b/sdk_v2/cpp/src/catalog/base_model_catalog.h index 97411e395..d7adb5d80 100644 --- a/sdk_v2/cpp/src/catalog/base_model_catalog.h +++ b/sdk_v2/cpp/src/catalog/base_model_catalog.h @@ -98,6 +98,7 @@ class BaseModelCatalog : public ICatalog { std::shared_ptr GetIndex() const; mutable bool populated_ = false; + mutable bool force_refresh_ = false; mutable std::mutex mutex_; mutable std::chrono::steady_clock::time_point next_refresh_at_{}; @@ -111,6 +112,7 @@ class BaseModelCatalog : public ICatalog { /// present. For new aliases, creates a new container. Rebuilds the lookup /// index when the model set actually changed. void IntegrateVariants(std::vector variants) const; + void IntegrateVariantsLocked(std::vector variants) const; /// Build lookup indices from the current models_ collection. /// Builds a complete new ModelIndex locally, then atomically swaps it into index_. diff --git a/sdk_v2/cpp/src/catalog/catalog_client.cc b/sdk_v2/cpp/src/catalog/catalog_client.cc index 6f0a51e38..65fc2e05c 100644 --- a/sdk_v2/cpp/src/catalog/catalog_client.cc +++ b/sdk_v2/cpp/src/catalog/catalog_client.cc @@ -1,22 +1,66 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. #include "catalog/catalog_client.h" +#include "ep_detection/ep_detector.h" +#include "telemetry/telemetry.h" +#include "telemetry/telemetry_redaction.h" #include "utils.h" #include #include +#include +#include #include namespace fl { +namespace { + +constexpr const char* kCatalogFetchFailure = "catalog request failed"; + +} // namespace + std::vector FetchAllModelInfosWithCachedModels( ICatalogClient& client, const std::vector& cached_model_ids, - ILogger& logger) { - // Step 1: Fetch latest catalog models (existing flow). - auto result = client.FetchAllModelInfos(); + ILogger& logger, + ITelemetry* telemetry, + const CatalogFetchInfo* base_info) { + auto now = [] { return std::chrono::steady_clock::now(); }; + auto elapsed_ms = [](std::chrono::steady_clock::time_point start) { + return std::chrono::duration_cast(std::chrono::steady_clock::now() - start) + .count(); + }; + auto emit = [&](const std::string& operation, ActionStatus status, int64_t duration_ms, + int32_t model_count, const std::string& error) { + if (telemetry == nullptr || base_info == nullptr) { + return; + } + CatalogFetchInfo info = *base_info; + info.operation = operation; + info.status = status; + info.duration_ms = duration_ms; + info.model_count = model_count; + info.error_message = error; + telemetry->RecordCatalogFetch(info); + }; + + // Step 1: Fetch latest catalog models (the primary catalog access). + std::vector result; + { + const auto start = now(); + try { + result = client.FetchAllModelInfos(); + } catch (const std::exception& ex) { + logger.Log(LogLevel::Warning, + fmt::format("catalog: failed to fetch models — {}", ScrubTelemetryErrorMessage(ex.what()))); + emit("FetchAll", ActionStatus::kFailure, elapsed_ms(start), 0, kCatalogFetchFailure); + throw; + } + emit("FetchAll", ActionStatus::kSuccess, elapsed_ms(start), static_cast(result.size()), ""); + } if (cached_model_ids.empty()) { return result; @@ -37,17 +81,23 @@ std::vector FetchAllModelInfosWithCachedModels( // Step 3: Look up unresolved IDs from the catalog (older versions, etc.) if (!unresolved_ids.empty()) { + const auto start = now(); try { auto additional = client.FetchModelsByIds(unresolved_ids); + const auto additional_count = static_cast(additional.size()); for (auto& info : additional) { resolved_ids.insert(info.model_id); result.push_back(std::move(info)); } + emit("FetchByIds", ActionStatus::kSuccess, elapsed_ms(start), additional_count, ""); } catch (const std::exception& ex) { + auto error_message = ScrubTelemetryErrorMessage(ex.what()); logger.Log(LogLevel::Warning, - fmt::format("catalog: failed to fetch cached model IDs — {}", ex.what())); + fmt::format("catalog: failed to fetch cached model IDs — {}", error_message)); + emit("FetchByIds", ActionStatus::kFailure, elapsed_ms(start), 0, kCatalogFetchFailure); } catch (...) { logger.Log(LogLevel::Warning, "catalog: failed to fetch cached model IDs — unknown error"); + emit("FetchByIds", ActionStatus::kFailure, elapsed_ms(start), 0, "unknown error"); } // Step 4: Create basic entries for any IDs still unresolved (BYO models). diff --git a/sdk_v2/cpp/src/catalog/catalog_client.h b/sdk_v2/cpp/src/catalog/catalog_client.h index e3afcfe78..af84e8453 100644 --- a/sdk_v2/cpp/src/catalog/catalog_client.h +++ b/sdk_v2/cpp/src/catalog/catalog_client.h @@ -12,6 +12,9 @@ namespace fl { +class ITelemetry; // forward declaration +struct CatalogFetchInfo; // forward declaration + /// Abstract catalog client. Implemented by the live Azure catalog client, /// which queries the Azure Foundry catalog REST API. class ICatalogClient { @@ -50,11 +53,16 @@ class ICatalogClient { }; /// Production helper that combines a catalog fetch with locally cached model -/// resolution and BYO synthesis. +/// resolution and BYO synthesis. When `telemetry` and `base_info` are provided, +/// emits a CatalogFetch event for the primary fetch and (if it runs) the +/// cached-id lookup, copying the endpoint/region/format/correlation fields from +/// `base_info` and filling in operation/status/duration/model_count/error. std::vector FetchAllModelInfosWithCachedModels( ICatalogClient& client, const std::vector& cached_model_ids, - ILogger& logger); + ILogger& logger, + ITelemetry* telemetry = nullptr, + const CatalogFetchInfo* base_info = nullptr); /// Construct a client for the live Azure Foundry catalog. /// - `ep_detector` limits results to models supported by this machine. diff --git a/sdk_v2/cpp/src/download/blob_download_state.cc b/sdk_v2/cpp/src/download/blob_download_state.cc index 07680bd73..b76ff7180 100644 --- a/sdk_v2/cpp/src/download/blob_download_state.cc +++ b/sdk_v2/cpp/src/download/blob_download_state.cc @@ -21,7 +21,7 @@ constexpr const char* kStateFileExtension = ".dlstate"; // bytes | field // -------|-------------------------------------------------------- // 0..3 | magic "FLDS" -// 4 | version (currently 1) +// 4 | version (currently 2) // 5..12 | blob_size (int64) // 13..16 | chunk_size (int32) // 17..20 | total_chunks (int32) @@ -29,14 +29,17 @@ constexpr const char* kStateFileExtension = ".dlstate"; // 25..28 | highest_completed_chunk (int32) // 29..32 | completed_count (int32) // 33..40 | last_modified_unix_ms (int64) -// 41..44 | trunc_bitmap_byte_len (uint32) -// 45.. | trunc_bitmap_byte_len bytes of bitmap data, copied directly out of +// 41..44 | blob_identity_len (uint32) +// 45.. | blob_identity bytes +// ... | trunc_bitmap_byte_len (uint32) +// ... | trunc_bitmap_byte_len bytes of bitmap data, copied directly out of // full_completion_bitmap starting at the byte offset implied by // bitmap_byte_aligned_start. constexpr char kMagic[4] = {'F', 'L', 'D', 'S'}; -constexpr uint8_t kVersion = 1; +constexpr uint8_t kVersion = 2; constexpr int32_t kBitsPerWord = 64; +constexpr uint32_t kMaxBlobIdentityLength = 4096; // Serialize a scalar field in host byte order. Every target we build for // (x64 / arm64) is little-endian, so the on-disk layout is little-endian in @@ -54,6 +57,28 @@ bool ReadNative(std::istream& in, T& out_value) { return static_cast(in); } +void WriteString(std::ostream& out, const std::string& value) { + WriteNative(out, static_cast(value.size())); + if (!value.empty()) { + out.write(value.data(), static_cast(value.size())); + } +} + +bool ReadString(std::istream& in, std::string& value) { + uint32_t size = 0; + if (!ReadNative(in, size)) { + return false; + } + if (size > kMaxBlobIdentityLength) { + return false; + } + value.resize(size); + if (size > 0) { + in.read(value.data(), size); + } + return static_cast(in); +} + int64_t NowUnixMs() { return std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) @@ -72,13 +97,15 @@ std::unique_ptr BlobDownloadState::CreateNew(std::string blob const std::filesystem::path& local_file_path, int64_t blob_size, int32_t chunk_size, - int32_t total_chunks) { + int32_t total_chunks, + std::string blob_identity) { auto state = std::make_unique(); state->blob_name = std::move(blob_name); state->local_file_path = local_file_path.string(); state->blob_size = blob_size; state->chunk_size = chunk_size; state->total_chunks = total_chunks; + state->blob_identity = std::move(blob_identity); state->bitmap_byte_aligned_start = 0; state->highest_completed_chunk = -1; state->completed_count = 0; @@ -93,6 +120,7 @@ std::unique_ptr BlobDownloadState::LoadState(std::string blob int64_t expected_blob_size, int32_t expected_chunk_size, int32_t expected_total_chunks, + std::string_view expected_blob_identity, ILogger& logger) { auto state_path = GetStateFilePath(local_file_path); std::error_code ec; @@ -122,17 +150,20 @@ std::unique_ptr BlobDownloadState::LoadState(std::string blob int32_t highest_completed_chunk = 0; int32_t completed_count = 0; int64_t last_modified_unix_ms = 0; + std::string blob_identity; uint32_t trunc_len = 0; if (!ReadNative(in, blob_size) || !ReadNative(in, chunk_size) || !ReadNative(in, total_chunks) || !ReadNative(in, bitmap_byte_aligned_start) || !ReadNative(in, highest_completed_chunk) || - !ReadNative(in, completed_count) || !ReadNative(in, last_modified_unix_ms) || !ReadNative(in, trunc_len)) { + !ReadNative(in, completed_count) || !ReadNative(in, last_modified_unix_ms) || + !ReadString(in, blob_identity) || !ReadNative(in, trunc_len)) { logger.Log(LogLevel::Warning, "Download state header truncated: " + state_path.string()); return nullptr; } // Sanity / compatibility checks. if (blob_size != expected_blob_size || chunk_size != expected_chunk_size || - total_chunks != expected_total_chunks) { + total_chunks != expected_total_chunks || + (!expected_blob_identity.empty() && blob_identity != expected_blob_identity)) { logger.Log(LogLevel::Information, "Download state for " + state_path.string() + " is incompatible with current blob layout; starting fresh"); @@ -191,6 +222,7 @@ std::unique_ptr BlobDownloadState::LoadState(std::string blob state->highest_completed_chunk = highest_completed_chunk; state->completed_count = completed_count; state->last_modified_unix_ms = last_modified_unix_ms; + state->blob_identity = std::move(blob_identity); state->full_completion_bitmap = std::move(bitmap); logger.Log(LogLevel::Information, @@ -200,6 +232,16 @@ std::unique_ptr BlobDownloadState::LoadState(std::string blob return state; } +std::unique_ptr BlobDownloadState::LoadState(std::string blob_name, + const std::filesystem::path& local_file_path, + int64_t expected_blob_size, + int32_t expected_chunk_size, + int32_t expected_total_chunks, + ILogger& logger) { + return LoadState(std::move(blob_name), local_file_path, expected_blob_size, expected_chunk_size, + expected_total_chunks, {}, logger); +} + int64_t BlobDownloadState::CalculateDownloadedSize() const noexcept { int64_t bytes = static_cast(completed_count) * chunk_size; // If the final chunk is partial and was completed, adjust the overcount. @@ -321,6 +363,7 @@ bool BlobDownloadState::SaveState(ILogger& logger) { WriteNative(out, highest_completed_chunk); WriteNative(out, completed_count); WriteNative(out, last_modified_unix_ms); + WriteString(out, blob_identity); WriteNative(out, trunc_len); if (trunc_len > 0) { auto* src = reinterpret_cast(full_completion_bitmap.data()) + byte_offset; diff --git a/sdk_v2/cpp/src/download/blob_download_state.h b/sdk_v2/cpp/src/download/blob_download_state.h index 54f114340..acd43be99 100644 --- a/sdk_v2/cpp/src/download/blob_download_state.h +++ b/sdk_v2/cpp/src/download/blob_download_state.h @@ -7,6 +7,7 @@ #include #include #include +#include #include namespace fl { @@ -35,6 +36,7 @@ class BlobDownloadState { int64_t blob_size = 0; int32_t chunk_size = 0; int32_t total_chunks = 0; + std::string blob_identity; /// Serialization marker (always a multiple of 8): chunks below this index are /// complete and dropped from the sidecar's truncated bitmap. The in-memory @@ -64,7 +66,8 @@ class BlobDownloadState { const std::filesystem::path& local_file_path, int64_t blob_size, int32_t chunk_size, - int32_t total_chunks); + int32_t total_chunks, + std::string blob_identity = {}); /// Load existing state from `.dlstate`. Returns nullptr if /// the file does not exist, is corrupted, or has incompatible @@ -73,6 +76,13 @@ class BlobDownloadState { /// and the partial download is no longer valid). /// `logger` receives diagnostics for corrupt/incompatible state files. Required: the /// downloader always has a logger, so there is no optional/null case to handle. + static std::unique_ptr LoadState(std::string blob_name, + const std::filesystem::path& local_file_path, + int64_t expected_blob_size, + int32_t expected_chunk_size, + int32_t expected_total_chunks, + std::string_view expected_blob_identity, + ILogger& logger); static std::unique_ptr LoadState(std::string blob_name, const std::filesystem::path& local_file_path, int64_t expected_blob_size, diff --git a/sdk_v2/cpp/src/download/blob_downloader.cc b/sdk_v2/cpp/src/download/blob_downloader.cc index c6a5e701a..d04539d28 100644 --- a/sdk_v2/cpp/src/download/blob_downloader.cc +++ b/sdk_v2/cpp/src/download/blob_downloader.cc @@ -49,6 +49,7 @@ constexpr size_t kStreamingBufferBytes = 64 * 1024; struct AzureBlobDownloader::ChunkContext { const Azure::Storage::Blobs::BlobClient& blob_client; const Azure::Core::Context& azure_ctx; + std::string blob_identity; }; AzureBlobDownloader::AzureBlobDownloader(ILogger& logger) : logger_(logger) {} @@ -63,6 +64,11 @@ std::vector AzureBlobDownloader::ListBlobs(const std::string& sas_ BlobItemInfo info; info.name = blob.Name; info.content_length = blob.BlobSize; + if (blob.Details.ETag.HasValue()) { + info.blob_identity = blob.Details.ETag.ToString(); + } else if (blob.VersionId.HasValue()) { + info.blob_identity = blob.VersionId.Value(); + } items.push_back(std::move(info)); } } @@ -74,9 +80,14 @@ std::vector AzureBlobDownloader::ListBlobs(const std::string& sas_ } } -int64_t AzureBlobDownloader::GetBlobSize(ChunkContext& ctx) { +AzureBlobDownloader::BlobProperties AzureBlobDownloader::GetBlobProperties(ChunkContext& ctx) { auto props = ctx.blob_client.GetProperties({}, ctx.azure_ctx).Value; - return props.BlobSize; + BlobProperties result; + result.content_length = props.BlobSize; + if (props.ETag.HasValue()) { + result.blob_identity = props.ETag.ToString(); + } + return result; } bool AzureBlobDownloader::IsCancellationRequested(const ChunkContext& ctx) const { @@ -88,6 +99,9 @@ void AzureBlobDownloader::DownloadChunkStreaming( const std::function& sink) { Azure::Storage::Blobs::DownloadBlobOptions range_opts; range_opts.Range = Azure::Core::Http::HttpRange{offset, size}; + if (!ctx.blob_identity.empty()) { + range_opts.AccessConditions.IfMatch = Azure::ETag(ctx.blob_identity); + } auto result = ctx.blob_client.Download(range_opts, ctx.azure_ctx); auto& body_stream = *result.Value.BodyStream; @@ -155,9 +169,11 @@ void AzureBlobDownloader::DownloadBlob(const std::string& sas_uri, // or by external cancellation; checked by workers between iterations. std::atomic internal_cancel{false}; - ChunkContext chunk_ctx{blob_client, azure_ctx}; + ChunkContext chunk_ctx{blob_client, azure_ctx, ""}; - int64_t blob_size = GetBlobSize(chunk_ctx); + auto blob_properties = GetBlobProperties(chunk_ctx); + int64_t blob_size = blob_properties.content_length; + chunk_ctx.blob_identity = std::move(blob_properties.blob_identity); if (blob_size == 0) { EnsureEmptyBlobFile(local_path); @@ -172,7 +188,7 @@ void AzureBlobDownloader::DownloadBlob(const std::string& sas_uri, // Resume from existing sidecar if it matches the current blob layout. auto state = BlobDownloadState::LoadState(blob_name, local_path, blob_size, static_cast(kChunkSize), - num_chunks, logger_); + num_chunks, chunk_ctx.blob_identity, logger_); if (state) { // Only trust the sidecar if the data file it describes is actually on disk // at full size. If the data file was truncated or removed (e.g. an external @@ -192,7 +208,8 @@ void AzureBlobDownloader::DownloadBlob(const std::string& sas_uri, if (!state) { state = BlobDownloadState::CreateNew(blob_name, local_path, blob_size, - static_cast(kChunkSize), num_chunks); + static_cast(kChunkSize), num_chunks, + chunk_ctx.blob_identity); // Persist the sidecar now, before Open() pre-allocates the data file. // IsDownloadNeeded treats "data file at full size + no sidecar" as a // completed download and skips it. The periodic save below does not run @@ -209,6 +226,28 @@ void AzureBlobDownloader::DownloadBlob(const std::string& sas_uri, } } + auto pending = state->GetPendingChunks(); + if (pending.empty()) { + // A complete sidecar means a previous attempt finished all chunks but did not reach finalization. Do not trust + // that state: a late close error may have left a full-size corrupt file. Start a fresh all-chunks pass. + logger_.Log(LogLevel::Information, + "Resume sidecar for '" + local_path + "' is complete but unfinalized; starting fresh"); + std::error_code remove_ec; + std::filesystem::remove(local_path, remove_ec); + if (remove_ec && std::filesystem::exists(local_path)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, + "failed to remove suspect completed blob before restart: " + local_path); + } + state = BlobDownloadState::CreateNew(blob_name, local_path, blob_size, + static_cast(kChunkSize), num_chunks, + chunk_ctx.blob_identity); + if (!state->SaveState(logger_)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, + "failed to persist reset download state for '" + local_path + "'"); + } + pending = state->GetPendingChunks(); + } + // Track cumulative bytes for progress reporting; seed with bytes already // present on disk so percent stays monotonic across resume. std::atomic bytes_completed{state->CalculateDownloadedSize()}; @@ -216,16 +255,6 @@ void AzureBlobDownloader::DownloadBlob(const std::string& sas_uri, bytes_written_cb(bytes_completed.load()); } - auto pending = state->GetPendingChunks(); - if (pending.empty()) { - // Already complete on disk — drop the sidecar. - BlobDownloadState::DeleteState(local_path, logger_); - if (bytes_written_cb) { - bytes_written_cb(blob_size); - } - return; - } - // Open the file writer once for the whole download. Open() pre-allocates // the file to blob_size if needed, preserving any existing bytes from a // resume. Concurrent WriteAt calls to disjoint ranges are thread-safe — the @@ -360,7 +389,54 @@ void AzureBlobDownloader::DownloadBlob(const std::string& sas_uri, // Release the OS handle before persisting / deleting the sidecar so any // observer that watches the data file sees a fully-closed handle. - writer.Close(); + try { + writer.Close(); + } catch (...) { + bool safe_to_delete_state = false; + std::error_code remove_ec; + if (std::filesystem::remove(local_path, remove_ec) || !std::filesystem::exists(local_path)) { + safe_to_delete_state = true; + } else { + logger_.Log(LogLevel::Warning, + "failed to remove blob file after close failure: " + local_path + + " (" + remove_ec.message() + ")"); + + const std::filesystem::path invalid_path = local_path + ".invalid"; + std::error_code cleanup_ec; + std::filesystem::remove(invalid_path, cleanup_ec); + std::error_code rename_ec; + std::filesystem::rename(local_path, invalid_path, rename_ec); + if (!rename_ec) { + safe_to_delete_state = true; + } else { + logger_.Log(LogLevel::Warning, + "failed to quarantine blob file after close failure: " + local_path + + " (" + rename_ec.message() + ")"); + + std::error_code resize_ec; + std::filesystem::resize_file(local_path, 0, resize_ec); + if (!resize_ec) { + safe_to_delete_state = true; + } else { + logger_.Log(LogLevel::Warning, + "failed to truncate blob file after close failure: " + local_path + + " (" + resize_ec.message() + ")"); + auto fresh_state = BlobDownloadState::CreateNew(blob_name, local_path, blob_size, + static_cast(kChunkSize), num_chunks, + chunk_ctx.blob_identity); + if (!fresh_state->SaveState(logger_)) { + logger_.Log(LogLevel::Error, + "failed to reset download state after close failure for '" + local_path + "'"); + } + } + } + } + + if (safe_to_delete_state) { + BlobDownloadState::DeleteState(local_path, logger_); + } + throw; + } const bool was_cancelled = cancelled && cancelled->load(std::memory_order_relaxed); if (first_error || was_cancelled) { @@ -443,7 +519,11 @@ bool IsDownloadNeeded(const BlobItemInfo& blob, const std::string& local_path) { void DownloadBlobsToDirectory(IBlobDownloader& downloader, const std::string& sas_uri, const std::string& output_directory, - const BlobDownloadOptions& options) { + const BlobDownloadOptions& options, + BlobDownloadStats* stats) { + using clock = std::chrono::steady_clock; + auto enum_start = clock::now(); + // Step 1: Enumerate all blobs auto all_blobs = downloader.ListBlobs(sas_uri); @@ -486,6 +566,11 @@ void DownloadBlobsToDirectory(IBlobDownloader& downloader, blobs_to_download.end()); if (blobs_to_download.empty()) { + if (stats != nullptr) { + stats->enumeration_ms = std::chrono::duration_cast( + clock::now() - enum_start) + .count(); + } return; } @@ -502,21 +587,37 @@ void DownloadBlobsToDirectory(IBlobDownloader& downloader, total_size += blob.content_length; } + if (stats != nullptr) { + stats->file_count = static_cast(blobs_to_download.size()); + stats->total_size_bytes = total_size; + stats->enumeration_ms = std::chrono::duration_cast( + clock::now() - enum_start) + .count(); + } + auto download_start = clock::now(); + // Step 5: Skip blobs already present at the expected size. Their bytes // count toward "downloaded" so the percentage stays accurate when this is a // resume of a partially-completed download. int64_t skipped_bytes = 0; + int32_t skipped_file_count = 0; blobs_to_download.erase( std::remove_if(blobs_to_download.begin(), blobs_to_download.end(), - [&skipped_bytes](const auto& pair) { - if (IsDownloadNeeded(pair.first, pair.second)) { - return false; - } - skipped_bytes += pair.first.content_length; - return true; + [&skipped_bytes, &skipped_file_count, &options](const auto& pair) { + if (!options.skip_completed_files || IsDownloadNeeded(pair.first, pair.second)) { + return false; + } + skipped_bytes += pair.first.content_length; + ++skipped_file_count; + return true; }), blobs_to_download.end()); + if (stats != nullptr) { + stats->already_cached_bytes = skipped_bytes; + stats->skipped_file_count = skipped_file_count; + } + // Step 6: Emit initial progress reflecting any already-on-disk bytes. // If everything was skipped, emit 100% directly and return. if (blobs_to_download.empty()) { @@ -596,6 +697,12 @@ void DownloadBlobsToDirectory(IBlobDownloader& downloader, if (options.progress) { options.progress(100.0f); } + + if (stats != nullptr) { + stats->download_ms = std::chrono::duration_cast( + clock::now() - download_start) + .count(); + } } } // namespace fl diff --git a/sdk_v2/cpp/src/download/blob_downloader.h b/sdk_v2/cpp/src/download/blob_downloader.h index 5d6849a6e..3e3c671a7 100644 --- a/sdk_v2/cpp/src/download/blob_downloader.h +++ b/sdk_v2/cpp/src/download/blob_downloader.h @@ -28,6 +28,10 @@ struct BlobDownloadOptions { /// Maximum concurrent chunk downloads per blob. Default matches C# desktop. int max_concurrency = 64; + /// When false, full-size files without a sidecar are redownloaded instead of + /// skipped. Use this while recovering an incomplete model directory. + bool skip_completed_files = false; + /// Progress callback (optional). Return non-zero to cancel the download. DownloadProgressFn progress; }; @@ -36,6 +40,7 @@ struct BlobDownloadOptions { struct BlobItemInfo { std::string name; int64_t content_length = 0; + std::string blob_identity{}; }; /// Interface for blob download operations. Enables testing via mock implementations. @@ -90,8 +95,14 @@ class AzureBlobDownloader : public IBlobDownloader { /// SDK BlobClient + Context pointers used by the production virtuals. struct ChunkContext; - /// Return the blob size in bytes. Production calls `BlobClient::GetProperties`. - virtual int64_t GetBlobSize(ChunkContext& ctx); + struct BlobProperties { + int64_t content_length = 0; + std::string blob_identity{}; + }; + + /// Return the blob size and identity from one properties read so chunk planning + /// and conditional range reads refer to the same remote blob version. + virtual BlobProperties GetBlobProperties(ChunkContext& ctx); /// Read `size` bytes starting at `offset` from the blob and forward them /// piecewise to `sink`. Pulls from the blob client referenced by `ctx`. @@ -121,9 +132,24 @@ class AzureBlobDownloader : public IBlobDownloader { /// High-level download function: enumerate, filter, and download all blobs from a SAS URI. /// Handles safetensors optimization, path prefix filtering, and progress reporting. /// Throws fl::Exception on failure. +/// @param stats When non-null, populated with byte/file counts and per-phase timings useful +/// for telemetry. Always populated when the function returns normally; partially +/// populated when it throws (use values only after a clean return). void DownloadBlobsToDirectory(IBlobDownloader& downloader, const std::string& sas_uri, const std::string& output_directory, - const BlobDownloadOptions& options); + const BlobDownloadOptions& options, + struct BlobDownloadStats* stats = nullptr); + +/// Aggregate statistics from one DownloadBlobsToDirectory call. Used by callers to +/// stamp telemetry events with download-shape data (file count, byte volume, timing). +struct BlobDownloadStats { + int64_t total_size_bytes = 0; + int64_t already_cached_bytes = 0; + int32_t file_count = 0; + int32_t skipped_file_count = 0; + int64_t enumeration_ms = 0; // Time spent listing blobs from the SAS URI. + int64_t download_ms = 0; // Time spent transferring blob bytes (excludes enumeration). +}; } // namespace fl diff --git a/sdk_v2/cpp/src/download/download_manager.cc b/sdk_v2/cpp/src/download/download_manager.cc index f7219e7e8..ab56e9238 100644 --- a/sdk_v2/cpp/src/download/download_manager.cc +++ b/sdk_v2/cpp/src/download/download_manager.cc @@ -6,6 +6,8 @@ #include "exception.h" #include "log_level.h" #include "logger.h" +#include "telemetry/download_tracker.h" +#include "telemetry/telemetry.h" #include "util/path_safety.h" #include "util/region_fallback.h" #include "utils.h" @@ -13,6 +15,7 @@ #include #include +#include #include #include #include @@ -25,6 +28,8 @@ namespace fl { namespace { const char* kDownloadSignalFileName = "download.tmp"; +const char* kDownloadSignalSidecarSafeMarker = "foundry-local-sidecar-safe-v1\n"; +const char* kDownloadSignalLegacyRepairMarker = "foundry-local-legacy-repair\n"; const char* kGenAIConfigFileName = "genai_config.json"; const char* kInferenceModelFileName = "inference_model.json"; const char* kDefaultRegistryRegion = "centralus"; @@ -42,7 +47,6 @@ bool HasInferenceModelJson(const std::string& model_path) { if (ec) { return false; } - for (const auto& entry : it) { if (entry.is_directory(ec)) { if (std::filesystem::exists(entry.path() / kInferenceModelFileName)) { @@ -54,6 +58,17 @@ bool HasInferenceModelJson(const std::string& model_path) { return false; } +bool HasSidecarSafeDownloadSignal(const std::filesystem::path& signal_path) { + std::ifstream signal(signal_path); + if (!signal.is_open()) { + return false; + } + + std::string marker; + std::getline(signal, marker); + return marker == "foundry-local-sidecar-safe-v1"; +} + /// Resolve the effective model path — the directory containing genai_config.json. /// For single-variant models this is model_path itself. /// For multi-variant models it's the first subdirectory containing genai_config.json. @@ -177,14 +192,15 @@ std::string ResolveRegion(const std::string& config_region, const ModelInfo& inf } // anonymous namespace DownloadManager::DownloadManager(std::string cache_directory, std::string_view catalog_region, int max_concurrency, - ILogger& logger, bool disable_region_fallback) + ILogger& logger, bool disable_region_fallback, ITelemetry* telemetry) : cache_directory_(std::move(cache_directory)), config_region_(NormalizeConfiguredRegion(catalog_region)), max_concurrency_(max_concurrency), logger_(logger), registry_client_(std::make_unique( kDefaultRegistryRegion, logger, std::make_unique(logger, !disable_region_fallback))), - blob_downloader_(std::make_unique(logger)) {} + blob_downloader_(std::make_unique(logger)), + telemetry_(telemetry) {} DownloadManager::~DownloadManager() = default; @@ -218,7 +234,7 @@ std::string DownloadManager::ComputeModelPath(const ModelInfo& info) const { auto version = std::string_view(info.model_id).substr(last_colon + 1); SanitizeForPathSegment(bare_id); SanitizeForPathSegment(version); - sanitized_model_dir = FixVersionSuffix(info.model_id); + sanitized_model_dir = SanitizeForPathSegment(FixVersionSuffix(info.model_id)); } std::filesystem::path full_path(cache_directory_); @@ -238,18 +254,38 @@ std::string DownloadManager::ComputeModelPath(const ModelInfo& info) const { } std::string DownloadManager::DownloadModel(const ModelInfo& info, - std::function progress_cb) { + std::function progress_cb, + const std::string& user_agent) { + using clock = std::chrono::steady_clock; + auto lock_wait_start = clock::now(); + // Serialize all model downloads in this process: only one runs at a time, so it // gets the full network and disk instead of competing with another download. // The cross-process file lock taken below extends the guarantee across every // process and app that shares this cache directory. std::unique_lock download_guard(download_mutex_); + + int64_t lock_wait_ms = std::chrono::duration_cast( + clock::now() - lock_wait_start) + .count(); + + // RAII telemetry tracker — emits a "Download" event on destruction with whatever + // fields have been populated. Default status is kFailure so abrupt exits (exceptions) + // are recorded as failures; the happy path explicitly sets kSuccess / kSkipped. + std::unique_ptr tracker; + if (telemetry_ != nullptr) { + tracker = std::make_unique(info.model_id, user_agent, *telemetry_); + tracker->SetLockWaitMs(lock_wait_ms); + tracker->SetMaxConcurrency(static_cast(max_concurrency_)); + } + auto model_path = ComputeModelPath(info); // Fast path: serve the cache without taking the cross-process lock. // A valid cache hit requires: directory exists, no in-progress signal file, and // inference_model.json is present (written by DownloadModel on successful completion). auto signal_path = std::filesystem::path(model_path) / kDownloadSignalFileName; + const bool model_dir_existed_before_download = std::filesystem::exists(model_path); if (std::filesystem::exists(model_path) && !std::filesystem::exists(signal_path) && HasInferenceModelJson(model_path)) { // Already cached and download was complete — cancellation request is @@ -258,10 +294,17 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, progress_cb(100.0f); } + if (tracker != nullptr) { + tracker->SetStatus(ActionStatus::kSkipped); + } return ResolveEffectiveModelPath(model_path); } if (info.uri.empty()) { + if (tracker != nullptr) { + auto ex = std::runtime_error("cannot download model: empty URI (asset_id)"); + tracker->RecordException(ex); + } FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "cannot download model: empty URI (asset_id)"); } @@ -312,13 +355,20 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, if (progress_cb) { progress_cb(100.0f); } + if (tracker != nullptr) { + tracker->SetStatus(ActionStatus::kSkipped); + tracker->SetDownloadWaitResult("CompletedByOtherProcess"); + } return ResolveEffectiveModelPath(model_path); } + const bool can_skip_completed_files = + !model_dir_existed_before_download || HasSidecarSafeDownloadSignal(signal_path); + // Create download signal file { std::ofstream signal(signal_path); - // Empty file — its presence indicates download is in progress + signal << (can_skip_completed_files ? kDownloadSignalSidecarSafeMarker : kDownloadSignalLegacyRepairMarker); } // Emit 0% immediately so callers know the download process has started. @@ -342,6 +392,7 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, // and the blob container has all files at the root or in variant subdirectories. download_opts.path_prefix = ""; download_opts.max_concurrency = max_concurrency_; + download_opts.skip_completed_files = can_skip_completed_files; if (progress_cb) { download_opts.progress = [&progress_cb](float percent) { @@ -349,8 +400,23 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, }; } + if (tracker != nullptr) { + tracker->BeginDownloadPhase(); + } + + BlobDownloadStats stats; DownloadBlobsToDirectory(*blob_downloader_, container.blob_sas_uri, - model_path, download_opts); + model_path, download_opts, &stats); + + if (tracker != nullptr) { + tracker->EndDownloadPhase(); + tracker->SetEnumerationMs(stats.enumeration_ms); + tracker->SetFileCount(stats.file_count); + tracker->SetTotalSizeBytes(stats.total_size_bytes); + tracker->SetAlreadyCachedBytes(stats.already_cached_bytes); + tracker->SetSkippedFileCount(stats.skipped_file_count); + tracker->SetDownloadWaitResult("Completed"); + } // Step 3: Write inference_model.json — use model_id (includes version) so the // local model scanner can match it back to catalog entries during startup. @@ -362,8 +428,15 @@ std::string DownloadManager::DownloadModel(const ModelInfo& info, // Step 5: Remove download signal — marks download as complete std::filesystem::remove(signal_path); + if (tracker != nullptr) { + tracker->SetStatus(ActionStatus::kSuccess); + } return ResolveEffectiveModelPath(model_path); - } catch (...) { + } catch (const std::exception& e) { + if (tracker != nullptr) { + tracker->SetDownloadWaitResult("Failed"); + tracker->RecordException(e); + } // Leave the signal file in place so the incomplete download is detected throw; } diff --git a/sdk_v2/cpp/src/download/download_manager.h b/sdk_v2/cpp/src/download/download_manager.h index 7099dcb8a..82ca3bb70 100644 --- a/sdk_v2/cpp/src/download/download_manager.h +++ b/sdk_v2/cpp/src/download/download_manager.h @@ -15,6 +15,7 @@ namespace fl { class ILogger; +class ITelemetry; /// Orchestrates the full model download flow: /// 1. Compute local cache path @@ -32,11 +33,15 @@ class DownloadManager { /// @param logger Logger forwarded to the registry client for retry diagnostics. /// @param disable_region_fallback When true, the registry uses a single region attempt /// with no cross-region fallback. + /// @param telemetry Optional telemetry sink. If non-null, a Download event is emitted + /// per DownloadModel call. nullptr is supported so tests and + /// embedders without telemetry continue to compile and run. DownloadManager(std::string cache_directory, std::string_view catalog_region, int max_concurrency, ILogger& logger, - bool disable_region_fallback = false); + bool disable_region_fallback = false, + ITelemetry* telemetry = nullptr); ~DownloadManager(); /// Override the model registry client (for testing). @@ -47,9 +52,13 @@ class DownloadManager { /// Download a model to the local cache. /// progress_cb reports 0.0 to 100.0 percentage. + /// user_agent identifies the calling API surface for telemetry attribution; pass an + /// empty string when called outside of an HTTP request context. /// Returns the local path where the model was downloaded. /// Throws fl::Exception on failure. - std::string DownloadModel(const ModelInfo& info, std::function progress_cb = nullptr); + std::string DownloadModel(const ModelInfo& info, + std::function progress_cb = nullptr, + const std::string& user_agent = ""); /// Check if a model is cached locally (directory exists and download is complete). bool IsModelCached(const ModelInfo& info) const; @@ -77,6 +86,7 @@ class DownloadManager { ILogger& logger_; std::unique_ptr registry_client_; std::unique_ptr blob_downloader_; + ITelemetry* telemetry_ = nullptr; /// Serializes all model downloads in this process: only one runs at a time, so /// each gets the full network/disk instead of competing with another download. diff --git a/sdk_v2/cpp/src/download/file_writer.cc b/sdk_v2/cpp/src/download/file_writer.cc index cdde85d24..f116eac44 100644 --- a/sdk_v2/cpp/src/download/file_writer.cc +++ b/sdk_v2/cpp/src/download/file_writer.cc @@ -58,7 +58,15 @@ void EnsureFileExistsAtSize(const fs::path& path, int64_t expected_size) { FileWriter::FileWriter(ILogger& logger) : logger_(logger) {} -FileWriter::~FileWriter() { Close(); } +FileWriter::~FileWriter() { + try { + Close(); + } catch (const std::exception& ex) { + logger_.Log(LogLevel::Warning, std::string("FileWriter: close failed in destructor: ") + ex.what()); + } catch (...) { + logger_.Log(LogLevel::Warning, "FileWriter: close failed in destructor with unknown error"); + } +} void FileWriter::Open(const fs::path& path, int64_t expected_size) { EnsureFileExistsAtSize(path, expected_size); @@ -81,7 +89,8 @@ void FileWriter::Close() { if (handle_ != platform::kInvalidFileHandle) { std::string error; if (!platform::CloseFile(handle_, error)) { - logger_.Log(LogLevel::Warning, "FileWriter: " + error); + handle_ = platform::kInvalidFileHandle; + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, "FileWriter " + error); } handle_ = platform::kInvalidFileHandle; } diff --git a/sdk_v2/cpp/src/ep_detection/cuda_ep_bootstrapper.cc b/sdk_v2/cpp/src/ep_detection/cuda_ep_bootstrapper.cc index 7857655c0..b7ea216be 100644 --- a/sdk_v2/cpp/src/ep_detection/cuda_ep_bootstrapper.cc +++ b/sdk_v2/cpp/src/ep_detection/cuda_ep_bootstrapper.cc @@ -8,6 +8,7 @@ #include "utils.h" #include "http/http_download.h" #include "util/zip_extract.h" +#include "util/sha256.h" #include @@ -16,7 +17,10 @@ #include #include #include +#include #include +#include +#include namespace { @@ -28,6 +32,8 @@ constexpr int kMaxInstallAttempts = 5; // CUDA EP package is built against the ONNX Runtime version we link against. constexpr const char* kDownloadUrl = "https://foundrypackages-ffhrdhbxb7gpdreh.b02.azurefd.net/cuda-ep-20260501-062935.zip"; +constexpr const char* kPackageSha256 = + "D0DBCE52D121954F2D86E01E6D3418690A5BBA787028CCEE448E4BCDDD4F01E1"; struct ExpectedBinary { const char* filename; @@ -35,6 +41,17 @@ struct ExpectedBinary { }; constexpr ExpectedBinary kExpectedBinaries[] = { + {"cublas64_12.dll", "9513540E4EC4C51EE9E7304138C2CC255C29A8C181F9E80C38EFA25738BECD99"}, + {"cublasLt64_12.dll", "B199D1FF892A81B7FD3D57BA1781549609B41500B36008FEF326038393AD46C7"}, + {"cudart64_12.dll", "C2C9A9C22A9BCBA90E261825968836787B331038047A26770CFFB7A583C28344"}, + {"cudnn64_9.dll", "0D1D71325EB5E91570AB8BA8E399E07BF717FFD76511B2407229A8F45E0B1305"}, + {"cudnn_adv64_9.dll", "6D66BCE22502C2582A9C0E5398EE8CC38ADDCE2C837EB6DB8786ABC650E48DD8"}, + {"cudnn_engines_precompiled64_9.dll", "B410C3B42921AFC6E668FF994FCE1BF12C5A8A9B1A9445EBEE61958BF49B1E0A"}, + {"cudnn_engines_runtime_compiled64_9.dll", "8E62214495C96B93C6333C084FEC49B43F272B7E1977A12FE62275E9070647EB"}, + {"cudnn_graph64_9.dll", "82F710B01D15D20C311009721C771B76360A4954EBF7B5F4A407B0F96587F568"}, + {"cudnn_heuristic64_9.dll", "50719EEFB6692074096BF83C87E9CD186F7CE5B953201DA33669C1277A61949B"}, + {"cudnn_ops64_9.dll", "49487537744256A3D4365C4792B03BF31130AD1FAEA0A13EAFA219620941D837"}, + {"cufft64_11.dll", "F4FEA9227B14843894AD5436725F9638B172171142C95291FC6AE7A493248221"}, {"onnxruntime_providers_cuda.dll", "DD540FCFECFBC68B4675C9ADF09C2858CF6B054563859D79598AA2524406A76F"}, {"onnxruntime-genai-cuda.dll", "BC953F8E2AAFC6219B2D723B65AB8F1A9426A6B7724D6A01ED756FAE8C3DE6AE"}, }; @@ -43,6 +60,15 @@ constexpr const char* kRegistrationName = "Foundry.CUDA"; constexpr const char* kCudaProviderDll = "onnxruntime_providers_cuda.dll"; constexpr const char* kCudaProviderOverrideEnv = "FOUNDRY_LOCAL_CUDA_EP_LIBRARY"; +std::vector> ExpectedBinaryHashes() { + std::vector> expected; + expected.reserve(std::size(kExpectedBinaries)); + for (const auto& binary : kExpectedBinaries) { + expected.emplace_back(binary.filename, binary.sha256); + } + return expected; +} + } // anonymous namespace namespace fl { @@ -122,10 +148,7 @@ bool CudaEpBootstrapper::DownloadAndRegister(bool force, FileLock lock(lock_path); // Check if package already exists and is valid - if (fl::VerifyEpBinaries(ep_dir, - {{kExpectedBinaries[0].filename, kExpectedBinaries[0].sha256}, - {kExpectedBinaries[1].filename, kExpectedBinaries[1].sha256}}, - "CUDA EP", logger)) { + if (fl::VerifyEpBinaries(ep_dir, ExpectedBinaryHashes(), "CUDA EP", logger)) { logger.Log(LogLevel::Information, "CUDA EP: package already valid, skipping download"); } else { // Clean up any partial install @@ -156,6 +179,11 @@ bool CudaEpBootstrapper::DownloadAndRegister(bool force, return false; } + if (!VerifyEpArchive(zip_path, kPackageSha256, "CUDA EP", logger)) { + logger.Log(LogLevel::Warning, "CUDA EP: downloaded archive verification failed"); + return false; + } + // Extract logger.Log(LogLevel::Information, "CUDA EP: extracting..."); @@ -168,10 +196,7 @@ bool CudaEpBootstrapper::DownloadAndRegister(bool force, std::filesystem::remove(zip_path); // Verify - if (!fl::VerifyEpBinaries(ep_dir, - {{kExpectedBinaries[0].filename, kExpectedBinaries[0].sha256}, - {kExpectedBinaries[1].filename, kExpectedBinaries[1].sha256}}, - "CUDA EP", logger)) { + if (!fl::VerifyEpBinaries(ep_dir, ExpectedBinaryHashes(), "CUDA EP", logger)) { logger.Log(LogLevel::Warning, "CUDA EP: verification failed after download"); return false; } diff --git a/sdk_v2/cpp/src/ep_detection/ep_detector.cc b/sdk_v2/cpp/src/ep_detection/ep_detector.cc index f60e6adf9..befad97ae 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_detector.cc +++ b/sdk_v2/cpp/src/ep_detection/ep_detector.cc @@ -4,36 +4,57 @@ #include "ep_detection/ep_bootstrapper.h" #include "logger.h" +#include "telemetry/ep_download_tracker.h" +#include "telemetry/telemetry.h" #include #include +#include #include namespace fl { EpDetector::EpDetector(const OrtApi& ort_api, OrtEnv& ort_env, std::vector> bootstrappers, - ILogger& logger) + ILogger& logger, + ITelemetry* telemetry) : ort_api_(ort_api), ort_env_(ort_env), bootstrappers_(std::move(bootstrappers)), - logger_(logger) { - // Populate both cache vectors exact-sized from bootstrappers_. After this point - // size and element addresses (including the EpInfo::name string storage backing - // flEpInfo::name) are immutable for the detector's lifetime — only is_registered - // is ever updated, in place, under cache_mutex_. + logger_(logger), + telemetry_(telemetry) { + // Populate the C++ cache once from bootstrappers_. Name string storage is stable + // because cached_eps_ size is fixed for the detector's lifetime. cached_eps_.reserve(bootstrappers_.size()); - cached_eps_c_.reserve(bootstrappers_.size()); for (const auto& bs : bootstrappers_) { cached_eps_.push_back(EpInfo{bs->Name(), bs->IsRegistered()}); - cached_eps_c_.push_back(flEpInfo{ + } + + PublishEpSnapshotLocked(); + PublishCApiSnapshotLocked(); +} + +void EpDetector::PublishEpSnapshotLocked() { + auto& snapshot = cached_eps_snapshots_.emplace_back(cached_eps_); + current_cached_eps_ = &snapshot; +} + +void EpDetector::PublishCApiSnapshotLocked() { + auto& snapshot = cached_eps_c_snapshots_.emplace_back(); + if (current_cached_eps_ == nullptr) { + current_cached_eps_ = &cached_eps_; + } + snapshot.reserve(current_cached_eps_->size()); + for (const auto& ep : *current_cached_eps_) { + snapshot.push_back(flEpInfo{ FOUNDRY_LOCAL_API_VERSION, - cached_eps_.back().name.c_str(), - bs->IsRegistered(), + ep.name.c_str(), + ep.is_registered, }); } + current_cached_eps_c_ = &snapshot; } std::map> EpDetector::GetAvailableDevicesToEPs() const { @@ -105,19 +126,20 @@ std::map> EpDetector::GetAvailableDevicesT } const std::vector& EpDetector::GetDiscoverableEps() const { - // Take the cache lock for strict correctness of the is_registered field reads. - // Vector size and element addresses are immutable after construction; only - // is_registered fields can be mutated (by DownloadAndRegisterEps under the same - // mutex). The lock is released when this function returns, so the snapshot may - // be stale by the time the caller reads individual fields — that is documented - // and acceptable. std::lock_guard lock(cache_mutex_); - return cached_eps_; + if (current_cached_eps_ == nullptr) { + static const std::vector empty; + return empty; + } + return *current_cached_eps_; } std::span EpDetector::GetDiscoverableEpsCApi() const { std::lock_guard lock(cache_mutex_); - return cached_eps_c_; + if (current_cached_eps_c_ == nullptr) { + return {}; + } + return *current_cached_eps_c_; } EpDownloadResult EpDetector::DownloadAndRegisterEps(const std::vector* names, @@ -134,9 +156,24 @@ EpDownloadResult EpDetector::DownloadAndRegisterEps(const std::vector unmatched_requested_names = names ? *names : std::vector{}; if (progress_cb) { wrapped_cb = [&progress_cb, &cancelled](const std::string& ep_name, float percent) -> bool { @@ -166,24 +203,73 @@ EpDownloadResult EpDetector::DownloadAndRegisterEps(const std::vectorend()) { continue; } + unmatched_requested_names.erase(std::remove(unmatched_requested_names.begin(), unmatched_requested_names.end(), + bs->Name()), + unmatched_requested_names.end()); } + ++telemetry_num_providers; logger_.Log(LogLevel::Information, "Downloading and registering EP: " + bs->Name()); + // Per-provider EPDownloadAndRegister event via EpDownloadTracker. When + // telemetry_ is null (e.g. unit tests instantiate EpDetector directly), + // the tracker is skipped — the event has no fallback emitter. + std::unique_ptr tracker; + const bool was_registered_before = bs->IsRegistered(); + if (telemetry_ != nullptr) { + tracker = std::make_unique(bs->Name(), /*user_agent=*/std::string{}, + telemetry_correlation_id, *telemetry_); + tracker->RecordInitialState(was_registered_before ? "Registered" : "NotPresent"); + } + + ++telemetry_attempts; // Reuse previously downloaded EP packages unless the caller explicitly asks // for a forced refresh. Downloading every time made the bootstrapper // re-fetch and re-register EPs on every invocation. - if (bs->DownloadAndRegister(/*force=*/false, wrapped_cb, logger_)) { + bool ok = false; + try { + ok = bs->DownloadAndRegister(/*force=*/false, wrapped_cb, logger_); + } catch (const std::exception& ex) { + if (tracker) { + tracker->RecordException(ex); + } + // Re-throw to preserve existing semantics — the wrapper RAII guard above + // resets download_in_progress_; the tracker dtor records the EP event. + throw; + } + + if (ok) { + ++telemetry_succeeded; + telemetry_resolved = true; result.registered_eps.push_back(bs->Name()); // Update cached registration state in place under the cache lock so // GetDiscoverableEps[C] readers see the new value. std::lock_guard cache_lock(cache_mutex_); + const bool registration_changed = !cached_eps_[i].is_registered; cached_eps_[i].is_registered = true; - cached_eps_c_[i].is_registered = true; + if (registration_changed) { + PublishEpSnapshotLocked(); + PublishCApiSnapshotLocked(); + } + + if (tracker) { + tracker->RecordDownloadComplete(ActionStatus::kSuccess, "Installed"); + tracker->RecordRegisterComplete(ActionStatus::kSuccess, "Registered"); + } } else { + ++telemetry_failed; result.failed_eps.push_back(bs->Name()); result.success = false; + if (tracker) { + // The bootstrapper conflated download + register and returned false. + // Record both phases as kFailure so backend dashboards can tell that + // this EP didn't reach Registered, without claiming a specific phase. + tracker->RecordDownloadComplete(ActionStatus::kFailure, + was_registered_before ? "Registered" : "NotPresent"); + tracker->RecordRegisterComplete(ActionStatus::kFailure, + was_registered_before ? "Registered" : "NotPresent"); + } } } @@ -191,12 +277,35 @@ EpDownloadResult EpDetector::DownloadAndRegisterEps(const std::vector( + std::chrono::steady_clock::now() - attempt_start) + .count(); + telemetry_->RecordEpDownloadAttempt(attempt_info); + } + return result; } diff --git a/sdk_v2/cpp/src/ep_detection/ep_detector.h b/sdk_v2/cpp/src/ep_detection/ep_detector.h index 254f1e43e..673ac4812 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_detector.h +++ b/sdk_v2/cpp/src/ep_detection/ep_detector.h @@ -3,6 +3,7 @@ #pragma once #include +#include #include #include #include @@ -23,6 +24,7 @@ struct OrtEnv; namespace fl { class ILogger; +class ITelemetry; /// Interface for detecting available hardware devices and execution providers. class IEpDetector { @@ -71,9 +73,15 @@ class EpDetector : public IEpDetector { /// @param ort_env The ORT environment singleton. /// @param bootstrappers EP bootstrappers for download/registration. /// @param logger Logger instance. + /// @param telemetry Optional telemetry sink. When non-null, DownloadAndRegisterEps + /// emits one EPDownloadAndRegister event per provider via + /// EpDownloadTracker, plus one aggregate EPDownloadAttempt + /// event for the whole call. Pass nullptr in tests / when no + /// telemetry sink is available. EpDetector(const OrtApi& ort_api, OrtEnv& ort_env, std::vector> bootstrappers, - ILogger& logger); + ILogger& logger, + ITelemetry* telemetry = nullptr); ~EpDetector() override = default; // Non-copyable, non-movable (owns bootstrappers and mutex state) @@ -92,14 +100,21 @@ class EpDetector : public IEpDetector { OrtEnv& ort_env_; std::vector> bootstrappers_; ILogger& logger_; + ITelemetry* telemetry_ = nullptr; std::mutex download_mutex_; std::atomic download_in_progress_{false}; mutable std::mutex cache_mutex_; - // Populated once in the constructor; size and element addresses (including name strings) - // are stable for the detector's lifetime. Only is_registered fields are mutated, under - // cache_mutex_. cached_eps_c_ mirrors cached_eps_ for the C ABI. + // cached_eps_ is updated under cache_mutex_. Each C ABI view is an immutable + // snapshot retained for the detector lifetime so previously returned spans are + // never mutated or freed while callers may still read them. std::vector cached_eps_; - std::vector cached_eps_c_; + std::deque> cached_eps_snapshots_; + const std::vector* current_cached_eps_ = nullptr; + std::deque> cached_eps_c_snapshots_; + const std::vector* current_cached_eps_c_ = nullptr; + + void PublishEpSnapshotLocked(); + void PublishCApiSnapshotLocked(); }; } // namespace fl diff --git a/sdk_v2/cpp/src/ep_detection/ep_utils.cc b/sdk_v2/cpp/src/ep_detection/ep_utils.cc index 5cd025b1b..2f8c53fc3 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_utils.cc +++ b/sdk_v2/cpp/src/ep_detection/ep_utils.cc @@ -9,6 +9,7 @@ #include #include +#include #ifdef _WIN32 #define WIN32_LEAN_AND_MEAN @@ -75,6 +76,30 @@ bool VerifyEpBinaries( return true; } +bool VerifyEpBinaries( + const std::filesystem::path& dir, + const std::vector>& expected, + std::string_view ep_name, + ILogger& logger) { + for (const auto& [filename, expected_hash] : expected) { + auto file_path = dir / filename; + + if (!std::filesystem::exists(file_path)) { + return false; + } + + auto hash = Sha256File(file_path); + if (CompareCaseInsensitive(hash, expected_hash) != 0) { + logger.Log(LogLevel::Warning, + fmt::format("{}: hash mismatch for {}: got {}, expected {}", + ep_name, filename, hash, expected_hash)); + return false; + } + } + + return true; +} + void PrependDirToProcessPath([[maybe_unused]] const std::filesystem::path& dir) { #ifdef _WIN32 DWORD len = GetEnvironmentVariableW(L"PATH", nullptr, 0); diff --git a/sdk_v2/cpp/src/ep_detection/ep_utils.h b/sdk_v2/cpp/src/ep_detection/ep_utils.h index ba4690e65..985248ab3 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_utils.h +++ b/sdk_v2/cpp/src/ep_detection/ep_utils.h @@ -4,8 +4,10 @@ #include #include +#include #include #include +#include namespace fl { @@ -37,6 +39,12 @@ bool VerifyEpBinaries( std::string_view ep_name, ILogger& logger); +bool VerifyEpBinaries( + const std::filesystem::path& dir, + const std::vector>& expected, + std::string_view ep_name, + ILogger& logger); + /// Prepend @p dir to the process `PATH` environment variable for the lifetime of the process. /// /// EP provider libraries (CUDA, WebGPU) delay-load sibling dependency DLLs from their own directory, diff --git a/sdk_v2/cpp/src/ep_detection/webgpu_ep_bootstrapper.cc b/sdk_v2/cpp/src/ep_detection/webgpu_ep_bootstrapper.cc index d8b63db52..2f5d38aab 100644 --- a/sdk_v2/cpp/src/ep_detection/webgpu_ep_bootstrapper.cc +++ b/sdk_v2/cpp/src/ep_detection/webgpu_ep_bootstrapper.cc @@ -56,7 +56,7 @@ const std::unordered_map kPackageMetada "0767448ABEADE590821E88E56C471E0F2DFE89EC9A7642D08ADC3DC94F14AB92"} }, }; -#elif defined(__APPLE__) +#elif defined(__APPLE__) && defined(__aarch64__) const std::unordered_map kPackageMetadata = { {"macos-arm64", {"https://foundrypackages-ffhrdhbxb7gpdreh.b02.azurefd.net/webgpu_ep_0.1.0_macos-arm64.zip", @@ -64,7 +64,7 @@ const std::unordered_map kPackageMetada "A08BCEBE097B555E23938FCC71A5FAAD461F586CAB0B63DC9D21E970F6CA4C87"} }, }; -#else +#elif defined(__linux__) && defined(__x86_64__) && !defined(__ANDROID__) const std::unordered_map kPackageMetadata = { {"linux-x64", {"https://foundrypackages-ffhrdhbxb7gpdreh.b02.azurefd.net/webgpu_ep_0.1.0_linux-x64.zip", @@ -72,6 +72,8 @@ const std::unordered_map kPackageMetada "CBDFF74E6569E3CF66B46F0D194D87CD3CF49E83B7AA46552C39B0218D58B215"} }, }; +#else +const std::unordered_map kPackageMetadata = {}; #endif // Platform-specific package metadata is baked into the binary to keep @@ -80,10 +82,12 @@ const std::unordered_map kPackageMetada constexpr const char* kPlatformKey = "win-arm64"; #elif defined(_WIN32) constexpr const char* kPlatformKey = "win-x64"; -#elif defined(__APPLE__) +#elif defined(__APPLE__) && defined(__aarch64__) constexpr const char* kPlatformKey = "macos-arm64"; -#else +#elif defined(__linux__) && defined(__x86_64__) && !defined(__ANDROID__) constexpr const char* kPlatformKey = "linux-x64"; +#else +constexpr const char* kPlatformKey = ""; #endif const WebGpuPackageMetadata& GetPackageMetadata() { @@ -124,6 +128,10 @@ bool WebGpuEpBootstrapper::IsRegistered() const { return registered_; } +bool WebGpuEpBootstrapper::IsSupported() { + return kPackageMetadata.find(kPlatformKey) != kPackageMetadata.end(); +} + bool WebGpuEpBootstrapper::DownloadAndRegister(bool force, const ProgressCallback& progress_cb, ILogger& logger) { diff --git a/sdk_v2/cpp/src/ep_detection/webgpu_ep_bootstrapper.h b/sdk_v2/cpp/src/ep_detection/webgpu_ep_bootstrapper.h index e80d4651d..21a147d32 100644 --- a/sdk_v2/cpp/src/ep_detection/webgpu_ep_bootstrapper.h +++ b/sdk_v2/cpp/src/ep_detection/webgpu_ep_bootstrapper.h @@ -30,6 +30,7 @@ class WebGpuEpBootstrapper : public IEpBootstrapper { WebGpuEpBootstrapper& operator=(const WebGpuEpBootstrapper&) = delete; const std::string& Name() const override; + static bool IsSupported(); bool IsRegistered() const override; bool DownloadAndRegister(bool force, const ProgressCallback& progress_cb, diff --git a/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc b/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc index 06a597ea9..94206864d 100644 --- a/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.cc @@ -74,11 +74,13 @@ std::string JoinTokens(const std::vector& token_texts) { } // namespace AudioSession::AudioSession(const fl::Model& catalog_model, GenAIModelInstance& model, - ILogger& logger, ITelemetry& telemetry) + ILogger& logger, ITelemetry& telemetry, bool session_ref_acquired) : Session(catalog_model, logger, telemetry), logger_(logger), model_(model) { logger_.Log(LogLevel::Debug, fmt::format("Creating AudioSession for model: {}", model.ModelId())); // Last so a throw above does not leak a refcount; nothing below can throw. - model_.AcquireSession(); + if (!session_ref_acquired) { + model_.AcquireSession(); + } } AudioSession::~AudioSession() { diff --git a/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h b/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h index 1a35d096a..28b9dd886 100644 --- a/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h +++ b/sdk_v2/cpp/src/inferencing/generative/audio/audio_session.h @@ -35,7 +35,8 @@ struct SpeechSegmentItem; /// AudioTranscriptionResponse payload. class AudioSession : public Session { public: - AudioSession(const fl::Model& catalog_model, GenAIModelInstance& model, ILogger& logger, ITelemetry& telemetry); + AudioSession(const fl::Model& catalog_model, GenAIModelInstance& model, ILogger& logger, ITelemetry& telemetry, + bool session_ref_acquired = false); ~AudioSession(); // Movable: transfers session refcount ownership to the moved-to instance. diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc index eb76ef94d..7515f26b9 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.cc @@ -48,11 +48,14 @@ void ApplyToolChoiceToContext(std::optional tool_choice, ToolCallC } // namespace -ChatSession::ChatSession(const fl::Model& catalog_model, GenAIModelInstance& model, ILogger& logger, ITelemetry& telemetry) +ChatSession::ChatSession(const fl::Model& catalog_model, GenAIModelInstance& model, ILogger& logger, + ITelemetry& telemetry, bool session_ref_acquired) : Session(catalog_model, logger, telemetry), logger_(logger), model_(model) { logger_.Log(LogLevel::Debug, fmt::format("Creating ChatSession for model: {}", model.ModelId())); // Last so a throw above does not leak a refcount; nothing below can throw. - model_.AcquireSession(); + if (!session_ref_acquired) { + model_.AcquireSession(); + } } ChatSession::~ChatSession() { diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h index 4bd761bc3..f3b9ef29b 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/chat_session.h @@ -39,7 +39,8 @@ class ChatSession : public Session { // The assistant reply is at history_[history_start + input_count] }; - ChatSession(const fl::Model& catalog_model, GenAIModelInstance& model, ILogger& logger, ITelemetry& telemetry); + ChatSession(const fl::Model& catalog_model, GenAIModelInstance& model, ILogger& logger, ITelemetry& telemetry, + bool session_ref_acquired = false); ~ChatSession(); // Movable: transfers session refcount ownership to the moved-to instance. diff --git a/sdk_v2/cpp/src/inferencing/generative/embeddings/embeddings_session.cc b/sdk_v2/cpp/src/inferencing/generative/embeddings/embeddings_session.cc index ae571dbf4..08a9a4425 100644 --- a/sdk_v2/cpp/src/inferencing/generative/embeddings/embeddings_session.cc +++ b/sdk_v2/cpp/src/inferencing/generative/embeddings/embeddings_session.cc @@ -22,13 +22,15 @@ namespace fl { EmbeddingsSession::EmbeddingsSession(const fl::Model& catalog_model, GenAIModelInstance& model, - ILogger& logger, ITelemetry& telemetry) + ILogger& logger, ITelemetry& telemetry, bool session_ref_acquired) : Session(catalog_model, logger, telemetry, /*allow_concurrent_requests=*/true), logger_(logger), model_(model) { logger_.Log(LogLevel::Debug, fmt::format("Creating EmbeddingsSession for model: {}", model.ModelId())); // Last so a throw above does not leak a refcount; nothing below can throw. - model_.AcquireSession(); + if (!session_ref_acquired) { + model_.AcquireSession(); + } } EmbeddingsSession::~EmbeddingsSession() { diff --git a/sdk_v2/cpp/src/inferencing/generative/embeddings/embeddings_session.h b/sdk_v2/cpp/src/inferencing/generative/embeddings/embeddings_session.h index 26f9a317d..99f751fea 100644 --- a/sdk_v2/cpp/src/inferencing/generative/embeddings/embeddings_session.h +++ b/sdk_v2/cpp/src/inferencing/generative/embeddings/embeddings_session.h @@ -22,7 +22,7 @@ class GenAIModelInstance; class EmbeddingsSession : public Session { public: EmbeddingsSession(const fl::Model& catalog_model, GenAIModelInstance& model, - ILogger& logger, ITelemetry& telemetry); + ILogger& logger, ITelemetry& telemetry, bool session_ref_acquired = false); ~EmbeddingsSession(); EmbeddingsSession(EmbeddingsSession&&) = delete; diff --git a/sdk_v2/cpp/src/inferencing/model_load_manager.cc b/sdk_v2/cpp/src/inferencing/model_load_manager.cc index 0bc321b9b..f10e09e91 100644 --- a/sdk_v2/cpp/src/inferencing/model_load_manager.cc +++ b/sdk_v2/cpp/src/inferencing/model_load_manager.cc @@ -57,11 +57,52 @@ ModelLoadManager::ModelLoadManager(IEpDetector& ep_detector, ILogger& logger) : ep_detector_(ep_detector), logger_(logger) {} ModelLoadManager::~ModelLoadManager() { - // Destroy all loaded models under the lock. + // Destroy all loaded models under the lock. If a caller violates the public + // lifetime contract and keeps direct sessions alive past Manager destruction, + // keep those model instances alive rather than leaving sessions with dangling + // GenAIModelInstance references. std::lock_guard lock(mutex_); + for (auto it = loaded_models_.begin(); it != loaded_models_.end();) { + auto live_sessions = it->second->SessionRefCount(); + if (live_sessions > 0) { + logger_.Log(LogLevel::Warning, + fmt::format("ModelLoadManager destroyed while model '{}' still has {} live session(s); " + "leaving model instance allocated to avoid dangling session references", + it->first, live_sessions)); + it->second.release(); + it = loaded_models_.erase(it); + } else { + ++it; + } + } loaded_models_.clear(); } +ModelLoadManager::LoadedModelLease::~LoadedModelLease() { + Reset(); +} + +ModelLoadManager::LoadedModelLease::LoadedModelLease(LoadedModelLease&& other) noexcept + : model_(other.model_) { + other.model_ = nullptr; +} + +ModelLoadManager::LoadedModelLease& ModelLoadManager::LoadedModelLease::operator=(LoadedModelLease&& other) noexcept { + if (this != &other) { + Reset(); + model_ = other.model_; + other.model_ = nullptr; + } + return *this; +} + +void ModelLoadManager::LoadedModelLease::Reset() noexcept { + if (model_ != nullptr) { + model_->ReleaseSession(); + model_ = nullptr; + } +} + bool ModelLoadManager::HasEP(const std::string& ep_name) const { const auto& device_map = ep_detector_.GetAvailableDevicesToEPs(); for (const auto& [device, eps] : device_map) { @@ -89,6 +130,10 @@ ModelLoadManager::LoadResult ModelLoadManager::LoadModel(std::string_view model_ std::string path_str(model_path); std::string id_str(model_id); std::lock_guard lock(mutex_); + if (shutdown_.load()) { + FL_LOG_AND_THROW(logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + "cannot load model during shutdown"); + } // Check if model is already loaded auto it = loaded_models_.find(id_str); @@ -202,24 +247,21 @@ void ModelLoadManager::RejectNewLoads() { } void ModelLoadManager::UnloadAll(std::chrono::milliseconds timeout) { - // Snapshot ids+pointers under the lock; the GenAIModelInstance pointers stay valid - // because (a) only this method or UnloadModel can erase entries, and (b) we serialize - // the per-id erase below. - std::vector> snapshot; + std::vector ids; { std::lock_guard lock(mutex_); - snapshot.reserve(loaded_models_.size()); + ids.reserve(loaded_models_.size()); for (auto& [id, instance] : loaded_models_) { - snapshot.emplace_back(id, instance.get()); + ids.emplace_back(id); } } - if (snapshot.empty()) { + if (ids.empty()) { return; } logger_.Log(LogLevel::Information, - fmt::format("Shutdown: unloading {} model(s)", snapshot.size())); + fmt::format("Shutdown: unloading {} model(s)", ids.size())); using clock = std::chrono::steady_clock; constexpr auto kPollInterval = std::chrono::milliseconds(50); @@ -229,27 +271,39 @@ void ModelLoadManager::UnloadAll(std::chrono::milliseconds timeout) { // time linearly with model count. auto deadline = clock::now() + timeout; - for (auto& [id, instance] : snapshot) { - while (instance->SessionRefCount() > 0 && clock::now() < deadline) { + for (const auto& id : ids) { + int remaining = 0; + bool unloaded = false; + while (true) { + { + std::lock_guard lock(mutex_); + auto it = loaded_models_.find(id); + if (it == loaded_models_.end()) { + remaining = 0; + unloaded = true; + break; + } + + remaining = it->second->SessionRefCount(); + if (remaining == 0) { + logger_.Log(LogLevel::Information, fmt::format("unloading model: {}", id)); + loaded_models_.erase(it); + unloaded = true; + break; + } + } + + if (clock::now() >= deadline) { + break; + } + std::this_thread::sleep_for(kPollInterval); } - auto remaining = instance->SessionRefCount(); - if (remaining > 0) { + if (!unloaded && remaining > 0) { logger_.Log(LogLevel::Warning, fmt::format("Shutdown: model '{}' still has {} session(s) after overall {}ms deadline; leaving loaded", id, remaining, timeout.count())); - continue; - } - - try { - UnloadModel(id); - } catch (const std::exception& ex) { - // A new session attached between our refcount poll and the lock acquisition inside - // UnloadModel. Log and move on — IsShutdownRequested-gated callers shouldn't be - // creating new sessions, but we don't crash shutdown over it. - logger_.Log(LogLevel::Warning, - fmt::format("Shutdown: failed to unload '{}': {}", id, ex.what())); } } } @@ -270,4 +324,17 @@ GenAIModelInstance* ModelLoadManager::GetLoadedModel(std::string_view model_id) return nullptr; } +ModelLoadManager::LoadedModelLease ModelLoadManager::AcquireLoadedModel(std::string_view model_id) { + std::lock_guard lock(mutex_); + + std::string id_str(model_id); + auto it = loaded_models_.find(id_str); + if (it == loaded_models_.end()) { + return {}; + } + + it->second->AcquireSession(); + return LoadedModelLease(it->second.get()); +} + } // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/model_load_manager.h b/sdk_v2/cpp/src/inferencing/model_load_manager.h index f582217ad..9082c3bfa 100644 --- a/sdk_v2/cpp/src/inferencing/model_load_manager.h +++ b/sdk_v2/cpp/src/inferencing/model_load_manager.h @@ -39,6 +39,33 @@ class ModelLoadManager { GenAIModelInstance* model = nullptr; // non-owning pointer; lifetime managed by this class }; + class LoadedModelLease { + public: + LoadedModelLease() = default; + ~LoadedModelLease(); + + LoadedModelLease(const LoadedModelLease&) = delete; + LoadedModelLease& operator=(const LoadedModelLease&) = delete; + + LoadedModelLease(LoadedModelLease&& other) noexcept; + LoadedModelLease& operator=(LoadedModelLease&& other) noexcept; + + explicit operator bool() const { return model_ != nullptr; } + GenAIModelInstance* get() const { return model_; } + GenAIModelInstance& operator*() const { return *model_; } + GenAIModelInstance* operator->() const { return model_; } + + void Reset() noexcept; + void Release() noexcept { model_ = nullptr; } + + private: + explicit LoadedModelLease(GenAIModelInstance* model) : model_(model) {} + + GenAIModelInstance* model_ = nullptr; + + friend class ModelLoadManager; + }; + ModelLoadManager(IEpDetector& ep_detector, ILogger& logger); ~ModelLoadManager(); @@ -67,6 +94,10 @@ class ModelLoadManager { /// The returned pointer is valid until UnloadModel is called for this model_id. GenAIModelInstance* GetLoadedModel(std::string_view model_id); + /// Get a loaded model by ID and increment its live-session refcount while + /// holding the load-manager lock. Returns an empty lease if not loaded. + LoadedModelLease AcquireLoadedModel(std::string_view model_id); + /// Reject all future LoadModel calls. Called by Manager::Shutdown(). /// Idempotent and thread-safe. void RejectNewLoads(); diff --git a/sdk_v2/cpp/src/inferencing/session/session.cc b/sdk_v2/cpp/src/inferencing/session/session.cc index e8f90ee69..137a9eb5a 100644 --- a/sdk_v2/cpp/src/inferencing/session/session.cc +++ b/sdk_v2/cpp/src/inferencing/session/session.cc @@ -16,6 +16,7 @@ #include +#include #include namespace fl { @@ -48,7 +49,7 @@ std::unique_ptr Session::Create(const fl::Model& model) { } auto& lm = mgr.GetModelLoadManager(); - auto* loaded = lm.GetLoadedModel(model.Id()); + auto loaded = lm.AcquireLoadedModel(model.Id()); if (!loaded) { FL_LOG_AND_THROW(logger, FOUNDRY_LOCAL_ERROR_INTERNAL, "loaded model not found in load manager"); } @@ -57,19 +58,22 @@ std::unique_ptr Session::Create(const fl::Model& model) { const auto& info = model.Info(); if (info.task == "chat-completion" || info.task == "vision-language-chat") { - auto session = std::make_unique(model, *loaded, logger, telemetry); + auto session = std::make_unique(model, *loaded, logger, telemetry, true); + loaded.Release(); tracker.SetStatus(ActionStatus::kSuccess); return session; } if (info.task == "automatic-speech-recognition") { - auto session = std::make_unique(model, *loaded, logger, telemetry); + auto session = std::make_unique(model, *loaded, logger, telemetry, true); + loaded.Release(); tracker.SetStatus(ActionStatus::kSuccess); return session; } if (info.task == "embeddings") { - auto session = std::make_unique(model, *loaded, logger, telemetry); + auto session = std::make_unique(model, *loaded, logger, telemetry, true); + loaded.Release(); tracker.SetStatus(ActionStatus::kSuccess); return session; } @@ -101,17 +105,45 @@ void Session::ProcessRequest(const Request& request, Response& response) { lock.lock(); } - ActionTracker tracker(Action::kSessionProcessRequest, telemetry_); + // Use the context the caller staged (an HTTP route stages an indirect child + // with the route's correlation id); otherwise mint a direct context per call + // for direct SDK use. + InvocationContext context = request_context_ ? *request_context_ : InvocationContext::Direct(); + context.EnsureCorrelationId(); + const bool streaming = static_cast(callback_fn_); + + ActionTracker tracker(Action::kSessionProcessRequest, telemetry_, context); tracker.SetModelId(CatalogModel().Id()); + const auto start = std::chrono::steady_clock::now(); try { ProcessRequestImpl(request, response); + if (request.canceled.load(std::memory_order_relaxed)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "request cancelled"); + } tracker.SetStatus(ActionStatus::kSuccess); } catch (const std::exception& ex) { tracker.RecordException(ex); throw; } + + // Per-inference Model event — emitted on success with whatever metrics this run + // produced, sharing the action's correlation id and indirect flag. TTFT and + // memory are not surfaced by the generators yet and stay at their unset values. + ModelUsageInfo usage; + usage.model_id = CatalogModel().Id(); + usage.execution_provider = ExecutionProvider(); + usage.user_agent = context.user_agent; + usage.correlation_id = context.correlation_id; + usage.indirect = context.indirect; + usage.stream = streaming; + usage.total_time_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); + usage.total_tokens = static_cast(response.usage.total_tokens); + usage.input_token_count = static_cast(response.usage.prompt_tokens); + telemetry_.RecordModelUsage(usage); } } // namespace fl diff --git a/sdk_v2/cpp/src/inferencing/session/session.h b/sdk_v2/cpp/src/inferencing/session/session.h index f3ec0e5f1..a92556c16 100644 --- a/sdk_v2/cpp/src/inferencing/session/session.h +++ b/sdk_v2/cpp/src/inferencing/session/session.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -15,6 +16,7 @@ #include "inferencing/session/request.h" #include "inferencing/session/response.h" #include "inferencing/session/types.h" +#include "telemetry/invocation_context.h" #include "util/key_value_pairs.h" namespace fl { @@ -100,6 +102,15 @@ class Session { callback_user_data_ = user_data; } + /// Telemetry context for the next ProcessRequest. HTTP handlers set this to an + /// indirect child of the route's context so the kSessionProcessRequest action + /// and the per-inference Model event are marked indirect and share the route's + /// correlation id. When unset (direct SDK use), ProcessRequest mints its own + /// direct context per call. + void SetRequestContext(InvocationContext context) { + request_context_ = std::move(context); + } + protected: Session(const fl::Model& catalog_model, ILogger& logger, ITelemetry& telemetry, bool allow_concurrent_requests = false); @@ -131,6 +142,10 @@ class Session { /// Requests are serialized if the derived class does not opt into concurrency via allow_concurrent_requests_. virtual void ProcessRequestImpl(const Request& request, Response& response) = 0; + /// Execution provider the loaded model runs on (e.g. "CPU", "CUDA"). Surfaced + /// on the per-inference Model telemetry event. Empty when not known. + virtual std::string ExecutionProvider() const { return {}; } + /// Create a per-request callback handler. Returns nullptr if no callback is set. /// The handler is owned by the caller (unique_ptr) and drains+joins on destruction. std::unique_ptr CreateCallbackHandler(const Request& request) { @@ -151,6 +166,7 @@ class Session { KeyValuePairs session_options_; StreamingCallbackFn callback_fn_; void* callback_user_data_ = nullptr; + std::optional request_context_; const bool allow_concurrent_requests_; mutable std::unique_ptr request_mutex_ = std::make_unique(); }; diff --git a/sdk_v2/cpp/src/inferencing/session/session_manager.cc b/sdk_v2/cpp/src/inferencing/session/session_manager.cc index a48e81bb6..59f1a86d7 100644 --- a/sdk_v2/cpp/src/inferencing/session/session_manager.cc +++ b/sdk_v2/cpp/src/inferencing/session/session_manager.cc @@ -85,13 +85,19 @@ size_t SessionManager::ActiveCount() const { // --- Session cache --- -std::unique_ptr SessionManager::CheckOut(const std::string& key) { +std::unique_ptr SessionManager::CheckOut(const std::string& key, const std::string& expected_model_id) { std::lock_guard lock(mutex_); auto it = cache_.find(key); if (it == cache_.end()) { return nullptr; } + if (!expected_model_id.empty() && it->second.model_id != expected_model_id) { + logger_.Log(LogLevel::Information, + fmt::format("SessionManager: cached session for '{}' uses model '{}', not requested model '{}'", + key, it->second.model_id, expected_model_id)); + return nullptr; + } auto session = std::move(it->second.session); lru_order_.erase(it->second.lru_iter); @@ -101,13 +107,19 @@ std::unique_ptr SessionManager::CheckOut(const std::string& key) { return session; } -void SessionManager::CheckIn(const std::string& key, std::unique_ptr session) { +void SessionManager::CheckIn(const std::string& key, std::unique_ptr session, std::string model_id) { // Collect evicted sessions to destroy outside the lock. std::vector> evicted; { std::lock_guard lock(mutex_); + if (shutting_down_.load()) { + logger_.Log(LogLevel::Debug, + fmt::format("SessionManager: dropping check-in for '{}' during shutdown", key)); + return; + } + // Replace existing entry for this key (if any) auto existing = cache_.find(key); if (existing != cache_.end()) { @@ -127,7 +139,7 @@ void SessionManager::CheckIn(const std::string& key, std::unique_ptr CheckOut(const std::string& key); + std::unique_ptr CheckOut(const std::string& key, const std::string& expected_model_id = {}); /// Insert a session into the cache under the given key. /// May evict the least-recently-used entry if at capacity. /// The session remains tracked (registered) while cached. - void CheckIn(const std::string& key, std::unique_ptr session); + void CheckIn(const std::string& key, std::unique_ptr session, std::string model_id = {}); /// Remove and destroy a cached session by key. Returns true if an entry was removed. /// @@ -94,6 +96,7 @@ class SessionManager : public ISessionManager { private: struct CacheEntry { std::unique_ptr session; + std::string model_id; std::list::iterator lru_iter; }; diff --git a/sdk_v2/cpp/src/manager.cc b/sdk_v2/cpp/src/manager.cc index 23c6ccb3f..901d8edc6 100644 --- a/sdk_v2/cpp/src/manager.cc +++ b/sdk_v2/cpp/src/manager.cc @@ -24,6 +24,10 @@ #include "spdlog_logger.h" #include "telemetry/telemetry_action_tracker.h" #include "telemetry/telemetry_logger.h" +#if FOUNDRY_LOCAL_HAS_1DS +#include "one_ds_tenant_token.h" // auto-generated header, in ${CMAKE_BINARY_DIR}/generated +#include "telemetry/one_ds_telemetry.h" +#endif #include "util/string_utils.h" #include "utils.h" @@ -178,6 +182,7 @@ std::unique_ptr Manager::s_instance_; Manager::Manager(const Configuration& config) : config_(config) { + try { config_.Validate(); const bool genai_verbose_logging = IsGenAIVerboseLoggingEnabled(); @@ -248,7 +253,11 @@ Manager::Manager(const Configuration& config) // Detected once and reused below for both the WinML-catalog skip-list and the // Foundry CUDA bootstrapper. HasNvidiaGpu() shells out to nvidia-smi, so caching // the result here avoids a second subprocess spawn. +#ifdef _WIN32 const bool has_nvidia_gpu = CudaEpBootstrapper::HasNvidiaGpu(); +#else + const bool has_nvidia_gpu = false; +#endif #if FOUNDRY_LOCAL_HAS_EP_CATALOG // WinML EPs — enumerate from the OS EP catalog (Windows 10 19H1+ reg-free runtime). @@ -285,17 +294,34 @@ Manager::Manager(const Configuration& config) const auto cache_dir = std::filesystem::path(*config_.model_cache_dir).parent_path(); - // CUDA EP — only if an NVIDIA GPU is detected +#ifdef _WIN32 + // CUDA EP — only if an NVIDIA GPU is detected. The current Foundry CUDA + // package contains Windows DLLs. if (has_nvidia_gpu) { const auto cuda_ep_dir = cache_dir / "cuda-ep"; bootstrappers.push_back(std::make_unique(cuda_ep_dir.string(), register_ep)); } +#endif - // WebGPU EP — always available (no hardware detection needed). - const auto webgpu_ep_dir = cache_dir / "webgpu-ep"; - bootstrappers.push_back(std::make_unique(webgpu_ep_dir.string(), register_ep)); + if (WebGpuEpBootstrapper::IsSupported()) { + const auto webgpu_ep_dir = cache_dir / "webgpu-ep"; + bootstrappers.push_back(std::make_unique(webgpu_ep_dir.string(), register_ep)); + } - ep_detector_ = std::make_unique(*ort_api_, *ort_env_, std::move(bootstrappers), *logger_); + // Telemetry must be constructed before subsystems that emit events so we can + // pass it to them at construction time (e.g. EpDetector emits EPDownloadAttempt + // / EPDownloadAndRegister events from DownloadAndRegisterEps). +#if FOUNDRY_LOCAL_HAS_1DS + // OneDsTelemetry includes a TelemetryLogger mirror, so local diagnostic logging + // is preserved whether or not 1DS upload is enabled. Suppression rules + // (CI environment, empty token) are enforced inside OneDsTelemetry. + telemetry_ = std::make_unique(kFoundryLocalTenantToken, config_.app_name, *logger_); +#else + telemetry_ = std::make_unique(config_.app_name, *logger_); +#endif + + ep_detector_ = std::make_unique(*ort_api_, *ort_env_, std::move(bootstrappers), *logger_, + telemetry_.get()); // Read configurable download concurrency (default 64) int download_concurrency = 64; @@ -320,10 +346,10 @@ Manager::Manager(const Configuration& config) config_.catalog_region.value_or("auto"), download_concurrency, *logger_, - disable_region_fallback); + disable_region_fallback, + telemetry_.get()); model_load_manager_ = std::make_unique(*ep_detector_, *logger_); session_manager_ = std::make_unique(*logger_); - telemetry_ = std::make_unique(config_.app_name, *logger_); catalog_ = std::make_unique( config_.catalog_urls, download_manager_->GetCacheDirectory(), @@ -333,7 +359,24 @@ Manager::Manager(const Configuration& config) *ep_detector_, *logger_, config_.external_service_url.has_value(), config_.catalog_region.value_or("auto"), - disable_region_fallback); + disable_region_fallback, + telemetry_.get()); + } catch (...) { + if (ort_api_ != nullptr && ort_env_ != nullptr) { + for (const auto& name : registered_ep_libraries_) { + OrtStatus* status = ort_api_->UnregisterExecutionProviderLibrary(ort_env_, name.c_str()); + if (status != nullptr) { + ort_api_->ReleaseStatus(status); + } + } + registered_ep_libraries_.clear(); + ort_api_->ReleaseEnv(ort_env_); + ort_env_ = nullptr; + } + SetOgaLogCallback(nullptr); + s_ort_logger.store(nullptr, std::memory_order_release); + throw; + } } Manager::~Manager() { @@ -359,8 +402,9 @@ Manager::~Manager() { model_load_manager_.reset(); download_manager_.reset(); catalog_.reset(); - telemetry_.reset(); + // ep_detector_ holds a raw ITelemetry* — destroy it before telemetry_. ep_detector_.reset(); + telemetry_.reset(); // Unregister EPs we registered, then drop our OrtEnv refcount. Best-effort: // log failures but don't throw from a destructor. @@ -409,7 +453,8 @@ Manager& Manager::Create(const Configuration& config) { // state: catch and log, then proceed. The Manager itself is fully constructed at this // point — only the post-construction signaling can fail, and it's not load-bearing. try { - created->telemetry_->RecordAction(Action::kCoreInitialize, ActionStatus::kSuccess, "", false, 0); + created->telemetry_->RecordAction(Action::kCoreInitialize, ActionStatus::kSuccess, + InvocationContext::Direct(), 0); } catch (const std::exception& ex) { created->GetLogger().Log(LogLevel::Error, fmt::format("telemetry RecordAction failed during Create: {}", ex.what())); @@ -437,34 +482,80 @@ void Manager::Destroy() { s_instance_.reset(); } +void Manager::RequestShutdown() { + std::lock_guard lock(s_mutex_); + if (s_instance_ != nullptr) { + s_instance_->Shutdown(); + } +} + ICatalog& Manager::GetCatalog() { return *catalog_; } void Manager::StartWebService() { - if (web_service_running_) { - FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "web service is already running"); - } - - if (config_.external_service_url.has_value()) { - FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, - "cannot start local web service when external_service_url is configured"); - } - ActionTracker tracker(Action::kCoreServiceStart, *telemetry_); #ifdef FOUNDRY_LOCAL_HAS_WEB_SERVICE - web_service_ = std::make_unique(*catalog_, *logger_, *config_.model_cache_dir, *model_load_manager_, - *session_manager_, *telemetry_, - [this]() { Shutdown(); }); + bool stop_after_start = false; + + { + std::lock_guard lock(web_service_mutex_); + + if (web_service_ != nullptr) { + FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "web service is already running"); + } + + if (config_.external_service_url.has_value()) { + FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + "cannot start local web service when external_service_url is configured"); + } + if (shutdown_requested_.load(std::memory_order_acquire)) { + FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "cannot start web service during shutdown"); + } - auto endpoints = config_.web_service_endpoints; - if (endpoints.empty()) { - endpoints.push_back("http://127.0.0.1:0"); + web_service_ = std::make_unique(*catalog_, *logger_, *config_.model_cache_dir, *model_load_manager_, + *session_manager_, *telemetry_, + []() { Manager::RequestShutdown(); }); + + auto endpoints = config_.web_service_endpoints; + if (endpoints.empty()) { + endpoints.push_back("http://127.0.0.1:0"); + } + + try { + bound_urls_ = web_service_->Start(endpoints); + web_service_running_ = true; + stop_after_start = shutdown_requested_.load(std::memory_order_acquire); + + if (!stop_after_start) { + // Open an app-usage session for the lifetime of the running service so events + // carry ext.app.sesId and the backend gets session duration. + try { + telemetry_->StartSession(); + web_service_session_started_ = true; + } catch (const std::exception& ex) { + logger_->Log(LogLevel::Warning, std::string("telemetry StartSession failed: ") + ex.what()); + } catch (...) { + logger_->Log(LogLevel::Warning, "telemetry StartSession failed with unknown error"); + } + } + } catch (...) { + if (web_service_) { + web_service_->Stop(); + } + web_service_.reset(); + web_service_running_ = false; + bound_urls_.clear(); + throw; + } + } + + if (stop_after_start) { + StopWebService(); + FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "web service stopped because shutdown was requested"); } - bound_urls_ = web_service_->Start(endpoints); - web_service_running_ = true; tracker.SetStatus(ActionStatus::kSuccess); #else FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, @@ -472,28 +563,40 @@ void Manager::StartWebService() { #endif } -const std::vector& Manager::GetWebServiceUrls() const { +std::vector Manager::GetWebServiceUrls() const { // No "not running" check: bound_urls_ is cleared in StopWebService() and is empty before // StartWebService(), so the empty vector is the documented "service is not running" signal // (see GetWebServiceEndpoints() docstring in foundry_local_cpp.h). + std::lock_guard lock(web_service_mutex_); return bound_urls_; } void Manager::StopWebService() { - if (!web_service_running_) { +#ifdef FOUNDRY_LOCAL_HAS_WEB_SERVICE + std::lock_guard lock(web_service_mutex_); + + if (web_service_ == nullptr) { // No-op rather than throw: the public-API contract treats StopWebService() as idempotent so // callers can shut down unconditionally without first probing service state. logger_->Log(LogLevel::Information, "StopWebService called but web service is not running; ignoring"); return; } - ActionTracker tracker(Action::kCoreServiceStop, *telemetry_); -#ifdef FOUNDRY_LOCAL_HAS_WEB_SERVICE web_service_->Stop(); web_service_.reset(); web_service_running_ = false; bound_urls_.clear(); + if (web_service_session_started_) { + web_service_session_started_ = false; + try { + telemetry_->EndSession(); + } catch (const std::exception& ex) { + logger_->Log(LogLevel::Warning, std::string("telemetry EndSession failed: ") + ex.what()); + } catch (...) { + logger_->Log(LogLevel::Warning, "telemetry EndSession failed with unknown error"); + } + } tracker.SetStatus(ActionStatus::kSuccess); #else FL_LOG_AND_THROW(*logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, @@ -509,9 +612,12 @@ void Manager::Shutdown() { logger_->Log(LogLevel::Information, "Shutdown requested"); - if (web_service_running_) { - StopWebService(); - } + model_load_manager_->RejectNewLoads(); + session_manager_->CancelAll(); + +#ifdef FOUNDRY_LOCAL_HAS_WEB_SERVICE + StopWebService(); +#endif // Order matters: // 1. Reject new loads so callers gated on IsShutdownRequested can stop early. @@ -519,8 +625,6 @@ void Manager::Shutdown() { // 3. Unload all models, polling per-model session refcount for direct-API users // who haven't dropped their flSession* yet. Bounded by timeout so a stuck // caller can't block process shutdown indefinitely. - model_load_manager_->RejectNewLoads(); - session_manager_->CancelAll(); session_manager_->WaitForDrain(); model_load_manager_->UnloadAll(); } diff --git a/sdk_v2/cpp/src/manager.h b/sdk_v2/cpp/src/manager.h index 4b5440db7..37e1ce82d 100644 --- a/sdk_v2/cpp/src/manager.h +++ b/sdk_v2/cpp/src/manager.h @@ -45,6 +45,9 @@ class Manager { /// Destroy the singleton and release all resources. static void Destroy(); + /// Request shutdown on the singleton if it still exists. + static void RequestShutdown(); + /// Get the shared catalog interface for querying models. /// The catalog is owned by the manager and shared across all consumers /// (web service, C API, etc.) so model state (e.g. IsLoaded) is consistent. @@ -81,7 +84,7 @@ class Manager { /// Get the bound service URLs. The returned reference is valid as long as /// the web service is running. Throws if web service is not running. - const std::vector& GetWebServiceUrls() const; + std::vector GetWebServiceUrls() const; /// Stop the embedded web service. void StopWebService(); @@ -120,9 +123,12 @@ class Manager { // released manually in ~Manager() after all // consumers (sessions, ep_detector_) are gone. // logger_ — everything logs through this, destroyed last + // telemetry_ — used by ep_detector_ and throughout; must + // outlive ep_detector_ because EpDetector holds + // a raw ITelemetry* for emitting EP events // ep_detector_ — detects HW acceleration; holds OrtEnv& (must - // outlive ort_env_ release in ~Manager()) - // telemetry_ — used throughout + // outlive ort_env_ release in ~Manager()) and + // a raw ITelemetry* // catalog_ — owns all Model instances. used by download_manager, model_load_manager, and web service // download_manager_ — uses ModelInfo owned by catalog // model_load_manager_ — holds loaded model state referencing catalog models @@ -135,14 +141,16 @@ class Manager { OrtEnv* ort_env_ = nullptr; std::vector registered_ep_libraries_; std::unique_ptr logger_; - std::unique_ptr ep_detector_; std::unique_ptr telemetry_; + std::unique_ptr ep_detector_; std::unique_ptr catalog_; std::unique_ptr download_manager_; std::unique_ptr model_load_manager_; std::unique_ptr session_manager_; std::atomic shutdown_requested_{false}; std::atomic web_service_running_{false}; + mutable std::mutex web_service_mutex_; + bool web_service_session_started_ = false; std::vector bound_urls_; #ifdef FOUNDRY_LOCAL_HAS_WEB_SERVICE diff --git a/sdk_v2/cpp/src/model.cc b/sdk_v2/cpp/src/model.cc index 1f06201ce..e036ed468 100644 --- a/sdk_v2/cpp/src/model.cc +++ b/sdk_v2/cpp/src/model.cc @@ -111,11 +111,13 @@ Model::Model(Model&& other) noexcept download_manager_(other.download_manager_), model_load_manager_(other.model_load_manager_), variants_(std::move(other.variants_)), - selected_variant_(other.selected_variant_) { + selected_variant_(other.selected_variant_.load(std::memory_order_acquire)), + explicit_variant_selected_(other.explicit_variant_selected_.load(std::memory_order_acquire)) { // After vector move, selected_variant_ still points into the transferred buffer. other.download_manager_ = nullptr; other.model_load_manager_ = nullptr; - other.selected_variant_ = nullptr; + other.selected_variant_.store(nullptr, std::memory_order_release); + other.explicit_variant_selected_.store(false, std::memory_order_release); } Model& Model::operator=(Model&& other) noexcept { @@ -126,10 +128,13 @@ Model& Model::operator=(Model&& other) noexcept { download_manager_ = other.download_manager_; model_load_manager_ = other.model_load_manager_; variants_ = std::move(other.variants_); - selected_variant_ = other.selected_variant_; + selected_variant_.store(other.selected_variant_.load(std::memory_order_acquire), std::memory_order_release); + explicit_variant_selected_.store(other.explicit_variant_selected_.load(std::memory_order_acquire), + std::memory_order_release); other.download_manager_ = nullptr; other.model_load_manager_ = nullptr; - other.selected_variant_ = nullptr; + other.selected_variant_.store(nullptr, std::memory_order_release); + other.explicit_variant_selected_.store(false, std::memory_order_release); } return *this; @@ -163,7 +168,7 @@ Model Model::FromModelInfo(ModelInfo info, Model Model::MakeContainer(Model first_variant) { Model container; container.variants_.push_back(std::make_unique(std::move(first_variant))); - container.selected_variant_ = container.variants_.back().get(); + container.selected_variant_.store(container.variants_.back().get(), std::memory_order_release); return container; } @@ -196,12 +201,14 @@ void Model::SelectDefaultVariant() { for (auto& v : variants_) { if (v->IsCached()) { - selected_variant_ = v.get(); + selected_variant_.store(v.get(), std::memory_order_release); + explicit_variant_selected_.store(false, std::memory_order_release); return; } } - selected_variant_ = variants_.front().get(); + selected_variant_.store(variants_.front().get(), std::memory_order_release); + explicit_variant_selected_.store(false, std::memory_order_release); } // --------------------------------------------------------------------------- @@ -209,24 +216,24 @@ void Model::SelectDefaultVariant() { // --------------------------------------------------------------------------- const std::string& Model::Id() const { - if (selected_variant_) { - return selected_variant_->Id(); + if (auto* selected = SelectedVariant()) { + return selected->Id(); } return info_.model_id; } const std::string& Model::Alias() const { - if (selected_variant_) { - return selected_variant_->Alias(); + if (auto* selected = SelectedVariant()) { + return selected->Alias(); } return info_.alias; } const ModelInfo& Model::Info() const { - if (selected_variant_) { - return selected_variant_->Info(); + if (auto* selected = SelectedVariant()) { + return selected->Info(); } return info_; @@ -251,16 +258,16 @@ std::vector Model::Variants() const { } bool Model::IsCached() const { - if (selected_variant_) { - return selected_variant_->IsCached(); + if (auto* selected = SelectedVariant()) { + return selected->IsCached(); } return cached_; } bool Model::IsLoaded() const { - if (selected_variant_) { - return selected_variant_->IsLoaded(); + if (auto* selected = SelectedVariant()) { + return selected->IsLoaded(); } // ModelLoadManager owns the authoritative loaded-instance map. The pointer is set at @@ -274,8 +281,8 @@ bool Model::IsLoaded() const { // --------------------------------------------------------------------------- void Model::Download(std::function progress_cb) { - if (selected_variant_) { - selected_variant_->Download(std::move(progress_cb)); + if (auto* selected = SelectedVariant()) { + selected->Download(std::move(progress_cb)); return; } @@ -296,16 +303,16 @@ void Model::Download(std::function progress_cb) { } const std::string& Model::GetPath() const { - if (selected_variant_) { - return selected_variant_->GetPath(); + if (auto* selected = SelectedVariant()) { + return selected->GetPath(); } return local_path_; } void Model::Load(ExecutionProvider ep) { - if (selected_variant_) { - selected_variant_->Load(ep); + if (auto* selected = SelectedVariant()) { + selected->Load(ep); return; } @@ -319,8 +326,8 @@ void Model::Load(ExecutionProvider ep) { } void Model::Unload() { - if (selected_variant_) { - selected_variant_->Unload(); + if (auto* selected = SelectedVariant()) { + selected->Unload(); return; } @@ -329,8 +336,8 @@ void Model::Unload() { } void Model::RemoveFromCache() { - if (selected_variant_) { - selected_variant_->RemoveFromCache(); + if (auto* selected = SelectedVariant()) { + selected->RemoveFromCache(); return; } @@ -357,9 +364,12 @@ void Model::SelectVariant(const Model& variant) { "with all variants available."); } + std::lock_guard lock(state_mutex_); + for (auto& v : variants_) { if (v.get() == &variant) { - selected_variant_ = v.get(); + selected_variant_.store(v.get(), std::memory_order_release); + explicit_variant_selected_.store(true, std::memory_order_release); return; } } @@ -434,8 +444,8 @@ Model::IOInfo IOInfoFromCache(const StaticIOCache& cache) { } // namespace Model::IOInfo Model::GetInputOutputInfo() const { - if (selected_variant_) { - return selected_variant_->GetInputOutputInfo(); + if (auto* selected = SelectedVariant()) { + return selected->GetInputOutputInfo(); } const auto& task = Info().task; diff --git a/sdk_v2/cpp/src/model.h b/sdk_v2/cpp/src/model.h index 90710768a..0dcb329c1 100644 --- a/sdk_v2/cpp/src/model.h +++ b/sdk_v2/cpp/src/model.h @@ -70,6 +70,9 @@ class Model { /// first cached variant if any, else the best variant. /// Requires IsContainer() to be true. void SelectDefaultVariant(); + bool HasExplicitVariantSelection() const { + return explicit_variant_selected_.load(std::memory_order_acquire); + } /// Best-first comparator: device priority asc, version desc, created-at desc, model_id asc. /// Exposed so callers that return Model* lists can produce consistent ordering with Variants(). @@ -116,7 +119,7 @@ class Model { IOInfo GetInputOutputInfo() const; /// True if this is a multi-variant container. - bool IsContainer() const { return selected_variant_ != nullptr; } + bool IsContainer() const { return SelectedVariant() != nullptr; } // --- Mutation methods --- @@ -169,11 +172,14 @@ class Model { // Container data (empty/null for leaves). unique_ptr keeps Model addresses // stable across vector growth/reordering. std::vector> variants_; - Model* selected_variant_ = nullptr; // non-null = this is a container + std::atomic selected_variant_{nullptr}; // non-null = this is a container + std::atomic explicit_variant_selected_{false}; // Guards variants_ across reader/writer threads (catalog refresh adding variants // while another thread enumerates via Variants()). mutable std::mutex state_mutex_; + + Model* SelectedVariant() const { return selected_variant_.load(std::memory_order_acquire); } }; } // namespace fl diff --git a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc index e7b2ac642..1ac716623 100644 --- a/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc +++ b/sdk_v2/cpp/src/service/audio_transcriptions_handler.cc @@ -7,6 +7,7 @@ #include "c_api_types.h" #include "catalog.h" #include "contracts/audio_transcriptions.h" +#include "exception.h" #include "inferencing/generative/audio/audio_session.h" #include "inferencing/model_load_manager.h" #include "inferencing/session/session_manager.h" @@ -57,13 +58,13 @@ std::shared_ptr AudioTranscriptionsHandler } std::shared_ptr AudioTranscriptionsHandler::ResolveModel( - const std::string& model_name, Model*& model, GenAIModelInstance*& loaded) { + const std::string& model_name, Model*& model, ModelLoadManager::LoadedModelLease& loaded) { model = ctx_.catalog.GetModelVariant(model_name); if (!model) { return ErrorResponse(Status::CODE_404, "Model not found", "No model matching '" + model_name + "'"); } - loaded = ctx_.model_load_manager.GetLoadedModel(model->Id()); + loaded = ctx_.model_load_manager.AcquireLoadedModel(model->Id()); if (!loaded) { return ErrorResponse(Status::CODE_400, "Model not loaded", "Model '" + model_name + "' must be loaded before inference"); @@ -87,16 +88,19 @@ void AudioTranscriptionsHandler::BuildOpenAIJsonRequest(const std::string& body, std::shared_ptr AudioTranscriptionsHandler::handle( const std::shared_ptr& request) { - ActionTracker tracker(Action::kOpenAIAudioTranscribe, ctx_.telemetry); + auto route_ctx = InvocationContext::Direct(GetUserAgent(request)); + auto tracker = std::make_unique(Action::kOpenAIAudioTranscribe, ctx_.telemetry, route_ctx); auto body_str = request->readBodyToString(); if (!body_str || body_str->empty()) { + tracker->SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Empty request body"); } // 1. Parse & validate AudioTranscriptionRequest req; if (auto err = ParseAndValidateRequest(body_str->c_str(), req)) { + tracker->SetStatus(ActionStatus::kClientError); return err; } @@ -108,41 +112,54 @@ std::shared_ptr AudioTranscriptionsHandler // 2. Validate file path try { if (!std::filesystem::exists(req.filename)) { + tracker->SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Audio file not found", "'" + req.filename + "'"); } } catch (const std::filesystem::filesystem_error& ex) { + tracker->SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Invalid file path", ex.what()); } // 3. Resolve model std::string model_name = req.model; Model* model = nullptr; - GenAIModelInstance* loaded = nullptr; + ModelLoadManager::LoadedModelLease loaded; if (auto err = ResolveModel(model_name, model, loaded)) { + tracker->SetStatus(ActionStatus::kClientError); return err; } - tracker.SetModelId(model_name); + tracker->SetModelId(model_name); // 4. Build an OPENAI_JSON-tagged TEXT request item — pass original body directly Request session_request; BuildOpenAIJsonRequest(body_str->c_str(), session_request); + // The session and the inference it drives are indirect children of this route. + auto session_ctx = route_ctx.AsIndirect(); + // 5. Dispatch to streaming or non-streaming try { - AudioSession session(*model, *loaded, ctx_.logger, ctx_.telemetry); + AudioSession session(*model, *loaded, ctx_.logger, ctx_.telemetry, true); + loaded.Release(); + { + ActionTracker create_tracker(Action::kSessionCreate, ctx_.telemetry, session_ctx); + create_tracker.SetModelId(model_name); + create_tracker.SetStatus(ActionStatus::kSuccess); + } + session.SetRequestContext(session_ctx); if (stream) { - tracker.SetStatus(ActionStatus::kSuccess); - return HandleStreaming(std::move(session), std::move(session_request)); + // The route action is recorded by the streaming thread on completion. + return HandleStreaming(std::move(session), std::move(session_request), std::move(tracker)); } else { SessionRegistration reg(ctx_.session_manager, session); auto response = HandleNonStreaming(session, session_request); - tracker.SetStatus(ActionStatus::kSuccess); + tracker->SetStatus(ActionStatus::kSuccess); return response; } } catch (const std::exception& ex) { - tracker.RecordException(ex); + tracker->RecordException(ex); ctx_.logger.Log(LogLevel::Error, fmt::format("Audio transcription inference failed: {}", ex.what())); return ErrorResponse(Status::CODE_500, "Inference failed", ex.what()); } catch (...) { @@ -177,18 +194,21 @@ std::shared_ptr AudioTranscriptionsHandler } std::shared_ptr AudioTranscriptionsHandler::HandleStreaming( - AudioSession&& session, Request session_request) { + AudioSession&& session, Request session_request, std::unique_ptr route_tracker) { auto body = std::make_shared(); auto body_ptr = body; auto& logger = ctx_.logger; - auto& tracker = ctx_.thread_tracker; + auto& thread_tracker = ctx_.thread_tracker; + auto stream_done = std::make_shared>(false); + auto stream_request = std::make_shared(std::move(session_request)); std::thread streaming_thread([bg_session = std::move(session), body_ptr, &logger, - req = std::move(session_request), &tracker, - &session_manager = ctx_.session_manager]() mutable { - SessionRegistration reg(session_manager, bg_session); - + req = stream_request, &thread_tracker, + route_tracker = std::move(route_tracker), + &session_manager = ctx_.session_manager, + stream_done]() mutable { try { + SessionRegistration reg(session_manager, bg_session); fl::Response bg_response; // Callback receives OPENAI_JSON-tagged TextItem chunks from AudioSession — just wrap in SSE framing. @@ -201,7 +221,9 @@ std::shared_ptr AudioTranscriptionsHandler if (item->type == FOUNDRY_LOCAL_ITEM_TEXT) { auto& text_item = static_cast(*item); - body_ptr->Push("data: " + text_item.text + "\n\n"); + if (!body_ptr->Push("data: " + text_item.text + "\n\n")) { + return 1; + } } else { logger.Log(LogLevel::Error, fmt::format("Unexpected item type {} in audio streaming callback", @@ -212,23 +234,40 @@ std::shared_ptr AudioTranscriptionsHandler }; bg_session.SetStreamingCallback(callback_fn); - bg_session.ProcessRequest(req, bg_response); + bg_session.ProcessRequest(*req, bg_response); + if (req->canceled.load(std::memory_order_relaxed)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "audio transcription stream cancelled"); + } // Send terminal event - body_ptr->Push("data: [DONE]\n\n"); + if (!body_ptr->Push("data: [DONE]\n\n")) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "audio transcription stream cancelled"); + } + + if (route_tracker) { + route_tracker->SetStatus(ActionStatus::kSuccess); + } } catch (const std::exception& ex) { logger.Log(LogLevel::Error, fmt::format("Audio streaming transcription failed: {}", ex.what())); // Push error to stream so client doesn't hang nlohmann::json error = {{"error", {{"message", ex.what()}}}}; body_ptr->Push("data: " + error.dump() + "\n\n"); + + if (route_tracker) { + route_tracker->RecordException(ex); + } } body_ptr->Finish(); - tracker.Remove(std::this_thread::get_id()); + stream_done->store(true, std::memory_order_release); + thread_tracker.NotifyCompleted(); }); - tracker.Track(std::move(streaming_thread)); + thread_tracker.Track(std::move(streaming_thread), stream_done, [body, stream_request] { + body->Abort(); + stream_request->canceled.store(true, std::memory_order_relaxed); + }); auto response = oatpp::web::protocol::http::outgoing::Response::createShared(Status::CODE_200, body); response->putHeader("Content-Type", "text/event-stream"); diff --git a/sdk_v2/cpp/src/service/audio_transcriptions_handler.h b/sdk_v2/cpp/src/service/audio_transcriptions_handler.h index 5dffeaa90..9f25345fa 100644 --- a/sdk_v2/cpp/src/service/audio_transcriptions_handler.h +++ b/sdk_v2/cpp/src/service/audio_transcriptions_handler.h @@ -4,6 +4,7 @@ #ifdef FOUNDRY_LOCAL_HAS_WEB_SERVICE +#include "inferencing/model_load_manager.h" #include "service/handler_utils.h" #include @@ -15,7 +16,7 @@ struct ServiceContext; struct AudioTranscriptionRequest; class AudioSession; class Model; -class GenAIModelInstance; +class ActionTracker; struct Request; // ======================================================================== @@ -35,7 +36,7 @@ class AudioTranscriptionsHandler : public HttpRequestHandler { /// Look up model in catalog and verify it's loaded and supports audio. std::shared_ptr ResolveModel(const std::string& model_name, - Model*& model, GenAIModelInstance*& loaded); + Model*& model, ModelLoadManager::LoadedModelLease& loaded); /// Build a Request with an OPENAI_JSON-tagged TEXT item from the original body string. void BuildOpenAIJsonRequest(const std::string& body, Request& session_request); @@ -43,8 +44,11 @@ class AudioTranscriptionsHandler : public HttpRequestHandler { /// Non-streaming inference — processes request, extracts the OPENAI_JSON-tagged TEXT response. std::shared_ptr HandleNonStreaming(AudioSession& session, Request& session_request); - /// Streaming inference — runs inference in background thread with SSE output. - std::shared_ptr HandleStreaming(AudioSession&& session, Request session_request); + /// Streaming inference — runs inference in a background thread with SSE output. + /// Takes ownership of the route ActionTracker so the route action records when + /// the stream finishes (real duration + terminal status). + std::shared_ptr HandleStreaming(AudioSession&& session, Request session_request, + std::unique_ptr route_tracker); ServiceContext& ctx_; }; diff --git a/sdk_v2/cpp/src/service/chat_completions_handler.cc b/sdk_v2/cpp/src/service/chat_completions_handler.cc index fcf13fb29..b4079ba5f 100644 --- a/sdk_v2/cpp/src/service/chat_completions_handler.cc +++ b/sdk_v2/cpp/src/service/chat_completions_handler.cc @@ -7,6 +7,7 @@ #include "c_api_types.h" #include "catalog.h" #include "contracts/chat_completions.h" +#include "exception.h" #include "inferencing/generative/chat/chat_session.h" #include "inferencing/model_load_manager.h" #include "inferencing/session/session.h" @@ -60,13 +61,13 @@ std::shared_ptr ChatCompletionsHandler::Pa } std::shared_ptr ChatCompletionsHandler::ResolveModel( - const std::string& model_name, Model*& model, GenAIModelInstance*& loaded) { + const std::string& model_name, Model*& model, ModelLoadManager::LoadedModelLease& loaded) { model = ctx_.catalog.GetModelVariant(model_name); if (!model) { return ErrorResponse(Status::CODE_404, "Model not found", "No model matching '" + model_name + "'"); } - loaded = ctx_.model_load_manager.GetLoadedModel(model->Id()); + loaded = ctx_.model_load_manager.AcquireLoadedModel(model->Id()); if (!loaded) { return ErrorResponse(Status::CODE_400, "Model not loaded", "Model '" + model_name + "' must be loaded before inference"); @@ -104,10 +105,12 @@ void ChatCompletionsHandler::BuildOpenAIJsonRequest(const std::string& body, con std::shared_ptr ChatCompletionsHandler::handle( const std::shared_ptr& request) { - ActionTracker tracker(Action::kOpenAIChatCompletions, ctx_.telemetry); + auto route_ctx = InvocationContext::Direct(GetUserAgent(request)); + auto tracker = std::make_unique(Action::kOpenAIChatCompletions, ctx_.telemetry, route_ctx); auto body_str = request->readBodyToString(); if (!body_str || body_str->empty()) { + tracker->SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Empty request body"); } @@ -117,6 +120,7 @@ std::shared_ptr ChatCompletionsHandler::ha // telemetry. How much do we care about that? Is it worth the double parsing? ChatCompletionRequest req; if (auto err = ParseAndValidateRequest(body_str->c_str(), req)) { + tracker->SetStatus(ActionStatus::kClientError); return err; } @@ -128,12 +132,13 @@ std::shared_ptr ChatCompletionsHandler::ha // 2. Resolve model std::string model_name = req.model; Model* model = nullptr; - GenAIModelInstance* loaded = nullptr; + ModelLoadManager::LoadedModelLease loaded; if (auto err = ResolveModel(model_name, model, loaded)) { + tracker->SetStatus(ActionStatus::kClientError); return err; } - tracker.SetModelId(model_name); + tracker->SetModelId(model_name); // 3. Build an OPENAI_JSON-tagged TEXT request item. Request session_request; @@ -145,21 +150,34 @@ std::shared_ptr ChatCompletionsHandler::ha include_usage_in_stream = req.stream_options->include_usage; } + // The session and the inference it drives happen as a consequence of this + // route, so they are indirect and reuse the route's correlation id. + auto session_ctx = route_ctx.AsIndirect(); + // 6. Run inference via ChatSession try { - ChatSession session(*model, *loaded, ctx_.logger, ctx_.telemetry); + ChatSession session(*model, *loaded, ctx_.logger, ctx_.telemetry, true); + loaded.Release(); + { + ActionTracker create_tracker(Action::kSessionCreate, ctx_.telemetry, session_ctx); + create_tracker.SetModelId(model_name); + create_tracker.SetStatus(ActionStatus::kSuccess); + } + session.SetRequestContext(session_ctx); if (stream) { - tracker.SetStatus(ActionStatus::kSuccess); - return HandleStreaming(std::move(session), std::move(session_request), include_usage_in_stream); + // The route action is recorded by the streaming thread when the stream + // finishes, so don't set its status here — move the tracker into the thread. + return HandleStreaming(std::move(session), std::move(session_request), include_usage_in_stream, + std::move(tracker)); } else { SessionRegistration reg(ctx_.session_manager, session); auto response = HandleNonStreaming(session, session_request); - tracker.SetStatus(ActionStatus::kSuccess); + tracker->SetStatus(ActionStatus::kSuccess); return response; } } catch (const std::exception& ex) { - tracker.RecordException(ex); + tracker->RecordException(ex); ctx_.logger.Log(LogLevel::Error, fmt::format("Chat completion inference failed: {}", ex.what())); return ErrorResponse(Status::CODE_500, "Inference failed", ex.what()); } @@ -189,19 +207,23 @@ std::shared_ptr ChatCompletionsHandler::Ha } std::shared_ptr ChatCompletionsHandler::HandleStreaming( - ChatSession&& session, Request session_request, bool include_usage) { + ChatSession&& session, Request session_request, bool include_usage, + std::unique_ptr route_tracker) { auto body = std::make_shared(); auto body_ptr = body; auto& logger = ctx_.logger; - auto& tracker = ctx_.thread_tracker; + auto& thread_tracker = ctx_.thread_tracker; + auto stream_done = std::make_shared>(false); + auto stream_request = std::make_shared(std::move(session_request)); std::thread streaming_thread([bg_session = std::move(session), body_ptr, &logger, - req = std::move(session_request), - include_usage, &tracker, - &session_manager = ctx_.session_manager]() mutable { - SessionRegistration reg(session_manager, bg_session); - + req = stream_request, + include_usage, &thread_tracker, + route_tracker = std::move(route_tracker), + &session_manager = ctx_.session_manager, + stream_done]() mutable { try { + SessionRegistration reg(session_manager, bg_session); fl::Response bg_response; // Callback receives OPENAI_JSON-tagged TextItem chunks from ChatSession — just wrap in SSE framing. @@ -214,7 +236,9 @@ std::shared_ptr ChatCompletionsHandler::Ha if (item->type == FOUNDRY_LOCAL_ITEM_TEXT) { auto& text_item = static_cast(*item); - body_ptr->Push("data: " + text_item.text + "\n\n"); + if (!body_ptr->Push("data: " + text_item.text + "\n\n")) { + return 1; + } } else { logger.Log(LogLevel::Error, fmt::format("Unexpected item type {} in chat streaming callback", static_cast(item->type))); @@ -224,7 +248,10 @@ std::shared_ptr ChatCompletionsHandler::Ha }; bg_session.SetStreamingCallback(callback_fn); - bg_session.ProcessRequest(req, bg_response); + bg_session.ProcessRequest(*req, bg_response); + if (req->canceled.load(std::memory_order_relaxed)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "chat completion stream cancelled"); + } // Usage chunk — only if stream_options.include_usage was true if (include_usage) { @@ -244,22 +271,43 @@ std::shared_ptr ChatCompletionsHandler::Ha usage.total_tokens = static_cast(bg_response.usage.total_tokens); usage_chunk.usage = std::move(usage); - body_ptr->Push("data: " + nlohmann::json(usage_chunk).dump() + "\n\n"); + if (!body_ptr->Push("data: " + nlohmann::json(usage_chunk).dump() + "\n\n")) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "chat completion stream cancelled"); + } } - body_ptr->Push("data: [DONE]\n\n"); + if (!body_ptr->Push("data: [DONE]\n\n")) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "chat completion stream cancelled"); + } + + // Inference streamed to completion — record the route action as a success. + if (route_tracker) { + route_tracker->SetStatus(ActionStatus::kSuccess); + } } catch (const std::exception& ex) { nlohmann::json err = { {"error", {{"message", ex.what()}, {"type", "server_error"}, {"param", nullptr}, {"code", nullptr}}}, }; body_ptr->Push("data: " + err.dump() + "\n\n"); + + // Mid-stream failure: record the exception. The route action keeps its + // default kFailure status. + if (route_tracker) { + route_tracker->RecordException(ex); + } } body_ptr->Finish(); - tracker.Remove(std::this_thread::get_id()); + // route_tracker is destroyed with this closure once the thread completes, + // recording the route action with the full streaming duration and final status. + stream_done->store(true, std::memory_order_release); + thread_tracker.NotifyCompleted(); }); - tracker.Track(std::move(streaming_thread)); + thread_tracker.Track(std::move(streaming_thread), stream_done, [body, stream_request] { + body->Abort(); + stream_request->canceled.store(true, std::memory_order_relaxed); + }); auto response = oatpp::web::protocol::http::outgoing::Response::createShared( Status::CODE_200, body); diff --git a/sdk_v2/cpp/src/service/chat_completions_handler.h b/sdk_v2/cpp/src/service/chat_completions_handler.h index ed483c4d0..bb6a010af 100644 --- a/sdk_v2/cpp/src/service/chat_completions_handler.h +++ b/sdk_v2/cpp/src/service/chat_completions_handler.h @@ -4,6 +4,7 @@ #ifdef FOUNDRY_LOCAL_HAS_WEB_SERVICE +#include "inferencing/model_load_manager.h" #include "service/handler_utils.h" #include @@ -15,7 +16,7 @@ struct ServiceContext; struct ChatCompletionRequest; class ChatSession; class Model; -class GenAIModelInstance; +class ActionTracker; struct Request; // ======================================================================== @@ -37,7 +38,7 @@ class ChatCompletionsHandler : public HttpRequestHandler { /// Look up model in catalog and verify it's loaded. Sets output pointers. /// Returns an error response on failure, nullptr on success. std::shared_ptr ResolveModel(const std::string& model_name, - Model*& model, GenAIModelInstance*& loaded); + Model*& model, ModelLoadManager::LoadedModelLease& loaded); /// Build a Request with an OPENAI_JSON-tagged TEXT item from the original body string. /// Populates catalog defaults and model-specific options (tool_call_start/end). @@ -47,9 +48,13 @@ class ChatCompletionsHandler : public HttpRequestHandler { /// Non-streaming inference — processes request, extracts the OPENAI_JSON-tagged TEXT response. std::shared_ptr HandleNonStreaming(ChatSession& session, Request& session_request); - /// Streaming inference — runs inference in background thread with SSE output. + /// Streaming inference — runs inference in a background thread with SSE output. + /// Takes ownership of the route ActionTracker so the route action is recorded + /// when the stream actually finishes (real duration + terminal status), + /// instead of when this handler returns. std::shared_ptr HandleStreaming(ChatSession&& session, Request session_request, - bool include_usage); + bool include_usage, + std::unique_ptr route_tracker); ServiceContext& ctx_; }; diff --git a/sdk_v2/cpp/src/service/embeddings_handler.cc b/sdk_v2/cpp/src/service/embeddings_handler.cc index b5f734b07..59e5aee1f 100644 --- a/sdk_v2/cpp/src/service/embeddings_handler.cc +++ b/sdk_v2/cpp/src/service/embeddings_handler.cc @@ -28,10 +28,12 @@ class EmbeddingsHandler : public HttpRequestHandler { explicit EmbeddingsHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr handle(const std::shared_ptr& request) override { - ActionTracker tracker(Action::kOpenAIEmbeddings, ctx_.telemetry); + auto route_ctx = InvocationContext::Direct(GetUserAgent(request)); + ActionTracker tracker(Action::kOpenAIEmbeddings, ctx_.telemetry, route_ctx); auto body_str = request->readBodyToString(); if (!body_str || body_str->empty()) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Empty request body"); } @@ -41,6 +43,7 @@ class EmbeddingsHandler : public HttpRequestHandler { auto j = nlohmann::json::parse(*body_str); req = j.get(); } catch (const nlohmann::json::exception& ex) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Invalid JSON", ex.what()); } @@ -53,6 +56,7 @@ class EmbeddingsHandler : public HttpRequestHandler { } if (inputs.empty()) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "\"input\" must not be empty"); } @@ -60,19 +64,31 @@ class EmbeddingsHandler : public HttpRequestHandler { std::string model_name = req.model; auto* model = ctx_.catalog.GetModelVariant(model_name); if (!model) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_404, "Model not found", model_name); } - auto* loaded = ctx_.model_load_manager.GetLoadedModel(model->Id()); + auto loaded = ctx_.model_load_manager.AcquireLoadedModel(model->Id()); if (!loaded) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Model not loaded", model_name); } tracker.SetModelId(model_name); + // The session and the inference it drives are indirect children of this route. + auto session_ctx = route_ctx.AsIndirect(); + // 4. Create session and process each input try { - EmbeddingsSession session(*model, *loaded, ctx_.logger, ctx_.telemetry); + EmbeddingsSession session(*model, *loaded, ctx_.logger, ctx_.telemetry, true); + loaded.Release(); + { + ActionTracker create_tracker(Action::kSessionCreate, ctx_.telemetry, session_ctx); + create_tracker.SetModelId(model_name); + create_tracker.SetStatus(ActionStatus::kSuccess); + } + session.SetRequestContext(session_ctx); SessionRegistration reg(ctx_.session_manager, session); Request session_request; diff --git a/sdk_v2/cpp/src/service/handler_utils.h b/sdk_v2/cpp/src/service/handler_utils.h index e25528fa3..0eda909b6 100644 --- a/sdk_v2/cpp/src/service/handler_utils.h +++ b/sdk_v2/cpp/src/service/handler_utils.h @@ -10,6 +10,10 @@ #include #include +#include "telemetry/telemetry_redaction.h" + +#include +#include #include #include #include @@ -62,6 +66,47 @@ inline std::string GenerateCompletionId(const std::string& prefix) { return ss.str(); } +/// Extract the User-Agent header from an incoming request ("" if absent), for +/// attribution on the telemetry events the request drives. +inline std::string GetUserAgent(const std::shared_ptr& request) { + if (!request) { + return {}; + } + auto ua = request->getHeader("User-Agent"); + if (!ua) { + return {}; + } + std::string scrubbed; + scrubbed.reserve(std::min(ua->size(), 256)); + size_t run_length = 0; + for (char ch : *ua) { + unsigned char c = static_cast(ch); + bool safe = std::isalnum(c) || ch == ' ' || ch == '.' || ch == '-' || ch == '_' || ch == '/' || + ch == '(' || ch == ')' || ch == ';'; + if (!safe) { + scrubbed.push_back('?'); + run_length = 0; + continue; + } + + if (std::isalnum(c)) { + ++run_length; + if (run_length > 32) { + scrubbed.push_back('?'); + continue; + } + } else { + run_length = 0; + } + scrubbed.push_back(ch); + } + constexpr size_t kMaxUserAgentLength = 256; + if (scrubbed.size() > kMaxUserAgentLength) { + scrubbed.resize(kMaxUserAgentLength); + } + return scrubbed; +} + // ======================================================================== // SSE stream body — feeds token-by-token SSE events to oatpp's chunked // transfer encoding. A producer thread pushes formatted SSE strings into @@ -70,13 +115,17 @@ inline std::string GenerateCompletionId(const std::string& prefix) { class SseStreamBody : public oatpp::web::protocol::http::outgoing::Body { public: - SseStreamBody() : done_(false) {} + SseStreamBody() : done_(false), aborted_(false) {} /// Push a formatted SSE event (e.g. "data: {...}\n\n") into the queue. - void Push(std::string chunk) { + bool Push(std::string chunk) { std::lock_guard lock(mutex_); + if (aborted_) { + return false; + } queue_.push(std::move(chunk)); cv_.notify_one(); + return true; } /// Signal that no more data will be pushed. @@ -86,6 +135,18 @@ class SseStreamBody : public oatpp::web::protocol::http::outgoing::Body { cv_.notify_one(); } + void Abort() { + std::lock_guard lock(mutex_); + aborted_ = true; + done_ = true; + cv_.notify_all(); + } + + bool IsAborted() const { + std::lock_guard lock(mutex_); + return aborted_; + } + // -- Body interface -- oatpp::v_io_size read(void* buffer, v_buff_size count, oatpp::async::Action& /*action*/) override { @@ -133,10 +194,11 @@ class SseStreamBody : public oatpp::web::protocol::http::outgoing::Body { v_int64 getKnownSize() override { return -1; } // unknown → chunked transfer private: - std::mutex mutex_; + mutable std::mutex mutex_; std::condition_variable cv_; std::queue queue_; bool done_; + bool aborted_; }; } // namespace fl diff --git a/sdk_v2/cpp/src/service/models_handlers.cc b/sdk_v2/cpp/src/service/models_handlers.cc index 93988d3b6..b2bb65f51 100644 --- a/sdk_v2/cpp/src/service/models_handlers.cc +++ b/sdk_v2/cpp/src/service/models_handlers.cc @@ -24,7 +24,10 @@ class ListLoadedModelsHandler : public HttpRequestHandler { public: explicit ListLoadedModelsHandler(ServiceContext& ctx) : ctx_(ctx) {} - std::shared_ptr handle(const std::shared_ptr&) override { + std::shared_ptr handle(const std::shared_ptr& request) override { + ActionTracker tracker(Action::kModelList, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); + auto loaded = ctx_.catalog.GetLoadedModels(); nlohmann::json names = nlohmann::json::array(); @@ -32,6 +35,7 @@ class ListLoadedModelsHandler : public HttpRequestHandler { names.push_back(model->Id()); } + tracker.SetStatus(ActionStatus::kSuccess); return JsonResponse(Status::CODE_200, names); } @@ -48,10 +52,12 @@ class LoadModelHandler : public HttpRequestHandler { explicit LoadModelHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr handle(const std::shared_ptr& request) override { - ActionTracker tracker(Action::kModelLoad, ctx_.telemetry); + ActionTracker tracker(Action::kModelLoad, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); auto name_raw = request->getPathVariable("name"); if (!name_raw) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Missing model name"); } @@ -59,6 +65,7 @@ class LoadModelHandler : public HttpRequestHandler { auto* model = ctx_.catalog.GetModel(name); if (!model) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_404, "Model not found", "No model matching '" + name + "'"); } @@ -69,6 +76,7 @@ class LoadModelHandler : public HttpRequestHandler { } if (!model->IsCached()) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Model not cached", "Model must be downloaded before loading"); } @@ -101,10 +109,12 @@ class UnloadModelHandler : public HttpRequestHandler { explicit UnloadModelHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr handle(const std::shared_ptr& request) override { - ActionTracker tracker(Action::kModelUnload, ctx_.telemetry); + ActionTracker tracker(Action::kModelUnload, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); auto name_raw = request->getPathVariable("name"); if (!name_raw) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Missing model name"); } @@ -112,6 +122,7 @@ class UnloadModelHandler : public HttpRequestHandler { auto* model = ctx_.catalog.GetModel(name); if (!model) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_404, "Model not found", "No model matching '" + name + "'"); } @@ -149,8 +160,9 @@ class OpenAIListModelsHandler : public HttpRequestHandler { public: explicit OpenAIListModelsHandler(ServiceContext& ctx) : ctx_(ctx) {} - std::shared_ptr handle(const std::shared_ptr&) override { - ActionTracker tracker(Action::kOpenAIModelList, ctx_.telemetry); + std::shared_ptr handle(const std::shared_ptr& request) override { + ActionTracker tracker(Action::kOpenAIModelList, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); auto models = ctx_.catalog.ListModels(); nlohmann::json data = nlohmann::json::array(); @@ -203,10 +215,12 @@ class OpenAIRetrieveModelHandler : public HttpRequestHandler { explicit OpenAIRetrieveModelHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr handle(const std::shared_ptr& request) override { - ActionTracker tracker(Action::kOpenAIModelRetrieve, ctx_.telemetry); + ActionTracker tracker(Action::kOpenAIModelRetrieve, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); auto name_raw = request->getPathVariable("name"); if (!name_raw) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Missing model name"); } @@ -214,6 +228,7 @@ class OpenAIRetrieveModelHandler : public HttpRequestHandler { auto* model = ctx_.catalog.GetModelVariant(name); if (!model) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_404, "Model not found", "No model matching '" + name + "'"); } diff --git a/sdk_v2/cpp/src/service/responses_handler.cc b/sdk_v2/cpp/src/service/responses_handler.cc index 0fdf6d8f7..cd9730227 100644 --- a/sdk_v2/cpp/src/service/responses_handler.cc +++ b/sdk_v2/cpp/src/service/responses_handler.cc @@ -6,6 +6,7 @@ #include "c_api_types.h" #include "catalog.h" +#include "exception.h" #include "inferencing/generative/chat/chat_session.h" #include "inferencing/generative/openresponses/response_converter.h" #include "inferencing/generative/openresponses/response_store.h" @@ -79,13 +80,13 @@ std::shared_ptr ResponsesHandler::ParseAnd } std::shared_ptr ResponsesHandler::ResolveModel( - const std::string& model_name, Model*& model, GenAIModelInstance*& loaded) { + const std::string& model_name, Model*& model, ModelLoadManager::LoadedModelLease& loaded) { model = ctx_.catalog.GetModelVariant(model_name); if (!model) { return ErrorResponse(Status::CODE_404, "Model not found", "No model matching '" + model_name + "'"); } - loaded = ctx_.model_load_manager.GetLoadedModel(model->Id()); + loaded = ctx_.model_load_manager.AcquireLoadedModel(model->Id()); if (!loaded) { return ErrorResponse(Status::CODE_400, "Model not loaded", "Model '" + model_name + "' must be loaded before inference"); @@ -130,10 +131,12 @@ void ResponsesHandler::LoadPreviousContext(const ResponseCreateParams& params, std::shared_ptr ResponsesHandler::handle( const std::shared_ptr& request) { - ActionTracker tracker(Action::kOpenAIResponsesCreate, ctx_.telemetry); + auto route_ctx = InvocationContext::Direct(GetUserAgent(request)); + auto tracker = std::make_unique(Action::kOpenAIResponsesCreate, ctx_.telemetry, route_ctx); auto body_str = request->readBodyToString(); if (!body_str || body_str->empty()) { + tracker->SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Empty request body"); } @@ -141,6 +144,7 @@ std::shared_ptr ResponsesHandler::handle( nlohmann::json req_json; ResponseCreateParams params; if (auto err = ParseAndValidateRequest(body_str->c_str(), req_json, params)) { + tracker->SetStatus(ActionStatus::kClientError); return err; } @@ -153,12 +157,13 @@ std::shared_ptr ResponsesHandler::handle( // 2. Resolve model std::string model_name = params.model; Model* model = nullptr; - GenAIModelInstance* loaded = nullptr; + ModelLoadManager::LoadedModelLease loaded; if (auto err = ResolveModel(model_name, model, loaded)) { + tracker->SetStatus(ActionStatus::kClientError); return err; } - tracker.SetModelId(model_name); + tracker->SetModelId(model_name); // 3. Load previous context if chaining via previous_response_id const nlohmann::json* previous_input = nullptr; @@ -173,9 +178,10 @@ std::shared_ptr ResponsesHandler::handle( std::string response_id = ResponseConverter::GenerateId("resp"); std::unique_ptr session; + const std::string resolved_model_id = model->Id(); if (params.previous_response_id) { - session = ctx_.session_manager.CheckOut(*params.previous_response_id); + session = ctx_.session_manager.CheckOut(*params.previous_response_id, resolved_model_id); if (session) { ctx_.logger.Log(LogLevel::Information, @@ -193,10 +199,19 @@ std::shared_ptr ResponsesHandler::handle( // happens here in the handler that owns the session lifetime. std::string tools_json = ResponseConverter::ExtractResponsesToolDefinitions(params, session_request); + // The session and the inference it drives happen as a consequence of this + // route, so they are indirect and reuse the route's correlation id. + auto session_ctx = route_ctx.AsIndirect(); + try { if (!session) { - session = std::make_unique(*model, *loaded, ctx_.logger, ctx_.telemetry); + ActionTracker create_tracker(Action::kSessionCreate, ctx_.telemetry, session_ctx); + create_tracker.SetModelId(model_name); + session = std::make_unique(*model, *loaded, ctx_.logger, ctx_.telemetry, true); + loaded.Release(); + create_tracker.SetStatus(ActionStatus::kSuccess); } + session->SetRequestContext(session_ctx); // Sessions can be reused via previous_response_id; clear any stale tool defs from the prior // turn before applying this request's tools so the request stays self-contained. @@ -208,22 +223,23 @@ std::shared_ptr ResponsesHandler::handle( if (params.stream) { ctx_.logger.Log(LogLevel::Debug, fmt::format("Creating streaming response {} for model {}", response_id, model_name)); - tracker.SetStatus(ActionStatus::kSuccess); - return HandleStreaming(std::move(session), std::move(session_request), model_name, - response_id, created_at, params, req_json); + // The route action is recorded by the streaming thread when the stream + // finishes — move the tracker in rather than marking success now. + return HandleStreaming(std::move(session), std::move(session_request), model_name, resolved_model_id, + response_id, created_at, params, req_json, std::move(tracker)); } else { ctx_.logger.Log(LogLevel::Debug, fmt::format("Creating response {} for model {}", response_id, model_name)); - auto response = HandleNonStreaming(std::move(session), session_request, model_name, + auto response = HandleNonStreaming(std::move(session), session_request, model_name, resolved_model_id, response_id, created_at, params, req_json); - tracker.SetStatus(ActionStatus::kSuccess); + tracker->SetStatus(ActionStatus::kSuccess); return response; } } catch (const std::exception& ex) { - tracker.RecordException(ex); + tracker->RecordException(ex); ctx_.logger.Log(LogLevel::Error, fmt::format("Response {} failed: {}", response_id, ex.what())); @@ -238,7 +254,7 @@ std::shared_ptr ResponsesHandler::handle( std::shared_ptr ResponsesHandler::HandleNonStreaming( std::unique_ptr session, Request& session_request, - const std::string& model_name, const std::string& response_id, + const std::string& model_name, const std::string& model_id, const std::string& response_id, int64_t created_at, const ResponseCreateParams& params, const nlohmann::json& req_json) { SessionRegistration reg(ctx_.session_manager, *session); @@ -270,7 +286,7 @@ std::shared_ptr ResponsesHandler::HandleNo session->SetStreamingCallback(nullptr); // Cache the session for potential reuse on the next turn - ctx_.session_manager.CheckIn(response_id, std::move(session)); + ctx_.session_manager.CheckIn(response_id, std::move(session), model_id); } return JsonResponse(Status::CODE_200, response_json); @@ -278,9 +294,9 @@ std::shared_ptr ResponsesHandler::HandleNo std::shared_ptr ResponsesHandler::HandleStreaming( std::unique_ptr session, Request session_request, - const std::string& model_name, const std::string& response_id, + const std::string& model_name, const std::string& model_id, const std::string& response_id, int64_t created_at, const ResponseCreateParams& params, - const nlohmann::json& req_json) { + const nlohmann::json& req_json, std::unique_ptr route_tracker) { auto body = std::make_shared(); auto initial_response = ResponseConverter::BuildInitialResponseObject(response_id, created_at, model_name, params); @@ -316,19 +332,23 @@ std::shared_ptr ResponsesHandler::HandleSt auto& logger = ctx_.logger; auto& session_manager = ctx_.session_manager; auto& tracker = ctx_.thread_tracker; + auto stream_done = std::make_shared>(false); + auto stream_request = std::make_shared(std::move(session_request)); // Background thread is required: oatpp needs the Response returned immediately so it can // start writing SSE events. ProcessRequest blocks until generation completes. std::thread streaming_thread([body_ptr, &logger, &session_manager, session = std::move(session), - req = std::move(session_request), + req = stream_request, model_name, response_id, created_at, + model_id, should_store, &store, req_copy = std::move(req_copy), params_copy = std::move(params_copy), - &tracker]() mutable { - SessionRegistration reg(session_manager, *session); - + route_tracker = std::move(route_tracker), + &tracker, + stream_done]() mutable { + bool terminal_sent = false; int seq = 2; std::string full_text; // concatenation of all visible runs, used for output_text in completed_response @@ -347,9 +367,10 @@ std::shared_ptr ResponsesHandler::HandleSt // Items that have been *closed* (or, for the currently-open item at end-of-stream, finalized in place). // Used to construct the final `output[]` array for the response.completed event. std::vector closed_items; + std::optional reg; auto push_event = [&](const std::string& event_name, const StreamEvent& ev) { - body_ptr->Push("event: " + event_name + "\ndata: " + nlohmann::json(ev).dump() + "\n\n"); + return body_ptr->Push("event: " + event_name + "\ndata: " + nlohmann::json(ev).dump() + "\n\n"); }; auto close_current = [&]() { @@ -474,6 +495,7 @@ std::shared_ptr ResponsesHandler::HandleSt }; try { + reg.emplace(session_manager, *session); fl::Response bg_response; fl::Session::StreamingCallbackFn callback_fn = [&](flStreamingCallbackData event, void* /*user_data*/) -> int { fl::ItemQueue* queue = reinterpret_cast(event.item_queue); @@ -511,7 +533,9 @@ std::shared_ptr ResponsesHandler::HandleSt delta.output_index = current_output_index; delta.item_id = current_id; delta.delta = text_item->text; - push_event("response.reasoning.delta", delta); + if (!push_event("response.reasoning.delta", delta)) { + return 1; + } } else { full_text += text_item->text; @@ -522,7 +546,9 @@ std::shared_ptr ResponsesHandler::HandleSt text_delta.content_index = 0; text_delta.item_id = current_id; text_delta.delta = text_item->text; - push_event("response.output_text.delta", text_delta); + if (!push_event("response.output_text.delta", text_delta)) { + return 1; + } } return 0; @@ -530,10 +556,16 @@ std::shared_ptr ResponsesHandler::HandleSt session->SetStreamingCallback(callback_fn); - session->ProcessRequest(req, bg_response); + session->ProcessRequest(*req, bg_response); + if (req->canceled.load(std::memory_order_relaxed)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "response stream cancelled"); + } // Close whatever item is still open at end-of-generation so the SSE stream is well-formed. close_current(); + if (req->canceled.load(std::memory_order_relaxed)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "response stream cancelled"); + } auto completed_response = ResponseConverter::BuildResponseObject( response_id, created_at, model_name, params_copy, std::move(closed_items), full_text, bg_response.usage); @@ -542,7 +574,9 @@ std::shared_ptr ResponsesHandler::HandleSt completed.type = StreamEventType::kResponseCompleted; completed.sequence_number = seq++; completed.response = completed_response; - push_event("response.completed", completed); + if (!push_event("response.completed", completed)) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "response stream cancelled"); + } // Store if requested if (should_store) { @@ -551,13 +585,23 @@ std::shared_ptr ResponsesHandler::HandleSt store.Store(response_id, response_json, std::move(input_items)); // Deregister before caching — see non-streaming path comment. - reg.Release(); + reg->Release(); // Clear per-request streaming callback before caching — see non-streaming path comment. session->SetStreamingCallback(nullptr); // Cache the session for potential reuse on the next turn - session_manager.CheckIn(response_id, std::move(session)); + session_manager.CheckIn(response_id, std::move(session), model_id); + } + + if (!body_ptr->Push("data: [DONE]\n\n")) { + FL_THROW(FOUNDRY_LOCAL_ERROR_OPERATION_CANCELLED, "response stream cancelled"); + } + terminal_sent = true; + + // Streamed to completion — record the route action as a success. + if (route_tracker) { + route_tracker->SetStatus(ActionStatus::kSuccess); } } catch (const std::exception& ex) { @@ -572,16 +616,26 @@ std::shared_ptr ResponsesHandler::HandleSt failed.sequence_number = seq++; failed.response = error_response; body_ptr->Push("event: response.failed\ndata: " + nlohmann::json(failed).dump() + "\n\n"); + + // Mid-stream failure: record the exception; the route action keeps kFailure. + if (route_tracker) { + route_tracker->RecordException(ex); + } } - // Terminal event per spec - body_ptr->Push("data: [DONE]\n\n"); + if (!terminal_sent) { + body_ptr->Push("data: [DONE]\n\n"); + } body_ptr->Finish(); + stream_done->store(true, std::memory_order_release); + tracker.NotifyCompleted(); - tracker.Remove(std::this_thread::get_id()); }); - tracker.Track(std::move(streaming_thread)); + tracker.Track(std::move(streaming_thread), stream_done, [body, stream_request] { + body->Abort(); + stream_request->canceled.store(true, std::memory_order_relaxed); + }); auto response = oatpp::web::protocol::http::outgoing::Response::createShared( Status::CODE_200, body); @@ -598,10 +652,12 @@ GetResponseHandler::GetResponseHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr GetResponseHandler::handle( const std::shared_ptr& request) { - ActionTracker tracker(Action::kOpenAIResponsesGet, ctx_.telemetry); + ActionTracker tracker(Action::kOpenAIResponsesGet, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); auto id = request->getPathVariable("id"); if (!id) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Missing response ID"); } @@ -609,6 +665,7 @@ std::shared_ptr GetResponseHandler::handle auto response = ctx_.response_store.Get(id->c_str()); if (!response) { + tracker.SetStatus(ActionStatus::kClientError); nlohmann::json error_body = { {"error", { {"message", "The response '" + std::string(id->c_str()) + "' does not exist."}, @@ -633,7 +690,8 @@ ListResponsesHandler::ListResponsesHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr ListResponsesHandler::handle( const std::shared_ptr& request) { - ActionTracker tracker(Action::kOpenAIResponsesList, ctx_.telemetry); + ActionTracker tracker(Action::kOpenAIResponsesList, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); // Parse query parameters auto limit_str = request->getQueryParameter("limit", "20"); @@ -688,10 +746,12 @@ DeleteResponseHandler::DeleteResponseHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr DeleteResponseHandler::handle( const std::shared_ptr& request) { - ActionTracker tracker(Action::kOpenAIResponsesDelete, ctx_.telemetry); + ActionTracker tracker(Action::kOpenAIResponsesDelete, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); auto id = request->getPathVariable("id"); if (!id) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Missing response ID"); } @@ -699,6 +759,7 @@ std::shared_ptr DeleteResponseHandler::han bool deleted = ctx_.response_store.Delete(id->c_str()); if (!deleted) { + tracker.SetStatus(ActionStatus::kClientError); nlohmann::json error_body = { {"error", { {"message", "The response '" + std::string(id->c_str()) + "' does not exist."}, @@ -733,10 +794,12 @@ GetInputItemsHandler::GetInputItemsHandler(ServiceContext& ctx) : ctx_(ctx) {} std::shared_ptr GetInputItemsHandler::handle( const std::shared_ptr& request) { - ActionTracker tracker(Action::kOpenAIResponsesGetInputItems, ctx_.telemetry); + ActionTracker tracker(Action::kOpenAIResponsesGetInputItems, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); auto id = request->getPathVariable("id"); if (!id) { + tracker.SetStatus(ActionStatus::kClientError); return ErrorResponse(Status::CODE_400, "Missing response ID"); } @@ -745,6 +808,7 @@ std::shared_ptr GetInputItemsHandler::hand // Check that response exists auto response = ctx_.response_store.Get(id->c_str()); if (!response) { + tracker.SetStatus(ActionStatus::kClientError); nlohmann::json error_body = { {"error", { {"message", "The response '" + std::string(id->c_str()) + "' does not exist."}, diff --git a/sdk_v2/cpp/src/service/responses_handler.h b/sdk_v2/cpp/src/service/responses_handler.h index da9f5f691..7ce6af59c 100644 --- a/sdk_v2/cpp/src/service/responses_handler.h +++ b/sdk_v2/cpp/src/service/responses_handler.h @@ -4,6 +4,7 @@ #ifdef FOUNDRY_LOCAL_HAS_WEB_SERVICE +#include "inferencing/model_load_manager.h" #include "service/handler_utils.h" #include @@ -15,7 +16,7 @@ struct ServiceContext; struct Request; class ChatSession; class Model; -class GenAIModelInstance; +class ActionTracker; namespace responses { struct ResponseCreateParams; @@ -43,7 +44,7 @@ class ResponsesHandler : public HttpRequestHandler { /// Look up model in catalog and verify it's loaded. Sets output pointers. /// Returns an error response on failure, nullptr on success. std::shared_ptr ResolveModel(const std::string& model_name, - Model*& model, GenAIModelInstance*& loaded); + Model*& model, ModelLoadManager::LoadedModelLease& loaded); /// Load previous response context when chaining via previous_response_id. /// The json storage objects are passed by reference because the output pointers alias into them. @@ -56,16 +57,17 @@ class ResponsesHandler : public HttpRequestHandler { // --- Inference dispatch --- std::shared_ptr HandleNonStreaming(std::unique_ptr session, Request& session_request, - const std::string& model_name, const std::string& response_id, - int64_t created_at, + const std::string& model_name, const std::string& model_id, + const std::string& response_id, int64_t created_at, const responses::ResponseCreateParams& params, const nlohmann::json& req_json); std::shared_ptr HandleStreaming(std::unique_ptr session, Request session_request, - const std::string& model_name, const std::string& response_id, - int64_t created_at, + const std::string& model_name, const std::string& model_id, + const std::string& response_id, int64_t created_at, const responses::ResponseCreateParams& params, - const nlohmann::json& req_json); + const nlohmann::json& req_json, + std::unique_ptr route_tracker); ServiceContext& ctx_; }; diff --git a/sdk_v2/cpp/src/service/web_service.cc b/sdk_v2/cpp/src/service/web_service.cc index 683ff7817..8672bf25c 100644 --- a/sdk_v2/cpp/src/service/web_service.cc +++ b/sdk_v2/cpp/src/service/web_service.cc @@ -13,6 +13,7 @@ #include "service/handler_utils.h" #include "service/models_handlers.h" #include "service/responses_handler.h" +#include "telemetry/telemetry_action_tracker.h" #include #include @@ -20,9 +21,11 @@ #include #include #include +#include #include #include +#include #include #include @@ -134,7 +137,9 @@ class StatusHandler : public HttpRequestHandler { public: explicit StatusHandler(ServiceContext& ctx) : ctx_(ctx) {} - std::shared_ptr handle(const std::shared_ptr&) override { + std::shared_ptr handle(const std::shared_ptr& request) override { + MaybeRecordStatusTelemetry(request); + nlohmann::json body = { {"modelCachePath", ctx_.model_cache_dir}, {"endpoints", ctx_.bound_urls}, @@ -149,7 +154,32 @@ class StatusHandler : public HttpRequestHandler { } private: + // /status is an orchestrator heartbeat that can be polled as often as every + // second, so record it at most once per hour per process — enough to confirm + // the endpoint is being used without letting it dominate telemetry volume. + void MaybeRecordStatusTelemetry(const std::shared_ptr& request) { + constexpr int64_t kIntervalMs = 3'600'000; + constexpr int64_t kNever = INT64_MIN; + const int64_t now_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + + int64_t last = last_status_emit_ms_.load(std::memory_order_relaxed); + if (last != kNever && now_ms - last < kIntervalMs) { + return; + } + // Exactly one caller wins this interval's slot; the rest skip. + if (!last_status_emit_ms_.compare_exchange_strong(last, now_ms, std::memory_order_relaxed)) { + return; + } + + ActionTracker tracker(Action::kServiceStatus, ctx_.telemetry, + InvocationContext::Direct(GetUserAgent(request))); + tracker.SetStatus(ActionStatus::kSuccess); + } + ServiceContext& ctx_; + std::atomic last_status_emit_ms_{INT64_MIN}; }; // ======================================================================== @@ -162,9 +192,13 @@ class ShutdownHandler : public HttpRequestHandler { : shutdown_fn_(std::move(shutdown_fn)) {} std::shared_ptr handle(const std::shared_ptr&) override { - shutdown_fn_(); - nlohmann::json body = {{"status", "shutting_down"}}; + std::thread([fn = shutdown_fn_] { + try { + fn(); + } catch (...) { + } + }).detach(); return JsonResponse(Status::CODE_200, body); } @@ -172,6 +206,45 @@ class ShutdownHandler : public HttpRequestHandler { std::function shutdown_fn_; }; +// ======================================================================== +// UnmatchedRouteInterceptor +// +// oatpp routes straight to handlers, so requests that match no route (unknown +// path or wrong method) never reach an ActionTracker and would be invisible to +// telemetry. This request interceptor runs before routing: when the router has +// no route for the request it records a kServiceRequestUnmatched action and +// replies 404 itself; otherwise it returns nullptr and normal routing proceeds. +// ======================================================================== + +class UnmatchedRouteInterceptor : public oatpp::web::server::interceptor::RequestInterceptor { + public: + UnmatchedRouteInterceptor(std::shared_ptr router, ITelemetry& telemetry) + : router_(std::move(router)), telemetry_(telemetry) {} + + std::shared_ptr intercept(const std::shared_ptr& request) override { + const auto& line = request->getStartingLine(); + if (router_->getRoute(line.method, line.path)) { + return nullptr; // a handler will serve this request + } + + { + ActionTracker tracker(Action::kServiceRequestUnmatched, telemetry_, + InvocationContext::Direct(GetUserAgent(request))); + tracker.SetStatus(ActionStatus::kClientError); + } + + nlohmann::json body = { + {"error", {{"message", "Not found"}, {"type", "invalid_request_error"}, + {"param", nullptr}, {"code", nullptr}}}, + }; + return JsonResponse(Status::CODE_404, body); + } + + private: + std::shared_ptr router_; + ITelemetry& telemetry_; +}; + // ======================================================================== // WebService implementation // ======================================================================== @@ -237,11 +310,13 @@ WebService::~WebService() { std::vector WebService::Start(const std::vector& endpoints) { auto& ctx = *impl_->context; - if (impl_->running.load()) { + if (impl_->running.exchange(true)) { ctx.logger.Log(LogLevel::Information, "Web service is already running."); FL_THROW(FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Web service is already running"); } + impl_->thread_tracker.Reset(); + // Create shared router and register all routes impl_->router = oatpp::web::server::HttpRouter::createShared(); @@ -272,93 +347,100 @@ std::vector WebService::Start(const std::vector& endpo impl_->connection_handler = oatpp::web::server::HttpConnectionHandler::createShared(impl_->router); + // Record requests that match no route (unknown path / wrong method), which + // otherwise bypass every handler's ActionTracker. + impl_->connection_handler->addRequestInterceptor( + std::make_shared(impl_->router, ctx.telemetry)); + std::vector bound_urls; - for (const auto& endpoint : endpoints) { - // Parse "http://host:port" — strip scheme, extract host:port - std::string host = "127.0.0.1"; - uint16_t port = 0; + try { + for (const auto& endpoint : endpoints) { + // Parse "http://host:port" — strip scheme, extract host:port + std::string host = "127.0.0.1"; + uint16_t port = 0; - std::string addr = endpoint; - auto scheme_end = addr.find("://"); - if (scheme_end != std::string::npos) { - addr = addr.substr(scheme_end + 3); - } + std::string addr = endpoint; + auto scheme_end = addr.find("://"); + if (scheme_end != std::string::npos) { + addr = addr.substr(scheme_end + 3); + } - if (!addr.empty() && addr.back() == '/') { - addr.pop_back(); - } + if (!addr.empty() && addr.back() == '/') { + addr.pop_back(); + } - auto colon = addr.rfind(':'); - if (colon != std::string::npos) { - host = addr.substr(0, colon); - port = static_cast(std::stoi(addr.substr(colon + 1))); - } else { - host = addr; - } + auto colon = addr.rfind(':'); + if (colon != std::string::npos) { + host = addr.substr(0, colon); + port = static_cast(std::stoi(addr.substr(colon + 1))); + } else { + host = addr; + } - auto tcp_provider = oatpp::network::tcp::server::ConnectionProvider::createShared({host.c_str(), port}); + auto tcp_provider = oatpp::network::tcp::server::ConnectionProvider::createShared({host.c_str(), port}); - // oatpp resolves ephemeral port 0 during construction via getsockname() - // and stores it in PROPERTY_PORT. getAddress().port is stale — use the property. - auto resolved_port = tcp_provider->getProperty(oatpp::network::ConnectionProvider::PROPERTY_PORT); + // oatpp resolves ephemeral port 0 during construction via getsockname() + // and stores it in PROPERTY_PORT. getAddress().port is stale — use the property. + auto resolved_port = tcp_provider->getProperty(oatpp::network::ConnectionProvider::PROPERTY_PORT); - if (resolved_port) { - port = static_cast(std::stoi(resolved_port.std_str())); - } + if (resolved_port) { + port = static_cast(std::stoi(resolved_port.std_str())); + } - // Wrap with our force-close provider so that connection invalidation also unblocks any pending recv() in worker - // threads (via CancelIoEx on Windows; shutdown() suffices on POSIX). Without this, HttpConnectionHandler::stop() - // can wait ~120s for a single keep-alive client to drop its connection. - auto provider = std::static_pointer_cast( - std::make_shared(tcp_provider)); + // Wrap with our force-close provider so that connection invalidation also unblocks any pending recv() in worker + // threads (via CancelIoEx on Windows; shutdown() suffices on POSIX). Without this, HttpConnectionHandler::stop() + // can wait ~120s for a single keep-alive client to drop its connection. + auto provider = std::static_pointer_cast( + std::make_shared(tcp_provider)); - auto server = std::make_shared(provider, impl_->connection_handler); + auto server = std::make_shared(provider, impl_->connection_handler); - impl_->providers.push_back(provider); - impl_->servers.push_back(server); + impl_->providers.push_back(provider); + impl_->servers.push_back(server); - // Start server on a background thread - impl_->listener_threads.emplace_back([server]() { - server->run(); - }); + // Start server on a background thread + impl_->listener_threads.emplace_back([server]() { + server->run(); + }); - // Wait until oatpp's server reports STATUS_RUNNING (i.e. the listener loop - // has started and is accepting connections). Bounded poll with a 5s timeout. - const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); - while (server->getStatus() != oatpp::network::Server::STATUS_RUNNING && - std::chrono::steady_clock::now() < deadline) { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } + // Wait until oatpp's server reports STATUS_RUNNING (i.e. the listener loop + // has started and is accepting connections). Bounded poll with a 5s timeout. + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (server->getStatus() != oatpp::network::Server::STATUS_RUNNING && + std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } - if (server->getStatus() != oatpp::network::Server::STATUS_RUNNING) { - // Tear down anything we already started in this call so we don't leak - // listener threads. `running` is still false, so the destructor would skip Stop(). - impl_->running.store(true); - Stop(); - FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, - "Web service failed to reach RUNNING state for endpoint ", endpoint, " within 5s"); - } + if (server->getStatus() != oatpp::network::Server::STATUS_RUNNING) { + // Tear down anything we already started in this call so we don't leak listener threads. + Stop(); + FL_THROW(FOUNDRY_LOCAL_ERROR_INTERNAL, + "Web service failed to reach RUNNING state for endpoint ", endpoint, " within 5s"); + } - std::string bound_url = "http://" + host + ":" + std::to_string(port); - bound_urls.push_back(bound_url); + std::string bound_url = "http://" + host + ":" + std::to_string(port); + bound_urls.push_back(bound_url); - ctx.logger.Log(LogLevel::Information, fmt::format("Web service listening on {}", bound_url)); + ctx.logger.Log(LogLevel::Information, fmt::format("Web service listening on {}", bound_url)); + } + } catch (...) { + Stop(); + throw; } ctx.bound_urls = bound_urls; - impl_->running.store(true); return bound_urls; } void WebService::Stop() { - if (!impl_->running.load()) { + if (!impl_->running.load() && impl_->servers.empty() && impl_->providers.empty() && + impl_->listener_threads.empty() && !impl_->connection_handler) { return; } - // Join streaming threads first — they may still be pushing to SSE bodies. - impl_->thread_tracker.JoinAll(); + impl_->thread_tracker.BeginStopping(); // Stop accepting new connections first, then stop server loops. for (auto& provider : impl_->providers) { @@ -369,6 +451,8 @@ void WebService::Stop() { server->stop(); } + impl_->thread_tracker.AbortAll(); + // Stop per-connection worker tasks before releasing router/handlers. // // HttpConnectionHandler::stop() invalidates every open connection (our ForceCloseConnectionProvider's invalidator @@ -380,6 +464,10 @@ void WebService::Stop() { impl_->connection_handler->stop(); } + // Join streaming producers after connection shutdown so no new streaming + // thread can be tracked while Stop() is taking its join snapshot. + impl_->thread_tracker.JoinAll(); + for (auto& thread : impl_->listener_threads) { if (thread.joinable()) { thread.join(); diff --git a/sdk_v2/cpp/src/service/web_service.h b/sdk_v2/cpp/src/service/web_service.h index b7d72bd17..fb44cfea2 100644 --- a/sdk_v2/cpp/src/service/web_service.h +++ b/sdk_v2/cpp/src/service/web_service.h @@ -4,6 +4,8 @@ #include "logger.h" +#include +#include #include #include #include @@ -21,47 +23,145 @@ class ResponseStore; /// Tracks streaming threads so they can be joined on shutdown. /// Handlers call Track() instead of std::thread::detach(). -/// Threads call Remove() when done to clean up immediately. class StreamingThreadTracker { + struct TrackedThread { + std::thread thread; + std::shared_ptr> done; + std::function abort; + }; + public: + StreamingThreadTracker() { + StartReaperLocked(); + } + + ~StreamingThreadTracker() { + StopReaper(); + JoinAll(); + } + /// Take ownership of a streaming thread. - void Track(std::thread t) { + void Track(std::thread t, std::shared_ptr> done, std::function abort) { std::lock_guard lock(mutex_); - threads_.push_back(std::move(t)); + StartReaperLocked(); + ReapCompletedLocked(); + if (stopping_) { + if (abort) { + abort(); + } + } + threads_.push_back(TrackedThread{std::move(t), std::move(done), std::move(abort)}); + if (threads_.back().done && threads_.back().done->load(std::memory_order_acquire)) { + completion_cv_.notify_one(); + } } - /// Called from within a thread to untrack itself after work is done. - /// Detaches the thread (can't join itself) and removes the entry. - void Remove(std::thread::id id) { + void NotifyCompleted() { + completion_cv_.notify_one(); + } + + void BeginStopping() { std::lock_guard lock(mutex_); - for (auto it = threads_.begin(); it != threads_.end(); ++it) { - if (it->get_id() == id) { - it->detach(); - threads_.erase(it); - return; + stopping_ = true; + AbortAllLocked(); + } + + void AbortAll() { + std::lock_guard lock(mutex_); + AbortAllLocked(); + } + + void Reset() { + std::lock_guard lock(mutex_); + stopping_ = false; + StartReaperLocked(); + } + + void AbortAllLocked() { + for (auto& tracked : threads_) { + if (tracked.abort) { + tracked.abort(); } } } /// Join all remaining threads. Called by WebService::Stop(). - /// Moves entries out before joining to avoid deadlock with Remove(). + /// Moves entries out before joining so completed streaming threads are cleaned up safely. void JoinAll() { - std::vector local; + StopReaper(); + + std::vector local; { std::lock_guard lock(mutex_); local = std::move(threads_); } - for (auto& t : local) { - if (t.joinable()) { - t.join(); + for (auto& tracked : local) { + if (tracked.thread.joinable()) { + tracked.thread.join(); } } } private: + bool HasCompletedLocked() const { + for (const auto& tracked : threads_) { + if (tracked.done && tracked.done->load(std::memory_order_acquire)) { + return true; + } + } + return false; + } + + void StartReaperLocked() { + if (reaper_thread_.joinable()) { + return; + } + reaper_stop_ = false; + reaper_thread_ = std::thread([this]() { ReaperLoop(); }); + } + + void StopReaper() { + { + std::lock_guard lock(mutex_); + reaper_stop_ = true; + } + completion_cv_.notify_all(); + if (reaper_thread_.joinable()) { + reaper_thread_.join(); + } + } + + void ReaperLoop() { + while (true) { + std::unique_lock lock(mutex_); + completion_cv_.wait(lock, [this]() { return reaper_stop_ || HasCompletedLocked(); }); + if (reaper_stop_) { + return; + } + ReapCompletedLocked(); + } + } + + void ReapCompletedLocked() { + for (auto it = threads_.begin(); it != threads_.end();) { + if (it->done && it->done->load(std::memory_order_acquire)) { + if (it->thread.joinable()) { + it->thread.join(); + } + it = threads_.erase(it); + } else { + ++it; + } + } + } + std::mutex mutex_; - std::vector threads_; + std::condition_variable completion_cv_; + std::vector threads_; + std::thread reaper_thread_; + bool stopping_ = false; + bool reaper_stop_ = false; }; /// Context shared with all HTTP controllers. diff --git a/sdk_v2/cpp/src/telemetry/download_tracker.cc b/sdk_v2/cpp/src/telemetry/download_tracker.cc new file mode 100644 index 000000000..67322f23b --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/download_tracker.cc @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "telemetry/download_tracker.h" + +#include + +namespace fl { + +DownloadTracker::DownloadTracker(std::string model_id, + std::string user_agent, + ITelemetry& telemetry) + : telemetry_(telemetry) { + info_.model_id = std::move(model_id); + info_.user_agent = std::move(user_agent); + info_.correlation_id = MakeGuidV4Hex(); + info_.status = ActionStatus::kFailure; + download_phase_start_ = std::chrono::steady_clock::now(); +} + +DownloadTracker::~DownloadTracker() { + // Emit the Download event regardless of outcome. The default status is + // kFailure so abrupt exits (exceptions) are recorded as failures. + try { + telemetry_.RecordDownload(info_); + } catch (...) { + } +} + +void DownloadTracker::RecordException(const std::exception& exception) { + try { + telemetry_.RecordException(Action::kModelFileDownload, exception, + InvocationContext{info_.user_agent, info_.correlation_id, /*indirect=*/false}); + } catch (...) { + } +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/download_tracker.h b/sdk_v2/cpp/src/telemetry/download_tracker.h new file mode 100644 index 000000000..240bf5b4a --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/download_tracker.h @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "telemetry/telemetry.h" + +#include +#include + +namespace fl { + +/// RAII tracker for the per-model "Download" event. +/// Caller updates fields as the download progresses; the event is emitted on +/// destruction. Status defaults to kFailure so that abrupt exits (exceptions) +/// are recorded as failures unless the caller calls SetStatus(kSuccess) on the +/// happy path or SetStatus(kSkipped) when the model was already cached. +class DownloadTracker { + public: + DownloadTracker(std::string model_id, + std::string user_agent, + ITelemetry& telemetry); + ~DownloadTracker(); + + // Non-copyable, non-movable + DownloadTracker(const DownloadTracker&) = delete; + DownloadTracker& operator=(const DownloadTracker&) = delete; + + void SetStatus(ActionStatus status) { info_.status = status; } + void SetLockWaitMs(int64_t v) { info_.lock_wait_ms = v; } + void SetEnumerationMs(int64_t v) { info_.enumeration_ms = v; } + void SetTotalSizeBytes(int64_t v) { info_.total_size_bytes = v; } + void SetAlreadyCachedBytes(int64_t v) { info_.already_cached_bytes = v; } + void SetFileCount(int32_t v) { info_.file_count = v; } + void SetSkippedFileCount(int32_t v) { info_.skipped_file_count = v; } + void SetDownloadWaitResult(std::string v) { info_.download_wait_result = std::move(v); } + void SetMaxConcurrency(int32_t v) { info_.max_concurrency = v; } + + /// Start the timer for the download phase. Use after lock wait and + /// enumeration are complete, so download_ms only reflects byte transfer. + void BeginDownloadPhase() { download_phase_start_ = std::chrono::steady_clock::now(); } + + /// Stop the timer for the download phase. The duration is captured into the + /// emitted event. Safe to call multiple times — the last call wins. + void EndDownloadPhase() { + info_.download_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - download_phase_start_) + .count(); + } + + void RecordException(const std::exception& exception); + + private: + ITelemetry& telemetry_; + DownloadInfo info_; + std::chrono::steady_clock::time_point download_phase_start_; +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/ep_download_tracker.cc b/sdk_v2/cpp/src/telemetry/ep_download_tracker.cc new file mode 100644 index 000000000..ae82ce96f --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/ep_download_tracker.cc @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "telemetry/ep_download_tracker.h" + +#include + +namespace fl { + +namespace { + +int64_t ElapsedMs(std::chrono::steady_clock::time_point start) { + return std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); +} + +} // namespace + +EpDownloadTracker::EpDownloadTracker(std::string provider_name, + std::string user_agent, + std::string correlation_id, + ITelemetry& telemetry) + : telemetry_(telemetry), + provider_name_(std::move(provider_name)), + user_agent_(std::move(user_agent)), + correlation_id_(std::move(correlation_id)), + stage_start_(std::chrono::steady_clock::now()) { +} + +EpDownloadTracker::~EpDownloadTracker() { + // Mirror neutron-server: if the caller didn't reach Done() or + // RecordRegisterComplete, assume the abrupt exit was an exception path and + // record any unfinished stage as kFailure. + try { + RecordEvent(ActionStatus::kFailure); + } catch (...) { + } +} + +void EpDownloadTracker::RecordInitialState(std::string ready_state) { + init_ready_state_ = std::move(ready_state); + stage_ = Stage::Download; + stage_start_ = std::chrono::steady_clock::now(); +} + +void EpDownloadTracker::RecordDownloadComplete(ActionStatus status, std::string ready_state) { + download_duration_ms_ = ElapsedMs(stage_start_); + download_ready_state_ = std::move(ready_state); + download_status_ = status; + stage_ = Stage::Register; + stage_start_ = std::chrono::steady_clock::now(); +} + +void EpDownloadTracker::RecordRegisterComplete(ActionStatus status, std::string ready_state) { + register_duration_ms_ = ElapsedMs(stage_start_); + register_ready_state_ = std::move(ready_state); + register_status_ = status; + stage_ = Stage::Final; +} + +void EpDownloadTracker::Done() { + RecordEvent(ActionStatus::kSkipped); +} + +void EpDownloadTracker::RecordException(const std::exception& ex) { + // The per-provider attempt happens as a consequence of the overall + // DownloadAndRegisterEps call, so it is indirect and shares its correlation id. + try { + telemetry_.RecordException(Action::kEpDownloadAndRegister, ex, + InvocationContext{user_agent_, correlation_id_, /*indirect=*/true}); + } catch (...) { + } +} + +void EpDownloadTracker::RecordEvent(ActionStatus incomplete_stage_status) { + if (recorded_event_) { + return; + } + recorded_event_ = true; + + if (stage_ == Stage::Download) { + download_duration_ms_ = ElapsedMs(stage_start_); + download_status_ = incomplete_stage_status; + } else if (stage_ == Stage::Register) { + register_duration_ms_ = ElapsedMs(stage_start_); + register_status_ = incomplete_stage_status; + } + + EpDownloadAndRegisterInfo info; + info.user_agent = user_agent_; + info.correlation_id = correlation_id_; + info.provider_name = provider_name_; + info.init_ready_state = init_ready_state_; + info.download_ready_state = download_ready_state_; + info.download_status = download_status_; + info.download_duration_ms = download_duration_ms_; + info.register_ready_state = register_ready_state_; + info.register_status = register_status_; + info.register_duration_ms = register_duration_ms_; + telemetry_.RecordEpDownloadAndRegister(info); +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/ep_download_tracker.h b/sdk_v2/cpp/src/telemetry/ep_download_tracker.h new file mode 100644 index 000000000..7376685bd --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/ep_download_tracker.h @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "telemetry/telemetry.h" + +#include +#include +#include + +namespace fl { + +/// RAII tracker for the per-provider "EPDownloadAndRegister" event. Mirrors the +/// neutron-server EPDownloadTracker. Stages advance Initial -> Download -> +/// Register -> Final. Each stage that the caller doesn't explicitly complete +/// is recorded with one of: +/// * kFailure on destruction (assumed exception path) +/// * kSkipped if Done() is called before destruction (early-exit without +/// exception, e.g. "EP already registered, nothing to download") +class EpDownloadTracker { + public: + EpDownloadTracker(std::string provider_name, + std::string user_agent, + std::string correlation_id, + ITelemetry& telemetry); + ~EpDownloadTracker(); + + // Non-copyable, non-movable + EpDownloadTracker(const EpDownloadTracker&) = delete; + EpDownloadTracker& operator=(const EpDownloadTracker&) = delete; + + /// Captures the EP's ready state before the download phase begins. Restarts + /// the stopwatch so subsequent timings measure the download phase only. + void RecordInitialState(std::string ready_state = "N/A"); + + /// Captures the download phase outcome and ready state, restarts the + /// stopwatch for the register phase. + void RecordDownloadComplete(ActionStatus status, std::string ready_state = "N/A"); + + /// Captures the register phase outcome and ready state. Stops the stopwatch. + void RecordRegisterComplete(ActionStatus status, std::string ready_state = "N/A"); + + /// Mark tracking as complete, filling any remaining stages with kSkipped + /// instead of the default kFailure (exception-assumed) status. Call this on + /// happy-path early exits (e.g. "EP already registered"). + void Done(); + + /// Record an exception associated with the bootstrap operation. Does not + /// finalize the EPDownloadAndRegister event — that happens on destruction. + void RecordException(const std::exception& ex); + + private: + enum class Stage { + Initial = 0, + Download = 1, + Register = 2, + Final = 3, + }; + + void RecordEvent(ActionStatus incomplete_stage_status); + + ITelemetry& telemetry_; + std::string provider_name_; + std::string user_agent_; + std::string correlation_id_; + std::string init_ready_state_ = "N/A"; + std::string download_ready_state_ = "N/A"; + std::string register_ready_state_ = "N/A"; + ActionStatus download_status_ = ActionStatus::kSkipped; + ActionStatus register_status_ = ActionStatus::kSkipped; + int64_t download_duration_ms_ = 0; + int64_t register_duration_ms_ = 0; + std::chrono::steady_clock::time_point stage_start_; + Stage stage_ = Stage::Initial; + bool recorded_event_ = false; +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/invocation_context.cc b/sdk_v2/cpp/src/telemetry/invocation_context.cc new file mode 100644 index 000000000..0b4daab04 --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/invocation_context.cc @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "telemetry/invocation_context.h" + +#include +#include +#include + +namespace fl { + +std::string MakeGuidV4Hex() { + // std::random_device + mt19937_64 is enough here — this is a correlation / + // session id, not a cryptographic identifier, so the OS UUID API is overkill. + std::random_device rd; + std::mt19937_64 gen{(static_cast(rd()) << 32) | rd()}; + uint64_t hi = gen(); + uint64_t lo = gen(); + + // Set version (4) and variant (10xx) bits. + hi = (hi & 0xFFFFFFFFFFFF0FFFULL) | 0x0000000000004000ULL; + lo = (lo & 0x3FFFFFFFFFFFFFFFULL) | 0x8000000000000000ULL; + + char buf[37]; + std::snprintf(buf, sizeof(buf), + "%08x-%04x-%04x-%04x-%012llx", + static_cast((hi >> 32) & 0xFFFFFFFFu), + static_cast((hi >> 16) & 0xFFFFu), + static_cast(hi & 0xFFFFu), + static_cast((lo >> 48) & 0xFFFFu), + static_cast(lo & 0x0000FFFFFFFFFFFFULL)); + return std::string(buf); +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/invocation_context.h b/sdk_v2/cpp/src/telemetry/invocation_context.h new file mode 100644 index 000000000..8bf30db56 --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/invocation_context.h @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include +#include + +namespace fl { + +/// Generate an RFC 4122 v4 UUID, hex-encoded with hyphens. Not a cryptographic +/// identifier — used for per-operation correlation and per-process session ids. +std::string MakeGuidV4Hex(); + +/// Per-operation telemetry context, threaded from an entry point through every +/// action it triggers. +/// +/// - `user_agent` identifies the calling client (SDK, CLI, Node, browser…). +/// - `correlation_id` groups every event emitted while servicing one logical +/// operation, so the route action, the inference it drives, +/// the Model metrics event and any error can be joined. +/// - `indirect` is true when this action happened as a consequence of +/// another action rather than a direct user/API call (e.g. a +/// session driven by an HTTP route, or a per-provider EP +/// download under an overall attempt). +struct InvocationContext { + std::string user_agent; + std::string correlation_id; + bool indirect = false; + + /// A direct, top-level context with a freshly generated correlation id. + static InvocationContext Direct(std::string user_agent = "") { + InvocationContext ctx; + ctx.user_agent = std::move(user_agent); + ctx.correlation_id = MakeGuidV4Hex(); + ctx.indirect = false; + return ctx; + } + + /// Derive a context for an action triggered by this one: same correlation id + /// and user agent, but marked indirect. + InvocationContext AsIndirect() const { + InvocationContext ctx = *this; + ctx.indirect = true; + return ctx; + } + + /// Guarantee a correlation id is present, generating one when empty. Lets an + /// entry point that received a default-constructed context still group its + /// events (e.g. an SDK caller that didn't build a Direct() context). + InvocationContext& EnsureCorrelationId() { + if (correlation_id.empty()) { + correlation_id = MakeGuidV4Hex(); + } + return *this; + } +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc new file mode 100644 index 000000000..d380d8e56 --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.cc @@ -0,0 +1,332 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// +// 1DS-backed ITelemetry implementation. Compiled only when FOUNDRY_LOCAL_USE_TELEMETRY=ON +// (which requires the cpp-client-telemetry vcpkg port to be available). + +#include "telemetry/one_ds_telemetry.h" + +#include "telemetry/telemetry_redaction.h" +#include "telemetry/telemetry_environment.h" + +#include + +// 1DS C++ SDK headers. The vcpkg port ships these via find_package(MSTelemetry CONFIG). +#include +#include +#include +#include + +// The LogManager macro must appear exactly once per binary that uses the +// "v1 classic" LogManager API surface. The macro instantiates the singleton +// statics for our module configuration. +LOGMANAGER_INSTANCE + +namespace fl { + +namespace { + +using ::Microsoft::Applications::Events::LogManager; +using MatILogger = ::Microsoft::Applications::Events::ILogger; +using ::Microsoft::Applications::Events::EventLatency_Normal; +using ::Microsoft::Applications::Events::EventProperties; +using ::Microsoft::Applications::Events::SessionState; + +void SetCommonContext(MatILogger* mat_logger, const TelemetryMetadata& m) { + // Process-wide context — stamped on every event uploaded through this ILogger. + mat_logger->SetContext("AppName", m.app_name); + mat_logger->SetContext("Version", m.version); + mat_logger->SetContext("AppVersion", m.version); + // Per-process correlation GUID, generated at startup. Lets the backend group + // every event from one FL run. (This is distinct from ext.app.sesId, the SDK's + // rotating usage-session id set via LogSession.) + mat_logger->SetContext("AppSessionGuid", m.app_session_guid); + mat_logger->SetContext("OsName", m.os_name); + mat_logger->SetContext("Platform", m.os_name); + mat_logger->SetContext("OsVersion", m.os_version); + mat_logger->SetContext("CpuArch", m.cpu_arch); +} + +EventProperties MakeEvent(const char* name, bool test_mode) { + EventProperties ev(name); + ev.SetLatency(EventLatency_Normal); + ev.SetLevel(DIAG_LEVEL_REQUIRED); + // `test` is stamped on every event so CI/test data is distinguishable in the backend + // when FOUNDRY_TESTING_MODE is set. In CI (IsCiEnvironment()=true) we never reach + // emission at all, so this only ever toggles for explicit test mode. + ev.SetProperty("test", test_mode); + return ev; +} + +void SafeLog(MatILogger* mat_logger, EventProperties& ev) { + if (mat_logger != nullptr) { + mat_logger->LogEvent(ev); + } +} + +MatILogger* GetMatLogger() { + // LogManager::GetLogger() returns nullptr until Initialize has been called. + return LogManager::GetLogger(); +} + +} // namespace + +OneDsTelemetry::OneDsTelemetry(const std::string& tenant_token, + const std::string& app_name, + ILogger& logger) + : local_log_(app_name, logger), + metadata_(BuildTelemetryMetadata(app_name)), + logger_(logger) { + const bool is_ci = TelemetryEnvironment::IsCiEnvironment(); + if (is_ci) { + logger_.Log(LogLevel::Information, + "[Telemetry] CI environment detected; 1DS upload disabled (events still logged locally)"); + return; + } + if (tenant_token.empty()) { + logger_.Log(LogLevel::Information, + "[Telemetry] Tenant token is empty; 1DS upload disabled (events still logged locally)"); + return; + } + + try { + auto* mat_logger = LogManager::Initialize(tenant_token); + if (mat_logger == nullptr) { + logger_.Log(LogLevel::Warning, + "[Telemetry] LogManager::Initialize returned null; 1DS upload disabled"); + return; + } + SetCommonContext(mat_logger, metadata_); + initialized_.store(true, std::memory_order_release); + logger_.Log(LogLevel::Information, + fmt::format("[Telemetry] 1DS initialized; AppName={} Version={} Os={} {} Arch={} TestMode={}", + metadata_.app_name, metadata_.version, metadata_.os_name, + metadata_.os_version, metadata_.cpu_arch, metadata_.test_mode)); + } catch (const std::exception& ex) { + logger_.Log(LogLevel::Warning, + fmt::format("[Telemetry] LogManager::Initialize threw: {}; 1DS upload disabled", ex.what())); + } catch (...) { + logger_.Log(LogLevel::Warning, + "[Telemetry] LogManager::Initialize threw unknown exception; 1DS upload disabled"); + } +} + +OneDsTelemetry::~OneDsTelemetry() { + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + try { + LogManager::FlushAndTeardown(); + } catch (...) { + // Best-effort: never throw from a destructor. + } +} + +void OneDsTelemetry::RecordAction(Action action, ActionStatus status, const InvocationContext& context, + int64_t duration_ms) { + try { + local_log_.RecordAction(action, status, context, duration_ms); + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto ev = MakeEvent("Action", metadata_.test_mode); + ev.SetProperty("Action", std::string(ActionToString(action))); + ev.SetProperty("Status", std::string(ActionStatusToString(status))); + ev.SetProperty("UserAgent", context.user_agent); + ev.SetProperty("CorrelationId", context.correlation_id); + ev.SetProperty("Direct", !context.indirect); + ev.SetProperty("TimeMs", duration_ms); + SafeLog(GetMatLogger(), ev); + } catch (...) { + } +} + +void OneDsTelemetry::RecordException(Action action, const std::exception& exception, + const InvocationContext& context) { + try { + local_log_.RecordException(action, exception, context); + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto ev = MakeEvent("Error", metadata_.test_mode); + ev.SetProperty("Action", std::string(ActionToString(action))); + ev.SetProperty("UserAgent", context.user_agent); + ev.SetProperty("CorrelationId", context.correlation_id); + ev.SetProperty("ExceptionType", "std::exception"); + ev.SetProperty("ExceptionMessage", ScrubTelemetryErrorMessage(exception.what())); + ev.SetProperty("InnerExceptionType", ""); + ev.SetProperty("InnerExceptionMessage", ""); + ev.SetProperty("StackTrace", ""); + ev.SetProperty("InnerStackTrace", ""); + SafeLog(GetMatLogger(), ev); + } catch (...) { + } +} + +void OneDsTelemetry::RecordModelUsage(const ModelUsageInfo& info) { + try { + local_log_.RecordModelUsage(info); + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto ev = MakeEvent("Model", metadata_.test_mode); + ev.SetProperty("ModelId", info.model_id); + ev.SetProperty("ExecutionProvider", info.execution_provider); + ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("CorrelationId", info.correlation_id); + ev.SetProperty("Stream", info.stream); + ev.SetProperty("Direct", !info.indirect); + ev.SetProperty("TimeToFirstTokenMs", info.time_to_first_token_ms); + ev.SetProperty("TotalTimeMs", info.total_time_ms); + ev.SetProperty("TotalTokens", static_cast(info.total_tokens)); + ev.SetProperty("InputTokenCount", static_cast(info.input_token_count)); + ev.SetProperty("NumMessages", static_cast(info.num_messages)); + ev.SetProperty("MemoryUsedMB", info.memory_used_mb); + ev.SetProperty("CpuTimeMs", info.cpu_time_ms); + ev.SetProperty("GpuMemoryUsedMB", info.gpu_memory_used_mb); + SafeLog(GetMatLogger(), ev); + } catch (...) { + } +} + +void OneDsTelemetry::RecordModelId(Action action, const std::string& model_id, + ActionStatus status, const InvocationContext& context) { + try { + local_log_.RecordModelId(action, model_id, status, context); + if (!initialized_.load(std::memory_order_acquire) || model_id.empty()) { + return; + } + auto ev = MakeEvent("ModelId", metadata_.test_mode); + ev.SetProperty("Action", std::string(ActionToString(action))); + ev.SetProperty("ModelId", model_id); + ev.SetProperty("Status", std::string(ActionStatusToString(status))); + ev.SetProperty("UserAgent", context.user_agent); + ev.SetProperty("CorrelationId", context.correlation_id); + SafeLog(GetMatLogger(), ev); + } catch (...) { + } +} + +void OneDsTelemetry::RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) { + try { + local_log_.RecordEpDownloadAttempt(info); + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto ev = MakeEvent("EPDownloadAttempt", metadata_.test_mode); + ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("CorrelationId", info.correlation_id); + ev.SetProperty("Attempts", static_cast(info.attempts)); + ev.SetProperty("NumProviders", static_cast(info.num_providers)); + ev.SetProperty("Succeeded", static_cast(info.succeeded)); + ev.SetProperty("Failed", static_cast(info.failed)); + ev.SetProperty("Resolved", info.resolved); + ev.SetProperty("Status", std::string(ActionStatusToString(info.status))); + ev.SetProperty("TimeMs", info.duration_ms); + SafeLog(GetMatLogger(), ev); + } catch (...) { + } +} + +void OneDsTelemetry::RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) { + try { + local_log_.RecordEpDownloadAndRegister(info); + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto ev = MakeEvent("EPDownloadAndRegister", metadata_.test_mode); + ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("CorrelationId", info.correlation_id); + ev.SetProperty("ProviderName", info.provider_name); + ev.SetProperty("InitReadyState", info.init_ready_state); + ev.SetProperty("DownloadReadyState", info.download_ready_state); + ev.SetProperty("DownloadStatus", std::string(ActionStatusToString(info.download_status))); + ev.SetProperty("DownloadTimeMs", info.download_duration_ms); + ev.SetProperty("RegisterReadyState", info.register_ready_state); + ev.SetProperty("RegisterStatus", std::string(ActionStatusToString(info.register_status))); + ev.SetProperty("RegisterTimeMs", info.register_duration_ms); + SafeLog(GetMatLogger(), ev); + } catch (...) { + } +} + +void OneDsTelemetry::RecordDownload(const DownloadInfo& info) { + try { + local_log_.RecordDownload(info); + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto ev = MakeEvent("Download", metadata_.test_mode); + ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("CorrelationId", info.correlation_id); + ev.SetProperty("ModelId", info.model_id); + ev.SetProperty("Status", std::string(ActionStatusToString(info.status))); + ev.SetProperty("LockWaitTimeMs", info.lock_wait_ms); + ev.SetProperty("EnumerationTimeMs", info.enumeration_ms); + ev.SetProperty("DownloadTimeMs", info.download_ms); + ev.SetProperty("TotalSizeBytes", info.total_size_bytes); + ev.SetProperty("AlreadyCachedBytes", info.already_cached_bytes); + ev.SetProperty("FileCount", static_cast(info.file_count)); + ev.SetProperty("SkippedFileCount", static_cast(info.skipped_file_count)); + ev.SetProperty("DownloadWaitResult", info.download_wait_result); + ev.SetProperty("MaxConcurrency", static_cast(info.max_concurrency)); + SafeLog(GetMatLogger(), ev); + } catch (...) { + } +} + +void OneDsTelemetry::RecordCatalogFetch(const CatalogFetchInfo& info) { + try { + local_log_.RecordCatalogFetch(info); + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto ev = MakeEvent("CatalogFetch", metadata_.test_mode); + ev.SetProperty("Operation", info.operation); + ev.SetProperty("Endpoint", info.endpoint); + ev.SetProperty("Region", info.region); + ev.SetProperty("Format", info.format); + ev.SetProperty("Status", std::string(ActionStatusToString(info.status))); + ev.SetProperty("TimeMs", info.duration_ms); + ev.SetProperty("ModelCount", static_cast(info.model_count)); + ev.SetProperty("ErrorMessage", ScrubTelemetryErrorMessage(info.error_message)); + ev.SetProperty("UserAgent", info.user_agent); + ev.SetProperty("CorrelationId", info.correlation_id); + SafeLog(GetMatLogger(), ev); + } catch (...) { + } +} + +void OneDsTelemetry::StartSession() { + try { + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto* mat_logger = GetMatLogger(); + if (mat_logger == nullptr) { + return; + } + // LogSession(Started) opens an app-usage session; the SDK stamps ext.app.sesId + // on subsequent events and records session duration on End. + auto ev = MakeEvent("Session", metadata_.test_mode); + mat_logger->LogSession(SessionState::Session_Started, ev); + } catch (...) { + } +} + +void OneDsTelemetry::EndSession() { + try { + if (!initialized_.load(std::memory_order_acquire)) { + return; + } + auto* mat_logger = GetMatLogger(); + if (mat_logger == nullptr) { + return; + } + auto ev = MakeEvent("Session", metadata_.test_mode); + mat_logger->LogSession(SessionState::Session_Ended, ev); + } catch (...) { + } +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h new file mode 100644 index 000000000..c1af838bd --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/one_ds_telemetry.h @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "telemetry/telemetry.h" +#include "telemetry/telemetry_logger.h" +#include "telemetry/telemetry_metadata.h" +#include "logger.h" + +#include +#include +#include + +namespace fl { + +/// 1DS-backed ITelemetry implementation. Built only when the +/// cpp-client-telemetry vcpkg port is available (find_package(MSTelemetry CONFIG) +/// succeeded and FOUNDRY_LOCAL_USE_TELEMETRY=ON). +/// +/// Lifecycle: +/// * Constructor: +/// - If TelemetryEnvironment::IsCiEnvironment() -> CI mode: skip 1DS Initialize +/// entirely. All RecordX calls become no-ops on the 1DS side; the embedded +/// TelemetryLogger still mirrors them to ILogger. +/// - Else if tenant_token is empty -> uninitialized: same behavior as CI mode. +/// Manager normally avoids constructing OneDsTelemetry when the token is +/// empty, but the guard here makes the class robust to misconfiguration. +/// - Else: call MAT::LogManager::Initialize(token), set process-wide common +/// context (app session GUID, version, OS info, app name, test flag). +/// +/// * RecordX: builds an EventProperties with the event-specific fields and the +/// `test` bool, then logs via ILogger->LogEvent. Local mirror via TelemetryLogger +/// always runs first so the diagnostic log is preserved regardless of upload. +/// +/// * Destructor: MAT::LogManager::FlushAndTeardown so events on disk are pushed +/// before the process exits. Safe to call even when initialization was skipped. +/// +/// Thread safety: 1DS LogManager methods are documented as thread-safe. The +/// `initialized_` flag is set once during construction; all other writes happen +/// only in the destructor. +class OneDsTelemetry : public ITelemetry { + public: + /// @param tenant_token 1DS ingestion token. Empty disables uploads (CI-equivalent). + /// @param app_name Configuration::app_name; stamped as AppName on every event. + /// @param logger Diagnostic logger; used by the embedded TelemetryLogger mirror. + OneDsTelemetry(const std::string& tenant_token, + const std::string& app_name, + ILogger& logger); + ~OneDsTelemetry() override; + + // Non-copyable, non-movable (singleton owns LogManager state). + OneDsTelemetry(const OneDsTelemetry&) = delete; + OneDsTelemetry& operator=(const OneDsTelemetry&) = delete; + + void RecordAction(Action action, ActionStatus status, const InvocationContext& context, + int64_t duration_ms) override; + + void RecordException(Action action, const std::exception& exception, + const InvocationContext& context) override; + + void RecordModelUsage(const ModelUsageInfo& info) override; + void RecordModelId(Action action, const std::string& model_id, + ActionStatus status, const InvocationContext& context) override; + void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) override; + void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) override; + void RecordDownload(const DownloadInfo& info) override; + void RecordCatalogFetch(const CatalogFetchInfo& info) override; + void StartSession() override; + void EndSession() override; + + /// True if 1DS Initialize succeeded (i.e. events are actually uploaded). + /// False in CI or when the tenant token was empty. + bool IsUploadEnabled() const { return initialized_; } + + private: + TelemetryLogger local_log_; // Always-on local mirror. + TelemetryMetadata metadata_; // Cached at construction. + std::atomic initialized_{false}; // True iff MAT::LogManager::Initialize ran. + ILogger& logger_; +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/one_ds_tenant_token.h.in b/sdk_v2/cpp/src/telemetry/one_ds_tenant_token.h.in new file mode 100644 index 000000000..2257ad0be --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/one_ds_tenant_token.h.in @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft. All rights reserved. +// Auto-generated by CMake from one_ds_tenant_token.h.in — do not edit manually. +// +// The 1DS tenant token is read by CMake from the FOUNDRY_LOCAL_TELEMETRY_TOKEN +// environment variable. Do not pass it with -D: that would place the secret in +// process argv and CMakeCache.txt. The default is an empty string so developer +// builds and forks don't accidentally ship a tenant binding. +// +// When the constant is empty, OneDsTelemetry::Initialize is skipped and the +// Manager falls back to TelemetryLogger (which only writes to the local +// ILogger sink — no upload). +#pragma once + +namespace fl { + +inline constexpr const char* kFoundryLocalTenantToken = "@FOUNDRY_LOCAL_TELEMETRY_TOKEN@"; + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry.cc b/sdk_v2/cpp/src/telemetry/telemetry.cc index 6d8fc06f6..28b75c27c 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry.cc @@ -22,10 +22,6 @@ std::string_view ActionToString(Action action) { return "ModelLoad"; case Action::kModelUnload: return "ModelUnload"; - case Action::kModelDownload: - return "ModelDownload"; - case Action::kModelDelete: - return "ModelDelete"; case Action::kModelList: return "ModelList"; case Action::kOpenAIChatCompletions: @@ -48,8 +44,18 @@ std::string_view ActionToString(Action action) { return "OpenAIResponsesDelete"; case Action::kOpenAIResponsesGetInputItems: return "OpenAIResponsesGetInputItems"; - case Action::kCoreAudioTranscribe: - return "CoreAudioTranscribe"; + case Action::kEpDownloadAttempt: + return "EpDownloadAttempt"; + case Action::kEpDownloadAndRegister: + return "EpDownloadAndRegister"; + case Action::kModelFileDownload: + return "ModelFileDownload"; + case Action::kModelInference: + return "ModelInference"; + case Action::kServiceRequestUnmatched: + return "ServiceRequestUnmatched"; + case Action::kServiceStatus: + return "ServiceStatus"; default: return "Unknown"; } @@ -65,6 +71,8 @@ std::string_view ActionStatusToString(ActionStatus status) { return "Invalid"; case ActionStatus::kSkipped: return "Skipped"; + case ActionStatus::kClientError: + return "ClientError"; default: return "Unknown"; } diff --git a/sdk_v2/cpp/src/telemetry/telemetry.h b/sdk_v2/cpp/src/telemetry/telemetry.h index 680dfcd9a..c5075e42c 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry.h +++ b/sdk_v2/cpp/src/telemetry/telemetry.h @@ -2,6 +2,8 @@ // Licensed under the MIT License. #pragma once +#include "telemetry/invocation_context.h" + #include #include #include @@ -9,8 +11,9 @@ namespace fl { -/// Telemetry action identifiers matching the C# ITelemetry.Action enum. -// TODO: This is a lie. The enum values don't match. Do they need to? +/// Telemetry action identifiers. The values are stable IDs for log greppability; +/// the over-the-wire field is the human-readable name produced by ActionToString. +/// The 1DS implementation emits the string, so changing numeric values is safe. enum class Action { kInvalid = 0, @@ -26,8 +29,6 @@ enum class Action { // Model management kModelLoad = 100, kModelUnload = 101, - kModelDownload = 102, - kModelDelete = 103, kModelList = 104, // OpenAI Chat/Audio/Embeddings APIs @@ -44,16 +45,28 @@ enum class Action { kOpenAIResponsesDelete = 303, kOpenAIResponsesGetInputItems = 304, - // Audio - kCoreAudioTranscribe = 400, + // EP catalog operations + kEpDownloadAttempt = 500, // Wraps the entire DownloadAndRegisterEps call + kEpDownloadAndRegister = 501, // One per-provider attempt within DownloadAndRegisterEps + + // Model file download + kModelFileDownload = 600, // Wraps the per-model DownloadManager flow + + // EP runtime usage (TimeToFirstToken / total tokens / memory) + kModelInference = 700, // The "Model" event in the C# implementation + + // HTTP service plumbing + kServiceRequestUnmatched = 800, // A request reached the service but matched no route / method + kServiceStatus = 801, // GET /status heartbeat (sampled — at most once per hour per process) }; /// Status of a tracked telemetry action. enum class ActionStatus { - kFailure = 0, + kFailure = 0, // Server-side / internal failure (maps to HTTP 5xx) kSuccess, kInvalid, kSkipped, + kClientError, // Rejected due to invalid client input (maps to HTTP 4xx) — not a service fault }; /// Convert Action to human-readable string. @@ -62,29 +75,126 @@ std::string_view ActionToString(Action action); /// Convert ActionStatus to human-readable string. std::string_view ActionStatusToString(ActionStatus status); +/// Payload for the EPDownloadAttempt event — emitted once per DownloadAndRegisterEps call. +struct EpDownloadAttemptInfo { + std::string user_agent; + std::string correlation_id; + int attempts = 0; // Total per-provider attempts made + int num_providers = 0; // Number of providers requested + int succeeded = 0; // Number of providers that registered successfully + int failed = 0; // Number of providers that failed + bool resolved = false; // True if at least one provider became Registered + ActionStatus status = ActionStatus::kInvalid; + int64_t duration_ms = 0; +}; + +/// Payload for the EPDownloadAndRegister event — emitted once per provider attempt. +struct EpDownloadAndRegisterInfo { + std::string user_agent; + std::string correlation_id; + std::string provider_name; + std::string init_ready_state; // EP state before this call (e.g. "NotPresent") + std::string download_ready_state; // EP state after the download phase (e.g. "Installed") + ActionStatus download_status = ActionStatus::kInvalid; + int64_t download_duration_ms = 0; + std::string register_ready_state; // EP state after the register phase (e.g. "Registered") + ActionStatus register_status = ActionStatus::kInvalid; + int64_t register_duration_ms = 0; +}; + +/// Payload for the Model event — emitted once per inference completion with token / memory metrics. +struct ModelUsageInfo { + std::string model_id; + std::string execution_provider; + std::string user_agent; + std::string correlation_id; + bool stream = false; // True if the inference was streamed (SSE) vs a single response + bool indirect = false; // True if the inference was driven by another action (e.g. an HTTP route) + int64_t time_to_first_token_ms = -1; + int64_t total_time_ms = 0; + int32_t total_tokens = 0; + int32_t input_token_count = 0; + uint64_t num_messages = 0; + int64_t memory_used_mb = -1; // -1 if not measured + int64_t cpu_time_ms = -1; // -1 if not measured + int64_t gpu_memory_used_mb = -1; // -1 if not measured +}; + +/// Payload for the Download event — emitted once per DownloadManager::DownloadModel call. +struct DownloadInfo { + std::string model_id; + std::string user_agent; + std::string correlation_id; + ActionStatus status = ActionStatus::kInvalid; + int64_t lock_wait_ms = 0; + int64_t enumeration_ms = 0; + int64_t download_ms = 0; + int64_t total_size_bytes = 0; + int64_t already_cached_bytes = 0; + int32_t file_count = 0; + int32_t skipped_file_count = 0; + std::string download_wait_result; // e.g. "Completed", "TimedOut", "AlreadyHeld" + int32_t max_concurrency = 0; +}; + +/// Payload for the CatalogFetch event — emitted once per access to a model +/// catalog source (the live Azure catalog or the embedded static snapshot). +struct CatalogFetchInfo { + std::string operation; // "FetchAll" (full catalog) or "FetchByIds" (cached-id lookup) + std::string endpoint; // catalog host (e.g. "ai.azure.com"), or "static" for the embedded snapshot + std::string region; // region parsed from the catalog URL (e.g. "eastus"); empty if not present + std::string format; // catalog API path/version after the region (e.g. "ux/v1.0") + ActionStatus status = ActionStatus::kInvalid; + int64_t duration_ms = 0; + int32_t model_count = 0; // models returned by this access + std::string error_message; // populated on failure + std::string user_agent; + std::string correlation_id; // shared across the accesses of one catalog refresh +}; + /// Abstract telemetry interface. -/// Implementations may send events to a telemetry backend (ETW, OpenTelemetry, etc.) -/// or simply log them. The stub TelemetryLogger logs via ILogger. +/// Implementations may send events to a telemetry backend (1DS, ETW, OpenTelemetry, …) +/// or simply log them. The OneDsTelemetry implementation sends to 1DS; the +/// TelemetryLogger stub formats them to the ILogger sink. class ITelemetry { public: virtual ~ITelemetry() = default; - /// Record a completed action with timing and status. + /// Record a completed action with timing and status. The context carries the + /// user agent, the correlation id grouping this operation's events, and whether + /// the action was indirect (triggered by another action). virtual void RecordAction(Action action, ActionStatus status, - const std::string& user_agent, - bool indirect, int64_t duration_ms) = 0; + const InvocationContext& context, int64_t duration_ms) = 0; /// Record an exception associated with an action. - virtual void RecordException(Action action, const std::exception& exception) = 0; + virtual void RecordException(Action action, const std::exception& exception, + const InvocationContext& context) = 0; + + /// Record model usage metrics after inference (Model event). + virtual void RecordModelUsage(const ModelUsageInfo& info) = 0; + + /// Record which model was used for an action (ModelId event). + virtual void RecordModelId(Action action, const std::string& model_id, + ActionStatus status, const InvocationContext& context) = 0; + + /// Record the result of a DownloadAndRegisterEps call (EPDownloadAttempt event). + virtual void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) = 0; + + /// Record one EP provider's download+register attempt (EPDownloadAndRegister event). + virtual void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) = 0; + + /// Record one model file download (Download event). + virtual void RecordDownload(const DownloadInfo& info) = 0; - /// Record model usage metrics after inference. - virtual void RecordModelUsage(const std::string& model_id, - int64_t prompt_tokens, - int64_t completion_tokens, - int64_t duration_ms) = 0; + /// Record one access to a model catalog source (CatalogFetch event). + virtual void RecordCatalogFetch(const CatalogFetchInfo& info) = 0; - /// Record which model was used for an action. - virtual void RecordModelId(Action action, const std::string& model_id) = 0; + /// Mark the start / end of an app-usage session via 1DS LogSession. Between + /// these the SDK stamps a session id (ext.app.sesId) on events and emits a + /// session start/end with duration — standard, cross-platform engagement + /// sessions. Default no-op for the non-1DS implementations. + virtual void StartSession() {} + virtual void EndSession() {} }; } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc b/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc index 7672cb1f4..50a960e35 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.cc @@ -2,24 +2,31 @@ // Licensed under the MIT License. #include "telemetry/telemetry_action_tracker.h" +#include + namespace fl { -ActionTracker::ActionTracker(Action action, ITelemetry& telemetry, const std::string& user_agent, bool indirect) +ActionTracker::ActionTracker(Action action, ITelemetry& telemetry, InvocationContext context) : action_(action), telemetry_(telemetry), - user_agent_(user_agent), - indirect_(indirect), + context_(std::move(context)), start_(std::chrono::steady_clock::now()) { + // Guarantee a correlation id so this action and anything it triggers can be + // grouped, even when the caller passed a default-constructed context. + context_.EnsureCorrelationId(); } ActionTracker::~ActionTracker() { auto end = std::chrono::steady_clock::now(); auto duration_ms = std::chrono::duration_cast(end - start_).count(); - telemetry_.RecordAction(action_, status_, user_agent_, indirect_, duration_ms); + try { + telemetry_.RecordAction(action_, status_, context_, duration_ms); - if (!model_id_.empty()) { - telemetry_.RecordModelId(action_, model_id_); + if (!model_id_.empty()) { + telemetry_.RecordModelId(action_, model_id_, status_, context_); + } + } catch (...) { } } @@ -28,7 +35,10 @@ void ActionTracker::SetStatus(ActionStatus status) { } void ActionTracker::RecordException(const std::exception& exception) { - telemetry_.RecordException(action_, exception); + try { + telemetry_.RecordException(action_, exception, context_); + } catch (...) { + } } void ActionTracker::SetModelId(const std::string& model_id) { diff --git a/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.h b/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.h index e06da6130..e19971eb2 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.h +++ b/sdk_v2/cpp/src/telemetry/telemetry_action_tracker.h @@ -14,9 +14,7 @@ namespace fl { /// Matches the C# ActionTracker (IDisposable) pattern. class ActionTracker { public: - ActionTracker(Action action, ITelemetry& telemetry, - const std::string& user_agent = "", - bool indirect = false); + ActionTracker(Action action, ITelemetry& telemetry, InvocationContext context = {}); ~ActionTracker(); // Non-copyable, non-movable @@ -32,11 +30,15 @@ class ActionTracker { /// Associate a model ID with this action. void SetModelId(const std::string& model_id); + /// This action's context, with its correlation id resolved. Use to derive a + /// child context (Context().AsIndirect()) for any caused-by action so all + /// events from one operation share a correlation id. + const InvocationContext& Context() const { return context_; } + private: Action action_; ITelemetry& telemetry_; - std::string user_agent_; - bool indirect_; + InvocationContext context_; ActionStatus status_ = ActionStatus::kFailure; std::string model_id_; std::chrono::steady_clock::time_point start_; diff --git a/sdk_v2/cpp/src/telemetry/telemetry_environment.cc b/sdk_v2/cpp/src/telemetry/telemetry_environment.cc new file mode 100644 index 000000000..a19838ba7 --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/telemetry_environment.cc @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "telemetry/telemetry_environment.h" + +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#endif + +namespace fl { + +namespace { + +// Mirrors neutron-server's CiEnvironmentVariableNames. Keep this in sync if the +// list there changes — telemetry behavior in CI must match across stacks. +constexpr std::array kCiEnvironmentVariableNames = { + "CI", // Generic CI flag used by many providers + "TF_BUILD", // Azure Pipelines + "GITHUB_ACTIONS", // GitHub Actions + "GITLAB_CI", // GitLab CI + "CIRCLECI", // CircleCI + "TRAVIS", // Travis CI + "JENKINS_URL", // Jenkins + "CODEBUILD_BUILD_ID", // AWS CodeBuild + "BUILDKITE", // Buildkite + "TEAMCITY_VERSION", // TeamCity + "APPVEYOR", // AppVeyor + "BITBUCKET_BUILD_NUMBER", // Bitbucket Pipelines + "SYSTEM_TEAMFOUNDATIONCOLLECTIONURI", // Azure DevOps +}; + +bool EqualsIgnoreCase(std::string_view a, std::string_view b) { + if (a.size() != b.size()) { + return false; + } + for (size_t i = 0; i < a.size(); ++i) { + if (std::tolower(static_cast(a[i])) != + std::tolower(static_cast(b[i]))) { + return false; + } + } + return true; +} + +std::string_view Trim(std::string_view s) { + auto is_ws = [](unsigned char c) { return std::isspace(c) != 0; }; + while (!s.empty() && is_ws(static_cast(s.front()))) { + s.remove_prefix(1); + } + while (!s.empty() && is_ws(static_cast(s.back()))) { + s.remove_suffix(1); + } + return s; +} + +} // namespace + +std::string TelemetryEnvironment::GetEnv(const char* name) { +#ifdef _WIN32 + // Use the W variant so we don't depend on the legacy CRT _CRT_SECURE_NO_WARNINGS. + // Env-var values are ASCII for the CI flags we care about; if a value is unicode + // we still get the bytes round-tripped correctly because we only do truthiness checks. + DWORD needed = ::GetEnvironmentVariableA(name, nullptr, 0); + if (needed == 0) { + return {}; + } + std::vector buf(needed); + DWORD written = ::GetEnvironmentVariableA(name, buf.data(), needed); + if (written == 0 || written >= needed) { + return {}; + } + return std::string(buf.data(), written); +#else + const char* value = std::getenv(name); + return value ? std::string(value) : std::string{}; +#endif +} + +bool TelemetryEnvironment::IsTruthyValue(std::string_view value) { + auto trimmed = Trim(value); + if (trimmed.empty()) { + return false; + } + return !EqualsIgnoreCase(trimmed, "0") && + !EqualsIgnoreCase(trimmed, "false") && + !EqualsIgnoreCase(trimmed, "no") && + !EqualsIgnoreCase(trimmed, "off"); +} + +bool TelemetryEnvironment::IsCiEnvironment() { + for (const char* name : kCiEnvironmentVariableNames) { + if (IsTruthyValue(GetEnv(name))) { + return true; + } + } + return false; +} + +bool TelemetryEnvironment::IsTestingMode() { + return IsTruthyValue(GetEnv("FOUNDRY_TESTING_MODE")); +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_environment.h b/sdk_v2/cpp/src/telemetry/telemetry_environment.h new file mode 100644 index 000000000..71f3e2a5a --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/telemetry_environment.h @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include +#include + +namespace fl { + +/// Static helpers for telemetry runtime gating. Ported from neutron-server's +/// TelemetryEnvironment.cs so the CI suppression behavior matches across stacks. +class TelemetryEnvironment { + public: + /// Returns true if any well-known CI environment variable is set to a truthy + /// value. The set matches neutron-server's TelemetryEnvironment.cs. + /// In CI, OneDsTelemetry skips Initialize entirely — no 1DS events emitted. + static bool IsCiEnvironment(); + + /// Returns true if the FOUNDRY_TESTING_MODE env var is set to a truthy value. + /// Outside CI, this stamps a `test=true` boolean on every emitted event but + /// does not suppress emission. In CI, this is moot — emission is suppressed. + static bool IsTestingMode(); + + /// Truthy-value semantics: a non-empty, non-whitespace string whose trimmed + /// value is not "0", "false", "no", or "off" (case-insensitive). + static bool IsTruthyValue(std::string_view value); + + /// Read an env var (cross-platform). Returns empty string if unset. + static std::string GetEnv(const char* name); +}; + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_logger.cc b/sdk_v2/cpp/src/telemetry/telemetry_logger.cc index 68d250513..60e44faba 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_logger.cc +++ b/sdk_v2/cpp/src/telemetry/telemetry_logger.cc @@ -2,6 +2,8 @@ // Licensed under the MIT License. #include "telemetry/telemetry_logger.h" +#include "telemetry/telemetry_redaction.h" + #include namespace fl { @@ -10,34 +12,94 @@ TelemetryLogger::TelemetryLogger(const std::string& app_name, ILogger& logger) : app_name_(app_name), logger_(logger) { } -void TelemetryLogger::RecordAction(Action action, ActionStatus status, const std::string& user_agent, - bool indirect, int64_t duration_ms) { +void TelemetryLogger::RecordAction(Action action, ActionStatus status, const InvocationContext& context, + int64_t duration_ms) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] AppName:{} UserAgent:{} Command:{} Status:{} Direct:{} Time:{}ms", - app_name_, user_agent, ActionToString(action), - ActionStatusToString(status), !indirect, duration_ms)); + fmt::format("[Telemetry] Action AppName={} UserAgent={} CorrelationId={} Action={} Status={} " + "Direct={} TimeMs={}", + app_name_, context.user_agent, context.correlation_id, ActionToString(action), + ActionStatusToString(status), !context.indirect, duration_ms)); } -void TelemetryLogger::RecordException(Action action, const std::exception& exception) { +void TelemetryLogger::RecordException(Action action, const std::exception& exception, + const InvocationContext& context) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] AppName:{} Command:{} Exception:{}", - app_name_, ActionToString(action), exception.what())); + fmt::format("[Telemetry] Error AppName={} UserAgent={} CorrelationId={} Action={} Exception={}", + app_name_, context.user_agent, context.correlation_id, ActionToString(action), + ScrubTelemetryErrorMessage(exception.what()))); } -void TelemetryLogger::RecordModelUsage(const std::string& model_id, - int64_t prompt_tokens, - int64_t completion_tokens, - int64_t duration_ms) { +void TelemetryLogger::RecordModelUsage(const ModelUsageInfo& info) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] AppName:{} ModelUsage: model={} prompt_tokens={} " - "completion_tokens={} duration={}ms", - app_name_, model_id, prompt_tokens, completion_tokens, duration_ms)); + fmt::format("[Telemetry] Model AppName={} UserAgent={} CorrelationId={} ModelId={} EP={} " + "Stream={} Direct={} TimeToFirstTokenMs={} " + "TotalTimeMs={} TotalTokens={} InputTokenCount={} NumMessages={} MemoryUsedMB={} " + "CpuTimeMs={} GpuMemoryUsedMB={}", + app_name_, info.user_agent, info.correlation_id, info.model_id, + info.execution_provider, info.stream, !info.indirect, + info.time_to_first_token_ms, info.total_time_ms, info.total_tokens, + info.input_token_count, info.num_messages, info.memory_used_mb, + info.cpu_time_ms, info.gpu_memory_used_mb)); } -void TelemetryLogger::RecordModelId(Action action, const std::string& model_id) { +void TelemetryLogger::RecordModelId(Action action, const std::string& model_id, + ActionStatus status, const InvocationContext& context) { logger_.Log(LogLevel::Debug, - fmt::format("[Telemetry] AppName:{} Command:{} ModelId:{}", - app_name_, ActionToString(action), model_id)); + fmt::format("[Telemetry] ModelId AppName={} UserAgent={} CorrelationId={} Action={} ModelId={} " + "Status={}", + app_name_, context.user_agent, context.correlation_id, ActionToString(action), + model_id, ActionStatusToString(status))); +} + +void TelemetryLogger::RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) { + logger_.Log(LogLevel::Debug, + fmt::format("[Telemetry] EPDownloadAttempt AppName={} UserAgent={} CorrelationId={} Attempts={} " + "NumProviders={} Succeeded={} Failed={} Resolved={} Status={} TimeMs={}", + app_name_, info.user_agent, info.correlation_id, info.attempts, info.num_providers, + info.succeeded, info.failed, info.resolved, + ActionStatusToString(info.status), info.duration_ms)); +} + +void TelemetryLogger::RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) { + logger_.Log(LogLevel::Debug, + fmt::format("[Telemetry] EPDownloadAndRegister AppName={} UserAgent={} CorrelationId={} Provider={} " + "InitReadyState={} DownloadReadyState={} DownloadStatus={} DownloadTimeMs={} " + "RegisterReadyState={} RegisterStatus={} RegisterTimeMs={}", + app_name_, info.user_agent, info.correlation_id, info.provider_name, + info.init_ready_state, info.download_ready_state, + ActionStatusToString(info.download_status), info.download_duration_ms, + info.register_ready_state, ActionStatusToString(info.register_status), + info.register_duration_ms)); +} + +void TelemetryLogger::RecordDownload(const DownloadInfo& info) { + logger_.Log(LogLevel::Debug, + fmt::format("[Telemetry] Download AppName={} UserAgent={} CorrelationId={} ModelId={} Status={} " + "LockWaitMs={} EnumerationMs={} DownloadMs={} TotalSizeBytes={} " + "AlreadyCachedBytes={} FileCount={} SkippedFileCount={} " + "DownloadWaitResult={} MaxConcurrency={}", + app_name_, info.user_agent, info.correlation_id, info.model_id, + ActionStatusToString(info.status), info.lock_wait_ms, + info.enumeration_ms, info.download_ms, info.total_size_bytes, + info.already_cached_bytes, info.file_count, info.skipped_file_count, + info.download_wait_result, info.max_concurrency)); +} + +void TelemetryLogger::RecordCatalogFetch(const CatalogFetchInfo& info) { + logger_.Log(LogLevel::Debug, + fmt::format("[Telemetry] CatalogFetch AppName={} Operation={} Endpoint={} Region={} Format={} " + "Status={} TimeMs={} ModelCount={} Error={} UserAgent={} CorrelationId={}", + app_name_, info.operation, info.endpoint, info.region, info.format, + ActionStatusToString(info.status), info.duration_ms, info.model_count, + ScrubTelemetryErrorMessage(info.error_message), info.user_agent, info.correlation_id)); +} + +void TelemetryLogger::StartSession() { + logger_.Log(LogLevel::Debug, fmt::format("[Telemetry] SessionStart AppName={}", app_name_)); +} + +void TelemetryLogger::EndSession() { + logger_.Log(LogLevel::Debug, fmt::format("[Telemetry] SessionEnd AppName={}", app_name_)); } } // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_logger.h b/sdk_v2/cpp/src/telemetry/telemetry_logger.h index 056838846..b07273987 100644 --- a/sdk_v2/cpp/src/telemetry/telemetry_logger.h +++ b/sdk_v2/cpp/src/telemetry/telemetry_logger.h @@ -9,23 +9,35 @@ namespace fl { -/// Stub ITelemetry implementation that logs telemetry events via ILogger. -/// Used as a fallback when no platform-specific telemetry backend is available. +/// ITelemetry implementation that formats telemetry events to ILogger. +/// Used: +/// 1. As the fallback when no 1DS backend is available (cpp-client-telemetry +/// not configured, or FOUNDRY_LOCAL_USE_TELEMETRY=OFF). +/// 2. Inside OneDsTelemetry to provide local diagnostic logging in addition +/// to the 1DS upload. +/// In both cases, the local logger receives every event regardless of CI / +/// FOUNDRY_TESTING_MODE state — those flags only gate the 1DS upload. class TelemetryLogger : public ITelemetry { public: TelemetryLogger(const std::string& app_name, ILogger& logger); - void RecordAction(Action action, ActionStatus status, const std::string& user_agent, - bool indirect, int64_t duration_ms) override; + void RecordAction(Action action, ActionStatus status, const InvocationContext& context, + int64_t duration_ms) override; - void RecordException(Action action, const std::exception& exception) override; + void RecordException(Action action, const std::exception& exception, + const InvocationContext& context) override; - void RecordModelUsage(const std::string& model_id, - int64_t prompt_tokens, - int64_t completion_tokens, - int64_t duration_ms) override; + void RecordModelUsage(const ModelUsageInfo& info) override; - void RecordModelId(Action action, const std::string& model_id) override; + void RecordModelId(Action action, const std::string& model_id, + ActionStatus status, const InvocationContext& context) override; + + void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) override; + void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) override; + void RecordDownload(const DownloadInfo& info) override; + void RecordCatalogFetch(const CatalogFetchInfo& info) override; + void StartSession() override; + void EndSession() override; private: std::string app_name_; diff --git a/sdk_v2/cpp/src/telemetry/telemetry_metadata.cc b/sdk_v2/cpp/src/telemetry/telemetry_metadata.cc new file mode 100644 index 000000000..9aba4ed5f --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/telemetry_metadata.cc @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "telemetry/telemetry_metadata.h" + +#include "telemetry/invocation_context.h" +#include "telemetry/telemetry_environment.h" +#include "version.h" + +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#else +#include +#endif + +namespace fl { + +namespace { + +#ifdef _WIN32 +std::string GetWindowsVersion() { + // GetVersionExA is deprecated and lies for unmanifested apps. The reliable + // approach is to read the build number directly from kernel32 via + // RtlGetVersion, or fall back to the OS version registry. For now, use + // GetVersionEx — the deprecation only affects apps without a manifest, and + // Foundry Local has a manifest declaring Win10 / Win11 compat. + OSVERSIONINFOEXA info{}; + info.dwOSVersionInfoSize = sizeof(info); +#pragma warning(suppress : 4996) // GetVersionExA deprecation + if (::GetVersionExA(reinterpret_cast(&info)) == 0) { + return "unknown"; + } + char buf[64]; + std::snprintf(buf, sizeof(buf), "%lu.%lu.%lu", + info.dwMajorVersion, info.dwMinorVersion, info.dwBuildNumber); + return std::string(buf); +} + +std::string GetCpuArch() { + SYSTEM_INFO si{}; + ::GetNativeSystemInfo(&si); + switch (si.wProcessorArchitecture) { + case PROCESSOR_ARCHITECTURE_AMD64: return "amd64"; + case PROCESSOR_ARCHITECTURE_ARM: return "arm"; + case PROCESSOR_ARCHITECTURE_ARM64: return "arm64"; + case PROCESSOR_ARCHITECTURE_IA64: return "ia64"; + case PROCESSOR_ARCHITECTURE_INTEL: return "x86"; + default: return "unknown"; + } +} +#else +struct PosixOsInfo { + std::string name; + std::string version; + std::string arch; +}; + +PosixOsInfo GetPosixOsInfo() { + PosixOsInfo out{"unknown", "unknown", "unknown"}; + ::utsname u{}; + if (::uname(&u) == 0) { + out.name = u.sysname; + out.version = u.release; + out.arch = u.machine; + } + return out; +} +#endif + +} // namespace + +TelemetryMetadata BuildTelemetryMetadata(std::string app_name) { + TelemetryMetadata m; + m.app_session_guid = MakeGuidV4Hex(); + m.version = FOUNDRY_LOCAL_VERSION; + m.app_name = std::move(app_name); + m.test_mode = TelemetryEnvironment::IsTestingMode(); + +#ifdef _WIN32 + m.os_name = "Windows"; + m.os_version = GetWindowsVersion(); + m.cpu_arch = GetCpuArch(); +#else + auto info = GetPosixOsInfo(); + m.os_name = info.name; + m.os_version = info.version; + m.cpu_arch = info.arch; +#endif + + return m; +} + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_metadata.h b/sdk_v2/cpp/src/telemetry/telemetry_metadata.h new file mode 100644 index 000000000..8d47fc6e2 --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/telemetry_metadata.h @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include + +namespace fl { + +/// Process-wide metadata stamped onto every 1DS event as common context. +/// Computed once at startup and cached. Cheap to copy. +struct TelemetryMetadata { + /// Hex-encoded random 128-bit GUID, generated once at startup. Stamped on + /// every event as `AppSessionGuid` so the backend can group all events from a + /// single FL process run. This is a stable per-process correlation id and is + /// distinct from the SDK's rotating usage-session id (ext.app.sesId), which is + /// driven separately via LogSession(Started/Ended). + std::string app_session_guid; + + /// Foundry Local SDK version (FoundryLocalGetVersionString). + std::string version; + + /// Configured app name (from Configuration::app_name). + std::string app_name; + + /// Free-form "Windows 11 10.0.26100 amd64" / "Linux 6.5.0 x86_64" / "macOS 14.4 arm64". + std::string os_name; // "Windows" / "Linux" / "Darwin" + std::string os_version; // "10.0.26100" / "6.5.0-azure" / "14.4" + std::string cpu_arch; // "amd64" / "arm64" / "x86" / ... + + /// True if FOUNDRY_TESTING_MODE was truthy at startup (stamped per-event as `test`). + bool test_mode = false; +}; + +/// Build the metadata for this process. Reads env vars and OS APIs once. +/// app_name comes from Configuration; version comes from FoundryLocalGetVersionString. +TelemetryMetadata BuildTelemetryMetadata(std::string app_name); + +} // namespace fl diff --git a/sdk_v2/cpp/src/telemetry/telemetry_redaction.h b/sdk_v2/cpp/src/telemetry/telemetry_redaction.h new file mode 100644 index 000000000..72f2fdf8e --- /dev/null +++ b/sdk_v2/cpp/src/telemetry/telemetry_redaction.h @@ -0,0 +1,161 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include +#include +#include + +namespace fl { +namespace telemetry_detail { + +inline bool LooksLikePath(std::string_view token) { + if (token.find("://") != std::string_view::npos) { + return true; + } + if (token.find('\\') != std::string_view::npos) { + return true; + } + if (token.size() >= 3 && std::isalpha(static_cast(token[0])) && token[1] == ':' && + (token[2] == '\\' || token[2] == '/')) { + return true; + } + if (token.size() >= 2 && token[0] == '~' && (token[1] == '/' || token[1] == '\\')) { + return true; + } + + int segments = 0; + for (size_t i = 0; i + 1 < token.size(); ++i) { + if (token[i] == '/' && token[i + 1] != '/') { + ++segments; + } + } + return segments >= 2; +} + +inline std::string RedactPathToken(std::string_view token) { + if (token.find("://") != std::string_view::npos) { + return "[url]"; + } + + const auto is_sep = [](char c) { return c == '/' || c == '\\'; }; + const auto query_start = token.find_first_of("?#"); + if (query_start != std::string_view::npos) { + token = token.substr(0, query_start); + } + + std::string normalized(token); + for (char& c : normalized) { + c = (c == '\\') ? '/' : static_cast(std::tolower(static_cast(c))); + } + + size_t safe_start = 0; + std::string redacted_user; + const auto guard = [&](std::string_view marker, bool has_user) { + for (size_t pos = normalized.find(marker); pos != std::string::npos; + pos = normalized.find(marker, pos + 1)) { + size_t end = pos + marker.size(); + if (has_user) { + while (end < token.size() && + (is_sep(token[end]) || + (token[end] == '.' && (end + 1 == token.size() || is_sep(token[end + 1]))))) { + ++end; + } + const size_t user_start = end; + while (end < token.size() && !is_sep(token[end])) { + ++end; + } + if (end > user_start && redacted_user.empty()) { + redacted_user.assign(token.data() + user_start, end - user_start); + for (char& c : redacted_user) { + c = static_cast(std::tolower(static_cast(c))); + } + } + } else { + --end; + } + if (end > safe_start) { + safe_start = end; + } + } + }; + guard("/home/", true); + guard("/users/", true); + guard("/root/", false); + guard("~/", false); + + size_t scan_end = token.size(); + while (scan_end > 0 && is_sep(token[scan_end - 1])) { + --scan_end; + } + + size_t tail_start = token.size(); + if (scan_end > 0) { + const size_t last_sep = token.find_last_of("/\\", scan_end - 1); + if (last_sep != std::string_view::npos) { + const size_t prev = + (last_sep == 0) ? std::string_view::npos : token.find_last_of("/\\", last_sep - 1); + tail_start = (prev == std::string_view::npos) ? last_sep : prev; + } + } + + const size_t keep_from = (tail_start > safe_start) ? tail_start : safe_start; + std::string out = "[path]"; + if (keep_from < token.size()) { + size_t pos = keep_from; + while (pos < token.size()) { + if (is_sep(token[pos])) { + out.push_back(token[pos++]); + continue; + } + const size_t segment_start = pos; + while (pos < token.size() && !is_sep(token[pos])) { + ++pos; + } + std::string segment(token.data() + segment_start, pos - segment_start); + std::string lowered = segment; + for (char& c : lowered) { + c = static_cast(std::tolower(static_cast(c))); + } + out += (!redacted_user.empty() && lowered == redacted_user) ? "[user]" : segment; + } + } + return out; +} + +} // namespace telemetry_detail + +inline constexpr size_t kMaxTelemetryErrorMessageLength = 256; + +inline std::string ScrubTelemetryErrorMessage(std::string_view message) { + std::string out; + out.reserve(message.size()); + + size_t i = 0; + while (i < message.size()) { + if (std::isspace(static_cast(message[i]))) { + out.push_back(message[i]); + ++i; + continue; + } + + const size_t start = i; + while (i < message.size() && !std::isspace(static_cast(message[i]))) { + ++i; + } + + const std::string_view token = message.substr(start, i - start); + if (telemetry_detail::LooksLikePath(token)) { + out += telemetry_detail::RedactPathToken(token); + } else { + out.append(token.data(), token.size()); + } + } + + if (out.size() > kMaxTelemetryErrorMessageLength) { + out.resize(kMaxTelemetryErrorMessageLength); + } + return out; +} + +} // namespace fl diff --git a/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc b/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc index 4e331d038..204bf3d62 100644 --- a/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc +++ b/sdk_v2/cpp/test/internal_api/base_model_catalog_test.cc @@ -9,6 +9,7 @@ // - Variant grouping behavior // #include "catalog/base_model_catalog.h" +#include "exception.h" #include "internal_api/test_helpers.h" #include "logger.h" #include "model.h" @@ -108,6 +109,38 @@ class QueryingTestCatalog : public BaseModelCatalog { mutable std::vector id_fetch_results_; }; +class FailingRefreshCatalog : public BaseModelCatalog { + public: + explicit FailingRefreshCatalog(ILogger& logger) : BaseModelCatalog("failing-refresh-catalog", logger) {} + + void SetModels(std::vector models) { + models_ = std::move(models); + } + + void FailNextFetch() { + fail_next_fetch_ = true; + } + + int FetchCount() const { + return fetch_count_; + } + + protected: + std::vector FetchModels() const override { + ++fetch_count_; + if (fail_next_fetch_) { + fail_next_fetch_ = false; + FL_THROW(FOUNDRY_LOCAL_ERROR_NETWORK, "simulated refresh failure"); + } + return std::move(models_); + } + + private: + mutable std::vector models_; + mutable bool fail_next_fetch_ = false; + mutable int fetch_count_ = 0; +}; + // Helper: create a Model from basic fields. static Model MakeModel(const std::string& model_id, const std::string& name, int version, const std::string& alias, @@ -208,6 +241,76 @@ TEST_F(BaseModelCatalogTest, GetModel_VariantsAccessible) { EXPECT_EQ(container->Variants().size(), 2u); } +TEST_F(BaseModelCatalogTest, RefreshMergesNewVariantsForExistingAlias) { + TestCatalog catalog(logger_); + catalog.AddModel(MakeModel("phi-3-mini-cpu:1", "phi-3-mini-cpu", 1, "phi-3")); + + Model* container = catalog.GetModel("phi-3"); + ASSERT_NE(container, nullptr); + EXPECT_EQ(container->Variants().size(), 1u); + EXPECT_EQ(catalog.GetModelVariant("phi-3-mini-gpu:1"), nullptr); + + catalog.AddModel(MakeModel("phi-3-mini-gpu:1", "phi-3-mini-gpu", 1, "phi-3")); + catalog.InvalidateCache(); + + auto refreshed = catalog.ListModels(); + ASSERT_EQ(refreshed.size(), 1u); + EXPECT_EQ(refreshed[0]->Variants().size(), 2u); + EXPECT_NE(catalog.GetModelVariant("phi-3-mini-gpu:1"), nullptr); + EXPECT_EQ(catalog.GetModel("phi-3")->Id(), "phi-3-mini-gpu:1"); +} + +TEST_F(BaseModelCatalogTest, RefreshPreservesExplicitVariantSelection) { + TestCatalog catalog(logger_); + catalog.AddModel(MakeModel("phi-3-mini-cpu:1", "phi-3-mini-cpu", 1, "phi-3")); + + Model* container = catalog.GetModel("phi-3"); + ASSERT_NE(container, nullptr); + Model* cpu_variant = catalog.GetModelVariant("phi-3-mini-cpu:1"); + ASSERT_NE(cpu_variant, nullptr); + container->SelectVariant(*cpu_variant); + + catalog.AddModel(MakeModel("phi-3-mini-gpu:1", "phi-3-mini-gpu", 1, "phi-3")); + catalog.InvalidateCache(); + + auto refreshed = catalog.ListModels(); + ASSERT_EQ(refreshed.size(), 1u); + EXPECT_EQ(refreshed[0]->Variants().size(), 2u); + EXPECT_EQ(container->Id(), "phi-3-mini-cpu:1"); +} + +TEST_F(BaseModelCatalogTest, GetLatestVersionReturnsLatestSameModelName) { + TestCatalog catalog(logger_); + catalog.AddModel(MakeModel("phi-3-mini-cpu:1", "phi-3-mini-cpu", 1, "phi-3")); + catalog.AddModel(MakeModel("phi-3-mini-gpu:2", "phi-3-mini-gpu", 2, "phi-3")); + + Model* cpu_variant = catalog.GetModelVariant("phi-3-mini-cpu:1"); + ASSERT_NE(cpu_variant, nullptr); + + Model* latest = catalog.GetLatestVersion(cpu_variant); + ASSERT_NE(latest, nullptr); + EXPECT_EQ(latest->Info().name, "phi-3-mini-cpu"); +} + +TEST_F(BaseModelCatalogTest, ForcedRefreshFailureBacksOffWhenCatalogAlreadyPopulated) { + FailingRefreshCatalog catalog(logger_); + std::vector models; + models.push_back(MakeModel("phi-3-mini-cpu:1", "phi-3-mini-cpu", 1, "phi-3")); + catalog.SetModels(std::move(models)); + + ASSERT_NE(catalog.GetModel("phi-3"), nullptr); + EXPECT_EQ(catalog.FetchCount(), 1); + + catalog.FailNextFetch(); + catalog.InvalidateCache(); + + EXPECT_EQ(catalog.ListModels().size(), 1u); + EXPECT_EQ(catalog.FetchCount(), 2); + + EXPECT_EQ(catalog.ListModels().size(), 1u); + EXPECT_EQ(catalog.FetchCount(), 2); +} + TEST_F(BaseModelCatalogTest, GetModel_NotFound_ReturnsNullptr) { TestCatalog catalog(logger_); catalog.AddModel(MakeModel("phi-3-mini:1", "phi-3-mini", 1, "phi-3")); diff --git a/sdk_v2/cpp/test/internal_api/download_test.cc b/sdk_v2/cpp/test/internal_api/download_test.cc index d8a5f9b5c..cc94b87e8 100644 --- a/sdk_v2/cpp/test/internal_api/download_test.cc +++ b/sdk_v2/cpp/test/internal_api/download_test.cc @@ -459,6 +459,7 @@ TEST(BlobDownloadTest, FiltersOutInferenceModelJson) { }; BlobDownloadOptions opts; + opts.skip_completed_files = true; DownloadBlobsToDirectory(mock, "https://test.blob/c?sig=x", tmpdir.string(), opts); ASSERT_EQ(mock.downloaded_blobs.size(), 2u); @@ -522,6 +523,7 @@ TEST(BlobDownloadTest, SkipsExistingFilesWithCorrectSize) { }; BlobDownloadOptions opts; + opts.skip_completed_files = true; DownloadBlobsToDirectory(mock, "https://test.blob/c?sig=x", tmpdir.string(), opts); // Only the missing blob should be downloaded. @@ -560,6 +562,7 @@ TEST(BlobDownloadTest, ReportsSkippedBytesInInitialProgress) { std::vector progress_values; BlobDownloadOptions opts; + opts.skip_completed_files = true; opts.progress = [&](float pct) { progress_values.push_back(pct); return 0; @@ -587,6 +590,7 @@ TEST(BlobDownloadTest, EmitsHundredPercentWhenEverythingIsCached) { std::vector progress_values; BlobDownloadOptions opts; + opts.skip_completed_files = true; opts.progress = [&](float pct) { progress_values.push_back(pct); return 0; @@ -892,6 +896,156 @@ TEST(DownloadManagerTest, FullDownloadFlow) { EXPECT_FALSE(progress_values.empty()); } +TEST(DownloadManagerTest, RetrySkipsCompletedBlobWithoutSidecar) { + auto tmpdir = TempPath::CreateTempDir(); + + auto manager = std::make_unique(tmpdir.string(), "eastus", 64, fl::test::NullLog()); + + auto registry = std::make_unique( + "eastus", fl::test::NullLog(), std::make_unique(fl::test::NullLog(), false), + [](const std::string&) { + return MakeRegistryResponse( + R"({"blobSasUri": "https://storage.blob.core.windows.net/container?sig=test"})"); + }); + manager->SetModelRegistryClient(std::move(registry)); + + auto mock_downloader = std::make_unique(); + auto* mock_downloader_ptr = mock_downloader.get(); + mock_downloader->blobs_to_return = { + {"weights.safetensors", 1024}, + {"config.json", 100}, + }; + manager->SetBlobDownloader(std::move(mock_downloader)); + + ModelInfo info; + info.model_id = "test-model:1"; + info.name = "test-model"; + info.uri = "azureml://registries/test/models/test-model/versions/1"; + info.string_properties[FOUNDRY_LOCAL_MODEL_PROP_PUBLISHER_STR] = "TestPublisher"; + + auto model_dir = fs::path(tmpdir.string()) / "TestPublisher" / "test-model-1"; + fs::create_directories(model_dir); + { + std::ofstream signal(model_dir / "download.tmp"); + signal << "foundry-local-sidecar-safe-v1\n"; + } + { + std::ofstream f(model_dir / "weights.safetensors", std::ios::binary); + f.seekp(1023); + f.put('\0'); + } + ASSERT_EQ(fs::file_size(model_dir / "weights.safetensors"), static_cast(1024)); + + manager->DownloadModel(info); + + ASSERT_EQ(mock_downloader_ptr->downloaded_blobs.size(), 1u); + EXPECT_EQ(mock_downloader_ptr->downloaded_blobs[0], "config.json"); +} + +TEST(DownloadManagerTest, LegacyIncompleteRetryRedownloadsFullSizeBlobWithoutSidecar) { + auto tmpdir = TempPath::CreateTempDir(); + + auto manager = std::make_unique(tmpdir.string(), "eastus", 64, fl::test::NullLog()); + + auto registry = std::make_unique( + "eastus", fl::test::NullLog(), std::make_unique(fl::test::NullLog(), false), + [](const std::string&) { + return MakeRegistryResponse( + R"({"blobSasUri": "https://storage.blob.core.windows.net/container?sig=test"})"); + }); + manager->SetModelRegistryClient(std::move(registry)); + + auto mock_downloader = std::make_unique(); + auto* mock_downloader_ptr = mock_downloader.get(); + mock_downloader->blobs_to_return = { + {"weights.safetensors", 1024}, + {"config.json", 100}, + }; + manager->SetBlobDownloader(std::move(mock_downloader)); + + ModelInfo info; + info.model_id = "test-model:1"; + info.name = "test-model"; + info.uri = "azureml://registries/test/models/test-model/versions/1"; + info.string_properties[FOUNDRY_LOCAL_MODEL_PROP_PUBLISHER_STR] = "TestPublisher"; + + auto model_dir = fs::path(tmpdir.string()) / "TestPublisher" / "test-model-1"; + fs::create_directories(model_dir); + { + std::ofstream signal(model_dir / "download.tmp"); + signal << "legacy-incomplete-download"; + } + { + std::ofstream f(model_dir / "weights.safetensors", std::ios::binary); + f.seekp(1023); + f.put('\0'); + } + + manager->DownloadModel(info); + + ASSERT_EQ(mock_downloader_ptr->downloaded_blobs.size(), 2u); + EXPECT_NE(std::find(mock_downloader_ptr->downloaded_blobs.begin(), mock_downloader_ptr->downloaded_blobs.end(), + "weights.safetensors"), + mock_downloader_ptr->downloaded_blobs.end()); + EXPECT_NE(std::find(mock_downloader_ptr->downloaded_blobs.begin(), mock_downloader_ptr->downloaded_blobs.end(), + "config.json"), + mock_downloader_ptr->downloaded_blobs.end()); +} + +TEST(DownloadManagerTest, FailedLegacyRepairDoesNotPromoteToSidecarSafeRetry) { + auto tmpdir = TempPath::CreateTempDir(); + + int registry_calls = 0; + auto manager = std::make_unique(tmpdir.string(), "eastus", 64, fl::test::NullLog()); + auto registry = std::make_unique( + "eastus", fl::test::NullLog(), std::make_unique(fl::test::NullLog(), false), + [®istry_calls](const std::string&) { + ++registry_calls; + if (registry_calls == 1) { + return MakeRegistryResponse("temporary failure", 503); + } + return MakeRegistryResponse( + R"({"blobSasUri": "https://storage.blob.core.windows.net/container?sig=test"})"); + }); + manager->SetModelRegistryClient(std::move(registry)); + + auto mock_downloader = std::make_unique(); + auto* mock_downloader_ptr = mock_downloader.get(); + mock_downloader->blobs_to_return = { + {"weights.safetensors", 1024}, + {"config.json", 100}, + }; + manager->SetBlobDownloader(std::move(mock_downloader)); + + ModelInfo info; + info.model_id = "test-model:1"; + info.name = "test-model"; + info.uri = "azureml://registries/test/models/test-model/versions/1"; + info.string_properties[FOUNDRY_LOCAL_MODEL_PROP_PUBLISHER_STR] = "TestPublisher"; + + auto model_dir = fs::path(tmpdir.string()) / "TestPublisher" / "test-model-1"; + fs::create_directories(model_dir); + { + std::ofstream signal(model_dir / "download.tmp"); + signal << "legacy-incomplete-download"; + } + { + std::ofstream f(model_dir / "weights.safetensors", std::ios::binary); + f.seekp(1023); + f.put('\0'); + } + + EXPECT_THROW(manager->DownloadModel(info), fl::Exception); + EXPECT_TRUE(mock_downloader_ptr->downloaded_blobs.empty()); + + manager->DownloadModel(info); + + ASSERT_EQ(mock_downloader_ptr->downloaded_blobs.size(), 2u); + EXPECT_NE(std::find(mock_downloader_ptr->downloaded_blobs.begin(), mock_downloader_ptr->downloaded_blobs.end(), + "weights.safetensors"), + mock_downloader_ptr->downloaded_blobs.end()); +} + // --- Region resolution: detected region drives the download endpoint --- // Run one download and return the registry URL the manager hit. @@ -1493,6 +1647,17 @@ TEST(DownloadManagerTest, RejectsColonInBareModelId) { EXPECT_THROW(manager.GetModelCachePath(info), fl::Exception); } +TEST(DownloadManagerTest, RejectsNonNumericVersionSuffix) { + auto tmpdir = TempPath::CreateTempDir(); + DownloadManager manager(tmpdir.string(), "eastus", 64, fl::test::NullLog()); + + ModelInfo info; + info.model_id = "test:preview"; + info.string_properties[FOUNDRY_LOCAL_MODEL_PROP_PUBLISHER_STR] = "Publisher"; + + EXPECT_THROW(manager.GetModelCachePath(info), fl::Exception); +} + TEST(DownloadManagerTest, RejectsTrailingDotInPublisher) { auto tmpdir = TempPath::CreateTempDir(); DownloadManager manager(tmpdir.string(), "eastus", 64, fl::test::NullLog()); @@ -1531,7 +1696,7 @@ TEST(DownloadManagerTest, AcceptsNormalModelIdAndPublisher) { // ======================================================================== // AzureBlobDownloader resume + cancel-cascade tests -// Use a subclass that overrides the protected GetBlobSize / DownloadChunkStreaming +// Use a subclass that overrides the protected GetBlobProperties / DownloadChunkStreaming // virtuals to bypass the real Azure SDK and simulate per-chunk behavior. // ======================================================================== @@ -1542,6 +1707,7 @@ namespace { class FakeChunkAzureDownloader : public AzureBlobDownloader { public: int64_t blob_size = 0; + std::string blob_identity; /// Per-call hook. Receives the chunk offset and size plus a `sink` callback /// that forwards bytes to the file writer. Allowed to: @@ -1566,7 +1732,9 @@ class FakeChunkAzureDownloader : public AzureBlobDownloader { FakeChunkAzureDownloader() : AzureBlobDownloader(fl::test::NullLog()) {} protected: - int64_t GetBlobSize(ChunkContext& /*ctx*/) override { return blob_size; } + BlobProperties GetBlobProperties(ChunkContext& /*ctx*/) override { + return BlobProperties{blob_size, blob_identity}; + } void DownloadChunkStreaming(ChunkContext& ctx, int64_t offset, int64_t size, std::vector& scratch, @@ -1797,6 +1965,42 @@ TEST(AzureBlobDownloaderResumeTest, CleansUpSidecarOnEmptyBlob) { EXPECT_EQ(d.chunk_call_count.load(), 0); } +TEST(AzureBlobDownloaderResumeTest, CompleteUnfinalizedSidecarDoesNotEmitRegressingProgress) { + auto tmpdir = TempPath::CreateTempDir(); + auto local = tmpdir.path() / "blob.bin"; + + constexpr int32_t kChunkSize = 2 * 1024 * 1024; + constexpr int32_t kNumChunks = 2; + constexpr int64_t kBlobSize = static_cast(kNumChunks) * kChunkSize; + + { + std::ofstream f(local, std::ios::binary); + f.seekp(kBlobSize - 1); + f.put('\0'); + } + { + auto state = BlobDownloadState::CreateNew("blob", local, kBlobSize, kChunkSize, kNumChunks); + for (int32_t i = 0; i < kNumChunks; ++i) { + state->MarkChunkComplete(i); + } + ASSERT_TRUE(state->SaveState(fl::test::NullLog())); + } + + FakeChunkAzureDownloader d; + d.blob_size = kBlobSize; + std::vector progress; + + d.DownloadBlob(/*sas_uri=*/"", "blob", local.string(), /*max_concurrency=*/2, + [&](int64_t bytes) { progress.push_back(bytes); }); + + ASSERT_FALSE(progress.empty()); + EXPECT_LT(progress.front(), kBlobSize); + for (size_t i = 1; i < progress.size(); ++i) { + EXPECT_GE(progress[i], progress[i - 1]); + } + EXPECT_EQ(progress.back(), kBlobSize); +} + TEST(AzureBlobDownloaderResumeTest, ChunkFailureCancelsInFlightPeersFast) { auto tmpdir = TempPath::CreateTempDir(); auto local = tmpdir.path() / "blob.bin"; diff --git a/sdk_v2/cpp/test/internal_api/ep_detector_test.cc b/sdk_v2/cpp/test/internal_api/ep_detector_test.cc index 8feaef302..c3e24ae70 100644 --- a/sdk_v2/cpp/test/internal_api/ep_detector_test.cc +++ b/sdk_v2/cpp/test/internal_api/ep_detector_test.cc @@ -199,24 +199,26 @@ TEST_F(EpDetectorTest, DownloadFiltered_UnknownNamesSkipped) { std::vector names = {"CUDAExecutionProvider", "NonExistentProvider"}; auto result = detector->DownloadAndRegisterEps(&names, nullptr); - EXPECT_TRUE(result.success); + EXPECT_FALSE(result.success); ASSERT_EQ(result.registered_eps.size(), 1u); EXPECT_EQ(result.registered_eps[0], "CUDAExecutionProvider"); - EXPECT_TRUE(result.failed_eps.empty()); + ASSERT_EQ(result.failed_eps.size(), 1u); + EXPECT_EQ(result.failed_eps[0], "NonExistentProvider"); EXPECT_TRUE(mocks[0]->download_called_); } -TEST_F(EpDetectorTest, DownloadFiltered_AllNamesUnknown_SucceedsWithNothing) { +TEST_F(EpDetectorTest, DownloadFiltered_AllNamesUnknownFailsWithRequestedName) { std::vector mocks; auto detector = MakeDetector(mocks, {{"CUDAExecutionProvider", true}}); std::vector names = {"FakeProvider"}; auto result = detector->DownloadAndRegisterEps(&names, nullptr); - EXPECT_TRUE(result.success); + EXPECT_FALSE(result.success); EXPECT_TRUE(result.registered_eps.empty()); - EXPECT_TRUE(result.failed_eps.empty()); + ASSERT_EQ(result.failed_eps.size(), 1u); + EXPECT_EQ(result.failed_eps[0], "FakeProvider"); EXPECT_FALSE(mocks[0]->download_called_); } \ No newline at end of file diff --git a/sdk_v2/cpp/test/internal_api/model_load_manager_test.cc b/sdk_v2/cpp/test/internal_api/model_load_manager_test.cc index 695f0e5d7..9d91a94f0 100644 --- a/sdk_v2/cpp/test/internal_api/model_load_manager_test.cc +++ b/sdk_v2/cpp/test/internal_api/model_load_manager_test.cc @@ -213,6 +213,19 @@ TEST_F(ModelLoadManagerUnloadTest, UnloadThrowsWhenSessionsLive) { instance_->ReleaseSession(); } +TEST_F(ModelLoadManagerUnloadTest, AcquiredLoadedModelLeaseBlocksUnloadUntilReleased) { + auto lease = mgr_->AcquireLoadedModel(fl::test::kTestChatModelAlias); + ASSERT_TRUE(lease); + EXPECT_EQ(lease.get(), instance_); + EXPECT_EQ(instance_->SessionRefCount(), 1); + + EXPECT_THROW(mgr_->UnloadModel(fl::test::kTestChatModelAlias), fl::Exception); + + lease.Reset(); + EXPECT_EQ(instance_->SessionRefCount(), 0); + EXPECT_TRUE(mgr_->UnloadModel(fl::test::kTestChatModelAlias)); +} + TEST_F(ModelLoadManagerUnloadTest, UnloadFailsWhenInUseAndSucceedsAfterSessionsReleased) { instance_->AcquireSession(); EXPECT_THROW(mgr_->UnloadModel(fl::test::kTestChatModelAlias), fl::Exception); diff --git a/sdk_v2/cpp/test/internal_api/null_telemetry.h b/sdk_v2/cpp/test/internal_api/null_telemetry.h index 6d2c11c16..c0a122fe6 100644 --- a/sdk_v2/cpp/test/internal_api/null_telemetry.h +++ b/sdk_v2/cpp/test/internal_api/null_telemetry.h @@ -10,17 +10,23 @@ namespace fl::test { class NullTelemetry : public ITelemetry { public: void RecordAction(Action /*action*/, ActionStatus /*status*/, - const std::string& /*user_agent*/, - bool /*indirect*/, int64_t /*duration_ms*/) override {} + const InvocationContext& /*context*/, int64_t /*duration_ms*/) override {} - void RecordException(Action /*action*/, const std::exception& /*exception*/) override {} + void RecordException(Action /*action*/, const std::exception& /*exception*/, + const InvocationContext& /*context*/) override {} - void RecordModelUsage(const std::string& /*model_id*/, - int64_t /*prompt_tokens*/, - int64_t /*completion_tokens*/, - int64_t /*duration_ms*/) override {} + void RecordModelUsage(const ModelUsageInfo& /*info*/) override {} - void RecordModelId(Action /*action*/, const std::string& /*model_id*/) override {} + void RecordModelId(Action /*action*/, const std::string& /*model_id*/, + ActionStatus /*status*/, const InvocationContext& /*context*/) override {} + + void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& /*info*/) override {} + + void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& /*info*/) override {} + + void RecordDownload(const DownloadInfo& /*info*/) override {} + + void RecordCatalogFetch(const CatalogFetchInfo& /*info*/) override {} }; } // namespace fl::test diff --git a/sdk_v2/cpp/test/internal_api/session_manager_test.cc b/sdk_v2/cpp/test/internal_api/session_manager_test.cc index 9e485781a..914fb5cc4 100644 --- a/sdk_v2/cpp/test/internal_api/session_manager_test.cc +++ b/sdk_v2/cpp/test/internal_api/session_manager_test.cc @@ -145,6 +145,31 @@ TEST_F(SessionManagerTest, CheckInAndCheckOutRoundTrip) { EXPECT_EQ(mgr.CacheSize(), 0u); } +TEST(SessionManagerStandaloneTest, CheckOutModelMismatchLeavesCachedSession) { + StderrLogger logger; + SessionManager mgr(logger); + + mgr.CheckIn("resp-1", nullptr, "model-a"); + + EXPECT_EQ(mgr.CheckOut("resp-1", "model-b"), nullptr); + EXPECT_EQ(mgr.CacheSize(), 1u); + + auto checked_out = mgr.CheckOut("resp-1", "model-a"); + EXPECT_EQ(checked_out, nullptr); + EXPECT_EQ(mgr.CacheSize(), 0u); +} + +TEST(SessionManagerStandaloneTest, CheckInAfterCancelAllDropsSession) { + StderrLogger logger; + SessionManager mgr(logger); + mgr.CancelAll(); + + mgr.CheckIn("resp-1", nullptr); + + EXPECT_EQ(mgr.CacheSize(), 0u); + EXPECT_EQ(mgr.CheckOut("resp-1"), nullptr); +} + TEST_F(SessionManagerTest, CheckOutRemovesFromCache) { SessionManager mgr(GetLogger()); auto session = MakeSession(); diff --git a/sdk_v2/cpp/test/internal_api/telemetry_test.cc b/sdk_v2/cpp/test/internal_api/telemetry_test.cc index 56683cd68..82fadcc1e 100644 --- a/sdk_v2/cpp/test/internal_api/telemetry_test.cc +++ b/sdk_v2/cpp/test/internal_api/telemetry_test.cc @@ -3,6 +3,7 @@ #include "logger.h" #include "telemetry/telemetry_action_tracker.h" #include "telemetry/telemetry_logger.h" +#include "telemetry/telemetry_redaction.h" #include @@ -42,38 +43,49 @@ struct ActionCall { class RecordingTelemetry : public ITelemetry { public: void RecordAction(Action action, ActionStatus status, - const std::string& user_agent, - bool indirect, int64_t duration_ms) override { - action_calls.push_back(ActionCall{action, status, user_agent, indirect, duration_ms}); + const InvocationContext& context, int64_t duration_ms) override { + action_calls.push_back( + ActionCall{action, status, context.user_agent, context.indirect, duration_ms}); } - void RecordException(Action action, const std::exception& exception) override { + void RecordException(Action action, const std::exception& exception, + const InvocationContext& /*context*/) override { exception_calls.emplace_back(action, exception.what()); } - void RecordModelUsage(const std::string& model_id, - int64_t prompt_tokens, - int64_t completion_tokens, - int64_t duration_ms) override { - model_usage_calls.push_back( - ModelUsageCall{model_id, prompt_tokens, completion_tokens, duration_ms}); + void RecordModelUsage(const ModelUsageInfo& info) override { + model_usage_calls.push_back(info); } - void RecordModelId(Action action, const std::string& model_id) override { + void RecordModelId(Action action, const std::string& model_id, + ActionStatus /*status*/, const InvocationContext& /*context*/) override { model_id_calls.emplace_back(action, model_id); } - struct ModelUsageCall { - std::string model_id; - int64_t prompt_tokens; - int64_t completion_tokens; - int64_t duration_ms; - }; + void RecordEpDownloadAttempt(const EpDownloadAttemptInfo& info) override { + ep_attempt_calls.push_back(info); + } + + void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo& info) override { + ep_register_calls.push_back(info); + } + + void RecordDownload(const DownloadInfo& info) override { + download_calls.push_back(info); + } + + void RecordCatalogFetch(const CatalogFetchInfo& info) override { + catalog_fetch_calls.push_back(info); + } std::vector action_calls; std::vector> exception_calls; - std::vector model_usage_calls; + std::vector model_usage_calls; std::vector> model_id_calls; + std::vector ep_attempt_calls; + std::vector ep_register_calls; + std::vector download_calls; + std::vector catalog_fetch_calls; }; } // namespace @@ -82,49 +94,97 @@ TEST(TelemetryLoggerTest, RecordActionIncludesConcreteFields) { RecordingLogger logger; TelemetryLogger telemetry("foundry-local", logger); - telemetry.RecordAction(Action::kModelDownload, ActionStatus::kSuccess, - "cli/1.0", false, 1234); + telemetry.RecordAction(Action::kModelFileDownload, ActionStatus::kSuccess, + InvocationContext{"cli/1.0", "corr-1", false}, 1234); ASSERT_EQ(logger.entries.size(), 1u); EXPECT_EQ(logger.entries[0].level, LogLevel::Debug); - EXPECT_NE(logger.entries[0].message.find("AppName:foundry-local"), std::string::npos); - EXPECT_NE(logger.entries[0].message.find("UserAgent:cli/1.0"), std::string::npos); - EXPECT_NE(logger.entries[0].message.find("Command:ModelDownload"), std::string::npos); - EXPECT_NE(logger.entries[0].message.find("Status:Success"), std::string::npos); - EXPECT_NE(logger.entries[0].message.find("Direct:true"), std::string::npos); - EXPECT_NE(logger.entries[0].message.find("Time:1234ms"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("AppName=foundry-local"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("UserAgent=cli/1.0"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("CorrelationId=corr-1"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("Action=ModelFileDownload"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("Status=Success"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("Direct=true"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("TimeMs=1234"), std::string::npos); } TEST(TelemetryLoggerTest, RecordExceptionAndModelEventsIncludeSpecificValues) { RecordingLogger logger; TelemetryLogger telemetry("foundry-local", logger); - telemetry.RecordException(Action::kModelLoad, std::runtime_error("config missing")); - telemetry.RecordModelUsage("phi-3-mini", 17, 31, 250); - telemetry.RecordModelId(Action::kModelLoad, "phi-3-mini"); + telemetry.RecordException(Action::kModelLoad, std::runtime_error("config missing"), InvocationContext{}); + + ModelUsageInfo usage; + usage.model_id = "phi-3-mini"; + usage.execution_provider = "CPU"; + usage.user_agent = "cli/1.0"; + usage.total_tokens = 31; + usage.input_token_count = 17; + usage.total_time_ms = 250; + telemetry.RecordModelUsage(usage); + + telemetry.RecordModelId(Action::kModelLoad, "phi-3-mini", ActionStatus::kSuccess, + InvocationContext{"cli/1.0", "", false}); ASSERT_EQ(logger.entries.size(), 3u); - EXPECT_NE(logger.entries[0].message.find("Command:ModelLoad"), std::string::npos); - EXPECT_NE(logger.entries[0].message.find("Exception:config missing"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("Action=ModelLoad"), std::string::npos); + EXPECT_NE(logger.entries[0].message.find("Exception=config missing"), std::string::npos); + + EXPECT_NE(logger.entries[1].message.find("Model "), std::string::npos); + EXPECT_NE(logger.entries[1].message.find("ModelId=phi-3-mini"), std::string::npos); + EXPECT_NE(logger.entries[1].message.find("InputTokenCount=17"), std::string::npos); + EXPECT_NE(logger.entries[1].message.find("TotalTokens=31"), std::string::npos); + EXPECT_NE(logger.entries[1].message.find("TotalTimeMs=250"), std::string::npos); + + EXPECT_NE(logger.entries[2].message.find("Action=ModelLoad"), std::string::npos); + EXPECT_NE(logger.entries[2].message.find("ModelId=phi-3-mini"), std::string::npos); +} + +TEST(TelemetryRedactionTest, ScrubsHomePathsAndKeepsUsefulTail) { + const std::string message = + "failed to open /home/alice/.cache/foundry/model/config.json and C:\\Users\\Bob\\AppData\\file.bin"; + + const auto scrubbed = ScrubTelemetryErrorMessage(message); + + EXPECT_EQ(scrubbed.find("alice"), std::string::npos); + EXPECT_EQ(scrubbed.find("Bob"), std::string::npos); + EXPECT_NE(scrubbed.find("[path]/model/config.json"), std::string::npos); + EXPECT_NE(scrubbed.find("[path]\\AppData\\file.bin"), std::string::npos); +} + +TEST(TelemetryRedactionTest, ScrubsRepeatedUserNameInRetainedTail) { + const auto scrubbed = ScrubTelemetryErrorMessage(R"(failed C:\Users\Alice\src\Alice\model.onnx)"); + + EXPECT_EQ(scrubbed.find("Alice"), std::string::npos); + EXPECT_EQ(scrubbed.find("alice"), std::string::npos); + EXPECT_NE(scrubbed.find("[path]\\[user]\\model.onnx"), std::string::npos); +} + +TEST(TelemetryRedactionTest, ScrubsUrlQueryAndFragment) { + const auto scrubbed = ScrubTelemetryErrorMessage( + "GET https://custom.example.com/private/catalog?token=secret#fragment failed"); + + EXPECT_EQ(scrubbed.find("custom.example.com"), std::string::npos); + EXPECT_EQ(scrubbed.find("token=secret"), std::string::npos); + EXPECT_EQ(scrubbed.find("#fragment"), std::string::npos); + EXPECT_NE(scrubbed.find("[url]"), std::string::npos); +} - EXPECT_NE(logger.entries[1].message.find("ModelUsage: model=phi-3-mini"), std::string::npos); - EXPECT_NE(logger.entries[1].message.find("prompt_tokens=17"), std::string::npos); - EXPECT_NE(logger.entries[1].message.find("completion_tokens=31"), std::string::npos); - EXPECT_NE(logger.entries[1].message.find("duration=250ms"), std::string::npos); +TEST(TelemetryRedactionTest, TruncatesLongErrorMessages) { + const auto scrubbed = ScrubTelemetryErrorMessage(std::string(300, 'x')); - EXPECT_NE(logger.entries[2].message.find("Command:ModelLoad"), std::string::npos); - EXPECT_NE(logger.entries[2].message.find("ModelId:phi-3-mini"), std::string::npos); + EXPECT_EQ(scrubbed.size(), kMaxTelemetryErrorMessageLength); } TEST(ActionTrackerTest, DestructorRecordsFailureByDefaultWithoutModelId) { RecordingTelemetry telemetry; { - ActionTracker tracker(Action::kModelDelete, telemetry, "cli/2.0", true); + ActionTracker tracker(Action::kModelFileDownload, telemetry, InvocationContext{"cli/2.0", "", true}); } ASSERT_EQ(telemetry.action_calls.size(), 1u); - EXPECT_EQ(telemetry.action_calls[0].action, Action::kModelDelete); + EXPECT_EQ(telemetry.action_calls[0].action, Action::kModelFileDownload); EXPECT_EQ(telemetry.action_calls[0].status, ActionStatus::kFailure); EXPECT_EQ(telemetry.action_calls[0].user_agent, "cli/2.0"); EXPECT_TRUE(telemetry.action_calls[0].indirect); @@ -136,7 +196,7 @@ TEST(ActionTrackerTest, RecordsExceptionSuccessAndModelId) { RecordingTelemetry telemetry; { - ActionTracker tracker(Action::kModelLoad, telemetry, "cli/3.0", false); + ActionTracker tracker(Action::kModelLoad, telemetry, InvocationContext{"cli/3.0", "", false}); tracker.RecordException(std::runtime_error("failed to load")); tracker.SetModelId("phi-3-mini"); tracker.SetStatus(ActionStatus::kSuccess); diff --git a/sdk_v2/cpp/test/internal_api/web_service_test.cc b/sdk_v2/cpp/test/internal_api/web_service_test.cc index 4b33be0a3..6f76601bc 100644 --- a/sdk_v2/cpp/test/internal_api/web_service_test.cc +++ b/sdk_v2/cpp/test/internal_api/web_service_test.cc @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -58,6 +59,63 @@ std::string TestHttpDelete(const std::string& url, const std::string& user_agent return http::HttpDelete(url, options); } +// Telemetry sink that records every event for assertions. +class CapturingTelemetry : public ITelemetry { + public: + struct ActionCall { + Action action; + ActionStatus status; + std::string user_agent; + std::string correlation_id; + bool indirect; + }; + + void RecordAction(Action action, ActionStatus status, const InvocationContext& context, + int64_t /*duration_ms*/) override { + std::lock_guard lock(mutex_); + actions.push_back({action, status, context.user_agent, context.correlation_id, context.indirect}); + } + void RecordException(Action, const std::exception&, const InvocationContext&) override {} + void RecordModelUsage(const ModelUsageInfo& info) override { + std::lock_guard lock(mutex_); + model_usages.push_back(info); + } + void RecordModelId(Action, const std::string&, ActionStatus, const InvocationContext&) override {} + void RecordEpDownloadAttempt(const EpDownloadAttemptInfo&) override {} + void RecordEpDownloadAndRegister(const EpDownloadAndRegisterInfo&) override {} + void RecordDownload(const DownloadInfo&) override {} + void RecordCatalogFetch(const CatalogFetchInfo& info) override { + std::lock_guard lock(mutex_); + catalog_fetches.push_back(info); + } + + int Count(Action action) { + std::lock_guard lock(mutex_); + int n = 0; + for (const auto& c : actions) { + if (c.action == action) { + ++n; + } + } + return n; + } + + std::optional Find(Action action) { + std::lock_guard lock(mutex_); + for (const auto& c : actions) { + if (c.action == action) { + return c; + } + } + return std::nullopt; + } + + std::vector actions; + std::vector model_usages; + std::vector catalog_fetches; + std::mutex mutex_; +}; + } // namespace // ======================================================================== @@ -485,6 +543,86 @@ TEST(WebServiceEmptyCatalogTest, LoadedModelsReturnsEmptyArray) { service.Stop(); } +// ======================================================================== +// Telemetry behaviors — capture events and assert coverage / classification +// ======================================================================== + +TEST(WebServiceTelemetryTest, UnmatchedRouteRecordsServiceRequestUnmatched) { + test::MockCatalog catalog; + StderrLogger logger; + test::CpuOnlyEpDetector ep_detector; + ModelLoadManager model_load_manager(ep_detector, logger); + SessionManager session_manager(logger); + CapturingTelemetry telemetry; + + WebService service(catalog, logger, "/tmp/test", model_load_manager, session_manager, telemetry, []() {}); + auto urls = service.Start({"http://127.0.0.1:0"}); + + // Unknown path: the interceptor replies 404, so TestHttpGet throws on the non-2xx. + try { + TestHttpGet(urls[0] + "/no/such/route", "probe-agent/9"); + } catch (...) { + } + + service.Stop(); + + ASSERT_EQ(telemetry.Count(Action::kServiceRequestUnmatched), 1); + auto call = telemetry.Find(Action::kServiceRequestUnmatched); + ASSERT_TRUE(call.has_value()); + EXPECT_EQ(call->status, ActionStatus::kClientError); + EXPECT_EQ(call->user_agent, "probe-agent/9"); + EXPECT_FALSE(call->indirect); + EXPECT_FALSE(call->correlation_id.empty()); +} + +TEST(WebServiceTelemetryTest, EmptyChatBodyRecordsClientErrorNotFailure) { + test::MockCatalog catalog; + StderrLogger logger; + test::CpuOnlyEpDetector ep_detector; + ModelLoadManager model_load_manager(ep_detector, logger); + SessionManager session_manager(logger); + CapturingTelemetry telemetry; + + WebService service(catalog, logger, "/tmp/test", model_load_manager, session_manager, telemetry, []() {}); + auto urls = service.Start({"http://127.0.0.1:0"}); + + // Empty body is rejected (400) before any model resolution. + try { + TestHttpPost(urls[0] + "/v1/chat/completions", ""); + } catch (...) { + } + + service.Stop(); + + auto call = telemetry.Find(Action::kOpenAIChatCompletions); + ASSERT_TRUE(call.has_value()) << "chat completions route action was not recorded"; + EXPECT_EQ(call->status, ActionStatus::kClientError); + EXPECT_FALSE(call->indirect); + // A 4xx reject performs no inference, so there is no Model event. + EXPECT_TRUE(telemetry.model_usages.empty()); +} + +TEST(WebServiceTelemetryTest, StatusEndpointIsSampledHourly) { + test::MockCatalog catalog; + StderrLogger logger; + test::CpuOnlyEpDetector ep_detector; + ModelLoadManager model_load_manager(ep_detector, logger); + SessionManager session_manager(logger); + CapturingTelemetry telemetry; + + WebService service(catalog, logger, "/tmp/test", model_load_manager, session_manager, telemetry, []() {}); + auto urls = service.Start({"http://127.0.0.1:0"}); + + // Three rapid heartbeats — only the first inside the hourly window is recorded. + TestHttpGet(urls[0] + "/status"); + TestHttpGet(urls[0] + "/status"); + TestHttpGet(urls[0] + "/status"); + + service.Stop(); + + EXPECT_EQ(telemetry.Count(Action::kServiceStatus), 1); +} + // ======================================================================== // Streaming validation tests — same errors apply with stream=true // ======================================================================== diff --git a/sdk_v2/cpp/vcpkg.json b/sdk_v2/cpp/vcpkg.json index 8a3d1037e..241b39995 100644 --- a/sdk_v2/cpp/vcpkg.json +++ b/sdk_v2/cpp/vcpkg.json @@ -2,7 +2,7 @@ "name": "foundry-local", "version-semver": "0.1.0", "description": "Foundry Local C++ SDK", - "builtin-baseline": "256acc64012b23a13041d8705805e1f23b43a024", + "builtin-baseline": "44819aa2a6c10e56065e2b0330e7d6c89d1d2574", "dependencies": [ "azure-storage-blobs-cpp", { @@ -19,7 +19,8 @@ }, "ms-gsl", "nlohmann-json", - "spdlog" + "spdlog", + { "name": "sqlite3", "default-features": false } ], "overrides": [ { "name": "nlohmann-json", "version": "3.11.3" }, @@ -32,6 +33,12 @@ "oatpp" ] }, + "telemetry": { + "description": "Build with 1DS (cpp-client-telemetry) telemetry uploads", + "dependencies": [ + "cpp-client-telemetry" + ] + }, "tests": { "description": "Build unit tests", "dependencies": [ diff --git a/sdk_v2/cs/src/Detail/NativeMethods.cs b/sdk_v2/cs/src/Detail/NativeMethods.cs index fa8e920d7..f5f9541bc 100644 --- a/sdk_v2/cs/src/Detail/NativeMethods.cs +++ b/sdk_v2/cs/src/Detail/NativeMethods.cs @@ -33,7 +33,7 @@ namespace Microsoft.AI.Foundry.Local.Detail.Interop; // ----------------------------------------------------------------------- public static partial class NativeMethods { - public const uint ApiVersion = 1; + public const uint ApiVersion = 2; public const string LibraryName = "foundry_local"; // The first P/Invoke through this class can come from any of several entry diff --git a/sdk_v2/js/native/src/catalog.cc b/sdk_v2/js/native/src/catalog.cc index 8372187a2..d8a210b90 100644 --- a/sdk_v2/js/native/src/catalog.cc +++ b/sdk_v2/js/native/src/catalog.cc @@ -5,6 +5,7 @@ #include "addon_data.h" #include "errors.h" #include "model.h" +#include "manager.h" #include @@ -16,10 +17,24 @@ namespace foundry_local_node { namespace { +std::shared_ptr LockManagerOrThrow( + const std::weak_ptr& manager_keepalive, + const std::shared_ptr& lifecycle) { + if (!lifecycle || lifecycle->disposed.load(std::memory_order_acquire)) { + throw foundry_local::Error("Manager has been disposed", FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + } + auto manager = manager_keepalive.lock(); + if (!manager) { + throw foundry_local::Error("Manager has been disposed", FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + } + return manager; +} + // Wrap a ModelList (rvalue) into a JS array of Model handles, each pinning the // passed-in manager reference. -Napi::Value WrapModelList(Napi::Env env, foundry_local::ModelList ml, - Napi::ObjectReference manager) { +Napi::Value WrapModelList(Napi::Env env, foundry_local::ModelList ml, Napi::ObjectReference manager, + std::weak_ptr manager_keepalive, + std::shared_ptr lifecycle) { auto list = std::make_shared(std::move(ml)); auto models = list->Models(); Napi::Array arr = Napi::Array::New(env, models.size()); @@ -28,6 +43,8 @@ Napi::Value WrapModelList(Napi::Env env, foundry_local::ModelList ml, token.impl = models[i].get(); token.keepalive = list; token.manager = Napi::Reference::New(manager.Value(), 1); + token.manager_keepalive = manager_keepalive; + token.lifecycle = lifecycle; arr.Set(static_cast(i), Model::NewInstance(env, std::move(token))); } return arr; @@ -35,7 +52,9 @@ Napi::Value WrapModelList(Napi::Env env, foundry_local::ModelList ml, // Wrap an owning unique_ptr into a JS Model (or undefined when null). Napi::Value WrapOwnedModelOrUndefined(Napi::Env env, std::unique_ptr owned, - Napi::ObjectReference manager) { + Napi::ObjectReference manager, + std::weak_ptr manager_keepalive, + std::shared_ptr lifecycle) { if (!owned) { return env.Undefined(); } @@ -46,11 +65,13 @@ Napi::Value WrapOwnedModelOrUndefined(Napi::Env env, std::unique_ptr>(std::move(owned)); token.keepalive = holder; token.manager = std::move(manager); + token.manager_keepalive = std::move(manager_keepalive); + token.lifecycle = std::move(lifecycle); return Model::NewInstance(env, std::move(token)); } -// Extract IModel* from a JS Model arg, or return nullptr if not a Model. -foundry_local::IModel* ExtractIModel(const Napi::Value& v) { +// Extract Model* from a JS Model arg, or return nullptr if not a Model. +Model* ExtractModel(const Napi::Value& v) { if (!v.IsObject()) { return nullptr; } @@ -63,8 +84,7 @@ foundry_local::IModel* ExtractIModel(const Napi::Value& v) { if (!obj.InstanceOf(ctor)) { return nullptr; } - Model* m = Napi::ObjectWrap::Unwrap(obj); - return m != nullptr ? m->native_impl() : nullptr; + return Napi::ObjectWrap::Unwrap(obj); } Napi::ObjectReference CloneManager(const Napi::ObjectReference& mgr) { @@ -109,11 +129,15 @@ Catalog::Catalog(const Napi::CallbackInfo& info) : Napi::ObjectWrap(inf } impl_ = token->impl; manager_ = std::move(token->manager); + manager_keepalive_ = std::move(token->manager_keepalive); + lifecycle_ = std::move(token->lifecycle); } Napi::Value Catalog::GetName(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); + (void)manager_alive; std::string_view name = impl_->GetName(); return Napi::String::New(env, std::string(name)); }); @@ -125,14 +149,20 @@ Napi::Value Catalog::GetModels(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked( - env, [&]() -> Napi::Value { return WrapModelList(env, impl_->GetModels(), std::move(mgr)); }); + env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); + (void)manager_alive; + return WrapModelList(env, impl_->GetModels(), std::move(mgr), manager_keepalive_, lifecycle_); + }); } Napi::Value Catalog::GetCachedModels(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked(env, [&]() -> Napi::Value { - return WrapModelList(env, impl_->GetCachedModels(), std::move(mgr)); + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); + (void)manager_alive; + return WrapModelList(env, impl_->GetCachedModels(), std::move(mgr), manager_keepalive_, lifecycle_); }); } @@ -140,7 +170,9 @@ Napi::Value Catalog::GetLoadedModels(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked(env, [&]() -> Napi::Value { - return WrapModelList(env, impl_->GetLoadedModels(), std::move(mgr)); + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); + (void)manager_alive; + return WrapModelList(env, impl_->GetLoadedModels(), std::move(mgr), manager_keepalive_, lifecycle_); }); } @@ -155,8 +187,10 @@ Napi::Value Catalog::GetModel(const Napi::CallbackInfo& info) { std::string alias = info[0].As(); Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); + (void)manager_alive; auto owned = impl_->GetModel(alias); - return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr)); + return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr), manager_keepalive_, lifecycle_); }); } @@ -169,8 +203,10 @@ Napi::Value Catalog::GetModelVariant(const Napi::CallbackInfo& info) { std::string model_id = info[0].As(); Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); + (void)manager_alive; auto owned = impl_->GetModelVariant(model_id); - return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr)); + return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr), manager_keepalive_, lifecycle_); }); } @@ -180,15 +216,20 @@ Napi::Value Catalog::GetLatestVersion(const Napi::CallbackInfo& info) { Napi::TypeError::New(env, "getLatestVersion(model: Model)").ThrowAsJavaScriptException(); return env.Undefined(); } - foundry_local::IModel* arg = ExtractIModel(info[0]); - if (arg == nullptr) { + Model* arg = ExtractModel(info[0]); + if (arg == nullptr || arg->native_impl() == nullptr) { Napi::TypeError::New(env, "getLatestVersion: argument must be a Model").ThrowAsJavaScriptException(); return env.Undefined(); } Napi::ObjectReference mgr = CloneManager(manager_); return CallChecked(env, [&]() -> Napi::Value { - auto owned = impl_->GetLatestVersion(*arg); - return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr)); + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); + if (arg->manager_disposed() || !arg->manager_keepalive()) { + throw foundry_local::Error("Manager has been disposed", FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + } + (void)manager_alive; + auto owned = impl_->GetLatestVersion(*arg->native_impl()); + return WrapOwnedModelOrUndefined(env, std::move(owned), std::move(mgr), manager_keepalive_, lifecycle_); }); } diff --git a/sdk_v2/js/native/src/catalog.h b/sdk_v2/js/native/src/catalog.h index 29ff89cd9..6d8ce8fa3 100644 --- a/sdk_v2/js/native/src/catalog.h +++ b/sdk_v2/js/native/src/catalog.h @@ -16,13 +16,18 @@ #include +#include #include namespace foundry_local_node { +struct ManagerLifecycle; + struct CatalogCtorToken { foundry_local::ICatalog* impl = nullptr; Napi::ObjectReference manager; // pins the owning Manager + std::weak_ptr manager_keepalive; + std::shared_ptr lifecycle; }; class Catalog : public Napi::ObjectWrap { @@ -43,6 +48,8 @@ class Catalog : public Napi::ObjectWrap { foundry_local::ICatalog* impl_ = nullptr; Napi::ObjectReference manager_; + std::weak_ptr manager_keepalive_; + std::shared_ptr lifecycle_; }; } // namespace foundry_local_node diff --git a/sdk_v2/js/native/src/manager.cc b/sdk_v2/js/native/src/manager.cc index 6d359b588..84ca2db89 100644 --- a/sdk_v2/js/native/src/manager.cc +++ b/sdk_v2/js/native/src/manager.cc @@ -20,6 +20,24 @@ namespace foundry_local_node { namespace { +struct WorkerLease { + explicit WorkerLease(std::shared_ptr lifecycle) : lifecycle_(std::move(lifecycle)) { + if (lifecycle_) { + lifecycle_->active_workers.fetch_add(1, std::memory_order_acq_rel); + } + } + ~WorkerLease() { + if (lifecycle_) { + lifecycle_->active_workers.fetch_sub(1, std::memory_order_acq_rel); + } + } + std::shared_ptr lifecycle_; +}; + +std::shared_ptr MakeWorkerLease(std::shared_ptr lifecycle) { + return std::make_shared(std::move(lifecycle)); +} + Napi::Value ConvertEndpoints(Napi::Env env, std::vector& endpoints) { Napi::Array out = Napi::Array::New(env, endpoints.size()); for (size_t i = 0; i < endpoints.size(); ++i) { @@ -193,7 +211,7 @@ Manager::Manager(const Napi::CallbackInfo& info) : Napi::ObjectWrap(inf } config.SetAdditionalOptions(kvp); } - impl_ = std::make_unique(std::move(config)); + impl_ = std::make_shared(std::move(config)); }); } @@ -227,6 +245,8 @@ Napi::Value Manager::GetCatalog(const Napi::CallbackInfo& info) { CatalogCtorToken token; token.impl = &cat; token.manager = std::move(owner); + token.manager_keepalive = impl_; + token.lifecycle = lifecycle_; return Catalog::NewInstance(env, std::move(token)); }); } @@ -234,6 +254,17 @@ Napi::Value Manager::GetCatalog(const Napi::CallbackInfo& info) { Napi::Value Manager::Dispose(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); // Idempotent — releasing an already-null unique_ptr is a no-op. + if (lifecycle_->active_sessions.load(std::memory_order_acquire) > 0) { + ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + "Manager has active sessions; dispose sessions before disposing the manager"); + return env.Undefined(); + } + if (lifecycle_->active_workers.load(std::memory_order_acquire) > 0) { + ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, + "Manager has active native workers; await them before disposing the manager"); + return env.Undefined(); + } + lifecycle_->disposed.store(true, std::memory_order_release); impl_.reset(); return env.Undefined(); } @@ -288,11 +319,13 @@ namespace { // progress callback. Mirrors the pattern in model.cc's DownloadWorker. class EpDownloadWorker : public Napi::AsyncWorker { public: - EpDownloadWorker(Napi::Env env, foundry_local::Manager* impl, std::vector ep_names, - Napi::ObjectReference owner, Napi::ThreadSafeFunction tsfn) + EpDownloadWorker(Napi::Env env, std::shared_ptr impl, std::shared_ptr worker_lease, + std::vector ep_names, Napi::ObjectReference owner, + Napi::ThreadSafeFunction tsfn) : Napi::AsyncWorker(env), deferred_(Napi::Promise::Deferred::New(env)), impl_(impl), + worker_lease_(std::move(worker_lease)), ep_names_(std::move(ep_names)), owner_(std::move(owner)), tsfn_(std::move(tsfn)) {} @@ -356,7 +389,8 @@ class EpDownloadWorker : public Napi::AsyncWorker { } Napi::Promise::Deferred deferred_; - foundry_local::Manager* impl_; + std::shared_ptr impl_; + std::shared_ptr worker_lease_; std::vector ep_names_; Napi::ObjectReference owner_; Napi::ThreadSafeFunction tsfn_; @@ -413,7 +447,8 @@ Napi::Value Manager::DownloadAndRegisterEps(const Napi::CallbackInfo& info) { } Napi::ObjectReference owner = Napi::Reference::New(info.This().As(), 1); - auto* w = new EpDownloadWorker(env, impl_.get(), std::move(ep_names), std::move(owner), std::move(tsfn)); + auto* w = new EpDownloadWorker(env, impl_, MakeWorkerLease(lifecycle_), std::move(ep_names), std::move(owner), + std::move(tsfn)); Napi::Promise p = w->Promise(); w->Queue(); return p; diff --git a/sdk_v2/js/native/src/manager.h b/sdk_v2/js/native/src/manager.h index 0cdd35207..c321a1c12 100644 --- a/sdk_v2/js/native/src/manager.h +++ b/sdk_v2/js/native/src/manager.h @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. // -// Napi::ObjectWrap over std::unique_ptr. +// Napi::ObjectWrap over std::shared_ptr. // // Surface: // - ctor accepts { appName, modelCacheDir?, serviceEndpoint? } @@ -15,10 +15,17 @@ #include +#include #include namespace foundry_local_node { +struct ManagerLifecycle { + std::atomic disposed{false}; + std::atomic active_sessions{0}; + std::atomic active_workers{0}; +}; + class Manager : public Napi::ObjectWrap { public: static Napi::Function Init(Napi::Env env); @@ -55,7 +62,8 @@ class Manager : public Napi::ObjectWrap { // on env and returns true. Callers should return env.Undefined() when true. bool ThrowIfDisposed(Napi::Env env); - std::unique_ptr impl_; + std::shared_ptr impl_; + std::shared_ptr lifecycle_ = std::make_shared(); }; } // namespace foundry_local_node diff --git a/sdk_v2/js/native/src/model.cc b/sdk_v2/js/native/src/model.cc index c7164778f..a6605acdc 100644 --- a/sdk_v2/js/native/src/model.cc +++ b/sdk_v2/js/native/src/model.cc @@ -4,6 +4,7 @@ #include "addon_data.h" #include "errors.h" +#include "manager.h" #include "promise_worker.h" #include @@ -18,6 +19,24 @@ namespace foundry_local_node { namespace { +struct WorkerLease { + explicit WorkerLease(std::shared_ptr lifecycle) : lifecycle_(std::move(lifecycle)) { + if (lifecycle_) { + lifecycle_->active_workers.fetch_add(1, std::memory_order_acq_rel); + } + } + ~WorkerLease() { + if (lifecycle_) { + lifecycle_->active_workers.fetch_sub(1, std::memory_order_acq_rel); + } + } + std::shared_ptr lifecycle_; +}; + +std::shared_ptr MakeWorkerLease(std::shared_ptr lifecycle) { + return std::make_shared(std::move(lifecycle)); +} + const char* DeviceTypeToString(flDeviceType dt) { switch (dt) { case FOUNDRY_LOCAL_DEVICE_CPU: @@ -73,6 +92,46 @@ void SetPromptTemplate(Napi::Env env, Napi::Object obj, const foundry_local::Mod obj.Set("promptTemplate", template_obj); } +std::shared_ptr LockManager( + const std::weak_ptr& manager_keepalive) { + return manager_keepalive.lock(); +} + +void ThrowFoundryLocalError(Napi::Env env, int code, const std::string& msg) { + Napi::Error err = Napi::Error::New(env, msg); + Napi::Object value = err.Value(); + value.Set("name", Napi::String::New(env, "FoundryLocalError")); + value.Set("code", Napi::Number::New(env, code)); + err.ThrowAsJavaScriptException(); +} + +std::shared_ptr LockManagerOrThrow( + const std::weak_ptr& manager_keepalive, + const std::shared_ptr& lifecycle) { + if (!lifecycle || lifecycle->disposed.load(std::memory_order_acquire)) { + throw foundry_local::Error("Manager has been disposed", FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + } + auto manager = LockManager(manager_keepalive); + if (!manager) { + throw foundry_local::Error("Manager has been disposed", FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + } + return manager; +} + +std::shared_ptr LockManagerOrThrowJs( + Napi::Env env, const std::weak_ptr& manager_keepalive, + const std::shared_ptr& lifecycle) { + if (!lifecycle || lifecycle->disposed.load(std::memory_order_acquire)) { + ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Manager has been disposed"); + return nullptr; + } + auto manager = LockManager(manager_keepalive); + if (!manager) { + ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Manager has been disposed"); + } + return manager; +} + void SetModelSettings(Napi::Env env, Napi::Object obj, const foundry_local::ModelInfo& info) { auto settings = info.GetModelSettings(); if (!settings.has_value()) { @@ -144,7 +203,9 @@ Napi::Object SnapshotModelInfo(Napi::Env env, const foundry_local::ModelInfo& in // Drain a ModelList into a JS array, with each entry wrapped as a JS Model // whose keepalive holds the shared ModelList. Napi::Array WrapModelList(Napi::Env env, std::shared_ptr list, - Napi::ObjectReference manager) { + Napi::ObjectReference manager, + std::weak_ptr manager_keepalive, + std::shared_ptr lifecycle) { auto models = list->Models(); Napi::Array arr = Napi::Array::New(env, models.size()); for (size_t i = 0; i < models.size(); ++i) { @@ -153,6 +214,8 @@ Napi::Array WrapModelList(Napi::Env env, std::shared_ptr::New(manager.Value(), 1); + token.manager_keepalive = manager_keepalive; + token.lifecycle = lifecycle; arr.Set(static_cast(i), Model::NewInstance(env, std::move(token))); } return arr; @@ -198,12 +261,20 @@ Model::Model(const Napi::CallbackInfo& info) : Napi::ObjectWrap(info) { } impl_ = token->impl; keepalive_ = std::move(token->keepalive); + manager_keepalive_ = std::move(token->manager_keepalive); + lifecycle_ = std::move(token->lifecycle); manager_ = std::move(token->manager); } +bool Model::manager_disposed() const noexcept { + return !lifecycle_ || lifecycle_->disposed.load(std::memory_order_acquire); +} + Napi::Value Model::GetInfo(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); + (void)manager_alive; foundry_local::ModelInfo mi = impl_->GetInfo(); Napi::Object snapshot = SnapshotModelInfo(env, mi); snapshot.Set("cached", Napi::Boolean::New(env, impl_->IsCached())); @@ -214,6 +285,8 @@ Napi::Value Model::GetInfo(const Napi::CallbackInfo& info) { Napi::Value Model::IsCached(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); + (void)manager_alive; return Napi::Boolean::New(env, impl_->IsCached()); }); } @@ -221,6 +294,8 @@ Napi::Value Model::IsCached(const Napi::CallbackInfo& info) { Napi::Value Model::IsLoaded(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); + (void)manager_alive; return Napi::Boolean::New(env, impl_->IsLoaded()); }); } @@ -228,6 +303,8 @@ Napi::Value Model::IsLoaded(const Napi::CallbackInfo& info) { Napi::Value Model::GetPath(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); + (void)manager_alive; std::string_view p = impl_->GetPath(); return Napi::String::New(env, std::string(p)); }); @@ -237,18 +314,19 @@ Napi::Value Model::GetVariants(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); Napi::ObjectReference owner_clone = Napi::Reference::New(manager_.Value(), 1); return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); + (void)manager_alive; auto list = std::make_shared(impl_->GetVariants()); - return WrapModelList(env, std::move(list), std::move(owner_clone)); + return WrapModelList(env, std::move(list), std::move(owner_clone), manager_keepalive_, lifecycle_); }); } // ── Async lifecycle ───────────────────────────────────────────────────────── // // Load/Unload/Download dispatch the underlying virtual call onto a libuv -// worker so the event loop stays responsive. The Model itself is pinned -// against GC for the duration of the worker via an ObjectReference to the -// parent Manager (the Manager owns the catalog whose ModelList views the -// IModel*). +// worker so the event loop stays responsive. Each worker captures the model's +// native keepalives so explicit Manager.dispose() or JS GC cannot release the +// underlying Manager/ModelList/owned IModel before the worker finishes. Napi::Value Model::Load(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); @@ -258,8 +336,16 @@ Napi::Value Model::Load(const Napi::CallbackInfo& info) { } Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); foundry_local::IModel* m = impl_; + auto keepalive = keepalive_; + auto manager_keepalive = LockManagerOrThrowJs(env, manager_keepalive_, lifecycle_); + if (!manager_keepalive) { + return env.Undefined(); + } + auto worker_lease = MakeWorkerLease(lifecycle_); return PromiseWorkerVoid::Run( - env, [m]() { m->Load(); }, std::move(owner)); + env, [m, keepalive = std::move(keepalive), manager_keepalive = std::move(manager_keepalive), + worker_lease = std::move(worker_lease)]() { m->Load(); }, + std::move(owner)); } Napi::Value Model::Unload(const Napi::CallbackInfo& info) { @@ -270,8 +356,16 @@ Napi::Value Model::Unload(const Napi::CallbackInfo& info) { } Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); foundry_local::IModel* m = impl_; + auto keepalive = keepalive_; + auto manager_keepalive = LockManagerOrThrowJs(env, manager_keepalive_, lifecycle_); + if (!manager_keepalive) { + return env.Undefined(); + } + auto worker_lease = MakeWorkerLease(lifecycle_); return PromiseWorkerVoid::Run( - env, [m]() { m->Unload(); }, std::move(owner)); + env, [m, keepalive = std::move(keepalive), manager_keepalive = std::move(manager_keepalive), + worker_lease = std::move(worker_lease)]() { m->Unload(); }, + std::move(owner)); } namespace { @@ -282,11 +376,15 @@ namespace { // worker queues and released in OnOK/OnError. class DownloadWorker : public Napi::AsyncWorker { public: - DownloadWorker(Napi::Env env, foundry_local::IModel* impl, Napi::ObjectReference owner, - Napi::ThreadSafeFunction tsfn) + DownloadWorker(Napi::Env env, foundry_local::IModel* impl, std::shared_ptr keepalive, + std::shared_ptr manager_keepalive, std::shared_ptr worker_lease, + Napi::ObjectReference owner, Napi::ThreadSafeFunction tsfn) : Napi::AsyncWorker(env), deferred_(Napi::Promise::Deferred::New(env)), impl_(impl), + keepalive_(std::move(keepalive)), + manager_keepalive_(std::move(manager_keepalive)), + worker_lease_(std::move(worker_lease)), owner_(std::move(owner)), tsfn_(std::move(tsfn)) {} @@ -350,6 +448,9 @@ class DownloadWorker : public Napi::AsyncWorker { Napi::Promise::Deferred deferred_; foundry_local::IModel* impl_; + std::shared_ptr keepalive_; + std::shared_ptr manager_keepalive_; + std::shared_ptr worker_lease_; Napi::ObjectReference owner_; Napi::ThreadSafeFunction tsfn_; std::string err_msg_; @@ -366,6 +467,11 @@ Napi::Value Model::Download(const Napi::CallbackInfo& info) { return env.Undefined(); } + auto manager_keepalive = LockManagerOrThrowJs(env, manager_keepalive_, lifecycle_); + if (!manager_keepalive) { + return env.Undefined(); + } + Napi::ThreadSafeFunction tsfn; if (info.Length() >= 1 && info[0].IsFunction()) { tsfn = Napi::ThreadSafeFunction::New(env, info[0].As(), @@ -379,7 +485,9 @@ Napi::Value Model::Download(const Napi::CallbackInfo& info) { } Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - auto* w = new DownloadWorker(env, impl_, std::move(owner), std::move(tsfn)); + auto* w = new DownloadWorker(env, impl_, keepalive_, std::move(manager_keepalive), MakeWorkerLease(lifecycle_), + std::move(owner), + std::move(tsfn)); Napi::Promise p = w->Promise(); w->Queue(); return p; @@ -395,6 +503,8 @@ Napi::Value Model::RemoveFromCache(const Napi::CallbackInfo& info) { // V1's contract is `removeFromCache(): void` so we do not bounce to a // worker. return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); + (void)manager_alive; impl_->RemoveFromCache(); return env.Undefined(); }); @@ -422,6 +532,10 @@ Napi::Value Model::SelectVariant(const Napi::CallbackInfo& info) { } return CallChecked(env, [&]() -> Napi::Value { + auto manager_alive = LockManagerOrThrow(manager_keepalive_, lifecycle_); + auto variant_manager_alive = LockManagerOrThrow(variant->manager_keepalive_, variant->lifecycle_); + (void)manager_alive; + (void)variant_manager_alive; impl_->SelectVariant(*variant->impl_); return env.Undefined(); }); diff --git a/sdk_v2/js/native/src/model.h b/sdk_v2/js/native/src/model.h index 3ae4b6e63..e90dc3977 100644 --- a/sdk_v2/js/native/src/model.h +++ b/sdk_v2/js/native/src/model.h @@ -32,6 +32,8 @@ namespace foundry_local_node { +struct ManagerLifecycle; + struct ModelCtorToken { // The IModel accessor. Never null when the token is constructed. foundry_local::IModel* impl = nullptr; @@ -39,6 +41,8 @@ struct ModelCtorToken { // or a std::shared_ptr) alive for the JS Model's // lifetime. std::shared_ptr keepalive; + std::weak_ptr manager_keepalive; + std::shared_ptr lifecycle; // Pins the parent Manager so its native handle (and the Catalog's flCatalog* // which the IModel views into) cannot be released first. Napi::ObjectReference manager; @@ -62,6 +66,9 @@ class Model : public Napi::ObjectWrap { // Internal accessor used by Session / ChatSession ctors so they can clone // the parent Manager ObjectReference and pin it for the session lifetime. const Napi::ObjectReference& manager() const noexcept { return manager_; } + std::shared_ptr manager_keepalive() const noexcept { return manager_keepalive_.lock(); } + std::shared_ptr manager_lifecycle() const noexcept { return lifecycle_; } + bool manager_disposed() const noexcept; private: Napi::Value GetInfo(const Napi::CallbackInfo& info); @@ -78,6 +85,8 @@ class Model : public Napi::ObjectWrap { foundry_local::IModel* impl_ = nullptr; std::shared_ptr keepalive_; + std::weak_ptr manager_keepalive_; + std::shared_ptr lifecycle_; Napi::ObjectReference manager_; }; diff --git a/sdk_v2/js/native/src/session.cc b/sdk_v2/js/native/src/session.cc index 0d1247463..d71f2896e 100644 --- a/sdk_v2/js/native/src/session.cc +++ b/sdk_v2/js/native/src/session.cc @@ -5,6 +5,7 @@ #include "addon_data.h" #include "errors.h" #include "items.h" +#include "manager.h" #include "model.h" #include "promise_worker.h" #include "request.h" @@ -22,6 +23,36 @@ namespace foundry_local_node { namespace { +struct SessionLease { + explicit SessionLease(std::shared_ptr lifecycle) + : lifecycle_(std::move(lifecycle)) { + if (lifecycle_) { + lifecycle_->active_sessions.fetch_add(1, std::memory_order_acq_rel); + } + } + + ~SessionLease() { + if (lifecycle_) { + lifecycle_->active_sessions.fetch_sub(1, std::memory_order_acq_rel); + } + } + + std::shared_ptr lifecycle_; +}; + +std::shared_ptr MakeSessionLease(std::shared_ptr lifecycle) { + return std::make_shared(std::move(lifecycle)); +} + +template +void ReleaseSessionState(std::shared_ptr& impl, + std::shared_ptr& manager_keepalive, + std::shared_ptr& session_lease) { + impl.reset(); + manager_keepalive.reset(); + session_lease.reset(); +} + const char* FinishReasonToString(flFinishReason r) { switch (r) { case FOUNDRY_LOCAL_FINISH_STOP: @@ -97,8 +128,10 @@ foundry_local::Request* UnwrapRequest(Napi::Env env, const Napi::Value& v) { // Pins both the Manager (so the Model handle the Session holds stays alive) // and the Request (so the C++ Request the worker reads stays alive). template -Napi::Value ProcessRequestOn(Napi::Env env, SessT* sess, const Napi::Value& request_arg, - Napi::ObjectReference manager_ref) { +Napi::Value ProcessRequestOn(Napi::Env env, std::shared_ptr sess, const Napi::Value& request_arg, + Napi::ObjectReference manager_ref, + std::shared_ptr manager_keepalive, + std::shared_ptr session_lease) { foundry_local::Request* req = UnwrapRequest(env, request_arg); if (req == nullptr) return env.Undefined(); // pending exception Napi::ObjectReference req_pin = Napi::Reference::New(request_arg.As(), 1); @@ -107,12 +140,15 @@ Napi::Value ProcessRequestOn(Napi::Env env, SessT* sess, const Napi::Value& requ struct Pins { Napi::ObjectReference manager; Napi::ObjectReference request; + std::shared_ptr native_manager; + std::shared_ptr session_lease; }; - auto pins = std::make_shared(Pins{std::move(manager_ref), std::move(req_pin)}); + auto pins = std::make_shared( + Pins{std::move(manager_ref), std::move(req_pin), std::move(manager_keepalive), std::move(session_lease)}); return PromiseWorker::Run( env, - [sess, req, pins]() -> Result { + [sess = std::move(sess), req, pins]() -> Result { (void)pins; // keepalive captured by reference count return std::make_shared(sess->ProcessRequest(*req)); }, @@ -148,6 +184,8 @@ struct StreamCtx { Napi::Promise::Deferred deferred; Napi::ObjectReference manager; Napi::ObjectReference request; + std::shared_ptr native_manager; + std::shared_ptr session_lease; std::shared_ptr response; std::string err_msg; int err_code = 0; @@ -180,9 +218,9 @@ void FinalizeStream(Napi::Env env, void* /*data*/, StreamCtx* ctx) { template class StreamWorker : public Napi::AsyncWorker { public: - static Napi::Promise Run(Napi::Env env, SessT* sess, foundry_local::Request* req, + static Napi::Promise Run(Napi::Env env, std::shared_ptr sess, foundry_local::Request* req, Napi::Function jsCallback, StreamCtx* ctx) { - auto* w = new StreamWorker(env, sess, req, jsCallback, ctx); + auto* w = new StreamWorker(env, std::move(sess), req, jsCallback, ctx); Napi::Promise p = ctx->deferred.Promise(); w->Queue(); return p; @@ -215,9 +253,6 @@ class StreamWorker : public Napi::AsyncWorker { return 0; }); ctx_->response = std::make_shared(sess_->ProcessRequest(*req_)); - // Drop the callback so any stale shared state in the lambda is released - // before the Session is re-used for a follow-up request. - sess_->SetStreamingCallback(nullptr); } catch (const foundry_local::Error& e) { ctx_->errored = true; ctx_->err_code = static_cast(e.Code()); @@ -230,6 +265,9 @@ class StreamWorker : public Napi::AsyncWorker { ctx_->errored = true; ctx_->err_msg = "Unknown native exception"; } + // Drop the callback so any stale shared state in the lambda is released + // before the Session is re-used for a follow-up request. + sess_->SetStreamingCallback(nullptr); } // Promise resolution happens in FinalizeStream — overriding OnOK/OnError @@ -239,10 +277,10 @@ class StreamWorker : public Napi::AsyncWorker { void OnError(const Napi::Error& /*unused*/) override { tsfn_.Release(); } private: - StreamWorker(Napi::Env env, SessT* sess, foundry_local::Request* req, + StreamWorker(Napi::Env env, std::shared_ptr sess, foundry_local::Request* req, Napi::Function jsCallback, StreamCtx* ctx) : Napi::AsyncWorker(env), - sess_(sess), + sess_(std::move(sess)), req_(req), ctx_(ctx), tsfn_(Napi::ThreadSafeFunction::New(env, jsCallback, "foundry_local_stream", @@ -250,15 +288,17 @@ class StreamWorker : public Napi::AsyncWorker { FinalizeStream, static_cast(nullptr))) {} - SessT* sess_; + std::shared_ptr sess_; foundry_local::Request* req_; StreamCtx* ctx_; Napi::ThreadSafeFunction tsfn_; }; template -Napi::Value ProcessStreamingRequestOn(Napi::Env env, SessT* sess, const Napi::CallbackInfo& info, - Napi::ObjectReference manager_ref) { +Napi::Value ProcessStreamingRequestOn(Napi::Env env, std::shared_ptr sess, const Napi::CallbackInfo& info, + Napi::ObjectReference manager_ref, + std::shared_ptr manager_keepalive, + std::shared_ptr session_lease) { if (info.Length() < 2 || !info[1].IsFunction()) { Napi::TypeError::New(env, "processStreamingRequest(request: Request, onItem: (item) => void)") .ThrowAsJavaScriptException(); @@ -272,12 +312,14 @@ Napi::Value ProcessStreamingRequestOn(Napi::Env env, SessT* sess, const Napi::Ca auto* ctx = new StreamCtx{Napi::Promise::Deferred::New(env), std::move(manager_ref), std::move(req_pin), + std::move(manager_keepalive), + std::move(session_lease), nullptr, "", 0, false, false}; - return StreamWorker::Run(env, sess, req, info[1].As(), ctx); + return StreamWorker::Run(env, std::move(sess), req, info[1].As(), ctx); } } // namespace @@ -322,7 +364,15 @@ ChatSession::ChatSession(const Napi::CallbackInfo& info) : Napi::ObjectWrap(*native); + auto manager_keepalive = model->manager_keepalive(); + if (model->manager_disposed() || !manager_keepalive) { + ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Manager has been disposed"); + return; + } + impl_ = std::make_shared(*native); + lifecycle_ = model->manager_lifecycle(); + session_lease_ = MakeSessionLease(lifecycle_); + manager_keepalive_ = std::move(manager_keepalive); } catch (const foundry_local::Error& e) { ThrowFoundryLocalError(env, static_cast(e.Code()), e.what()); return; @@ -333,6 +383,10 @@ ChatSession::ChatSession(const Napi::CallbackInfo& info) : Napi::ObjectWrap::New(model->manager().Value(), 1); } +ChatSession::~ChatSession() { + ReleaseSessionState(impl_, manager_keepalive_, session_lease_); +} + bool ChatSession::ThrowIfDisposed(Napi::Env env) { if (impl_ == nullptr) { ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, @@ -350,14 +404,14 @@ Napi::Value ChatSession::ProcessRequest(const Napi::CallbackInfo& info) { return env.Undefined(); } Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - return ProcessRequestOn(env, impl_.get(), info[0], std::move(owner)); + return ProcessRequestOn(env, impl_, info[0], std::move(owner), manager_keepalive_, session_lease_); } Napi::Value ChatSession::ProcessStreamingRequest(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); if (ThrowIfDisposed(env)) return env.Undefined(); Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - return ProcessStreamingRequestOn(env, impl_.get(), info, std::move(owner)); + return ProcessStreamingRequestOn(env, impl_, info, std::move(owner), manager_keepalive_, session_lease_); } Napi::Value ChatSession::SetOptions(const Napi::CallbackInfo& info) { @@ -437,7 +491,7 @@ Napi::Value ChatSession::UndoTurns(const Napi::CallbackInfo& info) { } Napi::Value ChatSession::Dispose(const Napi::CallbackInfo& info) { - impl_.reset(); + ReleaseSessionState(impl_, manager_keepalive_, session_lease_); return info.Env().Undefined(); } @@ -479,7 +533,15 @@ EmbeddingsSession::EmbeddingsSession(const Napi::CallbackInfo& info) return; } try { - impl_ = std::make_unique(*native); + auto manager_keepalive = model->manager_keepalive(); + if (model->manager_disposed() || !manager_keepalive) { + ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Manager has been disposed"); + return; + } + impl_ = std::make_shared(*native); + lifecycle_ = model->manager_lifecycle(); + session_lease_ = MakeSessionLease(lifecycle_); + manager_keepalive_ = std::move(manager_keepalive); } catch (const foundry_local::Error& e) { ThrowFoundryLocalError(env, static_cast(e.Code()), e.what()); return; @@ -490,6 +552,10 @@ EmbeddingsSession::EmbeddingsSession(const Napi::CallbackInfo& info) manager_ = Napi::Reference::New(model->manager().Value(), 1); } +EmbeddingsSession::~EmbeddingsSession() { + ReleaseSessionState(impl_, manager_keepalive_, session_lease_); +} + bool EmbeddingsSession::ThrowIfDisposed(Napi::Env env) { if (impl_ == nullptr) { ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, @@ -507,7 +573,7 @@ Napi::Value EmbeddingsSession::ProcessRequest(const Napi::CallbackInfo& info) { return env.Undefined(); } Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - return ProcessRequestOn(env, impl_.get(), info[0], std::move(owner)); + return ProcessRequestOn(env, impl_, info[0], std::move(owner), manager_keepalive_, session_lease_); } Napi::Value EmbeddingsSession::SetOptions(const Napi::CallbackInfo& info) { @@ -526,7 +592,7 @@ Napi::Value EmbeddingsSession::SetOptions(const Napi::CallbackInfo& info) { } Napi::Value EmbeddingsSession::Dispose(const Napi::CallbackInfo& info) { - impl_.reset(); + ReleaseSessionState(impl_, manager_keepalive_, session_lease_); return info.Env().Undefined(); } @@ -573,7 +639,15 @@ AudioSession::AudioSession(const Napi::CallbackInfo& info) return; } try { - impl_ = std::make_unique(*native); + auto manager_keepalive = model->manager_keepalive(); + if (model->manager_disposed() || !manager_keepalive) { + ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, "Manager has been disposed"); + return; + } + impl_ = std::make_shared(*native); + lifecycle_ = model->manager_lifecycle(); + session_lease_ = MakeSessionLease(lifecycle_); + manager_keepalive_ = std::move(manager_keepalive); } catch (const foundry_local::Error& e) { ThrowFoundryLocalError(env, static_cast(e.Code()), e.what()); return; @@ -584,6 +658,10 @@ AudioSession::AudioSession(const Napi::CallbackInfo& info) manager_ = Napi::Reference::New(model->manager().Value(), 1); } +AudioSession::~AudioSession() { + ReleaseSessionState(impl_, manager_keepalive_, session_lease_); +} + bool AudioSession::ThrowIfDisposed(Napi::Env env) { if (impl_ == nullptr) { ThrowFoundryLocalError(env, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, @@ -601,14 +679,14 @@ Napi::Value AudioSession::ProcessRequest(const Napi::CallbackInfo& info) { return env.Undefined(); } Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - return ProcessRequestOn(env, impl_.get(), info[0], std::move(owner)); + return ProcessRequestOn(env, impl_, info[0], std::move(owner), manager_keepalive_, session_lease_); } Napi::Value AudioSession::ProcessStreamingRequest(const Napi::CallbackInfo& info) { Napi::Env env = info.Env(); if (ThrowIfDisposed(env)) return env.Undefined(); Napi::ObjectReference owner = Napi::Reference::New(manager_.Value(), 1); - return ProcessStreamingRequestOn(env, impl_.get(), info, std::move(owner)); + return ProcessStreamingRequestOn(env, impl_, info, std::move(owner), manager_keepalive_, session_lease_); } Napi::Value AudioSession::SetOptions(const Napi::CallbackInfo& info) { @@ -627,7 +705,7 @@ Napi::Value AudioSession::SetOptions(const Napi::CallbackInfo& info) { } Napi::Value AudioSession::Dispose(const Napi::CallbackInfo& info) { - impl_.reset(); + ReleaseSessionState(impl_, manager_keepalive_, session_lease_); return info.Env().Undefined(); } diff --git a/sdk_v2/js/native/src/session.h b/sdk_v2/js/native/src/session.h index 2b1db7b86..796439e54 100644 --- a/sdk_v2/js/native/src/session.h +++ b/sdk_v2/js/native/src/session.h @@ -32,11 +32,14 @@ namespace foundry_local_node { +struct ManagerLifecycle; + class ChatSession : public Napi::ObjectWrap { public: static Napi::Function Init(Napi::Env env); explicit ChatSession(const Napi::CallbackInfo& info); + ~ChatSession(); private: Napi::Value ProcessRequest(const Napi::CallbackInfo& info); @@ -51,8 +54,11 @@ class ChatSession : public Napi::ObjectWrap { bool ThrowIfDisposed(Napi::Env env); - std::unique_ptr impl_; + std::shared_ptr impl_; Napi::ObjectReference manager_; + std::shared_ptr manager_keepalive_; + std::shared_ptr lifecycle_; + std::shared_ptr session_lease_; }; // Napi::ObjectWrap over foundry_local::EmbeddingsSession. @@ -73,6 +79,7 @@ class EmbeddingsSession : public Napi::ObjectWrap { static Napi::Function Init(Napi::Env env); explicit EmbeddingsSession(const Napi::CallbackInfo& info); + ~EmbeddingsSession(); private: Napi::Value ProcessRequest(const Napi::CallbackInfo& info); @@ -82,8 +89,11 @@ class EmbeddingsSession : public Napi::ObjectWrap { bool ThrowIfDisposed(Napi::Env env); - std::unique_ptr impl_; + std::shared_ptr impl_; Napi::ObjectReference manager_; + std::shared_ptr manager_keepalive_; + std::shared_ptr lifecycle_; + std::shared_ptr session_lease_; }; // Napi::ObjectWrap over foundry_local::AudioSession. @@ -101,6 +111,7 @@ class AudioSession : public Napi::ObjectWrap { static Napi::Function Init(Napi::Env env); explicit AudioSession(const Napi::CallbackInfo& info); + ~AudioSession(); private: Napi::Value ProcessRequest(const Napi::CallbackInfo& info); @@ -111,8 +122,11 @@ class AudioSession : public Napi::ObjectWrap { bool ThrowIfDisposed(Napi::Env env); - std::unique_ptr impl_; + std::shared_ptr impl_; Napi::ObjectReference manager_; + std::shared_ptr manager_keepalive_; + std::shared_ptr lifecycle_; + std::shared_ptr session_lease_; }; } // namespace foundry_local_node diff --git a/sdk_v2/js/script/copy-native.mjs b/sdk_v2/js/script/copy-native.mjs index 494d26671..34682f7dc 100644 --- a/sdk_v2/js/script/copy-native.mjs +++ b/sdk_v2/js/script/copy-native.mjs @@ -55,6 +55,7 @@ const wanted = (() => { "onnxruntime-genai.dll", "onnxruntime_providers_shared.dll", "Microsoft.Windows.AI.MachineLearning.dll", + "DirectML.dll", ]; } if (process.platform === "darwin") { diff --git a/sdk_v2/js/script/install-native.cjs b/sdk_v2/js/script/install-native.cjs index bbb372091..ae32ce0b4 100644 --- a/sdk_v2/js/script/install-native.cjs +++ b/sdk_v2/js/script/install-native.cjs @@ -30,7 +30,6 @@ const PLATFORM_MAP = { 'win32-x64': 'win-x64', 'win32-arm64': 'win-arm64', 'linux-x64': 'linux-x64', - 'linux-arm64': 'linux-arm64', 'darwin-arm64': 'osx-arm64', }; const platformKey = `${os.platform()}-${os.arch()}`; diff --git a/sdk_v2/js/script/pack-prebuilds.mjs b/sdk_v2/js/script/pack-prebuilds.mjs index 36e1d0913..5dbf558aa 100644 --- a/sdk_v2/js/script/pack-prebuilds.mjs +++ b/sdk_v2/js/script/pack-prebuilds.mjs @@ -43,18 +43,14 @@ const destDir = resolve(pkgRoot, "prebuilds", `${process.platform}-${process.arc mkdirSync(destDir, { recursive: true }); // foundry_local is required; ORT/GenAI siblings are excluded (fetched at install -// time). The WinML EP catalog DLL is an optional sibling bundled on Windows. +// time). The WinML EP catalog and DirectML DLLs are required siblings on Windows. const wanted = (() => { - if (process.platform === "win32") return ["foundry_local.dll"]; + if (process.platform === "win32") return ["foundry_local.dll", "Microsoft.Windows.AI.MachineLearning.dll", "DirectML.dll"]; if (process.platform === "darwin") return ["libfoundry_local.dylib"]; return ["libfoundry_local.so"]; })(); -// Optional native siblings — copied when present, skipped (with a warning) when -// not. The reg-free WinML 2.x runtime ships next to foundry_local.dll on Windows -// so WinML hardware EPs work out of the box without an install-time download. -const optional = - process.platform === "win32" ? ["Microsoft.Windows.AI.MachineLearning.dll"] : []; +const optional = []; let copied = 0; const available = new Set(readdirSync(sourceDir)); diff --git a/sdk_v2/js/src/catalog.ts b/sdk_v2/js/src/catalog.ts index 978a0f359..c08c8d579 100644 --- a/sdk_v2/js/src/catalog.ts +++ b/sdk_v2/js/src/catalog.ts @@ -4,6 +4,7 @@ // // The underlying native catalog operations are synchronous; the async surface here is for parity with the C# / // Python SDKs. `getModel`, `getModelVariant`, and `getLatestVersion` throw when the alias / id is not found. +// After the parent manager is disposed, native catalog methods throw a FoundryLocalError. import type { NativeCatalog, NativeModel } from "./detail/native.js"; import type { IModel } from "./imodel.js"; diff --git a/sdk_v2/js/src/foundryLocalManager.ts b/sdk_v2/js/src/foundryLocalManager.ts index 9cf9946cf..f6fc77782 100644 --- a/sdk_v2/js/src/foundryLocalManager.ts +++ b/sdk_v2/js/src/foundryLocalManager.ts @@ -14,11 +14,10 @@ import { import type { EpDownloadResult, EpInfo } from "./types.js"; // A native Manager holds process-global native resources. Dispose the live -// Manager on process exit so native teardown happens at a deterministic point -// rather than being left entirely to environment finalizers. `beforeExit` -// covers a natural event-loop drain; `exit` covers callers who invoke -// `process.exit()` themselves. Both are idempotent. The native layer permits -// only one live Manager at a time, so a single reference is enough. +// Manager when the event loop naturally drains so native teardown happens at a +// deterministic point rather than being left entirely to environment finalizers. +// Do not dispose during the synchronous `exit` event: native worker threads may +// still be unwinding, and freeing the process-global manager there can race them. let liveManager: FoundryLocalManager | undefined; let exitHandlersInstalled = false; @@ -42,9 +41,6 @@ function installExitHandlersOnce(): void { process.on("beforeExit", () => { disposeLiveManager(); }); - process.on("exit", () => { - disposeLiveManager(); - }); } export class FoundryLocalManager { @@ -215,10 +211,10 @@ export class FoundryLocalManager { * (and any method on a `Catalog` or `Model` obtained through this manager) throws a `FoundryLocalError`. */ dispose(): void { + this.#native.dispose(); if (liveManager === this) { liveManager = undefined; } - this.#native.dispose(); this.#catalog = undefined; this.#urls = []; } diff --git a/sdk_v2/js/test/manager-dispose.test.ts b/sdk_v2/js/test/manager-dispose.test.ts index fe7c7ba19..26bbe1c4e 100644 --- a/sdk_v2/js/test/manager-dispose.test.ts +++ b/sdk_v2/js/test/manager-dispose.test.ts @@ -62,6 +62,21 @@ describeIfBuilt("FoundryLocalManager.dispose", () => { } }); + it("a cached catalog handle throws after manager disposal and does not block a new manager", () => { + const mgr = freshManager("cached-catalog"); + const catalog = mgr.catalog; + mgr.dispose(); + + expect(() => catalog.name).toThrow(/disposed/i); + + const next = freshManager("after-cached-catalog"); + try { + expect(next.disposed).toBe(false); + } finally { + next.dispose(); + } + }); + it("Symbol.dispose is wired and idempotent", () => { const mgr = freshManager("symbol-dispose"); mgr[Symbol.dispose](); @@ -70,6 +85,15 @@ describeIfBuilt("FoundryLocalManager.dispose", () => { expect(mgr.disposed).toBe(true); }); + it("dispose() rejects while a native EP worker is in flight", async () => { + const mgr = freshManager("async-worker"); + const pending = mgr.downloadAndRegisterEps(["__not-a-provider__"]); + expect(() => mgr.dispose()).toThrow(/active native workers/i); + await expect(pending).rejects.toThrow(/not available/i); + mgr.dispose(); + expect(mgr.disposed).toBe(true); + }); + it("`using` declaration disposes at scope exit", () => { let captured: FoundryLocalManager | undefined; { diff --git a/sdk_v2/python/src/foundry_local_sdk/_native/api.py b/sdk_v2/python/src/foundry_local_sdk/_native/api.py index e2a392d3d..0f19e5188 100644 --- a/sdk_v2/python/src/foundry_local_sdk/_native/api.py +++ b/sdk_v2/python/src/foundry_local_sdk/_native/api.py @@ -16,8 +16,8 @@ from foundry_local_sdk._native.lib_loader import find_library, prepare_native_dependencies from foundry_local_sdk.exception import FoundryLocalException -# FOUNDRY_LOCAL_API_VERSION = 1 (from foundry_local_c.h) -_FOUNDRY_LOCAL_API_VERSION: int = 1 +# FOUNDRY_LOCAL_API_VERSION = 2 (from foundry_local_c.h) +_FOUNDRY_LOCAL_API_VERSION: int = 2 _lib_path = find_library() @@ -32,6 +32,7 @@ # module-level name so the GC never collects them (which would unload the DLLs # mid-process and crash any in-flight ORT call). _preloaded_native_deps: list = [] +_dll_directory_handles: list = [] if _lib_path is not None: _preloaded_native_deps = prepare_native_dependencies(_lib_path.parent) @@ -46,7 +47,7 @@ # foundry_local.dll resolves. ORT/GenAI were already preloaded by absolute path above. _dll_parent = _lib_path.parent.resolve() if _dll_parent.is_dir(): - os.add_dll_directory(str(_dll_parent)) + _dll_directory_handles.append(os.add_dll_directory(str(_dll_parent))) else: # Linux/macOS: preload libfoundry_local with RTLD_GLOBAL so its exported symbols satisfy the cffi # extension's NEEDED entry by symbol resolution rather than by file lookup. ORT/GenAI are already diff --git a/sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py b/sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py index 1369759c8..57d8b8868 100644 --- a/sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py +++ b/sdk_v2/python/src/foundry_local_sdk/_native/lib_loader.py @@ -15,6 +15,7 @@ import pathlib import platform import sys +import sysconfig logger = logging.getLogger(__name__) @@ -29,7 +30,7 @@ def _lib_name() -> str: def _platform_subdir() -> str: if sys.platform == "win32": - return "win-x64" + return "win-arm64" if sysconfig.get_platform().lower() == "win-arm64" else "win-x64" if sys.platform == "darwin": return "osx-arm64" if platform.machine() == "arm64" else "osx-x64" return "linux-x64" @@ -248,7 +249,7 @@ def prepare_native_dependencies(foundry_local_dir: pathlib.Path) -> list: if add_dll_directory is not None: for d in {ort_path.parent, genai_path.parent, foundry_local_dir}: try: - add_dll_directory(str(d)) + handles.append(add_dll_directory(str(d))) except OSError as exc: logger.warning("os.add_dll_directory(%s) failed: %s", d, exc) @@ -278,22 +279,11 @@ def prepare_native_dependencies(foundry_local_dir: pathlib.Path) -> list: logger.warning("Failed to preload ORT (%s): %s", ort_path, exc) return handles - # macOS only: GenAI's static initializer does its own dlopen("libonnxruntime.dylib"), - # which on Darwin only matches by leafname against dyld search paths or images - # whose install_name leaf is exactly "libonnxruntime.dylib". ORT's dylib has a - # versioned install_name instead (the soversion, e.g. "@rpath/libonnxruntime.1.dylib"), - # so neither match path succeeds and GenAI aborts in dyld init before any of our code - # runs. The second name GenAI tries is "/libonnxruntime.dylib", so a symlink - # there fixes it. Linux dlopen consults the loaded-soname table by leafname and finds - # our already-RTLD_GLOBAL'd image without help. - if sys.platform == "darwin": - symlink_path = genai_path.parent / "libonnxruntime.dylib" - try: - if not symlink_path.exists(): - symlink_path.symlink_to(ort_path) - logger.info("Created macOS GenAI->ORT symlink: %s -> %s", symlink_path, ort_path) - except OSError as exc: - logger.warning("Failed to create macOS ORT symlink at %s: %s", symlink_path, exc) + # GenAI's lazy InitApi() first checks ORT_LIB_PATH before falling back to a + # platform leafname. Point it at the exact ORT dylib/so we resolved instead + # of mutating the installed GenAI package to add an unversioned alias. + if "ORT_LIB_PATH" not in os.environ: + os.environ["ORT_LIB_PATH"] = str(ort_path) try: handles.append(ctypes.CDLL(str(genai_path), mode=ctypes.RTLD_GLOBAL)) diff --git a/sdk_v2/python/src/foundry_local_sdk/items.py b/sdk_v2/python/src/foundry_local_sdk/items.py index 0f1f3c14f..809fe2b62 100644 --- a/sdk_v2/python/src/foundry_local_sdk/items.py +++ b/sdk_v2/python/src/foundry_local_sdk/items.py @@ -11,7 +11,7 @@ from enum import IntEnum -_API_VERSION = 1 # FOUNDRY_LOCAL_API_VERSION +_API_VERSION = 2 # FOUNDRY_LOCAL_API_VERSION # Sentinel for absent time fields in flSpeech* structs. INT64_MIN, matches # C ABI FOUNDRY_LOCAL_DURATION_UNSET. cffi cdef does not process #define. _DURATION_UNSET: int = -(2**63) diff --git a/sdk_v2/python/src/foundry_local_sdk/request.py b/sdk_v2/python/src/foundry_local_sdk/request.py index fe6879e81..5c3a90ca5 100644 --- a/sdk_v2/python/src/foundry_local_sdk/request.py +++ b/sdk_v2/python/src/foundry_local_sdk/request.py @@ -12,7 +12,7 @@ from foundry_local_sdk.items import Item from foundry_local_sdk.session_types import RequestOptions -_API_VERSION = 1 # FOUNDRY_LOCAL_API_VERSION +_API_VERSION = 2 # FOUNDRY_LOCAL_API_VERSION class Request: diff --git a/sdk_v2/python/src/foundry_local_sdk/response.py b/sdk_v2/python/src/foundry_local_sdk/response.py index df6748bc3..8df2e2f24 100644 --- a/sdk_v2/python/src/foundry_local_sdk/response.py +++ b/sdk_v2/python/src/foundry_local_sdk/response.py @@ -12,7 +12,7 @@ from foundry_local_sdk.items import Item from foundry_local_sdk.session_types import FinishReason, TokenUsage -_API_VERSION = 1 # FOUNDRY_LOCAL_API_VERSION +_API_VERSION = 2 # FOUNDRY_LOCAL_API_VERSION class Response: diff --git a/sdk_v2/python/src/foundry_local_sdk/session.py b/sdk_v2/python/src/foundry_local_sdk/session.py index 136609085..e913c0138 100644 --- a/sdk_v2/python/src/foundry_local_sdk/session.py +++ b/sdk_v2/python/src/foundry_local_sdk/session.py @@ -17,7 +17,7 @@ from foundry_local_sdk.response import Response from foundry_local_sdk.session_types import RequestOptions -_API_VERSION = 1 # FOUNDRY_LOCAL_API_VERSION +_API_VERSION = 2 # FOUNDRY_LOCAL_API_VERSION # Sentinel placed on the stream queue by the background thread when inference finishes. _DONE = object()